Reformatting Conversion, FeatureSelection and Image Analytics of Transform to Width 85#3943

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
rayankrish merged 7 commits into from
Jul 3, 2019
Merged

Reformatting Conversion, FeatureSelection and Image Analytics of Transform to Width 85 #3943

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
9267d02
samples/dynamic/transforms/conversions formatted to 85 char
Jul 1, 2019
f2a18ed
samples/dynamic/transforms/featureselection formatted to 85 char
Jul 1, 2019
38c8938
ImageAnalytics
Jul 1, 2019
67080d1
ImageAnalytics
Jul 1, 2019
ecd2738
small corrections to formatting
Jul 2, 2019
91532b0
Merge branch 'transform85' of https://github.com/RadicalRayan/machine…
Jul 2, 2019
3deb533
minor tab and empty line changes
Jul 2, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 9 additions & 4 deletions docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Conversion/ConvertType.cs
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -20,17 +20,22 @@ public static void Example()
var data = mlContext.Data.LoadFromEnumerable(rawData);

// Construct the pipeline.
var pipeline = mlContext.Transforms.Conversion.ConvertType("SurvivedInt32", "Survived", DataKind.Int32);
var pipeline = mlContext.Transforms.Conversion.ConvertType(
"SurvivedInt32", "Survived", DataKind.Int32);

// Let's train our pipeline, and then apply it to the same data.
var transformer = pipeline.Fit(data);
var transformedData = transformer.Transform(data);

// Display original column 'Survived' (boolean) and converted column 'SurvivedInt32' (Int32)
var convertedData = mlContext.Data.CreateEnumerable<TransformedData>(transformedData, true);
// Display original column 'Survived' (boolean) and converted column
// SurvivedInt32' (Int32)
var convertedData = mlContext.Data.CreateEnumerable<TransformedData>(
transformedData, true);

foreach (var item in convertedData)

@codemzscodemzsJul 2, 2019

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

new line between any two lines that were broken down into multiple lines but no new line between multiple breakdowns of a single line.

{
Console.WriteLine("A:{0,-10} Aconv:{1}", item.Survived, item.SurvivedInt32);
Console.WriteLine("A:{0,-10} Aconv:{1}", item.Survived,
item.SurvivedInt32);
}

// Output
Expand Down
38 changes: 26 additions & 12 deletions docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Conversion/ConvertTypeMultiColumn.cs
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -4,22 +4,33 @@

namespace Samples.Dynamic
{
// This example illustrates how to convert multiple columns of different types to one type, in this case System.Single.
// This is often a useful data transformation before concatenating the features together and passing them to a particular estimator.
// This example illustrates how to convert multiple columns of different types
// to one type, in this case System.Single.
// This is often a useful data transformation before concatenating the features
// together and passing them to a particular estimator.
public static class ConvertTypeMultiColumn
{
public static void Example()
{
// Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging,
// as well as the source of randomness.
// Create a new ML context, for ML.NET operations. It can be used for
// exception tracking and logging, as well as the source of randomness.
var mlContext = new MLContext(seed: 1);

var rawData = new[] {
new InputData() { Feature1 = true, Feature2 = "0.4", Feature3 = DateTime.Now, Feature4 = 0.145},
new InputData() { Feature1 = false, Feature2 = "0.5", Feature3 = DateTime.Today, Feature4 = 3.14},
new InputData() { Feature1 = false, Feature2 = "14", Feature3 = DateTime.Today, Feature4 = 0.2046},
new InputData() { Feature1 = false, Feature2 = "23", Feature3 = DateTime.Now, Feature4 = 0.1206},
new InputData() { Feature1 = true, Feature2 = "8904", Feature3 = DateTime.UtcNow, Feature4 = 8.09},
new InputData() { Feature1 = true, Feature2 = "0.4",
Feature3 = DateTime.Now, Feature4 = 0.145},

new InputData() { Feature1 = false, Feature2 = "0.5",
Feature3 = DateTime.Today, Feature4 = 3.14},

new InputData() { Feature1 = false, Feature2 = "14",
Feature3 = DateTime.Today, Feature4 = 0.2046},

new InputData() { Feature1 = false, Feature2 = "23",
Feature3 = DateTime.Now, Feature4 = 0.1206},

new InputData() { Feature1 = true, Feature2 = "8904",
Feature3 = DateTime.UtcNow, Feature4 = 8.09},
};

// Convert the data to an IDataView.
Expand All@@ -37,17 +48,20 @@ public static void Example()

// Let's fit our pipeline to the data.
var transformer = pipeline.Fit(data);
// Transforming the same data. This will add the 4 columns defined in the pipeline, containing the converted
// Transforming the same data. This will add the 4 columns defined in
// the pipeline, containing the converted
// values of the initial columns.
var transformedData = transformer.Transform(data);

// Shape the transformed data as a strongly typed IEnumerable.
var convertedData = mlContext.Data.CreateEnumerable<TransformedData>(transformedData, true);
var convertedData = mlContext.Data.CreateEnumerable<TransformedData>(
transformedData, true);

// Printing the results.
Console.WriteLine("Converted1\t Converted2\t Converted3\t Converted4");
foreach (var item in convertedData)
Console.WriteLine($"\t{item.Converted1}\t {item.Converted2}\t\t {item.Converted3}\t {item.Converted4}");
Console.WriteLine($"\t{item.Converted1}\t {item.Converted2}\t\t " +
$"{item.Converted3}\t {item.Converted4}");

// Transformed data.
//
Expand Down
60 changes: 37 additions & 23 deletions docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Conversion/Hash.cs
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -9,8 +9,8 @@ public static class Hash
{
public static void Example()
{
// Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging,
// as well as the source of randomness.
// Create a new ML context, for ML.NET operations. It can be used for
// exception tracking and logging, as well as the source of randomness.
var mlContext = new MLContext(seed: 1);

// Get a small dataset as an IEnumerable.
Expand All@@ -24,30 +24,40 @@ public static void Example()

var data = mlContext.Data.LoadFromEnumerable(rawData);

// Construct the pipeline that would hash the two columns and store the results in new columns.
// The first transform hashes the string column and the second transform hashes the integer column.
// Construct the pipeline that would hash the two columns and store the
// results in new columns. The first transform hashes the string column
// and the second transform hashes the integer column.
//
// Hashing is not a reversible operation, so there is no way to retrive the original value from the hashed value.
// Sometimes, for debugging, or model explainability, users will need to know what values in the original columns generated
// the values in the hashed columns, since the algorithms will mostly use the hashed values for further computations.
// The Hash method will preserve the mapping from the original values to the hashed values in the Annotations of the
// newly created column (column populated with the hashed values).
//
// Setting the maximumNumberOfInverts parameters to -1 will preserve the full map.
// If that parameter is left to the default 0 value, the mapping is not preserved.
var pipeline = mlContext.Transforms.Conversion.Hash("CategoryHashed", "Category", numberOfBits: 16, maximumNumberOfInverts: -1)
.Append(mlContext.Transforms.Conversion.Hash("AgeHashed", "Age", numberOfBits: 8));
// Hashing is not a reversible operation, so there is no way to retrive
// the original value from the hashed value. Sometimes, for debugging,
// or model explainability, users will need to know what values in the
// original columns generated the values in the hashed columns, since
// the algorithms will mostly use the hashed values for further
// computations. The Hash method will preserve the mapping from the
// original values to the hashed values in the Annotations of the newly
// created column (column populated with the hashed values).
//
// Setting the maximumNumberOfInverts parameters to -1 will preserve the
// full map. If that parameter is left to the default 0 value, the
// mapping is not preserved.
var pipeline = mlContext.Transforms.Conversion.Hash("CategoryHashed",
"Category", numberOfBits: 16, maximumNumberOfInverts: -1)
.Append(mlContext.Transforms.Conversion.Hash("AgeHashed", "Age",
numberOfBits: 8));

// Let's fit our pipeline, and then apply it to the same data.
var transformer = pipeline.Fit(data);
var transformedData = transformer.Transform(data);

// Convert the post transformation from the IDataView format to an IEnumerable<TransformedData> for easy consumption.
var convertedData = mlContext.Data.CreateEnumerable<TransformedDataPoint>(transformedData, true);
// Convert the post transformation from the IDataView format to an
// IEnumerable <TransformedData> for easy consumption.
var convertedData = mlContext.Data.CreateEnumerable<
TransformedDataPoint>(transformedData, true);

Console.WriteLine("Category CategoryHashed\t Age\t AgeHashed");
foreach (var item in convertedData)
Console.WriteLine($"{item.Category}\t {item.CategoryHashed}\t\t {item.Age}\t {item.AgeHashed}");
Console.WriteLine($"{item.Category}\t {item.CategoryHashed}\t\t " +
$"{item.Age}\t {item.AgeHashed}");

// Expected data after the transformation.
//
Expand All@@ -58,20 +68,24 @@ public static void Example()
// MLB 36206 18 127
// MLS 6013 14 62

// For the Category column, where we set the maximumNumberOfInverts parameter, the names of the original categories,
// and their correspondance with the generated hash values is preserved in the Annotations in the format of indices and values.
// the indices array will have the hashed values, and the corresponding element, position-wise, in the values array will
// contain the original value.
// For the Category column, where we set the maximumNumberOfInverts
// parameter, the names of the original categories, and their
// correspondance with the generated hash values is preserved in the
// Annotations in the format of indices and values.the indices array
// will have the hashed values, and the corresponding element,
// position -wise, in the values array will contain the original value.
//
// See below for an example on how to retrieve the mapping.
var slotNames = new VBuffer<ReadOnlyMemory<char>>();
transformedData.Schema["CategoryHashed"].Annotations.GetValue("KeyValues", ref slotNames);
transformedData.Schema["CategoryHashed"].Annotations.GetValue(
"KeyValues", ref slotNames);

var indices = slotNames.GetIndices();
var categoryNames = slotNames.GetValues();

for (int i = 0; i < indices.Length; i++)
Console.WriteLine($"The original value of the {indices[i]} category is {categoryNames[i]}");
Console.WriteLine($"The original value of the {indices[i]} " +
$"category is {categoryNames[i]}");

// Output Data
//
Expand Down
57 changes: 38 additions & 19 deletions docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Conversion/KeyToValueToKey.cs
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -11,8 +11,8 @@ public class KeyToValueToKey
{
public static void Example()
{
// Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging,
// as well as the source of randomness.
// Create a new ML context, for ML.NET operations. It can be used for
// exception tracking and logging, as well as the source of randomness.
var mlContext = new MLContext();

// Get a small dataset as an IEnumerable.
Expand All@@ -27,25 +27,40 @@ public static void Example()

// A pipeline to convert the terms of the 'Review' column in
// making use of default settings.
var defaultPipeline = mlContext.Transforms.Text.TokenizeIntoWords("TokenizedText", nameof(DataPoint.Review))
.Append(mlContext.Transforms.Conversion.MapValueToKey(nameof(TransformedData.Keys), "TokenizedText"));
var defaultPipeline = mlContext.Transforms.Text.TokenizeIntoWords(
"TokenizedText", nameof(DataPoint.Review)).Append(mlContext
.Transforms.Conversion.MapValueToKey(nameof(TransformedData.Keys),
"TokenizedText"));

@codemzscodemzsJul 2, 2019

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

please move .Append to a new line.

// Another pipeline, that customizes the advanced settings of the ValueToKeyMappingEstimator.
// We can change the maximumNumberOfKeys to limit how many keys will get generated out of the set of words,
// and condition the order in which they get evaluated by changing keyOrdinality from the default ByOccurence (order in which they get encountered)
// to value/alphabetically.
var customizedPipeline = mlContext.Transforms.Text.TokenizeIntoWords("TokenizedText", nameof(DataPoint.Review))
.Append(mlContext.Transforms.Conversion.MapValueToKey(nameof(TransformedData.Keys), "TokenizedText", maximumNumberOfKeys: 10,
keyOrdinality: ValueToKeyMappingEstimator.KeyOrdinality.ByValue));
// Another pipeline, that customizes the advanced settings of the
// ValueToKeyMappingEstimator. We can change the maximumNumberOfKeys to
// limit how many keys will get generated out of the set of words, and
// condition the order in which they get evaluated by changing
// keyOrdinality from the default ByOccurence (order in which they get
// encountered) to value/alphabetically.
var customizedPipeline = mlContext.Transforms.Text.TokenizeIntoWords(
"TokenizedText", nameof(DataPoint.Review)).Append(mlContext
.Transforms.Conversion.MapValueToKey(nameof(TransformedData.Keys),
"TokenizedText", maximumNumberOfKeys: 10, keyOrdinality:
ValueToKeyMappingEstimator.KeyOrdinality.ByValue));

// The transformed data.
var transformedDataDefault = defaultPipeline.Fit(trainData).Transform(trainData);
var transformedDataCustomized = customizedPipeline.Fit(trainData).Transform(trainData);
var transformedDataDefault = defaultPipeline.Fit(trainData).Transform(
trainData);

var transformedDataCustomized = customizedPipeline.Fit(trainData)
.Transform(trainData);

// Getting the resulting data as an IEnumerable.
// This will contain the newly created columns.
IEnumerable<TransformedData> defaultData = mlContext.Data.CreateEnumerable<TransformedData>(transformedDataDefault, reuseRowObject: false);
IEnumerable<TransformedData> customizedData = mlContext.Data.CreateEnumerable<TransformedData>(transformedDataCustomized, reuseRowObject: false);
IEnumerable<TransformedData> defaultData = mlContext.Data.
CreateEnumerable<TransformedData>(transformedDataDefault,
reuseRowObject: false);

IEnumerable<TransformedData> customizedData = mlContext.Data.
CreateEnumerable<TransformedData>(transformedDataCustomized,
reuseRowObject: false);

Console.WriteLine($"Keys");
foreach (var dataRow in defaultData)
Console.WriteLine($"{string.Join(',', dataRow.Keys)}");
Expand All@@ -65,13 +80,17 @@ public static void Example()
// 8,2,9,7,6,4
// 3,10,0,0,0
// 3,10,0,0,0,8
// Retrieve the original values, by appending the KeyToValue etimator to the existing pipelines
// to convert the keys back to the strings.
var pipeline = defaultPipeline.Append(mlContext.Transforms.Conversion.MapKeyToValue(nameof(TransformedData.Keys)));
// Retrieve the original values, by appending the KeyToValue etimator to
// the existing pipelines to convert the keys back to the strings.
var pipeline = defaultPipeline.Append(mlContext.Transforms.Conversion
.MapKeyToValue(nameof(TransformedData.Keys)));

transformedDataDefault = pipeline.Fit(trainData).Transform(trainData);

// Preview of the DefaultColumnName column obtained.
var originalColumnBack = transformedDataDefault.GetColumn<VBuffer<ReadOnlyMemory<char>>>(transformedDataDefault.Schema[nameof(TransformedData.Keys)]);
var originalColumnBack = transformedDataDefault.GetColumn<VBuffer<
ReadOnlyMemory<char>>>(transformedDataDefault.Schema[nameof(
TransformedData.Keys)]);

foreach (var row in originalColumnBack)
{
Expand Down
21 changes: 13 additions & 8 deletions docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Conversion/MapKeyToBinaryVector.cs
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -7,14 +7,16 @@ namespace Samples.Dynamic
{
class MapKeyToBinaryVector
{
/// This example demonstrates the use of MapKeyToVector by mapping keys to floats[] of 0 and 1, representing the number in binary format.
/// Because the ML.NET KeyType maps the missing value to zero, counting starts at 1, so the uint values
/// converted to KeyTypes will appear skewed by one.
/// This example demonstrates the use of MapKeyToVector by mapping keys to
/// floats[] of 0 and 1, representing the number in binary format.
/// Because the ML.NET KeyType maps the missing value to zero, counting
/// starts at 1, so the uint values converted to KeyTypes will appear
/// skewed by one.
/// See https://github.com/dotnet/machinelearning/blob/master/docs/code/IDataViewTypeSystem.md#key-types
public static void Example()
{
// Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging,
// as well as the source of randomness.
// Create a new ML context, for ML.NET operations. It can be used for
// exception tracking and logging, as well as the source of randomness.
var mlContext = new MLContext();

// Get a small dataset as an IEnumerable.
Expand All@@ -30,18 +32,21 @@ public static void Example()
var data = mlContext.Data.LoadFromEnumerable(rawData);

// Constructs the ML.net pipeline
var pipeline = mlContext.Transforms.Conversion.MapKeyToBinaryVector("TimeframeVector", "Timeframe");
var pipeline = mlContext.Transforms.Conversion.MapKeyToBinaryVector(
"TimeframeVector", "Timeframe");

// Fits the pipeline to the data.
IDataView transformedData = pipeline.Fit(data).Transform(data);

// Getting the resulting data as an IEnumerable.
// This will contain the newly created columns.
IEnumerable<TransformedData> features = mlContext.Data.CreateEnumerable<TransformedData>(transformedData, reuseRowObject: false);
IEnumerable<TransformedData> features = mlContext.Data.CreateEnumerable<
TransformedData>(transformedData, reuseRowObject: false);

Console.WriteLine($" Timeframe TimeframeVector");
foreach (var featureRow in features)
Console.WriteLine($"{featureRow.Timeframe}\t\t\t{string.Join(',', featureRow.TimeframeVector)}");
Console.WriteLine($"{featureRow.Timeframe}\t\t\t" +
$"{string.Join(',', featureRow.TimeframeVector)}");

// Timeframe TimeframeVector
// 10 0,1,0,0,1 //binary representation of 9, the original value
Expand Down
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Add copy buttons to all 
 blocks\n(function() {\n function addCopyButtons() {\n document.querySelectorAll('pre code').forEach(function(codeBlock) {\n if (codeBlock.parentElement.hasAttribute('data-copy-added')) return;\n codeBlock.parentElement.setAttribute('data-copy-added', 'true');\n \n var btn = document.createElement('button');\n btn.textContent = 'Copy';\n btn.style.cssText = 'position:absolute;top:4px;right:4px;padding:2px 8px;font-size:11px;background:#4ecdc4;border:none;border-radius:4px;color:#1a1a2e;cursor:pointer;opacity:0.7;transition:opacity 0.2s;';\n btn.onmouseover = function() { this.style.opacity = '1'; };\n btn.onmouseout = function() { this.style.opacity = '0.7'; };\n btn.onclick = function() {\n navigator.clipboard.writeText(codeBlock.textContent).then(function() {\n btn.textContent = 'Copied!';\n setTimeout(function() { btn.textContent = 'Copy'; }, 1500);\n });\n };\n codeBlock.parentElement.style.position = 'relative';\n codeBlock.parentElement.appendChild(btn);\n });\n }\n \n addCopyButtons();\n \n // Re-run on dynamic content\n var observer = new MutationObserver(addCopyButtons);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Add Copy Buttons to Code Blocks");
}
} catch(__e) { console.warn('[Userscript:Add Copy Buttons to Code Blocks]', __e); }
})();
(function(){
try {
var __m = "github.com";
var __re = new RegExp('^' + "github\\.com" + '

Skip to content
Reformatting Conversion, FeatureSelection and Image Analytics of Transform to Width 85#3943

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

rayankrish
merged 7 commits into

from

Jul 3, 2019
 Merged


Reformatting Conversion, FeatureSelection and Image Analytics of Transform to Width 85
#3943
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


7 commits


Select commit
Hold shift + click to select a range
9267d02

samples/dynamic/transforms/conversions formatted to 85 char

Jul 1, 2019
f2a18ed

samples/dynamic/transforms/featureselection formatted to 85 char

Jul 1, 2019
38c8938

ImageAnalytics

Jul 1, 2019
67080d1

ImageAnalytics

Jul 1, 2019
ecd2738

small corrections to formatting

Jul 2, 2019
91532b0

Merge branch 'transform85' of https://github.com/RadicalRayan/machine…

Jul 2, 2019
3deb533

minor tab and empty line changes

Jul 2, 2019

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


13 changes: 9 additions & 4 deletions
docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Conversion/ConvertType.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -20,17 +20,22 @@ public static void Example()
 var data = mlContext.Data.LoadFromEnumerable(rawData);

 // Construct the pipeline.
 var pipeline = mlContext.Transforms.Conversion.ConvertType("SurvivedInt32", "Survived", DataKind.Int32);
 var pipeline = mlContext.Transforms.Conversion.ConvertType(
 "SurvivedInt32", "Survived", DataKind.Int32);

 // Let's train our pipeline, and then apply it to the same data.
 var transformer = pipeline.Fit(data);
 var transformedData = transformer.Transform(data);

 // Display original column 'Survived' (boolean) and converted column 'SurvivedInt32' (Int32)
 var convertedData = mlContext.Data.CreateEnumerable<TransformedData>(transformedData, true);
 // Display original column 'Survived' (boolean) and converted column 
 // SurvivedInt32' (Int32)
 var convertedData = mlContext.Data.CreateEnumerable<TransformedData>(
 transformedData, true);

 foreach (var item in convertedData)
@codemzscodemzsJul 2, 2019

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

new line between any two lines that were broken down into multiple lines but no new line between multiple breakdowns of a single line.
 {
 Console.WriteLine("A:{0,-10} Aconv:{1}", item.Survived, item.SurvivedInt32);
 Console.WriteLine("A:{0,-10} Aconv:{1}", item.Survived,
 item.SurvivedInt32);
 }

 // Output
Expand Down

38 changes: 26 additions & 12 deletions
docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Conversion/ConvertTypeMultiColumn.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -4,22 +4,33 @@

namespace Samples.Dynamic
{
 // This example illustrates how to convert multiple columns of different types to one type, in this case System.Single. 
 // This is often a useful data transformation before concatenating the features together and passing them to a particular estimator.
 // This example illustrates how to convert multiple columns of different types
 // to one type, in this case System.Single. 
 // This is often a useful data transformation before concatenating the features
 // together and passing them to a particular estimator.
 public static class ConvertTypeMultiColumn
 { 
 public static void Example()
 {
 // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, 
 // as well as the source of randomness.
 // Create a new ML context, for ML.NET operations. It can be used for
 // exception tracking and logging, as well as the source of randomness.
 var mlContext = new MLContext(seed: 1);

 var rawData = new[] {
 new InputData() { Feature1 = true, Feature2 = "0.4", Feature3 = DateTime.Now, Feature4 = 0.145},
 new InputData() { Feature1 = false, Feature2 = "0.5", Feature3 = DateTime.Today, Feature4 = 3.14},
 new InputData() { Feature1 = false, Feature2 = "14", Feature3 = DateTime.Today, Feature4 = 0.2046},
 new InputData() { Feature1 = false, Feature2 = "23", Feature3 = DateTime.Now, Feature4 = 0.1206},
 new InputData() { Feature1 = true, Feature2 = "8904", Feature3 = DateTime.UtcNow, Feature4 = 8.09},
 new InputData() { Feature1 = true, Feature2 = "0.4",
 Feature3 = DateTime.Now, Feature4 = 0.145},

 new InputData() { Feature1 = false, Feature2 = "0.5",
 Feature3 = DateTime.Today, Feature4 = 3.14},

 new InputData() { Feature1 = false, Feature2 = "14",
 Feature3 = DateTime.Today, Feature4 = 0.2046},

 new InputData() { Feature1 = false, Feature2 = "23",
 Feature3 = DateTime.Now, Feature4 = 0.1206},

 new InputData() { Feature1 = true, Feature2 = "8904",
 Feature3 = DateTime.UtcNow, Feature4 = 8.09},
 };

 // Convert the data to an IDataView.
Expand All@@ -37,17 +48,20 @@ public static void Example()

 // Let's fit our pipeline to the data.
 var transformer = pipeline.Fit(data);
 // Transforming the same data. This will add the 4 columns defined in the pipeline, containing the converted
 // Transforming the same data. This will add the 4 columns defined in
 // the pipeline, containing the converted
 // values of the initial columns. 
 var transformedData = transformer.Transform(data);

 // Shape the transformed data as a strongly typed IEnumerable.
 var convertedData = mlContext.Data.CreateEnumerable<TransformedData>(transformedData, true);
 var convertedData = mlContext.Data.CreateEnumerable<TransformedData>(
 transformedData, true);

 // Printing the results.
 Console.WriteLine("Converted1\t Converted2\t Converted3\t Converted4");
 foreach (var item in convertedData)
 Console.WriteLine($"\t{item.Converted1}\t {item.Converted2}\t\t {item.Converted3}\t {item.Converted4}");
 Console.WriteLine($"\t{item.Converted1}\t {item.Converted2}\t\t " +
 $"{item.Converted3}\t {item.Converted4}");

 // Transformed data.
 //
Expand Down

60 changes: 37 additions & 23 deletions
docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Conversion/Hash.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -9,8 +9,8 @@ public static class Hash
 {
 public static void Example()
 {
 // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, 
 // as well as the source of randomness.
 // Create a new ML context, for ML.NET operations. It can be used for
 // exception tracking and logging, as well as the source of randomness.
 var mlContext = new MLContext(seed: 1);

 // Get a small dataset as an IEnumerable.
Expand All@@ -24,30 +24,40 @@ public static void Example()

 var data = mlContext.Data.LoadFromEnumerable(rawData);

 // Construct the pipeline that would hash the two columns and store the results in new columns.
 // The first transform hashes the string column and the second transform hashes the integer column.
 // Construct the pipeline that would hash the two columns and store the
 // results in new columns. The first transform hashes the string column
 // and the second transform hashes the integer column.
 //
 // Hashing is not a reversible operation, so there is no way to retrive the original value from the hashed value. 
 // Sometimes, for debugging, or model explainability, users will need to know what values in the original columns generated
 // the values in the hashed columns, since the algorithms will mostly use the hashed values for further computations.
 // The Hash method will preserve the mapping from the original values to the hashed values in the Annotations of the 
 // newly created column (column populated with the hashed values). 
 // 
 // Setting the maximumNumberOfInverts parameters to -1 will preserve the full map. 
 // If that parameter is left to the default 0 value, the mapping is not preserved.
 var pipeline = mlContext.Transforms.Conversion.Hash("CategoryHashed", "Category", numberOfBits: 16, maximumNumberOfInverts: -1)
 .Append(mlContext.Transforms.Conversion.Hash("AgeHashed", "Age", numberOfBits: 8));
 // Hashing is not a reversible operation, so there is no way to retrive
 // the original value from the hashed value. Sometimes, for debugging,
 // or model explainability, users will need to know what values in the
 // original columns generated the values in the hashed columns, since
 // the algorithms will mostly use the hashed values for further
 // computations. The Hash method will preserve the mapping from the
 // original values to the hashed values in the Annotations of the newly
 // created column (column populated with the hashed values). 
 //
 // Setting the maximumNumberOfInverts parameters to -1 will preserve the
 // full map. If that parameter is left to the default 0 value, the
 // mapping is not preserved.
 var pipeline = mlContext.Transforms.Conversion.Hash("CategoryHashed",
 "Category", numberOfBits: 16, maximumNumberOfInverts: -1)
 .Append(mlContext.Transforms.Conversion.Hash("AgeHashed", "Age",
 numberOfBits: 8));

 // Let's fit our pipeline, and then apply it to the same data.
 var transformer = pipeline.Fit(data);
 var transformedData = transformer.Transform(data);

 // Convert the post transformation from the IDataView format to an IEnumerable<TransformedData> for easy consumption.
 var convertedData = mlContext.Data.CreateEnumerable<TransformedDataPoint>(transformedData, true);
 // Convert the post transformation from the IDataView format to an
 // IEnumerable <TransformedData> for easy consumption.
 var convertedData = mlContext.Data.CreateEnumerable<
 TransformedDataPoint>(transformedData, true);

 Console.WriteLine("Category CategoryHashed\t Age\t AgeHashed");
 foreach (var item in convertedData)
 Console.WriteLine($"{item.Category}\t {item.CategoryHashed}\t\t {item.Age}\t {item.AgeHashed}");
 Console.WriteLine($"{item.Category}\t {item.CategoryHashed}\t\t " +
 $"{item.Age}\t {item.AgeHashed}");

 // Expected data after the transformation.
 //
Expand All@@ -58,20 +68,24 @@ public static void Example()
 // MLB 36206 18 127
 // MLS 6013 14 62

 // For the Category column, where we set the maximumNumberOfInverts parameter, the names of the original categories, 
 // and their correspondance with the generated hash values is preserved in the Annotations in the format of indices and values. 
 // the indices array will have the hashed values, and the corresponding element, position-wise, in the values array will 
 // contain the original value. 
 // For the Category column, where we set the maximumNumberOfInverts
 // parameter, the names of the original categories, and their
 // correspondance with the generated hash values is preserved in the
 // Annotations in the format of indices and values.the indices array
 // will have the hashed values, and the corresponding element,
 // position -wise, in the values array will contain the original value. 
 //
 // See below for an example on how to retrieve the mapping. 
 var slotNames = new VBuffer<ReadOnlyMemory<char>>();
 transformedData.Schema["CategoryHashed"].Annotations.GetValue("KeyValues", ref slotNames);
 transformedData.Schema["CategoryHashed"].Annotations.GetValue(
 "KeyValues", ref slotNames);

 var indices = slotNames.GetIndices();
 var categoryNames = slotNames.GetValues();

 for (int i = 0; i < indices.Length; i++)
 Console.WriteLine($"The original value of the {indices[i]} category is {categoryNames[i]}");
 Console.WriteLine($"The original value of the {indices[i]} " +
 $"category is {categoryNames[i]}");

 // Output Data
 // 
Expand Down

57 changes: 38 additions & 19 deletions
docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Conversion/KeyToValueToKey.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -11,8 +11,8 @@ public class KeyToValueToKey
 {
 public static void Example()
 {
 // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, 
 // as well as the source of randomness.
 // Create a new ML context, for ML.NET operations. It can be used for
 // exception tracking and logging, as well as the source of randomness.
 var mlContext = new MLContext();

 // Get a small dataset as an IEnumerable.
Expand All@@ -27,25 +27,40 @@ public static void Example()

 // A pipeline to convert the terms of the 'Review' column in 
 // making use of default settings.
 var defaultPipeline = mlContext.Transforms.Text.TokenizeIntoWords("TokenizedText", nameof(DataPoint.Review))
 .Append(mlContext.Transforms.Conversion.MapValueToKey(nameof(TransformedData.Keys), "TokenizedText"));
 var defaultPipeline = mlContext.Transforms.Text.TokenizeIntoWords(
 "TokenizedText", nameof(DataPoint.Review)).Append(mlContext
 .Transforms.Conversion.MapValueToKey(nameof(TransformedData.Keys),
 "TokenizedText"));

@codemzscodemzsJul 2, 2019

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

please move .Append to a new line.
 // Another pipeline, that customizes the advanced settings of the ValueToKeyMappingEstimator.
 // We can change the maximumNumberOfKeys to limit how many keys will get generated out of the set of words, 
 // and condition the order in which they get evaluated by changing keyOrdinality from the default ByOccurence (order in which they get encountered) 
 // to value/alphabetically.
 var customizedPipeline = mlContext.Transforms.Text.TokenizeIntoWords("TokenizedText", nameof(DataPoint.Review))
 .Append(mlContext.Transforms.Conversion.MapValueToKey(nameof(TransformedData.Keys), "TokenizedText", maximumNumberOfKeys: 10,
 keyOrdinality: ValueToKeyMappingEstimator.KeyOrdinality.ByValue));
 // Another pipeline, that customizes the advanced settings of the
 // ValueToKeyMappingEstimator. We can change the maximumNumberOfKeys to
 // limit how many keys will get generated out of the set of words, and
 // condition the order in which they get evaluated by changing
 // keyOrdinality from the default ByOccurence (order in which they get
 // encountered) to value/alphabetically.
 var customizedPipeline = mlContext.Transforms.Text.TokenizeIntoWords(
 "TokenizedText", nameof(DataPoint.Review)).Append(mlContext
 .Transforms.Conversion.MapValueToKey(nameof(TransformedData.Keys),
 "TokenizedText", maximumNumberOfKeys: 10, keyOrdinality:
 ValueToKeyMappingEstimator.KeyOrdinality.ByValue));

 // The transformed data.
 var transformedDataDefault = defaultPipeline.Fit(trainData).Transform(trainData);
 var transformedDataCustomized = customizedPipeline.Fit(trainData).Transform(trainData);
 var transformedDataDefault = defaultPipeline.Fit(trainData).Transform(
 trainData);

 var transformedDataCustomized = customizedPipeline.Fit(trainData)
 .Transform(trainData);

 // Getting the resulting data as an IEnumerable.
 // This will contain the newly created columns.
 IEnumerable<TransformedData> defaultData = mlContext.Data.CreateEnumerable<TransformedData>(transformedDataDefault, reuseRowObject: false);
 IEnumerable<TransformedData> customizedData = mlContext.Data.CreateEnumerable<TransformedData>(transformedDataCustomized, reuseRowObject: false);
 IEnumerable<TransformedData> defaultData = mlContext.Data.
 CreateEnumerable<TransformedData>(transformedDataDefault,
 reuseRowObject: false);

 IEnumerable<TransformedData> customizedData = mlContext.Data.
 CreateEnumerable<TransformedData>(transformedDataCustomized,
 reuseRowObject: false);

 Console.WriteLine($"Keys");
 foreach (var dataRow in defaultData)
 Console.WriteLine($"{string.Join(',', dataRow.Keys)}");
Expand All@@ -65,13 +80,17 @@ public static void Example()
 // 8,2,9,7,6,4
 // 3,10,0,0,0
 // 3,10,0,0,0,8
 // Retrieve the original values, by appending the KeyToValue etimator to the existing pipelines
 // to convert the keys back to the strings.
 var pipeline = defaultPipeline.Append(mlContext.Transforms.Conversion.MapKeyToValue(nameof(TransformedData.Keys)));
 // Retrieve the original values, by appending the KeyToValue etimator to
 // the existing pipelines to convert the keys back to the strings.
 var pipeline = defaultPipeline.Append(mlContext.Transforms.Conversion
 .MapKeyToValue(nameof(TransformedData.Keys)));

 transformedDataDefault = pipeline.Fit(trainData).Transform(trainData);

 // Preview of the DefaultColumnName column obtained.
 var originalColumnBack = transformedDataDefault.GetColumn<VBuffer<ReadOnlyMemory<char>>>(transformedDataDefault.Schema[nameof(TransformedData.Keys)]);
 var originalColumnBack = transformedDataDefault.GetColumn<VBuffer<
 ReadOnlyMemory<char>>>(transformedDataDefault.Schema[nameof(
 TransformedData.Keys)]);

 foreach (var row in originalColumnBack)
 {
Expand Down

21 changes: 13 additions & 8 deletions
docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Conversion/MapKeyToBinaryVector.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -7,14 +7,16 @@ namespace Samples.Dynamic
{
 class MapKeyToBinaryVector
 {
 /// This example demonstrates the use of MapKeyToVector by mapping keys to floats[] of 0 and 1, representing the number in binary format. 
 /// Because the ML.NET KeyType maps the missing value to zero, counting starts at 1, so the uint values
 /// converted to KeyTypes will appear skewed by one. 
 /// This example demonstrates the use of MapKeyToVector by mapping keys to
 /// floats[] of 0 and 1, representing the number in binary format. 
 /// Because the ML.NET KeyType maps the missing value to zero, counting
 /// starts at 1, so the uint values converted to KeyTypes will appear
 /// skewed by one. 
 /// See https://github.com/dotnet/machinelearning/blob/master/docs/code/IDataViewTypeSystem.md#key-types
 public static void Example()
 {
 // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, 
 // as well as the source of randomness.
 // Create a new ML context, for ML.NET operations. It can be used for
 // exception tracking and logging, as well as the source of randomness.
 var mlContext = new MLContext();

 // Get a small dataset as an IEnumerable.
Expand All@@ -30,18 +32,21 @@ public static void Example()
 var data = mlContext.Data.LoadFromEnumerable(rawData);

 // Constructs the ML.net pipeline
 var pipeline = mlContext.Transforms.Conversion.MapKeyToBinaryVector("TimeframeVector", "Timeframe");
 var pipeline = mlContext.Transforms.Conversion.MapKeyToBinaryVector(
 "TimeframeVector", "Timeframe");

 // Fits the pipeline to the data.
 IDataView transformedData = pipeline.Fit(data).Transform(data);

 // Getting the resulting data as an IEnumerable.
 // This will contain the newly created columns.
 IEnumerable<TransformedData> features = mlContext.Data.CreateEnumerable<TransformedData>(transformedData, reuseRowObject: false);
 IEnumerable<TransformedData> features = mlContext.Data.CreateEnumerable<
 TransformedData>(transformedData, reuseRowObject: false);

 Console.WriteLine($" Timeframe TimeframeVector");
 foreach (var featureRow in features)
 Console.WriteLine($"{featureRow.Timeframe}\t\t\t{string.Join(',', featureRow.TimeframeVector)}");
 Console.WriteLine($"{featureRow.Timeframe}\t\t\t" +
 $"{string.Join(',', featureRow.TimeframeVector)}");

 // Timeframe TimeframeVector
 // 10 0,1,0,0,1 //binary representation of 9, the original value
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Force GitHub README to respect dark mode\n(function() {\n var style = document.createElement('style');\n style.textContent = '\n .markdown-body {\n color-scheme: dark light;\n }\n .markdown-body pre { background: #161b22 !important; }\n .markdown-body code { background: rgba(110, 118, 129, 0.4) !important; }\n .markdown-body table th, .markdown-body table td { border-color: #30363d !important; }\n .markdown-body img { background: #0d1117; }\n .markdown-body blockquote { border-left-color: #8b949e; }\n .markdown-body hr { border-color: #30363d; }\n ';\n document.head.appendChild(style);\n})();", "GitHub Dark Mode README Fix");
}
} catch(__e) { console.warn('[Userscript:GitHub Dark Mode README Fix]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
Reformatting Conversion, FeatureSelection and Image Analytics of Transform to Width 85#3943

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

rayankrish
merged 7 commits into

from

Jul 3, 2019
 Merged


Reformatting Conversion, FeatureSelection and Image Analytics of Transform to Width 85
#3943
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


7 commits


Select commit
Hold shift + click to select a range
9267d02

samples/dynamic/transforms/conversions formatted to 85 char

Jul 1, 2019
f2a18ed

samples/dynamic/transforms/featureselection formatted to 85 char

Jul 1, 2019
38c8938

ImageAnalytics

Jul 1, 2019
67080d1

ImageAnalytics

Jul 1, 2019
ecd2738

small corrections to formatting

Jul 2, 2019
91532b0

Merge branch 'transform85' of https://github.com/RadicalRayan/machine…

Jul 2, 2019
3deb533

minor tab and empty line changes

Jul 2, 2019

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


13 changes: 9 additions & 4 deletions
docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Conversion/ConvertType.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -20,17 +20,22 @@ public static void Example()
 var data = mlContext.Data.LoadFromEnumerable(rawData);

 // Construct the pipeline.
 var pipeline = mlContext.Transforms.Conversion.ConvertType("SurvivedInt32", "Survived", DataKind.Int32);
 var pipeline = mlContext.Transforms.Conversion.ConvertType(
 "SurvivedInt32", "Survived", DataKind.Int32);

 // Let's train our pipeline, and then apply it to the same data.
 var transformer = pipeline.Fit(data);
 var transformedData = transformer.Transform(data);

 // Display original column 'Survived' (boolean) and converted column 'SurvivedInt32' (Int32)
 var convertedData = mlContext.Data.CreateEnumerable<TransformedData>(transformedData, true);
 // Display original column 'Survived' (boolean) and converted column 
 // SurvivedInt32' (Int32)
 var convertedData = mlContext.Data.CreateEnumerable<TransformedData>(
 transformedData, true);

 foreach (var item in convertedData)
@codemzscodemzsJul 2, 2019

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

new line between any two lines that were broken down into multiple lines but no new line between multiple breakdowns of a single line.
 {
 Console.WriteLine("A:{0,-10} Aconv:{1}", item.Survived, item.SurvivedInt32);
 Console.WriteLine("A:{0,-10} Aconv:{1}", item.Survived,
 item.SurvivedInt32);
 }

 // Output
Expand Down

38 changes: 26 additions & 12 deletions
docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Conversion/ConvertTypeMultiColumn.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -4,22 +4,33 @@

namespace Samples.Dynamic
{
 // This example illustrates how to convert multiple columns of different types to one type, in this case System.Single. 
 // This is often a useful data transformation before concatenating the features together and passing them to a particular estimator.
 // This example illustrates how to convert multiple columns of different types
 // to one type, in this case System.Single. 
 // This is often a useful data transformation before concatenating the features
 // together and passing them to a particular estimator.
 public static class ConvertTypeMultiColumn
 { 
 public static void Example()
 {
 // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, 
 // as well as the source of randomness.
 // Create a new ML context, for ML.NET operations. It can be used for
 // exception tracking and logging, as well as the source of randomness.
 var mlContext = new MLContext(seed: 1);

 var rawData = new[] {
 new InputData() { Feature1 = true, Feature2 = "0.4", Feature3 = DateTime.Now, Feature4 = 0.145},
 new InputData() { Feature1 = false, Feature2 = "0.5", Feature3 = DateTime.Today, Feature4 = 3.14},
 new InputData() { Feature1 = false, Feature2 = "14", Feature3 = DateTime.Today, Feature4 = 0.2046},
 new InputData() { Feature1 = false, Feature2 = "23", Feature3 = DateTime.Now, Feature4 = 0.1206},
 new InputData() { Feature1 = true, Feature2 = "8904", Feature3 = DateTime.UtcNow, Feature4 = 8.09},
 new InputData() { Feature1 = true, Feature2 = "0.4",
 Feature3 = DateTime.Now, Feature4 = 0.145},

 new InputData() { Feature1 = false, Feature2 = "0.5",
 Feature3 = DateTime.Today, Feature4 = 3.14},

 new InputData() { Feature1 = false, Feature2 = "14",
 Feature3 = DateTime.Today, Feature4 = 0.2046},

 new InputData() { Feature1 = false, Feature2 = "23",
 Feature3 = DateTime.Now, Feature4 = 0.1206},

 new InputData() { Feature1 = true, Feature2 = "8904",
 Feature3 = DateTime.UtcNow, Feature4 = 8.09},
 };

 // Convert the data to an IDataView.
Expand All@@ -37,17 +48,20 @@ public static void Example()

 // Let's fit our pipeline to the data.
 var transformer = pipeline.Fit(data);
 // Transforming the same data. This will add the 4 columns defined in the pipeline, containing the converted
 // Transforming the same data. This will add the 4 columns defined in
 // the pipeline, containing the converted
 // values of the initial columns. 
 var transformedData = transformer.Transform(data);

 // Shape the transformed data as a strongly typed IEnumerable.
 var convertedData = mlContext.Data.CreateEnumerable<TransformedData>(transformedData, true);
 var convertedData = mlContext.Data.CreateEnumerable<TransformedData>(
 transformedData, true);

 // Printing the results.
 Console.WriteLine("Converted1\t Converted2\t Converted3\t Converted4");
 foreach (var item in convertedData)
 Console.WriteLine($"\t{item.Converted1}\t {item.Converted2}\t\t {item.Converted3}\t {item.Converted4}");
 Console.WriteLine($"\t{item.Converted1}\t {item.Converted2}\t\t " +
 $"{item.Converted3}\t {item.Converted4}");

 // Transformed data.
 //
Expand Down

60 changes: 37 additions & 23 deletions
docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Conversion/Hash.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -9,8 +9,8 @@ public static class Hash
 {
 public static void Example()
 {
 // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, 
 // as well as the source of randomness.
 // Create a new ML context, for ML.NET operations. It can be used for
 // exception tracking and logging, as well as the source of randomness.
 var mlContext = new MLContext(seed: 1);

 // Get a small dataset as an IEnumerable.
Expand All@@ -24,30 +24,40 @@ public static void Example()

 var data = mlContext.Data.LoadFromEnumerable(rawData);

 // Construct the pipeline that would hash the two columns and store the results in new columns.
 // The first transform hashes the string column and the second transform hashes the integer column.
 // Construct the pipeline that would hash the two columns and store the
 // results in new columns. The first transform hashes the string column
 // and the second transform hashes the integer column.
 //
 // Hashing is not a reversible operation, so there is no way to retrive the original value from the hashed value. 
 // Sometimes, for debugging, or model explainability, users will need to know what values in the original columns generated
 // the values in the hashed columns, since the algorithms will mostly use the hashed values for further computations.
 // The Hash method will preserve the mapping from the original values to the hashed values in the Annotations of the 
 // newly created column (column populated with the hashed values). 
 // 
 // Setting the maximumNumberOfInverts parameters to -1 will preserve the full map. 
 // If that parameter is left to the default 0 value, the mapping is not preserved.
 var pipeline = mlContext.Transforms.Conversion.Hash("CategoryHashed", "Category", numberOfBits: 16, maximumNumberOfInverts: -1)
 .Append(mlContext.Transforms.Conversion.Hash("AgeHashed", "Age", numberOfBits: 8));
 // Hashing is not a reversible operation, so there is no way to retrive
 // the original value from the hashed value. Sometimes, for debugging,
 // or model explainability, users will need to know what values in the
 // original columns generated the values in the hashed columns, since
 // the algorithms will mostly use the hashed values for further
 // computations. The Hash method will preserve the mapping from the
 // original values to the hashed values in the Annotations of the newly
 // created column (column populated with the hashed values). 
 //
 // Setting the maximumNumberOfInverts parameters to -1 will preserve the
 // full map. If that parameter is left to the default 0 value, the
 // mapping is not preserved.
 var pipeline = mlContext.Transforms.Conversion.Hash("CategoryHashed",
 "Category", numberOfBits: 16, maximumNumberOfInverts: -1)
 .Append(mlContext.Transforms.Conversion.Hash("AgeHashed", "Age",
 numberOfBits: 8));

 // Let's fit our pipeline, and then apply it to the same data.
 var transformer = pipeline.Fit(data);
 var transformedData = transformer.Transform(data);

 // Convert the post transformation from the IDataView format to an IEnumerable<TransformedData> for easy consumption.
 var convertedData = mlContext.Data.CreateEnumerable<TransformedDataPoint>(transformedData, true);
 // Convert the post transformation from the IDataView format to an
 // IEnumerable <TransformedData> for easy consumption.
 var convertedData = mlContext.Data.CreateEnumerable<
 TransformedDataPoint>(transformedData, true);

 Console.WriteLine("Category CategoryHashed\t Age\t AgeHashed");
 foreach (var item in convertedData)
 Console.WriteLine($"{item.Category}\t {item.CategoryHashed}\t\t {item.Age}\t {item.AgeHashed}");
 Console.WriteLine($"{item.Category}\t {item.CategoryHashed}\t\t " +
 $"{item.Age}\t {item.AgeHashed}");

 // Expected data after the transformation.
 //
Expand All@@ -58,20 +68,24 @@ public static void Example()
 // MLB 36206 18 127
 // MLS 6013 14 62

 // For the Category column, where we set the maximumNumberOfInverts parameter, the names of the original categories, 
 // and their correspondance with the generated hash values is preserved in the Annotations in the format of indices and values. 
 // the indices array will have the hashed values, and the corresponding element, position-wise, in the values array will 
 // contain the original value. 
 // For the Category column, where we set the maximumNumberOfInverts
 // parameter, the names of the original categories, and their
 // correspondance with the generated hash values is preserved in the
 // Annotations in the format of indices and values.the indices array
 // will have the hashed values, and the corresponding element,
 // position -wise, in the values array will contain the original value. 
 //
 // See below for an example on how to retrieve the mapping. 
 var slotNames = new VBuffer<ReadOnlyMemory<char>>();
 transformedData.Schema["CategoryHashed"].Annotations.GetValue("KeyValues", ref slotNames);
 transformedData.Schema["CategoryHashed"].Annotations.GetValue(
 "KeyValues", ref slotNames);

 var indices = slotNames.GetIndices();
 var categoryNames = slotNames.GetValues();

 for (int i = 0; i < indices.Length; i++)
 Console.WriteLine($"The original value of the {indices[i]} category is {categoryNames[i]}");
 Console.WriteLine($"The original value of the {indices[i]} " +
 $"category is {categoryNames[i]}");

 // Output Data
 // 
Expand Down

57 changes: 38 additions & 19 deletions
docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Conversion/KeyToValueToKey.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -11,8 +11,8 @@ public class KeyToValueToKey
 {
 public static void Example()
 {
 // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, 
 // as well as the source of randomness.
 // Create a new ML context, for ML.NET operations. It can be used for
 // exception tracking and logging, as well as the source of randomness.
 var mlContext = new MLContext();

 // Get a small dataset as an IEnumerable.
Expand All@@ -27,25 +27,40 @@ public static void Example()

 // A pipeline to convert the terms of the 'Review' column in 
 // making use of default settings.
 var defaultPipeline = mlContext.Transforms.Text.TokenizeIntoWords("TokenizedText", nameof(DataPoint.Review))
 .Append(mlContext.Transforms.Conversion.MapValueToKey(nameof(TransformedData.Keys), "TokenizedText"));
 var defaultPipeline = mlContext.Transforms.Text.TokenizeIntoWords(
 "TokenizedText", nameof(DataPoint.Review)).Append(mlContext
 .Transforms.Conversion.MapValueToKey(nameof(TransformedData.Keys),
 "TokenizedText"));

@codemzscodemzsJul 2, 2019

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

please move .Append to a new line.
 // Another pipeline, that customizes the advanced settings of the ValueToKeyMappingEstimator.
 // We can change the maximumNumberOfKeys to limit how many keys will get generated out of the set of words, 
 // and condition the order in which they get evaluated by changing keyOrdinality from the default ByOccurence (order in which they get encountered) 
 // to value/alphabetically.
 var customizedPipeline = mlContext.Transforms.Text.TokenizeIntoWords("TokenizedText", nameof(DataPoint.Review))
 .Append(mlContext.Transforms.Conversion.MapValueToKey(nameof(TransformedData.Keys), "TokenizedText", maximumNumberOfKeys: 10,
 keyOrdinality: ValueToKeyMappingEstimator.KeyOrdinality.ByValue));
 // Another pipeline, that customizes the advanced settings of the
 // ValueToKeyMappingEstimator. We can change the maximumNumberOfKeys to
 // limit how many keys will get generated out of the set of words, and
 // condition the order in which they get evaluated by changing
 // keyOrdinality from the default ByOccurence (order in which they get
 // encountered) to value/alphabetically.
 var customizedPipeline = mlContext.Transforms.Text.TokenizeIntoWords(
 "TokenizedText", nameof(DataPoint.Review)).Append(mlContext
 .Transforms.Conversion.MapValueToKey(nameof(TransformedData.Keys),
 "TokenizedText", maximumNumberOfKeys: 10, keyOrdinality:
 ValueToKeyMappingEstimator.KeyOrdinality.ByValue));

 // The transformed data.
 var transformedDataDefault = defaultPipeline.Fit(trainData).Transform(trainData);
 var transformedDataCustomized = customizedPipeline.Fit(trainData).Transform(trainData);
 var transformedDataDefault = defaultPipeline.Fit(trainData).Transform(
 trainData);

 var transformedDataCustomized = customizedPipeline.Fit(trainData)
 .Transform(trainData);

 // Getting the resulting data as an IEnumerable.
 // This will contain the newly created columns.
 IEnumerable<TransformedData> defaultData = mlContext.Data.CreateEnumerable<TransformedData>(transformedDataDefault, reuseRowObject: false);
 IEnumerable<TransformedData> customizedData = mlContext.Data.CreateEnumerable<TransformedData>(transformedDataCustomized, reuseRowObject: false);
 IEnumerable<TransformedData> defaultData = mlContext.Data.
 CreateEnumerable<TransformedData>(transformedDataDefault,
 reuseRowObject: false);

 IEnumerable<TransformedData> customizedData = mlContext.Data.
 CreateEnumerable<TransformedData>(transformedDataCustomized,
 reuseRowObject: false);

 Console.WriteLine($"Keys");
 foreach (var dataRow in defaultData)
 Console.WriteLine($"{string.Join(',', dataRow.Keys)}");
Expand All@@ -65,13 +80,17 @@ public static void Example()
 // 8,2,9,7,6,4
 // 3,10,0,0,0
 // 3,10,0,0,0,8
 // Retrieve the original values, by appending the KeyToValue etimator to the existing pipelines
 // to convert the keys back to the strings.
 var pipeline = defaultPipeline.Append(mlContext.Transforms.Conversion.MapKeyToValue(nameof(TransformedData.Keys)));
 // Retrieve the original values, by appending the KeyToValue etimator to
 // the existing pipelines to convert the keys back to the strings.
 var pipeline = defaultPipeline.Append(mlContext.Transforms.Conversion
 .MapKeyToValue(nameof(TransformedData.Keys)));

 transformedDataDefault = pipeline.Fit(trainData).Transform(trainData);

 // Preview of the DefaultColumnName column obtained.
 var originalColumnBack = transformedDataDefault.GetColumn<VBuffer<ReadOnlyMemory<char>>>(transformedDataDefault.Schema[nameof(TransformedData.Keys)]);
 var originalColumnBack = transformedDataDefault.GetColumn<VBuffer<
 ReadOnlyMemory<char>>>(transformedDataDefault.Schema[nameof(
 TransformedData.Keys)]);

 foreach (var row in originalColumnBack)
 {
Expand Down

21 changes: 13 additions & 8 deletions
docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Conversion/MapKeyToBinaryVector.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -7,14 +7,16 @@ namespace Samples.Dynamic
{
 class MapKeyToBinaryVector
 {
 /// This example demonstrates the use of MapKeyToVector by mapping keys to floats[] of 0 and 1, representing the number in binary format. 
 /// Because the ML.NET KeyType maps the missing value to zero, counting starts at 1, so the uint values
 /// converted to KeyTypes will appear skewed by one. 
 /// This example demonstrates the use of MapKeyToVector by mapping keys to
 /// floats[] of 0 and 1, representing the number in binary format. 
 /// Because the ML.NET KeyType maps the missing value to zero, counting
 /// starts at 1, so the uint values converted to KeyTypes will appear
 /// skewed by one. 
 /// See https://github.com/dotnet/machinelearning/blob/master/docs/code/IDataViewTypeSystem.md#key-types
 public static void Example()
 {
 // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, 
 // as well as the source of randomness.
 // Create a new ML context, for ML.NET operations. It can be used for
 // exception tracking and logging, as well as the source of randomness.
 var mlContext = new MLContext();

 // Get a small dataset as an IEnumerable.
Expand All@@ -30,18 +32,21 @@ public static void Example()
 var data = mlContext.Data.LoadFromEnumerable(rawData);

 // Constructs the ML.net pipeline
 var pipeline = mlContext.Transforms.Conversion.MapKeyToBinaryVector("TimeframeVector", "Timeframe");
 var pipeline = mlContext.Transforms.Conversion.MapKeyToBinaryVector(
 "TimeframeVector", "Timeframe");

 // Fits the pipeline to the data.
 IDataView transformedData = pipeline.Fit(data).Transform(data);

 // Getting the resulting data as an IEnumerable.
 // This will contain the newly created columns.
 IEnumerable<TransformedData> features = mlContext.Data.CreateEnumerable<TransformedData>(transformedData, reuseRowObject: false);
 IEnumerable<TransformedData> features = mlContext.Data.CreateEnumerable<
 TransformedData>(transformedData, reuseRowObject: false);

 Console.WriteLine($" Timeframe TimeframeVector");
 foreach (var featureRow in features)
 Console.WriteLine($"{featureRow.Timeframe}\t\t\t{string.Join(',', featureRow.TimeframeVector)}");
 Console.WriteLine($"{featureRow.Timeframe}\t\t\t" +
 $"{string.Join(',', featureRow.TimeframeVector)}");

 // Timeframe TimeframeVector
 // 10 0,1,0,0,1 //binary representation of 9, the original value
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Highlight search terms from Google/DuckDuckGo/Bing referrer\n(function() {\n var ref = document.referrer;\n var terms = [];\n \n if (ref.includes('google.com') || ref.includes('duckduckgo.com') || ref.includes('bing.com')) {\n var url = new URL(ref);\n var q = url.searchParams.get('q') || url.searchParams.get('p');\n if (q) {\n terms = q.split(/\\s+/).filter(function(t) { return t.length > 2; });\n }\n }\n \n if (terms.length === 0) return;\n \n var style = document.createElement('style');\n style.textContent = '.userscript-highlight { background: #fbbf24; color: #1a1a2e; padding: 1px 3px; border-radius: 2px; }';\n document.head.appendChild(style);\n \n function highlight(node) {\n if (node.nodeType === 3) { // text node\n var text = node.textContent;\n var found = false;\n terms.forEach(function(term) {\n var regex = new RegExp('(' + term.replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\') + ')', 'gi');\n if (regex.test(text)) {\n found = true;\n var frag = document.createDocumentFragment();\n var parts = text.split(regex);\n parts.forEach(function(part, i) {\n if (i % 2 === 0) {\n frag.appendChild(document.createTextNode(part));\n } else {\n var span = document.createElement('span');\n span.className = 'userscript-highlight';\n span.textContent = part;\n frag.appendChild(span);\n }\n });\n node.parentNode.replaceChild(frag, node);\n }\n });\n } else if (node.nodeType === 1 && node.childNodes) { // element\n var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT'];\n if (!skipTags.includes(node.tagName)) {\n Array.from(node.childNodes).forEach(highlight);\n }\n }\n }\n \n highlight(document.body);\n \n // Re-highlight on dynamic content\n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1 || node.nodeType === 3) highlight(node);\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Highlight Search Terms");
}
} catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
Reformatting Conversion, FeatureSelection and Image Analytics of Transform to Width 85#3943

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

rayankrish
merged 7 commits into

from

Jul 3, 2019
 Merged


Reformatting Conversion, FeatureSelection and Image Analytics of Transform to Width 85
#3943
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


7 commits


Select commit
Hold shift + click to select a range
9267d02

samples/dynamic/transforms/conversions formatted to 85 char

Jul 1, 2019
f2a18ed

samples/dynamic/transforms/featureselection formatted to 85 char

Jul 1, 2019
38c8938

ImageAnalytics

Jul 1, 2019
67080d1

ImageAnalytics

Jul 1, 2019
ecd2738

small corrections to formatting

Jul 2, 2019
91532b0

Merge branch 'transform85' of https://github.com/RadicalRayan/machine…

Jul 2, 2019
3deb533

minor tab and empty line changes

Jul 2, 2019

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


13 changes: 9 additions & 4 deletions
docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Conversion/ConvertType.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -20,17 +20,22 @@ public static void Example()
 var data = mlContext.Data.LoadFromEnumerable(rawData);

 // Construct the pipeline.
 var pipeline = mlContext.Transforms.Conversion.ConvertType("SurvivedInt32", "Survived", DataKind.Int32);
 var pipeline = mlContext.Transforms.Conversion.ConvertType(
 "SurvivedInt32", "Survived", DataKind.Int32);

 // Let's train our pipeline, and then apply it to the same data.
 var transformer = pipeline.Fit(data);
 var transformedData = transformer.Transform(data);

 // Display original column 'Survived' (boolean) and converted column 'SurvivedInt32' (Int32)
 var convertedData = mlContext.Data.CreateEnumerable<TransformedData>(transformedData, true);
 // Display original column 'Survived' (boolean) and converted column 
 // SurvivedInt32' (Int32)
 var convertedData = mlContext.Data.CreateEnumerable<TransformedData>(
 transformedData, true);

 foreach (var item in convertedData)
@codemzscodemzsJul 2, 2019

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

new line between any two lines that were broken down into multiple lines but no new line between multiple breakdowns of a single line.
 {
 Console.WriteLine("A:{0,-10} Aconv:{1}", item.Survived, item.SurvivedInt32);
 Console.WriteLine("A:{0,-10} Aconv:{1}", item.Survived,
 item.SurvivedInt32);
 }

 // Output
Expand Down

38 changes: 26 additions & 12 deletions
docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Conversion/ConvertTypeMultiColumn.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -4,22 +4,33 @@

namespace Samples.Dynamic
{
 // This example illustrates how to convert multiple columns of different types to one type, in this case System.Single. 
 // This is often a useful data transformation before concatenating the features together and passing them to a particular estimator.
 // This example illustrates how to convert multiple columns of different types
 // to one type, in this case System.Single. 
 // This is often a useful data transformation before concatenating the features
 // together and passing them to a particular estimator.
 public static class ConvertTypeMultiColumn
 { 
 public static void Example()
 {
 // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, 
 // as well as the source of randomness.
 // Create a new ML context, for ML.NET operations. It can be used for
 // exception tracking and logging, as well as the source of randomness.
 var mlContext = new MLContext(seed: 1);

 var rawData = new[] {
 new InputData() { Feature1 = true, Feature2 = "0.4", Feature3 = DateTime.Now, Feature4 = 0.145},
 new InputData() { Feature1 = false, Feature2 = "0.5", Feature3 = DateTime.Today, Feature4 = 3.14},
 new InputData() { Feature1 = false, Feature2 = "14", Feature3 = DateTime.Today, Feature4 = 0.2046},
 new InputData() { Feature1 = false, Feature2 = "23", Feature3 = DateTime.Now, Feature4 = 0.1206},
 new InputData() { Feature1 = true, Feature2 = "8904", Feature3 = DateTime.UtcNow, Feature4 = 8.09},
 new InputData() { Feature1 = true, Feature2 = "0.4",
 Feature3 = DateTime.Now, Feature4 = 0.145},

 new InputData() { Feature1 = false, Feature2 = "0.5",
 Feature3 = DateTime.Today, Feature4 = 3.14},

 new InputData() { Feature1 = false, Feature2 = "14",
 Feature3 = DateTime.Today, Feature4 = 0.2046},

 new InputData() { Feature1 = false, Feature2 = "23",
 Feature3 = DateTime.Now, Feature4 = 0.1206},

 new InputData() { Feature1 = true, Feature2 = "8904",
 Feature3 = DateTime.UtcNow, Feature4 = 8.09},
 };

 // Convert the data to an IDataView.
Expand All@@ -37,17 +48,20 @@ public static void Example()

 // Let's fit our pipeline to the data.
 var transformer = pipeline.Fit(data);
 // Transforming the same data. This will add the 4 columns defined in the pipeline, containing the converted
 // Transforming the same data. This will add the 4 columns defined in
 // the pipeline, containing the converted
 // values of the initial columns. 
 var transformedData = transformer.Transform(data);

 // Shape the transformed data as a strongly typed IEnumerable.
 var convertedData = mlContext.Data.CreateEnumerable<TransformedData>(transformedData, true);
 var convertedData = mlContext.Data.CreateEnumerable<TransformedData>(
 transformedData, true);

 // Printing the results.
 Console.WriteLine("Converted1\t Converted2\t Converted3\t Converted4");
 foreach (var item in convertedData)
 Console.WriteLine($"\t{item.Converted1}\t {item.Converted2}\t\t {item.Converted3}\t {item.Converted4}");
 Console.WriteLine($"\t{item.Converted1}\t {item.Converted2}\t\t " +
 $"{item.Converted3}\t {item.Converted4}");

 // Transformed data.
 //
Expand Down

60 changes: 37 additions & 23 deletions
docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Conversion/Hash.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -9,8 +9,8 @@ public static class Hash
 {
 public static void Example()
 {
 // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, 
 // as well as the source of randomness.
 // Create a new ML context, for ML.NET operations. It can be used for
 // exception tracking and logging, as well as the source of randomness.
 var mlContext = new MLContext(seed: 1);

 // Get a small dataset as an IEnumerable.
Expand All@@ -24,30 +24,40 @@ public static void Example()

 var data = mlContext.Data.LoadFromEnumerable(rawData);

 // Construct the pipeline that would hash the two columns and store the results in new columns.
 // The first transform hashes the string column and the second transform hashes the integer column.
 // Construct the pipeline that would hash the two columns and store the
 // results in new columns. The first transform hashes the string column
 // and the second transform hashes the integer column.
 //
 // Hashing is not a reversible operation, so there is no way to retrive the original value from the hashed value. 
 // Sometimes, for debugging, or model explainability, users will need to know what values in the original columns generated
 // the values in the hashed columns, since the algorithms will mostly use the hashed values for further computations.
 // The Hash method will preserve the mapping from the original values to the hashed values in the Annotations of the 
 // newly created column (column populated with the hashed values). 
 // 
 // Setting the maximumNumberOfInverts parameters to -1 will preserve the full map. 
 // If that parameter is left to the default 0 value, the mapping is not preserved.
 var pipeline = mlContext.Transforms.Conversion.Hash("CategoryHashed", "Category", numberOfBits: 16, maximumNumberOfInverts: -1)
 .Append(mlContext.Transforms.Conversion.Hash("AgeHashed", "Age", numberOfBits: 8));
 // Hashing is not a reversible operation, so there is no way to retrive
 // the original value from the hashed value. Sometimes, for debugging,
 // or model explainability, users will need to know what values in the
 // original columns generated the values in the hashed columns, since
 // the algorithms will mostly use the hashed values for further
 // computations. The Hash method will preserve the mapping from the
 // original values to the hashed values in the Annotations of the newly
 // created column (column populated with the hashed values). 
 //
 // Setting the maximumNumberOfInverts parameters to -1 will preserve the
 // full map. If that parameter is left to the default 0 value, the
 // mapping is not preserved.
 var pipeline = mlContext.Transforms.Conversion.Hash("CategoryHashed",
 "Category", numberOfBits: 16, maximumNumberOfInverts: -1)
 .Append(mlContext.Transforms.Conversion.Hash("AgeHashed", "Age",
 numberOfBits: 8));

 // Let's fit our pipeline, and then apply it to the same data.
 var transformer = pipeline.Fit(data);
 var transformedData = transformer.Transform(data);

 // Convert the post transformation from the IDataView format to an IEnumerable<TransformedData> for easy consumption.
 var convertedData = mlContext.Data.CreateEnumerable<TransformedDataPoint>(transformedData, true);
 // Convert the post transformation from the IDataView format to an
 // IEnumerable <TransformedData> for easy consumption.
 var convertedData = mlContext.Data.CreateEnumerable<
 TransformedDataPoint>(transformedData, true);

 Console.WriteLine("Category CategoryHashed\t Age\t AgeHashed");
 foreach (var item in convertedData)
 Console.WriteLine($"{item.Category}\t {item.CategoryHashed}\t\t {item.Age}\t {item.AgeHashed}");
 Console.WriteLine($"{item.Category}\t {item.CategoryHashed}\t\t " +
 $"{item.Age}\t {item.AgeHashed}");

 // Expected data after the transformation.
 //
Expand All@@ -58,20 +68,24 @@ public static void Example()
 // MLB 36206 18 127
 // MLS 6013 14 62

 // For the Category column, where we set the maximumNumberOfInverts parameter, the names of the original categories, 
 // and their correspondance with the generated hash values is preserved in the Annotations in the format of indices and values. 
 // the indices array will have the hashed values, and the corresponding element, position-wise, in the values array will 
 // contain the original value. 
 // For the Category column, where we set the maximumNumberOfInverts
 // parameter, the names of the original categories, and their
 // correspondance with the generated hash values is preserved in the
 // Annotations in the format of indices and values.the indices array
 // will have the hashed values, and the corresponding element,
 // position -wise, in the values array will contain the original value. 
 //
 // See below for an example on how to retrieve the mapping. 
 var slotNames = new VBuffer<ReadOnlyMemory<char>>();
 transformedData.Schema["CategoryHashed"].Annotations.GetValue("KeyValues", ref slotNames);
 transformedData.Schema["CategoryHashed"].Annotations.GetValue(
 "KeyValues", ref slotNames);

 var indices = slotNames.GetIndices();
 var categoryNames = slotNames.GetValues();

 for (int i = 0; i < indices.Length; i++)
 Console.WriteLine($"The original value of the {indices[i]} category is {categoryNames[i]}");
 Console.WriteLine($"The original value of the {indices[i]} " +
 $"category is {categoryNames[i]}");

 // Output Data
 // 
Expand Down

57 changes: 38 additions & 19 deletions
docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Conversion/KeyToValueToKey.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -11,8 +11,8 @@ public class KeyToValueToKey
 {
 public static void Example()
 {
 // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, 
 // as well as the source of randomness.
 // Create a new ML context, for ML.NET operations. It can be used for
 // exception tracking and logging, as well as the source of randomness.
 var mlContext = new MLContext();

 // Get a small dataset as an IEnumerable.
Expand All@@ -27,25 +27,40 @@ public static void Example()

 // A pipeline to convert the terms of the 'Review' column in 
 // making use of default settings.
 var defaultPipeline = mlContext.Transforms.Text.TokenizeIntoWords("TokenizedText", nameof(DataPoint.Review))
 .Append(mlContext.Transforms.Conversion.MapValueToKey(nameof(TransformedData.Keys), "TokenizedText"));
 var defaultPipeline = mlContext.Transforms.Text.TokenizeIntoWords(
 "TokenizedText", nameof(DataPoint.Review)).Append(mlContext
 .Transforms.Conversion.MapValueToKey(nameof(TransformedData.Keys),
 "TokenizedText"));

@codemzscodemzsJul 2, 2019

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

please move .Append to a new line.
 // Another pipeline, that customizes the advanced settings of the ValueToKeyMappingEstimator.
 // We can change the maximumNumberOfKeys to limit how many keys will get generated out of the set of words, 
 // and condition the order in which they get evaluated by changing keyOrdinality from the default ByOccurence (order in which they get encountered) 
 // to value/alphabetically.
 var customizedPipeline = mlContext.Transforms.Text.TokenizeIntoWords("TokenizedText", nameof(DataPoint.Review))
 .Append(mlContext.Transforms.Conversion.MapValueToKey(nameof(TransformedData.Keys), "TokenizedText", maximumNumberOfKeys: 10,
 keyOrdinality: ValueToKeyMappingEstimator.KeyOrdinality.ByValue));
 // Another pipeline, that customizes the advanced settings of the
 // ValueToKeyMappingEstimator. We can change the maximumNumberOfKeys to
 // limit how many keys will get generated out of the set of words, and
 // condition the order in which they get evaluated by changing
 // keyOrdinality from the default ByOccurence (order in which they get
 // encountered) to value/alphabetically.
 var customizedPipeline = mlContext.Transforms.Text.TokenizeIntoWords(
 "TokenizedText", nameof(DataPoint.Review)).Append(mlContext
 .Transforms.Conversion.MapValueToKey(nameof(TransformedData.Keys),
 "TokenizedText", maximumNumberOfKeys: 10, keyOrdinality:
 ValueToKeyMappingEstimator.KeyOrdinality.ByValue));

 // The transformed data.
 var transformedDataDefault = defaultPipeline.Fit(trainData).Transform(trainData);
 var transformedDataCustomized = customizedPipeline.Fit(trainData).Transform(trainData);
 var transformedDataDefault = defaultPipeline.Fit(trainData).Transform(
 trainData);

 var transformedDataCustomized = customizedPipeline.Fit(trainData)
 .Transform(trainData);

 // Getting the resulting data as an IEnumerable.
 // This will contain the newly created columns.
 IEnumerable<TransformedData> defaultData = mlContext.Data.CreateEnumerable<TransformedData>(transformedDataDefault, reuseRowObject: false);
 IEnumerable<TransformedData> customizedData = mlContext.Data.CreateEnumerable<TransformedData>(transformedDataCustomized, reuseRowObject: false);
 IEnumerable<TransformedData> defaultData = mlContext.Data.
 CreateEnumerable<TransformedData>(transformedDataDefault,
 reuseRowObject: false);

 IEnumerable<TransformedData> customizedData = mlContext.Data.
 CreateEnumerable<TransformedData>(transformedDataCustomized,
 reuseRowObject: false);

 Console.WriteLine($"Keys");
 foreach (var dataRow in defaultData)
 Console.WriteLine($"{string.Join(',', dataRow.Keys)}");
Expand All@@ -65,13 +80,17 @@ public static void Example()
 // 8,2,9,7,6,4
 // 3,10,0,0,0
 // 3,10,0,0,0,8
 // Retrieve the original values, by appending the KeyToValue etimator to the existing pipelines
 // to convert the keys back to the strings.
 var pipeline = defaultPipeline.Append(mlContext.Transforms.Conversion.MapKeyToValue(nameof(TransformedData.Keys)));
 // Retrieve the original values, by appending the KeyToValue etimator to
 // the existing pipelines to convert the keys back to the strings.
 var pipeline = defaultPipeline.Append(mlContext.Transforms.Conversion
 .MapKeyToValue(nameof(TransformedData.Keys)));

 transformedDataDefault = pipeline.Fit(trainData).Transform(trainData);

 // Preview of the DefaultColumnName column obtained.
 var originalColumnBack = transformedDataDefault.GetColumn<VBuffer<ReadOnlyMemory<char>>>(transformedDataDefault.Schema[nameof(TransformedData.Keys)]);
 var originalColumnBack = transformedDataDefault.GetColumn<VBuffer<
 ReadOnlyMemory<char>>>(transformedDataDefault.Schema[nameof(
 TransformedData.Keys)]);

 foreach (var row in originalColumnBack)
 {
Expand Down

21 changes: 13 additions & 8 deletions
docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Conversion/MapKeyToBinaryVector.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -7,14 +7,16 @@ namespace Samples.Dynamic
{
 class MapKeyToBinaryVector
 {
 /// This example demonstrates the use of MapKeyToVector by mapping keys to floats[] of 0 and 1, representing the number in binary format. 
 /// Because the ML.NET KeyType maps the missing value to zero, counting starts at 1, so the uint values
 /// converted to KeyTypes will appear skewed by one. 
 /// This example demonstrates the use of MapKeyToVector by mapping keys to
 /// floats[] of 0 and 1, representing the number in binary format. 
 /// Because the ML.NET KeyType maps the missing value to zero, counting
 /// starts at 1, so the uint values converted to KeyTypes will appear
 /// skewed by one. 
 /// See https://github.com/dotnet/machinelearning/blob/master/docs/code/IDataViewTypeSystem.md#key-types
 public static void Example()
 {
 // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, 
 // as well as the source of randomness.
 // Create a new ML context, for ML.NET operations. It can be used for
 // exception tracking and logging, as well as the source of randomness.
 var mlContext = new MLContext();

 // Get a small dataset as an IEnumerable.
Expand All@@ -30,18 +32,21 @@ public static void Example()
 var data = mlContext.Data.LoadFromEnumerable(rawData);

 // Constructs the ML.net pipeline
 var pipeline = mlContext.Transforms.Conversion.MapKeyToBinaryVector("TimeframeVector", "Timeframe");
 var pipeline = mlContext.Transforms.Conversion.MapKeyToBinaryVector(
 "TimeframeVector", "Timeframe");

 // Fits the pipeline to the data.
 IDataView transformedData = pipeline.Fit(data).Transform(data);

 // Getting the resulting data as an IEnumerable.
 // This will contain the newly created columns.
 IEnumerable<TransformedData> features = mlContext.Data.CreateEnumerable<TransformedData>(transformedData, reuseRowObject: false);
 IEnumerable<TransformedData> features = mlContext.Data.CreateEnumerable<
 TransformedData>(transformedData, reuseRowObject: false);

 Console.WriteLine($" Timeframe TimeframeVector");
 foreach (var featureRow in features)
 Console.WriteLine($"{featureRow.Timeframe}\t\t\t{string.Join(',', featureRow.TimeframeVector)}");
 Console.WriteLine($"{featureRow.Timeframe}\t\t\t" +
 $"{string.Join(',', featureRow.TimeframeVector)}");

 // Timeframe TimeframeVector
 // 10 0,1,0,0,1 //binary representation of 9, the original value
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Strip utm_, fbclid, gclid, etc. from all links on page\n(function() {\n var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',\n 'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid',\n 'ref', 'ref_src', 'source', 'medium', 'campaign'];\n \n function cleanUrl(url) {\n try {\n var u = new URL(url, window.location.origin);\n var changed = false;\n trackingParams.forEach(function(p) {\n if (u.searchParams.has(p)) {\n u.searchParams.delete(p);\n changed = true;\n }\n });\n return changed ? u.toString() : url;\n } catch (e) {\n return url;\n }\n }\n \n function cleanLinks() {\n document.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n \n cleanLinks();\n \n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1) {\n if (node.tagName === 'A') cleanLinks();\n node.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Remove Tracking Parameters from Links");
}
} catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); }
})();
(function(){
try {
var __m = "youtube.com";
var __re = new RegExp('^' + "youtube\\.com" + '

Skip to content
Reformatting Conversion, FeatureSelection and Image Analytics of Transform to Width 85#3943

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

rayankrish
merged 7 commits into

from

Jul 3, 2019
 Merged


Reformatting Conversion, FeatureSelection and Image Analytics of Transform to Width 85
#3943
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


7 commits


Select commit
Hold shift + click to select a range
9267d02

samples/dynamic/transforms/conversions formatted to 85 char

Jul 1, 2019
f2a18ed

samples/dynamic/transforms/featureselection formatted to 85 char

Jul 1, 2019
38c8938

ImageAnalytics

Jul 1, 2019
67080d1

ImageAnalytics

Jul 1, 2019
ecd2738

small corrections to formatting

Jul 2, 2019
91532b0

Merge branch 'transform85' of https://github.com/RadicalRayan/machine…

Jul 2, 2019
3deb533

minor tab and empty line changes

Jul 2, 2019

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


13 changes: 9 additions & 4 deletions
docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Conversion/ConvertType.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -20,17 +20,22 @@ public static void Example()
 var data = mlContext.Data.LoadFromEnumerable(rawData);

 // Construct the pipeline.
 var pipeline = mlContext.Transforms.Conversion.ConvertType("SurvivedInt32", "Survived", DataKind.Int32);
 var pipeline = mlContext.Transforms.Conversion.ConvertType(
 "SurvivedInt32", "Survived", DataKind.Int32);

 // Let's train our pipeline, and then apply it to the same data.
 var transformer = pipeline.Fit(data);
 var transformedData = transformer.Transform(data);

 // Display original column 'Survived' (boolean) and converted column 'SurvivedInt32' (Int32)
 var convertedData = mlContext.Data.CreateEnumerable<TransformedData>(transformedData, true);
 // Display original column 'Survived' (boolean) and converted column 
 // SurvivedInt32' (Int32)
 var convertedData = mlContext.Data.CreateEnumerable<TransformedData>(
 transformedData, true);

 foreach (var item in convertedData)
@codemzscodemzsJul 2, 2019

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

new line between any two lines that were broken down into multiple lines but no new line between multiple breakdowns of a single line.
 {
 Console.WriteLine("A:{0,-10} Aconv:{1}", item.Survived, item.SurvivedInt32);
 Console.WriteLine("A:{0,-10} Aconv:{1}", item.Survived,
 item.SurvivedInt32);
 }

 // Output
Expand Down

38 changes: 26 additions & 12 deletions
docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Conversion/ConvertTypeMultiColumn.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -4,22 +4,33 @@

namespace Samples.Dynamic
{
 // This example illustrates how to convert multiple columns of different types to one type, in this case System.Single. 
 // This is often a useful data transformation before concatenating the features together and passing them to a particular estimator.
 // This example illustrates how to convert multiple columns of different types
 // to one type, in this case System.Single. 
 // This is often a useful data transformation before concatenating the features
 // together and passing them to a particular estimator.
 public static class ConvertTypeMultiColumn
 { 
 public static void Example()
 {
 // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, 
 // as well as the source of randomness.
 // Create a new ML context, for ML.NET operations. It can be used for
 // exception tracking and logging, as well as the source of randomness.
 var mlContext = new MLContext(seed: 1);

 var rawData = new[] {
 new InputData() { Feature1 = true, Feature2 = "0.4", Feature3 = DateTime.Now, Feature4 = 0.145},
 new InputData() { Feature1 = false, Feature2 = "0.5", Feature3 = DateTime.Today, Feature4 = 3.14},
 new InputData() { Feature1 = false, Feature2 = "14", Feature3 = DateTime.Today, Feature4 = 0.2046},
 new InputData() { Feature1 = false, Feature2 = "23", Feature3 = DateTime.Now, Feature4 = 0.1206},
 new InputData() { Feature1 = true, Feature2 = "8904", Feature3 = DateTime.UtcNow, Feature4 = 8.09},
 new InputData() { Feature1 = true, Feature2 = "0.4",
 Feature3 = DateTime.Now, Feature4 = 0.145},

 new InputData() { Feature1 = false, Feature2 = "0.5",
 Feature3 = DateTime.Today, Feature4 = 3.14},

 new InputData() { Feature1 = false, Feature2 = "14",
 Feature3 = DateTime.Today, Feature4 = 0.2046},

 new InputData() { Feature1 = false, Feature2 = "23",
 Feature3 = DateTime.Now, Feature4 = 0.1206},

 new InputData() { Feature1 = true, Feature2 = "8904",
 Feature3 = DateTime.UtcNow, Feature4 = 8.09},
 };

 // Convert the data to an IDataView.
Expand All@@ -37,17 +48,20 @@ public static void Example()

 // Let's fit our pipeline to the data.
 var transformer = pipeline.Fit(data);
 // Transforming the same data. This will add the 4 columns defined in the pipeline, containing the converted
 // Transforming the same data. This will add the 4 columns defined in
 // the pipeline, containing the converted
 // values of the initial columns. 
 var transformedData = transformer.Transform(data);

 // Shape the transformed data as a strongly typed IEnumerable.
 var convertedData = mlContext.Data.CreateEnumerable<TransformedData>(transformedData, true);
 var convertedData = mlContext.Data.CreateEnumerable<TransformedData>(
 transformedData, true);

 // Printing the results.
 Console.WriteLine("Converted1\t Converted2\t Converted3\t Converted4");
 foreach (var item in convertedData)
 Console.WriteLine($"\t{item.Converted1}\t {item.Converted2}\t\t {item.Converted3}\t {item.Converted4}");
 Console.WriteLine($"\t{item.Converted1}\t {item.Converted2}\t\t " +
 $"{item.Converted3}\t {item.Converted4}");

 // Transformed data.
 //
Expand Down

60 changes: 37 additions & 23 deletions
docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Conversion/Hash.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -9,8 +9,8 @@ public static class Hash
 {
 public static void Example()
 {
 // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, 
 // as well as the source of randomness.
 // Create a new ML context, for ML.NET operations. It can be used for
 // exception tracking and logging, as well as the source of randomness.
 var mlContext = new MLContext(seed: 1);

 // Get a small dataset as an IEnumerable.
Expand All@@ -24,30 +24,40 @@ public static void Example()

 var data = mlContext.Data.LoadFromEnumerable(rawData);

 // Construct the pipeline that would hash the two columns and store the results in new columns.
 // The first transform hashes the string column and the second transform hashes the integer column.
 // Construct the pipeline that would hash the two columns and store the
 // results in new columns. The first transform hashes the string column
 // and the second transform hashes the integer column.
 //
 // Hashing is not a reversible operation, so there is no way to retrive the original value from the hashed value. 
 // Sometimes, for debugging, or model explainability, users will need to know what values in the original columns generated
 // the values in the hashed columns, since the algorithms will mostly use the hashed values for further computations.
 // The Hash method will preserve the mapping from the original values to the hashed values in the Annotations of the 
 // newly created column (column populated with the hashed values). 
 // 
 // Setting the maximumNumberOfInverts parameters to -1 will preserve the full map. 
 // If that parameter is left to the default 0 value, the mapping is not preserved.
 var pipeline = mlContext.Transforms.Conversion.Hash("CategoryHashed", "Category", numberOfBits: 16, maximumNumberOfInverts: -1)
 .Append(mlContext.Transforms.Conversion.Hash("AgeHashed", "Age", numberOfBits: 8));
 // Hashing is not a reversible operation, so there is no way to retrive
 // the original value from the hashed value. Sometimes, for debugging,
 // or model explainability, users will need to know what values in the
 // original columns generated the values in the hashed columns, since
 // the algorithms will mostly use the hashed values for further
 // computations. The Hash method will preserve the mapping from the
 // original values to the hashed values in the Annotations of the newly
 // created column (column populated with the hashed values). 
 //
 // Setting the maximumNumberOfInverts parameters to -1 will preserve the
 // full map. If that parameter is left to the default 0 value, the
 // mapping is not preserved.
 var pipeline = mlContext.Transforms.Conversion.Hash("CategoryHashed",
 "Category", numberOfBits: 16, maximumNumberOfInverts: -1)
 .Append(mlContext.Transforms.Conversion.Hash("AgeHashed", "Age",
 numberOfBits: 8));

 // Let's fit our pipeline, and then apply it to the same data.
 var transformer = pipeline.Fit(data);
 var transformedData = transformer.Transform(data);

 // Convert the post transformation from the IDataView format to an IEnumerable<TransformedData> for easy consumption.
 var convertedData = mlContext.Data.CreateEnumerable<TransformedDataPoint>(transformedData, true);
 // Convert the post transformation from the IDataView format to an
 // IEnumerable <TransformedData> for easy consumption.
 var convertedData = mlContext.Data.CreateEnumerable<
 TransformedDataPoint>(transformedData, true);

 Console.WriteLine("Category CategoryHashed\t Age\t AgeHashed");
 foreach (var item in convertedData)
 Console.WriteLine($"{item.Category}\t {item.CategoryHashed}\t\t {item.Age}\t {item.AgeHashed}");
 Console.WriteLine($"{item.Category}\t {item.CategoryHashed}\t\t " +
 $"{item.Age}\t {item.AgeHashed}");

 // Expected data after the transformation.
 //
Expand All@@ -58,20 +68,24 @@ public static void Example()
 // MLB 36206 18 127
 // MLS 6013 14 62

 // For the Category column, where we set the maximumNumberOfInverts parameter, the names of the original categories, 
 // and their correspondance with the generated hash values is preserved in the Annotations in the format of indices and values. 
 // the indices array will have the hashed values, and the corresponding element, position-wise, in the values array will 
 // contain the original value. 
 // For the Category column, where we set the maximumNumberOfInverts
 // parameter, the names of the original categories, and their
 // correspondance with the generated hash values is preserved in the
 // Annotations in the format of indices and values.the indices array
 // will have the hashed values, and the corresponding element,
 // position -wise, in the values array will contain the original value. 
 //
 // See below for an example on how to retrieve the mapping. 
 var slotNames = new VBuffer<ReadOnlyMemory<char>>();
 transformedData.Schema["CategoryHashed"].Annotations.GetValue("KeyValues", ref slotNames);
 transformedData.Schema["CategoryHashed"].Annotations.GetValue(
 "KeyValues", ref slotNames);

 var indices = slotNames.GetIndices();
 var categoryNames = slotNames.GetValues();

 for (int i = 0; i < indices.Length; i++)
 Console.WriteLine($"The original value of the {indices[i]} category is {categoryNames[i]}");
 Console.WriteLine($"The original value of the {indices[i]} " +
 $"category is {categoryNames[i]}");

 // Output Data
 // 
Expand Down

57 changes: 38 additions & 19 deletions
docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Conversion/KeyToValueToKey.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -11,8 +11,8 @@ public class KeyToValueToKey
 {
 public static void Example()
 {
 // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, 
 // as well as the source of randomness.
 // Create a new ML context, for ML.NET operations. It can be used for
 // exception tracking and logging, as well as the source of randomness.
 var mlContext = new MLContext();

 // Get a small dataset as an IEnumerable.
Expand All@@ -27,25 +27,40 @@ public static void Example()

 // A pipeline to convert the terms of the 'Review' column in 
 // making use of default settings.
 var defaultPipeline = mlContext.Transforms.Text.TokenizeIntoWords("TokenizedText", nameof(DataPoint.Review))
 .Append(mlContext.Transforms.Conversion.MapValueToKey(nameof(TransformedData.Keys), "TokenizedText"));
 var defaultPipeline = mlContext.Transforms.Text.TokenizeIntoWords(
 "TokenizedText", nameof(DataPoint.Review)).Append(mlContext
 .Transforms.Conversion.MapValueToKey(nameof(TransformedData.Keys),
 "TokenizedText"));

@codemzscodemzsJul 2, 2019

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

please move .Append to a new line.
 // Another pipeline, that customizes the advanced settings of the ValueToKeyMappingEstimator.
 // We can change the maximumNumberOfKeys to limit how many keys will get generated out of the set of words, 
 // and condition the order in which they get evaluated by changing keyOrdinality from the default ByOccurence (order in which they get encountered) 
 // to value/alphabetically.
 var customizedPipeline = mlContext.Transforms.Text.TokenizeIntoWords("TokenizedText", nameof(DataPoint.Review))
 .Append(mlContext.Transforms.Conversion.MapValueToKey(nameof(TransformedData.Keys), "TokenizedText", maximumNumberOfKeys: 10,
 keyOrdinality: ValueToKeyMappingEstimator.KeyOrdinality.ByValue));
 // Another pipeline, that customizes the advanced settings of the
 // ValueToKeyMappingEstimator. We can change the maximumNumberOfKeys to
 // limit how many keys will get generated out of the set of words, and
 // condition the order in which they get evaluated by changing
 // keyOrdinality from the default ByOccurence (order in which they get
 // encountered) to value/alphabetically.
 var customizedPipeline = mlContext.Transforms.Text.TokenizeIntoWords(
 "TokenizedText", nameof(DataPoint.Review)).Append(mlContext
 .Transforms.Conversion.MapValueToKey(nameof(TransformedData.Keys),
 "TokenizedText", maximumNumberOfKeys: 10, keyOrdinality:
 ValueToKeyMappingEstimator.KeyOrdinality.ByValue));

 // The transformed data.
 var transformedDataDefault = defaultPipeline.Fit(trainData).Transform(trainData);
 var transformedDataCustomized = customizedPipeline.Fit(trainData).Transform(trainData);
 var transformedDataDefault = defaultPipeline.Fit(trainData).Transform(
 trainData);

 var transformedDataCustomized = customizedPipeline.Fit(trainData)
 .Transform(trainData);

 // Getting the resulting data as an IEnumerable.
 // This will contain the newly created columns.
 IEnumerable<TransformedData> defaultData = mlContext.Data.CreateEnumerable<TransformedData>(transformedDataDefault, reuseRowObject: false);
 IEnumerable<TransformedData> customizedData = mlContext.Data.CreateEnumerable<TransformedData>(transformedDataCustomized, reuseRowObject: false);
 IEnumerable<TransformedData> defaultData = mlContext.Data.
 CreateEnumerable<TransformedData>(transformedDataDefault,
 reuseRowObject: false);

 IEnumerable<TransformedData> customizedData = mlContext.Data.
 CreateEnumerable<TransformedData>(transformedDataCustomized,
 reuseRowObject: false);

 Console.WriteLine($"Keys");
 foreach (var dataRow in defaultData)
 Console.WriteLine($"{string.Join(',', dataRow.Keys)}");
Expand All@@ -65,13 +80,17 @@ public static void Example()
 // 8,2,9,7,6,4
 // 3,10,0,0,0
 // 3,10,0,0,0,8
 // Retrieve the original values, by appending the KeyToValue etimator to the existing pipelines
 // to convert the keys back to the strings.
 var pipeline = defaultPipeline.Append(mlContext.Transforms.Conversion.MapKeyToValue(nameof(TransformedData.Keys)));
 // Retrieve the original values, by appending the KeyToValue etimator to
 // the existing pipelines to convert the keys back to the strings.
 var pipeline = defaultPipeline.Append(mlContext.Transforms.Conversion
 .MapKeyToValue(nameof(TransformedData.Keys)));

 transformedDataDefault = pipeline.Fit(trainData).Transform(trainData);

 // Preview of the DefaultColumnName column obtained.
 var originalColumnBack = transformedDataDefault.GetColumn<VBuffer<ReadOnlyMemory<char>>>(transformedDataDefault.Schema[nameof(TransformedData.Keys)]);
 var originalColumnBack = transformedDataDefault.GetColumn<VBuffer<
 ReadOnlyMemory<char>>>(transformedDataDefault.Schema[nameof(
 TransformedData.Keys)]);

 foreach (var row in originalColumnBack)
 {
Expand Down

21 changes: 13 additions & 8 deletions
docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Conversion/MapKeyToBinaryVector.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -7,14 +7,16 @@ namespace Samples.Dynamic
{
 class MapKeyToBinaryVector
 {
 /// This example demonstrates the use of MapKeyToVector by mapping keys to floats[] of 0 and 1, representing the number in binary format. 
 /// Because the ML.NET KeyType maps the missing value to zero, counting starts at 1, so the uint values
 /// converted to KeyTypes will appear skewed by one. 
 /// This example demonstrates the use of MapKeyToVector by mapping keys to
 /// floats[] of 0 and 1, representing the number in binary format. 
 /// Because the ML.NET KeyType maps the missing value to zero, counting
 /// starts at 1, so the uint values converted to KeyTypes will appear
 /// skewed by one. 
 /// See https://github.com/dotnet/machinelearning/blob/master/docs/code/IDataViewTypeSystem.md#key-types
 public static void Example()
 {
 // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, 
 // as well as the source of randomness.
 // Create a new ML context, for ML.NET operations. It can be used for
 // exception tracking and logging, as well as the source of randomness.
 var mlContext = new MLContext();

 // Get a small dataset as an IEnumerable.
Expand All@@ -30,18 +32,21 @@ public static void Example()
 var data = mlContext.Data.LoadFromEnumerable(rawData);

 // Constructs the ML.net pipeline
 var pipeline = mlContext.Transforms.Conversion.MapKeyToBinaryVector("TimeframeVector", "Timeframe");
 var pipeline = mlContext.Transforms.Conversion.MapKeyToBinaryVector(
 "TimeframeVector", "Timeframe");

 // Fits the pipeline to the data.
 IDataView transformedData = pipeline.Fit(data).Transform(data);

 // Getting the resulting data as an IEnumerable.
 // This will contain the newly created columns.
 IEnumerable<TransformedData> features = mlContext.Data.CreateEnumerable<TransformedData>(transformedData, reuseRowObject: false);
 IEnumerable<TransformedData> features = mlContext.Data.CreateEnumerable<
 TransformedData>(transformedData, reuseRowObject: false);

 Console.WriteLine($" Timeframe TimeframeVector");
 foreach (var featureRow in features)
 Console.WriteLine($"{featureRow.Timeframe}\t\t\t{string.Join(',', featureRow.TimeframeVector)}");
 Console.WriteLine($"{featureRow.Timeframe}\t\t\t" +
 $"{string.Join(',', featureRow.TimeframeVector)}");

 // Timeframe TimeframeVector
 // 10 0,1,0,0,1 //binary representation of 9, the original value
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Auto-enable theater mode on YouTube\n(function() {\n function tryTheater() {\n var btn = document.querySelector('button[aria-label=\"Theater mode\"], ytd-player #player button[title=\"Theater mode\"]');\n if (btn && !btn.classList.contains('activated')) {\n btn.click();\n }\n }\n \n // Try immediately\n tryTheater();\n \n // Try after navigation (SPA)\n var lastUrl = location.href;\n setInterval(function() {\n if (location.href !== lastUrl) {\n lastUrl = location.href;\n setTimeout(tryTheater, 500);\n }\n }, 1000);\n \n // Also try on player load\n var observer = new MutationObserver(tryTheater);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "YouTube Theater Mode Default");
}
} catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
Reformatting Conversion, FeatureSelection and Image Analytics of Transform to Width 85#3943

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

rayankrish
merged 7 commits into

from

Jul 3, 2019
 Merged


Reformatting Conversion, FeatureSelection and Image Analytics of Transform to Width 85
#3943
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


7 commits


Select commit
Hold shift + click to select a range
9267d02

samples/dynamic/transforms/conversions formatted to 85 char

Jul 1, 2019
f2a18ed

samples/dynamic/transforms/featureselection formatted to 85 char

Jul 1, 2019
38c8938

ImageAnalytics

Jul 1, 2019
67080d1

ImageAnalytics

Jul 1, 2019
ecd2738

small corrections to formatting

Jul 2, 2019
91532b0

Merge branch 'transform85' of https://github.com/RadicalRayan/machine…

Jul 2, 2019
3deb533

minor tab and empty line changes

Jul 2, 2019

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


13 changes: 9 additions & 4 deletions
docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Conversion/ConvertType.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -20,17 +20,22 @@ public static void Example()
 var data = mlContext.Data.LoadFromEnumerable(rawData);

 // Construct the pipeline.
 var pipeline = mlContext.Transforms.Conversion.ConvertType("SurvivedInt32", "Survived", DataKind.Int32);
 var pipeline = mlContext.Transforms.Conversion.ConvertType(
 "SurvivedInt32", "Survived", DataKind.Int32);

 // Let's train our pipeline, and then apply it to the same data.
 var transformer = pipeline.Fit(data);
 var transformedData = transformer.Transform(data);

 // Display original column 'Survived' (boolean) and converted column 'SurvivedInt32' (Int32)
 var convertedData = mlContext.Data.CreateEnumerable<TransformedData>(transformedData, true);
 // Display original column 'Survived' (boolean) and converted column 
 // SurvivedInt32' (Int32)
 var convertedData = mlContext.Data.CreateEnumerable<TransformedData>(
 transformedData, true);

 foreach (var item in convertedData)
@codemzscodemzsJul 2, 2019

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

new line between any two lines that were broken down into multiple lines but no new line between multiple breakdowns of a single line.
 {
 Console.WriteLine("A:{0,-10} Aconv:{1}", item.Survived, item.SurvivedInt32);
 Console.WriteLine("A:{0,-10} Aconv:{1}", item.Survived,
 item.SurvivedInt32);
 }

 // Output
Expand Down

38 changes: 26 additions & 12 deletions
docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Conversion/ConvertTypeMultiColumn.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -4,22 +4,33 @@

namespace Samples.Dynamic
{
 // This example illustrates how to convert multiple columns of different types to one type, in this case System.Single. 
 // This is often a useful data transformation before concatenating the features together and passing them to a particular estimator.
 // This example illustrates how to convert multiple columns of different types
 // to one type, in this case System.Single. 
 // This is often a useful data transformation before concatenating the features
 // together and passing them to a particular estimator.
 public static class ConvertTypeMultiColumn
 { 
 public static void Example()
 {
 // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, 
 // as well as the source of randomness.
 // Create a new ML context, for ML.NET operations. It can be used for
 // exception tracking and logging, as well as the source of randomness.
 var mlContext = new MLContext(seed: 1);

 var rawData = new[] {
 new InputData() { Feature1 = true, Feature2 = "0.4", Feature3 = DateTime.Now, Feature4 = 0.145},
 new InputData() { Feature1 = false, Feature2 = "0.5", Feature3 = DateTime.Today, Feature4 = 3.14},
 new InputData() { Feature1 = false, Feature2 = "14", Feature3 = DateTime.Today, Feature4 = 0.2046},
 new InputData() { Feature1 = false, Feature2 = "23", Feature3 = DateTime.Now, Feature4 = 0.1206},
 new InputData() { Feature1 = true, Feature2 = "8904", Feature3 = DateTime.UtcNow, Feature4 = 8.09},
 new InputData() { Feature1 = true, Feature2 = "0.4",
 Feature3 = DateTime.Now, Feature4 = 0.145},

 new InputData() { Feature1 = false, Feature2 = "0.5",
 Feature3 = DateTime.Today, Feature4 = 3.14},

 new InputData() { Feature1 = false, Feature2 = "14",
 Feature3 = DateTime.Today, Feature4 = 0.2046},

 new InputData() { Feature1 = false, Feature2 = "23",
 Feature3 = DateTime.Now, Feature4 = 0.1206},

 new InputData() { Feature1 = true, Feature2 = "8904",
 Feature3 = DateTime.UtcNow, Feature4 = 8.09},
 };

 // Convert the data to an IDataView.
Expand All@@ -37,17 +48,20 @@ public static void Example()

 // Let's fit our pipeline to the data.
 var transformer = pipeline.Fit(data);
 // Transforming the same data. This will add the 4 columns defined in the pipeline, containing the converted
 // Transforming the same data. This will add the 4 columns defined in
 // the pipeline, containing the converted
 // values of the initial columns. 
 var transformedData = transformer.Transform(data);

 // Shape the transformed data as a strongly typed IEnumerable.
 var convertedData = mlContext.Data.CreateEnumerable<TransformedData>(transformedData, true);
 var convertedData = mlContext.Data.CreateEnumerable<TransformedData>(
 transformedData, true);

 // Printing the results.
 Console.WriteLine("Converted1\t Converted2\t Converted3\t Converted4");
 foreach (var item in convertedData)
 Console.WriteLine($"\t{item.Converted1}\t {item.Converted2}\t\t {item.Converted3}\t {item.Converted4}");
 Console.WriteLine($"\t{item.Converted1}\t {item.Converted2}\t\t " +
 $"{item.Converted3}\t {item.Converted4}");

 // Transformed data.
 //
Expand Down

60 changes: 37 additions & 23 deletions
docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Conversion/Hash.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -9,8 +9,8 @@ public static class Hash
 {
 public static void Example()
 {
 // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, 
 // as well as the source of randomness.
 // Create a new ML context, for ML.NET operations. It can be used for
 // exception tracking and logging, as well as the source of randomness.
 var mlContext = new MLContext(seed: 1);

 // Get a small dataset as an IEnumerable.
Expand All@@ -24,30 +24,40 @@ public static void Example()

 var data = mlContext.Data.LoadFromEnumerable(rawData);

 // Construct the pipeline that would hash the two columns and store the results in new columns.
 // The first transform hashes the string column and the second transform hashes the integer column.
 // Construct the pipeline that would hash the two columns and store the
 // results in new columns. The first transform hashes the string column
 // and the second transform hashes the integer column.
 //
 // Hashing is not a reversible operation, so there is no way to retrive the original value from the hashed value. 
 // Sometimes, for debugging, or model explainability, users will need to know what values in the original columns generated
 // the values in the hashed columns, since the algorithms will mostly use the hashed values for further computations.
 // The Hash method will preserve the mapping from the original values to the hashed values in the Annotations of the 
 // newly created column (column populated with the hashed values). 
 // 
 // Setting the maximumNumberOfInverts parameters to -1 will preserve the full map. 
 // If that parameter is left to the default 0 value, the mapping is not preserved.
 var pipeline = mlContext.Transforms.Conversion.Hash("CategoryHashed", "Category", numberOfBits: 16, maximumNumberOfInverts: -1)
 .Append(mlContext.Transforms.Conversion.Hash("AgeHashed", "Age", numberOfBits: 8));
 // Hashing is not a reversible operation, so there is no way to retrive
 // the original value from the hashed value. Sometimes, for debugging,
 // or model explainability, users will need to know what values in the
 // original columns generated the values in the hashed columns, since
 // the algorithms will mostly use the hashed values for further
 // computations. The Hash method will preserve the mapping from the
 // original values to the hashed values in the Annotations of the newly
 // created column (column populated with the hashed values). 
 //
 // Setting the maximumNumberOfInverts parameters to -1 will preserve the
 // full map. If that parameter is left to the default 0 value, the
 // mapping is not preserved.
 var pipeline = mlContext.Transforms.Conversion.Hash("CategoryHashed",
 "Category", numberOfBits: 16, maximumNumberOfInverts: -1)
 .Append(mlContext.Transforms.Conversion.Hash("AgeHashed", "Age",
 numberOfBits: 8));

 // Let's fit our pipeline, and then apply it to the same data.
 var transformer = pipeline.Fit(data);
 var transformedData = transformer.Transform(data);

 // Convert the post transformation from the IDataView format to an IEnumerable<TransformedData> for easy consumption.
 var convertedData = mlContext.Data.CreateEnumerable<TransformedDataPoint>(transformedData, true);
 // Convert the post transformation from the IDataView format to an
 // IEnumerable <TransformedData> for easy consumption.
 var convertedData = mlContext.Data.CreateEnumerable<
 TransformedDataPoint>(transformedData, true);

 Console.WriteLine("Category CategoryHashed\t Age\t AgeHashed");
 foreach (var item in convertedData)
 Console.WriteLine($"{item.Category}\t {item.CategoryHashed}\t\t {item.Age}\t {item.AgeHashed}");
 Console.WriteLine($"{item.Category}\t {item.CategoryHashed}\t\t " +
 $"{item.Age}\t {item.AgeHashed}");

 // Expected data after the transformation.
 //
Expand All@@ -58,20 +68,24 @@ public static void Example()
 // MLB 36206 18 127
 // MLS 6013 14 62

 // For the Category column, where we set the maximumNumberOfInverts parameter, the names of the original categories, 
 // and their correspondance with the generated hash values is preserved in the Annotations in the format of indices and values. 
 // the indices array will have the hashed values, and the corresponding element, position-wise, in the values array will 
 // contain the original value. 
 // For the Category column, where we set the maximumNumberOfInverts
 // parameter, the names of the original categories, and their
 // correspondance with the generated hash values is preserved in the
 // Annotations in the format of indices and values.the indices array
 // will have the hashed values, and the corresponding element,
 // position -wise, in the values array will contain the original value. 
 //
 // See below for an example on how to retrieve the mapping. 
 var slotNames = new VBuffer<ReadOnlyMemory<char>>();
 transformedData.Schema["CategoryHashed"].Annotations.GetValue("KeyValues", ref slotNames);
 transformedData.Schema["CategoryHashed"].Annotations.GetValue(
 "KeyValues", ref slotNames);

 var indices = slotNames.GetIndices();
 var categoryNames = slotNames.GetValues();

 for (int i = 0; i < indices.Length; i++)
 Console.WriteLine($"The original value of the {indices[i]} category is {categoryNames[i]}");
 Console.WriteLine($"The original value of the {indices[i]} " +
 $"category is {categoryNames[i]}");

 // Output Data
 // 
Expand Down

57 changes: 38 additions & 19 deletions
docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Conversion/KeyToValueToKey.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -11,8 +11,8 @@ public class KeyToValueToKey
 {
 public static void Example()
 {
 // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, 
 // as well as the source of randomness.
 // Create a new ML context, for ML.NET operations. It can be used for
 // exception tracking and logging, as well as the source of randomness.
 var mlContext = new MLContext();

 // Get a small dataset as an IEnumerable.
Expand All@@ -27,25 +27,40 @@ public static void Example()

 // A pipeline to convert the terms of the 'Review' column in 
 // making use of default settings.
 var defaultPipeline = mlContext.Transforms.Text.TokenizeIntoWords("TokenizedText", nameof(DataPoint.Review))
 .Append(mlContext.Transforms.Conversion.MapValueToKey(nameof(TransformedData.Keys), "TokenizedText"));
 var defaultPipeline = mlContext.Transforms.Text.TokenizeIntoWords(
 "TokenizedText", nameof(DataPoint.Review)).Append(mlContext
 .Transforms.Conversion.MapValueToKey(nameof(TransformedData.Keys),
 "TokenizedText"));

@codemzscodemzsJul 2, 2019

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

please move .Append to a new line.
 // Another pipeline, that customizes the advanced settings of the ValueToKeyMappingEstimator.
 // We can change the maximumNumberOfKeys to limit how many keys will get generated out of the set of words, 
 // and condition the order in which they get evaluated by changing keyOrdinality from the default ByOccurence (order in which they get encountered) 
 // to value/alphabetically.
 var customizedPipeline = mlContext.Transforms.Text.TokenizeIntoWords("TokenizedText", nameof(DataPoint.Review))
 .Append(mlContext.Transforms.Conversion.MapValueToKey(nameof(TransformedData.Keys), "TokenizedText", maximumNumberOfKeys: 10,
 keyOrdinality: ValueToKeyMappingEstimator.KeyOrdinality.ByValue));
 // Another pipeline, that customizes the advanced settings of the
 // ValueToKeyMappingEstimator. We can change the maximumNumberOfKeys to
 // limit how many keys will get generated out of the set of words, and
 // condition the order in which they get evaluated by changing
 // keyOrdinality from the default ByOccurence (order in which they get
 // encountered) to value/alphabetically.
 var customizedPipeline = mlContext.Transforms.Text.TokenizeIntoWords(
 "TokenizedText", nameof(DataPoint.Review)).Append(mlContext
 .Transforms.Conversion.MapValueToKey(nameof(TransformedData.Keys),
 "TokenizedText", maximumNumberOfKeys: 10, keyOrdinality:
 ValueToKeyMappingEstimator.KeyOrdinality.ByValue));

 // The transformed data.
 var transformedDataDefault = defaultPipeline.Fit(trainData).Transform(trainData);
 var transformedDataCustomized = customizedPipeline.Fit(trainData).Transform(trainData);
 var transformedDataDefault = defaultPipeline.Fit(trainData).Transform(
 trainData);

 var transformedDataCustomized = customizedPipeline.Fit(trainData)
 .Transform(trainData);

 // Getting the resulting data as an IEnumerable.
 // This will contain the newly created columns.
 IEnumerable<TransformedData> defaultData = mlContext.Data.CreateEnumerable<TransformedData>(transformedDataDefault, reuseRowObject: false);
 IEnumerable<TransformedData> customizedData = mlContext.Data.CreateEnumerable<TransformedData>(transformedDataCustomized, reuseRowObject: false);
 IEnumerable<TransformedData> defaultData = mlContext.Data.
 CreateEnumerable<TransformedData>(transformedDataDefault,
 reuseRowObject: false);

 IEnumerable<TransformedData> customizedData = mlContext.Data.
 CreateEnumerable<TransformedData>(transformedDataCustomized,
 reuseRowObject: false);

 Console.WriteLine($"Keys");
 foreach (var dataRow in defaultData)
 Console.WriteLine($"{string.Join(',', dataRow.Keys)}");
Expand All@@ -65,13 +80,17 @@ public static void Example()
 // 8,2,9,7,6,4
 // 3,10,0,0,0
 // 3,10,0,0,0,8
 // Retrieve the original values, by appending the KeyToValue etimator to the existing pipelines
 // to convert the keys back to the strings.
 var pipeline = defaultPipeline.Append(mlContext.Transforms.Conversion.MapKeyToValue(nameof(TransformedData.Keys)));
 // Retrieve the original values, by appending the KeyToValue etimator to
 // the existing pipelines to convert the keys back to the strings.
 var pipeline = defaultPipeline.Append(mlContext.Transforms.Conversion
 .MapKeyToValue(nameof(TransformedData.Keys)));

 transformedDataDefault = pipeline.Fit(trainData).Transform(trainData);

 // Preview of the DefaultColumnName column obtained.
 var originalColumnBack = transformedDataDefault.GetColumn<VBuffer<ReadOnlyMemory<char>>>(transformedDataDefault.Schema[nameof(TransformedData.Keys)]);
 var originalColumnBack = transformedDataDefault.GetColumn<VBuffer<
 ReadOnlyMemory<char>>>(transformedDataDefault.Schema[nameof(
 TransformedData.Keys)]);

 foreach (var row in originalColumnBack)
 {
Expand Down

21 changes: 13 additions & 8 deletions
docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Conversion/MapKeyToBinaryVector.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -7,14 +7,16 @@ namespace Samples.Dynamic
{
 class MapKeyToBinaryVector
 {
 /// This example demonstrates the use of MapKeyToVector by mapping keys to floats[] of 0 and 1, representing the number in binary format. 
 /// Because the ML.NET KeyType maps the missing value to zero, counting starts at 1, so the uint values
 /// converted to KeyTypes will appear skewed by one. 
 /// This example demonstrates the use of MapKeyToVector by mapping keys to
 /// floats[] of 0 and 1, representing the number in binary format. 
 /// Because the ML.NET KeyType maps the missing value to zero, counting
 /// starts at 1, so the uint values converted to KeyTypes will appear
 /// skewed by one. 
 /// See https://github.com/dotnet/machinelearning/blob/master/docs/code/IDataViewTypeSystem.md#key-types
 public static void Example()
 {
 // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, 
 // as well as the source of randomness.
 // Create a new ML context, for ML.NET operations. It can be used for
 // exception tracking and logging, as well as the source of randomness.
 var mlContext = new MLContext();

 // Get a small dataset as an IEnumerable.
Expand All@@ -30,18 +32,21 @@ public static void Example()
 var data = mlContext.Data.LoadFromEnumerable(rawData);

 // Constructs the ML.net pipeline
 var pipeline = mlContext.Transforms.Conversion.MapKeyToBinaryVector("TimeframeVector", "Timeframe");
 var pipeline = mlContext.Transforms.Conversion.MapKeyToBinaryVector(
 "TimeframeVector", "Timeframe");

 // Fits the pipeline to the data.
 IDataView transformedData = pipeline.Fit(data).Transform(data);

 // Getting the resulting data as an IEnumerable.
 // This will contain the newly created columns.
 IEnumerable<TransformedData> features = mlContext.Data.CreateEnumerable<TransformedData>(transformedData, reuseRowObject: false);
 IEnumerable<TransformedData> features = mlContext.Data.CreateEnumerable<
 TransformedData>(transformedData, reuseRowObject: false);

 Console.WriteLine($" Timeframe TimeframeVector");
 foreach (var featureRow in features)
 Console.WriteLine($"{featureRow.Timeframe}\t\t\t{string.Join(',', featureRow.TimeframeVector)}");
 Console.WriteLine($"{featureRow.Timeframe}\t\t\t" +
 $"{string.Join(',', featureRow.TimeframeVector)}");

 // Timeframe TimeframeVector
 // 10 0,1,0,0,1 //binary representation of 9, the original value
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Remove or un-stick sticky/fixed headers that block content\n(function() {\n function unstick() {\n document.querySelectorAll('header, nav, [role=\"banner\"], .header, .navbar, .sticky, .fixed-top, [style*=\"position: fixed\"], [style*=\"position:sticky\"]').forEach(function(el) {\n if (el.style.position === 'fixed' || el.style.position === 'sticky' || \n getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') {\n el.style.position = 'static';\n el.style.top = 'auto';\n el.style.zIndex = 'auto';\n }\n });\n }\n \n unstick();\n \n var observer = new MutationObserver(unstick);\n observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] });\n})();", "Kill Sticky Headers");
}
} catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
Reformatting Conversion, FeatureSelection and Image Analytics of Transform to Width 85#3943

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

rayankrish
merged 7 commits into

from

Jul 3, 2019
 Merged


Reformatting Conversion, FeatureSelection and Image Analytics of Transform to Width 85
#3943
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


7 commits


Select commit
Hold shift + click to select a range
9267d02

samples/dynamic/transforms/conversions formatted to 85 char

Jul 1, 2019
f2a18ed

samples/dynamic/transforms/featureselection formatted to 85 char

Jul 1, 2019
38c8938

ImageAnalytics

Jul 1, 2019
67080d1

ImageAnalytics

Jul 1, 2019
ecd2738

small corrections to formatting

Jul 2, 2019
91532b0

Merge branch 'transform85' of https://github.com/RadicalRayan/machine…

Jul 2, 2019
3deb533

minor tab and empty line changes

Jul 2, 2019

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


13 changes: 9 additions & 4 deletions
docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Conversion/ConvertType.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -20,17 +20,22 @@ public static void Example()
 var data = mlContext.Data.LoadFromEnumerable(rawData);

 // Construct the pipeline.
 var pipeline = mlContext.Transforms.Conversion.ConvertType("SurvivedInt32", "Survived", DataKind.Int32);
 var pipeline = mlContext.Transforms.Conversion.ConvertType(
 "SurvivedInt32", "Survived", DataKind.Int32);

 // Let's train our pipeline, and then apply it to the same data.
 var transformer = pipeline.Fit(data);
 var transformedData = transformer.Transform(data);

 // Display original column 'Survived' (boolean) and converted column 'SurvivedInt32' (Int32)
 var convertedData = mlContext.Data.CreateEnumerable<TransformedData>(transformedData, true);
 // Display original column 'Survived' (boolean) and converted column 
 // SurvivedInt32' (Int32)
 var convertedData = mlContext.Data.CreateEnumerable<TransformedData>(
 transformedData, true);

 foreach (var item in convertedData)
@codemzscodemzsJul 2, 2019

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

new line between any two lines that were broken down into multiple lines but no new line between multiple breakdowns of a single line.
 {
 Console.WriteLine("A:{0,-10} Aconv:{1}", item.Survived, item.SurvivedInt32);
 Console.WriteLine("A:{0,-10} Aconv:{1}", item.Survived,
 item.SurvivedInt32);
 }

 // Output
Expand Down

38 changes: 26 additions & 12 deletions
docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Conversion/ConvertTypeMultiColumn.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -4,22 +4,33 @@

namespace Samples.Dynamic
{
 // This example illustrates how to convert multiple columns of different types to one type, in this case System.Single. 
 // This is often a useful data transformation before concatenating the features together and passing them to a particular estimator.
 // This example illustrates how to convert multiple columns of different types
 // to one type, in this case System.Single. 
 // This is often a useful data transformation before concatenating the features
 // together and passing them to a particular estimator.
 public static class ConvertTypeMultiColumn
 { 
 public static void Example()
 {
 // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, 
 // as well as the source of randomness.
 // Create a new ML context, for ML.NET operations. It can be used for
 // exception tracking and logging, as well as the source of randomness.
 var mlContext = new MLContext(seed: 1);

 var rawData = new[] {
 new InputData() { Feature1 = true, Feature2 = "0.4", Feature3 = DateTime.Now, Feature4 = 0.145},
 new InputData() { Feature1 = false, Feature2 = "0.5", Feature3 = DateTime.Today, Feature4 = 3.14},
 new InputData() { Feature1 = false, Feature2 = "14", Feature3 = DateTime.Today, Feature4 = 0.2046},
 new InputData() { Feature1 = false, Feature2 = "23", Feature3 = DateTime.Now, Feature4 = 0.1206},
 new InputData() { Feature1 = true, Feature2 = "8904", Feature3 = DateTime.UtcNow, Feature4 = 8.09},
 new InputData() { Feature1 = true, Feature2 = "0.4",
 Feature3 = DateTime.Now, Feature4 = 0.145},

 new InputData() { Feature1 = false, Feature2 = "0.5",
 Feature3 = DateTime.Today, Feature4 = 3.14},

 new InputData() { Feature1 = false, Feature2 = "14",
 Feature3 = DateTime.Today, Feature4 = 0.2046},

 new InputData() { Feature1 = false, Feature2 = "23",
 Feature3 = DateTime.Now, Feature4 = 0.1206},

 new InputData() { Feature1 = true, Feature2 = "8904",
 Feature3 = DateTime.UtcNow, Feature4 = 8.09},
 };

 // Convert the data to an IDataView.
Expand All@@ -37,17 +48,20 @@ public static void Example()

 // Let's fit our pipeline to the data.
 var transformer = pipeline.Fit(data);
 // Transforming the same data. This will add the 4 columns defined in the pipeline, containing the converted
 // Transforming the same data. This will add the 4 columns defined in
 // the pipeline, containing the converted
 // values of the initial columns. 
 var transformedData = transformer.Transform(data);

 // Shape the transformed data as a strongly typed IEnumerable.
 var convertedData = mlContext.Data.CreateEnumerable<TransformedData>(transformedData, true);
 var convertedData = mlContext.Data.CreateEnumerable<TransformedData>(
 transformedData, true);

 // Printing the results.
 Console.WriteLine("Converted1\t Converted2\t Converted3\t Converted4");
 foreach (var item in convertedData)
 Console.WriteLine($"\t{item.Converted1}\t {item.Converted2}\t\t {item.Converted3}\t {item.Converted4}");
 Console.WriteLine($"\t{item.Converted1}\t {item.Converted2}\t\t " +
 $"{item.Converted3}\t {item.Converted4}");

 // Transformed data.
 //
Expand Down

60 changes: 37 additions & 23 deletions
docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Conversion/Hash.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -9,8 +9,8 @@ public static class Hash
 {
 public static void Example()
 {
 // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, 
 // as well as the source of randomness.
 // Create a new ML context, for ML.NET operations. It can be used for
 // exception tracking and logging, as well as the source of randomness.
 var mlContext = new MLContext(seed: 1);

 // Get a small dataset as an IEnumerable.
Expand All@@ -24,30 +24,40 @@ public static void Example()

 var data = mlContext.Data.LoadFromEnumerable(rawData);

 // Construct the pipeline that would hash the two columns and store the results in new columns.
 // The first transform hashes the string column and the second transform hashes the integer column.
 // Construct the pipeline that would hash the two columns and store the
 // results in new columns. The first transform hashes the string column
 // and the second transform hashes the integer column.
 //
 // Hashing is not a reversible operation, so there is no way to retrive the original value from the hashed value. 
 // Sometimes, for debugging, or model explainability, users will need to know what values in the original columns generated
 // the values in the hashed columns, since the algorithms will mostly use the hashed values for further computations.
 // The Hash method will preserve the mapping from the original values to the hashed values in the Annotations of the 
 // newly created column (column populated with the hashed values). 
 // 
 // Setting the maximumNumberOfInverts parameters to -1 will preserve the full map. 
 // If that parameter is left to the default 0 value, the mapping is not preserved.
 var pipeline = mlContext.Transforms.Conversion.Hash("CategoryHashed", "Category", numberOfBits: 16, maximumNumberOfInverts: -1)
 .Append(mlContext.Transforms.Conversion.Hash("AgeHashed", "Age", numberOfBits: 8));
 // Hashing is not a reversible operation, so there is no way to retrive
 // the original value from the hashed value. Sometimes, for debugging,
 // or model explainability, users will need to know what values in the
 // original columns generated the values in the hashed columns, since
 // the algorithms will mostly use the hashed values for further
 // computations. The Hash method will preserve the mapping from the
 // original values to the hashed values in the Annotations of the newly
 // created column (column populated with the hashed values). 
 //
 // Setting the maximumNumberOfInverts parameters to -1 will preserve the
 // full map. If that parameter is left to the default 0 value, the
 // mapping is not preserved.
 var pipeline = mlContext.Transforms.Conversion.Hash("CategoryHashed",
 "Category", numberOfBits: 16, maximumNumberOfInverts: -1)
 .Append(mlContext.Transforms.Conversion.Hash("AgeHashed", "Age",
 numberOfBits: 8));

 // Let's fit our pipeline, and then apply it to the same data.
 var transformer = pipeline.Fit(data);
 var transformedData = transformer.Transform(data);

 // Convert the post transformation from the IDataView format to an IEnumerable<TransformedData> for easy consumption.
 var convertedData = mlContext.Data.CreateEnumerable<TransformedDataPoint>(transformedData, true);
 // Convert the post transformation from the IDataView format to an
 // IEnumerable <TransformedData> for easy consumption.
 var convertedData = mlContext.Data.CreateEnumerable<
 TransformedDataPoint>(transformedData, true);

 Console.WriteLine("Category CategoryHashed\t Age\t AgeHashed");
 foreach (var item in convertedData)
 Console.WriteLine($"{item.Category}\t {item.CategoryHashed}\t\t {item.Age}\t {item.AgeHashed}");
 Console.WriteLine($"{item.Category}\t {item.CategoryHashed}\t\t " +
 $"{item.Age}\t {item.AgeHashed}");

 // Expected data after the transformation.
 //
Expand All@@ -58,20 +68,24 @@ public static void Example()
 // MLB 36206 18 127
 // MLS 6013 14 62

 // For the Category column, where we set the maximumNumberOfInverts parameter, the names of the original categories, 
 // and their correspondance with the generated hash values is preserved in the Annotations in the format of indices and values. 
 // the indices array will have the hashed values, and the corresponding element, position-wise, in the values array will 
 // contain the original value. 
 // For the Category column, where we set the maximumNumberOfInverts
 // parameter, the names of the original categories, and their
 // correspondance with the generated hash values is preserved in the
 // Annotations in the format of indices and values.the indices array
 // will have the hashed values, and the corresponding element,
 // position -wise, in the values array will contain the original value. 
 //
 // See below for an example on how to retrieve the mapping. 
 var slotNames = new VBuffer<ReadOnlyMemory<char>>();
 transformedData.Schema["CategoryHashed"].Annotations.GetValue("KeyValues", ref slotNames);
 transformedData.Schema["CategoryHashed"].Annotations.GetValue(
 "KeyValues", ref slotNames);

 var indices = slotNames.GetIndices();
 var categoryNames = slotNames.GetValues();

 for (int i = 0; i < indices.Length; i++)
 Console.WriteLine($"The original value of the {indices[i]} category is {categoryNames[i]}");
 Console.WriteLine($"The original value of the {indices[i]} " +
 $"category is {categoryNames[i]}");

 // Output Data
 // 
Expand Down

57 changes: 38 additions & 19 deletions
docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Conversion/KeyToValueToKey.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -11,8 +11,8 @@ public class KeyToValueToKey
 {
 public static void Example()
 {
 // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, 
 // as well as the source of randomness.
 // Create a new ML context, for ML.NET operations. It can be used for
 // exception tracking and logging, as well as the source of randomness.
 var mlContext = new MLContext();

 // Get a small dataset as an IEnumerable.
Expand All@@ -27,25 +27,40 @@ public static void Example()

 // A pipeline to convert the terms of the 'Review' column in 
 // making use of default settings.
 var defaultPipeline = mlContext.Transforms.Text.TokenizeIntoWords("TokenizedText", nameof(DataPoint.Review))
 .Append(mlContext.Transforms.Conversion.MapValueToKey(nameof(TransformedData.Keys), "TokenizedText"));
 var defaultPipeline = mlContext.Transforms.Text.TokenizeIntoWords(
 "TokenizedText", nameof(DataPoint.Review)).Append(mlContext
 .Transforms.Conversion.MapValueToKey(nameof(TransformedData.Keys),
 "TokenizedText"));

@codemzscodemzsJul 2, 2019

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

please move .Append to a new line.
 // Another pipeline, that customizes the advanced settings of the ValueToKeyMappingEstimator.
 // We can change the maximumNumberOfKeys to limit how many keys will get generated out of the set of words, 
 // and condition the order in which they get evaluated by changing keyOrdinality from the default ByOccurence (order in which they get encountered) 
 // to value/alphabetically.
 var customizedPipeline = mlContext.Transforms.Text.TokenizeIntoWords("TokenizedText", nameof(DataPoint.Review))
 .Append(mlContext.Transforms.Conversion.MapValueToKey(nameof(TransformedData.Keys), "TokenizedText", maximumNumberOfKeys: 10,
 keyOrdinality: ValueToKeyMappingEstimator.KeyOrdinality.ByValue));
 // Another pipeline, that customizes the advanced settings of the
 // ValueToKeyMappingEstimator. We can change the maximumNumberOfKeys to
 // limit how many keys will get generated out of the set of words, and
 // condition the order in which they get evaluated by changing
 // keyOrdinality from the default ByOccurence (order in which they get
 // encountered) to value/alphabetically.
 var customizedPipeline = mlContext.Transforms.Text.TokenizeIntoWords(
 "TokenizedText", nameof(DataPoint.Review)).Append(mlContext
 .Transforms.Conversion.MapValueToKey(nameof(TransformedData.Keys),
 "TokenizedText", maximumNumberOfKeys: 10, keyOrdinality:
 ValueToKeyMappingEstimator.KeyOrdinality.ByValue));

 // The transformed data.
 var transformedDataDefault = defaultPipeline.Fit(trainData).Transform(trainData);
 var transformedDataCustomized = customizedPipeline.Fit(trainData).Transform(trainData);
 var transformedDataDefault = defaultPipeline.Fit(trainData).Transform(
 trainData);

 var transformedDataCustomized = customizedPipeline.Fit(trainData)
 .Transform(trainData);

 // Getting the resulting data as an IEnumerable.
 // This will contain the newly created columns.
 IEnumerable<TransformedData> defaultData = mlContext.Data.CreateEnumerable<TransformedData>(transformedDataDefault, reuseRowObject: false);
 IEnumerable<TransformedData> customizedData = mlContext.Data.CreateEnumerable<TransformedData>(transformedDataCustomized, reuseRowObject: false);
 IEnumerable<TransformedData> defaultData = mlContext.Data.
 CreateEnumerable<TransformedData>(transformedDataDefault,
 reuseRowObject: false);

 IEnumerable<TransformedData> customizedData = mlContext.Data.
 CreateEnumerable<TransformedData>(transformedDataCustomized,
 reuseRowObject: false);

 Console.WriteLine($"Keys");
 foreach (var dataRow in defaultData)
 Console.WriteLine($"{string.Join(',', dataRow.Keys)}");
Expand All@@ -65,13 +80,17 @@ public static void Example()
 // 8,2,9,7,6,4
 // 3,10,0,0,0
 // 3,10,0,0,0,8
 // Retrieve the original values, by appending the KeyToValue etimator to the existing pipelines
 // to convert the keys back to the strings.
 var pipeline = defaultPipeline.Append(mlContext.Transforms.Conversion.MapKeyToValue(nameof(TransformedData.Keys)));
 // Retrieve the original values, by appending the KeyToValue etimator to
 // the existing pipelines to convert the keys back to the strings.
 var pipeline = defaultPipeline.Append(mlContext.Transforms.Conversion
 .MapKeyToValue(nameof(TransformedData.Keys)));

 transformedDataDefault = pipeline.Fit(trainData).Transform(trainData);

 // Preview of the DefaultColumnName column obtained.
 var originalColumnBack = transformedDataDefault.GetColumn<VBuffer<ReadOnlyMemory<char>>>(transformedDataDefault.Schema[nameof(TransformedData.Keys)]);
 var originalColumnBack = transformedDataDefault.GetColumn<VBuffer<
 ReadOnlyMemory<char>>>(transformedDataDefault.Schema[nameof(
 TransformedData.Keys)]);

 foreach (var row in originalColumnBack)
 {
Expand Down

21 changes: 13 additions & 8 deletions
docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Conversion/MapKeyToBinaryVector.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -7,14 +7,16 @@ namespace Samples.Dynamic
{
 class MapKeyToBinaryVector
 {
 /// This example demonstrates the use of MapKeyToVector by mapping keys to floats[] of 0 and 1, representing the number in binary format. 
 /// Because the ML.NET KeyType maps the missing value to zero, counting starts at 1, so the uint values
 /// converted to KeyTypes will appear skewed by one. 
 /// This example demonstrates the use of MapKeyToVector by mapping keys to
 /// floats[] of 0 and 1, representing the number in binary format. 
 /// Because the ML.NET KeyType maps the missing value to zero, counting
 /// starts at 1, so the uint values converted to KeyTypes will appear
 /// skewed by one. 
 /// See https://github.com/dotnet/machinelearning/blob/master/docs/code/IDataViewTypeSystem.md#key-types
 public static void Example()
 {
 // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, 
 // as well as the source of randomness.
 // Create a new ML context, for ML.NET operations. It can be used for
 // exception tracking and logging, as well as the source of randomness.
 var mlContext = new MLContext();

 // Get a small dataset as an IEnumerable.
Expand All@@ -30,18 +32,21 @@ public static void Example()
 var data = mlContext.Data.LoadFromEnumerable(rawData);

 // Constructs the ML.net pipeline
 var pipeline = mlContext.Transforms.Conversion.MapKeyToBinaryVector("TimeframeVector", "Timeframe");
 var pipeline = mlContext.Transforms.Conversion.MapKeyToBinaryVector(
 "TimeframeVector", "Timeframe");

 // Fits the pipeline to the data.
 IDataView transformedData = pipeline.Fit(data).Transform(data);

 // Getting the resulting data as an IEnumerable.
 // This will contain the newly created columns.
 IEnumerable<TransformedData> features = mlContext.Data.CreateEnumerable<TransformedData>(transformedData, reuseRowObject: false);
 IEnumerable<TransformedData> features = mlContext.Data.CreateEnumerable<
 TransformedData>(transformedData, reuseRowObject: false);

 Console.WriteLine($" Timeframe TimeframeVector");
 foreach (var featureRow in features)
 Console.WriteLine($"{featureRow.Timeframe}\t\t\t{string.Join(',', featureRow.TimeframeVector)}");
 Console.WriteLine($"{featureRow.Timeframe}\t\t\t" +
 $"{string.Join(',', featureRow.TimeframeVector)}");

 // Timeframe TimeframeVector
 // 10 0,1,0,0,1 //binary representation of 9, the original value
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Universal Dark Mode - works on any site\n(function() {\n var enabled = true;\n \n function applyDarkMode() {\n if (!enabled) return;\n \n // Create style element if it doesn't exist\n var style = document.getElementById('universal-dark-mode-style');\n if (!style) {\n style = document.createElement('style');\n style.id = 'universal-dark-mode-style';\n document.head.appendChild(style);\n }\n \n // Dark mode CSS - inverts colors but preserves images/video\n style.textContent = '\n /* Invert everything except media */\n html {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #1a1a2e !important;\n }\n \n /* Restore images, videos, iframes, canvas */\n img, video, iframe, canvas, svg, picture, [style*=\"background-image\"] {\n filter: invert(1) hue-rotate(180deg) !important;\n }\n \n /* Preserve specific elements that should not be inverted */\n .no-dark-mode, .no-dark-mode *,\n [data-theme=\"light\"], [data-theme=\"light\"],\n .ace_editor, .ace_editor *,\n .CodeMirror, .CodeMirror *,\n .monaco-editor, .monaco-editor *,\n .markdown-body pre, .markdown-body pre *,\n .highlight, .highlight *,\n pre code, pre code * {\n filter: none !important;\n }\n \n /* Fix common UI elements */\n .modal, .popup, .dropdown-menu, .tooltip, .popover {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #2d2d44 !important;\n border-color: #444 !important;\n }\n \n /* Scrollbars */\n ::-webkit-scrollbar { background: #1a1a2e !important; }\n ::-webkit-scrollbar-thumb { background: #444 !important; }\n ::-webkit-scrollbar-thumb:hover { background: #555 !important; }\n \n /* Selection */\n ::selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ::-moz-selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ';\n }\n \n function removeDarkMode() {\n var style = document.getElementById('universal-dark-mode-style');\n if (style) style.remove();\n }\n \n // Toggle with Alt+Shift+D\n document.addEventListener('keydown', function(e) {\n if (e.altKey && e.shiftKey && e.key === 'D') {\n e.preventDefault();\n enabled = !enabled;\n if (enabled) {\n applyDarkMode();\n console.log('[Universal Dark Mode] Enabled');\n } else {\n removeDarkMode();\n console.log('[Universal Dark Mode] Disabled');\n }\n }\n });\n \n // Apply on load\n applyDarkMode();\n \n // Re-apply on dynamic content\n var observer = new MutationObserver(function(mutations) {\n if (enabled && !document.getElementById('universal-dark-mode-style')) {\n applyDarkMode();\n }\n });\n observer.observe(document.head, { childList: true });\n \n console.log('[Universal Dark Mode] Loaded - Press Alt+Shift+D to toggle');\n})();", "Universal Dark Mode");
}
} catch(__e) { console.warn('[Userscript:Universal Dark Mode]', __e); }
})();
})();

Skip to content
Reformatting Conversion, FeatureSelection and Image Analytics of Transform to Width 85#3943

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

rayankrish
merged 7 commits into

from

Jul 3, 2019
 Merged


Reformatting Conversion, FeatureSelection and Image Analytics of Transform to Width 85
#3943
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


7 commits


Select commit
Hold shift + click to select a range
9267d02

samples/dynamic/transforms/conversions formatted to 85 char

Jul 1, 2019
f2a18ed

samples/dynamic/transforms/featureselection formatted to 85 char

Jul 1, 2019
38c8938

ImageAnalytics

Jul 1, 2019
67080d1

ImageAnalytics

Jul 1, 2019
ecd2738

small corrections to formatting

Jul 2, 2019
91532b0

Merge branch 'transform85' of https://github.com/RadicalRayan/machine…

Jul 2, 2019
3deb533

minor tab and empty line changes

Jul 2, 2019

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


13 changes: 9 additions & 4 deletions
docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Conversion/ConvertType.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -20,17 +20,22 @@ public static void Example()
 var data = mlContext.Data.LoadFromEnumerable(rawData);

 // Construct the pipeline.
 var pipeline = mlContext.Transforms.Conversion.ConvertType("SurvivedInt32", "Survived", DataKind.Int32);
 var pipeline = mlContext.Transforms.Conversion.ConvertType(
 "SurvivedInt32", "Survived", DataKind.Int32);

 // Let's train our pipeline, and then apply it to the same data.
 var transformer = pipeline.Fit(data);
 var transformedData = transformer.Transform(data);

 // Display original column 'Survived' (boolean) and converted column 'SurvivedInt32' (Int32)
 var convertedData = mlContext.Data.CreateEnumerable<TransformedData>(transformedData, true);
 // Display original column 'Survived' (boolean) and converted column 
 // SurvivedInt32' (Int32)
 var convertedData = mlContext.Data.CreateEnumerable<TransformedData>(
 transformedData, true);

 foreach (var item in convertedData)
@codemzscodemzsJul 2, 2019

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

new line between any two lines that were broken down into multiple lines but no new line between multiple breakdowns of a single line.
 {
 Console.WriteLine("A:{0,-10} Aconv:{1}", item.Survived, item.SurvivedInt32);
 Console.WriteLine("A:{0,-10} Aconv:{1}", item.Survived,
 item.SurvivedInt32);
 }

 // Output
Expand Down

38 changes: 26 additions & 12 deletions
docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Conversion/ConvertTypeMultiColumn.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -4,22 +4,33 @@

namespace Samples.Dynamic
{
 // This example illustrates how to convert multiple columns of different types to one type, in this case System.Single. 
 // This is often a useful data transformation before concatenating the features together and passing them to a particular estimator.
 // This example illustrates how to convert multiple columns of different types
 // to one type, in this case System.Single. 
 // This is often a useful data transformation before concatenating the features
 // together and passing them to a particular estimator.
 public static class ConvertTypeMultiColumn
 { 
 public static void Example()
 {
 // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, 
 // as well as the source of randomness.
 // Create a new ML context, for ML.NET operations. It can be used for
 // exception tracking and logging, as well as the source of randomness.
 var mlContext = new MLContext(seed: 1);

 var rawData = new[] {
 new InputData() { Feature1 = true, Feature2 = "0.4", Feature3 = DateTime.Now, Feature4 = 0.145},
 new InputData() { Feature1 = false, Feature2 = "0.5", Feature3 = DateTime.Today, Feature4 = 3.14},
 new InputData() { Feature1 = false, Feature2 = "14", Feature3 = DateTime.Today, Feature4 = 0.2046},
 new InputData() { Feature1 = false, Feature2 = "23", Feature3 = DateTime.Now, Feature4 = 0.1206},
 new InputData() { Feature1 = true, Feature2 = "8904", Feature3 = DateTime.UtcNow, Feature4 = 8.09},
 new InputData() { Feature1 = true, Feature2 = "0.4",
 Feature3 = DateTime.Now, Feature4 = 0.145},

 new InputData() { Feature1 = false, Feature2 = "0.5",
 Feature3 = DateTime.Today, Feature4 = 3.14},

 new InputData() { Feature1 = false, Feature2 = "14",
 Feature3 = DateTime.Today, Feature4 = 0.2046},

 new InputData() { Feature1 = false, Feature2 = "23",
 Feature3 = DateTime.Now, Feature4 = 0.1206},

 new InputData() { Feature1 = true, Feature2 = "8904",
 Feature3 = DateTime.UtcNow, Feature4 = 8.09},
 };

 // Convert the data to an IDataView.
Expand All@@ -37,17 +48,20 @@ public static void Example()

 // Let's fit our pipeline to the data.
 var transformer = pipeline.Fit(data);
 // Transforming the same data. This will add the 4 columns defined in the pipeline, containing the converted
 // Transforming the same data. This will add the 4 columns defined in
 // the pipeline, containing the converted
 // values of the initial columns. 
 var transformedData = transformer.Transform(data);

 // Shape the transformed data as a strongly typed IEnumerable.
 var convertedData = mlContext.Data.CreateEnumerable<TransformedData>(transformedData, true);
 var convertedData = mlContext.Data.CreateEnumerable<TransformedData>(
 transformedData, true);

 // Printing the results.
 Console.WriteLine("Converted1\t Converted2\t Converted3\t Converted4");
 foreach (var item in convertedData)
 Console.WriteLine($"\t{item.Converted1}\t {item.Converted2}\t\t {item.Converted3}\t {item.Converted4}");
 Console.WriteLine($"\t{item.Converted1}\t {item.Converted2}\t\t " +
 $"{item.Converted3}\t {item.Converted4}");

 // Transformed data.
 //
Expand Down

60 changes: 37 additions & 23 deletions
docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Conversion/Hash.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -9,8 +9,8 @@ public static class Hash
 {
 public static void Example()
 {
 // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, 
 // as well as the source of randomness.
 // Create a new ML context, for ML.NET operations. It can be used for
 // exception tracking and logging, as well as the source of randomness.
 var mlContext = new MLContext(seed: 1);

 // Get a small dataset as an IEnumerable.
Expand All@@ -24,30 +24,40 @@ public static void Example()

 var data = mlContext.Data.LoadFromEnumerable(rawData);

 // Construct the pipeline that would hash the two columns and store the results in new columns.
 // The first transform hashes the string column and the second transform hashes the integer column.
 // Construct the pipeline that would hash the two columns and store the
 // results in new columns. The first transform hashes the string column
 // and the second transform hashes the integer column.
 //
 // Hashing is not a reversible operation, so there is no way to retrive the original value from the hashed value. 
 // Sometimes, for debugging, or model explainability, users will need to know what values in the original columns generated
 // the values in the hashed columns, since the algorithms will mostly use the hashed values for further computations.
 // The Hash method will preserve the mapping from the original values to the hashed values in the Annotations of the 
 // newly created column (column populated with the hashed values). 
 // 
 // Setting the maximumNumberOfInverts parameters to -1 will preserve the full map. 
 // If that parameter is left to the default 0 value, the mapping is not preserved.
 var pipeline = mlContext.Transforms.Conversion.Hash("CategoryHashed", "Category", numberOfBits: 16, maximumNumberOfInverts: -1)
 .Append(mlContext.Transforms.Conversion.Hash("AgeHashed", "Age", numberOfBits: 8));
 // Hashing is not a reversible operation, so there is no way to retrive
 // the original value from the hashed value. Sometimes, for debugging,
 // or model explainability, users will need to know what values in the
 // original columns generated the values in the hashed columns, since
 // the algorithms will mostly use the hashed values for further
 // computations. The Hash method will preserve the mapping from the
 // original values to the hashed values in the Annotations of the newly
 // created column (column populated with the hashed values). 
 //
 // Setting the maximumNumberOfInverts parameters to -1 will preserve the
 // full map. If that parameter is left to the default 0 value, the
 // mapping is not preserved.
 var pipeline = mlContext.Transforms.Conversion.Hash("CategoryHashed",
 "Category", numberOfBits: 16, maximumNumberOfInverts: -1)
 .Append(mlContext.Transforms.Conversion.Hash("AgeHashed", "Age",
 numberOfBits: 8));

 // Let's fit our pipeline, and then apply it to the same data.
 var transformer = pipeline.Fit(data);
 var transformedData = transformer.Transform(data);

 // Convert the post transformation from the IDataView format to an IEnumerable<TransformedData> for easy consumption.
 var convertedData = mlContext.Data.CreateEnumerable<TransformedDataPoint>(transformedData, true);
 // Convert the post transformation from the IDataView format to an
 // IEnumerable <TransformedData> for easy consumption.
 var convertedData = mlContext.Data.CreateEnumerable<
 TransformedDataPoint>(transformedData, true);

 Console.WriteLine("Category CategoryHashed\t Age\t AgeHashed");
 foreach (var item in convertedData)
 Console.WriteLine($"{item.Category}\t {item.CategoryHashed}\t\t {item.Age}\t {item.AgeHashed}");
 Console.WriteLine($"{item.Category}\t {item.CategoryHashed}\t\t " +
 $"{item.Age}\t {item.AgeHashed}");

 // Expected data after the transformation.
 //
Expand All@@ -58,20 +68,24 @@ public static void Example()
 // MLB 36206 18 127
 // MLS 6013 14 62

 // For the Category column, where we set the maximumNumberOfInverts parameter, the names of the original categories, 
 // and their correspondance with the generated hash values is preserved in the Annotations in the format of indices and values. 
 // the indices array will have the hashed values, and the corresponding element, position-wise, in the values array will 
 // contain the original value. 
 // For the Category column, where we set the maximumNumberOfInverts
 // parameter, the names of the original categories, and their
 // correspondance with the generated hash values is preserved in the
 // Annotations in the format of indices and values.the indices array
 // will have the hashed values, and the corresponding element,
 // position -wise, in the values array will contain the original value. 
 //
 // See below for an example on how to retrieve the mapping. 
 var slotNames = new VBuffer<ReadOnlyMemory<char>>();
 transformedData.Schema["CategoryHashed"].Annotations.GetValue("KeyValues", ref slotNames);
 transformedData.Schema["CategoryHashed"].Annotations.GetValue(
 "KeyValues", ref slotNames);

 var indices = slotNames.GetIndices();
 var categoryNames = slotNames.GetValues();

 for (int i = 0; i < indices.Length; i++)
 Console.WriteLine($"The original value of the {indices[i]} category is {categoryNames[i]}");
 Console.WriteLine($"The original value of the {indices[i]} " +
 $"category is {categoryNames[i]}");

 // Output Data
 // 
Expand Down

57 changes: 38 additions & 19 deletions
docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Conversion/KeyToValueToKey.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -11,8 +11,8 @@ public class KeyToValueToKey
 {
 public static void Example()
 {
 // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, 
 // as well as the source of randomness.
 // Create a new ML context, for ML.NET operations. It can be used for
 // exception tracking and logging, as well as the source of randomness.
 var mlContext = new MLContext();

 // Get a small dataset as an IEnumerable.
Expand All@@ -27,25 +27,40 @@ public static void Example()

 // A pipeline to convert the terms of the 'Review' column in 
 // making use of default settings.
 var defaultPipeline = mlContext.Transforms.Text.TokenizeIntoWords("TokenizedText", nameof(DataPoint.Review))
 .Append(mlContext.Transforms.Conversion.MapValueToKey(nameof(TransformedData.Keys), "TokenizedText"));
 var defaultPipeline = mlContext.Transforms.Text.TokenizeIntoWords(
 "TokenizedText", nameof(DataPoint.Review)).Append(mlContext
 .Transforms.Conversion.MapValueToKey(nameof(TransformedData.Keys),
 "TokenizedText"));

@codemzscodemzsJul 2, 2019

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

please move .Append to a new line.
 // Another pipeline, that customizes the advanced settings of the ValueToKeyMappingEstimator.
 // We can change the maximumNumberOfKeys to limit how many keys will get generated out of the set of words, 
 // and condition the order in which they get evaluated by changing keyOrdinality from the default ByOccurence (order in which they get encountered) 
 // to value/alphabetically.
 var customizedPipeline = mlContext.Transforms.Text.TokenizeIntoWords("TokenizedText", nameof(DataPoint.Review))
 .Append(mlContext.Transforms.Conversion.MapValueToKey(nameof(TransformedData.Keys), "TokenizedText", maximumNumberOfKeys: 10,
 keyOrdinality: ValueToKeyMappingEstimator.KeyOrdinality.ByValue));
 // Another pipeline, that customizes the advanced settings of the
 // ValueToKeyMappingEstimator. We can change the maximumNumberOfKeys to
 // limit how many keys will get generated out of the set of words, and
 // condition the order in which they get evaluated by changing
 // keyOrdinality from the default ByOccurence (order in which they get
 // encountered) to value/alphabetically.
 var customizedPipeline = mlContext.Transforms.Text.TokenizeIntoWords(
 "TokenizedText", nameof(DataPoint.Review)).Append(mlContext
 .Transforms.Conversion.MapValueToKey(nameof(TransformedData.Keys),
 "TokenizedText", maximumNumberOfKeys: 10, keyOrdinality:
 ValueToKeyMappingEstimator.KeyOrdinality.ByValue));

 // The transformed data.
 var transformedDataDefault = defaultPipeline.Fit(trainData).Transform(trainData);
 var transformedDataCustomized = customizedPipeline.Fit(trainData).Transform(trainData);
 var transformedDataDefault = defaultPipeline.Fit(trainData).Transform(
 trainData);

 var transformedDataCustomized = customizedPipeline.Fit(trainData)
 .Transform(trainData);

 // Getting the resulting data as an IEnumerable.
 // This will contain the newly created columns.
 IEnumerable<TransformedData> defaultData = mlContext.Data.CreateEnumerable<TransformedData>(transformedDataDefault, reuseRowObject: false);
 IEnumerable<TransformedData> customizedData = mlContext.Data.CreateEnumerable<TransformedData>(transformedDataCustomized, reuseRowObject: false);
 IEnumerable<TransformedData> defaultData = mlContext.Data.
 CreateEnumerable<TransformedData>(transformedDataDefault,
 reuseRowObject: false);

 IEnumerable<TransformedData> customizedData = mlContext.Data.
 CreateEnumerable<TransformedData>(transformedDataCustomized,
 reuseRowObject: false);

 Console.WriteLine($"Keys");
 foreach (var dataRow in defaultData)
 Console.WriteLine($"{string.Join(',', dataRow.Keys)}");
Expand All@@ -65,13 +80,17 @@ public static void Example()
 // 8,2,9,7,6,4
 // 3,10,0,0,0
 // 3,10,0,0,0,8
 // Retrieve the original values, by appending the KeyToValue etimator to the existing pipelines
 // to convert the keys back to the strings.
 var pipeline = defaultPipeline.Append(mlContext.Transforms.Conversion.MapKeyToValue(nameof(TransformedData.Keys)));
 // Retrieve the original values, by appending the KeyToValue etimator to
 // the existing pipelines to convert the keys back to the strings.
 var pipeline = defaultPipeline.Append(mlContext.Transforms.Conversion
 .MapKeyToValue(nameof(TransformedData.Keys)));

 transformedDataDefault = pipeline.Fit(trainData).Transform(trainData);

 // Preview of the DefaultColumnName column obtained.
 var originalColumnBack = transformedDataDefault.GetColumn<VBuffer<ReadOnlyMemory<char>>>(transformedDataDefault.Schema[nameof(TransformedData.Keys)]);
 var originalColumnBack = transformedDataDefault.GetColumn<VBuffer<
 ReadOnlyMemory<char>>>(transformedDataDefault.Schema[nameof(
 TransformedData.Keys)]);

 foreach (var row in originalColumnBack)
 {
Expand Down

21 changes: 13 additions & 8 deletions
docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Conversion/MapKeyToBinaryVector.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -7,14 +7,16 @@ namespace Samples.Dynamic
{
 class MapKeyToBinaryVector
 {
 /// This example demonstrates the use of MapKeyToVector by mapping keys to floats[] of 0 and 1, representing the number in binary format. 
 /// Because the ML.NET KeyType maps the missing value to zero, counting starts at 1, so the uint values
 /// converted to KeyTypes will appear skewed by one. 
 /// This example demonstrates the use of MapKeyToVector by mapping keys to
 /// floats[] of 0 and 1, representing the number in binary format. 
 /// Because the ML.NET KeyType maps the missing value to zero, counting
 /// starts at 1, so the uint values converted to KeyTypes will appear
 /// skewed by one. 
 /// See https://github.com/dotnet/machinelearning/blob/master/docs/code/IDataViewTypeSystem.md#key-types
 public static void Example()
 {
 // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, 
 // as well as the source of randomness.
 // Create a new ML context, for ML.NET operations. It can be used for
 // exception tracking and logging, as well as the source of randomness.
 var mlContext = new MLContext();

 // Get a small dataset as an IEnumerable.
Expand All@@ -30,18 +32,21 @@ public static void Example()
 var data = mlContext.Data.LoadFromEnumerable(rawData);

 // Constructs the ML.net pipeline
 var pipeline = mlContext.Transforms.Conversion.MapKeyToBinaryVector("TimeframeVector", "Timeframe");
 var pipeline = mlContext.Transforms.Conversion.MapKeyToBinaryVector(
 "TimeframeVector", "Timeframe");

 // Fits the pipeline to the data.
 IDataView transformedData = pipeline.Fit(data).Transform(data);

 // Getting the resulting data as an IEnumerable.
 // This will contain the newly created columns.
 IEnumerable<TransformedData> features = mlContext.Data.CreateEnumerable<TransformedData>(transformedData, reuseRowObject: false);
 IEnumerable<TransformedData> features = mlContext.Data.CreateEnumerable<
 TransformedData>(transformedData, reuseRowObject: false);

 Console.WriteLine($" Timeframe TimeframeVector");
 foreach (var featureRow in features)
 Console.WriteLine($"{featureRow.Timeframe}\t\t\t{string.Join(',', featureRow.TimeframeVector)}");
 Console.WriteLine($"{featureRow.Timeframe}\t\t\t" +
 $"{string.Join(',', featureRow.TimeframeVector)}");

 // Timeframe TimeframeVector
 // 10 0,1,0,0,1 //binary representation of 9, the original value
Expand Down
Loading