Adding support for training metrics in PipelineSweeperMacro + new graph variable outputs#152

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
george-microsoft merged 15 commits into from
May 23, 2018
Merged

Adding support for training metrics in PipelineSweeperMacro + new graph variable outputs #152

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
05482eb
Adding support for training metrics in PipelineSweeperMacro and neede…
george-microsoft May 14, 2018
b789159
Changes for revieer comments.
george-microsoft May 14, 2018
76dd270
Changes to address reviewer comments.
george-microsoft May 14, 2018
483a50b
More changes for reviewer commenrts.
george-microsoft May 17, 2018
d42a522
Changes for reviewer comments.
george-microsoft May 18, 2018
fbee51c
Changes for latest round of reviewer comments.
george-microsoft May 21, 2018
0285c87
Changed where XML comment was placed.
george-microsoft May 21, 2018
6f92693
Added more tests (uncommented and fixed) for auto inference. Changed …
george-microsoft May 22, 2018
e006ee1
Added another test, TestPipelineSweeperMacroNoTransforms.
george-microsoft May 22, 2018
90d96a3
Updated tests to include warning disabling (following Zeeshan S's exa…
george-microsoft May 22, 2018
1d1c303
Merge branch 'master' into AutoML-GraphChanges
george-microsoft May 22, 2018
ce9792f
Changes to checks in AutoMlUtils (more correct usage of them).
george-microsoft May 22, 2018
a828266
Merge branch 'AutoML-GraphChanges' of https://github.com/george-micro…
george-microsoft May 22, 2018
c642f68
Fixing issue with ExceptParam using value and not name of parameters.
george-microsoft May 22, 2018
e066db0
Fixing errors on use of ExceptParam.
george-microsoft May 22, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 7 additions & 7 deletions src/Microsoft.ML.Data/EntryPoints/InputBuilder.cs
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -832,21 +832,21 @@ public static class SweepableDiscreteParam
public static class PipelineSweeperSupportedMetrics
{
public new static string ToString() => "SupportedMetric";
public const string Auc = "Auc";
public const string Auc = "AUC";
public const string AccuracyMicro = "AccuracyMicro";
public const string AccuracyMacro = "AccuracyMacro";
public const string F1 = "F1";
public const string AuPrc = "AuPrc";
public const string AuPrc = "AUPRC";
public const string TopKAccuracy = "TopKAccuracy";
public const string L1 = "L1";
public const string L2 = "L2";
public const string Rms = "Rms";
public const string Rms = "RMS";
public const string LossFn = "LossFn";
public const string RSquared = "RSquared";
public const string LogLoss = "LogLoss";
public const string LogLossReduction = "LogLossReduction";
public const string Ndcg = "Ndcg";
public const string Dcg = "Dcg";
public const string Ndcg = "NDCG";
public const string Dcg = "DCG";
public const string PositivePrecision = "PositivePrecision";
public const string PositiveRecall = "PositiveRecall";
public const string NegativePrecision = "NegativePrecision";
Expand All@@ -858,9 +858,9 @@ public static class PipelineSweeperSupportedMetrics
public const string ThreshAtK = "ThreshAtK";
public const string ThreshAtP = "ThreshAtP";
public const string ThreshAtNumPos = "ThreshAtNumPos";
public const string Nmi = "Nmi";
public const string Nmi = "NMI";
public const string AvgMinScore = "AvgMinScore";
public const string Dbi = "Dbi";
public const string Dbi = "DBI";
}
}
}
20 changes: 12 additions & 8 deletions src/Microsoft.ML.PipelineInference/AutoInference.cs
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -158,7 +158,8 @@ private bool GetDataVariableName(IExceptionContext ectx, string nameOfData, JTok
return false;

string dataVar = firstNodeInputs.Value<String>(nameOfData);
ectx.Check(VariableBinding.IsValidVariableName(ectx, dataVar), $"Invalid variable name {dataVar}.");
if (!VariableBinding.IsValidVariableName(ectx, dataVar))
throw ectx.ExceptParam(nameof(nameOfData), $"Invalid variable name {dataVar}.");

variableName = dataVar.Substring(1);
return true;
Expand All@@ -172,12 +173,14 @@ private bool GetDataVariableName(IExceptionContext ectx, string nameOfData, JTok
public sealed class RunSummary
{
public double MetricValue { get; }
public double TrainingMetricValue { get; }
public int NumRowsInTraining { get; }
public long RunTimeMilliseconds { get; }

public RunSummary(double metricValue, int numRows, long runTimeMilliseconds)
public RunSummary(double metricValue, int numRows, long runTimeMilliseconds, double trainingMetricValue)
{
MetricValue = metricValue;
TrainingMetricValue = trainingMetricValue;
NumRowsInTraining = numRows;
RunTimeMilliseconds = runTimeMilliseconds;
}
Expand DownExpand Up@@ -303,7 +306,7 @@ private void MainLearningLoop(int batchSize, int numOfTrainingRows)
var stopwatch = new Stopwatch();
var probabilityUtils = new Sweeper.Algorithms.SweeperProbabilityUtils(_host);

while (!_terminator.ShouldTerminate(_history))
while (!_terminator.ShouldTerminate(_history))
{
// Get next set of candidates
var currentBatchSize = batchSize;
Expand DownExpand Up@@ -341,16 +344,17 @@ private void ProcessPipeline(Sweeper.Algorithms.SweeperProbabilityUtils utils, S

// Run pipeline, and time how long it takes
stopwatch.Restart();
double d = candidate.RunTrainTestExperiment(_trainData.Take(randomizedNumberOfRows),
_testData, Metric, TrainerKind);
candidate.RunTrainTestExperiment(_trainData.Take(randomizedNumberOfRows),
_testData, Metric, TrainerKind, out var testMetricVal, out var trainMetricVal);
stopwatch.Stop();

// Handle key collisions on sorted list
while (_sortedSampledElements.ContainsKey(d))
d += 1e-10;
while (_sortedSampledElements.ContainsKey(testMetricVal))
testMetricVal += 1e-10;

// Save performance score
candidate.PerformanceSummary = new RunSummary(d, randomizedNumberOfRows, stopwatch.ElapsedMilliseconds);
candidate.PerformanceSummary =
new RunSummary(testMetricVal, randomizedNumberOfRows, stopwatch.ElapsedMilliseconds, trainMetricVal);
_sortedSampledElements.Add(candidate.PerformanceSummary.MetricValue, candidate);
_history.Add(candidate);
}
Expand Down
33 changes: 24 additions & 9 deletions src/Microsoft.ML.PipelineInference/AutoMlUtils.cs
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -15,21 +15,34 @@ namespace Microsoft.ML.Runtime.PipelineInference
{
public static class AutoMlUtils
{
public static AutoInference.RunSummary ExtractRunSummary(IHostEnvironment env, IDataView data, string metricColumnName)
public static double ExtractValueFromIDV(IHostEnvironment env, IDataView result, string columnName)
{
double metricValue = 0;
int numRows = 0;
var schema = data.Schema;
schema.TryGetColumnIndex(metricColumnName, out var metricCol);
Contracts.CheckValue(env, nameof(env));
env.CheckValue(result, nameof(result));
env.CheckNonEmpty(columnName, nameof(columnName));

using (var cursor = data.GetRowCursor(col => col == metricCol))
double outputValue = 0;
var schema = result.Schema;
if (!schema.TryGetColumnIndex(columnName, out var metricCol))
throw env.ExceptParam(nameof(columnName), $"Schema does not contain column: {columnName}");

using (var cursor = result.GetRowCursor(col => col == metricCol))
{
var getter = cursor.GetGetter<double>(metricCol);
cursor.MoveNext();
getter(ref metricValue);
bool moved = cursor.MoveNext();
env.Check(moved, "Expected an IDataView with a single row. Results dataset has no rows to extract.");
getter(ref outputValue);
env.Check(!cursor.MoveNext(), "Expected an IDataView with a single row. Results dataset has too many rows.");
}

return new AutoInference.RunSummary(metricValue, numRows, 0);
return outputValue;
}

public static AutoInference.RunSummary ExtractRunSummary(IHostEnvironment env, IDataView result, string metricColumnName, IDataView trainResult = null)
{
double testingMetricValue = ExtractValueFromIDV(env, result, metricColumnName);
double trainingMetricValue = trainResult != null ? ExtractValueFromIDV(env, trainResult, metricColumnName) : double.MinValue;
return new AutoInference.RunSummary(testingMetricValue, 0, 0, trainingMetricValue);
}

public static CommonInputs.IEvaluatorInput CloneEvaluatorInstance(CommonInputs.IEvaluatorInput evalInput) =>
Expand DownExpand Up@@ -618,5 +631,7 @@ public static Tuple<string, string[]>[] ConvertToSweepArgumentStrings(TlcModule.
}
return results;
}

public static string GenerateOverallTrainingMetricVarName(Guid id) => $"Var_Training_OM_{id:N}";
}
}
19 changes: 14 additions & 5 deletions src/Microsoft.ML.PipelineInference/Macros/PipelineSweeperMacro.cs
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -65,18 +65,24 @@ public static Output ExtractSweepResult(IHostEnvironment env, ResultInput input)
var col1 = new KeyValuePair<string, ColumnType>("Graph", TextType.Instance);
var col2 = new KeyValuePair<string, ColumnType>("MetricValue", PrimitiveType.FromKind(DataKind.R8));
var col3 = new KeyValuePair<string, ColumnType>("PipelineId", TextType.Instance);
var col4 = new KeyValuePair<string, ColumnType>("TrainingMetricValue", PrimitiveType.FromKind(DataKind.R8));
var col5 = new KeyValuePair<string, ColumnType>("FirstInput", TextType.Instance);
var col6 = new KeyValuePair<string, ColumnType>("PredictorModel", TextType.Instance);

if (rows.Count == 0)
{
var host = env.Register("ExtractSweepResult");
outputView = new EmptyDataView(host, new SimpleSchema(host, col1, col2, col3));
outputView = new EmptyDataView(host, new SimpleSchema(host, col1, col2, col3, col4, col5, col6));
}
else
{
var builder = new ArrayDataViewBuilder(env);
builder.AddColumn(col1.Key, (PrimitiveType)col1.Value, rows.Select(r => new DvText(r.GraphJson)).ToArray());
builder.AddColumn(col2.Key, (PrimitiveType)col2.Value, rows.Select(r => r.MetricValue).ToArray());
builder.AddColumn(col3.Key, (PrimitiveType)col3.Value, rows.Select(r => new DvText(r.PipelineId)).ToArray());
builder.AddColumn(col4.Key, (PrimitiveType)col4.Value, rows.Select(r => r.TrainingMetricValue).ToArray());
builder.AddColumn(col5.Key, (PrimitiveType)col5.Value, rows.Select(r => new DvText(r.FirstInput)).ToArray());
builder.AddColumn(col6.Key, (PrimitiveType)col6.Value, rows.Select(r => new DvText(r.PredictorModel)).ToArray());
outputView = builder.GetDataView();
}
return new Output { Results = outputView, State = autoMlState };
Expand DownExpand Up@@ -132,11 +138,11 @@ public static CommonOutputs.MacroOutput<Output> PipelineSweep(
// Extract performance summaries and assign to previous candidate pipelines.
foreach (var pipeline in autoMlState.BatchCandidates)
{
if (node.Context.TryGetVariable(ExperimentUtils.GenerateOverallMetricVarName(pipeline.UniqueId),
out var v))
if (node.Context.TryGetVariable(ExperimentUtils.GenerateOverallMetricVarName(pipeline.UniqueId), out var v) &&
node.Context.TryGetVariable(AutoMlUtils.GenerateOverallTrainingMetricVarName(pipeline.UniqueId), out var v2))
{
pipeline.PerformanceSummary =
AutoMlUtils.ExtractRunSummary(env, (IDataView)v.Value, autoMlState.Metric.Name);
AutoMlUtils.ExtractRunSummary(env, (IDataView)v.Value, autoMlState.Metric.Name, (IDataView)v2.Value);
autoMlState.AddEvaluated(pipeline);
}
}
Expand DownExpand Up@@ -168,14 +174,17 @@ public static CommonOutputs.MacroOutput<Output> PipelineSweep(
{
// Add train test experiments to current graph for candidate pipeline
var subgraph = new Experiment(env);
var trainTestOutput = p.AddAsTrainTest(training, testing, autoMlState.TrainerKind, subgraph);
var trainTestOutput = p.AddAsTrainTest(training, testing, autoMlState.TrainerKind, subgraph, true);

// Change variable name to reference pipeline ID in output map, context and entrypoint output.
var uniqueName = ExperimentUtils.GenerateOverallMetricVarName(p.UniqueId);
var uniqueNameTraining = AutoMlUtils.GenerateOverallTrainingMetricVarName(p.UniqueId);
var sgNode = EntryPointNode.ValidateNodes(env, node.Context,
new JArray(subgraph.GetNodes().Last()), node.Catalog).Last();
sgNode.RenameOutputVariable(trainTestOutput.OverallMetrics.VarName, uniqueName, cascadeChanges: true);
sgNode.RenameOutputVariable(trainTestOutput.TrainingOverallMetrics.VarName, uniqueNameTraining, cascadeChanges: true);
trainTestOutput.OverallMetrics.VarName = uniqueName;
trainTestOutput.TrainingOverallMetrics.VarName = uniqueNameTraining;
expNodes.Add(sgNode);

// Store indicators, to pass to next iteration of macro.
Expand Down
1 change: 1 addition & 0 deletions src/Microsoft.ML.PipelineInference/Microsoft.ML.PipelineInference.csproj
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -17,6 +17,7 @@
<ProjectReference Include="..\Microsoft.ML.Core\Microsoft.ML.Core.csproj" />
<ProjectReference Include="..\Microsoft.ML.StandardLearners\Microsoft.ML.StandardLearners.csproj" />
<ProjectReference Include="..\Microsoft.ML.Sweeper\Microsoft.ML.Sweeper.csproj" />
<ProjectReference Include="..\Microsoft.ML\Microsoft.ML.csproj" />
</ItemGroup>

</Project>
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Add copy buttons to all 
 blocks\n(function() {\n function addCopyButtons() {\n document.querySelectorAll('pre code').forEach(function(codeBlock) {\n if (codeBlock.parentElement.hasAttribute('data-copy-added')) return;\n codeBlock.parentElement.setAttribute('data-copy-added', 'true');\n \n var btn = document.createElement('button');\n btn.textContent = 'Copy';\n btn.style.cssText = 'position:absolute;top:4px;right:4px;padding:2px 8px;font-size:11px;background:#4ecdc4;border:none;border-radius:4px;color:#1a1a2e;cursor:pointer;opacity:0.7;transition:opacity 0.2s;';\n btn.onmouseover = function() { this.style.opacity = '1'; };\n btn.onmouseout = function() { this.style.opacity = '0.7'; };\n btn.onclick = function() {\n navigator.clipboard.writeText(codeBlock.textContent).then(function() {\n btn.textContent = 'Copied!';\n setTimeout(function() { btn.textContent = 'Copy'; }, 1500);\n });\n };\n codeBlock.parentElement.style.position = 'relative';\n codeBlock.parentElement.appendChild(btn);\n });\n }\n \n addCopyButtons();\n \n // Re-run on dynamic content\n var observer = new MutationObserver(addCopyButtons);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Add Copy Buttons to Code Blocks");
}
} catch(__e) { console.warn('[Userscript:Add Copy Buttons to Code Blocks]', __e); }
})();
(function(){
try {
var __m = "github.com";
var __re = new RegExp('^' + "github\\.com" + '

Skip to content
Adding support for training metrics in PipelineSweeperMacro + new graph variable outputs#152

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

george-microsoft
merged 15 commits into

from

May 23, 2018
 Merged


Adding support for training metrics in PipelineSweeperMacro + new graph variable outputs
#152
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


15 commits


Select commit
Hold shift + click to select a range
05482eb

Adding support for training metrics in PipelineSweeperMacro and neede…


george-microsoft May 14, 2018
b789159

Changes for revieer comments.


george-microsoft May 14, 2018
76dd270

Changes to address reviewer comments.


george-microsoft May 14, 2018
483a50b

More changes for reviewer commenrts.


george-microsoft May 17, 2018
d42a522

Changes for reviewer comments.


george-microsoft May 18, 2018
fbee51c

Changes for latest round of reviewer comments.


george-microsoft May 21, 2018
0285c87

Changed where XML comment was placed.


george-microsoft May 21, 2018
6f92693

Added more tests (uncommented and fixed) for auto inference. Changed …


george-microsoft May 22, 2018
e006ee1

Added another test, TestPipelineSweeperMacroNoTransforms.


george-microsoft May 22, 2018
90d96a3

Updated tests to include warning disabling (following Zeeshan S's exa…


george-microsoft May 22, 2018
1d1c303

Merge branch 'master' into AutoML-GraphChanges


george-microsoft May 22, 2018
ce9792f

Changes to checks in AutoMlUtils (more correct usage of them).


george-microsoft May 22, 2018
a828266

Merge branch 'AutoML-GraphChanges' of https://github.com/george-micro…


george-microsoft May 22, 2018
c642f68

Fixing issue with ExceptParam using value and not name of parameters.


george-microsoft May 22, 2018
e066db0

Fixing errors on use of ExceptParam.


george-microsoft May 22, 2018

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


14 changes: 7 additions & 7 deletions
src/Microsoft.ML.Data/EntryPoints/InputBuilder.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -832,21 +832,21 @@ public static class SweepableDiscreteParam
 public static class PipelineSweeperSupportedMetrics
 {
 public new static string ToString() => "SupportedMetric";
 public const string Auc = "Auc";
 public const string Auc = "AUC";
 public const string AccuracyMicro = "AccuracyMicro";
 public const string AccuracyMacro = "AccuracyMacro";
 public const string F1 = "F1";
 public const string AuPrc = "AuPrc";
 public const string AuPrc = "AUPRC";
 public const string TopKAccuracy = "TopKAccuracy";
 public const string L1 = "L1";
 public const string L2 = "L2";
 public const string Rms = "Rms";
 public const string Rms = "RMS";
 public const string LossFn = "LossFn";
 public const string RSquared = "RSquared";
 public const string LogLoss = "LogLoss";
 public const string LogLossReduction = "LogLossReduction";
 public const string Ndcg = "Ndcg";
 public const string Dcg = "Dcg";
 public const string Ndcg = "NDCG";
 public const string Dcg = "DCG";
 public const string PositivePrecision = "PositivePrecision";
 public const string PositiveRecall = "PositiveRecall";
 public const string NegativePrecision = "NegativePrecision";
Expand All@@ -858,9 +858,9 @@ public static class PipelineSweeperSupportedMetrics
 public const string ThreshAtK = "ThreshAtK";
 public const string ThreshAtP = "ThreshAtP";
 public const string ThreshAtNumPos = "ThreshAtNumPos";
 public const string Nmi = "Nmi";
 public const string Nmi = "NMI";
 public const string AvgMinScore = "AvgMinScore";
 public const string Dbi = "Dbi";
 public const string Dbi = "DBI";
 }
 }
}

20 changes: 12 additions & 8 deletions
src/Microsoft.ML.PipelineInference/AutoInference.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -158,7 +158,8 @@ private bool GetDataVariableName(IExceptionContext ectx, string nameOfData, JTok
 return false;

 string dataVar = firstNodeInputs.Value<String>(nameOfData);
 ectx.Check(VariableBinding.IsValidVariableName(ectx, dataVar), $"Invalid variable name {dataVar}.");
 if (!VariableBinding.IsValidVariableName(ectx, dataVar))
 throw ectx.ExceptParam(nameof(nameOfData), $"Invalid variable name {dataVar}.");

 variableName = dataVar.Substring(1);
 return true;
Expand All@@ -172,12 +173,14 @@ private bool GetDataVariableName(IExceptionContext ectx, string nameOfData, JTok
 public sealed class RunSummary
 {
 public double MetricValue { get; }
 public double TrainingMetricValue { get; }
 public int NumRowsInTraining { get; }
 public long RunTimeMilliseconds { get; }

 public RunSummary(double metricValue, int numRows, long runTimeMilliseconds)
 public RunSummary(double metricValue, int numRows, long runTimeMilliseconds, double trainingMetricValue)
 {
 MetricValue = metricValue;
 TrainingMetricValue = trainingMetricValue;
 NumRowsInTraining = numRows;
 RunTimeMilliseconds = runTimeMilliseconds;
 }
Expand DownExpand Up@@ -303,7 +306,7 @@ private void MainLearningLoop(int batchSize, int numOfTrainingRows)
 var stopwatch = new Stopwatch();
 var probabilityUtils = new Sweeper.Algorithms.SweeperProbabilityUtils(_host);

while (!_terminator.ShouldTerminate(_history))
 while (!_terminator.ShouldTerminate(_history))
 {
 // Get next set of candidates
 var currentBatchSize = batchSize;
Expand DownExpand Up@@ -341,16 +344,17 @@ private void ProcessPipeline(Sweeper.Algorithms.SweeperProbabilityUtils utils, S

 // Run pipeline, and time how long it takes
 stopwatch.Restart();
double d = candidate.RunTrainTestExperiment(_trainData.Take(randomizedNumberOfRows),
 _testData, Metric, TrainerKind);
 candidate.RunTrainTestExperiment(_trainData.Take(randomizedNumberOfRows),
 _testData, Metric, TrainerKind, out var testMetricVal, out var trainMetricVal);
 stopwatch.Stop();

 // Handle key collisions on sorted list
 while (_sortedSampledElements.ContainsKey(d))
d += 1e-10;
 while (_sortedSampledElements.ContainsKey(testMetricVal))
testMetricVal += 1e-10;

 // Save performance score
 candidate.PerformanceSummary = new RunSummary(d, randomizedNumberOfRows, stopwatch.ElapsedMilliseconds);
 candidate.PerformanceSummary = 
 new RunSummary(testMetricVal, randomizedNumberOfRows, stopwatch.ElapsedMilliseconds, trainMetricVal);
 _sortedSampledElements.Add(candidate.PerformanceSummary.MetricValue, candidate);
 _history.Add(candidate);
 }
Expand Down

33 changes: 24 additions & 9 deletions
src/Microsoft.ML.PipelineInference/AutoMlUtils.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -15,21 +15,34 @@ namespace Microsoft.ML.Runtime.PipelineInference
{
 public static class AutoMlUtils
 {
 public static AutoInference.RunSummary ExtractRunSummary(IHostEnvironment env, IDataView data, string metricColumnName)
 public static double ExtractValueFromIDV(IHostEnvironment env, IDataView result, string columnName)
 {
 double metricValue = 0;
 int numRows = 0;
 var schema = data.Schema;
 schema.TryGetColumnIndex(metricColumnName, out var metricCol);
 Contracts.CheckValue(env, nameof(env));
 env.CheckValue(result, nameof(result));
 env.CheckNonEmpty(columnName, nameof(columnName));

 using (var cursor = data.GetRowCursor(col => col == metricCol))
 double outputValue = 0;
 var schema = result.Schema;
 if (!schema.TryGetColumnIndex(columnName, out var metricCol))
 throw env.ExceptParam(nameof(columnName), $"Schema does not contain column: {columnName}");

 using (var cursor = result.GetRowCursor(col => col == metricCol))
 {
 var getter = cursor.GetGetter<double>(metricCol);
 cursor.MoveNext();
 getter(ref metricValue);
 bool moved = cursor.MoveNext();
 env.Check(moved, "Expected an IDataView with a single row. Results dataset has no rows to extract.");
 getter(ref outputValue);
 env.Check(!cursor.MoveNext(), "Expected an IDataView with a single row. Results dataset has too many rows.");
 }

 return new AutoInference.RunSummary(metricValue, numRows, 0);
 return outputValue;
 }

 public static AutoInference.RunSummary ExtractRunSummary(IHostEnvironment env, IDataView result, string metricColumnName, IDataView trainResult = null)
 {
 double testingMetricValue = ExtractValueFromIDV(env, result, metricColumnName);
 double trainingMetricValue = trainResult != null ? ExtractValueFromIDV(env, trainResult, metricColumnName) : double.MinValue;
 return new AutoInference.RunSummary(testingMetricValue, 0, 0, trainingMetricValue);
 }

 public static CommonInputs.IEvaluatorInput CloneEvaluatorInstance(CommonInputs.IEvaluatorInput evalInput) =>
Expand DownExpand Up@@ -618,5 +631,7 @@ public static Tuple<string, string[]>[] ConvertToSweepArgumentStrings(TlcModule.
 }
 return results;
 }

 public static string GenerateOverallTrainingMetricVarName(Guid id) => $"Var_Training_OM_{id:N}";
 }
}

19 changes: 14 additions & 5 deletions
src/Microsoft.ML.PipelineInference/Macros/PipelineSweeperMacro.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -65,18 +65,24 @@ public static Output ExtractSweepResult(IHostEnvironment env, ResultInput input)
 var col1 = new KeyValuePair<string, ColumnType>("Graph", TextType.Instance);
 var col2 = new KeyValuePair<string, ColumnType>("MetricValue", PrimitiveType.FromKind(DataKind.R8));
 var col3 = new KeyValuePair<string, ColumnType>("PipelineId", TextType.Instance);
 var col4 = new KeyValuePair<string, ColumnType>("TrainingMetricValue", PrimitiveType.FromKind(DataKind.R8));
 var col5 = new KeyValuePair<string, ColumnType>("FirstInput", TextType.Instance);
 var col6 = new KeyValuePair<string, ColumnType>("PredictorModel", TextType.Instance);

 if (rows.Count == 0)
 {
 var host = env.Register("ExtractSweepResult");
 outputView = new EmptyDataView(host, new SimpleSchema(host, col1, col2, col3));
 outputView = new EmptyDataView(host, new SimpleSchema(host, col1, col2, col3, col4, col5, col6));
 }
 else
 {
 var builder = new ArrayDataViewBuilder(env);
 builder.AddColumn(col1.Key, (PrimitiveType)col1.Value, rows.Select(r => new DvText(r.GraphJson)).ToArray());
 builder.AddColumn(col2.Key, (PrimitiveType)col2.Value, rows.Select(r => r.MetricValue).ToArray());
 builder.AddColumn(col3.Key, (PrimitiveType)col3.Value, rows.Select(r => new DvText(r.PipelineId)).ToArray());
 builder.AddColumn(col4.Key, (PrimitiveType)col4.Value, rows.Select(r => r.TrainingMetricValue).ToArray());
 builder.AddColumn(col5.Key, (PrimitiveType)col5.Value, rows.Select(r => new DvText(r.FirstInput)).ToArray());
 builder.AddColumn(col6.Key, (PrimitiveType)col6.Value, rows.Select(r => new DvText(r.PredictorModel)).ToArray());
 outputView = builder.GetDataView();
 }
 return new Output { Results = outputView, State = autoMlState };
Expand DownExpand Up@@ -132,11 +138,11 @@ public static CommonOutputs.MacroOutput<Output> PipelineSweep(
 // Extract performance summaries and assign to previous candidate pipelines.
 foreach (var pipeline in autoMlState.BatchCandidates)
 {
 if (node.Context.TryGetVariable(ExperimentUtils.GenerateOverallMetricVarName(pipeline.UniqueId),
 out var v))
 if (node.Context.TryGetVariable(ExperimentUtils.GenerateOverallMetricVarName(pipeline.UniqueId), out var v) &&
node.Context.TryGetVariable(AutoMlUtils.GenerateOverallTrainingMetricVarName(pipeline.UniqueId), out var v2))
 {
 pipeline.PerformanceSummary =
 AutoMlUtils.ExtractRunSummary(env, (IDataView)v.Value, autoMlState.Metric.Name);
 AutoMlUtils.ExtractRunSummary(env, (IDataView)v.Value, autoMlState.Metric.Name, (IDataView)v2.Value);
 autoMlState.AddEvaluated(pipeline);
 }
 }
Expand DownExpand Up@@ -168,14 +174,17 @@ public static CommonOutputs.MacroOutput<Output> PipelineSweep(
 {
 // Add train test experiments to current graph for candidate pipeline
 var subgraph = new Experiment(env);
 var trainTestOutput = p.AddAsTrainTest(training, testing, autoMlState.TrainerKind, subgraph);
 var trainTestOutput = p.AddAsTrainTest(training, testing, autoMlState.TrainerKind, subgraph, true);

 // Change variable name to reference pipeline ID in output map, context and entrypoint output.
 var uniqueName = ExperimentUtils.GenerateOverallMetricVarName(p.UniqueId);
 var uniqueNameTraining = AutoMlUtils.GenerateOverallTrainingMetricVarName(p.UniqueId);
 var sgNode = EntryPointNode.ValidateNodes(env, node.Context,
 new JArray(subgraph.GetNodes().Last()), node.Catalog).Last();
 sgNode.RenameOutputVariable(trainTestOutput.OverallMetrics.VarName, uniqueName, cascadeChanges: true);
 sgNode.RenameOutputVariable(trainTestOutput.TrainingOverallMetrics.VarName, uniqueNameTraining, cascadeChanges: true);
 trainTestOutput.OverallMetrics.VarName = uniqueName;
 trainTestOutput.TrainingOverallMetrics.VarName = uniqueNameTraining;
 expNodes.Add(sgNode);

 // Store indicators, to pass to next iteration of macro.
Expand Down

1 change: 1 addition & 0 deletions
src/Microsoft.ML.PipelineInference/Microsoft.ML.PipelineInference.csproj

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -17,6 +17,7 @@
 <ProjectReference Include="..\Microsoft.ML.Core\Microsoft.ML.Core.csproj" />
 <ProjectReference Include="..\Microsoft.ML.StandardLearners\Microsoft.ML.StandardLearners.csproj" />
 <ProjectReference Include="..\Microsoft.ML.Sweeper\Microsoft.ML.Sweeper.csproj" />
 <ProjectReference Include="..\Microsoft.ML\Microsoft.ML.csproj" />
 </ItemGroup>

</Project>
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Force GitHub README to respect dark mode\n(function() {\n var style = document.createElement('style');\n style.textContent = '\n .markdown-body {\n color-scheme: dark light;\n }\n .markdown-body pre { background: #161b22 !important; }\n .markdown-body code { background: rgba(110, 118, 129, 0.4) !important; }\n .markdown-body table th, .markdown-body table td { border-color: #30363d !important; }\n .markdown-body img { background: #0d1117; }\n .markdown-body blockquote { border-left-color: #8b949e; }\n .markdown-body hr { border-color: #30363d; }\n ';\n document.head.appendChild(style);\n})();", "GitHub Dark Mode README Fix");
}
} catch(__e) { console.warn('[Userscript:GitHub Dark Mode README Fix]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
Adding support for training metrics in PipelineSweeperMacro + new graph variable outputs#152

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

george-microsoft
merged 15 commits into

from

May 23, 2018
 Merged


Adding support for training metrics in PipelineSweeperMacro + new graph variable outputs
#152
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


15 commits


Select commit
Hold shift + click to select a range
05482eb

Adding support for training metrics in PipelineSweeperMacro and neede…


george-microsoft May 14, 2018
b789159

Changes for revieer comments.


george-microsoft May 14, 2018
76dd270

Changes to address reviewer comments.


george-microsoft May 14, 2018
483a50b

More changes for reviewer commenrts.


george-microsoft May 17, 2018
d42a522

Changes for reviewer comments.


george-microsoft May 18, 2018
fbee51c

Changes for latest round of reviewer comments.


george-microsoft May 21, 2018
0285c87

Changed where XML comment was placed.


george-microsoft May 21, 2018
6f92693

Added more tests (uncommented and fixed) for auto inference. Changed …


george-microsoft May 22, 2018
e006ee1

Added another test, TestPipelineSweeperMacroNoTransforms.


george-microsoft May 22, 2018
90d96a3

Updated tests to include warning disabling (following Zeeshan S's exa…


george-microsoft May 22, 2018
1d1c303

Merge branch 'master' into AutoML-GraphChanges


george-microsoft May 22, 2018
ce9792f

Changes to checks in AutoMlUtils (more correct usage of them).


george-microsoft May 22, 2018
a828266

Merge branch 'AutoML-GraphChanges' of https://github.com/george-micro…


george-microsoft May 22, 2018
c642f68

Fixing issue with ExceptParam using value and not name of parameters.


george-microsoft May 22, 2018
e066db0

Fixing errors on use of ExceptParam.


george-microsoft May 22, 2018

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


14 changes: 7 additions & 7 deletions
src/Microsoft.ML.Data/EntryPoints/InputBuilder.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -832,21 +832,21 @@ public static class SweepableDiscreteParam
 public static class PipelineSweeperSupportedMetrics
 {
 public new static string ToString() => "SupportedMetric";
 public const string Auc = "Auc";
 public const string Auc = "AUC";
 public const string AccuracyMicro = "AccuracyMicro";
 public const string AccuracyMacro = "AccuracyMacro";
 public const string F1 = "F1";
 public const string AuPrc = "AuPrc";
 public const string AuPrc = "AUPRC";
 public const string TopKAccuracy = "TopKAccuracy";
 public const string L1 = "L1";
 public const string L2 = "L2";
 public const string Rms = "Rms";
 public const string Rms = "RMS";
 public const string LossFn = "LossFn";
 public const string RSquared = "RSquared";
 public const string LogLoss = "LogLoss";
 public const string LogLossReduction = "LogLossReduction";
 public const string Ndcg = "Ndcg";
 public const string Dcg = "Dcg";
 public const string Ndcg = "NDCG";
 public const string Dcg = "DCG";
 public const string PositivePrecision = "PositivePrecision";
 public const string PositiveRecall = "PositiveRecall";
 public const string NegativePrecision = "NegativePrecision";
Expand All@@ -858,9 +858,9 @@ public static class PipelineSweeperSupportedMetrics
 public const string ThreshAtK = "ThreshAtK";
 public const string ThreshAtP = "ThreshAtP";
 public const string ThreshAtNumPos = "ThreshAtNumPos";
 public const string Nmi = "Nmi";
 public const string Nmi = "NMI";
 public const string AvgMinScore = "AvgMinScore";
 public const string Dbi = "Dbi";
 public const string Dbi = "DBI";
 }
 }
}

20 changes: 12 additions & 8 deletions
src/Microsoft.ML.PipelineInference/AutoInference.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -158,7 +158,8 @@ private bool GetDataVariableName(IExceptionContext ectx, string nameOfData, JTok
 return false;

 string dataVar = firstNodeInputs.Value<String>(nameOfData);
 ectx.Check(VariableBinding.IsValidVariableName(ectx, dataVar), $"Invalid variable name {dataVar}.");
 if (!VariableBinding.IsValidVariableName(ectx, dataVar))
 throw ectx.ExceptParam(nameof(nameOfData), $"Invalid variable name {dataVar}.");

 variableName = dataVar.Substring(1);
 return true;
Expand All@@ -172,12 +173,14 @@ private bool GetDataVariableName(IExceptionContext ectx, string nameOfData, JTok
 public sealed class RunSummary
 {
 public double MetricValue { get; }
 public double TrainingMetricValue { get; }
 public int NumRowsInTraining { get; }
 public long RunTimeMilliseconds { get; }

 public RunSummary(double metricValue, int numRows, long runTimeMilliseconds)
 public RunSummary(double metricValue, int numRows, long runTimeMilliseconds, double trainingMetricValue)
 {
 MetricValue = metricValue;
 TrainingMetricValue = trainingMetricValue;
 NumRowsInTraining = numRows;
 RunTimeMilliseconds = runTimeMilliseconds;
 }
Expand DownExpand Up@@ -303,7 +306,7 @@ private void MainLearningLoop(int batchSize, int numOfTrainingRows)
 var stopwatch = new Stopwatch();
 var probabilityUtils = new Sweeper.Algorithms.SweeperProbabilityUtils(_host);

while (!_terminator.ShouldTerminate(_history))
 while (!_terminator.ShouldTerminate(_history))
 {
 // Get next set of candidates
 var currentBatchSize = batchSize;
Expand DownExpand Up@@ -341,16 +344,17 @@ private void ProcessPipeline(Sweeper.Algorithms.SweeperProbabilityUtils utils, S

 // Run pipeline, and time how long it takes
 stopwatch.Restart();
double d = candidate.RunTrainTestExperiment(_trainData.Take(randomizedNumberOfRows),
 _testData, Metric, TrainerKind);
 candidate.RunTrainTestExperiment(_trainData.Take(randomizedNumberOfRows),
 _testData, Metric, TrainerKind, out var testMetricVal, out var trainMetricVal);
 stopwatch.Stop();

 // Handle key collisions on sorted list
 while (_sortedSampledElements.ContainsKey(d))
d += 1e-10;
 while (_sortedSampledElements.ContainsKey(testMetricVal))
testMetricVal += 1e-10;

 // Save performance score
 candidate.PerformanceSummary = new RunSummary(d, randomizedNumberOfRows, stopwatch.ElapsedMilliseconds);
 candidate.PerformanceSummary = 
 new RunSummary(testMetricVal, randomizedNumberOfRows, stopwatch.ElapsedMilliseconds, trainMetricVal);
 _sortedSampledElements.Add(candidate.PerformanceSummary.MetricValue, candidate);
 _history.Add(candidate);
 }
Expand Down

33 changes: 24 additions & 9 deletions
src/Microsoft.ML.PipelineInference/AutoMlUtils.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -15,21 +15,34 @@ namespace Microsoft.ML.Runtime.PipelineInference
{
 public static class AutoMlUtils
 {
 public static AutoInference.RunSummary ExtractRunSummary(IHostEnvironment env, IDataView data, string metricColumnName)
 public static double ExtractValueFromIDV(IHostEnvironment env, IDataView result, string columnName)
 {
 double metricValue = 0;
 int numRows = 0;
 var schema = data.Schema;
 schema.TryGetColumnIndex(metricColumnName, out var metricCol);
 Contracts.CheckValue(env, nameof(env));
 env.CheckValue(result, nameof(result));
 env.CheckNonEmpty(columnName, nameof(columnName));

 using (var cursor = data.GetRowCursor(col => col == metricCol))
 double outputValue = 0;
 var schema = result.Schema;
 if (!schema.TryGetColumnIndex(columnName, out var metricCol))
 throw env.ExceptParam(nameof(columnName), $"Schema does not contain column: {columnName}");

 using (var cursor = result.GetRowCursor(col => col == metricCol))
 {
 var getter = cursor.GetGetter<double>(metricCol);
 cursor.MoveNext();
 getter(ref metricValue);
 bool moved = cursor.MoveNext();
 env.Check(moved, "Expected an IDataView with a single row. Results dataset has no rows to extract.");
 getter(ref outputValue);
 env.Check(!cursor.MoveNext(), "Expected an IDataView with a single row. Results dataset has too many rows.");
 }

 return new AutoInference.RunSummary(metricValue, numRows, 0);
 return outputValue;
 }

 public static AutoInference.RunSummary ExtractRunSummary(IHostEnvironment env, IDataView result, string metricColumnName, IDataView trainResult = null)
 {
 double testingMetricValue = ExtractValueFromIDV(env, result, metricColumnName);
 double trainingMetricValue = trainResult != null ? ExtractValueFromIDV(env, trainResult, metricColumnName) : double.MinValue;
 return new AutoInference.RunSummary(testingMetricValue, 0, 0, trainingMetricValue);
 }

 public static CommonInputs.IEvaluatorInput CloneEvaluatorInstance(CommonInputs.IEvaluatorInput evalInput) =>
Expand DownExpand Up@@ -618,5 +631,7 @@ public static Tuple<string, string[]>[] ConvertToSweepArgumentStrings(TlcModule.
 }
 return results;
 }

 public static string GenerateOverallTrainingMetricVarName(Guid id) => $"Var_Training_OM_{id:N}";
 }
}

19 changes: 14 additions & 5 deletions
src/Microsoft.ML.PipelineInference/Macros/PipelineSweeperMacro.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -65,18 +65,24 @@ public static Output ExtractSweepResult(IHostEnvironment env, ResultInput input)
 var col1 = new KeyValuePair<string, ColumnType>("Graph", TextType.Instance);
 var col2 = new KeyValuePair<string, ColumnType>("MetricValue", PrimitiveType.FromKind(DataKind.R8));
 var col3 = new KeyValuePair<string, ColumnType>("PipelineId", TextType.Instance);
 var col4 = new KeyValuePair<string, ColumnType>("TrainingMetricValue", PrimitiveType.FromKind(DataKind.R8));
 var col5 = new KeyValuePair<string, ColumnType>("FirstInput", TextType.Instance);
 var col6 = new KeyValuePair<string, ColumnType>("PredictorModel", TextType.Instance);

 if (rows.Count == 0)
 {
 var host = env.Register("ExtractSweepResult");
 outputView = new EmptyDataView(host, new SimpleSchema(host, col1, col2, col3));
 outputView = new EmptyDataView(host, new SimpleSchema(host, col1, col2, col3, col4, col5, col6));
 }
 else
 {
 var builder = new ArrayDataViewBuilder(env);
 builder.AddColumn(col1.Key, (PrimitiveType)col1.Value, rows.Select(r => new DvText(r.GraphJson)).ToArray());
 builder.AddColumn(col2.Key, (PrimitiveType)col2.Value, rows.Select(r => r.MetricValue).ToArray());
 builder.AddColumn(col3.Key, (PrimitiveType)col3.Value, rows.Select(r => new DvText(r.PipelineId)).ToArray());
 builder.AddColumn(col4.Key, (PrimitiveType)col4.Value, rows.Select(r => r.TrainingMetricValue).ToArray());
 builder.AddColumn(col5.Key, (PrimitiveType)col5.Value, rows.Select(r => new DvText(r.FirstInput)).ToArray());
 builder.AddColumn(col6.Key, (PrimitiveType)col6.Value, rows.Select(r => new DvText(r.PredictorModel)).ToArray());
 outputView = builder.GetDataView();
 }
 return new Output { Results = outputView, State = autoMlState };
Expand DownExpand Up@@ -132,11 +138,11 @@ public static CommonOutputs.MacroOutput<Output> PipelineSweep(
 // Extract performance summaries and assign to previous candidate pipelines.
 foreach (var pipeline in autoMlState.BatchCandidates)
 {
 if (node.Context.TryGetVariable(ExperimentUtils.GenerateOverallMetricVarName(pipeline.UniqueId),
 out var v))
 if (node.Context.TryGetVariable(ExperimentUtils.GenerateOverallMetricVarName(pipeline.UniqueId), out var v) &&
node.Context.TryGetVariable(AutoMlUtils.GenerateOverallTrainingMetricVarName(pipeline.UniqueId), out var v2))
 {
 pipeline.PerformanceSummary =
 AutoMlUtils.ExtractRunSummary(env, (IDataView)v.Value, autoMlState.Metric.Name);
 AutoMlUtils.ExtractRunSummary(env, (IDataView)v.Value, autoMlState.Metric.Name, (IDataView)v2.Value);
 autoMlState.AddEvaluated(pipeline);
 }
 }
Expand DownExpand Up@@ -168,14 +174,17 @@ public static CommonOutputs.MacroOutput<Output> PipelineSweep(
 {
 // Add train test experiments to current graph for candidate pipeline
 var subgraph = new Experiment(env);
 var trainTestOutput = p.AddAsTrainTest(training, testing, autoMlState.TrainerKind, subgraph);
 var trainTestOutput = p.AddAsTrainTest(training, testing, autoMlState.TrainerKind, subgraph, true);

 // Change variable name to reference pipeline ID in output map, context and entrypoint output.
 var uniqueName = ExperimentUtils.GenerateOverallMetricVarName(p.UniqueId);
 var uniqueNameTraining = AutoMlUtils.GenerateOverallTrainingMetricVarName(p.UniqueId);
 var sgNode = EntryPointNode.ValidateNodes(env, node.Context,
 new JArray(subgraph.GetNodes().Last()), node.Catalog).Last();
 sgNode.RenameOutputVariable(trainTestOutput.OverallMetrics.VarName, uniqueName, cascadeChanges: true);
 sgNode.RenameOutputVariable(trainTestOutput.TrainingOverallMetrics.VarName, uniqueNameTraining, cascadeChanges: true);
 trainTestOutput.OverallMetrics.VarName = uniqueName;
 trainTestOutput.TrainingOverallMetrics.VarName = uniqueNameTraining;
 expNodes.Add(sgNode);

 // Store indicators, to pass to next iteration of macro.
Expand Down

1 change: 1 addition & 0 deletions
src/Microsoft.ML.PipelineInference/Microsoft.ML.PipelineInference.csproj

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -17,6 +17,7 @@
 <ProjectReference Include="..\Microsoft.ML.Core\Microsoft.ML.Core.csproj" />
 <ProjectReference Include="..\Microsoft.ML.StandardLearners\Microsoft.ML.StandardLearners.csproj" />
 <ProjectReference Include="..\Microsoft.ML.Sweeper\Microsoft.ML.Sweeper.csproj" />
 <ProjectReference Include="..\Microsoft.ML\Microsoft.ML.csproj" />
 </ItemGroup>

</Project>
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Highlight search terms from Google/DuckDuckGo/Bing referrer\n(function() {\n var ref = document.referrer;\n var terms = [];\n \n if (ref.includes('google.com') || ref.includes('duckduckgo.com') || ref.includes('bing.com')) {\n var url = new URL(ref);\n var q = url.searchParams.get('q') || url.searchParams.get('p');\n if (q) {\n terms = q.split(/\\s+/).filter(function(t) { return t.length > 2; });\n }\n }\n \n if (terms.length === 0) return;\n \n var style = document.createElement('style');\n style.textContent = '.userscript-highlight { background: #fbbf24; color: #1a1a2e; padding: 1px 3px; border-radius: 2px; }';\n document.head.appendChild(style);\n \n function highlight(node) {\n if (node.nodeType === 3) { // text node\n var text = node.textContent;\n var found = false;\n terms.forEach(function(term) {\n var regex = new RegExp('(' + term.replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\') + ')', 'gi');\n if (regex.test(text)) {\n found = true;\n var frag = document.createDocumentFragment();\n var parts = text.split(regex);\n parts.forEach(function(part, i) {\n if (i % 2 === 0) {\n frag.appendChild(document.createTextNode(part));\n } else {\n var span = document.createElement('span');\n span.className = 'userscript-highlight';\n span.textContent = part;\n frag.appendChild(span);\n }\n });\n node.parentNode.replaceChild(frag, node);\n }\n });\n } else if (node.nodeType === 1 && node.childNodes) { // element\n var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT'];\n if (!skipTags.includes(node.tagName)) {\n Array.from(node.childNodes).forEach(highlight);\n }\n }\n }\n \n highlight(document.body);\n \n // Re-highlight on dynamic content\n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1 || node.nodeType === 3) highlight(node);\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Highlight Search Terms");
}
} catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
Adding support for training metrics in PipelineSweeperMacro + new graph variable outputs#152

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

george-microsoft
merged 15 commits into

from

May 23, 2018
 Merged


Adding support for training metrics in PipelineSweeperMacro + new graph variable outputs
#152
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


15 commits


Select commit
Hold shift + click to select a range
05482eb

Adding support for training metrics in PipelineSweeperMacro and neede…


george-microsoft May 14, 2018
b789159

Changes for revieer comments.


george-microsoft May 14, 2018
76dd270

Changes to address reviewer comments.


george-microsoft May 14, 2018
483a50b

More changes for reviewer commenrts.


george-microsoft May 17, 2018
d42a522

Changes for reviewer comments.


george-microsoft May 18, 2018
fbee51c

Changes for latest round of reviewer comments.


george-microsoft May 21, 2018
0285c87

Changed where XML comment was placed.


george-microsoft May 21, 2018
6f92693

Added more tests (uncommented and fixed) for auto inference. Changed …


george-microsoft May 22, 2018
e006ee1

Added another test, TestPipelineSweeperMacroNoTransforms.


george-microsoft May 22, 2018
90d96a3

Updated tests to include warning disabling (following Zeeshan S's exa…


george-microsoft May 22, 2018
1d1c303

Merge branch 'master' into AutoML-GraphChanges


george-microsoft May 22, 2018
ce9792f

Changes to checks in AutoMlUtils (more correct usage of them).


george-microsoft May 22, 2018
a828266

Merge branch 'AutoML-GraphChanges' of https://github.com/george-micro…


george-microsoft May 22, 2018
c642f68

Fixing issue with ExceptParam using value and not name of parameters.


george-microsoft May 22, 2018
e066db0

Fixing errors on use of ExceptParam.


george-microsoft May 22, 2018

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


14 changes: 7 additions & 7 deletions
src/Microsoft.ML.Data/EntryPoints/InputBuilder.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -832,21 +832,21 @@ public static class SweepableDiscreteParam
 public static class PipelineSweeperSupportedMetrics
 {
 public new static string ToString() => "SupportedMetric";
 public const string Auc = "Auc";
 public const string Auc = "AUC";
 public const string AccuracyMicro = "AccuracyMicro";
 public const string AccuracyMacro = "AccuracyMacro";
 public const string F1 = "F1";
 public const string AuPrc = "AuPrc";
 public const string AuPrc = "AUPRC";
 public const string TopKAccuracy = "TopKAccuracy";
 public const string L1 = "L1";
 public const string L2 = "L2";
 public const string Rms = "Rms";
 public const string Rms = "RMS";
 public const string LossFn = "LossFn";
 public const string RSquared = "RSquared";
 public const string LogLoss = "LogLoss";
 public const string LogLossReduction = "LogLossReduction";
 public const string Ndcg = "Ndcg";
 public const string Dcg = "Dcg";
 public const string Ndcg = "NDCG";
 public const string Dcg = "DCG";
 public const string PositivePrecision = "PositivePrecision";
 public const string PositiveRecall = "PositiveRecall";
 public const string NegativePrecision = "NegativePrecision";
Expand All@@ -858,9 +858,9 @@ public static class PipelineSweeperSupportedMetrics
 public const string ThreshAtK = "ThreshAtK";
 public const string ThreshAtP = "ThreshAtP";
 public const string ThreshAtNumPos = "ThreshAtNumPos";
 public const string Nmi = "Nmi";
 public const string Nmi = "NMI";
 public const string AvgMinScore = "AvgMinScore";
 public const string Dbi = "Dbi";
 public const string Dbi = "DBI";
 }
 }
}

20 changes: 12 additions & 8 deletions
src/Microsoft.ML.PipelineInference/AutoInference.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -158,7 +158,8 @@ private bool GetDataVariableName(IExceptionContext ectx, string nameOfData, JTok
 return false;

 string dataVar = firstNodeInputs.Value<String>(nameOfData);
 ectx.Check(VariableBinding.IsValidVariableName(ectx, dataVar), $"Invalid variable name {dataVar}.");
 if (!VariableBinding.IsValidVariableName(ectx, dataVar))
 throw ectx.ExceptParam(nameof(nameOfData), $"Invalid variable name {dataVar}.");

 variableName = dataVar.Substring(1);
 return true;
Expand All@@ -172,12 +173,14 @@ private bool GetDataVariableName(IExceptionContext ectx, string nameOfData, JTok
 public sealed class RunSummary
 {
 public double MetricValue { get; }
 public double TrainingMetricValue { get; }
 public int NumRowsInTraining { get; }
 public long RunTimeMilliseconds { get; }

 public RunSummary(double metricValue, int numRows, long runTimeMilliseconds)
 public RunSummary(double metricValue, int numRows, long runTimeMilliseconds, double trainingMetricValue)
 {
 MetricValue = metricValue;
 TrainingMetricValue = trainingMetricValue;
 NumRowsInTraining = numRows;
 RunTimeMilliseconds = runTimeMilliseconds;
 }
Expand DownExpand Up@@ -303,7 +306,7 @@ private void MainLearningLoop(int batchSize, int numOfTrainingRows)
 var stopwatch = new Stopwatch();
 var probabilityUtils = new Sweeper.Algorithms.SweeperProbabilityUtils(_host);

while (!_terminator.ShouldTerminate(_history))
 while (!_terminator.ShouldTerminate(_history))
 {
 // Get next set of candidates
 var currentBatchSize = batchSize;
Expand DownExpand Up@@ -341,16 +344,17 @@ private void ProcessPipeline(Sweeper.Algorithms.SweeperProbabilityUtils utils, S

 // Run pipeline, and time how long it takes
 stopwatch.Restart();
double d = candidate.RunTrainTestExperiment(_trainData.Take(randomizedNumberOfRows),
 _testData, Metric, TrainerKind);
 candidate.RunTrainTestExperiment(_trainData.Take(randomizedNumberOfRows),
 _testData, Metric, TrainerKind, out var testMetricVal, out var trainMetricVal);
 stopwatch.Stop();

 // Handle key collisions on sorted list
 while (_sortedSampledElements.ContainsKey(d))
d += 1e-10;
 while (_sortedSampledElements.ContainsKey(testMetricVal))
testMetricVal += 1e-10;

 // Save performance score
 candidate.PerformanceSummary = new RunSummary(d, randomizedNumberOfRows, stopwatch.ElapsedMilliseconds);
 candidate.PerformanceSummary = 
 new RunSummary(testMetricVal, randomizedNumberOfRows, stopwatch.ElapsedMilliseconds, trainMetricVal);
 _sortedSampledElements.Add(candidate.PerformanceSummary.MetricValue, candidate);
 _history.Add(candidate);
 }
Expand Down

33 changes: 24 additions & 9 deletions
src/Microsoft.ML.PipelineInference/AutoMlUtils.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -15,21 +15,34 @@ namespace Microsoft.ML.Runtime.PipelineInference
{
 public static class AutoMlUtils
 {
 public static AutoInference.RunSummary ExtractRunSummary(IHostEnvironment env, IDataView data, string metricColumnName)
 public static double ExtractValueFromIDV(IHostEnvironment env, IDataView result, string columnName)
 {
 double metricValue = 0;
 int numRows = 0;
 var schema = data.Schema;
 schema.TryGetColumnIndex(metricColumnName, out var metricCol);
 Contracts.CheckValue(env, nameof(env));
 env.CheckValue(result, nameof(result));
 env.CheckNonEmpty(columnName, nameof(columnName));

 using (var cursor = data.GetRowCursor(col => col == metricCol))
 double outputValue = 0;
 var schema = result.Schema;
 if (!schema.TryGetColumnIndex(columnName, out var metricCol))
 throw env.ExceptParam(nameof(columnName), $"Schema does not contain column: {columnName}");

 using (var cursor = result.GetRowCursor(col => col == metricCol))
 {
 var getter = cursor.GetGetter<double>(metricCol);
 cursor.MoveNext();
 getter(ref metricValue);
 bool moved = cursor.MoveNext();
 env.Check(moved, "Expected an IDataView with a single row. Results dataset has no rows to extract.");
 getter(ref outputValue);
 env.Check(!cursor.MoveNext(), "Expected an IDataView with a single row. Results dataset has too many rows.");
 }

 return new AutoInference.RunSummary(metricValue, numRows, 0);
 return outputValue;
 }

 public static AutoInference.RunSummary ExtractRunSummary(IHostEnvironment env, IDataView result, string metricColumnName, IDataView trainResult = null)
 {
 double testingMetricValue = ExtractValueFromIDV(env, result, metricColumnName);
 double trainingMetricValue = trainResult != null ? ExtractValueFromIDV(env, trainResult, metricColumnName) : double.MinValue;
 return new AutoInference.RunSummary(testingMetricValue, 0, 0, trainingMetricValue);
 }

 public static CommonInputs.IEvaluatorInput CloneEvaluatorInstance(CommonInputs.IEvaluatorInput evalInput) =>
Expand DownExpand Up@@ -618,5 +631,7 @@ public static Tuple<string, string[]>[] ConvertToSweepArgumentStrings(TlcModule.
 }
 return results;
 }

 public static string GenerateOverallTrainingMetricVarName(Guid id) => $"Var_Training_OM_{id:N}";
 }
}

19 changes: 14 additions & 5 deletions
src/Microsoft.ML.PipelineInference/Macros/PipelineSweeperMacro.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -65,18 +65,24 @@ public static Output ExtractSweepResult(IHostEnvironment env, ResultInput input)
 var col1 = new KeyValuePair<string, ColumnType>("Graph", TextType.Instance);
 var col2 = new KeyValuePair<string, ColumnType>("MetricValue", PrimitiveType.FromKind(DataKind.R8));
 var col3 = new KeyValuePair<string, ColumnType>("PipelineId", TextType.Instance);
 var col4 = new KeyValuePair<string, ColumnType>("TrainingMetricValue", PrimitiveType.FromKind(DataKind.R8));
 var col5 = new KeyValuePair<string, ColumnType>("FirstInput", TextType.Instance);
 var col6 = new KeyValuePair<string, ColumnType>("PredictorModel", TextType.Instance);

 if (rows.Count == 0)
 {
 var host = env.Register("ExtractSweepResult");
 outputView = new EmptyDataView(host, new SimpleSchema(host, col1, col2, col3));
 outputView = new EmptyDataView(host, new SimpleSchema(host, col1, col2, col3, col4, col5, col6));
 }
 else
 {
 var builder = new ArrayDataViewBuilder(env);
 builder.AddColumn(col1.Key, (PrimitiveType)col1.Value, rows.Select(r => new DvText(r.GraphJson)).ToArray());
 builder.AddColumn(col2.Key, (PrimitiveType)col2.Value, rows.Select(r => r.MetricValue).ToArray());
 builder.AddColumn(col3.Key, (PrimitiveType)col3.Value, rows.Select(r => new DvText(r.PipelineId)).ToArray());
 builder.AddColumn(col4.Key, (PrimitiveType)col4.Value, rows.Select(r => r.TrainingMetricValue).ToArray());
 builder.AddColumn(col5.Key, (PrimitiveType)col5.Value, rows.Select(r => new DvText(r.FirstInput)).ToArray());
 builder.AddColumn(col6.Key, (PrimitiveType)col6.Value, rows.Select(r => new DvText(r.PredictorModel)).ToArray());
 outputView = builder.GetDataView();
 }
 return new Output { Results = outputView, State = autoMlState };
Expand DownExpand Up@@ -132,11 +138,11 @@ public static CommonOutputs.MacroOutput<Output> PipelineSweep(
 // Extract performance summaries and assign to previous candidate pipelines.
 foreach (var pipeline in autoMlState.BatchCandidates)
 {
 if (node.Context.TryGetVariable(ExperimentUtils.GenerateOverallMetricVarName(pipeline.UniqueId),
 out var v))
 if (node.Context.TryGetVariable(ExperimentUtils.GenerateOverallMetricVarName(pipeline.UniqueId), out var v) &&
node.Context.TryGetVariable(AutoMlUtils.GenerateOverallTrainingMetricVarName(pipeline.UniqueId), out var v2))
 {
 pipeline.PerformanceSummary =
 AutoMlUtils.ExtractRunSummary(env, (IDataView)v.Value, autoMlState.Metric.Name);
 AutoMlUtils.ExtractRunSummary(env, (IDataView)v.Value, autoMlState.Metric.Name, (IDataView)v2.Value);
 autoMlState.AddEvaluated(pipeline);
 }
 }
Expand DownExpand Up@@ -168,14 +174,17 @@ public static CommonOutputs.MacroOutput<Output> PipelineSweep(
 {
 // Add train test experiments to current graph for candidate pipeline
 var subgraph = new Experiment(env);
 var trainTestOutput = p.AddAsTrainTest(training, testing, autoMlState.TrainerKind, subgraph);
 var trainTestOutput = p.AddAsTrainTest(training, testing, autoMlState.TrainerKind, subgraph, true);

 // Change variable name to reference pipeline ID in output map, context and entrypoint output.
 var uniqueName = ExperimentUtils.GenerateOverallMetricVarName(p.UniqueId);
 var uniqueNameTraining = AutoMlUtils.GenerateOverallTrainingMetricVarName(p.UniqueId);
 var sgNode = EntryPointNode.ValidateNodes(env, node.Context,
 new JArray(subgraph.GetNodes().Last()), node.Catalog).Last();
 sgNode.RenameOutputVariable(trainTestOutput.OverallMetrics.VarName, uniqueName, cascadeChanges: true);
 sgNode.RenameOutputVariable(trainTestOutput.TrainingOverallMetrics.VarName, uniqueNameTraining, cascadeChanges: true);
 trainTestOutput.OverallMetrics.VarName = uniqueName;
 trainTestOutput.TrainingOverallMetrics.VarName = uniqueNameTraining;
 expNodes.Add(sgNode);

 // Store indicators, to pass to next iteration of macro.
Expand Down

1 change: 1 addition & 0 deletions
src/Microsoft.ML.PipelineInference/Microsoft.ML.PipelineInference.csproj

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -17,6 +17,7 @@
 <ProjectReference Include="..\Microsoft.ML.Core\Microsoft.ML.Core.csproj" />
 <ProjectReference Include="..\Microsoft.ML.StandardLearners\Microsoft.ML.StandardLearners.csproj" />
 <ProjectReference Include="..\Microsoft.ML.Sweeper\Microsoft.ML.Sweeper.csproj" />
 <ProjectReference Include="..\Microsoft.ML\Microsoft.ML.csproj" />
 </ItemGroup>

</Project>
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Strip utm_, fbclid, gclid, etc. from all links on page\n(function() {\n var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',\n 'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid',\n 'ref', 'ref_src', 'source', 'medium', 'campaign'];\n \n function cleanUrl(url) {\n try {\n var u = new URL(url, window.location.origin);\n var changed = false;\n trackingParams.forEach(function(p) {\n if (u.searchParams.has(p)) {\n u.searchParams.delete(p);\n changed = true;\n }\n });\n return changed ? u.toString() : url;\n } catch (e) {\n return url;\n }\n }\n \n function cleanLinks() {\n document.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n \n cleanLinks();\n \n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1) {\n if (node.tagName === 'A') cleanLinks();\n node.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Remove Tracking Parameters from Links");
}
} catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); }
})();
(function(){
try {
var __m = "youtube.com";
var __re = new RegExp('^' + "youtube\\.com" + '

Skip to content
Adding support for training metrics in PipelineSweeperMacro + new graph variable outputs#152

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

george-microsoft
merged 15 commits into

from

May 23, 2018
 Merged


Adding support for training metrics in PipelineSweeperMacro + new graph variable outputs
#152
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


15 commits


Select commit
Hold shift + click to select a range
05482eb

Adding support for training metrics in PipelineSweeperMacro and neede…


george-microsoft May 14, 2018
b789159

Changes for revieer comments.


george-microsoft May 14, 2018
76dd270

Changes to address reviewer comments.


george-microsoft May 14, 2018
483a50b

More changes for reviewer commenrts.


george-microsoft May 17, 2018
d42a522

Changes for reviewer comments.


george-microsoft May 18, 2018
fbee51c

Changes for latest round of reviewer comments.


george-microsoft May 21, 2018
0285c87

Changed where XML comment was placed.


george-microsoft May 21, 2018
6f92693

Added more tests (uncommented and fixed) for auto inference. Changed …


george-microsoft May 22, 2018
e006ee1

Added another test, TestPipelineSweeperMacroNoTransforms.


george-microsoft May 22, 2018
90d96a3

Updated tests to include warning disabling (following Zeeshan S's exa…


george-microsoft May 22, 2018
1d1c303

Merge branch 'master' into AutoML-GraphChanges


george-microsoft May 22, 2018
ce9792f

Changes to checks in AutoMlUtils (more correct usage of them).


george-microsoft May 22, 2018
a828266

Merge branch 'AutoML-GraphChanges' of https://github.com/george-micro…


george-microsoft May 22, 2018
c642f68

Fixing issue with ExceptParam using value and not name of parameters.


george-microsoft May 22, 2018
e066db0

Fixing errors on use of ExceptParam.


george-microsoft May 22, 2018

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


14 changes: 7 additions & 7 deletions
src/Microsoft.ML.Data/EntryPoints/InputBuilder.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -832,21 +832,21 @@ public static class SweepableDiscreteParam
 public static class PipelineSweeperSupportedMetrics
 {
 public new static string ToString() => "SupportedMetric";
 public const string Auc = "Auc";
 public const string Auc = "AUC";
 public const string AccuracyMicro = "AccuracyMicro";
 public const string AccuracyMacro = "AccuracyMacro";
 public const string F1 = "F1";
 public const string AuPrc = "AuPrc";
 public const string AuPrc = "AUPRC";
 public const string TopKAccuracy = "TopKAccuracy";
 public const string L1 = "L1";
 public const string L2 = "L2";
 public const string Rms = "Rms";
 public const string Rms = "RMS";
 public const string LossFn = "LossFn";
 public const string RSquared = "RSquared";
 public const string LogLoss = "LogLoss";
 public const string LogLossReduction = "LogLossReduction";
 public const string Ndcg = "Ndcg";
 public const string Dcg = "Dcg";
 public const string Ndcg = "NDCG";
 public const string Dcg = "DCG";
 public const string PositivePrecision = "PositivePrecision";
 public const string PositiveRecall = "PositiveRecall";
 public const string NegativePrecision = "NegativePrecision";
Expand All@@ -858,9 +858,9 @@ public static class PipelineSweeperSupportedMetrics
 public const string ThreshAtK = "ThreshAtK";
 public const string ThreshAtP = "ThreshAtP";
 public const string ThreshAtNumPos = "ThreshAtNumPos";
 public const string Nmi = "Nmi";
 public const string Nmi = "NMI";
 public const string AvgMinScore = "AvgMinScore";
 public const string Dbi = "Dbi";
 public const string Dbi = "DBI";
 }
 }
}

20 changes: 12 additions & 8 deletions
src/Microsoft.ML.PipelineInference/AutoInference.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -158,7 +158,8 @@ private bool GetDataVariableName(IExceptionContext ectx, string nameOfData, JTok
 return false;

 string dataVar = firstNodeInputs.Value<String>(nameOfData);
 ectx.Check(VariableBinding.IsValidVariableName(ectx, dataVar), $"Invalid variable name {dataVar}.");
 if (!VariableBinding.IsValidVariableName(ectx, dataVar))
 throw ectx.ExceptParam(nameof(nameOfData), $"Invalid variable name {dataVar}.");

 variableName = dataVar.Substring(1);
 return true;
Expand All@@ -172,12 +173,14 @@ private bool GetDataVariableName(IExceptionContext ectx, string nameOfData, JTok
 public sealed class RunSummary
 {
 public double MetricValue { get; }
 public double TrainingMetricValue { get; }
 public int NumRowsInTraining { get; }
 public long RunTimeMilliseconds { get; }

 public RunSummary(double metricValue, int numRows, long runTimeMilliseconds)
 public RunSummary(double metricValue, int numRows, long runTimeMilliseconds, double trainingMetricValue)
 {
 MetricValue = metricValue;
 TrainingMetricValue = trainingMetricValue;
 NumRowsInTraining = numRows;
 RunTimeMilliseconds = runTimeMilliseconds;
 }
Expand DownExpand Up@@ -303,7 +306,7 @@ private void MainLearningLoop(int batchSize, int numOfTrainingRows)
 var stopwatch = new Stopwatch();
 var probabilityUtils = new Sweeper.Algorithms.SweeperProbabilityUtils(_host);

while (!_terminator.ShouldTerminate(_history))
 while (!_terminator.ShouldTerminate(_history))
 {
 // Get next set of candidates
 var currentBatchSize = batchSize;
Expand DownExpand Up@@ -341,16 +344,17 @@ private void ProcessPipeline(Sweeper.Algorithms.SweeperProbabilityUtils utils, S

 // Run pipeline, and time how long it takes
 stopwatch.Restart();
double d = candidate.RunTrainTestExperiment(_trainData.Take(randomizedNumberOfRows),
 _testData, Metric, TrainerKind);
 candidate.RunTrainTestExperiment(_trainData.Take(randomizedNumberOfRows),
 _testData, Metric, TrainerKind, out var testMetricVal, out var trainMetricVal);
 stopwatch.Stop();

 // Handle key collisions on sorted list
 while (_sortedSampledElements.ContainsKey(d))
d += 1e-10;
 while (_sortedSampledElements.ContainsKey(testMetricVal))
testMetricVal += 1e-10;

 // Save performance score
 candidate.PerformanceSummary = new RunSummary(d, randomizedNumberOfRows, stopwatch.ElapsedMilliseconds);
 candidate.PerformanceSummary = 
 new RunSummary(testMetricVal, randomizedNumberOfRows, stopwatch.ElapsedMilliseconds, trainMetricVal);
 _sortedSampledElements.Add(candidate.PerformanceSummary.MetricValue, candidate);
 _history.Add(candidate);
 }
Expand Down

33 changes: 24 additions & 9 deletions
src/Microsoft.ML.PipelineInference/AutoMlUtils.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -15,21 +15,34 @@ namespace Microsoft.ML.Runtime.PipelineInference
{
 public static class AutoMlUtils
 {
 public static AutoInference.RunSummary ExtractRunSummary(IHostEnvironment env, IDataView data, string metricColumnName)
 public static double ExtractValueFromIDV(IHostEnvironment env, IDataView result, string columnName)
 {
 double metricValue = 0;
 int numRows = 0;
 var schema = data.Schema;
 schema.TryGetColumnIndex(metricColumnName, out var metricCol);
 Contracts.CheckValue(env, nameof(env));
 env.CheckValue(result, nameof(result));
 env.CheckNonEmpty(columnName, nameof(columnName));

 using (var cursor = data.GetRowCursor(col => col == metricCol))
 double outputValue = 0;
 var schema = result.Schema;
 if (!schema.TryGetColumnIndex(columnName, out var metricCol))
 throw env.ExceptParam(nameof(columnName), $"Schema does not contain column: {columnName}");

 using (var cursor = result.GetRowCursor(col => col == metricCol))
 {
 var getter = cursor.GetGetter<double>(metricCol);
 cursor.MoveNext();
 getter(ref metricValue);
 bool moved = cursor.MoveNext();
 env.Check(moved, "Expected an IDataView with a single row. Results dataset has no rows to extract.");
 getter(ref outputValue);
 env.Check(!cursor.MoveNext(), "Expected an IDataView with a single row. Results dataset has too many rows.");
 }

 return new AutoInference.RunSummary(metricValue, numRows, 0);
 return outputValue;
 }

 public static AutoInference.RunSummary ExtractRunSummary(IHostEnvironment env, IDataView result, string metricColumnName, IDataView trainResult = null)
 {
 double testingMetricValue = ExtractValueFromIDV(env, result, metricColumnName);
 double trainingMetricValue = trainResult != null ? ExtractValueFromIDV(env, trainResult, metricColumnName) : double.MinValue;
 return new AutoInference.RunSummary(testingMetricValue, 0, 0, trainingMetricValue);
 }

 public static CommonInputs.IEvaluatorInput CloneEvaluatorInstance(CommonInputs.IEvaluatorInput evalInput) =>
Expand DownExpand Up@@ -618,5 +631,7 @@ public static Tuple<string, string[]>[] ConvertToSweepArgumentStrings(TlcModule.
 }
 return results;
 }

 public static string GenerateOverallTrainingMetricVarName(Guid id) => $"Var_Training_OM_{id:N}";
 }
}

19 changes: 14 additions & 5 deletions
src/Microsoft.ML.PipelineInference/Macros/PipelineSweeperMacro.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -65,18 +65,24 @@ public static Output ExtractSweepResult(IHostEnvironment env, ResultInput input)
 var col1 = new KeyValuePair<string, ColumnType>("Graph", TextType.Instance);
 var col2 = new KeyValuePair<string, ColumnType>("MetricValue", PrimitiveType.FromKind(DataKind.R8));
 var col3 = new KeyValuePair<string, ColumnType>("PipelineId", TextType.Instance);
 var col4 = new KeyValuePair<string, ColumnType>("TrainingMetricValue", PrimitiveType.FromKind(DataKind.R8));
 var col5 = new KeyValuePair<string, ColumnType>("FirstInput", TextType.Instance);
 var col6 = new KeyValuePair<string, ColumnType>("PredictorModel", TextType.Instance);

 if (rows.Count == 0)
 {
 var host = env.Register("ExtractSweepResult");
 outputView = new EmptyDataView(host, new SimpleSchema(host, col1, col2, col3));
 outputView = new EmptyDataView(host, new SimpleSchema(host, col1, col2, col3, col4, col5, col6));
 }
 else
 {
 var builder = new ArrayDataViewBuilder(env);
 builder.AddColumn(col1.Key, (PrimitiveType)col1.Value, rows.Select(r => new DvText(r.GraphJson)).ToArray());
 builder.AddColumn(col2.Key, (PrimitiveType)col2.Value, rows.Select(r => r.MetricValue).ToArray());
 builder.AddColumn(col3.Key, (PrimitiveType)col3.Value, rows.Select(r => new DvText(r.PipelineId)).ToArray());
 builder.AddColumn(col4.Key, (PrimitiveType)col4.Value, rows.Select(r => r.TrainingMetricValue).ToArray());
 builder.AddColumn(col5.Key, (PrimitiveType)col5.Value, rows.Select(r => new DvText(r.FirstInput)).ToArray());
 builder.AddColumn(col6.Key, (PrimitiveType)col6.Value, rows.Select(r => new DvText(r.PredictorModel)).ToArray());
 outputView = builder.GetDataView();
 }
 return new Output { Results = outputView, State = autoMlState };
Expand DownExpand Up@@ -132,11 +138,11 @@ public static CommonOutputs.MacroOutput<Output> PipelineSweep(
 // Extract performance summaries and assign to previous candidate pipelines.
 foreach (var pipeline in autoMlState.BatchCandidates)
 {
 if (node.Context.TryGetVariable(ExperimentUtils.GenerateOverallMetricVarName(pipeline.UniqueId),
 out var v))
 if (node.Context.TryGetVariable(ExperimentUtils.GenerateOverallMetricVarName(pipeline.UniqueId), out var v) &&
node.Context.TryGetVariable(AutoMlUtils.GenerateOverallTrainingMetricVarName(pipeline.UniqueId), out var v2))
 {
 pipeline.PerformanceSummary =
 AutoMlUtils.ExtractRunSummary(env, (IDataView)v.Value, autoMlState.Metric.Name);
 AutoMlUtils.ExtractRunSummary(env, (IDataView)v.Value, autoMlState.Metric.Name, (IDataView)v2.Value);
 autoMlState.AddEvaluated(pipeline);
 }
 }
Expand DownExpand Up@@ -168,14 +174,17 @@ public static CommonOutputs.MacroOutput<Output> PipelineSweep(
 {
 // Add train test experiments to current graph for candidate pipeline
 var subgraph = new Experiment(env);
 var trainTestOutput = p.AddAsTrainTest(training, testing, autoMlState.TrainerKind, subgraph);
 var trainTestOutput = p.AddAsTrainTest(training, testing, autoMlState.TrainerKind, subgraph, true);

 // Change variable name to reference pipeline ID in output map, context and entrypoint output.
 var uniqueName = ExperimentUtils.GenerateOverallMetricVarName(p.UniqueId);
 var uniqueNameTraining = AutoMlUtils.GenerateOverallTrainingMetricVarName(p.UniqueId);
 var sgNode = EntryPointNode.ValidateNodes(env, node.Context,
 new JArray(subgraph.GetNodes().Last()), node.Catalog).Last();
 sgNode.RenameOutputVariable(trainTestOutput.OverallMetrics.VarName, uniqueName, cascadeChanges: true);
 sgNode.RenameOutputVariable(trainTestOutput.TrainingOverallMetrics.VarName, uniqueNameTraining, cascadeChanges: true);
 trainTestOutput.OverallMetrics.VarName = uniqueName;
 trainTestOutput.TrainingOverallMetrics.VarName = uniqueNameTraining;
 expNodes.Add(sgNode);

 // Store indicators, to pass to next iteration of macro.
Expand Down

1 change: 1 addition & 0 deletions
src/Microsoft.ML.PipelineInference/Microsoft.ML.PipelineInference.csproj

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -17,6 +17,7 @@
 <ProjectReference Include="..\Microsoft.ML.Core\Microsoft.ML.Core.csproj" />
 <ProjectReference Include="..\Microsoft.ML.StandardLearners\Microsoft.ML.StandardLearners.csproj" />
 <ProjectReference Include="..\Microsoft.ML.Sweeper\Microsoft.ML.Sweeper.csproj" />
 <ProjectReference Include="..\Microsoft.ML\Microsoft.ML.csproj" />
 </ItemGroup>

</Project>
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Auto-enable theater mode on YouTube\n(function() {\n function tryTheater() {\n var btn = document.querySelector('button[aria-label=\"Theater mode\"], ytd-player #player button[title=\"Theater mode\"]');\n if (btn && !btn.classList.contains('activated')) {\n btn.click();\n }\n }\n \n // Try immediately\n tryTheater();\n \n // Try after navigation (SPA)\n var lastUrl = location.href;\n setInterval(function() {\n if (location.href !== lastUrl) {\n lastUrl = location.href;\n setTimeout(tryTheater, 500);\n }\n }, 1000);\n \n // Also try on player load\n var observer = new MutationObserver(tryTheater);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "YouTube Theater Mode Default");
}
} catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
Adding support for training metrics in PipelineSweeperMacro + new graph variable outputs#152

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

george-microsoft
merged 15 commits into

from

May 23, 2018
 Merged


Adding support for training metrics in PipelineSweeperMacro + new graph variable outputs
#152
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


15 commits


Select commit
Hold shift + click to select a range
05482eb

Adding support for training metrics in PipelineSweeperMacro and neede…


george-microsoft May 14, 2018
b789159

Changes for revieer comments.


george-microsoft May 14, 2018
76dd270

Changes to address reviewer comments.


george-microsoft May 14, 2018
483a50b

More changes for reviewer commenrts.


george-microsoft May 17, 2018
d42a522

Changes for reviewer comments.


george-microsoft May 18, 2018
fbee51c

Changes for latest round of reviewer comments.


george-microsoft May 21, 2018
0285c87

Changed where XML comment was placed.


george-microsoft May 21, 2018
6f92693

Added more tests (uncommented and fixed) for auto inference. Changed …


george-microsoft May 22, 2018
e006ee1

Added another test, TestPipelineSweeperMacroNoTransforms.


george-microsoft May 22, 2018
90d96a3

Updated tests to include warning disabling (following Zeeshan S's exa…


george-microsoft May 22, 2018
1d1c303

Merge branch 'master' into AutoML-GraphChanges


george-microsoft May 22, 2018
ce9792f

Changes to checks in AutoMlUtils (more correct usage of them).


george-microsoft May 22, 2018
a828266

Merge branch 'AutoML-GraphChanges' of https://github.com/george-micro…


george-microsoft May 22, 2018
c642f68

Fixing issue with ExceptParam using value and not name of parameters.


george-microsoft May 22, 2018
e066db0

Fixing errors on use of ExceptParam.


george-microsoft May 22, 2018

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


14 changes: 7 additions & 7 deletions
src/Microsoft.ML.Data/EntryPoints/InputBuilder.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -832,21 +832,21 @@ public static class SweepableDiscreteParam
 public static class PipelineSweeperSupportedMetrics
 {
 public new static string ToString() => "SupportedMetric";
 public const string Auc = "Auc";
 public const string Auc = "AUC";
 public const string AccuracyMicro = "AccuracyMicro";
 public const string AccuracyMacro = "AccuracyMacro";
 public const string F1 = "F1";
 public const string AuPrc = "AuPrc";
 public const string AuPrc = "AUPRC";
 public const string TopKAccuracy = "TopKAccuracy";
 public const string L1 = "L1";
 public const string L2 = "L2";
 public const string Rms = "Rms";
 public const string Rms = "RMS";
 public const string LossFn = "LossFn";
 public const string RSquared = "RSquared";
 public const string LogLoss = "LogLoss";
 public const string LogLossReduction = "LogLossReduction";
 public const string Ndcg = "Ndcg";
 public const string Dcg = "Dcg";
 public const string Ndcg = "NDCG";
 public const string Dcg = "DCG";
 public const string PositivePrecision = "PositivePrecision";
 public const string PositiveRecall = "PositiveRecall";
 public const string NegativePrecision = "NegativePrecision";
Expand All@@ -858,9 +858,9 @@ public static class PipelineSweeperSupportedMetrics
 public const string ThreshAtK = "ThreshAtK";
 public const string ThreshAtP = "ThreshAtP";
 public const string ThreshAtNumPos = "ThreshAtNumPos";
 public const string Nmi = "Nmi";
 public const string Nmi = "NMI";
 public const string AvgMinScore = "AvgMinScore";
 public const string Dbi = "Dbi";
 public const string Dbi = "DBI";
 }
 }
}

20 changes: 12 additions & 8 deletions
src/Microsoft.ML.PipelineInference/AutoInference.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -158,7 +158,8 @@ private bool GetDataVariableName(IExceptionContext ectx, string nameOfData, JTok
 return false;

 string dataVar = firstNodeInputs.Value<String>(nameOfData);
 ectx.Check(VariableBinding.IsValidVariableName(ectx, dataVar), $"Invalid variable name {dataVar}.");
 if (!VariableBinding.IsValidVariableName(ectx, dataVar))
 throw ectx.ExceptParam(nameof(nameOfData), $"Invalid variable name {dataVar}.");

 variableName = dataVar.Substring(1);
 return true;
Expand All@@ -172,12 +173,14 @@ private bool GetDataVariableName(IExceptionContext ectx, string nameOfData, JTok
 public sealed class RunSummary
 {
 public double MetricValue { get; }
 public double TrainingMetricValue { get; }
 public int NumRowsInTraining { get; }
 public long RunTimeMilliseconds { get; }

 public RunSummary(double metricValue, int numRows, long runTimeMilliseconds)
 public RunSummary(double metricValue, int numRows, long runTimeMilliseconds, double trainingMetricValue)
 {
 MetricValue = metricValue;
 TrainingMetricValue = trainingMetricValue;
 NumRowsInTraining = numRows;
 RunTimeMilliseconds = runTimeMilliseconds;
 }
Expand DownExpand Up@@ -303,7 +306,7 @@ private void MainLearningLoop(int batchSize, int numOfTrainingRows)
 var stopwatch = new Stopwatch();
 var probabilityUtils = new Sweeper.Algorithms.SweeperProbabilityUtils(_host);

while (!_terminator.ShouldTerminate(_history))
 while (!_terminator.ShouldTerminate(_history))
 {
 // Get next set of candidates
 var currentBatchSize = batchSize;
Expand DownExpand Up@@ -341,16 +344,17 @@ private void ProcessPipeline(Sweeper.Algorithms.SweeperProbabilityUtils utils, S

 // Run pipeline, and time how long it takes
 stopwatch.Restart();
double d = candidate.RunTrainTestExperiment(_trainData.Take(randomizedNumberOfRows),
 _testData, Metric, TrainerKind);
 candidate.RunTrainTestExperiment(_trainData.Take(randomizedNumberOfRows),
 _testData, Metric, TrainerKind, out var testMetricVal, out var trainMetricVal);
 stopwatch.Stop();

 // Handle key collisions on sorted list
 while (_sortedSampledElements.ContainsKey(d))
d += 1e-10;
 while (_sortedSampledElements.ContainsKey(testMetricVal))
testMetricVal += 1e-10;

 // Save performance score
 candidate.PerformanceSummary = new RunSummary(d, randomizedNumberOfRows, stopwatch.ElapsedMilliseconds);
 candidate.PerformanceSummary = 
 new RunSummary(testMetricVal, randomizedNumberOfRows, stopwatch.ElapsedMilliseconds, trainMetricVal);
 _sortedSampledElements.Add(candidate.PerformanceSummary.MetricValue, candidate);
 _history.Add(candidate);
 }
Expand Down

33 changes: 24 additions & 9 deletions
src/Microsoft.ML.PipelineInference/AutoMlUtils.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -15,21 +15,34 @@ namespace Microsoft.ML.Runtime.PipelineInference
{
 public static class AutoMlUtils
 {
 public static AutoInference.RunSummary ExtractRunSummary(IHostEnvironment env, IDataView data, string metricColumnName)
 public static double ExtractValueFromIDV(IHostEnvironment env, IDataView result, string columnName)
 {
 double metricValue = 0;
 int numRows = 0;
 var schema = data.Schema;
 schema.TryGetColumnIndex(metricColumnName, out var metricCol);
 Contracts.CheckValue(env, nameof(env));
 env.CheckValue(result, nameof(result));
 env.CheckNonEmpty(columnName, nameof(columnName));

 using (var cursor = data.GetRowCursor(col => col == metricCol))
 double outputValue = 0;
 var schema = result.Schema;
 if (!schema.TryGetColumnIndex(columnName, out var metricCol))
 throw env.ExceptParam(nameof(columnName), $"Schema does not contain column: {columnName}");

 using (var cursor = result.GetRowCursor(col => col == metricCol))
 {
 var getter = cursor.GetGetter<double>(metricCol);
 cursor.MoveNext();
 getter(ref metricValue);
 bool moved = cursor.MoveNext();
 env.Check(moved, "Expected an IDataView with a single row. Results dataset has no rows to extract.");
 getter(ref outputValue);
 env.Check(!cursor.MoveNext(), "Expected an IDataView with a single row. Results dataset has too many rows.");
 }

 return new AutoInference.RunSummary(metricValue, numRows, 0);
 return outputValue;
 }

 public static AutoInference.RunSummary ExtractRunSummary(IHostEnvironment env, IDataView result, string metricColumnName, IDataView trainResult = null)
 {
 double testingMetricValue = ExtractValueFromIDV(env, result, metricColumnName);
 double trainingMetricValue = trainResult != null ? ExtractValueFromIDV(env, trainResult, metricColumnName) : double.MinValue;
 return new AutoInference.RunSummary(testingMetricValue, 0, 0, trainingMetricValue);
 }

 public static CommonInputs.IEvaluatorInput CloneEvaluatorInstance(CommonInputs.IEvaluatorInput evalInput) =>
Expand DownExpand Up@@ -618,5 +631,7 @@ public static Tuple<string, string[]>[] ConvertToSweepArgumentStrings(TlcModule.
 }
 return results;
 }

 public static string GenerateOverallTrainingMetricVarName(Guid id) => $"Var_Training_OM_{id:N}";
 }
}

19 changes: 14 additions & 5 deletions
src/Microsoft.ML.PipelineInference/Macros/PipelineSweeperMacro.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -65,18 +65,24 @@ public static Output ExtractSweepResult(IHostEnvironment env, ResultInput input)
 var col1 = new KeyValuePair<string, ColumnType>("Graph", TextType.Instance);
 var col2 = new KeyValuePair<string, ColumnType>("MetricValue", PrimitiveType.FromKind(DataKind.R8));
 var col3 = new KeyValuePair<string, ColumnType>("PipelineId", TextType.Instance);
 var col4 = new KeyValuePair<string, ColumnType>("TrainingMetricValue", PrimitiveType.FromKind(DataKind.R8));
 var col5 = new KeyValuePair<string, ColumnType>("FirstInput", TextType.Instance);
 var col6 = new KeyValuePair<string, ColumnType>("PredictorModel", TextType.Instance);

 if (rows.Count == 0)
 {
 var host = env.Register("ExtractSweepResult");
 outputView = new EmptyDataView(host, new SimpleSchema(host, col1, col2, col3));
 outputView = new EmptyDataView(host, new SimpleSchema(host, col1, col2, col3, col4, col5, col6));
 }
 else
 {
 var builder = new ArrayDataViewBuilder(env);
 builder.AddColumn(col1.Key, (PrimitiveType)col1.Value, rows.Select(r => new DvText(r.GraphJson)).ToArray());
 builder.AddColumn(col2.Key, (PrimitiveType)col2.Value, rows.Select(r => r.MetricValue).ToArray());
 builder.AddColumn(col3.Key, (PrimitiveType)col3.Value, rows.Select(r => new DvText(r.PipelineId)).ToArray());
 builder.AddColumn(col4.Key, (PrimitiveType)col4.Value, rows.Select(r => r.TrainingMetricValue).ToArray());
 builder.AddColumn(col5.Key, (PrimitiveType)col5.Value, rows.Select(r => new DvText(r.FirstInput)).ToArray());
 builder.AddColumn(col6.Key, (PrimitiveType)col6.Value, rows.Select(r => new DvText(r.PredictorModel)).ToArray());
 outputView = builder.GetDataView();
 }
 return new Output { Results = outputView, State = autoMlState };
Expand DownExpand Up@@ -132,11 +138,11 @@ public static CommonOutputs.MacroOutput<Output> PipelineSweep(
 // Extract performance summaries and assign to previous candidate pipelines.
 foreach (var pipeline in autoMlState.BatchCandidates)
 {
 if (node.Context.TryGetVariable(ExperimentUtils.GenerateOverallMetricVarName(pipeline.UniqueId),
 out var v))
 if (node.Context.TryGetVariable(ExperimentUtils.GenerateOverallMetricVarName(pipeline.UniqueId), out var v) &&
node.Context.TryGetVariable(AutoMlUtils.GenerateOverallTrainingMetricVarName(pipeline.UniqueId), out var v2))
 {
 pipeline.PerformanceSummary =
 AutoMlUtils.ExtractRunSummary(env, (IDataView)v.Value, autoMlState.Metric.Name);
 AutoMlUtils.ExtractRunSummary(env, (IDataView)v.Value, autoMlState.Metric.Name, (IDataView)v2.Value);
 autoMlState.AddEvaluated(pipeline);
 }
 }
Expand DownExpand Up@@ -168,14 +174,17 @@ public static CommonOutputs.MacroOutput<Output> PipelineSweep(
 {
 // Add train test experiments to current graph for candidate pipeline
 var subgraph = new Experiment(env);
 var trainTestOutput = p.AddAsTrainTest(training, testing, autoMlState.TrainerKind, subgraph);
 var trainTestOutput = p.AddAsTrainTest(training, testing, autoMlState.TrainerKind, subgraph, true);

 // Change variable name to reference pipeline ID in output map, context and entrypoint output.
 var uniqueName = ExperimentUtils.GenerateOverallMetricVarName(p.UniqueId);
 var uniqueNameTraining = AutoMlUtils.GenerateOverallTrainingMetricVarName(p.UniqueId);
 var sgNode = EntryPointNode.ValidateNodes(env, node.Context,
 new JArray(subgraph.GetNodes().Last()), node.Catalog).Last();
 sgNode.RenameOutputVariable(trainTestOutput.OverallMetrics.VarName, uniqueName, cascadeChanges: true);
 sgNode.RenameOutputVariable(trainTestOutput.TrainingOverallMetrics.VarName, uniqueNameTraining, cascadeChanges: true);
 trainTestOutput.OverallMetrics.VarName = uniqueName;
 trainTestOutput.TrainingOverallMetrics.VarName = uniqueNameTraining;
 expNodes.Add(sgNode);

 // Store indicators, to pass to next iteration of macro.
Expand Down

1 change: 1 addition & 0 deletions
src/Microsoft.ML.PipelineInference/Microsoft.ML.PipelineInference.csproj

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -17,6 +17,7 @@
 <ProjectReference Include="..\Microsoft.ML.Core\Microsoft.ML.Core.csproj" />
 <ProjectReference Include="..\Microsoft.ML.StandardLearners\Microsoft.ML.StandardLearners.csproj" />
 <ProjectReference Include="..\Microsoft.ML.Sweeper\Microsoft.ML.Sweeper.csproj" />
 <ProjectReference Include="..\Microsoft.ML\Microsoft.ML.csproj" />
 </ItemGroup>

</Project>
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Remove or un-stick sticky/fixed headers that block content\n(function() {\n function unstick() {\n document.querySelectorAll('header, nav, [role=\"banner\"], .header, .navbar, .sticky, .fixed-top, [style*=\"position: fixed\"], [style*=\"position:sticky\"]').forEach(function(el) {\n if (el.style.position === 'fixed' || el.style.position === 'sticky' || \n getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') {\n el.style.position = 'static';\n el.style.top = 'auto';\n el.style.zIndex = 'auto';\n }\n });\n }\n \n unstick();\n \n var observer = new MutationObserver(unstick);\n observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] });\n})();", "Kill Sticky Headers");
}
} catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
Adding support for training metrics in PipelineSweeperMacro + new graph variable outputs#152

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

george-microsoft
merged 15 commits into

from

May 23, 2018
 Merged


Adding support for training metrics in PipelineSweeperMacro + new graph variable outputs
#152
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


15 commits


Select commit
Hold shift + click to select a range
05482eb

Adding support for training metrics in PipelineSweeperMacro and neede…


george-microsoft May 14, 2018
b789159

Changes for revieer comments.


george-microsoft May 14, 2018
76dd270

Changes to address reviewer comments.


george-microsoft May 14, 2018
483a50b

More changes for reviewer commenrts.


george-microsoft May 17, 2018
d42a522

Changes for reviewer comments.


george-microsoft May 18, 2018
fbee51c

Changes for latest round of reviewer comments.


george-microsoft May 21, 2018
0285c87

Changed where XML comment was placed.


george-microsoft May 21, 2018
6f92693

Added more tests (uncommented and fixed) for auto inference. Changed …


george-microsoft May 22, 2018
e006ee1

Added another test, TestPipelineSweeperMacroNoTransforms.


george-microsoft May 22, 2018
90d96a3

Updated tests to include warning disabling (following Zeeshan S's exa…


george-microsoft May 22, 2018
1d1c303

Merge branch 'master' into AutoML-GraphChanges


george-microsoft May 22, 2018
ce9792f

Changes to checks in AutoMlUtils (more correct usage of them).


george-microsoft May 22, 2018
a828266

Merge branch 'AutoML-GraphChanges' of https://github.com/george-micro…


george-microsoft May 22, 2018
c642f68

Fixing issue with ExceptParam using value and not name of parameters.


george-microsoft May 22, 2018
e066db0

Fixing errors on use of ExceptParam.


george-microsoft May 22, 2018

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


14 changes: 7 additions & 7 deletions
src/Microsoft.ML.Data/EntryPoints/InputBuilder.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -832,21 +832,21 @@ public static class SweepableDiscreteParam
 public static class PipelineSweeperSupportedMetrics
 {
 public new static string ToString() => "SupportedMetric";
 public const string Auc = "Auc";
 public const string Auc = "AUC";
 public const string AccuracyMicro = "AccuracyMicro";
 public const string AccuracyMacro = "AccuracyMacro";
 public const string F1 = "F1";
 public const string AuPrc = "AuPrc";
 public const string AuPrc = "AUPRC";
 public const string TopKAccuracy = "TopKAccuracy";
 public const string L1 = "L1";
 public const string L2 = "L2";
 public const string Rms = "Rms";
 public const string Rms = "RMS";
 public const string LossFn = "LossFn";
 public const string RSquared = "RSquared";
 public const string LogLoss = "LogLoss";
 public const string LogLossReduction = "LogLossReduction";
 public const string Ndcg = "Ndcg";
 public const string Dcg = "Dcg";
 public const string Ndcg = "NDCG";
 public const string Dcg = "DCG";
 public const string PositivePrecision = "PositivePrecision";
 public const string PositiveRecall = "PositiveRecall";
 public const string NegativePrecision = "NegativePrecision";
Expand All@@ -858,9 +858,9 @@ public static class PipelineSweeperSupportedMetrics
 public const string ThreshAtK = "ThreshAtK";
 public const string ThreshAtP = "ThreshAtP";
 public const string ThreshAtNumPos = "ThreshAtNumPos";
 public const string Nmi = "Nmi";
 public const string Nmi = "NMI";
 public const string AvgMinScore = "AvgMinScore";
 public const string Dbi = "Dbi";
 public const string Dbi = "DBI";
 }
 }
}

20 changes: 12 additions & 8 deletions
src/Microsoft.ML.PipelineInference/AutoInference.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -158,7 +158,8 @@ private bool GetDataVariableName(IExceptionContext ectx, string nameOfData, JTok
 return false;

 string dataVar = firstNodeInputs.Value<String>(nameOfData);
 ectx.Check(VariableBinding.IsValidVariableName(ectx, dataVar), $"Invalid variable name {dataVar}.");
 if (!VariableBinding.IsValidVariableName(ectx, dataVar))
 throw ectx.ExceptParam(nameof(nameOfData), $"Invalid variable name {dataVar}.");

 variableName = dataVar.Substring(1);
 return true;
Expand All@@ -172,12 +173,14 @@ private bool GetDataVariableName(IExceptionContext ectx, string nameOfData, JTok
 public sealed class RunSummary
 {
 public double MetricValue { get; }
 public double TrainingMetricValue { get; }
 public int NumRowsInTraining { get; }
 public long RunTimeMilliseconds { get; }

 public RunSummary(double metricValue, int numRows, long runTimeMilliseconds)
 public RunSummary(double metricValue, int numRows, long runTimeMilliseconds, double trainingMetricValue)
 {
 MetricValue = metricValue;
 TrainingMetricValue = trainingMetricValue;
 NumRowsInTraining = numRows;
 RunTimeMilliseconds = runTimeMilliseconds;
 }
Expand DownExpand Up@@ -303,7 +306,7 @@ private void MainLearningLoop(int batchSize, int numOfTrainingRows)
 var stopwatch = new Stopwatch();
 var probabilityUtils = new Sweeper.Algorithms.SweeperProbabilityUtils(_host);

while (!_terminator.ShouldTerminate(_history))
 while (!_terminator.ShouldTerminate(_history))
 {
 // Get next set of candidates
 var currentBatchSize = batchSize;
Expand DownExpand Up@@ -341,16 +344,17 @@ private void ProcessPipeline(Sweeper.Algorithms.SweeperProbabilityUtils utils, S

 // Run pipeline, and time how long it takes
 stopwatch.Restart();
double d = candidate.RunTrainTestExperiment(_trainData.Take(randomizedNumberOfRows),
 _testData, Metric, TrainerKind);
 candidate.RunTrainTestExperiment(_trainData.Take(randomizedNumberOfRows),
 _testData, Metric, TrainerKind, out var testMetricVal, out var trainMetricVal);
 stopwatch.Stop();

 // Handle key collisions on sorted list
 while (_sortedSampledElements.ContainsKey(d))
d += 1e-10;
 while (_sortedSampledElements.ContainsKey(testMetricVal))
testMetricVal += 1e-10;

 // Save performance score
 candidate.PerformanceSummary = new RunSummary(d, randomizedNumberOfRows, stopwatch.ElapsedMilliseconds);
 candidate.PerformanceSummary = 
 new RunSummary(testMetricVal, randomizedNumberOfRows, stopwatch.ElapsedMilliseconds, trainMetricVal);
 _sortedSampledElements.Add(candidate.PerformanceSummary.MetricValue, candidate);
 _history.Add(candidate);
 }
Expand Down

33 changes: 24 additions & 9 deletions
src/Microsoft.ML.PipelineInference/AutoMlUtils.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -15,21 +15,34 @@ namespace Microsoft.ML.Runtime.PipelineInference
{
 public static class AutoMlUtils
 {
 public static AutoInference.RunSummary ExtractRunSummary(IHostEnvironment env, IDataView data, string metricColumnName)
 public static double ExtractValueFromIDV(IHostEnvironment env, IDataView result, string columnName)
 {
 double metricValue = 0;
 int numRows = 0;
 var schema = data.Schema;
 schema.TryGetColumnIndex(metricColumnName, out var metricCol);
 Contracts.CheckValue(env, nameof(env));
 env.CheckValue(result, nameof(result));
 env.CheckNonEmpty(columnName, nameof(columnName));

 using (var cursor = data.GetRowCursor(col => col == metricCol))
 double outputValue = 0;
 var schema = result.Schema;
 if (!schema.TryGetColumnIndex(columnName, out var metricCol))
 throw env.ExceptParam(nameof(columnName), $"Schema does not contain column: {columnName}");

 using (var cursor = result.GetRowCursor(col => col == metricCol))
 {
 var getter = cursor.GetGetter<double>(metricCol);
 cursor.MoveNext();
 getter(ref metricValue);
 bool moved = cursor.MoveNext();
 env.Check(moved, "Expected an IDataView with a single row. Results dataset has no rows to extract.");
 getter(ref outputValue);
 env.Check(!cursor.MoveNext(), "Expected an IDataView with a single row. Results dataset has too many rows.");
 }

 return new AutoInference.RunSummary(metricValue, numRows, 0);
 return outputValue;
 }

 public static AutoInference.RunSummary ExtractRunSummary(IHostEnvironment env, IDataView result, string metricColumnName, IDataView trainResult = null)
 {
 double testingMetricValue = ExtractValueFromIDV(env, result, metricColumnName);
 double trainingMetricValue = trainResult != null ? ExtractValueFromIDV(env, trainResult, metricColumnName) : double.MinValue;
 return new AutoInference.RunSummary(testingMetricValue, 0, 0, trainingMetricValue);
 }

 public static CommonInputs.IEvaluatorInput CloneEvaluatorInstance(CommonInputs.IEvaluatorInput evalInput) =>
Expand DownExpand Up@@ -618,5 +631,7 @@ public static Tuple<string, string[]>[] ConvertToSweepArgumentStrings(TlcModule.
 }
 return results;
 }

 public static string GenerateOverallTrainingMetricVarName(Guid id) => $"Var_Training_OM_{id:N}";
 }
}

19 changes: 14 additions & 5 deletions
src/Microsoft.ML.PipelineInference/Macros/PipelineSweeperMacro.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -65,18 +65,24 @@ public static Output ExtractSweepResult(IHostEnvironment env, ResultInput input)
 var col1 = new KeyValuePair<string, ColumnType>("Graph", TextType.Instance);
 var col2 = new KeyValuePair<string, ColumnType>("MetricValue", PrimitiveType.FromKind(DataKind.R8));
 var col3 = new KeyValuePair<string, ColumnType>("PipelineId", TextType.Instance);
 var col4 = new KeyValuePair<string, ColumnType>("TrainingMetricValue", PrimitiveType.FromKind(DataKind.R8));
 var col5 = new KeyValuePair<string, ColumnType>("FirstInput", TextType.Instance);
 var col6 = new KeyValuePair<string, ColumnType>("PredictorModel", TextType.Instance);

 if (rows.Count == 0)
 {
 var host = env.Register("ExtractSweepResult");
 outputView = new EmptyDataView(host, new SimpleSchema(host, col1, col2, col3));
 outputView = new EmptyDataView(host, new SimpleSchema(host, col1, col2, col3, col4, col5, col6));
 }
 else
 {
 var builder = new ArrayDataViewBuilder(env);
 builder.AddColumn(col1.Key, (PrimitiveType)col1.Value, rows.Select(r => new DvText(r.GraphJson)).ToArray());
 builder.AddColumn(col2.Key, (PrimitiveType)col2.Value, rows.Select(r => r.MetricValue).ToArray());
 builder.AddColumn(col3.Key, (PrimitiveType)col3.Value, rows.Select(r => new DvText(r.PipelineId)).ToArray());
 builder.AddColumn(col4.Key, (PrimitiveType)col4.Value, rows.Select(r => r.TrainingMetricValue).ToArray());
 builder.AddColumn(col5.Key, (PrimitiveType)col5.Value, rows.Select(r => new DvText(r.FirstInput)).ToArray());
 builder.AddColumn(col6.Key, (PrimitiveType)col6.Value, rows.Select(r => new DvText(r.PredictorModel)).ToArray());
 outputView = builder.GetDataView();
 }
 return new Output { Results = outputView, State = autoMlState };
Expand DownExpand Up@@ -132,11 +138,11 @@ public static CommonOutputs.MacroOutput<Output> PipelineSweep(
 // Extract performance summaries and assign to previous candidate pipelines.
 foreach (var pipeline in autoMlState.BatchCandidates)
 {
 if (node.Context.TryGetVariable(ExperimentUtils.GenerateOverallMetricVarName(pipeline.UniqueId),
 out var v))
 if (node.Context.TryGetVariable(ExperimentUtils.GenerateOverallMetricVarName(pipeline.UniqueId), out var v) &&
node.Context.TryGetVariable(AutoMlUtils.GenerateOverallTrainingMetricVarName(pipeline.UniqueId), out var v2))
 {
 pipeline.PerformanceSummary =
 AutoMlUtils.ExtractRunSummary(env, (IDataView)v.Value, autoMlState.Metric.Name);
 AutoMlUtils.ExtractRunSummary(env, (IDataView)v.Value, autoMlState.Metric.Name, (IDataView)v2.Value);
 autoMlState.AddEvaluated(pipeline);
 }
 }
Expand DownExpand Up@@ -168,14 +174,17 @@ public static CommonOutputs.MacroOutput<Output> PipelineSweep(
 {
 // Add train test experiments to current graph for candidate pipeline
 var subgraph = new Experiment(env);
 var trainTestOutput = p.AddAsTrainTest(training, testing, autoMlState.TrainerKind, subgraph);
 var trainTestOutput = p.AddAsTrainTest(training, testing, autoMlState.TrainerKind, subgraph, true);

 // Change variable name to reference pipeline ID in output map, context and entrypoint output.
 var uniqueName = ExperimentUtils.GenerateOverallMetricVarName(p.UniqueId);
 var uniqueNameTraining = AutoMlUtils.GenerateOverallTrainingMetricVarName(p.UniqueId);
 var sgNode = EntryPointNode.ValidateNodes(env, node.Context,
 new JArray(subgraph.GetNodes().Last()), node.Catalog).Last();
 sgNode.RenameOutputVariable(trainTestOutput.OverallMetrics.VarName, uniqueName, cascadeChanges: true);
 sgNode.RenameOutputVariable(trainTestOutput.TrainingOverallMetrics.VarName, uniqueNameTraining, cascadeChanges: true);
 trainTestOutput.OverallMetrics.VarName = uniqueName;
 trainTestOutput.TrainingOverallMetrics.VarName = uniqueNameTraining;
 expNodes.Add(sgNode);

 // Store indicators, to pass to next iteration of macro.
Expand Down

1 change: 1 addition & 0 deletions
src/Microsoft.ML.PipelineInference/Microsoft.ML.PipelineInference.csproj

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -17,6 +17,7 @@
 <ProjectReference Include="..\Microsoft.ML.Core\Microsoft.ML.Core.csproj" />
 <ProjectReference Include="..\Microsoft.ML.StandardLearners\Microsoft.ML.StandardLearners.csproj" />
 <ProjectReference Include="..\Microsoft.ML.Sweeper\Microsoft.ML.Sweeper.csproj" />
 <ProjectReference Include="..\Microsoft.ML\Microsoft.ML.csproj" />
 </ItemGroup>

</Project>
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Universal Dark Mode - works on any site\n(function() {\n var enabled = true;\n \n function applyDarkMode() {\n if (!enabled) return;\n \n // Create style element if it doesn't exist\n var style = document.getElementById('universal-dark-mode-style');\n if (!style) {\n style = document.createElement('style');\n style.id = 'universal-dark-mode-style';\n document.head.appendChild(style);\n }\n \n // Dark mode CSS - inverts colors but preserves images/video\n style.textContent = '\n /* Invert everything except media */\n html {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #1a1a2e !important;\n }\n \n /* Restore images, videos, iframes, canvas */\n img, video, iframe, canvas, svg, picture, [style*=\"background-image\"] {\n filter: invert(1) hue-rotate(180deg) !important;\n }\n \n /* Preserve specific elements that should not be inverted */\n .no-dark-mode, .no-dark-mode *,\n [data-theme=\"light\"], [data-theme=\"light\"],\n .ace_editor, .ace_editor *,\n .CodeMirror, .CodeMirror *,\n .monaco-editor, .monaco-editor *,\n .markdown-body pre, .markdown-body pre *,\n .highlight, .highlight *,\n pre code, pre code * {\n filter: none !important;\n }\n \n /* Fix common UI elements */\n .modal, .popup, .dropdown-menu, .tooltip, .popover {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #2d2d44 !important;\n border-color: #444 !important;\n }\n \n /* Scrollbars */\n ::-webkit-scrollbar { background: #1a1a2e !important; }\n ::-webkit-scrollbar-thumb { background: #444 !important; }\n ::-webkit-scrollbar-thumb:hover { background: #555 !important; }\n \n /* Selection */\n ::selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ::-moz-selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ';\n }\n \n function removeDarkMode() {\n var style = document.getElementById('universal-dark-mode-style');\n if (style) style.remove();\n }\n \n // Toggle with Alt+Shift+D\n document.addEventListener('keydown', function(e) {\n if (e.altKey && e.shiftKey && e.key === 'D') {\n e.preventDefault();\n enabled = !enabled;\n if (enabled) {\n applyDarkMode();\n console.log('[Universal Dark Mode] Enabled');\n } else {\n removeDarkMode();\n console.log('[Universal Dark Mode] Disabled');\n }\n }\n });\n \n // Apply on load\n applyDarkMode();\n \n // Re-apply on dynamic content\n var observer = new MutationObserver(function(mutations) {\n if (enabled && !document.getElementById('universal-dark-mode-style')) {\n applyDarkMode();\n }\n });\n observer.observe(document.head, { childList: true });\n \n console.log('[Universal Dark Mode] Loaded - Press Alt+Shift+D to toggle');\n})();", "Universal Dark Mode");
}
} catch(__e) { console.warn('[Userscript:Universal Dark Mode]', __e); }
})();
})();

Skip to content
Adding support for training metrics in PipelineSweeperMacro + new graph variable outputs#152

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

george-microsoft
merged 15 commits into

from

May 23, 2018
 Merged


Adding support for training metrics in PipelineSweeperMacro + new graph variable outputs
#152
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


15 commits


Select commit
Hold shift + click to select a range
05482eb

Adding support for training metrics in PipelineSweeperMacro and neede…


george-microsoft May 14, 2018
b789159

Changes for revieer comments.


george-microsoft May 14, 2018
76dd270

Changes to address reviewer comments.


george-microsoft May 14, 2018
483a50b

More changes for reviewer commenrts.


george-microsoft May 17, 2018
d42a522

Changes for reviewer comments.


george-microsoft May 18, 2018
fbee51c

Changes for latest round of reviewer comments.


george-microsoft May 21, 2018
0285c87

Changed where XML comment was placed.


george-microsoft May 21, 2018
6f92693

Added more tests (uncommented and fixed) for auto inference. Changed …


george-microsoft May 22, 2018
e006ee1

Added another test, TestPipelineSweeperMacroNoTransforms.


george-microsoft May 22, 2018
90d96a3

Updated tests to include warning disabling (following Zeeshan S's exa…


george-microsoft May 22, 2018
1d1c303

Merge branch 'master' into AutoML-GraphChanges


george-microsoft May 22, 2018
ce9792f

Changes to checks in AutoMlUtils (more correct usage of them).


george-microsoft May 22, 2018
a828266

Merge branch 'AutoML-GraphChanges' of https://github.com/george-micro…


george-microsoft May 22, 2018
c642f68

Fixing issue with ExceptParam using value and not name of parameters.


george-microsoft May 22, 2018
e066db0

Fixing errors on use of ExceptParam.


george-microsoft May 22, 2018

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


14 changes: 7 additions & 7 deletions
src/Microsoft.ML.Data/EntryPoints/InputBuilder.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -832,21 +832,21 @@ public static class SweepableDiscreteParam
 public static class PipelineSweeperSupportedMetrics
 {
 public new static string ToString() => "SupportedMetric";
 public const string Auc = "Auc";
 public const string Auc = "AUC";
 public const string AccuracyMicro = "AccuracyMicro";
 public const string AccuracyMacro = "AccuracyMacro";
 public const string F1 = "F1";
 public const string AuPrc = "AuPrc";
 public const string AuPrc = "AUPRC";
 public const string TopKAccuracy = "TopKAccuracy";
 public const string L1 = "L1";
 public const string L2 = "L2";
 public const string Rms = "Rms";
 public const string Rms = "RMS";
 public const string LossFn = "LossFn";
 public const string RSquared = "RSquared";
 public const string LogLoss = "LogLoss";
 public const string LogLossReduction = "LogLossReduction";
 public const string Ndcg = "Ndcg";
 public const string Dcg = "Dcg";
 public const string Ndcg = "NDCG";
 public const string Dcg = "DCG";
 public const string PositivePrecision = "PositivePrecision";
 public const string PositiveRecall = "PositiveRecall";
 public const string NegativePrecision = "NegativePrecision";
Expand All@@ -858,9 +858,9 @@ public static class PipelineSweeperSupportedMetrics
 public const string ThreshAtK = "ThreshAtK";
 public const string ThreshAtP = "ThreshAtP";
 public const string ThreshAtNumPos = "ThreshAtNumPos";
 public const string Nmi = "Nmi";
 public const string Nmi = "NMI";
 public const string AvgMinScore = "AvgMinScore";
 public const string Dbi = "Dbi";
 public const string Dbi = "DBI";
 }
 }
}

20 changes: 12 additions & 8 deletions
src/Microsoft.ML.PipelineInference/AutoInference.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -158,7 +158,8 @@ private bool GetDataVariableName(IExceptionContext ectx, string nameOfData, JTok
 return false;

 string dataVar = firstNodeInputs.Value<String>(nameOfData);
 ectx.Check(VariableBinding.IsValidVariableName(ectx, dataVar), $"Invalid variable name {dataVar}.");
 if (!VariableBinding.IsValidVariableName(ectx, dataVar))
 throw ectx.ExceptParam(nameof(nameOfData), $"Invalid variable name {dataVar}.");

 variableName = dataVar.Substring(1);
 return true;
Expand All@@ -172,12 +173,14 @@ private bool GetDataVariableName(IExceptionContext ectx, string nameOfData, JTok
 public sealed class RunSummary
 {
 public double MetricValue { get; }
 public double TrainingMetricValue { get; }
 public int NumRowsInTraining { get; }
 public long RunTimeMilliseconds { get; }

 public RunSummary(double metricValue, int numRows, long runTimeMilliseconds)
 public RunSummary(double metricValue, int numRows, long runTimeMilliseconds, double trainingMetricValue)
 {
 MetricValue = metricValue;
 TrainingMetricValue = trainingMetricValue;
 NumRowsInTraining = numRows;
 RunTimeMilliseconds = runTimeMilliseconds;
 }
Expand DownExpand Up@@ -303,7 +306,7 @@ private void MainLearningLoop(int batchSize, int numOfTrainingRows)
 var stopwatch = new Stopwatch();
 var probabilityUtils = new Sweeper.Algorithms.SweeperProbabilityUtils(_host);

while (!_terminator.ShouldTerminate(_history))
 while (!_terminator.ShouldTerminate(_history))
 {
 // Get next set of candidates
 var currentBatchSize = batchSize;
Expand DownExpand Up@@ -341,16 +344,17 @@ private void ProcessPipeline(Sweeper.Algorithms.SweeperProbabilityUtils utils, S

 // Run pipeline, and time how long it takes
 stopwatch.Restart();
double d = candidate.RunTrainTestExperiment(_trainData.Take(randomizedNumberOfRows),
 _testData, Metric, TrainerKind);
 candidate.RunTrainTestExperiment(_trainData.Take(randomizedNumberOfRows),
 _testData, Metric, TrainerKind, out var testMetricVal, out var trainMetricVal);
 stopwatch.Stop();

 // Handle key collisions on sorted list
 while (_sortedSampledElements.ContainsKey(d))
d += 1e-10;
 while (_sortedSampledElements.ContainsKey(testMetricVal))
testMetricVal += 1e-10;

 // Save performance score
 candidate.PerformanceSummary = new RunSummary(d, randomizedNumberOfRows, stopwatch.ElapsedMilliseconds);
 candidate.PerformanceSummary = 
 new RunSummary(testMetricVal, randomizedNumberOfRows, stopwatch.ElapsedMilliseconds, trainMetricVal);
 _sortedSampledElements.Add(candidate.PerformanceSummary.MetricValue, candidate);
 _history.Add(candidate);
 }
Expand Down

33 changes: 24 additions & 9 deletions
src/Microsoft.ML.PipelineInference/AutoMlUtils.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -15,21 +15,34 @@ namespace Microsoft.ML.Runtime.PipelineInference
{
 public static class AutoMlUtils
 {
 public static AutoInference.RunSummary ExtractRunSummary(IHostEnvironment env, IDataView data, string metricColumnName)
 public static double ExtractValueFromIDV(IHostEnvironment env, IDataView result, string columnName)
 {
 double metricValue = 0;
 int numRows = 0;
 var schema = data.Schema;
 schema.TryGetColumnIndex(metricColumnName, out var metricCol);
 Contracts.CheckValue(env, nameof(env));
 env.CheckValue(result, nameof(result));
 env.CheckNonEmpty(columnName, nameof(columnName));

 using (var cursor = data.GetRowCursor(col => col == metricCol))
 double outputValue = 0;
 var schema = result.Schema;
 if (!schema.TryGetColumnIndex(columnName, out var metricCol))
 throw env.ExceptParam(nameof(columnName), $"Schema does not contain column: {columnName}");

 using (var cursor = result.GetRowCursor(col => col == metricCol))
 {
 var getter = cursor.GetGetter<double>(metricCol);
 cursor.MoveNext();
 getter(ref metricValue);
 bool moved = cursor.MoveNext();
 env.Check(moved, "Expected an IDataView with a single row. Results dataset has no rows to extract.");
 getter(ref outputValue);
 env.Check(!cursor.MoveNext(), "Expected an IDataView with a single row. Results dataset has too many rows.");
 }

 return new AutoInference.RunSummary(metricValue, numRows, 0);
 return outputValue;
 }

 public static AutoInference.RunSummary ExtractRunSummary(IHostEnvironment env, IDataView result, string metricColumnName, IDataView trainResult = null)
 {
 double testingMetricValue = ExtractValueFromIDV(env, result, metricColumnName);
 double trainingMetricValue = trainResult != null ? ExtractValueFromIDV(env, trainResult, metricColumnName) : double.MinValue;
 return new AutoInference.RunSummary(testingMetricValue, 0, 0, trainingMetricValue);
 }

 public static CommonInputs.IEvaluatorInput CloneEvaluatorInstance(CommonInputs.IEvaluatorInput evalInput) =>
Expand DownExpand Up@@ -618,5 +631,7 @@ public static Tuple<string, string[]>[] ConvertToSweepArgumentStrings(TlcModule.
 }
 return results;
 }

 public static string GenerateOverallTrainingMetricVarName(Guid id) => $"Var_Training_OM_{id:N}";
 }
}

19 changes: 14 additions & 5 deletions
src/Microsoft.ML.PipelineInference/Macros/PipelineSweeperMacro.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -65,18 +65,24 @@ public static Output ExtractSweepResult(IHostEnvironment env, ResultInput input)
 var col1 = new KeyValuePair<string, ColumnType>("Graph", TextType.Instance);
 var col2 = new KeyValuePair<string, ColumnType>("MetricValue", PrimitiveType.FromKind(DataKind.R8));
 var col3 = new KeyValuePair<string, ColumnType>("PipelineId", TextType.Instance);
 var col4 = new KeyValuePair<string, ColumnType>("TrainingMetricValue", PrimitiveType.FromKind(DataKind.R8));
 var col5 = new KeyValuePair<string, ColumnType>("FirstInput", TextType.Instance);
 var col6 = new KeyValuePair<string, ColumnType>("PredictorModel", TextType.Instance);

 if (rows.Count == 0)
 {
 var host = env.Register("ExtractSweepResult");
 outputView = new EmptyDataView(host, new SimpleSchema(host, col1, col2, col3));
 outputView = new EmptyDataView(host, new SimpleSchema(host, col1, col2, col3, col4, col5, col6));
 }
 else
 {
 var builder = new ArrayDataViewBuilder(env);
 builder.AddColumn(col1.Key, (PrimitiveType)col1.Value, rows.Select(r => new DvText(r.GraphJson)).ToArray());
 builder.AddColumn(col2.Key, (PrimitiveType)col2.Value, rows.Select(r => r.MetricValue).ToArray());
 builder.AddColumn(col3.Key, (PrimitiveType)col3.Value, rows.Select(r => new DvText(r.PipelineId)).ToArray());
 builder.AddColumn(col4.Key, (PrimitiveType)col4.Value, rows.Select(r => r.TrainingMetricValue).ToArray());
 builder.AddColumn(col5.Key, (PrimitiveType)col5.Value, rows.Select(r => new DvText(r.FirstInput)).ToArray());
 builder.AddColumn(col6.Key, (PrimitiveType)col6.Value, rows.Select(r => new DvText(r.PredictorModel)).ToArray());
 outputView = builder.GetDataView();
 }
 return new Output { Results = outputView, State = autoMlState };
Expand DownExpand Up@@ -132,11 +138,11 @@ public static CommonOutputs.MacroOutput<Output> PipelineSweep(
 // Extract performance summaries and assign to previous candidate pipelines.
 foreach (var pipeline in autoMlState.BatchCandidates)
 {
 if (node.Context.TryGetVariable(ExperimentUtils.GenerateOverallMetricVarName(pipeline.UniqueId),
 out var v))
 if (node.Context.TryGetVariable(ExperimentUtils.GenerateOverallMetricVarName(pipeline.UniqueId), out var v) &&
node.Context.TryGetVariable(AutoMlUtils.GenerateOverallTrainingMetricVarName(pipeline.UniqueId), out var v2))
 {
 pipeline.PerformanceSummary =
 AutoMlUtils.ExtractRunSummary(env, (IDataView)v.Value, autoMlState.Metric.Name);
 AutoMlUtils.ExtractRunSummary(env, (IDataView)v.Value, autoMlState.Metric.Name, (IDataView)v2.Value);
 autoMlState.AddEvaluated(pipeline);
 }
 }
Expand DownExpand Up@@ -168,14 +174,17 @@ public static CommonOutputs.MacroOutput<Output> PipelineSweep(
 {
 // Add train test experiments to current graph for candidate pipeline
 var subgraph = new Experiment(env);
 var trainTestOutput = p.AddAsTrainTest(training, testing, autoMlState.TrainerKind, subgraph);
 var trainTestOutput = p.AddAsTrainTest(training, testing, autoMlState.TrainerKind, subgraph, true);

 // Change variable name to reference pipeline ID in output map, context and entrypoint output.
 var uniqueName = ExperimentUtils.GenerateOverallMetricVarName(p.UniqueId);
 var uniqueNameTraining = AutoMlUtils.GenerateOverallTrainingMetricVarName(p.UniqueId);
 var sgNode = EntryPointNode.ValidateNodes(env, node.Context,
 new JArray(subgraph.GetNodes().Last()), node.Catalog).Last();
 sgNode.RenameOutputVariable(trainTestOutput.OverallMetrics.VarName, uniqueName, cascadeChanges: true);
 sgNode.RenameOutputVariable(trainTestOutput.TrainingOverallMetrics.VarName, uniqueNameTraining, cascadeChanges: true);
 trainTestOutput.OverallMetrics.VarName = uniqueName;
 trainTestOutput.TrainingOverallMetrics.VarName = uniqueNameTraining;
 expNodes.Add(sgNode);

 // Store indicators, to pass to next iteration of macro.
Expand Down

1 change: 1 addition & 0 deletions
src/Microsoft.ML.PipelineInference/Microsoft.ML.PipelineInference.csproj

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -17,6 +17,7 @@
 <ProjectReference Include="..\Microsoft.ML.Core\Microsoft.ML.Core.csproj" />
 <ProjectReference Include="..\Microsoft.ML.StandardLearners\Microsoft.ML.StandardLearners.csproj" />
 <ProjectReference Include="..\Microsoft.ML.Sweeper\Microsoft.ML.Sweeper.csproj" />
 <ProjectReference Include="..\Microsoft.ML\Microsoft.ML.csproj" />
 </ItemGroup>

</Project>
Loading