[AutoML] Rev AutoML public API; add required native references to AutoML projects#3364

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
daholste merged 9 commits into from
Apr 16, 2019
Merged

[AutoML] Rev AutoML public API; add required native references to AutoML projects #3364

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
6f0ea50
Convert CacheBeforeTrainer from nullable bool to enum
daholste Apr 16, 2019
e2aa6b9
Add ExperimentResult & CrossValidationExperimentResult classes
daholste Apr 16, 2019
966f704
AutoML test updates
daholste Apr 16, 2019
82572be
Updates for CLI
daholste Apr 16, 2019
8dcce53
In AutoML SDK, write logs to channel in MLContext
daholste Apr 16, 2019
14d2ee4
Consume AutoML SDK logs from new stream in CLI
daholste Apr 16, 2019
d799b0a
Include native references in required projects
daholste Apr 16, 2019
4b40e49
whitespace fix
daholste Apr 16, 2019
741d1c2
Unit test fixes
daholste Apr 16, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 5 additions & 27 deletions src/Microsoft.ML.Auto/API/BinaryClassificationExperiment.cs
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -132,7 +132,7 @@ public enum BinaryClassificationTrainer
/// <summary>
/// AutoML experiment on binary classification datasets.
/// </summary>
public sealed class BinaryClassificationExperiment : ExperimentBase<BinaryClassificationMetrics>
public sealed class BinaryClassificationExperiment : ExperimentBase<BinaryClassificationMetrics, BinaryExperimentSettings>
{
internal BinaryClassificationExperiment(MLContext context, BinaryExperimentSettings settings)
: base(context,
Expand All@@ -143,37 +143,15 @@ internal BinaryClassificationExperiment(MLContext context, BinaryExperimentSetti
TrainerExtensionUtil.GetTrainerNames(settings.Trainers))
{
}
}

/// <summary>
/// Extension methods that operate over binary experiment run results.
/// </summary>
public static class BinaryExperimentResultExtensions
{
/// <summary>
/// Select the best run from an enumeration of experiment runs.
/// </summary>
/// <param name="results">Enumeration of AutoML experiment run results.</param>
/// <param name="metric">Metric to consider when selecting the best run.</param>
/// <returns>The best experiment run.</returns>
public static RunDetail<BinaryClassificationMetrics> Best(this IEnumerable<RunDetail<BinaryClassificationMetrics>> results, BinaryClassificationMetric metric = BinaryClassificationMetric.Accuracy)
private protected override RunDetail<BinaryClassificationMetrics> GetBestRun(IEnumerable<RunDetail<BinaryClassificationMetrics>> results)
{
var metricsAgent = new BinaryMetricsAgent(null, metric);
var isMetricMaximizing = new OptimizingMetricInfo(metric).IsMaximizing;
return BestResultUtil.GetBestRun(results, metricsAgent, isMetricMaximizing);
return BestResultUtil.GetBestRun(results, MetricsAgent, OptimizingMetricInfo.IsMaximizing);
}

/// <summary>
/// Select the best run from an enumeration of experiment cross validation runs.
/// </summary>
/// <param name="results">Enumeration of AutoML experiment cross validation run results.</param>
/// <param name="metric">Metric to consider when selecting the best run.</param>
/// <returns>The best experiment run.</returns>
public static CrossValidationRunDetail<BinaryClassificationMetrics> Best(this IEnumerable<CrossValidationRunDetail<BinaryClassificationMetrics>> results, BinaryClassificationMetric metric = BinaryClassificationMetric.Accuracy)
private protected override CrossValidationRunDetail<BinaryClassificationMetrics> GetBestCrossValRun(IEnumerable<CrossValidationRunDetail<BinaryClassificationMetrics>> results)
{
var metricsAgent = new BinaryMetricsAgent(null, metric);
var isMetricMaximizing = new OptimizingMetricInfo(metric).IsMaximizing;
return BestResultUtil.GetBestRun(results, metricsAgent, isMetricMaximizing);
return BestResultUtil.GetBestRun(results, MetricsAgent, OptimizingMetricInfo.IsMaximizing);
}
}
}
4 changes: 2 additions & 2 deletions src/Microsoft.ML.Auto/API/ColumnInference.cs
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -28,7 +28,7 @@ public sealed class ColumnInferenceResults
/// <remarks>
/// <para>Contains the inferred purposes of each column. See <see cref="Auto.ColumnInformation"/> for more details.</para>
/// <para>This can be fed to the AutoML API when running an experiment.
/// See <typeref cref="ExperimentBase{TMetrics}.Execute(IDataView, ColumnInformation, IEstimator{ITransformer}, System.IProgress{RunDetail{TMetrics}})" />
/// See <typeref cref="ExperimentBase{TMetrics, TExperimentSettings}.Execute(IDataView, ColumnInformation, IEstimator{ITransformer}, System.IProgress{RunDetail{TMetrics}})" />
/// for example.</para>
/// </remarks>
public ColumnInformation ColumnInformation { get; internal set; } = new ColumnInformation();
Expand All@@ -42,7 +42,7 @@ public sealed class ColumnInferenceResults
/// it enumerates the dataset columns that AutoML should treat as categorical,
/// the columns AutoML should ignore, which column is the label, etc.</para>
/// <para><see cref="ColumnInformation"/> can be fed to the AutoML API when running an experiment.
/// See <typeref cref="ExperimentBase{TMetrics}.Execute(IDataView, ColumnInformation, IEstimator{ITransformer}, System.IProgress{RunDetail{TMetrics}})" />
/// See <typeref cref="ExperimentBase{TMetrics, TExperimentSettings}.Execute(IDataView, ColumnInformation, IEstimator{ITransformer}, System.IProgress{RunDetail{TMetrics}})" />
/// for example.</para>
/// </remarks>
public sealed class ColumnInformation
Expand Down
99 changes: 56 additions & 43 deletions src/Microsoft.ML.Auto/API/ExperimentBase.cs
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -12,27 +12,32 @@ namespace Microsoft.ML.Auto
/// (like <see cref="BinaryClassificationExperiment"/>) inherit from this class.
/// </summary>
/// <typeparam name="TMetrics">Metrics type used by task-specific AutoML experiments.</typeparam>
public abstract class ExperimentBase<TMetrics> where TMetrics : class
/// <typeparam name="TExperimentSettings">Experiment settings type.</typeparam>
public abstract class ExperimentBase<TMetrics, TExperimentSettings>
where TMetrics : class
where TExperimentSettings : ExperimentSettings
{
private protected readonly MLContext Context;
private protected readonly IMetricsAgent<TMetrics> MetricsAgent;
private protected readonly OptimizingMetricInfo OptimizingMetricInfo;
private protected readonly TExperimentSettings Settings;

private readonly IMetricsAgent<TMetrics> _metricsAgent;
private readonly OptimizingMetricInfo _optimizingMetricInfo;
private readonly ExperimentSettings _settings;
private readonly AutoMLLogger _logger;
private readonly TaskKind _task;
private readonly IEnumerable<TrainerName> _trainerWhitelist;

internal ExperimentBase(MLContext context,
IMetricsAgent<TMetrics> metricsAgent,
OptimizingMetricInfo optimizingMetricInfo,
ExperimentSettings settings,
TExperimentSettings settings,
TaskKind task,
IEnumerable<TrainerName> trainerWhitelist)
{
Context = context;
_metricsAgent = metricsAgent;
_optimizingMetricInfo = optimizingMetricInfo;
_settings = settings;
MetricsAgent = metricsAgent;
OptimizingMetricInfo = optimizingMetricInfo;
Settings = settings;
_logger = new AutoMLLogger(context);
_task = task;
_trainerWhitelist = trainerWhitelist;
}
Expand All@@ -53,12 +58,11 @@ internal ExperimentBase(MLContext context,
/// <see cref="IProgress{T}.Report(T)"/> after each model it produces during the
/// course of the experiment.
/// </param>
/// <returns>An enumeration of all the runs in an experiment. See <see cref="RunDetail{TMetrics}"/>
/// for more information on the contents of a run.</returns>
/// <returns>The experiment result.</returns>
/// <remarks>
/// Depending on the size of your data, the AutoML experiment could take a long time to execute.
/// </remarks>
public IEnumerable<RunDetail<TMetrics>> Execute(IDataView trainData, string labelColumnName = DefaultColumnNames.Label,
public ExperimentResult<TMetrics> Execute(IDataView trainData, string labelColumnName = DefaultColumnNames.Label,
string samplingKeyColumn = null, IEstimator<ITransformer> preFeaturizer = null, IProgress<RunDetail<TMetrics>> progressHandler = null)
{
var columnInformation = new ColumnInformation()
Expand All@@ -83,12 +87,11 @@ public IEnumerable<RunDetail<TMetrics>> Execute(IDataView trainData, string labe
/// <see cref="IProgress{T}.Report(T)"/> after each model it produces during the
/// course of the experiment.
/// </param>
/// <returns>An enumeration of all the runs in an experiment. See <see cref="RunDetail{TMetrics}"/>
/// for more information on the contents of a run.</returns>
/// <returns>The experiment result.</returns>
/// <remarks>
/// Depending on the size of your data, the AutoML experiment could take a long time to execute.
/// </remarks>
public IEnumerable<RunDetail<TMetrics>> Execute(IDataView trainData, ColumnInformation columnInformation,
public ExperimentResult<TMetrics> Execute(IDataView trainData, ColumnInformation columnInformation,
IEstimator<ITransformer> preFeaturizer = null, IProgress<RunDetail<TMetrics>> progressHandler = null)
{
// Cross val threshold for # of dataset rows --
Expand DownExpand Up@@ -126,12 +129,11 @@ public IEnumerable<RunDetail<TMetrics>> Execute(IDataView trainData, ColumnInfor
/// <see cref="IProgress{T}.Report(T)"/> after each model it produces during the
/// course of the experiment.
/// </param>
/// <returns>An enumeration of all the runs in an experiment. See <see cref="RunDetail{TMetrics}"/>
/// for more information on the contents of a run.</returns>
/// <returns>The experiment result.</returns>
/// <remarks>
/// Depending on the size of your data, the AutoML experiment could take a long time to execute.
/// </remarks>
public IEnumerable<RunDetail<TMetrics>> Execute(IDataView trainData, IDataView validationData, string labelColumnName = DefaultColumnNames.Label, IEstimator<ITransformer> preFeaturizer = null, IProgress<RunDetail<TMetrics>> progressHandler = null)
public ExperimentResult<TMetrics> Execute(IDataView trainData, IDataView validationData, string labelColumnName = DefaultColumnNames.Label, IEstimator<ITransformer> preFeaturizer = null, IProgress<RunDetail<TMetrics>> progressHandler = null)
{
var columnInformation = new ColumnInformation() { LabelColumnName = labelColumnName };
return Execute(trainData, validationData, columnInformation, preFeaturizer, progressHandler);
Expand All@@ -152,12 +154,11 @@ public IEnumerable<RunDetail<TMetrics>> Execute(IDataView trainData, IDataView v
/// <see cref="IProgress{T}.Report(T)"/> after each model it produces during the
/// course of the experiment.
/// </param>
/// <returns>An enumeration of all the runs in an experiment. See <see cref="RunDetail{TMetrics}"/>
/// for more information on the contents of a run.</returns>
/// <returns>The experiment result.</returns>
/// <remarks>
/// Depending on the size of your data, the AutoML experiment could take a long time to execute.
/// </remarks>
public IEnumerable<RunDetail<TMetrics>> Execute(IDataView trainData, IDataView validationData, ColumnInformation columnInformation, IEstimator<ITransformer> preFeaturizer = null, IProgress<RunDetail<TMetrics>> progressHandler = null)
public ExperimentResult<TMetrics> Execute(IDataView trainData, IDataView validationData, ColumnInformation columnInformation, IEstimator<ITransformer> preFeaturizer = null, IProgress<RunDetail<TMetrics>> progressHandler = null)
{
if (validationData == null)
{
Expand All@@ -183,12 +184,11 @@ public IEnumerable<RunDetail<TMetrics>> Execute(IDataView trainData, IDataView v
/// <see cref="IProgress{T}.Report(T)"/> after each model it produces during the
/// course of the experiment.
/// </param>
/// <returns>An enumeration of all the runs in an experiment. See <see cref="RunDetail{TMetrics}"/>
/// for more information on the contents of a run.</returns>
/// <returns>The cross validation experiment result.</returns>
/// <remarks>
/// Depending on the size of your data, the AutoML experiment could take a long time to execute.
/// </remarks>
public IEnumerable<CrossValidationRunDetail<TMetrics>> Execute(IDataView trainData, uint numberOfCVFolds, ColumnInformation columnInformation = null, IEstimator<ITransformer> preFeaturizer = null, IProgress<CrossValidationRunDetail<TMetrics>> progressHandler = null)
public CrossValidationExperimentResult<TMetrics> Execute(IDataView trainData, uint numberOfCVFolds, ColumnInformation columnInformation = null, IEstimator<ITransformer> preFeaturizer = null, IProgress<CrossValidationRunDetail<TMetrics>> progressHandler = null)
{
UserInputValidationUtil.ValidateNumberOfCVFoldsArg(numberOfCVFolds);
var splitResult = SplitUtil.CrossValSplit(Context, trainData, numberOfCVFolds, columnInformation?.SamplingKeyColumnName);
Expand All@@ -211,12 +211,11 @@ public IEnumerable<CrossValidationRunDetail<TMetrics>> Execute(IDataView trainDa
/// <see cref="IProgress{T}.Report(T)"/> after each model it produces during the
/// course of the experiment.
/// </param>
/// <returns>An enumeration of all the runs in an experiment. See <see cref="RunDetail{TMetrics}"/>
/// for more information on the contents of a run.</returns>
/// <returns>The cross validation experiment result.</returns>
/// <remarks>
/// Depending on the size of your data, the AutoML experiment could take a long time to execute.
/// </remarks>
public IEnumerable<CrossValidationRunDetail<TMetrics>> Execute(IDataView trainData,
public CrossValidationExperimentResult<TMetrics> Execute(IDataView trainData,
uint numberOfCVFolds, string labelColumnName = DefaultColumnNames.Label,
string samplingKeyColumn = null, IEstimator<ITransformer> preFeaturizer = null,
Progress<CrossValidationRunDetail<TMetrics>> progressHandler = null)
Expand All@@ -229,7 +228,11 @@ public IEnumerable<CrossValidationRunDetail<TMetrics>> Execute(IDataView trainDa
return Execute(trainData, numberOfCVFolds, columnInformation, preFeaturizer, progressHandler);
}

private IEnumerable<RunDetail<TMetrics>> ExecuteTrainValidate(IDataView trainData,
private protected abstract CrossValidationRunDetail<TMetrics> GetBestCrossValRun(IEnumerable<CrossValidationRunDetail<TMetrics>> results);

private protected abstract RunDetail<TMetrics> GetBestRun(IEnumerable<RunDetail<TMetrics>> results);

private ExperimentResult<TMetrics> ExecuteTrainValidate(IDataView trainData,
ColumnInformation columnInfo,
IDataView validationData,
IEstimator<ITransformer> preFeaturizer,
Expand All@@ -247,13 +250,13 @@ private IEnumerable<RunDetail<TMetrics>> ExecuteTrainValidate(IDataView trainDat
validationData = preprocessorTransform.Transform(validationData);
}

var runner = new TrainValidateRunner<TMetrics>(Context, trainData, validationData, columnInfo.LabelColumnName, _metricsAgent,
preFeaturizer, preprocessorTransform, _settings.DebugLogger);
var runner = new TrainValidateRunner<TMetrics>(Context, trainData, validationData, columnInfo.LabelColumnName, MetricsAgent,
preFeaturizer, preprocessorTransform, _logger);
var columns = DatasetColumnInfoUtil.GetDatasetColumnInfo(Context, trainData, columnInfo);
return Execute(columnInfo, columns, preFeaturizer, progressHandler, runner);
}

private IEnumerable<CrossValidationRunDetail<TMetrics>> ExecuteCrossVal(IDataView[] trainDatasets,
private CrossValidationExperimentResult<TMetrics> ExecuteCrossVal(IDataView[] trainDatasets,
ColumnInformation columnInfo,
IDataView[] validationDatasets,
IEstimator<ITransformer> preFeaturizer,
Expand All@@ -266,13 +269,21 @@ private IEnumerable<CrossValidationRunDetail<TMetrics>> ExecuteCrossVal(IDataVie
ITransformer[] preprocessorTransforms = null;
(trainDatasets, validationDatasets, preprocessorTransforms) = ApplyPreFeaturizerCrossVal(trainDatasets, validationDatasets, preFeaturizer);

var runner = new CrossValRunner<TMetrics>(Context, trainDatasets, validationDatasets, _metricsAgent, preFeaturizer,
preprocessorTransforms, columnInfo.LabelColumnName, _settings.DebugLogger);
var runner = new CrossValRunner<TMetrics>(Context, trainDatasets, validationDatasets, MetricsAgent, preFeaturizer,
preprocessorTransforms, columnInfo.LabelColumnName, _logger);
var columns = DatasetColumnInfoUtil.GetDatasetColumnInfo(Context, trainDatasets[0], columnInfo);
return Execute(columnInfo, columns, preFeaturizer, progressHandler, runner);

// Execute experiment & get all pipelines run
var experiment = new Experiment<CrossValidationRunDetail<TMetrics>, TMetrics>(Context, _task, OptimizingMetricInfo, progressHandler,
Settings, MetricsAgent, _trainerWhitelist, columns, runner, _logger);
var runDetails = experiment.Execute();

var bestRun = GetBestCrossValRun(runDetails);
var experimentResult = new CrossValidationExperimentResult<TMetrics>(runDetails, bestRun);
return experimentResult;
}

private IEnumerable<RunDetail<TMetrics>> ExecuteCrossValSummary(IDataView[] trainDatasets,
private ExperimentResult<TMetrics> ExecuteCrossValSummary(IDataView[] trainDatasets,
ColumnInformation columnInfo,
IDataView[] validationDatasets,
IEstimator<ITransformer> preFeaturizer,
Expand All@@ -285,24 +296,26 @@ private IEnumerable<RunDetail<TMetrics>> ExecuteCrossValSummary(IDataView[] trai
ITransformer[] preprocessorTransforms = null;
(trainDatasets, validationDatasets, preprocessorTransforms) = ApplyPreFeaturizerCrossVal(trainDatasets, validationDatasets, preFeaturizer);

var runner = new CrossValSummaryRunner<TMetrics>(Context, trainDatasets, validationDatasets, _metricsAgent, preFeaturizer,
preprocessorTransforms, columnInfo.LabelColumnName, _optimizingMetricInfo, _settings.DebugLogger);
var runner = new CrossValSummaryRunner<TMetrics>(Context, trainDatasets, validationDatasets, MetricsAgent, preFeaturizer,
preprocessorTransforms, columnInfo.LabelColumnName, OptimizingMetricInfo, _logger);
var columns = DatasetColumnInfoUtil.GetDatasetColumnInfo(Context, trainDatasets[0], columnInfo);
return Execute(columnInfo, columns, preFeaturizer, progressHandler, runner);
}

private IEnumerable<TRunDetail> Execute<TRunDetail>(ColumnInformation columnInfo,
private ExperimentResult<TMetrics> Execute(ColumnInformation columnInfo,
DatasetColumnInfo[] columns,
IEstimator<ITransformer> preFeaturizer,
IProgress<TRunDetail> progressHandler,
IRunner<TRunDetail> runner)
where TRunDetail : RunDetail
IProgress<RunDetail<TMetrics>> progressHandler,
IRunner<RunDetail<TMetrics>> runner)
{
// Execute experiment & get all pipelines run
var experiment = new Experiment<TRunDetail, TMetrics>(Context, _task, _optimizingMetricInfo, progressHandler,
_settings, _metricsAgent, _trainerWhitelist, columns, runner);
var experiment = new Experiment<RunDetail<TMetrics>, TMetrics>(Context, _task, OptimizingMetricInfo, progressHandler,
Settings, MetricsAgent, _trainerWhitelist, columns, runner, _logger);
var runDetails = experiment.Execute();

return experiment.Execute();
var bestRun = GetBestRun(runDetails);
var experimentResult = new ExperimentResult<TMetrics>(runDetails, bestRun);
return experimentResult;
}

private static (IDataView[] trainDatasets, IDataView[] validDatasets, ITransformer[] preprocessorTransforms)
Expand Down
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Add copy buttons to all 
 blocks\n(function() {\n function addCopyButtons() {\n document.querySelectorAll('pre code').forEach(function(codeBlock) {\n if (codeBlock.parentElement.hasAttribute('data-copy-added')) return;\n codeBlock.parentElement.setAttribute('data-copy-added', 'true');\n \n var btn = document.createElement('button');\n btn.textContent = 'Copy';\n btn.style.cssText = 'position:absolute;top:4px;right:4px;padding:2px 8px;font-size:11px;background:#4ecdc4;border:none;border-radius:4px;color:#1a1a2e;cursor:pointer;opacity:0.7;transition:opacity 0.2s;';\n btn.onmouseover = function() { this.style.opacity = '1'; };\n btn.onmouseout = function() { this.style.opacity = '0.7'; };\n btn.onclick = function() {\n navigator.clipboard.writeText(codeBlock.textContent).then(function() {\n btn.textContent = 'Copied!';\n setTimeout(function() { btn.textContent = 'Copy'; }, 1500);\n });\n };\n codeBlock.parentElement.style.position = 'relative';\n codeBlock.parentElement.appendChild(btn);\n });\n }\n \n addCopyButtons();\n \n // Re-run on dynamic content\n var observer = new MutationObserver(addCopyButtons);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Add Copy Buttons to Code Blocks");
}
} catch(__e) { console.warn('[Userscript:Add Copy Buttons to Code Blocks]', __e); }
})();
(function(){
try {
var __m = "github.com";
var __re = new RegExp('^' + "github\\.com" + '

Skip to content
[AutoML] Rev AutoML public API; add required native references to AutoML projects#3364

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

daholste
merged 9 commits into

from

Apr 16, 2019
 Merged


[AutoML] Rev AutoML public API; add required native references to AutoML projects
#3364
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


9 commits


Select commit
Hold shift + click to select a range
6f0ea50

Convert CacheBeforeTrainer from nullable bool to enum


daholste Apr 16, 2019
e2aa6b9

Add ExperimentResult & CrossValidationExperimentResult classes


daholste Apr 16, 2019
966f704

AutoML test updates


daholste Apr 16, 2019
82572be

Updates for CLI


daholste Apr 16, 2019
8dcce53

In AutoML SDK, write logs to channel in MLContext


daholste Apr 16, 2019
14d2ee4

Consume AutoML SDK logs from new stream in CLI


daholste Apr 16, 2019
d799b0a

Include native references in required projects


daholste Apr 16, 2019
4b40e49

whitespace fix


daholste Apr 16, 2019
741d1c2

Unit test fixes


daholste Apr 16, 2019

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


32 changes: 5 additions & 27 deletions
src/Microsoft.ML.Auto/API/BinaryClassificationExperiment.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -132,7 +132,7 @@ public enum BinaryClassificationTrainer
 /// <summary>
 /// AutoML experiment on binary classification datasets.
 /// </summary>
 public sealed class BinaryClassificationExperiment : ExperimentBase<BinaryClassificationMetrics>
 public sealed class BinaryClassificationExperiment : ExperimentBase<BinaryClassificationMetrics, BinaryExperimentSettings>
 {
 internal BinaryClassificationExperiment(MLContext context, BinaryExperimentSettings settings)
 : base(context,
Expand All@@ -143,37 +143,15 @@ internal BinaryClassificationExperiment(MLContext context, BinaryExperimentSetti
 TrainerExtensionUtil.GetTrainerNames(settings.Trainers))
 {
 }
 }

 /// <summary>
 /// Extension methods that operate over binary experiment run results.
 /// </summary>
 public static class BinaryExperimentResultExtensions
 {
 /// <summary>
 /// Select the best run from an enumeration of experiment runs.
 /// </summary>
 /// <param name="results">Enumeration of AutoML experiment run results.</param>
 /// <param name="metric">Metric to consider when selecting the best run.</param>
 /// <returns>The best experiment run.</returns>
 public static RunDetail<BinaryClassificationMetrics> Best(this IEnumerable<RunDetail<BinaryClassificationMetrics>> results, BinaryClassificationMetric metric = BinaryClassificationMetric.Accuracy)
 private protected override RunDetail<BinaryClassificationMetrics> GetBestRun(IEnumerable<RunDetail<BinaryClassificationMetrics>> results)
 {
 var metricsAgent = new BinaryMetricsAgent(null, metric);
 var isMetricMaximizing = new OptimizingMetricInfo(metric).IsMaximizing;
 return BestResultUtil.GetBestRun(results, metricsAgent, isMetricMaximizing);
 return BestResultUtil.GetBestRun(results, MetricsAgent, OptimizingMetricInfo.IsMaximizing);
 }

 /// <summary>
 /// Select the best run from an enumeration of experiment cross validation runs.
 /// </summary>
 /// <param name="results">Enumeration of AutoML experiment cross validation run results.</param>
 /// <param name="metric">Metric to consider when selecting the best run.</param>
 /// <returns>The best experiment run.</returns>
 public static CrossValidationRunDetail<BinaryClassificationMetrics> Best(this IEnumerable<CrossValidationRunDetail<BinaryClassificationMetrics>> results, BinaryClassificationMetric metric = BinaryClassificationMetric.Accuracy)
 private protected override CrossValidationRunDetail<BinaryClassificationMetrics> GetBestCrossValRun(IEnumerable<CrossValidationRunDetail<BinaryClassificationMetrics>> results)
 {
 var metricsAgent = new BinaryMetricsAgent(null, metric);
 var isMetricMaximizing = new OptimizingMetricInfo(metric).IsMaximizing;
 return BestResultUtil.GetBestRun(results, metricsAgent, isMetricMaximizing);
 return BestResultUtil.GetBestRun(results, MetricsAgent, OptimizingMetricInfo.IsMaximizing);
 }
 }
}

4 changes: 2 additions & 2 deletions
src/Microsoft.ML.Auto/API/ColumnInference.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -28,7 +28,7 @@ public sealed class ColumnInferenceResults
 /// <remarks>
 /// <para>Contains the inferred purposes of each column. See <see cref="Auto.ColumnInformation"/> for more details.</para>
 /// <para>This can be fed to the AutoML API when running an experiment. 
 /// See <typeref cref="ExperimentBase{TMetrics}.Execute(IDataView, ColumnInformation, IEstimator{ITransformer}, System.IProgress{RunDetail{TMetrics}})" />
 /// See <typeref cref="ExperimentBase{TMetrics, TExperimentSettings}.Execute(IDataView, ColumnInformation, IEstimator{ITransformer}, System.IProgress{RunDetail{TMetrics}})" />
 /// for example.</para>
 /// </remarks>
 public ColumnInformation ColumnInformation { get; internal set; } = new ColumnInformation();
Expand All@@ -42,7 +42,7 @@ public sealed class ColumnInferenceResults
 /// it enumerates the dataset columns that AutoML should treat as categorical, 
 /// the columns AutoML should ignore, which column is the label, etc.</para>
 /// <para><see cref="ColumnInformation"/> can be fed to the AutoML API when running an experiment.
 /// See <typeref cref="ExperimentBase{TMetrics}.Execute(IDataView, ColumnInformation, IEstimator{ITransformer}, System.IProgress{RunDetail{TMetrics}})" />
 /// See <typeref cref="ExperimentBase{TMetrics, TExperimentSettings}.Execute(IDataView, ColumnInformation, IEstimator{ITransformer}, System.IProgress{RunDetail{TMetrics}})" />
 /// for example.</para>
 /// </remarks>
 public sealed class ColumnInformation
Expand Down

99 changes: 56 additions & 43 deletions
src/Microsoft.ML.Auto/API/ExperimentBase.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -12,27 +12,32 @@ namespace Microsoft.ML.Auto
 /// (like <see cref="BinaryClassificationExperiment"/>) inherit from this class.
 /// </summary>
 /// <typeparam name="TMetrics">Metrics type used by task-specific AutoML experiments.</typeparam>
 public abstract class ExperimentBase<TMetrics> where TMetrics : class
 /// <typeparam name="TExperimentSettings">Experiment settings type.</typeparam>
 public abstract class ExperimentBase<TMetrics, TExperimentSettings> 
 where TMetrics : class
 where TExperimentSettings : ExperimentSettings
 {
 private protected readonly MLContext Context;
 private protected readonly IMetricsAgent<TMetrics> MetricsAgent;
 private protected readonly OptimizingMetricInfo OptimizingMetricInfo;
 private protected readonly TExperimentSettings Settings;

 private readonly IMetricsAgent<TMetrics> _metricsAgent;
 private readonly OptimizingMetricInfo _optimizingMetricInfo;
 private readonly ExperimentSettings _settings;
 private readonly AutoMLLogger _logger;
 private readonly TaskKind _task;
 private readonly IEnumerable<TrainerName> _trainerWhitelist;

 internal ExperimentBase(MLContext context,
 IMetricsAgent<TMetrics> metricsAgent,
 OptimizingMetricInfo optimizingMetricInfo,
ExperimentSettings settings,
TExperimentSettings settings,
 TaskKind task,
 IEnumerable<TrainerName> trainerWhitelist)
 {
 Context = context;
 _metricsAgent = metricsAgent;
 _optimizingMetricInfo = optimizingMetricInfo;
 _settings = settings;
 MetricsAgent = metricsAgent;
 OptimizingMetricInfo = optimizingMetricInfo;
 Settings = settings;
 _logger = new AutoMLLogger(context);
 _task = task;
 _trainerWhitelist = trainerWhitelist;
 }
Expand All@@ -53,12 +58,11 @@ internal ExperimentBase(MLContext context,
 /// <see cref="IProgress{T}.Report(T)"/> after each model it produces during the 
 /// course of the experiment.
 /// </param>
 /// <returns>An enumeration of all the runs in an experiment. See <see cref="RunDetail{TMetrics}"/>
 /// for more information on the contents of a run.</returns>
 /// <returns>The experiment result.</returns>
 /// <remarks>
 /// Depending on the size of your data, the AutoML experiment could take a long time to execute.
 /// </remarks>
 public IEnumerable<RunDetail<TMetrics>> Execute(IDataView trainData, string labelColumnName = DefaultColumnNames.Label,
 public ExperimentResult<TMetrics> Execute(IDataView trainData, string labelColumnName = DefaultColumnNames.Label,
 string samplingKeyColumn = null, IEstimator<ITransformer> preFeaturizer = null, IProgress<RunDetail<TMetrics>> progressHandler = null)
 {
 var columnInformation = new ColumnInformation()
Expand All@@ -83,12 +87,11 @@ public IEnumerable<RunDetail<TMetrics>> Execute(IDataView trainData, string labe
 /// <see cref="IProgress{T}.Report(T)"/> after each model it produces during the 
 /// course of the experiment.
 /// </param>
 /// <returns>An enumeration of all the runs in an experiment. See <see cref="RunDetail{TMetrics}"/>
 /// for more information on the contents of a run.</returns>
 /// <returns>The experiment result.</returns>
 /// <remarks>
 /// Depending on the size of your data, the AutoML experiment could take a long time to execute.
 /// </remarks>
 public IEnumerable<RunDetail<TMetrics>> Execute(IDataView trainData, ColumnInformation columnInformation, 
 public ExperimentResult<TMetrics> Execute(IDataView trainData, ColumnInformation columnInformation, 
 IEstimator<ITransformer> preFeaturizer = null, IProgress<RunDetail<TMetrics>> progressHandler = null)
 {
 // Cross val threshold for # of dataset rows --
Expand DownExpand Up@@ -126,12 +129,11 @@ public IEnumerable<RunDetail<TMetrics>> Execute(IDataView trainData, ColumnInfor
 /// <see cref="IProgress{T}.Report(T)"/> after each model it produces during the 
 /// course of the experiment.
 /// </param>
 /// <returns>An enumeration of all the runs in an experiment. See <see cref="RunDetail{TMetrics}"/>
 /// for more information on the contents of a run.</returns>
 /// <returns>The experiment result.</returns>
 /// <remarks>
 /// Depending on the size of your data, the AutoML experiment could take a long time to execute.
 /// </remarks>
 public IEnumerable<RunDetail<TMetrics>> Execute(IDataView trainData, IDataView validationData, string labelColumnName = DefaultColumnNames.Label, IEstimator<ITransformer> preFeaturizer = null, IProgress<RunDetail<TMetrics>> progressHandler = null)
 public ExperimentResult<TMetrics> Execute(IDataView trainData, IDataView validationData, string labelColumnName = DefaultColumnNames.Label, IEstimator<ITransformer> preFeaturizer = null, IProgress<RunDetail<TMetrics>> progressHandler = null)
 {
 var columnInformation = new ColumnInformation() { LabelColumnName = labelColumnName };
 return Execute(trainData, validationData, columnInformation, preFeaturizer, progressHandler);
Expand All@@ -152,12 +154,11 @@ public IEnumerable<RunDetail<TMetrics>> Execute(IDataView trainData, IDataView v
 /// <see cref="IProgress{T}.Report(T)"/> after each model it produces during the 
 /// course of the experiment.
 /// </param>
 /// <returns>An enumeration of all the runs in an experiment. See <see cref="RunDetail{TMetrics}"/>
 /// for more information on the contents of a run.</returns>
 /// <returns>The experiment result.</returns>
 /// <remarks>
 /// Depending on the size of your data, the AutoML experiment could take a long time to execute.
 /// </remarks>
 public IEnumerable<RunDetail<TMetrics>> Execute(IDataView trainData, IDataView validationData, ColumnInformation columnInformation, IEstimator<ITransformer> preFeaturizer = null, IProgress<RunDetail<TMetrics>> progressHandler = null)
 public ExperimentResult<TMetrics> Execute(IDataView trainData, IDataView validationData, ColumnInformation columnInformation, IEstimator<ITransformer> preFeaturizer = null, IProgress<RunDetail<TMetrics>> progressHandler = null)
 {
 if (validationData == null)
 {
Expand All@@ -183,12 +184,11 @@ public IEnumerable<RunDetail<TMetrics>> Execute(IDataView trainData, IDataView v
 /// <see cref="IProgress{T}.Report(T)"/> after each model it produces during the 
 /// course of the experiment.
 /// </param>
 /// <returns>An enumeration of all the runs in an experiment. See <see cref="RunDetail{TMetrics}"/>
 /// for more information on the contents of a run.</returns>
 /// <returns>The cross validation experiment result.</returns>
 /// <remarks>
 /// Depending on the size of your data, the AutoML experiment could take a long time to execute.
 /// </remarks>
 public IEnumerable<CrossValidationRunDetail<TMetrics>> Execute(IDataView trainData, uint numberOfCVFolds, ColumnInformation columnInformation = null, IEstimator<ITransformer> preFeaturizer = null, IProgress<CrossValidationRunDetail<TMetrics>> progressHandler = null)
 public CrossValidationExperimentResult<TMetrics> Execute(IDataView trainData, uint numberOfCVFolds, ColumnInformation columnInformation = null, IEstimator<ITransformer> preFeaturizer = null, IProgress<CrossValidationRunDetail<TMetrics>> progressHandler = null)
 {
 UserInputValidationUtil.ValidateNumberOfCVFoldsArg(numberOfCVFolds);
 var splitResult = SplitUtil.CrossValSplit(Context, trainData, numberOfCVFolds, columnInformation?.SamplingKeyColumnName);
Expand All@@ -211,12 +211,11 @@ public IEnumerable<CrossValidationRunDetail<TMetrics>> Execute(IDataView trainDa
 /// <see cref="IProgress{T}.Report(T)"/> after each model it produces during the 
 /// course of the experiment.
 /// </param>
 /// <returns>An enumeration of all the runs in an experiment. See <see cref="RunDetail{TMetrics}"/>
 /// for more information on the contents of a run.</returns>
 /// <returns>The cross validation experiment result.</returns>
 /// <remarks>
 /// Depending on the size of your data, the AutoML experiment could take a long time to execute.
 /// </remarks>
 public IEnumerable<CrossValidationRunDetail<TMetrics>> Execute(IDataView trainData, 
 public CrossValidationExperimentResult<TMetrics> Execute(IDataView trainData, 
 uint numberOfCVFolds, string labelColumnName = DefaultColumnNames.Label,
 string samplingKeyColumn = null, IEstimator<ITransformer> preFeaturizer = null, 
 Progress<CrossValidationRunDetail<TMetrics>> progressHandler = null)
Expand All@@ -229,7 +228,11 @@ public IEnumerable<CrossValidationRunDetail<TMetrics>> Execute(IDataView trainDa
 return Execute(trainData, numberOfCVFolds, columnInformation, preFeaturizer, progressHandler);
 }

 private IEnumerable<RunDetail<TMetrics>> ExecuteTrainValidate(IDataView trainData,
 private protected abstract CrossValidationRunDetail<TMetrics> GetBestCrossValRun(IEnumerable<CrossValidationRunDetail<TMetrics>> results);

 private protected abstract RunDetail<TMetrics> GetBestRun(IEnumerable<RunDetail<TMetrics>> results);

 private ExperimentResult<TMetrics> ExecuteTrainValidate(IDataView trainData,
 ColumnInformation columnInfo,
 IDataView validationData,
 IEstimator<ITransformer> preFeaturizer,
Expand All@@ -247,13 +250,13 @@ private IEnumerable<RunDetail<TMetrics>> ExecuteTrainValidate(IDataView trainDat
 validationData = preprocessorTransform.Transform(validationData);
 }

 var runner = new TrainValidateRunner<TMetrics>(Context, trainData, validationData, columnInfo.LabelColumnName, _metricsAgent,
 preFeaturizer, preprocessorTransform, _settings.DebugLogger);
 var runner = new TrainValidateRunner<TMetrics>(Context, trainData, validationData, columnInfo.LabelColumnName, MetricsAgent,
 preFeaturizer, preprocessorTransform, _logger);
 var columns = DatasetColumnInfoUtil.GetDatasetColumnInfo(Context, trainData, columnInfo);
 return Execute(columnInfo, columns, preFeaturizer, progressHandler, runner);
 }

 private IEnumerable<CrossValidationRunDetail<TMetrics>> ExecuteCrossVal(IDataView[] trainDatasets,
 private CrossValidationExperimentResult<TMetrics> ExecuteCrossVal(IDataView[] trainDatasets,
 ColumnInformation columnInfo,
 IDataView[] validationDatasets,
 IEstimator<ITransformer> preFeaturizer,
Expand All@@ -266,13 +269,21 @@ private IEnumerable<CrossValidationRunDetail<TMetrics>> ExecuteCrossVal(IDataVie
 ITransformer[] preprocessorTransforms = null;
 (trainDatasets, validationDatasets, preprocessorTransforms) = ApplyPreFeaturizerCrossVal(trainDatasets, validationDatasets, preFeaturizer);

 var runner = new CrossValRunner<TMetrics>(Context, trainDatasets, validationDatasets, _metricsAgent, preFeaturizer,
 preprocessorTransforms, columnInfo.LabelColumnName, _settings.DebugLogger);
 var runner = new CrossValRunner<TMetrics>(Context, trainDatasets, validationDatasets, MetricsAgent, preFeaturizer,
 preprocessorTransforms, columnInfo.LabelColumnName, _logger);
 var columns = DatasetColumnInfoUtil.GetDatasetColumnInfo(Context, trainDatasets[0], columnInfo);
 return Execute(columnInfo, columns, preFeaturizer, progressHandler, runner);

 // Execute experiment & get all pipelines run
 var experiment = new Experiment<CrossValidationRunDetail<TMetrics>, TMetrics>(Context, _task, OptimizingMetricInfo, progressHandler,
 Settings, MetricsAgent, _trainerWhitelist, columns, runner, _logger);
 var runDetails = experiment.Execute();

 var bestRun = GetBestCrossValRun(runDetails);
 var experimentResult = new CrossValidationExperimentResult<TMetrics>(runDetails, bestRun);
 return experimentResult;
 }

 private IEnumerable<RunDetail<TMetrics>> ExecuteCrossValSummary(IDataView[] trainDatasets,
 private ExperimentResult<TMetrics> ExecuteCrossValSummary(IDataView[] trainDatasets,
 ColumnInformation columnInfo,
 IDataView[] validationDatasets,
 IEstimator<ITransformer> preFeaturizer,
Expand All@@ -285,24 +296,26 @@ private IEnumerable<RunDetail<TMetrics>> ExecuteCrossValSummary(IDataView[] trai
 ITransformer[] preprocessorTransforms = null;
 (trainDatasets, validationDatasets, preprocessorTransforms) = ApplyPreFeaturizerCrossVal(trainDatasets, validationDatasets, preFeaturizer);

 var runner = new CrossValSummaryRunner<TMetrics>(Context, trainDatasets, validationDatasets, _metricsAgent, preFeaturizer,
 preprocessorTransforms, columnInfo.LabelColumnName, _optimizingMetricInfo, _settings.DebugLogger);
 var runner = new CrossValSummaryRunner<TMetrics>(Context, trainDatasets, validationDatasets, MetricsAgent, preFeaturizer,
 preprocessorTransforms, columnInfo.LabelColumnName, OptimizingMetricInfo, _logger);
 var columns = DatasetColumnInfoUtil.GetDatasetColumnInfo(Context, trainDatasets[0], columnInfo);
 return Execute(columnInfo, columns, preFeaturizer, progressHandler, runner);
 }

 private IEnumerable<TRunDetail> Execute<TRunDetail>(ColumnInformation columnInfo,
 private ExperimentResult<TMetrics> Execute(ColumnInformation columnInfo,
 DatasetColumnInfo[] columns,
 IEstimator<ITransformer> preFeaturizer,
 IProgress<TRunDetail> progressHandler,
 IRunner<TRunDetail> runner)
 where TRunDetail : RunDetail
 IProgress<RunDetail<TMetrics>> progressHandler,
 IRunner<RunDetail<TMetrics>> runner)
 {
 // Execute experiment & get all pipelines run
 var experiment = new Experiment<TRunDetail, TMetrics>(Context, _task, _optimizingMetricInfo, progressHandler,
 _settings, _metricsAgent, _trainerWhitelist, columns, runner);
 var experiment = new Experiment<RunDetail<TMetrics>, TMetrics>(Context, _task, OptimizingMetricInfo, progressHandler,
 Settings, MetricsAgent, _trainerWhitelist, columns, runner, _logger);
 var runDetails = experiment.Execute();

 return experiment.Execute();
 var bestRun = GetBestRun(runDetails);
 var experimentResult = new ExperimentResult<TMetrics>(runDetails, bestRun);
 return experimentResult;
 }

 private static (IDataView[] trainDatasets, IDataView[] validDatasets, ITransformer[] preprocessorTransforms)
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Force GitHub README to respect dark mode\n(function() {\n var style = document.createElement('style');\n style.textContent = '\n .markdown-body {\n color-scheme: dark light;\n }\n .markdown-body pre { background: #161b22 !important; }\n .markdown-body code { background: rgba(110, 118, 129, 0.4) !important; }\n .markdown-body table th, .markdown-body table td { border-color: #30363d !important; }\n .markdown-body img { background: #0d1117; }\n .markdown-body blockquote { border-left-color: #8b949e; }\n .markdown-body hr { border-color: #30363d; }\n ';\n document.head.appendChild(style);\n})();", "GitHub Dark Mode README Fix");
}
} catch(__e) { console.warn('[Userscript:GitHub Dark Mode README Fix]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
[AutoML] Rev AutoML public API; add required native references to AutoML projects#3364

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

daholste
merged 9 commits into

from

Apr 16, 2019
 Merged


[AutoML] Rev AutoML public API; add required native references to AutoML projects
#3364
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


9 commits


Select commit
Hold shift + click to select a range
6f0ea50

Convert CacheBeforeTrainer from nullable bool to enum


daholste Apr 16, 2019
e2aa6b9

Add ExperimentResult & CrossValidationExperimentResult classes


daholste Apr 16, 2019
966f704

AutoML test updates


daholste Apr 16, 2019
82572be

Updates for CLI


daholste Apr 16, 2019
8dcce53

In AutoML SDK, write logs to channel in MLContext


daholste Apr 16, 2019
14d2ee4

Consume AutoML SDK logs from new stream in CLI


daholste Apr 16, 2019
d799b0a

Include native references in required projects


daholste Apr 16, 2019
4b40e49

whitespace fix


daholste Apr 16, 2019
741d1c2

Unit test fixes


daholste Apr 16, 2019

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


32 changes: 5 additions & 27 deletions
src/Microsoft.ML.Auto/API/BinaryClassificationExperiment.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -132,7 +132,7 @@ public enum BinaryClassificationTrainer
 /// <summary>
 /// AutoML experiment on binary classification datasets.
 /// </summary>
 public sealed class BinaryClassificationExperiment : ExperimentBase<BinaryClassificationMetrics>
 public sealed class BinaryClassificationExperiment : ExperimentBase<BinaryClassificationMetrics, BinaryExperimentSettings>
 {
 internal BinaryClassificationExperiment(MLContext context, BinaryExperimentSettings settings)
 : base(context,
Expand All@@ -143,37 +143,15 @@ internal BinaryClassificationExperiment(MLContext context, BinaryExperimentSetti
 TrainerExtensionUtil.GetTrainerNames(settings.Trainers))
 {
 }
 }

 /// <summary>
 /// Extension methods that operate over binary experiment run results.
 /// </summary>
 public static class BinaryExperimentResultExtensions
 {
 /// <summary>
 /// Select the best run from an enumeration of experiment runs.
 /// </summary>
 /// <param name="results">Enumeration of AutoML experiment run results.</param>
 /// <param name="metric">Metric to consider when selecting the best run.</param>
 /// <returns>The best experiment run.</returns>
 public static RunDetail<BinaryClassificationMetrics> Best(this IEnumerable<RunDetail<BinaryClassificationMetrics>> results, BinaryClassificationMetric metric = BinaryClassificationMetric.Accuracy)
 private protected override RunDetail<BinaryClassificationMetrics> GetBestRun(IEnumerable<RunDetail<BinaryClassificationMetrics>> results)
 {
 var metricsAgent = new BinaryMetricsAgent(null, metric);
 var isMetricMaximizing = new OptimizingMetricInfo(metric).IsMaximizing;
 return BestResultUtil.GetBestRun(results, metricsAgent, isMetricMaximizing);
 return BestResultUtil.GetBestRun(results, MetricsAgent, OptimizingMetricInfo.IsMaximizing);
 }

 /// <summary>
 /// Select the best run from an enumeration of experiment cross validation runs.
 /// </summary>
 /// <param name="results">Enumeration of AutoML experiment cross validation run results.</param>
 /// <param name="metric">Metric to consider when selecting the best run.</param>
 /// <returns>The best experiment run.</returns>
 public static CrossValidationRunDetail<BinaryClassificationMetrics> Best(this IEnumerable<CrossValidationRunDetail<BinaryClassificationMetrics>> results, BinaryClassificationMetric metric = BinaryClassificationMetric.Accuracy)
 private protected override CrossValidationRunDetail<BinaryClassificationMetrics> GetBestCrossValRun(IEnumerable<CrossValidationRunDetail<BinaryClassificationMetrics>> results)
 {
 var metricsAgent = new BinaryMetricsAgent(null, metric);
 var isMetricMaximizing = new OptimizingMetricInfo(metric).IsMaximizing;
 return BestResultUtil.GetBestRun(results, metricsAgent, isMetricMaximizing);
 return BestResultUtil.GetBestRun(results, MetricsAgent, OptimizingMetricInfo.IsMaximizing);
 }
 }
}

4 changes: 2 additions & 2 deletions
src/Microsoft.ML.Auto/API/ColumnInference.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -28,7 +28,7 @@ public sealed class ColumnInferenceResults
 /// <remarks>
 /// <para>Contains the inferred purposes of each column. See <see cref="Auto.ColumnInformation"/> for more details.</para>
 /// <para>This can be fed to the AutoML API when running an experiment. 
 /// See <typeref cref="ExperimentBase{TMetrics}.Execute(IDataView, ColumnInformation, IEstimator{ITransformer}, System.IProgress{RunDetail{TMetrics}})" />
 /// See <typeref cref="ExperimentBase{TMetrics, TExperimentSettings}.Execute(IDataView, ColumnInformation, IEstimator{ITransformer}, System.IProgress{RunDetail{TMetrics}})" />
 /// for example.</para>
 /// </remarks>
 public ColumnInformation ColumnInformation { get; internal set; } = new ColumnInformation();
Expand All@@ -42,7 +42,7 @@ public sealed class ColumnInferenceResults
 /// it enumerates the dataset columns that AutoML should treat as categorical, 
 /// the columns AutoML should ignore, which column is the label, etc.</para>
 /// <para><see cref="ColumnInformation"/> can be fed to the AutoML API when running an experiment.
 /// See <typeref cref="ExperimentBase{TMetrics}.Execute(IDataView, ColumnInformation, IEstimator{ITransformer}, System.IProgress{RunDetail{TMetrics}})" />
 /// See <typeref cref="ExperimentBase{TMetrics, TExperimentSettings}.Execute(IDataView, ColumnInformation, IEstimator{ITransformer}, System.IProgress{RunDetail{TMetrics}})" />
 /// for example.</para>
 /// </remarks>
 public sealed class ColumnInformation
Expand Down

99 changes: 56 additions & 43 deletions
src/Microsoft.ML.Auto/API/ExperimentBase.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -12,27 +12,32 @@ namespace Microsoft.ML.Auto
 /// (like <see cref="BinaryClassificationExperiment"/>) inherit from this class.
 /// </summary>
 /// <typeparam name="TMetrics">Metrics type used by task-specific AutoML experiments.</typeparam>
 public abstract class ExperimentBase<TMetrics> where TMetrics : class
 /// <typeparam name="TExperimentSettings">Experiment settings type.</typeparam>
 public abstract class ExperimentBase<TMetrics, TExperimentSettings> 
 where TMetrics : class
 where TExperimentSettings : ExperimentSettings
 {
 private protected readonly MLContext Context;
 private protected readonly IMetricsAgent<TMetrics> MetricsAgent;
 private protected readonly OptimizingMetricInfo OptimizingMetricInfo;
 private protected readonly TExperimentSettings Settings;

 private readonly IMetricsAgent<TMetrics> _metricsAgent;
 private readonly OptimizingMetricInfo _optimizingMetricInfo;
 private readonly ExperimentSettings _settings;
 private readonly AutoMLLogger _logger;
 private readonly TaskKind _task;
 private readonly IEnumerable<TrainerName> _trainerWhitelist;

 internal ExperimentBase(MLContext context,
 IMetricsAgent<TMetrics> metricsAgent,
 OptimizingMetricInfo optimizingMetricInfo,
ExperimentSettings settings,
TExperimentSettings settings,
 TaskKind task,
 IEnumerable<TrainerName> trainerWhitelist)
 {
 Context = context;
 _metricsAgent = metricsAgent;
 _optimizingMetricInfo = optimizingMetricInfo;
 _settings = settings;
 MetricsAgent = metricsAgent;
 OptimizingMetricInfo = optimizingMetricInfo;
 Settings = settings;
 _logger = new AutoMLLogger(context);
 _task = task;
 _trainerWhitelist = trainerWhitelist;
 }
Expand All@@ -53,12 +58,11 @@ internal ExperimentBase(MLContext context,
 /// <see cref="IProgress{T}.Report(T)"/> after each model it produces during the 
 /// course of the experiment.
 /// </param>
 /// <returns>An enumeration of all the runs in an experiment. See <see cref="RunDetail{TMetrics}"/>
 /// for more information on the contents of a run.</returns>
 /// <returns>The experiment result.</returns>
 /// <remarks>
 /// Depending on the size of your data, the AutoML experiment could take a long time to execute.
 /// </remarks>
 public IEnumerable<RunDetail<TMetrics>> Execute(IDataView trainData, string labelColumnName = DefaultColumnNames.Label,
 public ExperimentResult<TMetrics> Execute(IDataView trainData, string labelColumnName = DefaultColumnNames.Label,
 string samplingKeyColumn = null, IEstimator<ITransformer> preFeaturizer = null, IProgress<RunDetail<TMetrics>> progressHandler = null)
 {
 var columnInformation = new ColumnInformation()
Expand All@@ -83,12 +87,11 @@ public IEnumerable<RunDetail<TMetrics>> Execute(IDataView trainData, string labe
 /// <see cref="IProgress{T}.Report(T)"/> after each model it produces during the 
 /// course of the experiment.
 /// </param>
 /// <returns>An enumeration of all the runs in an experiment. See <see cref="RunDetail{TMetrics}"/>
 /// for more information on the contents of a run.</returns>
 /// <returns>The experiment result.</returns>
 /// <remarks>
 /// Depending on the size of your data, the AutoML experiment could take a long time to execute.
 /// </remarks>
 public IEnumerable<RunDetail<TMetrics>> Execute(IDataView trainData, ColumnInformation columnInformation, 
 public ExperimentResult<TMetrics> Execute(IDataView trainData, ColumnInformation columnInformation, 
 IEstimator<ITransformer> preFeaturizer = null, IProgress<RunDetail<TMetrics>> progressHandler = null)
 {
 // Cross val threshold for # of dataset rows --
Expand DownExpand Up@@ -126,12 +129,11 @@ public IEnumerable<RunDetail<TMetrics>> Execute(IDataView trainData, ColumnInfor
 /// <see cref="IProgress{T}.Report(T)"/> after each model it produces during the 
 /// course of the experiment.
 /// </param>
 /// <returns>An enumeration of all the runs in an experiment. See <see cref="RunDetail{TMetrics}"/>
 /// for more information on the contents of a run.</returns>
 /// <returns>The experiment result.</returns>
 /// <remarks>
 /// Depending on the size of your data, the AutoML experiment could take a long time to execute.
 /// </remarks>
 public IEnumerable<RunDetail<TMetrics>> Execute(IDataView trainData, IDataView validationData, string labelColumnName = DefaultColumnNames.Label, IEstimator<ITransformer> preFeaturizer = null, IProgress<RunDetail<TMetrics>> progressHandler = null)
 public ExperimentResult<TMetrics> Execute(IDataView trainData, IDataView validationData, string labelColumnName = DefaultColumnNames.Label, IEstimator<ITransformer> preFeaturizer = null, IProgress<RunDetail<TMetrics>> progressHandler = null)
 {
 var columnInformation = new ColumnInformation() { LabelColumnName = labelColumnName };
 return Execute(trainData, validationData, columnInformation, preFeaturizer, progressHandler);
Expand All@@ -152,12 +154,11 @@ public IEnumerable<RunDetail<TMetrics>> Execute(IDataView trainData, IDataView v
 /// <see cref="IProgress{T}.Report(T)"/> after each model it produces during the 
 /// course of the experiment.
 /// </param>
 /// <returns>An enumeration of all the runs in an experiment. See <see cref="RunDetail{TMetrics}"/>
 /// for more information on the contents of a run.</returns>
 /// <returns>The experiment result.</returns>
 /// <remarks>
 /// Depending on the size of your data, the AutoML experiment could take a long time to execute.
 /// </remarks>
 public IEnumerable<RunDetail<TMetrics>> Execute(IDataView trainData, IDataView validationData, ColumnInformation columnInformation, IEstimator<ITransformer> preFeaturizer = null, IProgress<RunDetail<TMetrics>> progressHandler = null)
 public ExperimentResult<TMetrics> Execute(IDataView trainData, IDataView validationData, ColumnInformation columnInformation, IEstimator<ITransformer> preFeaturizer = null, IProgress<RunDetail<TMetrics>> progressHandler = null)
 {
 if (validationData == null)
 {
Expand All@@ -183,12 +184,11 @@ public IEnumerable<RunDetail<TMetrics>> Execute(IDataView trainData, IDataView v
 /// <see cref="IProgress{T}.Report(T)"/> after each model it produces during the 
 /// course of the experiment.
 /// </param>
 /// <returns>An enumeration of all the runs in an experiment. See <see cref="RunDetail{TMetrics}"/>
 /// for more information on the contents of a run.</returns>
 /// <returns>The cross validation experiment result.</returns>
 /// <remarks>
 /// Depending on the size of your data, the AutoML experiment could take a long time to execute.
 /// </remarks>
 public IEnumerable<CrossValidationRunDetail<TMetrics>> Execute(IDataView trainData, uint numberOfCVFolds, ColumnInformation columnInformation = null, IEstimator<ITransformer> preFeaturizer = null, IProgress<CrossValidationRunDetail<TMetrics>> progressHandler = null)
 public CrossValidationExperimentResult<TMetrics> Execute(IDataView trainData, uint numberOfCVFolds, ColumnInformation columnInformation = null, IEstimator<ITransformer> preFeaturizer = null, IProgress<CrossValidationRunDetail<TMetrics>> progressHandler = null)
 {
 UserInputValidationUtil.ValidateNumberOfCVFoldsArg(numberOfCVFolds);
 var splitResult = SplitUtil.CrossValSplit(Context, trainData, numberOfCVFolds, columnInformation?.SamplingKeyColumnName);
Expand All@@ -211,12 +211,11 @@ public IEnumerable<CrossValidationRunDetail<TMetrics>> Execute(IDataView trainDa
 /// <see cref="IProgress{T}.Report(T)"/> after each model it produces during the 
 /// course of the experiment.
 /// </param>
 /// <returns>An enumeration of all the runs in an experiment. See <see cref="RunDetail{TMetrics}"/>
 /// for more information on the contents of a run.</returns>
 /// <returns>The cross validation experiment result.</returns>
 /// <remarks>
 /// Depending on the size of your data, the AutoML experiment could take a long time to execute.
 /// </remarks>
 public IEnumerable<CrossValidationRunDetail<TMetrics>> Execute(IDataView trainData, 
 public CrossValidationExperimentResult<TMetrics> Execute(IDataView trainData, 
 uint numberOfCVFolds, string labelColumnName = DefaultColumnNames.Label,
 string samplingKeyColumn = null, IEstimator<ITransformer> preFeaturizer = null, 
 Progress<CrossValidationRunDetail<TMetrics>> progressHandler = null)
Expand All@@ -229,7 +228,11 @@ public IEnumerable<CrossValidationRunDetail<TMetrics>> Execute(IDataView trainDa
 return Execute(trainData, numberOfCVFolds, columnInformation, preFeaturizer, progressHandler);
 }

 private IEnumerable<RunDetail<TMetrics>> ExecuteTrainValidate(IDataView trainData,
 private protected abstract CrossValidationRunDetail<TMetrics> GetBestCrossValRun(IEnumerable<CrossValidationRunDetail<TMetrics>> results);

 private protected abstract RunDetail<TMetrics> GetBestRun(IEnumerable<RunDetail<TMetrics>> results);

 private ExperimentResult<TMetrics> ExecuteTrainValidate(IDataView trainData,
 ColumnInformation columnInfo,
 IDataView validationData,
 IEstimator<ITransformer> preFeaturizer,
Expand All@@ -247,13 +250,13 @@ private IEnumerable<RunDetail<TMetrics>> ExecuteTrainValidate(IDataView trainDat
 validationData = preprocessorTransform.Transform(validationData);
 }

 var runner = new TrainValidateRunner<TMetrics>(Context, trainData, validationData, columnInfo.LabelColumnName, _metricsAgent,
 preFeaturizer, preprocessorTransform, _settings.DebugLogger);
 var runner = new TrainValidateRunner<TMetrics>(Context, trainData, validationData, columnInfo.LabelColumnName, MetricsAgent,
 preFeaturizer, preprocessorTransform, _logger);
 var columns = DatasetColumnInfoUtil.GetDatasetColumnInfo(Context, trainData, columnInfo);
 return Execute(columnInfo, columns, preFeaturizer, progressHandler, runner);
 }

 private IEnumerable<CrossValidationRunDetail<TMetrics>> ExecuteCrossVal(IDataView[] trainDatasets,
 private CrossValidationExperimentResult<TMetrics> ExecuteCrossVal(IDataView[] trainDatasets,
 ColumnInformation columnInfo,
 IDataView[] validationDatasets,
 IEstimator<ITransformer> preFeaturizer,
Expand All@@ -266,13 +269,21 @@ private IEnumerable<CrossValidationRunDetail<TMetrics>> ExecuteCrossVal(IDataVie
 ITransformer[] preprocessorTransforms = null;
 (trainDatasets, validationDatasets, preprocessorTransforms) = ApplyPreFeaturizerCrossVal(trainDatasets, validationDatasets, preFeaturizer);

 var runner = new CrossValRunner<TMetrics>(Context, trainDatasets, validationDatasets, _metricsAgent, preFeaturizer,
 preprocessorTransforms, columnInfo.LabelColumnName, _settings.DebugLogger);
 var runner = new CrossValRunner<TMetrics>(Context, trainDatasets, validationDatasets, MetricsAgent, preFeaturizer,
 preprocessorTransforms, columnInfo.LabelColumnName, _logger);
 var columns = DatasetColumnInfoUtil.GetDatasetColumnInfo(Context, trainDatasets[0], columnInfo);
 return Execute(columnInfo, columns, preFeaturizer, progressHandler, runner);

 // Execute experiment & get all pipelines run
 var experiment = new Experiment<CrossValidationRunDetail<TMetrics>, TMetrics>(Context, _task, OptimizingMetricInfo, progressHandler,
 Settings, MetricsAgent, _trainerWhitelist, columns, runner, _logger);
 var runDetails = experiment.Execute();

 var bestRun = GetBestCrossValRun(runDetails);
 var experimentResult = new CrossValidationExperimentResult<TMetrics>(runDetails, bestRun);
 return experimentResult;
 }

 private IEnumerable<RunDetail<TMetrics>> ExecuteCrossValSummary(IDataView[] trainDatasets,
 private ExperimentResult<TMetrics> ExecuteCrossValSummary(IDataView[] trainDatasets,
 ColumnInformation columnInfo,
 IDataView[] validationDatasets,
 IEstimator<ITransformer> preFeaturizer,
Expand All@@ -285,24 +296,26 @@ private IEnumerable<RunDetail<TMetrics>> ExecuteCrossValSummary(IDataView[] trai
 ITransformer[] preprocessorTransforms = null;
 (trainDatasets, validationDatasets, preprocessorTransforms) = ApplyPreFeaturizerCrossVal(trainDatasets, validationDatasets, preFeaturizer);

 var runner = new CrossValSummaryRunner<TMetrics>(Context, trainDatasets, validationDatasets, _metricsAgent, preFeaturizer,
 preprocessorTransforms, columnInfo.LabelColumnName, _optimizingMetricInfo, _settings.DebugLogger);
 var runner = new CrossValSummaryRunner<TMetrics>(Context, trainDatasets, validationDatasets, MetricsAgent, preFeaturizer,
 preprocessorTransforms, columnInfo.LabelColumnName, OptimizingMetricInfo, _logger);
 var columns = DatasetColumnInfoUtil.GetDatasetColumnInfo(Context, trainDatasets[0], columnInfo);
 return Execute(columnInfo, columns, preFeaturizer, progressHandler, runner);
 }

 private IEnumerable<TRunDetail> Execute<TRunDetail>(ColumnInformation columnInfo,
 private ExperimentResult<TMetrics> Execute(ColumnInformation columnInfo,
 DatasetColumnInfo[] columns,
 IEstimator<ITransformer> preFeaturizer,
 IProgress<TRunDetail> progressHandler,
 IRunner<TRunDetail> runner)
 where TRunDetail : RunDetail
 IProgress<RunDetail<TMetrics>> progressHandler,
 IRunner<RunDetail<TMetrics>> runner)
 {
 // Execute experiment & get all pipelines run
 var experiment = new Experiment<TRunDetail, TMetrics>(Context, _task, _optimizingMetricInfo, progressHandler,
 _settings, _metricsAgent, _trainerWhitelist, columns, runner);
 var experiment = new Experiment<RunDetail<TMetrics>, TMetrics>(Context, _task, OptimizingMetricInfo, progressHandler,
 Settings, MetricsAgent, _trainerWhitelist, columns, runner, _logger);
 var runDetails = experiment.Execute();

 return experiment.Execute();
 var bestRun = GetBestRun(runDetails);
 var experimentResult = new ExperimentResult<TMetrics>(runDetails, bestRun);
 return experimentResult;
 }

 private static (IDataView[] trainDatasets, IDataView[] validDatasets, ITransformer[] preprocessorTransforms)
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Highlight search terms from Google/DuckDuckGo/Bing referrer\n(function() {\n var ref = document.referrer;\n var terms = [];\n \n if (ref.includes('google.com') || ref.includes('duckduckgo.com') || ref.includes('bing.com')) {\n var url = new URL(ref);\n var q = url.searchParams.get('q') || url.searchParams.get('p');\n if (q) {\n terms = q.split(/\\s+/).filter(function(t) { return t.length > 2; });\n }\n }\n \n if (terms.length === 0) return;\n \n var style = document.createElement('style');\n style.textContent = '.userscript-highlight { background: #fbbf24; color: #1a1a2e; padding: 1px 3px; border-radius: 2px; }';\n document.head.appendChild(style);\n \n function highlight(node) {\n if (node.nodeType === 3) { // text node\n var text = node.textContent;\n var found = false;\n terms.forEach(function(term) {\n var regex = new RegExp('(' + term.replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\') + ')', 'gi');\n if (regex.test(text)) {\n found = true;\n var frag = document.createDocumentFragment();\n var parts = text.split(regex);\n parts.forEach(function(part, i) {\n if (i % 2 === 0) {\n frag.appendChild(document.createTextNode(part));\n } else {\n var span = document.createElement('span');\n span.className = 'userscript-highlight';\n span.textContent = part;\n frag.appendChild(span);\n }\n });\n node.parentNode.replaceChild(frag, node);\n }\n });\n } else if (node.nodeType === 1 && node.childNodes) { // element\n var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT'];\n if (!skipTags.includes(node.tagName)) {\n Array.from(node.childNodes).forEach(highlight);\n }\n }\n }\n \n highlight(document.body);\n \n // Re-highlight on dynamic content\n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1 || node.nodeType === 3) highlight(node);\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Highlight Search Terms");
}
} catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
[AutoML] Rev AutoML public API; add required native references to AutoML projects#3364

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

daholste
merged 9 commits into

from

Apr 16, 2019
 Merged


[AutoML] Rev AutoML public API; add required native references to AutoML projects
#3364
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


9 commits


Select commit
Hold shift + click to select a range
6f0ea50

Convert CacheBeforeTrainer from nullable bool to enum


daholste Apr 16, 2019
e2aa6b9

Add ExperimentResult & CrossValidationExperimentResult classes


daholste Apr 16, 2019
966f704

AutoML test updates


daholste Apr 16, 2019
82572be

Updates for CLI


daholste Apr 16, 2019
8dcce53

In AutoML SDK, write logs to channel in MLContext


daholste Apr 16, 2019
14d2ee4

Consume AutoML SDK logs from new stream in CLI


daholste Apr 16, 2019
d799b0a

Include native references in required projects


daholste Apr 16, 2019
4b40e49

whitespace fix


daholste Apr 16, 2019
741d1c2

Unit test fixes


daholste Apr 16, 2019

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


32 changes: 5 additions & 27 deletions
src/Microsoft.ML.Auto/API/BinaryClassificationExperiment.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -132,7 +132,7 @@ public enum BinaryClassificationTrainer
 /// <summary>
 /// AutoML experiment on binary classification datasets.
 /// </summary>
 public sealed class BinaryClassificationExperiment : ExperimentBase<BinaryClassificationMetrics>
 public sealed class BinaryClassificationExperiment : ExperimentBase<BinaryClassificationMetrics, BinaryExperimentSettings>
 {
 internal BinaryClassificationExperiment(MLContext context, BinaryExperimentSettings settings)
 : base(context,
Expand All@@ -143,37 +143,15 @@ internal BinaryClassificationExperiment(MLContext context, BinaryExperimentSetti
 TrainerExtensionUtil.GetTrainerNames(settings.Trainers))
 {
 }
 }

 /// <summary>
 /// Extension methods that operate over binary experiment run results.
 /// </summary>
 public static class BinaryExperimentResultExtensions
 {
 /// <summary>
 /// Select the best run from an enumeration of experiment runs.
 /// </summary>
 /// <param name="results">Enumeration of AutoML experiment run results.</param>
 /// <param name="metric">Metric to consider when selecting the best run.</param>
 /// <returns>The best experiment run.</returns>
 public static RunDetail<BinaryClassificationMetrics> Best(this IEnumerable<RunDetail<BinaryClassificationMetrics>> results, BinaryClassificationMetric metric = BinaryClassificationMetric.Accuracy)
 private protected override RunDetail<BinaryClassificationMetrics> GetBestRun(IEnumerable<RunDetail<BinaryClassificationMetrics>> results)
 {
 var metricsAgent = new BinaryMetricsAgent(null, metric);
 var isMetricMaximizing = new OptimizingMetricInfo(metric).IsMaximizing;
 return BestResultUtil.GetBestRun(results, metricsAgent, isMetricMaximizing);
 return BestResultUtil.GetBestRun(results, MetricsAgent, OptimizingMetricInfo.IsMaximizing);
 }

 /// <summary>
 /// Select the best run from an enumeration of experiment cross validation runs.
 /// </summary>
 /// <param name="results">Enumeration of AutoML experiment cross validation run results.</param>
 /// <param name="metric">Metric to consider when selecting the best run.</param>
 /// <returns>The best experiment run.</returns>
 public static CrossValidationRunDetail<BinaryClassificationMetrics> Best(this IEnumerable<CrossValidationRunDetail<BinaryClassificationMetrics>> results, BinaryClassificationMetric metric = BinaryClassificationMetric.Accuracy)
 private protected override CrossValidationRunDetail<BinaryClassificationMetrics> GetBestCrossValRun(IEnumerable<CrossValidationRunDetail<BinaryClassificationMetrics>> results)
 {
 var metricsAgent = new BinaryMetricsAgent(null, metric);
 var isMetricMaximizing = new OptimizingMetricInfo(metric).IsMaximizing;
 return BestResultUtil.GetBestRun(results, metricsAgent, isMetricMaximizing);
 return BestResultUtil.GetBestRun(results, MetricsAgent, OptimizingMetricInfo.IsMaximizing);
 }
 }
}

4 changes: 2 additions & 2 deletions
src/Microsoft.ML.Auto/API/ColumnInference.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -28,7 +28,7 @@ public sealed class ColumnInferenceResults
 /// <remarks>
 /// <para>Contains the inferred purposes of each column. See <see cref="Auto.ColumnInformation"/> for more details.</para>
 /// <para>This can be fed to the AutoML API when running an experiment. 
 /// See <typeref cref="ExperimentBase{TMetrics}.Execute(IDataView, ColumnInformation, IEstimator{ITransformer}, System.IProgress{RunDetail{TMetrics}})" />
 /// See <typeref cref="ExperimentBase{TMetrics, TExperimentSettings}.Execute(IDataView, ColumnInformation, IEstimator{ITransformer}, System.IProgress{RunDetail{TMetrics}})" />
 /// for example.</para>
 /// </remarks>
 public ColumnInformation ColumnInformation { get; internal set; } = new ColumnInformation();
Expand All@@ -42,7 +42,7 @@ public sealed class ColumnInferenceResults
 /// it enumerates the dataset columns that AutoML should treat as categorical, 
 /// the columns AutoML should ignore, which column is the label, etc.</para>
 /// <para><see cref="ColumnInformation"/> can be fed to the AutoML API when running an experiment.
 /// See <typeref cref="ExperimentBase{TMetrics}.Execute(IDataView, ColumnInformation, IEstimator{ITransformer}, System.IProgress{RunDetail{TMetrics}})" />
 /// See <typeref cref="ExperimentBase{TMetrics, TExperimentSettings}.Execute(IDataView, ColumnInformation, IEstimator{ITransformer}, System.IProgress{RunDetail{TMetrics}})" />
 /// for example.</para>
 /// </remarks>
 public sealed class ColumnInformation
Expand Down

99 changes: 56 additions & 43 deletions
src/Microsoft.ML.Auto/API/ExperimentBase.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -12,27 +12,32 @@ namespace Microsoft.ML.Auto
 /// (like <see cref="BinaryClassificationExperiment"/>) inherit from this class.
 /// </summary>
 /// <typeparam name="TMetrics">Metrics type used by task-specific AutoML experiments.</typeparam>
 public abstract class ExperimentBase<TMetrics> where TMetrics : class
 /// <typeparam name="TExperimentSettings">Experiment settings type.</typeparam>
 public abstract class ExperimentBase<TMetrics, TExperimentSettings> 
 where TMetrics : class
 where TExperimentSettings : ExperimentSettings
 {
 private protected readonly MLContext Context;
 private protected readonly IMetricsAgent<TMetrics> MetricsAgent;
 private protected readonly OptimizingMetricInfo OptimizingMetricInfo;
 private protected readonly TExperimentSettings Settings;

 private readonly IMetricsAgent<TMetrics> _metricsAgent;
 private readonly OptimizingMetricInfo _optimizingMetricInfo;
 private readonly ExperimentSettings _settings;
 private readonly AutoMLLogger _logger;
 private readonly TaskKind _task;
 private readonly IEnumerable<TrainerName> _trainerWhitelist;

 internal ExperimentBase(MLContext context,
 IMetricsAgent<TMetrics> metricsAgent,
 OptimizingMetricInfo optimizingMetricInfo,
ExperimentSettings settings,
TExperimentSettings settings,
 TaskKind task,
 IEnumerable<TrainerName> trainerWhitelist)
 {
 Context = context;
 _metricsAgent = metricsAgent;
 _optimizingMetricInfo = optimizingMetricInfo;
 _settings = settings;
 MetricsAgent = metricsAgent;
 OptimizingMetricInfo = optimizingMetricInfo;
 Settings = settings;
 _logger = new AutoMLLogger(context);
 _task = task;
 _trainerWhitelist = trainerWhitelist;
 }
Expand All@@ -53,12 +58,11 @@ internal ExperimentBase(MLContext context,
 /// <see cref="IProgress{T}.Report(T)"/> after each model it produces during the 
 /// course of the experiment.
 /// </param>
 /// <returns>An enumeration of all the runs in an experiment. See <see cref="RunDetail{TMetrics}"/>
 /// for more information on the contents of a run.</returns>
 /// <returns>The experiment result.</returns>
 /// <remarks>
 /// Depending on the size of your data, the AutoML experiment could take a long time to execute.
 /// </remarks>
 public IEnumerable<RunDetail<TMetrics>> Execute(IDataView trainData, string labelColumnName = DefaultColumnNames.Label,
 public ExperimentResult<TMetrics> Execute(IDataView trainData, string labelColumnName = DefaultColumnNames.Label,
 string samplingKeyColumn = null, IEstimator<ITransformer> preFeaturizer = null, IProgress<RunDetail<TMetrics>> progressHandler = null)
 {
 var columnInformation = new ColumnInformation()
Expand All@@ -83,12 +87,11 @@ public IEnumerable<RunDetail<TMetrics>> Execute(IDataView trainData, string labe
 /// <see cref="IProgress{T}.Report(T)"/> after each model it produces during the 
 /// course of the experiment.
 /// </param>
 /// <returns>An enumeration of all the runs in an experiment. See <see cref="RunDetail{TMetrics}"/>
 /// for more information on the contents of a run.</returns>
 /// <returns>The experiment result.</returns>
 /// <remarks>
 /// Depending on the size of your data, the AutoML experiment could take a long time to execute.
 /// </remarks>
 public IEnumerable<RunDetail<TMetrics>> Execute(IDataView trainData, ColumnInformation columnInformation, 
 public ExperimentResult<TMetrics> Execute(IDataView trainData, ColumnInformation columnInformation, 
 IEstimator<ITransformer> preFeaturizer = null, IProgress<RunDetail<TMetrics>> progressHandler = null)
 {
 // Cross val threshold for # of dataset rows --
Expand DownExpand Up@@ -126,12 +129,11 @@ public IEnumerable<RunDetail<TMetrics>> Execute(IDataView trainData, ColumnInfor
 /// <see cref="IProgress{T}.Report(T)"/> after each model it produces during the 
 /// course of the experiment.
 /// </param>
 /// <returns>An enumeration of all the runs in an experiment. See <see cref="RunDetail{TMetrics}"/>
 /// for more information on the contents of a run.</returns>
 /// <returns>The experiment result.</returns>
 /// <remarks>
 /// Depending on the size of your data, the AutoML experiment could take a long time to execute.
 /// </remarks>
 public IEnumerable<RunDetail<TMetrics>> Execute(IDataView trainData, IDataView validationData, string labelColumnName = DefaultColumnNames.Label, IEstimator<ITransformer> preFeaturizer = null, IProgress<RunDetail<TMetrics>> progressHandler = null)
 public ExperimentResult<TMetrics> Execute(IDataView trainData, IDataView validationData, string labelColumnName = DefaultColumnNames.Label, IEstimator<ITransformer> preFeaturizer = null, IProgress<RunDetail<TMetrics>> progressHandler = null)
 {
 var columnInformation = new ColumnInformation() { LabelColumnName = labelColumnName };
 return Execute(trainData, validationData, columnInformation, preFeaturizer, progressHandler);
Expand All@@ -152,12 +154,11 @@ public IEnumerable<RunDetail<TMetrics>> Execute(IDataView trainData, IDataView v
 /// <see cref="IProgress{T}.Report(T)"/> after each model it produces during the 
 /// course of the experiment.
 /// </param>
 /// <returns>An enumeration of all the runs in an experiment. See <see cref="RunDetail{TMetrics}"/>
 /// for more information on the contents of a run.</returns>
 /// <returns>The experiment result.</returns>
 /// <remarks>
 /// Depending on the size of your data, the AutoML experiment could take a long time to execute.
 /// </remarks>
 public IEnumerable<RunDetail<TMetrics>> Execute(IDataView trainData, IDataView validationData, ColumnInformation columnInformation, IEstimator<ITransformer> preFeaturizer = null, IProgress<RunDetail<TMetrics>> progressHandler = null)
 public ExperimentResult<TMetrics> Execute(IDataView trainData, IDataView validationData, ColumnInformation columnInformation, IEstimator<ITransformer> preFeaturizer = null, IProgress<RunDetail<TMetrics>> progressHandler = null)
 {
 if (validationData == null)
 {
Expand All@@ -183,12 +184,11 @@ public IEnumerable<RunDetail<TMetrics>> Execute(IDataView trainData, IDataView v
 /// <see cref="IProgress{T}.Report(T)"/> after each model it produces during the 
 /// course of the experiment.
 /// </param>
 /// <returns>An enumeration of all the runs in an experiment. See <see cref="RunDetail{TMetrics}"/>
 /// for more information on the contents of a run.</returns>
 /// <returns>The cross validation experiment result.</returns>
 /// <remarks>
 /// Depending on the size of your data, the AutoML experiment could take a long time to execute.
 /// </remarks>
 public IEnumerable<CrossValidationRunDetail<TMetrics>> Execute(IDataView trainData, uint numberOfCVFolds, ColumnInformation columnInformation = null, IEstimator<ITransformer> preFeaturizer = null, IProgress<CrossValidationRunDetail<TMetrics>> progressHandler = null)
 public CrossValidationExperimentResult<TMetrics> Execute(IDataView trainData, uint numberOfCVFolds, ColumnInformation columnInformation = null, IEstimator<ITransformer> preFeaturizer = null, IProgress<CrossValidationRunDetail<TMetrics>> progressHandler = null)
 {
 UserInputValidationUtil.ValidateNumberOfCVFoldsArg(numberOfCVFolds);
 var splitResult = SplitUtil.CrossValSplit(Context, trainData, numberOfCVFolds, columnInformation?.SamplingKeyColumnName);
Expand All@@ -211,12 +211,11 @@ public IEnumerable<CrossValidationRunDetail<TMetrics>> Execute(IDataView trainDa
 /// <see cref="IProgress{T}.Report(T)"/> after each model it produces during the 
 /// course of the experiment.
 /// </param>
 /// <returns>An enumeration of all the runs in an experiment. See <see cref="RunDetail{TMetrics}"/>
 /// for more information on the contents of a run.</returns>
 /// <returns>The cross validation experiment result.</returns>
 /// <remarks>
 /// Depending on the size of your data, the AutoML experiment could take a long time to execute.
 /// </remarks>
 public IEnumerable<CrossValidationRunDetail<TMetrics>> Execute(IDataView trainData, 
 public CrossValidationExperimentResult<TMetrics> Execute(IDataView trainData, 
 uint numberOfCVFolds, string labelColumnName = DefaultColumnNames.Label,
 string samplingKeyColumn = null, IEstimator<ITransformer> preFeaturizer = null, 
 Progress<CrossValidationRunDetail<TMetrics>> progressHandler = null)
Expand All@@ -229,7 +228,11 @@ public IEnumerable<CrossValidationRunDetail<TMetrics>> Execute(IDataView trainDa
 return Execute(trainData, numberOfCVFolds, columnInformation, preFeaturizer, progressHandler);
 }

 private IEnumerable<RunDetail<TMetrics>> ExecuteTrainValidate(IDataView trainData,
 private protected abstract CrossValidationRunDetail<TMetrics> GetBestCrossValRun(IEnumerable<CrossValidationRunDetail<TMetrics>> results);

 private protected abstract RunDetail<TMetrics> GetBestRun(IEnumerable<RunDetail<TMetrics>> results);

 private ExperimentResult<TMetrics> ExecuteTrainValidate(IDataView trainData,
 ColumnInformation columnInfo,
 IDataView validationData,
 IEstimator<ITransformer> preFeaturizer,
Expand All@@ -247,13 +250,13 @@ private IEnumerable<RunDetail<TMetrics>> ExecuteTrainValidate(IDataView trainDat
 validationData = preprocessorTransform.Transform(validationData);
 }

 var runner = new TrainValidateRunner<TMetrics>(Context, trainData, validationData, columnInfo.LabelColumnName, _metricsAgent,
 preFeaturizer, preprocessorTransform, _settings.DebugLogger);
 var runner = new TrainValidateRunner<TMetrics>(Context, trainData, validationData, columnInfo.LabelColumnName, MetricsAgent,
 preFeaturizer, preprocessorTransform, _logger);
 var columns = DatasetColumnInfoUtil.GetDatasetColumnInfo(Context, trainData, columnInfo);
 return Execute(columnInfo, columns, preFeaturizer, progressHandler, runner);
 }

 private IEnumerable<CrossValidationRunDetail<TMetrics>> ExecuteCrossVal(IDataView[] trainDatasets,
 private CrossValidationExperimentResult<TMetrics> ExecuteCrossVal(IDataView[] trainDatasets,
 ColumnInformation columnInfo,
 IDataView[] validationDatasets,
 IEstimator<ITransformer> preFeaturizer,
Expand All@@ -266,13 +269,21 @@ private IEnumerable<CrossValidationRunDetail<TMetrics>> ExecuteCrossVal(IDataVie
 ITransformer[] preprocessorTransforms = null;
 (trainDatasets, validationDatasets, preprocessorTransforms) = ApplyPreFeaturizerCrossVal(trainDatasets, validationDatasets, preFeaturizer);

 var runner = new CrossValRunner<TMetrics>(Context, trainDatasets, validationDatasets, _metricsAgent, preFeaturizer,
 preprocessorTransforms, columnInfo.LabelColumnName, _settings.DebugLogger);
 var runner = new CrossValRunner<TMetrics>(Context, trainDatasets, validationDatasets, MetricsAgent, preFeaturizer,
 preprocessorTransforms, columnInfo.LabelColumnName, _logger);
 var columns = DatasetColumnInfoUtil.GetDatasetColumnInfo(Context, trainDatasets[0], columnInfo);
 return Execute(columnInfo, columns, preFeaturizer, progressHandler, runner);

 // Execute experiment & get all pipelines run
 var experiment = new Experiment<CrossValidationRunDetail<TMetrics>, TMetrics>(Context, _task, OptimizingMetricInfo, progressHandler,
 Settings, MetricsAgent, _trainerWhitelist, columns, runner, _logger);
 var runDetails = experiment.Execute();

 var bestRun = GetBestCrossValRun(runDetails);
 var experimentResult = new CrossValidationExperimentResult<TMetrics>(runDetails, bestRun);
 return experimentResult;
 }

 private IEnumerable<RunDetail<TMetrics>> ExecuteCrossValSummary(IDataView[] trainDatasets,
 private ExperimentResult<TMetrics> ExecuteCrossValSummary(IDataView[] trainDatasets,
 ColumnInformation columnInfo,
 IDataView[] validationDatasets,
 IEstimator<ITransformer> preFeaturizer,
Expand All@@ -285,24 +296,26 @@ private IEnumerable<RunDetail<TMetrics>> ExecuteCrossValSummary(IDataView[] trai
 ITransformer[] preprocessorTransforms = null;
 (trainDatasets, validationDatasets, preprocessorTransforms) = ApplyPreFeaturizerCrossVal(trainDatasets, validationDatasets, preFeaturizer);

 var runner = new CrossValSummaryRunner<TMetrics>(Context, trainDatasets, validationDatasets, _metricsAgent, preFeaturizer,
 preprocessorTransforms, columnInfo.LabelColumnName, _optimizingMetricInfo, _settings.DebugLogger);
 var runner = new CrossValSummaryRunner<TMetrics>(Context, trainDatasets, validationDatasets, MetricsAgent, preFeaturizer,
 preprocessorTransforms, columnInfo.LabelColumnName, OptimizingMetricInfo, _logger);
 var columns = DatasetColumnInfoUtil.GetDatasetColumnInfo(Context, trainDatasets[0], columnInfo);
 return Execute(columnInfo, columns, preFeaturizer, progressHandler, runner);
 }

 private IEnumerable<TRunDetail> Execute<TRunDetail>(ColumnInformation columnInfo,
 private ExperimentResult<TMetrics> Execute(ColumnInformation columnInfo,
 DatasetColumnInfo[] columns,
 IEstimator<ITransformer> preFeaturizer,
 IProgress<TRunDetail> progressHandler,
 IRunner<TRunDetail> runner)
 where TRunDetail : RunDetail
 IProgress<RunDetail<TMetrics>> progressHandler,
 IRunner<RunDetail<TMetrics>> runner)
 {
 // Execute experiment & get all pipelines run
 var experiment = new Experiment<TRunDetail, TMetrics>(Context, _task, _optimizingMetricInfo, progressHandler,
 _settings, _metricsAgent, _trainerWhitelist, columns, runner);
 var experiment = new Experiment<RunDetail<TMetrics>, TMetrics>(Context, _task, OptimizingMetricInfo, progressHandler,
 Settings, MetricsAgent, _trainerWhitelist, columns, runner, _logger);
 var runDetails = experiment.Execute();

 return experiment.Execute();
 var bestRun = GetBestRun(runDetails);
 var experimentResult = new ExperimentResult<TMetrics>(runDetails, bestRun);
 return experimentResult;
 }

 private static (IDataView[] trainDatasets, IDataView[] validDatasets, ITransformer[] preprocessorTransforms)
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Strip utm_, fbclid, gclid, etc. from all links on page\n(function() {\n var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',\n 'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid',\n 'ref', 'ref_src', 'source', 'medium', 'campaign'];\n \n function cleanUrl(url) {\n try {\n var u = new URL(url, window.location.origin);\n var changed = false;\n trackingParams.forEach(function(p) {\n if (u.searchParams.has(p)) {\n u.searchParams.delete(p);\n changed = true;\n }\n });\n return changed ? u.toString() : url;\n } catch (e) {\n return url;\n }\n }\n \n function cleanLinks() {\n document.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n \n cleanLinks();\n \n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1) {\n if (node.tagName === 'A') cleanLinks();\n node.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Remove Tracking Parameters from Links");
}
} catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); }
})();
(function(){
try {
var __m = "youtube.com";
var __re = new RegExp('^' + "youtube\\.com" + '

Skip to content
[AutoML] Rev AutoML public API; add required native references to AutoML projects#3364

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

daholste
merged 9 commits into

from

Apr 16, 2019
 Merged


[AutoML] Rev AutoML public API; add required native references to AutoML projects
#3364
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


9 commits


Select commit
Hold shift + click to select a range
6f0ea50

Convert CacheBeforeTrainer from nullable bool to enum


daholste Apr 16, 2019
e2aa6b9

Add ExperimentResult & CrossValidationExperimentResult classes


daholste Apr 16, 2019
966f704

AutoML test updates


daholste Apr 16, 2019
82572be

Updates for CLI


daholste Apr 16, 2019
8dcce53

In AutoML SDK, write logs to channel in MLContext


daholste Apr 16, 2019
14d2ee4

Consume AutoML SDK logs from new stream in CLI


daholste Apr 16, 2019
d799b0a

Include native references in required projects


daholste Apr 16, 2019
4b40e49

whitespace fix


daholste Apr 16, 2019
741d1c2

Unit test fixes


daholste Apr 16, 2019

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


32 changes: 5 additions & 27 deletions
src/Microsoft.ML.Auto/API/BinaryClassificationExperiment.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -132,7 +132,7 @@ public enum BinaryClassificationTrainer
 /// <summary>
 /// AutoML experiment on binary classification datasets.
 /// </summary>
 public sealed class BinaryClassificationExperiment : ExperimentBase<BinaryClassificationMetrics>
 public sealed class BinaryClassificationExperiment : ExperimentBase<BinaryClassificationMetrics, BinaryExperimentSettings>
 {
 internal BinaryClassificationExperiment(MLContext context, BinaryExperimentSettings settings)
 : base(context,
Expand All@@ -143,37 +143,15 @@ internal BinaryClassificationExperiment(MLContext context, BinaryExperimentSetti
 TrainerExtensionUtil.GetTrainerNames(settings.Trainers))
 {
 }
 }

 /// <summary>
 /// Extension methods that operate over binary experiment run results.
 /// </summary>
 public static class BinaryExperimentResultExtensions
 {
 /// <summary>
 /// Select the best run from an enumeration of experiment runs.
 /// </summary>
 /// <param name="results">Enumeration of AutoML experiment run results.</param>
 /// <param name="metric">Metric to consider when selecting the best run.</param>
 /// <returns>The best experiment run.</returns>
 public static RunDetail<BinaryClassificationMetrics> Best(this IEnumerable<RunDetail<BinaryClassificationMetrics>> results, BinaryClassificationMetric metric = BinaryClassificationMetric.Accuracy)
 private protected override RunDetail<BinaryClassificationMetrics> GetBestRun(IEnumerable<RunDetail<BinaryClassificationMetrics>> results)
 {
 var metricsAgent = new BinaryMetricsAgent(null, metric);
 var isMetricMaximizing = new OptimizingMetricInfo(metric).IsMaximizing;
 return BestResultUtil.GetBestRun(results, metricsAgent, isMetricMaximizing);
 return BestResultUtil.GetBestRun(results, MetricsAgent, OptimizingMetricInfo.IsMaximizing);
 }

 /// <summary>
 /// Select the best run from an enumeration of experiment cross validation runs.
 /// </summary>
 /// <param name="results">Enumeration of AutoML experiment cross validation run results.</param>
 /// <param name="metric">Metric to consider when selecting the best run.</param>
 /// <returns>The best experiment run.</returns>
 public static CrossValidationRunDetail<BinaryClassificationMetrics> Best(this IEnumerable<CrossValidationRunDetail<BinaryClassificationMetrics>> results, BinaryClassificationMetric metric = BinaryClassificationMetric.Accuracy)
 private protected override CrossValidationRunDetail<BinaryClassificationMetrics> GetBestCrossValRun(IEnumerable<CrossValidationRunDetail<BinaryClassificationMetrics>> results)
 {
 var metricsAgent = new BinaryMetricsAgent(null, metric);
 var isMetricMaximizing = new OptimizingMetricInfo(metric).IsMaximizing;
 return BestResultUtil.GetBestRun(results, metricsAgent, isMetricMaximizing);
 return BestResultUtil.GetBestRun(results, MetricsAgent, OptimizingMetricInfo.IsMaximizing);
 }
 }
}

4 changes: 2 additions & 2 deletions
src/Microsoft.ML.Auto/API/ColumnInference.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -28,7 +28,7 @@ public sealed class ColumnInferenceResults
 /// <remarks>
 /// <para>Contains the inferred purposes of each column. See <see cref="Auto.ColumnInformation"/> for more details.</para>
 /// <para>This can be fed to the AutoML API when running an experiment. 
 /// See <typeref cref="ExperimentBase{TMetrics}.Execute(IDataView, ColumnInformation, IEstimator{ITransformer}, System.IProgress{RunDetail{TMetrics}})" />
 /// See <typeref cref="ExperimentBase{TMetrics, TExperimentSettings}.Execute(IDataView, ColumnInformation, IEstimator{ITransformer}, System.IProgress{RunDetail{TMetrics}})" />
 /// for example.</para>
 /// </remarks>
 public ColumnInformation ColumnInformation { get; internal set; } = new ColumnInformation();
Expand All@@ -42,7 +42,7 @@ public sealed class ColumnInferenceResults
 /// it enumerates the dataset columns that AutoML should treat as categorical, 
 /// the columns AutoML should ignore, which column is the label, etc.</para>
 /// <para><see cref="ColumnInformation"/> can be fed to the AutoML API when running an experiment.
 /// See <typeref cref="ExperimentBase{TMetrics}.Execute(IDataView, ColumnInformation, IEstimator{ITransformer}, System.IProgress{RunDetail{TMetrics}})" />
 /// See <typeref cref="ExperimentBase{TMetrics, TExperimentSettings}.Execute(IDataView, ColumnInformation, IEstimator{ITransformer}, System.IProgress{RunDetail{TMetrics}})" />
 /// for example.</para>
 /// </remarks>
 public sealed class ColumnInformation
Expand Down

99 changes: 56 additions & 43 deletions
src/Microsoft.ML.Auto/API/ExperimentBase.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -12,27 +12,32 @@ namespace Microsoft.ML.Auto
 /// (like <see cref="BinaryClassificationExperiment"/>) inherit from this class.
 /// </summary>
 /// <typeparam name="TMetrics">Metrics type used by task-specific AutoML experiments.</typeparam>
 public abstract class ExperimentBase<TMetrics> where TMetrics : class
 /// <typeparam name="TExperimentSettings">Experiment settings type.</typeparam>
 public abstract class ExperimentBase<TMetrics, TExperimentSettings> 
 where TMetrics : class
 where TExperimentSettings : ExperimentSettings
 {
 private protected readonly MLContext Context;
 private protected readonly IMetricsAgent<TMetrics> MetricsAgent;
 private protected readonly OptimizingMetricInfo OptimizingMetricInfo;
 private protected readonly TExperimentSettings Settings;

 private readonly IMetricsAgent<TMetrics> _metricsAgent;
 private readonly OptimizingMetricInfo _optimizingMetricInfo;
 private readonly ExperimentSettings _settings;
 private readonly AutoMLLogger _logger;
 private readonly TaskKind _task;
 private readonly IEnumerable<TrainerName> _trainerWhitelist;

 internal ExperimentBase(MLContext context,
 IMetricsAgent<TMetrics> metricsAgent,
 OptimizingMetricInfo optimizingMetricInfo,
ExperimentSettings settings,
TExperimentSettings settings,
 TaskKind task,
 IEnumerable<TrainerName> trainerWhitelist)
 {
 Context = context;
 _metricsAgent = metricsAgent;
 _optimizingMetricInfo = optimizingMetricInfo;
 _settings = settings;
 MetricsAgent = metricsAgent;
 OptimizingMetricInfo = optimizingMetricInfo;
 Settings = settings;
 _logger = new AutoMLLogger(context);
 _task = task;
 _trainerWhitelist = trainerWhitelist;
 }
Expand All@@ -53,12 +58,11 @@ internal ExperimentBase(MLContext context,
 /// <see cref="IProgress{T}.Report(T)"/> after each model it produces during the 
 /// course of the experiment.
 /// </param>
 /// <returns>An enumeration of all the runs in an experiment. See <see cref="RunDetail{TMetrics}"/>
 /// for more information on the contents of a run.</returns>
 /// <returns>The experiment result.</returns>
 /// <remarks>
 /// Depending on the size of your data, the AutoML experiment could take a long time to execute.
 /// </remarks>
 public IEnumerable<RunDetail<TMetrics>> Execute(IDataView trainData, string labelColumnName = DefaultColumnNames.Label,
 public ExperimentResult<TMetrics> Execute(IDataView trainData, string labelColumnName = DefaultColumnNames.Label,
 string samplingKeyColumn = null, IEstimator<ITransformer> preFeaturizer = null, IProgress<RunDetail<TMetrics>> progressHandler = null)
 {
 var columnInformation = new ColumnInformation()
Expand All@@ -83,12 +87,11 @@ public IEnumerable<RunDetail<TMetrics>> Execute(IDataView trainData, string labe
 /// <see cref="IProgress{T}.Report(T)"/> after each model it produces during the 
 /// course of the experiment.
 /// </param>
 /// <returns>An enumeration of all the runs in an experiment. See <see cref="RunDetail{TMetrics}"/>
 /// for more information on the contents of a run.</returns>
 /// <returns>The experiment result.</returns>
 /// <remarks>
 /// Depending on the size of your data, the AutoML experiment could take a long time to execute.
 /// </remarks>
 public IEnumerable<RunDetail<TMetrics>> Execute(IDataView trainData, ColumnInformation columnInformation, 
 public ExperimentResult<TMetrics> Execute(IDataView trainData, ColumnInformation columnInformation, 
 IEstimator<ITransformer> preFeaturizer = null, IProgress<RunDetail<TMetrics>> progressHandler = null)
 {
 // Cross val threshold for # of dataset rows --
Expand DownExpand Up@@ -126,12 +129,11 @@ public IEnumerable<RunDetail<TMetrics>> Execute(IDataView trainData, ColumnInfor
 /// <see cref="IProgress{T}.Report(T)"/> after each model it produces during the 
 /// course of the experiment.
 /// </param>
 /// <returns>An enumeration of all the runs in an experiment. See <see cref="RunDetail{TMetrics}"/>
 /// for more information on the contents of a run.</returns>
 /// <returns>The experiment result.</returns>
 /// <remarks>
 /// Depending on the size of your data, the AutoML experiment could take a long time to execute.
 /// </remarks>
 public IEnumerable<RunDetail<TMetrics>> Execute(IDataView trainData, IDataView validationData, string labelColumnName = DefaultColumnNames.Label, IEstimator<ITransformer> preFeaturizer = null, IProgress<RunDetail<TMetrics>> progressHandler = null)
 public ExperimentResult<TMetrics> Execute(IDataView trainData, IDataView validationData, string labelColumnName = DefaultColumnNames.Label, IEstimator<ITransformer> preFeaturizer = null, IProgress<RunDetail<TMetrics>> progressHandler = null)
 {
 var columnInformation = new ColumnInformation() { LabelColumnName = labelColumnName };
 return Execute(trainData, validationData, columnInformation, preFeaturizer, progressHandler);
Expand All@@ -152,12 +154,11 @@ public IEnumerable<RunDetail<TMetrics>> Execute(IDataView trainData, IDataView v
 /// <see cref="IProgress{T}.Report(T)"/> after each model it produces during the 
 /// course of the experiment.
 /// </param>
 /// <returns>An enumeration of all the runs in an experiment. See <see cref="RunDetail{TMetrics}"/>
 /// for more information on the contents of a run.</returns>
 /// <returns>The experiment result.</returns>
 /// <remarks>
 /// Depending on the size of your data, the AutoML experiment could take a long time to execute.
 /// </remarks>
 public IEnumerable<RunDetail<TMetrics>> Execute(IDataView trainData, IDataView validationData, ColumnInformation columnInformation, IEstimator<ITransformer> preFeaturizer = null, IProgress<RunDetail<TMetrics>> progressHandler = null)
 public ExperimentResult<TMetrics> Execute(IDataView trainData, IDataView validationData, ColumnInformation columnInformation, IEstimator<ITransformer> preFeaturizer = null, IProgress<RunDetail<TMetrics>> progressHandler = null)
 {
 if (validationData == null)
 {
Expand All@@ -183,12 +184,11 @@ public IEnumerable<RunDetail<TMetrics>> Execute(IDataView trainData, IDataView v
 /// <see cref="IProgress{T}.Report(T)"/> after each model it produces during the 
 /// course of the experiment.
 /// </param>
 /// <returns>An enumeration of all the runs in an experiment. See <see cref="RunDetail{TMetrics}"/>
 /// for more information on the contents of a run.</returns>
 /// <returns>The cross validation experiment result.</returns>
 /// <remarks>
 /// Depending on the size of your data, the AutoML experiment could take a long time to execute.
 /// </remarks>
 public IEnumerable<CrossValidationRunDetail<TMetrics>> Execute(IDataView trainData, uint numberOfCVFolds, ColumnInformation columnInformation = null, IEstimator<ITransformer> preFeaturizer = null, IProgress<CrossValidationRunDetail<TMetrics>> progressHandler = null)
 public CrossValidationExperimentResult<TMetrics> Execute(IDataView trainData, uint numberOfCVFolds, ColumnInformation columnInformation = null, IEstimator<ITransformer> preFeaturizer = null, IProgress<CrossValidationRunDetail<TMetrics>> progressHandler = null)
 {
 UserInputValidationUtil.ValidateNumberOfCVFoldsArg(numberOfCVFolds);
 var splitResult = SplitUtil.CrossValSplit(Context, trainData, numberOfCVFolds, columnInformation?.SamplingKeyColumnName);
Expand All@@ -211,12 +211,11 @@ public IEnumerable<CrossValidationRunDetail<TMetrics>> Execute(IDataView trainDa
 /// <see cref="IProgress{T}.Report(T)"/> after each model it produces during the 
 /// course of the experiment.
 /// </param>
 /// <returns>An enumeration of all the runs in an experiment. See <see cref="RunDetail{TMetrics}"/>
 /// for more information on the contents of a run.</returns>
 /// <returns>The cross validation experiment result.</returns>
 /// <remarks>
 /// Depending on the size of your data, the AutoML experiment could take a long time to execute.
 /// </remarks>
 public IEnumerable<CrossValidationRunDetail<TMetrics>> Execute(IDataView trainData, 
 public CrossValidationExperimentResult<TMetrics> Execute(IDataView trainData, 
 uint numberOfCVFolds, string labelColumnName = DefaultColumnNames.Label,
 string samplingKeyColumn = null, IEstimator<ITransformer> preFeaturizer = null, 
 Progress<CrossValidationRunDetail<TMetrics>> progressHandler = null)
Expand All@@ -229,7 +228,11 @@ public IEnumerable<CrossValidationRunDetail<TMetrics>> Execute(IDataView trainDa
 return Execute(trainData, numberOfCVFolds, columnInformation, preFeaturizer, progressHandler);
 }

 private IEnumerable<RunDetail<TMetrics>> ExecuteTrainValidate(IDataView trainData,
 private protected abstract CrossValidationRunDetail<TMetrics> GetBestCrossValRun(IEnumerable<CrossValidationRunDetail<TMetrics>> results);

 private protected abstract RunDetail<TMetrics> GetBestRun(IEnumerable<RunDetail<TMetrics>> results);

 private ExperimentResult<TMetrics> ExecuteTrainValidate(IDataView trainData,
 ColumnInformation columnInfo,
 IDataView validationData,
 IEstimator<ITransformer> preFeaturizer,
Expand All@@ -247,13 +250,13 @@ private IEnumerable<RunDetail<TMetrics>> ExecuteTrainValidate(IDataView trainDat
 validationData = preprocessorTransform.Transform(validationData);
 }

 var runner = new TrainValidateRunner<TMetrics>(Context, trainData, validationData, columnInfo.LabelColumnName, _metricsAgent,
 preFeaturizer, preprocessorTransform, _settings.DebugLogger);
 var runner = new TrainValidateRunner<TMetrics>(Context, trainData, validationData, columnInfo.LabelColumnName, MetricsAgent,
 preFeaturizer, preprocessorTransform, _logger);
 var columns = DatasetColumnInfoUtil.GetDatasetColumnInfo(Context, trainData, columnInfo);
 return Execute(columnInfo, columns, preFeaturizer, progressHandler, runner);
 }

 private IEnumerable<CrossValidationRunDetail<TMetrics>> ExecuteCrossVal(IDataView[] trainDatasets,
 private CrossValidationExperimentResult<TMetrics> ExecuteCrossVal(IDataView[] trainDatasets,
 ColumnInformation columnInfo,
 IDataView[] validationDatasets,
 IEstimator<ITransformer> preFeaturizer,
Expand All@@ -266,13 +269,21 @@ private IEnumerable<CrossValidationRunDetail<TMetrics>> ExecuteCrossVal(IDataVie
 ITransformer[] preprocessorTransforms = null;
 (trainDatasets, validationDatasets, preprocessorTransforms) = ApplyPreFeaturizerCrossVal(trainDatasets, validationDatasets, preFeaturizer);

 var runner = new CrossValRunner<TMetrics>(Context, trainDatasets, validationDatasets, _metricsAgent, preFeaturizer,
 preprocessorTransforms, columnInfo.LabelColumnName, _settings.DebugLogger);
 var runner = new CrossValRunner<TMetrics>(Context, trainDatasets, validationDatasets, MetricsAgent, preFeaturizer,
 preprocessorTransforms, columnInfo.LabelColumnName, _logger);
 var columns = DatasetColumnInfoUtil.GetDatasetColumnInfo(Context, trainDatasets[0], columnInfo);
 return Execute(columnInfo, columns, preFeaturizer, progressHandler, runner);

 // Execute experiment & get all pipelines run
 var experiment = new Experiment<CrossValidationRunDetail<TMetrics>, TMetrics>(Context, _task, OptimizingMetricInfo, progressHandler,
 Settings, MetricsAgent, _trainerWhitelist, columns, runner, _logger);
 var runDetails = experiment.Execute();

 var bestRun = GetBestCrossValRun(runDetails);
 var experimentResult = new CrossValidationExperimentResult<TMetrics>(runDetails, bestRun);
 return experimentResult;
 }

 private IEnumerable<RunDetail<TMetrics>> ExecuteCrossValSummary(IDataView[] trainDatasets,
 private ExperimentResult<TMetrics> ExecuteCrossValSummary(IDataView[] trainDatasets,
 ColumnInformation columnInfo,
 IDataView[] validationDatasets,
 IEstimator<ITransformer> preFeaturizer,
Expand All@@ -285,24 +296,26 @@ private IEnumerable<RunDetail<TMetrics>> ExecuteCrossValSummary(IDataView[] trai
 ITransformer[] preprocessorTransforms = null;
 (trainDatasets, validationDatasets, preprocessorTransforms) = ApplyPreFeaturizerCrossVal(trainDatasets, validationDatasets, preFeaturizer);

 var runner = new CrossValSummaryRunner<TMetrics>(Context, trainDatasets, validationDatasets, _metricsAgent, preFeaturizer,
 preprocessorTransforms, columnInfo.LabelColumnName, _optimizingMetricInfo, _settings.DebugLogger);
 var runner = new CrossValSummaryRunner<TMetrics>(Context, trainDatasets, validationDatasets, MetricsAgent, preFeaturizer,
 preprocessorTransforms, columnInfo.LabelColumnName, OptimizingMetricInfo, _logger);
 var columns = DatasetColumnInfoUtil.GetDatasetColumnInfo(Context, trainDatasets[0], columnInfo);
 return Execute(columnInfo, columns, preFeaturizer, progressHandler, runner);
 }

 private IEnumerable<TRunDetail> Execute<TRunDetail>(ColumnInformation columnInfo,
 private ExperimentResult<TMetrics> Execute(ColumnInformation columnInfo,
 DatasetColumnInfo[] columns,
 IEstimator<ITransformer> preFeaturizer,
 IProgress<TRunDetail> progressHandler,
 IRunner<TRunDetail> runner)
 where TRunDetail : RunDetail
 IProgress<RunDetail<TMetrics>> progressHandler,
 IRunner<RunDetail<TMetrics>> runner)
 {
 // Execute experiment & get all pipelines run
 var experiment = new Experiment<TRunDetail, TMetrics>(Context, _task, _optimizingMetricInfo, progressHandler,
 _settings, _metricsAgent, _trainerWhitelist, columns, runner);
 var experiment = new Experiment<RunDetail<TMetrics>, TMetrics>(Context, _task, OptimizingMetricInfo, progressHandler,
 Settings, MetricsAgent, _trainerWhitelist, columns, runner, _logger);
 var runDetails = experiment.Execute();

 return experiment.Execute();
 var bestRun = GetBestRun(runDetails);
 var experimentResult = new ExperimentResult<TMetrics>(runDetails, bestRun);
 return experimentResult;
 }

 private static (IDataView[] trainDatasets, IDataView[] validDatasets, ITransformer[] preprocessorTransforms)
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Auto-enable theater mode on YouTube\n(function() {\n function tryTheater() {\n var btn = document.querySelector('button[aria-label=\"Theater mode\"], ytd-player #player button[title=\"Theater mode\"]');\n if (btn && !btn.classList.contains('activated')) {\n btn.click();\n }\n }\n \n // Try immediately\n tryTheater();\n \n // Try after navigation (SPA)\n var lastUrl = location.href;\n setInterval(function() {\n if (location.href !== lastUrl) {\n lastUrl = location.href;\n setTimeout(tryTheater, 500);\n }\n }, 1000);\n \n // Also try on player load\n var observer = new MutationObserver(tryTheater);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "YouTube Theater Mode Default");
}
} catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
[AutoML] Rev AutoML public API; add required native references to AutoML projects#3364

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

daholste
merged 9 commits into

from

Apr 16, 2019
 Merged


[AutoML] Rev AutoML public API; add required native references to AutoML projects
#3364
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


9 commits


Select commit
Hold shift + click to select a range
6f0ea50

Convert CacheBeforeTrainer from nullable bool to enum


daholste Apr 16, 2019
e2aa6b9

Add ExperimentResult & CrossValidationExperimentResult classes


daholste Apr 16, 2019
966f704

AutoML test updates


daholste Apr 16, 2019
82572be

Updates for CLI


daholste Apr 16, 2019
8dcce53

In AutoML SDK, write logs to channel in MLContext


daholste Apr 16, 2019
14d2ee4

Consume AutoML SDK logs from new stream in CLI


daholste Apr 16, 2019
d799b0a

Include native references in required projects


daholste Apr 16, 2019
4b40e49

whitespace fix


daholste Apr 16, 2019
741d1c2

Unit test fixes


daholste Apr 16, 2019

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


32 changes: 5 additions & 27 deletions
src/Microsoft.ML.Auto/API/BinaryClassificationExperiment.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -132,7 +132,7 @@ public enum BinaryClassificationTrainer
 /// <summary>
 /// AutoML experiment on binary classification datasets.
 /// </summary>
 public sealed class BinaryClassificationExperiment : ExperimentBase<BinaryClassificationMetrics>
 public sealed class BinaryClassificationExperiment : ExperimentBase<BinaryClassificationMetrics, BinaryExperimentSettings>
 {
 internal BinaryClassificationExperiment(MLContext context, BinaryExperimentSettings settings)
 : base(context,
Expand All@@ -143,37 +143,15 @@ internal BinaryClassificationExperiment(MLContext context, BinaryExperimentSetti
 TrainerExtensionUtil.GetTrainerNames(settings.Trainers))
 {
 }
 }

 /// <summary>
 /// Extension methods that operate over binary experiment run results.
 /// </summary>
 public static class BinaryExperimentResultExtensions
 {
 /// <summary>
 /// Select the best run from an enumeration of experiment runs.
 /// </summary>
 /// <param name="results">Enumeration of AutoML experiment run results.</param>
 /// <param name="metric">Metric to consider when selecting the best run.</param>
 /// <returns>The best experiment run.</returns>
 public static RunDetail<BinaryClassificationMetrics> Best(this IEnumerable<RunDetail<BinaryClassificationMetrics>> results, BinaryClassificationMetric metric = BinaryClassificationMetric.Accuracy)
 private protected override RunDetail<BinaryClassificationMetrics> GetBestRun(IEnumerable<RunDetail<BinaryClassificationMetrics>> results)
 {
 var metricsAgent = new BinaryMetricsAgent(null, metric);
 var isMetricMaximizing = new OptimizingMetricInfo(metric).IsMaximizing;
 return BestResultUtil.GetBestRun(results, metricsAgent, isMetricMaximizing);
 return BestResultUtil.GetBestRun(results, MetricsAgent, OptimizingMetricInfo.IsMaximizing);
 }

 /// <summary>
 /// Select the best run from an enumeration of experiment cross validation runs.
 /// </summary>
 /// <param name="results">Enumeration of AutoML experiment cross validation run results.</param>
 /// <param name="metric">Metric to consider when selecting the best run.</param>
 /// <returns>The best experiment run.</returns>
 public static CrossValidationRunDetail<BinaryClassificationMetrics> Best(this IEnumerable<CrossValidationRunDetail<BinaryClassificationMetrics>> results, BinaryClassificationMetric metric = BinaryClassificationMetric.Accuracy)
 private protected override CrossValidationRunDetail<BinaryClassificationMetrics> GetBestCrossValRun(IEnumerable<CrossValidationRunDetail<BinaryClassificationMetrics>> results)
 {
 var metricsAgent = new BinaryMetricsAgent(null, metric);
 var isMetricMaximizing = new OptimizingMetricInfo(metric).IsMaximizing;
 return BestResultUtil.GetBestRun(results, metricsAgent, isMetricMaximizing);
 return BestResultUtil.GetBestRun(results, MetricsAgent, OptimizingMetricInfo.IsMaximizing);
 }
 }
}

4 changes: 2 additions & 2 deletions
src/Microsoft.ML.Auto/API/ColumnInference.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -28,7 +28,7 @@ public sealed class ColumnInferenceResults
 /// <remarks>
 /// <para>Contains the inferred purposes of each column. See <see cref="Auto.ColumnInformation"/> for more details.</para>
 /// <para>This can be fed to the AutoML API when running an experiment. 
 /// See <typeref cref="ExperimentBase{TMetrics}.Execute(IDataView, ColumnInformation, IEstimator{ITransformer}, System.IProgress{RunDetail{TMetrics}})" />
 /// See <typeref cref="ExperimentBase{TMetrics, TExperimentSettings}.Execute(IDataView, ColumnInformation, IEstimator{ITransformer}, System.IProgress{RunDetail{TMetrics}})" />
 /// for example.</para>
 /// </remarks>
 public ColumnInformation ColumnInformation { get; internal set; } = new ColumnInformation();
Expand All@@ -42,7 +42,7 @@ public sealed class ColumnInferenceResults
 /// it enumerates the dataset columns that AutoML should treat as categorical, 
 /// the columns AutoML should ignore, which column is the label, etc.</para>
 /// <para><see cref="ColumnInformation"/> can be fed to the AutoML API when running an experiment.
 /// See <typeref cref="ExperimentBase{TMetrics}.Execute(IDataView, ColumnInformation, IEstimator{ITransformer}, System.IProgress{RunDetail{TMetrics}})" />
 /// See <typeref cref="ExperimentBase{TMetrics, TExperimentSettings}.Execute(IDataView, ColumnInformation, IEstimator{ITransformer}, System.IProgress{RunDetail{TMetrics}})" />
 /// for example.</para>
 /// </remarks>
 public sealed class ColumnInformation
Expand Down

99 changes: 56 additions & 43 deletions
src/Microsoft.ML.Auto/API/ExperimentBase.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -12,27 +12,32 @@ namespace Microsoft.ML.Auto
 /// (like <see cref="BinaryClassificationExperiment"/>) inherit from this class.
 /// </summary>
 /// <typeparam name="TMetrics">Metrics type used by task-specific AutoML experiments.</typeparam>
 public abstract class ExperimentBase<TMetrics> where TMetrics : class
 /// <typeparam name="TExperimentSettings">Experiment settings type.</typeparam>
 public abstract class ExperimentBase<TMetrics, TExperimentSettings> 
 where TMetrics : class
 where TExperimentSettings : ExperimentSettings
 {
 private protected readonly MLContext Context;
 private protected readonly IMetricsAgent<TMetrics> MetricsAgent;
 private protected readonly OptimizingMetricInfo OptimizingMetricInfo;
 private protected readonly TExperimentSettings Settings;

 private readonly IMetricsAgent<TMetrics> _metricsAgent;
 private readonly OptimizingMetricInfo _optimizingMetricInfo;
 private readonly ExperimentSettings _settings;
 private readonly AutoMLLogger _logger;
 private readonly TaskKind _task;
 private readonly IEnumerable<TrainerName> _trainerWhitelist;

 internal ExperimentBase(MLContext context,
 IMetricsAgent<TMetrics> metricsAgent,
 OptimizingMetricInfo optimizingMetricInfo,
ExperimentSettings settings,
TExperimentSettings settings,
 TaskKind task,
 IEnumerable<TrainerName> trainerWhitelist)
 {
 Context = context;
 _metricsAgent = metricsAgent;
 _optimizingMetricInfo = optimizingMetricInfo;
 _settings = settings;
 MetricsAgent = metricsAgent;
 OptimizingMetricInfo = optimizingMetricInfo;
 Settings = settings;
 _logger = new AutoMLLogger(context);
 _task = task;
 _trainerWhitelist = trainerWhitelist;
 }
Expand All@@ -53,12 +58,11 @@ internal ExperimentBase(MLContext context,
 /// <see cref="IProgress{T}.Report(T)"/> after each model it produces during the 
 /// course of the experiment.
 /// </param>
 /// <returns>An enumeration of all the runs in an experiment. See <see cref="RunDetail{TMetrics}"/>
 /// for more information on the contents of a run.</returns>
 /// <returns>The experiment result.</returns>
 /// <remarks>
 /// Depending on the size of your data, the AutoML experiment could take a long time to execute.
 /// </remarks>
 public IEnumerable<RunDetail<TMetrics>> Execute(IDataView trainData, string labelColumnName = DefaultColumnNames.Label,
 public ExperimentResult<TMetrics> Execute(IDataView trainData, string labelColumnName = DefaultColumnNames.Label,
 string samplingKeyColumn = null, IEstimator<ITransformer> preFeaturizer = null, IProgress<RunDetail<TMetrics>> progressHandler = null)
 {
 var columnInformation = new ColumnInformation()
Expand All@@ -83,12 +87,11 @@ public IEnumerable<RunDetail<TMetrics>> Execute(IDataView trainData, string labe
 /// <see cref="IProgress{T}.Report(T)"/> after each model it produces during the 
 /// course of the experiment.
 /// </param>
 /// <returns>An enumeration of all the runs in an experiment. See <see cref="RunDetail{TMetrics}"/>
 /// for more information on the contents of a run.</returns>
 /// <returns>The experiment result.</returns>
 /// <remarks>
 /// Depending on the size of your data, the AutoML experiment could take a long time to execute.
 /// </remarks>
 public IEnumerable<RunDetail<TMetrics>> Execute(IDataView trainData, ColumnInformation columnInformation, 
 public ExperimentResult<TMetrics> Execute(IDataView trainData, ColumnInformation columnInformation, 
 IEstimator<ITransformer> preFeaturizer = null, IProgress<RunDetail<TMetrics>> progressHandler = null)
 {
 // Cross val threshold for # of dataset rows --
Expand DownExpand Up@@ -126,12 +129,11 @@ public IEnumerable<RunDetail<TMetrics>> Execute(IDataView trainData, ColumnInfor
 /// <see cref="IProgress{T}.Report(T)"/> after each model it produces during the 
 /// course of the experiment.
 /// </param>
 /// <returns>An enumeration of all the runs in an experiment. See <see cref="RunDetail{TMetrics}"/>
 /// for more information on the contents of a run.</returns>
 /// <returns>The experiment result.</returns>
 /// <remarks>
 /// Depending on the size of your data, the AutoML experiment could take a long time to execute.
 /// </remarks>
 public IEnumerable<RunDetail<TMetrics>> Execute(IDataView trainData, IDataView validationData, string labelColumnName = DefaultColumnNames.Label, IEstimator<ITransformer> preFeaturizer = null, IProgress<RunDetail<TMetrics>> progressHandler = null)
 public ExperimentResult<TMetrics> Execute(IDataView trainData, IDataView validationData, string labelColumnName = DefaultColumnNames.Label, IEstimator<ITransformer> preFeaturizer = null, IProgress<RunDetail<TMetrics>> progressHandler = null)
 {
 var columnInformation = new ColumnInformation() { LabelColumnName = labelColumnName };
 return Execute(trainData, validationData, columnInformation, preFeaturizer, progressHandler);
Expand All@@ -152,12 +154,11 @@ public IEnumerable<RunDetail<TMetrics>> Execute(IDataView trainData, IDataView v
 /// <see cref="IProgress{T}.Report(T)"/> after each model it produces during the 
 /// course of the experiment.
 /// </param>
 /// <returns>An enumeration of all the runs in an experiment. See <see cref="RunDetail{TMetrics}"/>
 /// for more information on the contents of a run.</returns>
 /// <returns>The experiment result.</returns>
 /// <remarks>
 /// Depending on the size of your data, the AutoML experiment could take a long time to execute.
 /// </remarks>
 public IEnumerable<RunDetail<TMetrics>> Execute(IDataView trainData, IDataView validationData, ColumnInformation columnInformation, IEstimator<ITransformer> preFeaturizer = null, IProgress<RunDetail<TMetrics>> progressHandler = null)
 public ExperimentResult<TMetrics> Execute(IDataView trainData, IDataView validationData, ColumnInformation columnInformation, IEstimator<ITransformer> preFeaturizer = null, IProgress<RunDetail<TMetrics>> progressHandler = null)
 {
 if (validationData == null)
 {
Expand All@@ -183,12 +184,11 @@ public IEnumerable<RunDetail<TMetrics>> Execute(IDataView trainData, IDataView v
 /// <see cref="IProgress{T}.Report(T)"/> after each model it produces during the 
 /// course of the experiment.
 /// </param>
 /// <returns>An enumeration of all the runs in an experiment. See <see cref="RunDetail{TMetrics}"/>
 /// for more information on the contents of a run.</returns>
 /// <returns>The cross validation experiment result.</returns>
 /// <remarks>
 /// Depending on the size of your data, the AutoML experiment could take a long time to execute.
 /// </remarks>
 public IEnumerable<CrossValidationRunDetail<TMetrics>> Execute(IDataView trainData, uint numberOfCVFolds, ColumnInformation columnInformation = null, IEstimator<ITransformer> preFeaturizer = null, IProgress<CrossValidationRunDetail<TMetrics>> progressHandler = null)
 public CrossValidationExperimentResult<TMetrics> Execute(IDataView trainData, uint numberOfCVFolds, ColumnInformation columnInformation = null, IEstimator<ITransformer> preFeaturizer = null, IProgress<CrossValidationRunDetail<TMetrics>> progressHandler = null)
 {
 UserInputValidationUtil.ValidateNumberOfCVFoldsArg(numberOfCVFolds);
 var splitResult = SplitUtil.CrossValSplit(Context, trainData, numberOfCVFolds, columnInformation?.SamplingKeyColumnName);
Expand All@@ -211,12 +211,11 @@ public IEnumerable<CrossValidationRunDetail<TMetrics>> Execute(IDataView trainDa
 /// <see cref="IProgress{T}.Report(T)"/> after each model it produces during the 
 /// course of the experiment.
 /// </param>
 /// <returns>An enumeration of all the runs in an experiment. See <see cref="RunDetail{TMetrics}"/>
 /// for more information on the contents of a run.</returns>
 /// <returns>The cross validation experiment result.</returns>
 /// <remarks>
 /// Depending on the size of your data, the AutoML experiment could take a long time to execute.
 /// </remarks>
 public IEnumerable<CrossValidationRunDetail<TMetrics>> Execute(IDataView trainData, 
 public CrossValidationExperimentResult<TMetrics> Execute(IDataView trainData, 
 uint numberOfCVFolds, string labelColumnName = DefaultColumnNames.Label,
 string samplingKeyColumn = null, IEstimator<ITransformer> preFeaturizer = null, 
 Progress<CrossValidationRunDetail<TMetrics>> progressHandler = null)
Expand All@@ -229,7 +228,11 @@ public IEnumerable<CrossValidationRunDetail<TMetrics>> Execute(IDataView trainDa
 return Execute(trainData, numberOfCVFolds, columnInformation, preFeaturizer, progressHandler);
 }

 private IEnumerable<RunDetail<TMetrics>> ExecuteTrainValidate(IDataView trainData,
 private protected abstract CrossValidationRunDetail<TMetrics> GetBestCrossValRun(IEnumerable<CrossValidationRunDetail<TMetrics>> results);

 private protected abstract RunDetail<TMetrics> GetBestRun(IEnumerable<RunDetail<TMetrics>> results);

 private ExperimentResult<TMetrics> ExecuteTrainValidate(IDataView trainData,
 ColumnInformation columnInfo,
 IDataView validationData,
 IEstimator<ITransformer> preFeaturizer,
Expand All@@ -247,13 +250,13 @@ private IEnumerable<RunDetail<TMetrics>> ExecuteTrainValidate(IDataView trainDat
 validationData = preprocessorTransform.Transform(validationData);
 }

 var runner = new TrainValidateRunner<TMetrics>(Context, trainData, validationData, columnInfo.LabelColumnName, _metricsAgent,
 preFeaturizer, preprocessorTransform, _settings.DebugLogger);
 var runner = new TrainValidateRunner<TMetrics>(Context, trainData, validationData, columnInfo.LabelColumnName, MetricsAgent,
 preFeaturizer, preprocessorTransform, _logger);
 var columns = DatasetColumnInfoUtil.GetDatasetColumnInfo(Context, trainData, columnInfo);
 return Execute(columnInfo, columns, preFeaturizer, progressHandler, runner);
 }

 private IEnumerable<CrossValidationRunDetail<TMetrics>> ExecuteCrossVal(IDataView[] trainDatasets,
 private CrossValidationExperimentResult<TMetrics> ExecuteCrossVal(IDataView[] trainDatasets,
 ColumnInformation columnInfo,
 IDataView[] validationDatasets,
 IEstimator<ITransformer> preFeaturizer,
Expand All@@ -266,13 +269,21 @@ private IEnumerable<CrossValidationRunDetail<TMetrics>> ExecuteCrossVal(IDataVie
 ITransformer[] preprocessorTransforms = null;
 (trainDatasets, validationDatasets, preprocessorTransforms) = ApplyPreFeaturizerCrossVal(trainDatasets, validationDatasets, preFeaturizer);

 var runner = new CrossValRunner<TMetrics>(Context, trainDatasets, validationDatasets, _metricsAgent, preFeaturizer,
 preprocessorTransforms, columnInfo.LabelColumnName, _settings.DebugLogger);
 var runner = new CrossValRunner<TMetrics>(Context, trainDatasets, validationDatasets, MetricsAgent, preFeaturizer,
 preprocessorTransforms, columnInfo.LabelColumnName, _logger);
 var columns = DatasetColumnInfoUtil.GetDatasetColumnInfo(Context, trainDatasets[0], columnInfo);
 return Execute(columnInfo, columns, preFeaturizer, progressHandler, runner);

 // Execute experiment & get all pipelines run
 var experiment = new Experiment<CrossValidationRunDetail<TMetrics>, TMetrics>(Context, _task, OptimizingMetricInfo, progressHandler,
 Settings, MetricsAgent, _trainerWhitelist, columns, runner, _logger);
 var runDetails = experiment.Execute();

 var bestRun = GetBestCrossValRun(runDetails);
 var experimentResult = new CrossValidationExperimentResult<TMetrics>(runDetails, bestRun);
 return experimentResult;
 }

 private IEnumerable<RunDetail<TMetrics>> ExecuteCrossValSummary(IDataView[] trainDatasets,
 private ExperimentResult<TMetrics> ExecuteCrossValSummary(IDataView[] trainDatasets,
 ColumnInformation columnInfo,
 IDataView[] validationDatasets,
 IEstimator<ITransformer> preFeaturizer,
Expand All@@ -285,24 +296,26 @@ private IEnumerable<RunDetail<TMetrics>> ExecuteCrossValSummary(IDataView[] trai
 ITransformer[] preprocessorTransforms = null;
 (trainDatasets, validationDatasets, preprocessorTransforms) = ApplyPreFeaturizerCrossVal(trainDatasets, validationDatasets, preFeaturizer);

 var runner = new CrossValSummaryRunner<TMetrics>(Context, trainDatasets, validationDatasets, _metricsAgent, preFeaturizer,
 preprocessorTransforms, columnInfo.LabelColumnName, _optimizingMetricInfo, _settings.DebugLogger);
 var runner = new CrossValSummaryRunner<TMetrics>(Context, trainDatasets, validationDatasets, MetricsAgent, preFeaturizer,
 preprocessorTransforms, columnInfo.LabelColumnName, OptimizingMetricInfo, _logger);
 var columns = DatasetColumnInfoUtil.GetDatasetColumnInfo(Context, trainDatasets[0], columnInfo);
 return Execute(columnInfo, columns, preFeaturizer, progressHandler, runner);
 }

 private IEnumerable<TRunDetail> Execute<TRunDetail>(ColumnInformation columnInfo,
 private ExperimentResult<TMetrics> Execute(ColumnInformation columnInfo,
 DatasetColumnInfo[] columns,
 IEstimator<ITransformer> preFeaturizer,
 IProgress<TRunDetail> progressHandler,
 IRunner<TRunDetail> runner)
 where TRunDetail : RunDetail
 IProgress<RunDetail<TMetrics>> progressHandler,
 IRunner<RunDetail<TMetrics>> runner)
 {
 // Execute experiment & get all pipelines run
 var experiment = new Experiment<TRunDetail, TMetrics>(Context, _task, _optimizingMetricInfo, progressHandler,
 _settings, _metricsAgent, _trainerWhitelist, columns, runner);
 var experiment = new Experiment<RunDetail<TMetrics>, TMetrics>(Context, _task, OptimizingMetricInfo, progressHandler,
 Settings, MetricsAgent, _trainerWhitelist, columns, runner, _logger);
 var runDetails = experiment.Execute();

 return experiment.Execute();
 var bestRun = GetBestRun(runDetails);
 var experimentResult = new ExperimentResult<TMetrics>(runDetails, bestRun);
 return experimentResult;
 }

 private static (IDataView[] trainDatasets, IDataView[] validDatasets, ITransformer[] preprocessorTransforms)
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Remove or un-stick sticky/fixed headers that block content\n(function() {\n function unstick() {\n document.querySelectorAll('header, nav, [role=\"banner\"], .header, .navbar, .sticky, .fixed-top, [style*=\"position: fixed\"], [style*=\"position:sticky\"]').forEach(function(el) {\n if (el.style.position === 'fixed' || el.style.position === 'sticky' || \n getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') {\n el.style.position = 'static';\n el.style.top = 'auto';\n el.style.zIndex = 'auto';\n }\n });\n }\n \n unstick();\n \n var observer = new MutationObserver(unstick);\n observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] });\n})();", "Kill Sticky Headers");
}
} catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
[AutoML] Rev AutoML public API; add required native references to AutoML projects#3364

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

daholste
merged 9 commits into

from

Apr 16, 2019
 Merged


[AutoML] Rev AutoML public API; add required native references to AutoML projects
#3364
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


9 commits


Select commit
Hold shift + click to select a range
6f0ea50

Convert CacheBeforeTrainer from nullable bool to enum


daholste Apr 16, 2019
e2aa6b9

Add ExperimentResult & CrossValidationExperimentResult classes


daholste Apr 16, 2019
966f704

AutoML test updates


daholste Apr 16, 2019
82572be

Updates for CLI


daholste Apr 16, 2019
8dcce53

In AutoML SDK, write logs to channel in MLContext


daholste Apr 16, 2019
14d2ee4

Consume AutoML SDK logs from new stream in CLI


daholste Apr 16, 2019
d799b0a

Include native references in required projects


daholste Apr 16, 2019
4b40e49

whitespace fix


daholste Apr 16, 2019
741d1c2

Unit test fixes


daholste Apr 16, 2019

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


32 changes: 5 additions & 27 deletions
src/Microsoft.ML.Auto/API/BinaryClassificationExperiment.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -132,7 +132,7 @@ public enum BinaryClassificationTrainer
 /// <summary>
 /// AutoML experiment on binary classification datasets.
 /// </summary>
 public sealed class BinaryClassificationExperiment : ExperimentBase<BinaryClassificationMetrics>
 public sealed class BinaryClassificationExperiment : ExperimentBase<BinaryClassificationMetrics, BinaryExperimentSettings>
 {
 internal BinaryClassificationExperiment(MLContext context, BinaryExperimentSettings settings)
 : base(context,
Expand All@@ -143,37 +143,15 @@ internal BinaryClassificationExperiment(MLContext context, BinaryExperimentSetti
 TrainerExtensionUtil.GetTrainerNames(settings.Trainers))
 {
 }
 }

 /// <summary>
 /// Extension methods that operate over binary experiment run results.
 /// </summary>
 public static class BinaryExperimentResultExtensions
 {
 /// <summary>
 /// Select the best run from an enumeration of experiment runs.
 /// </summary>
 /// <param name="results">Enumeration of AutoML experiment run results.</param>
 /// <param name="metric">Metric to consider when selecting the best run.</param>
 /// <returns>The best experiment run.</returns>
 public static RunDetail<BinaryClassificationMetrics> Best(this IEnumerable<RunDetail<BinaryClassificationMetrics>> results, BinaryClassificationMetric metric = BinaryClassificationMetric.Accuracy)
 private protected override RunDetail<BinaryClassificationMetrics> GetBestRun(IEnumerable<RunDetail<BinaryClassificationMetrics>> results)
 {
 var metricsAgent = new BinaryMetricsAgent(null, metric);
 var isMetricMaximizing = new OptimizingMetricInfo(metric).IsMaximizing;
 return BestResultUtil.GetBestRun(results, metricsAgent, isMetricMaximizing);
 return BestResultUtil.GetBestRun(results, MetricsAgent, OptimizingMetricInfo.IsMaximizing);
 }

 /// <summary>
 /// Select the best run from an enumeration of experiment cross validation runs.
 /// </summary>
 /// <param name="results">Enumeration of AutoML experiment cross validation run results.</param>
 /// <param name="metric">Metric to consider when selecting the best run.</param>
 /// <returns>The best experiment run.</returns>
 public static CrossValidationRunDetail<BinaryClassificationMetrics> Best(this IEnumerable<CrossValidationRunDetail<BinaryClassificationMetrics>> results, BinaryClassificationMetric metric = BinaryClassificationMetric.Accuracy)
 private protected override CrossValidationRunDetail<BinaryClassificationMetrics> GetBestCrossValRun(IEnumerable<CrossValidationRunDetail<BinaryClassificationMetrics>> results)
 {
 var metricsAgent = new BinaryMetricsAgent(null, metric);
 var isMetricMaximizing = new OptimizingMetricInfo(metric).IsMaximizing;
 return BestResultUtil.GetBestRun(results, metricsAgent, isMetricMaximizing);
 return BestResultUtil.GetBestRun(results, MetricsAgent, OptimizingMetricInfo.IsMaximizing);
 }
 }
}

4 changes: 2 additions & 2 deletions
src/Microsoft.ML.Auto/API/ColumnInference.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -28,7 +28,7 @@ public sealed class ColumnInferenceResults
 /// <remarks>
 /// <para>Contains the inferred purposes of each column. See <see cref="Auto.ColumnInformation"/> for more details.</para>
 /// <para>This can be fed to the AutoML API when running an experiment. 
 /// See <typeref cref="ExperimentBase{TMetrics}.Execute(IDataView, ColumnInformation, IEstimator{ITransformer}, System.IProgress{RunDetail{TMetrics}})" />
 /// See <typeref cref="ExperimentBase{TMetrics, TExperimentSettings}.Execute(IDataView, ColumnInformation, IEstimator{ITransformer}, System.IProgress{RunDetail{TMetrics}})" />
 /// for example.</para>
 /// </remarks>
 public ColumnInformation ColumnInformation { get; internal set; } = new ColumnInformation();
Expand All@@ -42,7 +42,7 @@ public sealed class ColumnInferenceResults
 /// it enumerates the dataset columns that AutoML should treat as categorical, 
 /// the columns AutoML should ignore, which column is the label, etc.</para>
 /// <para><see cref="ColumnInformation"/> can be fed to the AutoML API when running an experiment.
 /// See <typeref cref="ExperimentBase{TMetrics}.Execute(IDataView, ColumnInformation, IEstimator{ITransformer}, System.IProgress{RunDetail{TMetrics}})" />
 /// See <typeref cref="ExperimentBase{TMetrics, TExperimentSettings}.Execute(IDataView, ColumnInformation, IEstimator{ITransformer}, System.IProgress{RunDetail{TMetrics}})" />
 /// for example.</para>
 /// </remarks>
 public sealed class ColumnInformation
Expand Down

99 changes: 56 additions & 43 deletions
src/Microsoft.ML.Auto/API/ExperimentBase.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -12,27 +12,32 @@ namespace Microsoft.ML.Auto
 /// (like <see cref="BinaryClassificationExperiment"/>) inherit from this class.
 /// </summary>
 /// <typeparam name="TMetrics">Metrics type used by task-specific AutoML experiments.</typeparam>
 public abstract class ExperimentBase<TMetrics> where TMetrics : class
 /// <typeparam name="TExperimentSettings">Experiment settings type.</typeparam>
 public abstract class ExperimentBase<TMetrics, TExperimentSettings> 
 where TMetrics : class
 where TExperimentSettings : ExperimentSettings
 {
 private protected readonly MLContext Context;
 private protected readonly IMetricsAgent<TMetrics> MetricsAgent;
 private protected readonly OptimizingMetricInfo OptimizingMetricInfo;
 private protected readonly TExperimentSettings Settings;

 private readonly IMetricsAgent<TMetrics> _metricsAgent;
 private readonly OptimizingMetricInfo _optimizingMetricInfo;
 private readonly ExperimentSettings _settings;
 private readonly AutoMLLogger _logger;
 private readonly TaskKind _task;
 private readonly IEnumerable<TrainerName> _trainerWhitelist;

 internal ExperimentBase(MLContext context,
 IMetricsAgent<TMetrics> metricsAgent,
 OptimizingMetricInfo optimizingMetricInfo,
ExperimentSettings settings,
TExperimentSettings settings,
 TaskKind task,
 IEnumerable<TrainerName> trainerWhitelist)
 {
 Context = context;
 _metricsAgent = metricsAgent;
 _optimizingMetricInfo = optimizingMetricInfo;
 _settings = settings;
 MetricsAgent = metricsAgent;
 OptimizingMetricInfo = optimizingMetricInfo;
 Settings = settings;
 _logger = new AutoMLLogger(context);
 _task = task;
 _trainerWhitelist = trainerWhitelist;
 }
Expand All@@ -53,12 +58,11 @@ internal ExperimentBase(MLContext context,
 /// <see cref="IProgress{T}.Report(T)"/> after each model it produces during the 
 /// course of the experiment.
 /// </param>
 /// <returns>An enumeration of all the runs in an experiment. See <see cref="RunDetail{TMetrics}"/>
 /// for more information on the contents of a run.</returns>
 /// <returns>The experiment result.</returns>
 /// <remarks>
 /// Depending on the size of your data, the AutoML experiment could take a long time to execute.
 /// </remarks>
 public IEnumerable<RunDetail<TMetrics>> Execute(IDataView trainData, string labelColumnName = DefaultColumnNames.Label,
 public ExperimentResult<TMetrics> Execute(IDataView trainData, string labelColumnName = DefaultColumnNames.Label,
 string samplingKeyColumn = null, IEstimator<ITransformer> preFeaturizer = null, IProgress<RunDetail<TMetrics>> progressHandler = null)
 {
 var columnInformation = new ColumnInformation()
Expand All@@ -83,12 +87,11 @@ public IEnumerable<RunDetail<TMetrics>> Execute(IDataView trainData, string labe
 /// <see cref="IProgress{T}.Report(T)"/> after each model it produces during the 
 /// course of the experiment.
 /// </param>
 /// <returns>An enumeration of all the runs in an experiment. See <see cref="RunDetail{TMetrics}"/>
 /// for more information on the contents of a run.</returns>
 /// <returns>The experiment result.</returns>
 /// <remarks>
 /// Depending on the size of your data, the AutoML experiment could take a long time to execute.
 /// </remarks>
 public IEnumerable<RunDetail<TMetrics>> Execute(IDataView trainData, ColumnInformation columnInformation, 
 public ExperimentResult<TMetrics> Execute(IDataView trainData, ColumnInformation columnInformation, 
 IEstimator<ITransformer> preFeaturizer = null, IProgress<RunDetail<TMetrics>> progressHandler = null)
 {
 // Cross val threshold for # of dataset rows --
Expand DownExpand Up@@ -126,12 +129,11 @@ public IEnumerable<RunDetail<TMetrics>> Execute(IDataView trainData, ColumnInfor
 /// <see cref="IProgress{T}.Report(T)"/> after each model it produces during the 
 /// course of the experiment.
 /// </param>
 /// <returns>An enumeration of all the runs in an experiment. See <see cref="RunDetail{TMetrics}"/>
 /// for more information on the contents of a run.</returns>
 /// <returns>The experiment result.</returns>
 /// <remarks>
 /// Depending on the size of your data, the AutoML experiment could take a long time to execute.
 /// </remarks>
 public IEnumerable<RunDetail<TMetrics>> Execute(IDataView trainData, IDataView validationData, string labelColumnName = DefaultColumnNames.Label, IEstimator<ITransformer> preFeaturizer = null, IProgress<RunDetail<TMetrics>> progressHandler = null)
 public ExperimentResult<TMetrics> Execute(IDataView trainData, IDataView validationData, string labelColumnName = DefaultColumnNames.Label, IEstimator<ITransformer> preFeaturizer = null, IProgress<RunDetail<TMetrics>> progressHandler = null)
 {
 var columnInformation = new ColumnInformation() { LabelColumnName = labelColumnName };
 return Execute(trainData, validationData, columnInformation, preFeaturizer, progressHandler);
Expand All@@ -152,12 +154,11 @@ public IEnumerable<RunDetail<TMetrics>> Execute(IDataView trainData, IDataView v
 /// <see cref="IProgress{T}.Report(T)"/> after each model it produces during the 
 /// course of the experiment.
 /// </param>
 /// <returns>An enumeration of all the runs in an experiment. See <see cref="RunDetail{TMetrics}"/>
 /// for more information on the contents of a run.</returns>
 /// <returns>The experiment result.</returns>
 /// <remarks>
 /// Depending on the size of your data, the AutoML experiment could take a long time to execute.
 /// </remarks>
 public IEnumerable<RunDetail<TMetrics>> Execute(IDataView trainData, IDataView validationData, ColumnInformation columnInformation, IEstimator<ITransformer> preFeaturizer = null, IProgress<RunDetail<TMetrics>> progressHandler = null)
 public ExperimentResult<TMetrics> Execute(IDataView trainData, IDataView validationData, ColumnInformation columnInformation, IEstimator<ITransformer> preFeaturizer = null, IProgress<RunDetail<TMetrics>> progressHandler = null)
 {
 if (validationData == null)
 {
Expand All@@ -183,12 +184,11 @@ public IEnumerable<RunDetail<TMetrics>> Execute(IDataView trainData, IDataView v
 /// <see cref="IProgress{T}.Report(T)"/> after each model it produces during the 
 /// course of the experiment.
 /// </param>
 /// <returns>An enumeration of all the runs in an experiment. See <see cref="RunDetail{TMetrics}"/>
 /// for more information on the contents of a run.</returns>
 /// <returns>The cross validation experiment result.</returns>
 /// <remarks>
 /// Depending on the size of your data, the AutoML experiment could take a long time to execute.
 /// </remarks>
 public IEnumerable<CrossValidationRunDetail<TMetrics>> Execute(IDataView trainData, uint numberOfCVFolds, ColumnInformation columnInformation = null, IEstimator<ITransformer> preFeaturizer = null, IProgress<CrossValidationRunDetail<TMetrics>> progressHandler = null)
 public CrossValidationExperimentResult<TMetrics> Execute(IDataView trainData, uint numberOfCVFolds, ColumnInformation columnInformation = null, IEstimator<ITransformer> preFeaturizer = null, IProgress<CrossValidationRunDetail<TMetrics>> progressHandler = null)
 {
 UserInputValidationUtil.ValidateNumberOfCVFoldsArg(numberOfCVFolds);
 var splitResult = SplitUtil.CrossValSplit(Context, trainData, numberOfCVFolds, columnInformation?.SamplingKeyColumnName);
Expand All@@ -211,12 +211,11 @@ public IEnumerable<CrossValidationRunDetail<TMetrics>> Execute(IDataView trainDa
 /// <see cref="IProgress{T}.Report(T)"/> after each model it produces during the 
 /// course of the experiment.
 /// </param>
 /// <returns>An enumeration of all the runs in an experiment. See <see cref="RunDetail{TMetrics}"/>
 /// for more information on the contents of a run.</returns>
 /// <returns>The cross validation experiment result.</returns>
 /// <remarks>
 /// Depending on the size of your data, the AutoML experiment could take a long time to execute.
 /// </remarks>
 public IEnumerable<CrossValidationRunDetail<TMetrics>> Execute(IDataView trainData, 
 public CrossValidationExperimentResult<TMetrics> Execute(IDataView trainData, 
 uint numberOfCVFolds, string labelColumnName = DefaultColumnNames.Label,
 string samplingKeyColumn = null, IEstimator<ITransformer> preFeaturizer = null, 
 Progress<CrossValidationRunDetail<TMetrics>> progressHandler = null)
Expand All@@ -229,7 +228,11 @@ public IEnumerable<CrossValidationRunDetail<TMetrics>> Execute(IDataView trainDa
 return Execute(trainData, numberOfCVFolds, columnInformation, preFeaturizer, progressHandler);
 }

 private IEnumerable<RunDetail<TMetrics>> ExecuteTrainValidate(IDataView trainData,
 private protected abstract CrossValidationRunDetail<TMetrics> GetBestCrossValRun(IEnumerable<CrossValidationRunDetail<TMetrics>> results);

 private protected abstract RunDetail<TMetrics> GetBestRun(IEnumerable<RunDetail<TMetrics>> results);

 private ExperimentResult<TMetrics> ExecuteTrainValidate(IDataView trainData,
 ColumnInformation columnInfo,
 IDataView validationData,
 IEstimator<ITransformer> preFeaturizer,
Expand All@@ -247,13 +250,13 @@ private IEnumerable<RunDetail<TMetrics>> ExecuteTrainValidate(IDataView trainDat
 validationData = preprocessorTransform.Transform(validationData);
 }

 var runner = new TrainValidateRunner<TMetrics>(Context, trainData, validationData, columnInfo.LabelColumnName, _metricsAgent,
 preFeaturizer, preprocessorTransform, _settings.DebugLogger);
 var runner = new TrainValidateRunner<TMetrics>(Context, trainData, validationData, columnInfo.LabelColumnName, MetricsAgent,
 preFeaturizer, preprocessorTransform, _logger);
 var columns = DatasetColumnInfoUtil.GetDatasetColumnInfo(Context, trainData, columnInfo);
 return Execute(columnInfo, columns, preFeaturizer, progressHandler, runner);
 }

 private IEnumerable<CrossValidationRunDetail<TMetrics>> ExecuteCrossVal(IDataView[] trainDatasets,
 private CrossValidationExperimentResult<TMetrics> ExecuteCrossVal(IDataView[] trainDatasets,
 ColumnInformation columnInfo,
 IDataView[] validationDatasets,
 IEstimator<ITransformer> preFeaturizer,
Expand All@@ -266,13 +269,21 @@ private IEnumerable<CrossValidationRunDetail<TMetrics>> ExecuteCrossVal(IDataVie
 ITransformer[] preprocessorTransforms = null;
 (trainDatasets, validationDatasets, preprocessorTransforms) = ApplyPreFeaturizerCrossVal(trainDatasets, validationDatasets, preFeaturizer);

 var runner = new CrossValRunner<TMetrics>(Context, trainDatasets, validationDatasets, _metricsAgent, preFeaturizer,
 preprocessorTransforms, columnInfo.LabelColumnName, _settings.DebugLogger);
 var runner = new CrossValRunner<TMetrics>(Context, trainDatasets, validationDatasets, MetricsAgent, preFeaturizer,
 preprocessorTransforms, columnInfo.LabelColumnName, _logger);
 var columns = DatasetColumnInfoUtil.GetDatasetColumnInfo(Context, trainDatasets[0], columnInfo);
 return Execute(columnInfo, columns, preFeaturizer, progressHandler, runner);

 // Execute experiment & get all pipelines run
 var experiment = new Experiment<CrossValidationRunDetail<TMetrics>, TMetrics>(Context, _task, OptimizingMetricInfo, progressHandler,
 Settings, MetricsAgent, _trainerWhitelist, columns, runner, _logger);
 var runDetails = experiment.Execute();

 var bestRun = GetBestCrossValRun(runDetails);
 var experimentResult = new CrossValidationExperimentResult<TMetrics>(runDetails, bestRun);
 return experimentResult;
 }

 private IEnumerable<RunDetail<TMetrics>> ExecuteCrossValSummary(IDataView[] trainDatasets,
 private ExperimentResult<TMetrics> ExecuteCrossValSummary(IDataView[] trainDatasets,
 ColumnInformation columnInfo,
 IDataView[] validationDatasets,
 IEstimator<ITransformer> preFeaturizer,
Expand All@@ -285,24 +296,26 @@ private IEnumerable<RunDetail<TMetrics>> ExecuteCrossValSummary(IDataView[] trai
 ITransformer[] preprocessorTransforms = null;
 (trainDatasets, validationDatasets, preprocessorTransforms) = ApplyPreFeaturizerCrossVal(trainDatasets, validationDatasets, preFeaturizer);

 var runner = new CrossValSummaryRunner<TMetrics>(Context, trainDatasets, validationDatasets, _metricsAgent, preFeaturizer,
 preprocessorTransforms, columnInfo.LabelColumnName, _optimizingMetricInfo, _settings.DebugLogger);
 var runner = new CrossValSummaryRunner<TMetrics>(Context, trainDatasets, validationDatasets, MetricsAgent, preFeaturizer,
 preprocessorTransforms, columnInfo.LabelColumnName, OptimizingMetricInfo, _logger);
 var columns = DatasetColumnInfoUtil.GetDatasetColumnInfo(Context, trainDatasets[0], columnInfo);
 return Execute(columnInfo, columns, preFeaturizer, progressHandler, runner);
 }

 private IEnumerable<TRunDetail> Execute<TRunDetail>(ColumnInformation columnInfo,
 private ExperimentResult<TMetrics> Execute(ColumnInformation columnInfo,
 DatasetColumnInfo[] columns,
 IEstimator<ITransformer> preFeaturizer,
 IProgress<TRunDetail> progressHandler,
 IRunner<TRunDetail> runner)
 where TRunDetail : RunDetail
 IProgress<RunDetail<TMetrics>> progressHandler,
 IRunner<RunDetail<TMetrics>> runner)
 {
 // Execute experiment & get all pipelines run
 var experiment = new Experiment<TRunDetail, TMetrics>(Context, _task, _optimizingMetricInfo, progressHandler,
 _settings, _metricsAgent, _trainerWhitelist, columns, runner);
 var experiment = new Experiment<RunDetail<TMetrics>, TMetrics>(Context, _task, OptimizingMetricInfo, progressHandler,
 Settings, MetricsAgent, _trainerWhitelist, columns, runner, _logger);
 var runDetails = experiment.Execute();

 return experiment.Execute();
 var bestRun = GetBestRun(runDetails);
 var experimentResult = new ExperimentResult<TMetrics>(runDetails, bestRun);
 return experimentResult;
 }

 private static (IDataView[] trainDatasets, IDataView[] validDatasets, ITransformer[] preprocessorTransforms)
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Universal Dark Mode - works on any site\n(function() {\n var enabled = true;\n \n function applyDarkMode() {\n if (!enabled) return;\n \n // Create style element if it doesn't exist\n var style = document.getElementById('universal-dark-mode-style');\n if (!style) {\n style = document.createElement('style');\n style.id = 'universal-dark-mode-style';\n document.head.appendChild(style);\n }\n \n // Dark mode CSS - inverts colors but preserves images/video\n style.textContent = '\n /* Invert everything except media */\n html {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #1a1a2e !important;\n }\n \n /* Restore images, videos, iframes, canvas */\n img, video, iframe, canvas, svg, picture, [style*=\"background-image\"] {\n filter: invert(1) hue-rotate(180deg) !important;\n }\n \n /* Preserve specific elements that should not be inverted */\n .no-dark-mode, .no-dark-mode *,\n [data-theme=\"light\"], [data-theme=\"light\"],\n .ace_editor, .ace_editor *,\n .CodeMirror, .CodeMirror *,\n .monaco-editor, .monaco-editor *,\n .markdown-body pre, .markdown-body pre *,\n .highlight, .highlight *,\n pre code, pre code * {\n filter: none !important;\n }\n \n /* Fix common UI elements */\n .modal, .popup, .dropdown-menu, .tooltip, .popover {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #2d2d44 !important;\n border-color: #444 !important;\n }\n \n /* Scrollbars */\n ::-webkit-scrollbar { background: #1a1a2e !important; }\n ::-webkit-scrollbar-thumb { background: #444 !important; }\n ::-webkit-scrollbar-thumb:hover { background: #555 !important; }\n \n /* Selection */\n ::selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ::-moz-selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ';\n }\n \n function removeDarkMode() {\n var style = document.getElementById('universal-dark-mode-style');\n if (style) style.remove();\n }\n \n // Toggle with Alt+Shift+D\n document.addEventListener('keydown', function(e) {\n if (e.altKey && e.shiftKey && e.key === 'D') {\n e.preventDefault();\n enabled = !enabled;\n if (enabled) {\n applyDarkMode();\n console.log('[Universal Dark Mode] Enabled');\n } else {\n removeDarkMode();\n console.log('[Universal Dark Mode] Disabled');\n }\n }\n });\n \n // Apply on load\n applyDarkMode();\n \n // Re-apply on dynamic content\n var observer = new MutationObserver(function(mutations) {\n if (enabled && !document.getElementById('universal-dark-mode-style')) {\n applyDarkMode();\n }\n });\n observer.observe(document.head, { childList: true });\n \n console.log('[Universal Dark Mode] Loaded - Press Alt+Shift+D to toggle');\n})();", "Universal Dark Mode");
}
} catch(__e) { console.warn('[Userscript:Universal Dark Mode]', __e); }
})();
})();

Skip to content
[AutoML] Rev AutoML public API; add required native references to AutoML projects#3364

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

daholste
merged 9 commits into

from

Apr 16, 2019
 Merged


[AutoML] Rev AutoML public API; add required native references to AutoML projects
#3364
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


9 commits


Select commit
Hold shift + click to select a range
6f0ea50

Convert CacheBeforeTrainer from nullable bool to enum


daholste Apr 16, 2019
e2aa6b9

Add ExperimentResult & CrossValidationExperimentResult classes


daholste Apr 16, 2019
966f704

AutoML test updates


daholste Apr 16, 2019
82572be

Updates for CLI


daholste Apr 16, 2019
8dcce53

In AutoML SDK, write logs to channel in MLContext


daholste Apr 16, 2019
14d2ee4

Consume AutoML SDK logs from new stream in CLI


daholste Apr 16, 2019
d799b0a

Include native references in required projects


daholste Apr 16, 2019
4b40e49

whitespace fix


daholste Apr 16, 2019
741d1c2

Unit test fixes


daholste Apr 16, 2019

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


32 changes: 5 additions & 27 deletions
src/Microsoft.ML.Auto/API/BinaryClassificationExperiment.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -132,7 +132,7 @@ public enum BinaryClassificationTrainer
 /// <summary>
 /// AutoML experiment on binary classification datasets.
 /// </summary>
 public sealed class BinaryClassificationExperiment : ExperimentBase<BinaryClassificationMetrics>
 public sealed class BinaryClassificationExperiment : ExperimentBase<BinaryClassificationMetrics, BinaryExperimentSettings>
 {
 internal BinaryClassificationExperiment(MLContext context, BinaryExperimentSettings settings)
 : base(context,
Expand All@@ -143,37 +143,15 @@ internal BinaryClassificationExperiment(MLContext context, BinaryExperimentSetti
 TrainerExtensionUtil.GetTrainerNames(settings.Trainers))
 {
 }
 }

 /// <summary>
 /// Extension methods that operate over binary experiment run results.
 /// </summary>
 public static class BinaryExperimentResultExtensions
 {
 /// <summary>
 /// Select the best run from an enumeration of experiment runs.
 /// </summary>
 /// <param name="results">Enumeration of AutoML experiment run results.</param>
 /// <param name="metric">Metric to consider when selecting the best run.</param>
 /// <returns>The best experiment run.</returns>
 public static RunDetail<BinaryClassificationMetrics> Best(this IEnumerable<RunDetail<BinaryClassificationMetrics>> results, BinaryClassificationMetric metric = BinaryClassificationMetric.Accuracy)
 private protected override RunDetail<BinaryClassificationMetrics> GetBestRun(IEnumerable<RunDetail<BinaryClassificationMetrics>> results)
 {
 var metricsAgent = new BinaryMetricsAgent(null, metric);
 var isMetricMaximizing = new OptimizingMetricInfo(metric).IsMaximizing;
 return BestResultUtil.GetBestRun(results, metricsAgent, isMetricMaximizing);
 return BestResultUtil.GetBestRun(results, MetricsAgent, OptimizingMetricInfo.IsMaximizing);
 }

 /// <summary>
 /// Select the best run from an enumeration of experiment cross validation runs.
 /// </summary>
 /// <param name="results">Enumeration of AutoML experiment cross validation run results.</param>
 /// <param name="metric">Metric to consider when selecting the best run.</param>
 /// <returns>The best experiment run.</returns>
 public static CrossValidationRunDetail<BinaryClassificationMetrics> Best(this IEnumerable<CrossValidationRunDetail<BinaryClassificationMetrics>> results, BinaryClassificationMetric metric = BinaryClassificationMetric.Accuracy)
 private protected override CrossValidationRunDetail<BinaryClassificationMetrics> GetBestCrossValRun(IEnumerable<CrossValidationRunDetail<BinaryClassificationMetrics>> results)
 {
 var metricsAgent = new BinaryMetricsAgent(null, metric);
 var isMetricMaximizing = new OptimizingMetricInfo(metric).IsMaximizing;
 return BestResultUtil.GetBestRun(results, metricsAgent, isMetricMaximizing);
 return BestResultUtil.GetBestRun(results, MetricsAgent, OptimizingMetricInfo.IsMaximizing);
 }
 }
}

4 changes: 2 additions & 2 deletions
src/Microsoft.ML.Auto/API/ColumnInference.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -28,7 +28,7 @@ public sealed class ColumnInferenceResults
 /// <remarks>
 /// <para>Contains the inferred purposes of each column. See <see cref="Auto.ColumnInformation"/> for more details.</para>
 /// <para>This can be fed to the AutoML API when running an experiment. 
 /// See <typeref cref="ExperimentBase{TMetrics}.Execute(IDataView, ColumnInformation, IEstimator{ITransformer}, System.IProgress{RunDetail{TMetrics}})" />
 /// See <typeref cref="ExperimentBase{TMetrics, TExperimentSettings}.Execute(IDataView, ColumnInformation, IEstimator{ITransformer}, System.IProgress{RunDetail{TMetrics}})" />
 /// for example.</para>
 /// </remarks>
 public ColumnInformation ColumnInformation { get; internal set; } = new ColumnInformation();
Expand All@@ -42,7 +42,7 @@ public sealed class ColumnInferenceResults
 /// it enumerates the dataset columns that AutoML should treat as categorical, 
 /// the columns AutoML should ignore, which column is the label, etc.</para>
 /// <para><see cref="ColumnInformation"/> can be fed to the AutoML API when running an experiment.
 /// See <typeref cref="ExperimentBase{TMetrics}.Execute(IDataView, ColumnInformation, IEstimator{ITransformer}, System.IProgress{RunDetail{TMetrics}})" />
 /// See <typeref cref="ExperimentBase{TMetrics, TExperimentSettings}.Execute(IDataView, ColumnInformation, IEstimator{ITransformer}, System.IProgress{RunDetail{TMetrics}})" />
 /// for example.</para>
 /// </remarks>
 public sealed class ColumnInformation
Expand Down

99 changes: 56 additions & 43 deletions
src/Microsoft.ML.Auto/API/ExperimentBase.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -12,27 +12,32 @@ namespace Microsoft.ML.Auto
 /// (like <see cref="BinaryClassificationExperiment"/>) inherit from this class.
 /// </summary>
 /// <typeparam name="TMetrics">Metrics type used by task-specific AutoML experiments.</typeparam>
 public abstract class ExperimentBase<TMetrics> where TMetrics : class
 /// <typeparam name="TExperimentSettings">Experiment settings type.</typeparam>
 public abstract class ExperimentBase<TMetrics, TExperimentSettings> 
 where TMetrics : class
 where TExperimentSettings : ExperimentSettings
 {
 private protected readonly MLContext Context;
 private protected readonly IMetricsAgent<TMetrics> MetricsAgent;
 private protected readonly OptimizingMetricInfo OptimizingMetricInfo;
 private protected readonly TExperimentSettings Settings;

 private readonly IMetricsAgent<TMetrics> _metricsAgent;
 private readonly OptimizingMetricInfo _optimizingMetricInfo;
 private readonly ExperimentSettings _settings;
 private readonly AutoMLLogger _logger;
 private readonly TaskKind _task;
 private readonly IEnumerable<TrainerName> _trainerWhitelist;

 internal ExperimentBase(MLContext context,
 IMetricsAgent<TMetrics> metricsAgent,
 OptimizingMetricInfo optimizingMetricInfo,
ExperimentSettings settings,
TExperimentSettings settings,
 TaskKind task,
 IEnumerable<TrainerName> trainerWhitelist)
 {
 Context = context;
 _metricsAgent = metricsAgent;
 _optimizingMetricInfo = optimizingMetricInfo;
 _settings = settings;
 MetricsAgent = metricsAgent;
 OptimizingMetricInfo = optimizingMetricInfo;
 Settings = settings;
 _logger = new AutoMLLogger(context);
 _task = task;
 _trainerWhitelist = trainerWhitelist;
 }
Expand All@@ -53,12 +58,11 @@ internal ExperimentBase(MLContext context,
 /// <see cref="IProgress{T}.Report(T)"/> after each model it produces during the 
 /// course of the experiment.
 /// </param>
 /// <returns>An enumeration of all the runs in an experiment. See <see cref="RunDetail{TMetrics}"/>
 /// for more information on the contents of a run.</returns>
 /// <returns>The experiment result.</returns>
 /// <remarks>
 /// Depending on the size of your data, the AutoML experiment could take a long time to execute.
 /// </remarks>
 public IEnumerable<RunDetail<TMetrics>> Execute(IDataView trainData, string labelColumnName = DefaultColumnNames.Label,
 public ExperimentResult<TMetrics> Execute(IDataView trainData, string labelColumnName = DefaultColumnNames.Label,
 string samplingKeyColumn = null, IEstimator<ITransformer> preFeaturizer = null, IProgress<RunDetail<TMetrics>> progressHandler = null)
 {
 var columnInformation = new ColumnInformation()
Expand All@@ -83,12 +87,11 @@ public IEnumerable<RunDetail<TMetrics>> Execute(IDataView trainData, string labe
 /// <see cref="IProgress{T}.Report(T)"/> after each model it produces during the 
 /// course of the experiment.
 /// </param>
 /// <returns>An enumeration of all the runs in an experiment. See <see cref="RunDetail{TMetrics}"/>
 /// for more information on the contents of a run.</returns>
 /// <returns>The experiment result.</returns>
 /// <remarks>
 /// Depending on the size of your data, the AutoML experiment could take a long time to execute.
 /// </remarks>
 public IEnumerable<RunDetail<TMetrics>> Execute(IDataView trainData, ColumnInformation columnInformation, 
 public ExperimentResult<TMetrics> Execute(IDataView trainData, ColumnInformation columnInformation, 
 IEstimator<ITransformer> preFeaturizer = null, IProgress<RunDetail<TMetrics>> progressHandler = null)
 {
 // Cross val threshold for # of dataset rows --
Expand DownExpand Up@@ -126,12 +129,11 @@ public IEnumerable<RunDetail<TMetrics>> Execute(IDataView trainData, ColumnInfor
 /// <see cref="IProgress{T}.Report(T)"/> after each model it produces during the 
 /// course of the experiment.
 /// </param>
 /// <returns>An enumeration of all the runs in an experiment. See <see cref="RunDetail{TMetrics}"/>
 /// for more information on the contents of a run.</returns>
 /// <returns>The experiment result.</returns>
 /// <remarks>
 /// Depending on the size of your data, the AutoML experiment could take a long time to execute.
 /// </remarks>
 public IEnumerable<RunDetail<TMetrics>> Execute(IDataView trainData, IDataView validationData, string labelColumnName = DefaultColumnNames.Label, IEstimator<ITransformer> preFeaturizer = null, IProgress<RunDetail<TMetrics>> progressHandler = null)
 public ExperimentResult<TMetrics> Execute(IDataView trainData, IDataView validationData, string labelColumnName = DefaultColumnNames.Label, IEstimator<ITransformer> preFeaturizer = null, IProgress<RunDetail<TMetrics>> progressHandler = null)
 {
 var columnInformation = new ColumnInformation() { LabelColumnName = labelColumnName };
 return Execute(trainData, validationData, columnInformation, preFeaturizer, progressHandler);
Expand All@@ -152,12 +154,11 @@ public IEnumerable<RunDetail<TMetrics>> Execute(IDataView trainData, IDataView v
 /// <see cref="IProgress{T}.Report(T)"/> after each model it produces during the 
 /// course of the experiment.
 /// </param>
 /// <returns>An enumeration of all the runs in an experiment. See <see cref="RunDetail{TMetrics}"/>
 /// for more information on the contents of a run.</returns>
 /// <returns>The experiment result.</returns>
 /// <remarks>
 /// Depending on the size of your data, the AutoML experiment could take a long time to execute.
 /// </remarks>
 public IEnumerable<RunDetail<TMetrics>> Execute(IDataView trainData, IDataView validationData, ColumnInformation columnInformation, IEstimator<ITransformer> preFeaturizer = null, IProgress<RunDetail<TMetrics>> progressHandler = null)
 public ExperimentResult<TMetrics> Execute(IDataView trainData, IDataView validationData, ColumnInformation columnInformation, IEstimator<ITransformer> preFeaturizer = null, IProgress<RunDetail<TMetrics>> progressHandler = null)
 {
 if (validationData == null)
 {
Expand All@@ -183,12 +184,11 @@ public IEnumerable<RunDetail<TMetrics>> Execute(IDataView trainData, IDataView v
 /// <see cref="IProgress{T}.Report(T)"/> after each model it produces during the 
 /// course of the experiment.
 /// </param>
 /// <returns>An enumeration of all the runs in an experiment. See <see cref="RunDetail{TMetrics}"/>
 /// for more information on the contents of a run.</returns>
 /// <returns>The cross validation experiment result.</returns>
 /// <remarks>
 /// Depending on the size of your data, the AutoML experiment could take a long time to execute.
 /// </remarks>
 public IEnumerable<CrossValidationRunDetail<TMetrics>> Execute(IDataView trainData, uint numberOfCVFolds, ColumnInformation columnInformation = null, IEstimator<ITransformer> preFeaturizer = null, IProgress<CrossValidationRunDetail<TMetrics>> progressHandler = null)
 public CrossValidationExperimentResult<TMetrics> Execute(IDataView trainData, uint numberOfCVFolds, ColumnInformation columnInformation = null, IEstimator<ITransformer> preFeaturizer = null, IProgress<CrossValidationRunDetail<TMetrics>> progressHandler = null)
 {
 UserInputValidationUtil.ValidateNumberOfCVFoldsArg(numberOfCVFolds);
 var splitResult = SplitUtil.CrossValSplit(Context, trainData, numberOfCVFolds, columnInformation?.SamplingKeyColumnName);
Expand All@@ -211,12 +211,11 @@ public IEnumerable<CrossValidationRunDetail<TMetrics>> Execute(IDataView trainDa
 /// <see cref="IProgress{T}.Report(T)"/> after each model it produces during the 
 /// course of the experiment.
 /// </param>
 /// <returns>An enumeration of all the runs in an experiment. See <see cref="RunDetail{TMetrics}"/>
 /// for more information on the contents of a run.</returns>
 /// <returns>The cross validation experiment result.</returns>
 /// <remarks>
 /// Depending on the size of your data, the AutoML experiment could take a long time to execute.
 /// </remarks>
 public IEnumerable<CrossValidationRunDetail<TMetrics>> Execute(IDataView trainData, 
 public CrossValidationExperimentResult<TMetrics> Execute(IDataView trainData, 
 uint numberOfCVFolds, string labelColumnName = DefaultColumnNames.Label,
 string samplingKeyColumn = null, IEstimator<ITransformer> preFeaturizer = null, 
 Progress<CrossValidationRunDetail<TMetrics>> progressHandler = null)
Expand All@@ -229,7 +228,11 @@ public IEnumerable<CrossValidationRunDetail<TMetrics>> Execute(IDataView trainDa
 return Execute(trainData, numberOfCVFolds, columnInformation, preFeaturizer, progressHandler);
 }

 private IEnumerable<RunDetail<TMetrics>> ExecuteTrainValidate(IDataView trainData,
 private protected abstract CrossValidationRunDetail<TMetrics> GetBestCrossValRun(IEnumerable<CrossValidationRunDetail<TMetrics>> results);

 private protected abstract RunDetail<TMetrics> GetBestRun(IEnumerable<RunDetail<TMetrics>> results);

 private ExperimentResult<TMetrics> ExecuteTrainValidate(IDataView trainData,
 ColumnInformation columnInfo,
 IDataView validationData,
 IEstimator<ITransformer> preFeaturizer,
Expand All@@ -247,13 +250,13 @@ private IEnumerable<RunDetail<TMetrics>> ExecuteTrainValidate(IDataView trainDat
 validationData = preprocessorTransform.Transform(validationData);
 }

 var runner = new TrainValidateRunner<TMetrics>(Context, trainData, validationData, columnInfo.LabelColumnName, _metricsAgent,
 preFeaturizer, preprocessorTransform, _settings.DebugLogger);
 var runner = new TrainValidateRunner<TMetrics>(Context, trainData, validationData, columnInfo.LabelColumnName, MetricsAgent,
 preFeaturizer, preprocessorTransform, _logger);
 var columns = DatasetColumnInfoUtil.GetDatasetColumnInfo(Context, trainData, columnInfo);
 return Execute(columnInfo, columns, preFeaturizer, progressHandler, runner);
 }

 private IEnumerable<CrossValidationRunDetail<TMetrics>> ExecuteCrossVal(IDataView[] trainDatasets,
 private CrossValidationExperimentResult<TMetrics> ExecuteCrossVal(IDataView[] trainDatasets,
 ColumnInformation columnInfo,
 IDataView[] validationDatasets,
 IEstimator<ITransformer> preFeaturizer,
Expand All@@ -266,13 +269,21 @@ private IEnumerable<CrossValidationRunDetail<TMetrics>> ExecuteCrossVal(IDataVie
 ITransformer[] preprocessorTransforms = null;
 (trainDatasets, validationDatasets, preprocessorTransforms) = ApplyPreFeaturizerCrossVal(trainDatasets, validationDatasets, preFeaturizer);

 var runner = new CrossValRunner<TMetrics>(Context, trainDatasets, validationDatasets, _metricsAgent, preFeaturizer,
 preprocessorTransforms, columnInfo.LabelColumnName, _settings.DebugLogger);
 var runner = new CrossValRunner<TMetrics>(Context, trainDatasets, validationDatasets, MetricsAgent, preFeaturizer,
 preprocessorTransforms, columnInfo.LabelColumnName, _logger);
 var columns = DatasetColumnInfoUtil.GetDatasetColumnInfo(Context, trainDatasets[0], columnInfo);
 return Execute(columnInfo, columns, preFeaturizer, progressHandler, runner);

 // Execute experiment & get all pipelines run
 var experiment = new Experiment<CrossValidationRunDetail<TMetrics>, TMetrics>(Context, _task, OptimizingMetricInfo, progressHandler,
 Settings, MetricsAgent, _trainerWhitelist, columns, runner, _logger);
 var runDetails = experiment.Execute();

 var bestRun = GetBestCrossValRun(runDetails);
 var experimentResult = new CrossValidationExperimentResult<TMetrics>(runDetails, bestRun);
 return experimentResult;
 }

 private IEnumerable<RunDetail<TMetrics>> ExecuteCrossValSummary(IDataView[] trainDatasets,
 private ExperimentResult<TMetrics> ExecuteCrossValSummary(IDataView[] trainDatasets,
 ColumnInformation columnInfo,
 IDataView[] validationDatasets,
 IEstimator<ITransformer> preFeaturizer,
Expand All@@ -285,24 +296,26 @@ private IEnumerable<RunDetail<TMetrics>> ExecuteCrossValSummary(IDataView[] trai
 ITransformer[] preprocessorTransforms = null;
 (trainDatasets, validationDatasets, preprocessorTransforms) = ApplyPreFeaturizerCrossVal(trainDatasets, validationDatasets, preFeaturizer);

 var runner = new CrossValSummaryRunner<TMetrics>(Context, trainDatasets, validationDatasets, _metricsAgent, preFeaturizer,
 preprocessorTransforms, columnInfo.LabelColumnName, _optimizingMetricInfo, _settings.DebugLogger);
 var runner = new CrossValSummaryRunner<TMetrics>(Context, trainDatasets, validationDatasets, MetricsAgent, preFeaturizer,
 preprocessorTransforms, columnInfo.LabelColumnName, OptimizingMetricInfo, _logger);
 var columns = DatasetColumnInfoUtil.GetDatasetColumnInfo(Context, trainDatasets[0], columnInfo);
 return Execute(columnInfo, columns, preFeaturizer, progressHandler, runner);
 }

 private IEnumerable<TRunDetail> Execute<TRunDetail>(ColumnInformation columnInfo,
 private ExperimentResult<TMetrics> Execute(ColumnInformation columnInfo,
 DatasetColumnInfo[] columns,
 IEstimator<ITransformer> preFeaturizer,
 IProgress<TRunDetail> progressHandler,
 IRunner<TRunDetail> runner)
 where TRunDetail : RunDetail
 IProgress<RunDetail<TMetrics>> progressHandler,
 IRunner<RunDetail<TMetrics>> runner)
 {
 // Execute experiment & get all pipelines run
 var experiment = new Experiment<TRunDetail, TMetrics>(Context, _task, _optimizingMetricInfo, progressHandler,
 _settings, _metricsAgent, _trainerWhitelist, columns, runner);
 var experiment = new Experiment<RunDetail<TMetrics>, TMetrics>(Context, _task, OptimizingMetricInfo, progressHandler,
 Settings, MetricsAgent, _trainerWhitelist, columns, runner, _logger);
 var runDetails = experiment.Execute();

 return experiment.Execute();
 var bestRun = GetBestRun(runDetails);
 var experimentResult = new ExperimentResult<TMetrics>(runDetails, bestRun);
 return experimentResult;
 }

 private static (IDataView[] trainDatasets, IDataView[] validDatasets, ITransformer[] preprocessorTransforms)
Expand Down
Loading