Add SrCnn entire API by implementing function

docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectEntireAnomalyBySrCnn.cs

@@ -0,0 +1,99 @@
+using System;
+using System.Collections.Generic;
+using Microsoft.ML;
+using Microsoft.ML.Data;
+using Microsoft.ML.TimeSeries;
+
+namespace Samples.Dynamic
+{
+ public static class DetectEntireAnomalyBySrCnn
+ {
+ public static void Example()
+ {
+ // Create a new ML context, for ML.NET operations. It can be used for
+ // exception tracking and logging, 
+ // as well as the source of randomness.
+ var ml = new MLContext();
+
+ // Generate sample series data with an anomaly
+ var data = new List<TimeSeriesData>();
+ for (int index = 0; index < 20; index++)
+ {
+ data.Add(new TimeSeriesData { Value = 5 });
+ }
+ data.Add(new TimeSeriesData { Value = 10 });
+ for (int index = 0; index < 5; index++)
+ {
+ data.Add(new TimeSeriesData { Value = 5 });
+ }
+
+ // Convert data to IDataView.
+ var dataView = ml.Data.LoadFromEnumerable(data);
+
+ // Setup the detection arguments
+ string outputColumnName = nameof(SrCnnAnomalyDetection.Prediction);
+ string inputColumnName = nameof(TimeSeriesData.Value);
+
+ // Do batch anomaly detection
+ var outputDataView = ml.AnomalyDetection.DetectEntireAnomalyBySrCnn(dataView, outputColumnName, inputColumnName,
+ threshold: 0.35, batchSize: 512, sensitivity: 90.0, detectMode: SrCnnDetectMode.AnomalyAndMargin);
+
+ // Getting the data of the newly created column as an IEnumerable of
+ // SrCnnAnomalyDetection.
+ var predictionColumn = ml.Data.CreateEnumerable<SrCnnAnomalyDetection>(
+ outputDataView, reuseRowObject: false);
+
+ Console.WriteLine("Index\tData\tAnomaly\tAnomalyScore\tMag\tExpectedValue\tBoundaryUnit\tUpperBoundary\tLowerBoundary");
+
+ int k = 0;
+ foreach (var prediction in predictionColumn)
+ {
+ PrintPrediction(k, data[k].Value, prediction);
+ k++;
+ }
+ //Index Data Anomaly AnomalyScore Mag ExpectedValue BoundaryUnit UpperBoundary LowerBoundary
+ //0 5.00 0 0.00 0.21 5.00 5.00 5.01 4.99
+ //1 5.00 0 0.00 0.11 5.00 5.00 5.01 4.99
+ //2 5.00 0 0.00 0.03 5.00 5.00 5.01 4.99
+ //3 5.00 0 0.00 0.01 5.00 5.00 5.01 4.99
+ //4 5.00 0 0.00 0.03 5.00 5.00 5.01 4.99
+ //5 5.00 0 0.00 0.06 5.00 5.00 5.01 4.99
+ //6 5.00 0 0.00 0.02 5.00 5.00 5.01 4.99
+ //7 5.00 0 0.00 0.01 5.00 5.00 5.01 4.99
+ //8 5.00 0 0.00 0.01 5.00 5.00 5.01 4.99
+ //9 5.00 0 0.00 0.01 5.00 5.00 5.01 4.99
+ //10 5.00 0 0.00 0.00 5.00 5.00 5.01 4.99
+ //11 5.00 0 0.00 0.01 5.00 5.00 5.01 4.99
+ //12 5.00 0 0.00 0.01 5.00 5.00 5.01 4.99
+ //13 5.00 0 0.00 0.02 5.00 5.00 5.01 4.99
+ //14 5.00 0 0.00 0.07 5.00 5.00 5.01 4.99
+ //15 5.00 0 0.00 0.08 5.00 5.00 5.01 4.99
+ //16 5.00 0 0.00 0.02 5.00 5.00 5.01 4.99
+ //17 5.00 0 0.00 0.05 5.00 5.00 5.01 4.99
+ //18 5.00 0 0.00 0.12 5.00 5.00 5.01 4.99
+ //19 5.00 0 0.00 0.17 5.00 5.00 5.01 4.99
+ //20 10.00 1 0.50 0.80 5.00 5.00 5.01 4.99
+ //21 5.00 0 0.00 0.16 5.00 5.00 5.01 4.99
+ //22 5.00 0 0.00 0.11 5.00 5.00 5.01 4.99
+ //23 5.00 0 0.00 0.05 5.00 5.00 5.01 4.99
+ //24 5.00 0 0.00 0.11 5.00 5.00 5.01 4.99
+ //25 5.00 0 0.00 0.19 5.00 5.00 5.01 4.99
+ }
+
+ private static void PrintPrediction(int idx, double value, SrCnnAnomalyDetection prediction) =>
+ Console.WriteLine("{0}\t{1:0.00}\t{2}\t\t{3:0.00}\t{4:0.00}\t\t{5:0.00}\t\t{6:0.00}\t\t{7:0.00}\t\t{8:0.00}",
+ idx, value, prediction.Prediction[0], prediction.Prediction[1], prediction.Prediction[2],
+ prediction.Prediction[3], prediction.Prediction[4], prediction.Prediction[5], prediction.Prediction[6]);
+
+ private class TimeSeriesData
+ {
+ public double Value { get; set; }
+ }
+
+ private class SrCnnAnomalyDetection
+ {
+ [VectorType]
+ public double[] Prediction { get; set; }
+ }
+ }
+}

src/Microsoft.ML.Data/DataView/BatchDataViewMapperBase.cs

@@ -0,0 +1,173 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using Microsoft.ML.Runtime;
+
+namespace Microsoft.ML.Data.DataView
+{
+ internal abstract class BatchDataViewMapperBase<TInput, TBatch> : IDataView
+ {
+ public bool CanShuffle => false;
+
+ public DataViewSchema Schema => SchemaBindings.AsSchema;
+
+ private readonly IDataView _source;
+ protected readonly IHost Host;
+
+ protected BatchDataViewMapperBase(IHostEnvironment env, string registrationName, IDataView input)
+ {
+ Contracts.CheckValue(env, nameof(env));
+ Host = env.Register(registrationName);
+ _source = input;
+ }
+
+ public long? GetRowCount() => _source.GetRowCount();
+
+ public DataViewRowCursor GetRowCursor(IEnumerable<DataViewSchema.Column> columnsNeeded, Random rand = null)
+ {
+ Host.CheckValue(columnsNeeded, nameof(columnsNeeded));
+ Host.CheckValueOrNull(rand);
+
+ var predicate = RowCursorUtils.FromColumnsToPredicate(columnsNeeded, SchemaBindings.AsSchema);
+
+ // If we aren't selecting any of the output columns, don't construct our cursor.
+ // Note that because we cannot support random due to the inherently
+ // stratified nature, neither can we allow the base data to be shuffled,
+ // even if it supports shuffling.
+ if (!SchemaBindings.AnyNewColumnsActive(predicate))
+ {
+ var activeInput = SchemaBindings.GetActiveInput(predicate);
+ var inputCursor = _source.GetRowCursor(_source.Schema.Where(c => activeInput[c.Index]), null);
+ return new BindingsWrappedRowCursor(Host, inputCursor, SchemaBindings);
+ }
+ var active = SchemaBindings.GetActive(predicate);
+ Contracts.Assert(active.Length == SchemaBindings.ColumnCount);
+
+ // REVIEW: We can get a different input predicate for the input cursor and for the lookahead cursor. The lookahead
+ // cursor is only used for getting the values from the input column, so it only needs that column activated. The
+ // other cursor is used to get source columns, so it needs the rest of them activated.
+ var predInput = GetSchemaBindingDependencies(predicate);
+ var inputCols = _source.Schema.Where(c => predInput(c.Index));
+ return new Cursor(this, _source.GetRowCursor(inputCols), _source.GetRowCursor(inputCols), active);
+ }
+
+ public DataViewRowCursor[] GetRowCursorSet(IEnumerable<DataViewSchema.Column> columnsNeeded, int n, Random rand = null)
+ {
+ return new[] { GetRowCursor(columnsNeeded, rand) };
+ }
+
+ protected abstract ColumnBindingsBase SchemaBindings { get; }
+ protected abstract TBatch CreateBatch(DataViewRowCursor input);
+ protected abstract void ProcessBatch(TBatch currentBatch);
+ protected abstract void ProcessExample(TBatch currentBatch, TInput currentInput);
+ protected abstract Func<bool> GetLastInBatchDelegate(DataViewRowCursor lookAheadCursor);
+ protected abstract Func<bool> GetIsNewBatchDelegate(DataViewRowCursor lookAheadCursor);
+ protected abstract ValueGetter<TInput> GetLookAheadGetter(DataViewRowCursor lookAheadCursor);
+ protected abstract Delegate[] CreateGetters(DataViewRowCursor input, TBatch currentBatch, bool[] active);
+ protected abstract Func<int, bool> GetSchemaBindingDependencies(Func<int, bool> predicate);
+
+ private sealed class Cursor : RootCursorBase
+ {
+ private readonly BatchDataViewMapperBase<TInput, TBatch> _parent;
+ private readonly DataViewRowCursor _lookAheadCursor;
+ private readonly DataViewRowCursor _input;
+
+ private readonly bool[] _active;
+ private readonly Delegate[] _getters;
+
+ private readonly TBatch _currentBatch;
+ private readonly Func<bool> _lastInBatchInLookAheadCursorDel;
+ private readonly Func<bool> _firstInBatchInInputCursorDel;
+ private readonly ValueGetter<TInput> _inputGetterInLookAheadCursor;
+ private TInput _currentInput;
+
+ public override long Batch => 0;
+
+ public override DataViewSchema Schema => _parent.Schema;
+
+ public Cursor(BatchDataViewMapperBase<TInput, TBatch> parent, DataViewRowCursor input, DataViewRowCursor lookAheadCursor, bool[] active)
+ : base(parent.Host)
+ {
+ _parent = parent;
+ _input = input;
+ _lookAheadCursor = lookAheadCursor;
+ _active = active;
+
+ _currentBatch = _parent.CreateBatch(_input);
+
+ _getters = _parent.CreateGetters(_input, _currentBatch, _active);
+
+ _lastInBatchInLookAheadCursorDel = _parent.GetLastInBatchDelegate(_lookAheadCursor);
+ _firstInBatchInInputCursorDel = _parent.GetIsNewBatchDelegate(_input);
+ _inputGetterInLookAheadCursor = _parent.GetLookAheadGetter(_lookAheadCursor);
+ }
+
+ public override ValueGetter<TValue> GetGetter<TValue>(DataViewSchema.Column column)
+ {
+ Contracts.CheckParam(IsColumnActive(column), nameof(column), "requested column is not active");
+
+ var col = _parent.SchemaBindings.MapColumnIndex(out bool isSrc, column.Index);
+ if (isSrc)
+ {
+ Contracts.AssertValue(_input);
+ return _input.GetGetter<TValue>(_input.Schema[col]);
+ }
+
+ Ch.AssertValue(_getters);
+ var getter = _getters[col];
+ Ch.Assert(getter != null);
+ var fn = getter as ValueGetter<TValue>;
+ if (fn == null)
+ throw Ch.Except("Invalid TValue in GetGetter: '{0}'", typeof(TValue));
+ return fn;
+ }
+
+ public override ValueGetter<DataViewRowId> GetIdGetter()
+ {
+ return
+ (ref DataViewRowId val) =>
+ {
+ Ch.Check(IsGood, "Cannot call ID getter in current state");
+ val = new DataViewRowId((ulong)Position, 0);
+ };
+ }
+
+ public override bool IsColumnActive(DataViewSchema.Column column)
+ {
+ Ch.Check(column.Index < _parent.SchemaBindings.AsSchema.Count);
+ return _active[column.Index];
+ }
+
+ protected override bool MoveNextCore()
+ {
+ if (!_input.MoveNext())
+ return false;
+ if (!_firstInBatchInInputCursorDel())
+ return true;
+
+ // If we are here, this means that _input.MoveNext() has gotten us to the beginning of the next batch,
+ // so now we need to look ahead at the entire next batch in the _lookAheadCursor.
+ // The _lookAheadCursor's position should be on the last row of the previous batch (or -1).
+ Ch.Assert(_lastInBatchInLookAheadCursorDel());
+
+ var good = _lookAheadCursor.MoveNext();
+ // The two cursors should have the same number of elements, so if _input.MoveNext() returned true,
+ // then it must return true here too.
+ Ch.Assert(good);
+
+ do
+ {
+ _inputGetterInLookAheadCursor(ref _currentInput);
+ _parent.ProcessExample(_currentBatch, _currentInput);
+ } while (!_lastInBatchInLookAheadCursorDel() && _lookAheadCursor.MoveNext());
+
+ _parent.ProcessBatch(_currentBatch);
+ return true;
+ }
+ }
+ }
+}

src/Microsoft.ML.TimeSeries/ExtensionsCatalog.cs

@@ -2,8 +2,6 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 // See the LICENSE file in the project root for more information.
 
-using System;
-using System.Reflection;
 using Microsoft.ML.Data;
 using Microsoft.ML.Runtime;
 using Microsoft.ML.TimeSeries;
@@ -150,6 +148,35 @@ public static SrCnnAnomalyEstimator DetectAnomalyBySrCnn(this TransformsCatalog
  int windowSize = 64, int backAddWindowSize = 5, int lookaheadWindowSize = 5, int averageingWindowSize = 3, int judgementWindowSize = 21, double threshold = 0.3)
  => new SrCnnAnomalyEstimator(CatalogUtils.GetEnvironment(catalog), outputColumnName, windowSize, backAddWindowSize, lookaheadWindowSize, averageingWindowSize, judgementWindowSize, threshold, inputColumnName);
 
+ /// <summary>
+ /// Create <see cref="SrCnnEntireAnomalyDetector"/>, which detects timeseries anomalies for entire input using SRCNN algorithm.
+ /// </summary>
+ /// <param name="catalog">The AnomalyDetectionCatalog.</param>
+ /// <param name="input">Input DataView.</param>
+ /// <param name="outputColumnName">Name of the column resulting from data processing of <paramref name="inputColumnName"/>.
+ /// The column data is a vector of <see cref="System.Double"/>. The length of this vector varies depending on <paramref name="detectMode"/>.</param>
+ /// <param name="inputColumnName">Name of column to process. The column data must be <see cref="System.Double"/>.</param>
+ /// <param name="threshold">The threshold to determine anomaly, score larger than the threshold is considered as anomaly. Must be in [0,1]. Default value is 0.3.</param>
+ /// <param name="batchSize">Divide the input data into batches to fit srcnn model.
+ /// When set to -1, use the whole input to fit model instead of batch by batch, when set to a positive integer, use this number as batch size.
+ /// Must be -1 or a positive integer no less than 12. Default value is 1024.</param>
+ /// <param name="sensitivity">Sensitivity of boundaries, only useful when srCnnDetectMode is AnomalyAndMargin. Must be in [0,100]. Default value is 99.</param>
+ /// <param name="detectMode">An enum type of <see cref="SrCnnDetectMode"/>.
+ /// When set to AnomalyOnly, the output vector would be a 3-element Double vector of (IsAnomaly, RawScore, Mag).
+ /// When set to AnomalyAndExpectedValue, the output vector would be a 4-element Double vector of (IsAnomaly, RawScore, Mag, ExpectedValue).
+ /// When set to AnomalyAndMargin, the output vector would be a 7-element Double vector of (IsAnomaly, AnomalyScore, Mag, ExpectedValue, BoundaryUnit, UpperBoundary, LowerBoundary).
+ /// Default value is AnomalyOnly.</param>
+ /// <example>
+ /// <format type="text/markdown">
+ /// <![CDATA[
+ /// [!code-csharp[DetectEntireAnomalyBySrCnn](~/../docs/samples/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectEntireAnomalyBySrCnn.cs)]
+ /// ]]>
+ /// </format>
+ /// </example>
+ public static IDataView DetectEntireAnomalyBySrCnn(this AnomalyDetectionCatalog catalog, IDataView input, string outputColumnName, string inputColumnName,
+ double threshold = 0.3, int batchSize = 1024, double sensitivity = 99, SrCnnDetectMode detectMode = SrCnnDetectMode.AnomalyOnly)
+ => new SrCnnEntireAnomalyDetector(CatalogUtils.GetEnvironment(catalog), input, inputColumnName, outputColumnName, threshold, batchSize, sensitivity, detectMode);
+
  /// <summary>
  /// Create <see cref="RootCause"/>, which localizes root causes using decision tree algorithm.
  /// </summary>

docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectEntireAnomalyBySrCnn.cs

@@ -0,0 +1,99 @@
+using System;
+using System.Collections.Generic;
+using Microsoft.ML;
+using Microsoft.ML.Data;
+using Microsoft.ML.TimeSeries;
+
+namespace Samples.Dynamic
+{
+ public static class DetectEntireAnomalyBySrCnn
+ {
+ public static void Example()
+ {
+ // Create a new ML context, for ML.NET operations. It can be used for
+ // exception tracking and logging, 
+ // as well as the source of randomness.
+ var ml = new MLContext();
+
+ // Generate sample series data with an anomaly
+ var data = new List<TimeSeriesData>();
+ for (int index = 0; index < 20; index++)
+ {
+ data.Add(new TimeSeriesData { Value = 5 });
+ }
+ data.Add(new TimeSeriesData { Value = 10 });
+ for (int index = 0; index < 5; index++)
+ {
+ data.Add(new TimeSeriesData { Value = 5 });
+ }
+
+ // Convert data to IDataView.
+ var dataView = ml.Data.LoadFromEnumerable(data);
+
+ // Setup the detection arguments
+ string outputColumnName = nameof(SrCnnAnomalyDetection.Prediction);
+ string inputColumnName = nameof(TimeSeriesData.Value);
+
+ // Do batch anomaly detection
+ var outputDataView = ml.AnomalyDetection.DetectEntireAnomalyBySrCnn(dataView, outputColumnName, inputColumnName,
+ threshold: 0.35, batchSize: 512, sensitivity: 90.0, detectMode: SrCnnDetectMode.AnomalyAndMargin);
+
+ // Getting the data of the newly created column as an IEnumerable of
+ // SrCnnAnomalyDetection.
+ var predictionColumn = ml.Data.CreateEnumerable<SrCnnAnomalyDetection>(
+ outputDataView, reuseRowObject: false);
+
+ Console.WriteLine("Index\tData\tAnomaly\tAnomalyScore\tMag\tExpectedValue\tBoundaryUnit\tUpperBoundary\tLowerBoundary");
+
+ int k = 0;
+ foreach (var prediction in predictionColumn)
+ {
+ PrintPrediction(k, data[k].Value, prediction);
+ k++;
+ }
+ //Index Data Anomaly AnomalyScore Mag ExpectedValue BoundaryUnit UpperBoundary LowerBoundary
+ //0 5.00 0 0.00 0.21 5.00 5.00 5.01 4.99
+ //1 5.00 0 0.00 0.11 5.00 5.00 5.01 4.99
+ //2 5.00 0 0.00 0.03 5.00 5.00 5.01 4.99
+ //3 5.00 0 0.00 0.01 5.00 5.00 5.01 4.99
+ //4 5.00 0 0.00 0.03 5.00 5.00 5.01 4.99
+ //5 5.00 0 0.00 0.06 5.00 5.00 5.01 4.99
+ //6 5.00 0 0.00 0.02 5.00 5.00 5.01 4.99
+ //7 5.00 0 0.00 0.01 5.00 5.00 5.01 4.99
+ //8 5.00 0 0.00 0.01 5.00 5.00 5.01 4.99
+ //9 5.00 0 0.00 0.01 5.00 5.00 5.01 4.99
+ //10 5.00 0 0.00 0.00 5.00 5.00 5.01 4.99
+ //11 5.00 0 0.00 0.01 5.00 5.00 5.01 4.99
+ //12 5.00 0 0.00 0.01 5.00 5.00 5.01 4.99
+ //13 5.00 0 0.00 0.02 5.00 5.00 5.01 4.99
+ //14 5.00 0 0.00 0.07 5.00 5.00 5.01 4.99
+ //15 5.00 0 0.00 0.08 5.00 5.00 5.01 4.99
+ //16 5.00 0 0.00 0.02 5.00 5.00 5.01 4.99
+ //17 5.00 0 0.00 0.05 5.00 5.00 5.01 4.99
+ //18 5.00 0 0.00 0.12 5.00 5.00 5.01 4.99
+ //19 5.00 0 0.00 0.17 5.00 5.00 5.01 4.99
+ //20 10.00 1 0.50 0.80 5.00 5.00 5.01 4.99
+ //21 5.00 0 0.00 0.16 5.00 5.00 5.01 4.99
+ //22 5.00 0 0.00 0.11 5.00 5.00 5.01 4.99
+ //23 5.00 0 0.00 0.05 5.00 5.00 5.01 4.99
+ //24 5.00 0 0.00 0.11 5.00 5.00 5.01 4.99
+ //25 5.00 0 0.00 0.19 5.00 5.00 5.01 4.99
+ }
+
+ private static void PrintPrediction(int idx, double value, SrCnnAnomalyDetection prediction) =>
+ Console.WriteLine("{0}\t{1:0.00}\t{2}\t\t{3:0.00}\t{4:0.00}\t\t{5:0.00}\t\t{6:0.00}\t\t{7:0.00}\t\t{8:0.00}",
+ idx, value, prediction.Prediction[0], prediction.Prediction[1], prediction.Prediction[2],
+ prediction.Prediction[3], prediction.Prediction[4], prediction.Prediction[5], prediction.Prediction[6]);
+
+ private class TimeSeriesData
+ {
+ public double Value { get; set; }
+ }
+
+ private class SrCnnAnomalyDetection
+ {
+ [VectorType]
+ public double[] Prediction { get; set; }
+ }
+ }
+}

src/Microsoft.ML.Data/DataView/BatchDataViewMapperBase.cs

@@ -0,0 +1,173 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using Microsoft.ML.Runtime;
+
+namespace Microsoft.ML.Data.DataView
+{
+ internal abstract class BatchDataViewMapperBase<TInput, TBatch> : IDataView
+ {
+ public bool CanShuffle => false;
+
+ public DataViewSchema Schema => SchemaBindings.AsSchema;
+
+ private readonly IDataView _source;
+ protected readonly IHost Host;
+
+ protected BatchDataViewMapperBase(IHostEnvironment env, string registrationName, IDataView input)
+ {
+ Contracts.CheckValue(env, nameof(env));
+ Host = env.Register(registrationName);
+ _source = input;
+ }
+
+ public long? GetRowCount() => _source.GetRowCount();
+
+ public DataViewRowCursor GetRowCursor(IEnumerable<DataViewSchema.Column> columnsNeeded, Random rand = null)
+ {
+ Host.CheckValue(columnsNeeded, nameof(columnsNeeded));
+ Host.CheckValueOrNull(rand);
+
+ var predicate = RowCursorUtils.FromColumnsToPredicate(columnsNeeded, SchemaBindings.AsSchema);
+
+ // If we aren't selecting any of the output columns, don't construct our cursor.
+ // Note that because we cannot support random due to the inherently
+ // stratified nature, neither can we allow the base data to be shuffled,
+ // even if it supports shuffling.
+ if (!SchemaBindings.AnyNewColumnsActive(predicate))
+ {
+ var activeInput = SchemaBindings.GetActiveInput(predicate);
+ var inputCursor = _source.GetRowCursor(_source.Schema.Where(c => activeInput[c.Index]), null);
+ return new BindingsWrappedRowCursor(Host, inputCursor, SchemaBindings);
+ }
+ var active = SchemaBindings.GetActive(predicate);
+ Contracts.Assert(active.Length == SchemaBindings.ColumnCount);
+
+ // REVIEW: We can get a different input predicate for the input cursor and for the lookahead cursor. The lookahead
+ // cursor is only used for getting the values from the input column, so it only needs that column activated. The
+ // other cursor is used to get source columns, so it needs the rest of them activated.
+ var predInput = GetSchemaBindingDependencies(predicate);
+ var inputCols = _source.Schema.Where(c => predInput(c.Index));
+ return new Cursor(this, _source.GetRowCursor(inputCols), _source.GetRowCursor(inputCols), active);
+ }
+
+ public DataViewRowCursor[] GetRowCursorSet(IEnumerable<DataViewSchema.Column> columnsNeeded, int n, Random rand = null)
+ {
+ return new[] { GetRowCursor(columnsNeeded, rand) };
+ }
+
+ protected abstract ColumnBindingsBase SchemaBindings { get; }
+ protected abstract TBatch CreateBatch(DataViewRowCursor input);
+ protected abstract void ProcessBatch(TBatch currentBatch);
+ protected abstract void ProcessExample(TBatch currentBatch, TInput currentInput);
+ protected abstract Func<bool> GetLastInBatchDelegate(DataViewRowCursor lookAheadCursor);
+ protected abstract Func<bool> GetIsNewBatchDelegate(DataViewRowCursor lookAheadCursor);
+ protected abstract ValueGetter<TInput> GetLookAheadGetter(DataViewRowCursor lookAheadCursor);
+ protected abstract Delegate[] CreateGetters(DataViewRowCursor input, TBatch currentBatch, bool[] active);
+ protected abstract Func<int, bool> GetSchemaBindingDependencies(Func<int, bool> predicate);
+
+ private sealed class Cursor : RootCursorBase
+ {
+ private readonly BatchDataViewMapperBase<TInput, TBatch> _parent;
+ private readonly DataViewRowCursor _lookAheadCursor;
+ private readonly DataViewRowCursor _input;
+
+ private readonly bool[] _active;
+ private readonly Delegate[] _getters;
+
+ private readonly TBatch _currentBatch;
+ private readonly Func<bool> _lastInBatchInLookAheadCursorDel;
+ private readonly Func<bool> _firstInBatchInInputCursorDel;
+ private readonly ValueGetter<TInput> _inputGetterInLookAheadCursor;
+ private TInput _currentInput;
+
+ public override long Batch => 0;
+
+ public override DataViewSchema Schema => _parent.Schema;
+
+ public Cursor(BatchDataViewMapperBase<TInput, TBatch> parent, DataViewRowCursor input, DataViewRowCursor lookAheadCursor, bool[] active)
+ : base(parent.Host)
+ {
+ _parent = parent;
+ _input = input;
+ _lookAheadCursor = lookAheadCursor;
+ _active = active;
+
+ _currentBatch = _parent.CreateBatch(_input);
+
+ _getters = _parent.CreateGetters(_input, _currentBatch, _active);
+
+ _lastInBatchInLookAheadCursorDel = _parent.GetLastInBatchDelegate(_lookAheadCursor);
+ _firstInBatchInInputCursorDel = _parent.GetIsNewBatchDelegate(_input);
+ _inputGetterInLookAheadCursor = _parent.GetLookAheadGetter(_lookAheadCursor);
+ }
+
+ public override ValueGetter<TValue> GetGetter<TValue>(DataViewSchema.Column column)
+ {
+ Contracts.CheckParam(IsColumnActive(column), nameof(column), "requested column is not active");
+
+ var col = _parent.SchemaBindings.MapColumnIndex(out bool isSrc, column.Index);
+ if (isSrc)
+ {
+ Contracts.AssertValue(_input);
+ return _input.GetGetter<TValue>(_input.Schema[col]);
+ }
+
+ Ch.AssertValue(_getters);
+ var getter = _getters[col];
+ Ch.Assert(getter != null);
+ var fn = getter as ValueGetter<TValue>;
+ if (fn == null)
+ throw Ch.Except("Invalid TValue in GetGetter: '{0}'", typeof(TValue));
+ return fn;
+ }
+
+ public override ValueGetter<DataViewRowId> GetIdGetter()
+ {
+ return
+ (ref DataViewRowId val) =>
+ {
+ Ch.Check(IsGood, "Cannot call ID getter in current state");
+ val = new DataViewRowId((ulong)Position, 0);
+ };
+ }
+
+ public override bool IsColumnActive(DataViewSchema.Column column)
+ {
+ Ch.Check(column.Index < _parent.SchemaBindings.AsSchema.Count);
+ return _active[column.Index];
+ }
+
+ protected override bool MoveNextCore()
+ {
+ if (!_input.MoveNext())
+ return false;
+ if (!_firstInBatchInInputCursorDel())
+ return true;
+
+ // If we are here, this means that _input.MoveNext() has gotten us to the beginning of the next batch,
+ // so now we need to look ahead at the entire next batch in the _lookAheadCursor.
+ // The _lookAheadCursor's position should be on the last row of the previous batch (or -1).
+ Ch.Assert(_lastInBatchInLookAheadCursorDel());
+
+ var good = _lookAheadCursor.MoveNext();
+ // The two cursors should have the same number of elements, so if _input.MoveNext() returned true,
+ // then it must return true here too.
+ Ch.Assert(good);
+
+ do
+ {
+ _inputGetterInLookAheadCursor(ref _currentInput);
+ _parent.ProcessExample(_currentBatch, _currentInput);
+ } while (!_lastInBatchInLookAheadCursorDel() && _lookAheadCursor.MoveNext());
+
+ _parent.ProcessBatch(_currentBatch);
+ return true;
+ }
+ }
+ }
+}

src/Microsoft.ML.TimeSeries/ExtensionsCatalog.cs

@@ -2,8 +2,6 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 // See the LICENSE file in the project root for more information.
 
-using System;
-using System.Reflection;
 using Microsoft.ML.Data;
 using Microsoft.ML.Runtime;
 using Microsoft.ML.TimeSeries;
@@ -150,6 +148,35 @@ public static SrCnnAnomalyEstimator DetectAnomalyBySrCnn(this TransformsCatalog
  int windowSize = 64, int backAddWindowSize = 5, int lookaheadWindowSize = 5, int averageingWindowSize = 3, int judgementWindowSize = 21, double threshold = 0.3)
  => new SrCnnAnomalyEstimator(CatalogUtils.GetEnvironment(catalog), outputColumnName, windowSize, backAddWindowSize, lookaheadWindowSize, averageingWindowSize, judgementWindowSize, threshold, inputColumnName);
 
+ /// <summary>
+ /// Create <see cref="SrCnnEntireAnomalyDetector"/>, which detects timeseries anomalies for entire input using SRCNN algorithm.
+ /// </summary>
+ /// <param name="catalog">The AnomalyDetectionCatalog.</param>
+ /// <param name="input">Input DataView.</param>
+ /// <param name="outputColumnName">Name of the column resulting from data processing of <paramref name="inputColumnName"/>.
+ /// The column data is a vector of <see cref="System.Double"/>. The length of this vector varies depending on <paramref name="detectMode"/>.</param>
+ /// <param name="inputColumnName">Name of column to process. The column data must be <see cref="System.Double"/>.</param>
+ /// <param name="threshold">The threshold to determine anomaly, score larger than the threshold is considered as anomaly. Must be in [0,1]. Default value is 0.3.</param>
+ /// <param name="batchSize">Divide the input data into batches to fit srcnn model.
+ /// When set to -1, use the whole input to fit model instead of batch by batch, when set to a positive integer, use this number as batch size.
+ /// Must be -1 or a positive integer no less than 12. Default value is 1024.</param>
+ /// <param name="sensitivity">Sensitivity of boundaries, only useful when srCnnDetectMode is AnomalyAndMargin. Must be in [0,100]. Default value is 99.</param>
+ /// <param name="detectMode">An enum type of <see cref="SrCnnDetectMode"/>.
+ /// When set to AnomalyOnly, the output vector would be a 3-element Double vector of (IsAnomaly, RawScore, Mag).
+ /// When set to AnomalyAndExpectedValue, the output vector would be a 4-element Double vector of (IsAnomaly, RawScore, Mag, ExpectedValue).
+ /// When set to AnomalyAndMargin, the output vector would be a 7-element Double vector of (IsAnomaly, AnomalyScore, Mag, ExpectedValue, BoundaryUnit, UpperBoundary, LowerBoundary).
+ /// Default value is AnomalyOnly.</param>
+ /// <example>
+ /// <format type="text/markdown">
+ /// <![CDATA[
+ /// [!code-csharp[DetectEntireAnomalyBySrCnn](~/../docs/samples/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectEntireAnomalyBySrCnn.cs)]
+ /// ]]>
+ /// </format>
+ /// </example>
+ public static IDataView DetectEntireAnomalyBySrCnn(this AnomalyDetectionCatalog catalog, IDataView input, string outputColumnName, string inputColumnName,
+ double threshold = 0.3, int batchSize = 1024, double sensitivity = 99, SrCnnDetectMode detectMode = SrCnnDetectMode.AnomalyOnly)
+ => new SrCnnEntireAnomalyDetector(CatalogUtils.GetEnvironment(catalog), input, inputColumnName, outputColumnName, threshold, batchSize, sensitivity, detectMode);
+
  /// <summary>
  /// Create <see cref="RootCause"/>, which localizes root causes using decision tree algorithm.
  /// </summary>

docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectEntireAnomalyBySrCnn.cs

@@ -0,0 +1,99 @@
+using System;
+using System.Collections.Generic;
+using Microsoft.ML;
+using Microsoft.ML.Data;
+using Microsoft.ML.TimeSeries;
+
+namespace Samples.Dynamic
+{
+ public static class DetectEntireAnomalyBySrCnn
+ {
+ public static void Example()
+ {
+ // Create a new ML context, for ML.NET operations. It can be used for
+ // exception tracking and logging, 
+ // as well as the source of randomness.
+ var ml = new MLContext();
+
+ // Generate sample series data with an anomaly
+ var data = new List<TimeSeriesData>();
+ for (int index = 0; index < 20; index++)
+ {
+ data.Add(new TimeSeriesData { Value = 5 });
+ }
+ data.Add(new TimeSeriesData { Value = 10 });
+ for (int index = 0; index < 5; index++)
+ {
+ data.Add(new TimeSeriesData { Value = 5 });
+ }
+
+ // Convert data to IDataView.
+ var dataView = ml.Data.LoadFromEnumerable(data);
+
+ // Setup the detection arguments
+ string outputColumnName = nameof(SrCnnAnomalyDetection.Prediction);
+ string inputColumnName = nameof(TimeSeriesData.Value);
+
+ // Do batch anomaly detection
+ var outputDataView = ml.AnomalyDetection.DetectEntireAnomalyBySrCnn(dataView, outputColumnName, inputColumnName,
+ threshold: 0.35, batchSize: 512, sensitivity: 90.0, detectMode: SrCnnDetectMode.AnomalyAndMargin);
+
+ // Getting the data of the newly created column as an IEnumerable of
+ // SrCnnAnomalyDetection.
+ var predictionColumn = ml.Data.CreateEnumerable<SrCnnAnomalyDetection>(
+ outputDataView, reuseRowObject: false);
+
+ Console.WriteLine("Index\tData\tAnomaly\tAnomalyScore\tMag\tExpectedValue\tBoundaryUnit\tUpperBoundary\tLowerBoundary");
+
+ int k = 0;
+ foreach (var prediction in predictionColumn)
+ {
+ PrintPrediction(k, data[k].Value, prediction);
+ k++;
+ }
+ //Index Data Anomaly AnomalyScore Mag ExpectedValue BoundaryUnit UpperBoundary LowerBoundary
+ //0 5.00 0 0.00 0.21 5.00 5.00 5.01 4.99
+ //1 5.00 0 0.00 0.11 5.00 5.00 5.01 4.99
+ //2 5.00 0 0.00 0.03 5.00 5.00 5.01 4.99
+ //3 5.00 0 0.00 0.01 5.00 5.00 5.01 4.99
+ //4 5.00 0 0.00 0.03 5.00 5.00 5.01 4.99
+ //5 5.00 0 0.00 0.06 5.00 5.00 5.01 4.99
+ //6 5.00 0 0.00 0.02 5.00 5.00 5.01 4.99
+ //7 5.00 0 0.00 0.01 5.00 5.00 5.01 4.99
+ //8 5.00 0 0.00 0.01 5.00 5.00 5.01 4.99
+ //9 5.00 0 0.00 0.01 5.00 5.00 5.01 4.99
+ //10 5.00 0 0.00 0.00 5.00 5.00 5.01 4.99
+ //11 5.00 0 0.00 0.01 5.00 5.00 5.01 4.99
+ //12 5.00 0 0.00 0.01 5.00 5.00 5.01 4.99
+ //13 5.00 0 0.00 0.02 5.00 5.00 5.01 4.99
+ //14 5.00 0 0.00 0.07 5.00 5.00 5.01 4.99
+ //15 5.00 0 0.00 0.08 5.00 5.00 5.01 4.99
+ //16 5.00 0 0.00 0.02 5.00 5.00 5.01 4.99
+ //17 5.00 0 0.00 0.05 5.00 5.00 5.01 4.99
+ //18 5.00 0 0.00 0.12 5.00 5.00 5.01 4.99
+ //19 5.00 0 0.00 0.17 5.00 5.00 5.01 4.99
+ //20 10.00 1 0.50 0.80 5.00 5.00 5.01 4.99
+ //21 5.00 0 0.00 0.16 5.00 5.00 5.01 4.99
+ //22 5.00 0 0.00 0.11 5.00 5.00 5.01 4.99
+ //23 5.00 0 0.00 0.05 5.00 5.00 5.01 4.99
+ //24 5.00 0 0.00 0.11 5.00 5.00 5.01 4.99
+ //25 5.00 0 0.00 0.19 5.00 5.00 5.01 4.99
+ }
+
+ private static void PrintPrediction(int idx, double value, SrCnnAnomalyDetection prediction) =>
+ Console.WriteLine("{0}\t{1:0.00}\t{2}\t\t{3:0.00}\t{4:0.00}\t\t{5:0.00}\t\t{6:0.00}\t\t{7:0.00}\t\t{8:0.00}",
+ idx, value, prediction.Prediction[0], prediction.Prediction[1], prediction.Prediction[2],
+ prediction.Prediction[3], prediction.Prediction[4], prediction.Prediction[5], prediction.Prediction[6]);
+
+ private class TimeSeriesData
+ {
+ public double Value { get; set; }
+ }
+
+ private class SrCnnAnomalyDetection
+ {
+ [VectorType]
+ public double[] Prediction { get; set; }
+ }
+ }
+}

src/Microsoft.ML.Data/DataView/BatchDataViewMapperBase.cs

@@ -0,0 +1,173 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using Microsoft.ML.Runtime;
+
+namespace Microsoft.ML.Data.DataView
+{
+ internal abstract class BatchDataViewMapperBase<TInput, TBatch> : IDataView
+ {
+ public bool CanShuffle => false;
+
+ public DataViewSchema Schema => SchemaBindings.AsSchema;
+
+ private readonly IDataView _source;
+ protected readonly IHost Host;
+
+ protected BatchDataViewMapperBase(IHostEnvironment env, string registrationName, IDataView input)
+ {
+ Contracts.CheckValue(env, nameof(env));
+ Host = env.Register(registrationName);
+ _source = input;
+ }
+
+ public long? GetRowCount() => _source.GetRowCount();
+
+ public DataViewRowCursor GetRowCursor(IEnumerable<DataViewSchema.Column> columnsNeeded, Random rand = null)
+ {
+ Host.CheckValue(columnsNeeded, nameof(columnsNeeded));
+ Host.CheckValueOrNull(rand);
+
+ var predicate = RowCursorUtils.FromColumnsToPredicate(columnsNeeded, SchemaBindings.AsSchema);
+
+ // If we aren't selecting any of the output columns, don't construct our cursor.
+ // Note that because we cannot support random due to the inherently
+ // stratified nature, neither can we allow the base data to be shuffled,
+ // even if it supports shuffling.
+ if (!SchemaBindings.AnyNewColumnsActive(predicate))
+ {
+ var activeInput = SchemaBindings.GetActiveInput(predicate);
+ var inputCursor = _source.GetRowCursor(_source.Schema.Where(c => activeInput[c.Index]), null);
+ return new BindingsWrappedRowCursor(Host, inputCursor, SchemaBindings);
+ }
+ var active = SchemaBindings.GetActive(predicate);
+ Contracts.Assert(active.Length == SchemaBindings.ColumnCount);
+
+ // REVIEW: We can get a different input predicate for the input cursor and for the lookahead cursor. The lookahead
+ // cursor is only used for getting the values from the input column, so it only needs that column activated. The
+ // other cursor is used to get source columns, so it needs the rest of them activated.
+ var predInput = GetSchemaBindingDependencies(predicate);
+ var inputCols = _source.Schema.Where(c => predInput(c.Index));
+ return new Cursor(this, _source.GetRowCursor(inputCols), _source.GetRowCursor(inputCols), active);
+ }
+
+ public DataViewRowCursor[] GetRowCursorSet(IEnumerable<DataViewSchema.Column> columnsNeeded, int n, Random rand = null)
+ {
+ return new[] { GetRowCursor(columnsNeeded, rand) };
+ }
+
+ protected abstract ColumnBindingsBase SchemaBindings { get; }
+ protected abstract TBatch CreateBatch(DataViewRowCursor input);
+ protected abstract void ProcessBatch(TBatch currentBatch);
+ protected abstract void ProcessExample(TBatch currentBatch, TInput currentInput);
+ protected abstract Func<bool> GetLastInBatchDelegate(DataViewRowCursor lookAheadCursor);
+ protected abstract Func<bool> GetIsNewBatchDelegate(DataViewRowCursor lookAheadCursor);
+ protected abstract ValueGetter<TInput> GetLookAheadGetter(DataViewRowCursor lookAheadCursor);
+ protected abstract Delegate[] CreateGetters(DataViewRowCursor input, TBatch currentBatch, bool[] active);
+ protected abstract Func<int, bool> GetSchemaBindingDependencies(Func<int, bool> predicate);
+
+ private sealed class Cursor : RootCursorBase
+ {
+ private readonly BatchDataViewMapperBase<TInput, TBatch> _parent;
+ private readonly DataViewRowCursor _lookAheadCursor;
+ private readonly DataViewRowCursor _input;
+
+ private readonly bool[] _active;
+ private readonly Delegate[] _getters;
+
+ private readonly TBatch _currentBatch;
+ private readonly Func<bool> _lastInBatchInLookAheadCursorDel;
+ private readonly Func<bool> _firstInBatchInInputCursorDel;
+ private readonly ValueGetter<TInput> _inputGetterInLookAheadCursor;
+ private TInput _currentInput;
+
+ public override long Batch => 0;
+
+ public override DataViewSchema Schema => _parent.Schema;
+
+ public Cursor(BatchDataViewMapperBase<TInput, TBatch> parent, DataViewRowCursor input, DataViewRowCursor lookAheadCursor, bool[] active)
+ : base(parent.Host)
+ {
+ _parent = parent;
+ _input = input;
+ _lookAheadCursor = lookAheadCursor;
+ _active = active;
+
+ _currentBatch = _parent.CreateBatch(_input);
+
+ _getters = _parent.CreateGetters(_input, _currentBatch, _active);
+
+ _lastInBatchInLookAheadCursorDel = _parent.GetLastInBatchDelegate(_lookAheadCursor);
+ _firstInBatchInInputCursorDel = _parent.GetIsNewBatchDelegate(_input);
+ _inputGetterInLookAheadCursor = _parent.GetLookAheadGetter(_lookAheadCursor);
+ }
+
+ public override ValueGetter<TValue> GetGetter<TValue>(DataViewSchema.Column column)
+ {
+ Contracts.CheckParam(IsColumnActive(column), nameof(column), "requested column is not active");
+
+ var col = _parent.SchemaBindings.MapColumnIndex(out bool isSrc, column.Index);
+ if (isSrc)
+ {
+ Contracts.AssertValue(_input);
+ return _input.GetGetter<TValue>(_input.Schema[col]);
+ }
+
+ Ch.AssertValue(_getters);
+ var getter = _getters[col];
+ Ch.Assert(getter != null);
+ var fn = getter as ValueGetter<TValue>;
+ if (fn == null)
+ throw Ch.Except("Invalid TValue in GetGetter: '{0}'", typeof(TValue));
+ return fn;
+ }
+
+ public override ValueGetter<DataViewRowId> GetIdGetter()
+ {
+ return
+ (ref DataViewRowId val) =>
+ {
+ Ch.Check(IsGood, "Cannot call ID getter in current state");
+ val = new DataViewRowId((ulong)Position, 0);
+ };
+ }
+
+ public override bool IsColumnActive(DataViewSchema.Column column)
+ {
+ Ch.Check(column.Index < _parent.SchemaBindings.AsSchema.Count);
+ return _active[column.Index];
+ }
+
+ protected override bool MoveNextCore()
+ {
+ if (!_input.MoveNext())
+ return false;
+ if (!_firstInBatchInInputCursorDel())
+ return true;
+
+ // If we are here, this means that _input.MoveNext() has gotten us to the beginning of the next batch,
+ // so now we need to look ahead at the entire next batch in the _lookAheadCursor.
+ // The _lookAheadCursor's position should be on the last row of the previous batch (or -1).
+ Ch.Assert(_lastInBatchInLookAheadCursorDel());
+
+ var good = _lookAheadCursor.MoveNext();
+ // The two cursors should have the same number of elements, so if _input.MoveNext() returned true,
+ // then it must return true here too.
+ Ch.Assert(good);
+
+ do
+ {
+ _inputGetterInLookAheadCursor(ref _currentInput);
+ _parent.ProcessExample(_currentBatch, _currentInput);
+ } while (!_lastInBatchInLookAheadCursorDel() && _lookAheadCursor.MoveNext());
+
+ _parent.ProcessBatch(_currentBatch);
+ return true;
+ }
+ }
+ }
+}

src/Microsoft.ML.TimeSeries/ExtensionsCatalog.cs

@@ -2,8 +2,6 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 // See the LICENSE file in the project root for more information.
 
-using System;
-using System.Reflection;
 using Microsoft.ML.Data;
 using Microsoft.ML.Runtime;
 using Microsoft.ML.TimeSeries;
@@ -150,6 +148,35 @@ public static SrCnnAnomalyEstimator DetectAnomalyBySrCnn(this TransformsCatalog
  int windowSize = 64, int backAddWindowSize = 5, int lookaheadWindowSize = 5, int averageingWindowSize = 3, int judgementWindowSize = 21, double threshold = 0.3)
  => new SrCnnAnomalyEstimator(CatalogUtils.GetEnvironment(catalog), outputColumnName, windowSize, backAddWindowSize, lookaheadWindowSize, averageingWindowSize, judgementWindowSize, threshold, inputColumnName);
 
+ /// <summary>
+ /// Create <see cref="SrCnnEntireAnomalyDetector"/>, which detects timeseries anomalies for entire input using SRCNN algorithm.
+ /// </summary>
+ /// <param name="catalog">The AnomalyDetectionCatalog.</param>
+ /// <param name="input">Input DataView.</param>
+ /// <param name="outputColumnName">Name of the column resulting from data processing of <paramref name="inputColumnName"/>.
+ /// The column data is a vector of <see cref="System.Double"/>. The length of this vector varies depending on <paramref name="detectMode"/>.</param>
+ /// <param name="inputColumnName">Name of column to process. The column data must be <see cref="System.Double"/>.</param>
+ /// <param name="threshold">The threshold to determine anomaly, score larger than the threshold is considered as anomaly. Must be in [0,1]. Default value is 0.3.</param>
+ /// <param name="batchSize">Divide the input data into batches to fit srcnn model.
+ /// When set to -1, use the whole input to fit model instead of batch by batch, when set to a positive integer, use this number as batch size.
+ /// Must be -1 or a positive integer no less than 12. Default value is 1024.</param>
+ /// <param name="sensitivity">Sensitivity of boundaries, only useful when srCnnDetectMode is AnomalyAndMargin. Must be in [0,100]. Default value is 99.</param>
+ /// <param name="detectMode">An enum type of <see cref="SrCnnDetectMode"/>.
+ /// When set to AnomalyOnly, the output vector would be a 3-element Double vector of (IsAnomaly, RawScore, Mag).
+ /// When set to AnomalyAndExpectedValue, the output vector would be a 4-element Double vector of (IsAnomaly, RawScore, Mag, ExpectedValue).
+ /// When set to AnomalyAndMargin, the output vector would be a 7-element Double vector of (IsAnomaly, AnomalyScore, Mag, ExpectedValue, BoundaryUnit, UpperBoundary, LowerBoundary).
+ /// Default value is AnomalyOnly.</param>
+ /// <example>
+ /// <format type="text/markdown">
+ /// <![CDATA[
+ /// [!code-csharp[DetectEntireAnomalyBySrCnn](~/../docs/samples/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectEntireAnomalyBySrCnn.cs)]
+ /// ]]>
+ /// </format>
+ /// </example>
+ public static IDataView DetectEntireAnomalyBySrCnn(this AnomalyDetectionCatalog catalog, IDataView input, string outputColumnName, string inputColumnName,
+ double threshold = 0.3, int batchSize = 1024, double sensitivity = 99, SrCnnDetectMode detectMode = SrCnnDetectMode.AnomalyOnly)
+ => new SrCnnEntireAnomalyDetector(CatalogUtils.GetEnvironment(catalog), input, inputColumnName, outputColumnName, threshold, batchSize, sensitivity, detectMode);
+
  /// <summary>
  /// Create <see cref="RootCause"/>, which localizes root causes using decision tree algorithm.
  /// </summary>

docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectEntireAnomalyBySrCnn.cs

@@ -0,0 +1,99 @@
+using System;
+using System.Collections.Generic;
+using Microsoft.ML;
+using Microsoft.ML.Data;
+using Microsoft.ML.TimeSeries;
+
+namespace Samples.Dynamic
+{
+ public static class DetectEntireAnomalyBySrCnn
+ {
+ public static void Example()
+ {
+ // Create a new ML context, for ML.NET operations. It can be used for
+ // exception tracking and logging, 
+ // as well as the source of randomness.
+ var ml = new MLContext();
+
+ // Generate sample series data with an anomaly
+ var data = new List<TimeSeriesData>();
+ for (int index = 0; index < 20; index++)
+ {
+ data.Add(new TimeSeriesData { Value = 5 });
+ }
+ data.Add(new TimeSeriesData { Value = 10 });
+ for (int index = 0; index < 5; index++)
+ {
+ data.Add(new TimeSeriesData { Value = 5 });
+ }
+
+ // Convert data to IDataView.
+ var dataView = ml.Data.LoadFromEnumerable(data);
+
+ // Setup the detection arguments
+ string outputColumnName = nameof(SrCnnAnomalyDetection.Prediction);
+ string inputColumnName = nameof(TimeSeriesData.Value);
+
+ // Do batch anomaly detection
+ var outputDataView = ml.AnomalyDetection.DetectEntireAnomalyBySrCnn(dataView, outputColumnName, inputColumnName,
+ threshold: 0.35, batchSize: 512, sensitivity: 90.0, detectMode: SrCnnDetectMode.AnomalyAndMargin);
+
+ // Getting the data of the newly created column as an IEnumerable of
+ // SrCnnAnomalyDetection.
+ var predictionColumn = ml.Data.CreateEnumerable<SrCnnAnomalyDetection>(
+ outputDataView, reuseRowObject: false);
+
+ Console.WriteLine("Index\tData\tAnomaly\tAnomalyScore\tMag\tExpectedValue\tBoundaryUnit\tUpperBoundary\tLowerBoundary");
+
+ int k = 0;
+ foreach (var prediction in predictionColumn)
+ {
+ PrintPrediction(k, data[k].Value, prediction);
+ k++;
+ }
+ //Index Data Anomaly AnomalyScore Mag ExpectedValue BoundaryUnit UpperBoundary LowerBoundary
+ //0 5.00 0 0.00 0.21 5.00 5.00 5.01 4.99
+ //1 5.00 0 0.00 0.11 5.00 5.00 5.01 4.99
+ //2 5.00 0 0.00 0.03 5.00 5.00 5.01 4.99
+ //3 5.00 0 0.00 0.01 5.00 5.00 5.01 4.99
+ //4 5.00 0 0.00 0.03 5.00 5.00 5.01 4.99
+ //5 5.00 0 0.00 0.06 5.00 5.00 5.01 4.99
+ //6 5.00 0 0.00 0.02 5.00 5.00 5.01 4.99
+ //7 5.00 0 0.00 0.01 5.00 5.00 5.01 4.99
+ //8 5.00 0 0.00 0.01 5.00 5.00 5.01 4.99
+ //9 5.00 0 0.00 0.01 5.00 5.00 5.01 4.99
+ //10 5.00 0 0.00 0.00 5.00 5.00 5.01 4.99
+ //11 5.00 0 0.00 0.01 5.00 5.00 5.01 4.99
+ //12 5.00 0 0.00 0.01 5.00 5.00 5.01 4.99
+ //13 5.00 0 0.00 0.02 5.00 5.00 5.01 4.99
+ //14 5.00 0 0.00 0.07 5.00 5.00 5.01 4.99
+ //15 5.00 0 0.00 0.08 5.00 5.00 5.01 4.99
+ //16 5.00 0 0.00 0.02 5.00 5.00 5.01 4.99
+ //17 5.00 0 0.00 0.05 5.00 5.00 5.01 4.99
+ //18 5.00 0 0.00 0.12 5.00 5.00 5.01 4.99
+ //19 5.00 0 0.00 0.17 5.00 5.00 5.01 4.99
+ //20 10.00 1 0.50 0.80 5.00 5.00 5.01 4.99
+ //21 5.00 0 0.00 0.16 5.00 5.00 5.01 4.99
+ //22 5.00 0 0.00 0.11 5.00 5.00 5.01 4.99
+ //23 5.00 0 0.00 0.05 5.00 5.00 5.01 4.99
+ //24 5.00 0 0.00 0.11 5.00 5.00 5.01 4.99
+ //25 5.00 0 0.00 0.19 5.00 5.00 5.01 4.99
+ }
+
+ private static void PrintPrediction(int idx, double value, SrCnnAnomalyDetection prediction) =>
+ Console.WriteLine("{0}\t{1:0.00}\t{2}\t\t{3:0.00}\t{4:0.00}\t\t{5:0.00}\t\t{6:0.00}\t\t{7:0.00}\t\t{8:0.00}",
+ idx, value, prediction.Prediction[0], prediction.Prediction[1], prediction.Prediction[2],
+ prediction.Prediction[3], prediction.Prediction[4], prediction.Prediction[5], prediction.Prediction[6]);
+
+ private class TimeSeriesData
+ {
+ public double Value { get; set; }
+ }
+
+ private class SrCnnAnomalyDetection
+ {
+ [VectorType]
+ public double[] Prediction { get; set; }
+ }
+ }
+}

src/Microsoft.ML.Data/DataView/BatchDataViewMapperBase.cs

@@ -0,0 +1,173 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using Microsoft.ML.Runtime;
+
+namespace Microsoft.ML.Data.DataView
+{
+ internal abstract class BatchDataViewMapperBase<TInput, TBatch> : IDataView
+ {
+ public bool CanShuffle => false;
+
+ public DataViewSchema Schema => SchemaBindings.AsSchema;
+
+ private readonly IDataView _source;
+ protected readonly IHost Host;
+
+ protected BatchDataViewMapperBase(IHostEnvironment env, string registrationName, IDataView input)
+ {
+ Contracts.CheckValue(env, nameof(env));
+ Host = env.Register(registrationName);
+ _source = input;
+ }
+
+ public long? GetRowCount() => _source.GetRowCount();
+
+ public DataViewRowCursor GetRowCursor(IEnumerable<DataViewSchema.Column> columnsNeeded, Random rand = null)
+ {
+ Host.CheckValue(columnsNeeded, nameof(columnsNeeded));
+ Host.CheckValueOrNull(rand);
+
+ var predicate = RowCursorUtils.FromColumnsToPredicate(columnsNeeded, SchemaBindings.AsSchema);
+
+ // If we aren't selecting any of the output columns, don't construct our cursor.
+ // Note that because we cannot support random due to the inherently
+ // stratified nature, neither can we allow the base data to be shuffled,
+ // even if it supports shuffling.
+ if (!SchemaBindings.AnyNewColumnsActive(predicate))
+ {
+ var activeInput = SchemaBindings.GetActiveInput(predicate);
+ var inputCursor = _source.GetRowCursor(_source.Schema.Where(c => activeInput[c.Index]), null);
+ return new BindingsWrappedRowCursor(Host, inputCursor, SchemaBindings);
+ }
+ var active = SchemaBindings.GetActive(predicate);
+ Contracts.Assert(active.Length == SchemaBindings.ColumnCount);
+
+ // REVIEW: We can get a different input predicate for the input cursor and for the lookahead cursor. The lookahead
+ // cursor is only used for getting the values from the input column, so it only needs that column activated. The
+ // other cursor is used to get source columns, so it needs the rest of them activated.
+ var predInput = GetSchemaBindingDependencies(predicate);
+ var inputCols = _source.Schema.Where(c => predInput(c.Index));
+ return new Cursor(this, _source.GetRowCursor(inputCols), _source.GetRowCursor(inputCols), active);
+ }
+
+ public DataViewRowCursor[] GetRowCursorSet(IEnumerable<DataViewSchema.Column> columnsNeeded, int n, Random rand = null)
+ {
+ return new[] { GetRowCursor(columnsNeeded, rand) };
+ }
+
+ protected abstract ColumnBindingsBase SchemaBindings { get; }
+ protected abstract TBatch CreateBatch(DataViewRowCursor input);
+ protected abstract void ProcessBatch(TBatch currentBatch);
+ protected abstract void ProcessExample(TBatch currentBatch, TInput currentInput);
+ protected abstract Func<bool> GetLastInBatchDelegate(DataViewRowCursor lookAheadCursor);
+ protected abstract Func<bool> GetIsNewBatchDelegate(DataViewRowCursor lookAheadCursor);
+ protected abstract ValueGetter<TInput> GetLookAheadGetter(DataViewRowCursor lookAheadCursor);
+ protected abstract Delegate[] CreateGetters(DataViewRowCursor input, TBatch currentBatch, bool[] active);
+ protected abstract Func<int, bool> GetSchemaBindingDependencies(Func<int, bool> predicate);
+
+ private sealed class Cursor : RootCursorBase
+ {
+ private readonly BatchDataViewMapperBase<TInput, TBatch> _parent;
+ private readonly DataViewRowCursor _lookAheadCursor;
+ private readonly DataViewRowCursor _input;
+
+ private readonly bool[] _active;
+ private readonly Delegate[] _getters;
+
+ private readonly TBatch _currentBatch;
+ private readonly Func<bool> _lastInBatchInLookAheadCursorDel;
+ private readonly Func<bool> _firstInBatchInInputCursorDel;
+ private readonly ValueGetter<TInput> _inputGetterInLookAheadCursor;
+ private TInput _currentInput;
+
+ public override long Batch => 0;
+
+ public override DataViewSchema Schema => _parent.Schema;
+
+ public Cursor(BatchDataViewMapperBase<TInput, TBatch> parent, DataViewRowCursor input, DataViewRowCursor lookAheadCursor, bool[] active)
+ : base(parent.Host)
+ {
+ _parent = parent;
+ _input = input;
+ _lookAheadCursor = lookAheadCursor;
+ _active = active;
+
+ _currentBatch = _parent.CreateBatch(_input);
+
+ _getters = _parent.CreateGetters(_input, _currentBatch, _active);
+
+ _lastInBatchInLookAheadCursorDel = _parent.GetLastInBatchDelegate(_lookAheadCursor);
+ _firstInBatchInInputCursorDel = _parent.GetIsNewBatchDelegate(_input);
+ _inputGetterInLookAheadCursor = _parent.GetLookAheadGetter(_lookAheadCursor);
+ }
+
+ public override ValueGetter<TValue> GetGetter<TValue>(DataViewSchema.Column column)
+ {
+ Contracts.CheckParam(IsColumnActive(column), nameof(column), "requested column is not active");
+
+ var col = _parent.SchemaBindings.MapColumnIndex(out bool isSrc, column.Index);
+ if (isSrc)
+ {
+ Contracts.AssertValue(_input);
+ return _input.GetGetter<TValue>(_input.Schema[col]);
+ }
+
+ Ch.AssertValue(_getters);
+ var getter = _getters[col];
+ Ch.Assert(getter != null);
+ var fn = getter as ValueGetter<TValue>;
+ if (fn == null)
+ throw Ch.Except("Invalid TValue in GetGetter: '{0}'", typeof(TValue));
+ return fn;
+ }
+
+ public override ValueGetter<DataViewRowId> GetIdGetter()
+ {
+ return
+ (ref DataViewRowId val) =>
+ {
+ Ch.Check(IsGood, "Cannot call ID getter in current state");
+ val = new DataViewRowId((ulong)Position, 0);
+ };
+ }
+
+ public override bool IsColumnActive(DataViewSchema.Column column)
+ {
+ Ch.Check(column.Index < _parent.SchemaBindings.AsSchema.Count);
+ return _active[column.Index];
+ }
+
+ protected override bool MoveNextCore()
+ {
+ if (!_input.MoveNext())
+ return false;
+ if (!_firstInBatchInInputCursorDel())
+ return true;
+
+ // If we are here, this means that _input.MoveNext() has gotten us to the beginning of the next batch,
+ // so now we need to look ahead at the entire next batch in the _lookAheadCursor.
+ // The _lookAheadCursor's position should be on the last row of the previous batch (or -1).
+ Ch.Assert(_lastInBatchInLookAheadCursorDel());
+
+ var good = _lookAheadCursor.MoveNext();
+ // The two cursors should have the same number of elements, so if _input.MoveNext() returned true,
+ // then it must return true here too.
+ Ch.Assert(good);
+
+ do
+ {
+ _inputGetterInLookAheadCursor(ref _currentInput);
+ _parent.ProcessExample(_currentBatch, _currentInput);
+ } while (!_lastInBatchInLookAheadCursorDel() && _lookAheadCursor.MoveNext());
+
+ _parent.ProcessBatch(_currentBatch);
+ return true;
+ }
+ }
+ }
+}

src/Microsoft.ML.TimeSeries/ExtensionsCatalog.cs

@@ -2,8 +2,6 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 // See the LICENSE file in the project root for more information.
 
-using System;
-using System.Reflection;
 using Microsoft.ML.Data;
 using Microsoft.ML.Runtime;
 using Microsoft.ML.TimeSeries;
@@ -150,6 +148,35 @@ public static SrCnnAnomalyEstimator DetectAnomalyBySrCnn(this TransformsCatalog
  int windowSize = 64, int backAddWindowSize = 5, int lookaheadWindowSize = 5, int averageingWindowSize = 3, int judgementWindowSize = 21, double threshold = 0.3)
  => new SrCnnAnomalyEstimator(CatalogUtils.GetEnvironment(catalog), outputColumnName, windowSize, backAddWindowSize, lookaheadWindowSize, averageingWindowSize, judgementWindowSize, threshold, inputColumnName);
 
+ /// <summary>
+ /// Create <see cref="SrCnnEntireAnomalyDetector"/>, which detects timeseries anomalies for entire input using SRCNN algorithm.
+ /// </summary>
+ /// <param name="catalog">The AnomalyDetectionCatalog.</param>
+ /// <param name="input">Input DataView.</param>
+ /// <param name="outputColumnName">Name of the column resulting from data processing of <paramref name="inputColumnName"/>.
+ /// The column data is a vector of <see cref="System.Double"/>. The length of this vector varies depending on <paramref name="detectMode"/>.</param>
+ /// <param name="inputColumnName">Name of column to process. The column data must be <see cref="System.Double"/>.</param>
+ /// <param name="threshold">The threshold to determine anomaly, score larger than the threshold is considered as anomaly. Must be in [0,1]. Default value is 0.3.</param>
+ /// <param name="batchSize">Divide the input data into batches to fit srcnn model.
+ /// When set to -1, use the whole input to fit model instead of batch by batch, when set to a positive integer, use this number as batch size.
+ /// Must be -1 or a positive integer no less than 12. Default value is 1024.</param>
+ /// <param name="sensitivity">Sensitivity of boundaries, only useful when srCnnDetectMode is AnomalyAndMargin. Must be in [0,100]. Default value is 99.</param>
+ /// <param name="detectMode">An enum type of <see cref="SrCnnDetectMode"/>.
+ /// When set to AnomalyOnly, the output vector would be a 3-element Double vector of (IsAnomaly, RawScore, Mag).
+ /// When set to AnomalyAndExpectedValue, the output vector would be a 4-element Double vector of (IsAnomaly, RawScore, Mag, ExpectedValue).
+ /// When set to AnomalyAndMargin, the output vector would be a 7-element Double vector of (IsAnomaly, AnomalyScore, Mag, ExpectedValue, BoundaryUnit, UpperBoundary, LowerBoundary).
+ /// Default value is AnomalyOnly.</param>
+ /// <example>
+ /// <format type="text/markdown">
+ /// <![CDATA[
+ /// [!code-csharp[DetectEntireAnomalyBySrCnn](~/../docs/samples/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectEntireAnomalyBySrCnn.cs)]
+ /// ]]>
+ /// </format>
+ /// </example>
+ public static IDataView DetectEntireAnomalyBySrCnn(this AnomalyDetectionCatalog catalog, IDataView input, string outputColumnName, string inputColumnName,
+ double threshold = 0.3, int batchSize = 1024, double sensitivity = 99, SrCnnDetectMode detectMode = SrCnnDetectMode.AnomalyOnly)
+ => new SrCnnEntireAnomalyDetector(CatalogUtils.GetEnvironment(catalog), input, inputColumnName, outputColumnName, threshold, batchSize, sensitivity, detectMode);
+
  /// <summary>
  /// Create <see cref="RootCause"/>, which localizes root causes using decision tree algorithm.
  /// </summary>

docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectEntireAnomalyBySrCnn.cs

@@ -0,0 +1,99 @@
+using System;
+using System.Collections.Generic;
+using Microsoft.ML;
+using Microsoft.ML.Data;
+using Microsoft.ML.TimeSeries;
+
+namespace Samples.Dynamic
+{
+ public static class DetectEntireAnomalyBySrCnn
+ {
+ public static void Example()
+ {
+ // Create a new ML context, for ML.NET operations. It can be used for
+ // exception tracking and logging, 
+ // as well as the source of randomness.
+ var ml = new MLContext();
+
+ // Generate sample series data with an anomaly
+ var data = new List<TimeSeriesData>();
+ for (int index = 0; index < 20; index++)
+ {
+ data.Add(new TimeSeriesData { Value = 5 });
+ }
+ data.Add(new TimeSeriesData { Value = 10 });
+ for (int index = 0; index < 5; index++)
+ {
+ data.Add(new TimeSeriesData { Value = 5 });
+ }
+
+ // Convert data to IDataView.
+ var dataView = ml.Data.LoadFromEnumerable(data);
+
+ // Setup the detection arguments
+ string outputColumnName = nameof(SrCnnAnomalyDetection.Prediction);
+ string inputColumnName = nameof(TimeSeriesData.Value);
+
+ // Do batch anomaly detection
+ var outputDataView = ml.AnomalyDetection.DetectEntireAnomalyBySrCnn(dataView, outputColumnName, inputColumnName,
+ threshold: 0.35, batchSize: 512, sensitivity: 90.0, detectMode: SrCnnDetectMode.AnomalyAndMargin);
+
+ // Getting the data of the newly created column as an IEnumerable of
+ // SrCnnAnomalyDetection.
+ var predictionColumn = ml.Data.CreateEnumerable<SrCnnAnomalyDetection>(
+ outputDataView, reuseRowObject: false);
+
+ Console.WriteLine("Index\tData\tAnomaly\tAnomalyScore\tMag\tExpectedValue\tBoundaryUnit\tUpperBoundary\tLowerBoundary");
+
+ int k = 0;
+ foreach (var prediction in predictionColumn)
+ {
+ PrintPrediction(k, data[k].Value, prediction);
+ k++;
+ }
+ //Index Data Anomaly AnomalyScore Mag ExpectedValue BoundaryUnit UpperBoundary LowerBoundary
+ //0 5.00 0 0.00 0.21 5.00 5.00 5.01 4.99
+ //1 5.00 0 0.00 0.11 5.00 5.00 5.01 4.99
+ //2 5.00 0 0.00 0.03 5.00 5.00 5.01 4.99
+ //3 5.00 0 0.00 0.01 5.00 5.00 5.01 4.99
+ //4 5.00 0 0.00 0.03 5.00 5.00 5.01 4.99
+ //5 5.00 0 0.00 0.06 5.00 5.00 5.01 4.99
+ //6 5.00 0 0.00 0.02 5.00 5.00 5.01 4.99
+ //7 5.00 0 0.00 0.01 5.00 5.00 5.01 4.99
+ //8 5.00 0 0.00 0.01 5.00 5.00 5.01 4.99
+ //9 5.00 0 0.00 0.01 5.00 5.00 5.01 4.99
+ //10 5.00 0 0.00 0.00 5.00 5.00 5.01 4.99
+ //11 5.00 0 0.00 0.01 5.00 5.00 5.01 4.99
+ //12 5.00 0 0.00 0.01 5.00 5.00 5.01 4.99
+ //13 5.00 0 0.00 0.02 5.00 5.00 5.01 4.99
+ //14 5.00 0 0.00 0.07 5.00 5.00 5.01 4.99
+ //15 5.00 0 0.00 0.08 5.00 5.00 5.01 4.99
+ //16 5.00 0 0.00 0.02 5.00 5.00 5.01 4.99
+ //17 5.00 0 0.00 0.05 5.00 5.00 5.01 4.99
+ //18 5.00 0 0.00 0.12 5.00 5.00 5.01 4.99
+ //19 5.00 0 0.00 0.17 5.00 5.00 5.01 4.99
+ //20 10.00 1 0.50 0.80 5.00 5.00 5.01 4.99
+ //21 5.00 0 0.00 0.16 5.00 5.00 5.01 4.99
+ //22 5.00 0 0.00 0.11 5.00 5.00 5.01 4.99
+ //23 5.00 0 0.00 0.05 5.00 5.00 5.01 4.99
+ //24 5.00 0 0.00 0.11 5.00 5.00 5.01 4.99
+ //25 5.00 0 0.00 0.19 5.00 5.00 5.01 4.99
+ }
+
+ private static void PrintPrediction(int idx, double value, SrCnnAnomalyDetection prediction) =>
+ Console.WriteLine("{0}\t{1:0.00}\t{2}\t\t{3:0.00}\t{4:0.00}\t\t{5:0.00}\t\t{6:0.00}\t\t{7:0.00}\t\t{8:0.00}",
+ idx, value, prediction.Prediction[0], prediction.Prediction[1], prediction.Prediction[2],
+ prediction.Prediction[3], prediction.Prediction[4], prediction.Prediction[5], prediction.Prediction[6]);
+
+ private class TimeSeriesData
+ {
+ public double Value { get; set; }
+ }
+
+ private class SrCnnAnomalyDetection
+ {
+ [VectorType]
+ public double[] Prediction { get; set; }
+ }
+ }
+}

src/Microsoft.ML.Data/DataView/BatchDataViewMapperBase.cs

@@ -0,0 +1,173 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using Microsoft.ML.Runtime;
+
+namespace Microsoft.ML.Data.DataView
+{
+ internal abstract class BatchDataViewMapperBase<TInput, TBatch> : IDataView
+ {
+ public bool CanShuffle => false;
+
+ public DataViewSchema Schema => SchemaBindings.AsSchema;
+
+ private readonly IDataView _source;
+ protected readonly IHost Host;
+
+ protected BatchDataViewMapperBase(IHostEnvironment env, string registrationName, IDataView input)
+ {
+ Contracts.CheckValue(env, nameof(env));
+ Host = env.Register(registrationName);
+ _source = input;
+ }
+
+ public long? GetRowCount() => _source.GetRowCount();
+
+ public DataViewRowCursor GetRowCursor(IEnumerable<DataViewSchema.Column> columnsNeeded, Random rand = null)
+ {
+ Host.CheckValue(columnsNeeded, nameof(columnsNeeded));
+ Host.CheckValueOrNull(rand);
+
+ var predicate = RowCursorUtils.FromColumnsToPredicate(columnsNeeded, SchemaBindings.AsSchema);
+
+ // If we aren't selecting any of the output columns, don't construct our cursor.
+ // Note that because we cannot support random due to the inherently
+ // stratified nature, neither can we allow the base data to be shuffled,
+ // even if it supports shuffling.
+ if (!SchemaBindings.AnyNewColumnsActive(predicate))
+ {
+ var activeInput = SchemaBindings.GetActiveInput(predicate);
+ var inputCursor = _source.GetRowCursor(_source.Schema.Where(c => activeInput[c.Index]), null);
+ return new BindingsWrappedRowCursor(Host, inputCursor, SchemaBindings);
+ }
+ var active = SchemaBindings.GetActive(predicate);
+ Contracts.Assert(active.Length == SchemaBindings.ColumnCount);
+
+ // REVIEW: We can get a different input predicate for the input cursor and for the lookahead cursor. The lookahead
+ // cursor is only used for getting the values from the input column, so it only needs that column activated. The
+ // other cursor is used to get source columns, so it needs the rest of them activated.
+ var predInput = GetSchemaBindingDependencies(predicate);
+ var inputCols = _source.Schema.Where(c => predInput(c.Index));
+ return new Cursor(this, _source.GetRowCursor(inputCols), _source.GetRowCursor(inputCols), active);
+ }
+
+ public DataViewRowCursor[] GetRowCursorSet(IEnumerable<DataViewSchema.Column> columnsNeeded, int n, Random rand = null)
+ {
+ return new[] { GetRowCursor(columnsNeeded, rand) };
+ }
+
+ protected abstract ColumnBindingsBase SchemaBindings { get; }
+ protected abstract TBatch CreateBatch(DataViewRowCursor input);
+ protected abstract void ProcessBatch(TBatch currentBatch);
+ protected abstract void ProcessExample(TBatch currentBatch, TInput currentInput);
+ protected abstract Func<bool> GetLastInBatchDelegate(DataViewRowCursor lookAheadCursor);
+ protected abstract Func<bool> GetIsNewBatchDelegate(DataViewRowCursor lookAheadCursor);
+ protected abstract ValueGetter<TInput> GetLookAheadGetter(DataViewRowCursor lookAheadCursor);
+ protected abstract Delegate[] CreateGetters(DataViewRowCursor input, TBatch currentBatch, bool[] active);
+ protected abstract Func<int, bool> GetSchemaBindingDependencies(Func<int, bool> predicate);
+
+ private sealed class Cursor : RootCursorBase
+ {
+ private readonly BatchDataViewMapperBase<TInput, TBatch> _parent;
+ private readonly DataViewRowCursor _lookAheadCursor;
+ private readonly DataViewRowCursor _input;
+
+ private readonly bool[] _active;
+ private readonly Delegate[] _getters;
+
+ private readonly TBatch _currentBatch;
+ private readonly Func<bool> _lastInBatchInLookAheadCursorDel;
+ private readonly Func<bool> _firstInBatchInInputCursorDel;
+ private readonly ValueGetter<TInput> _inputGetterInLookAheadCursor;
+ private TInput _currentInput;
+
+ public override long Batch => 0;
+
+ public override DataViewSchema Schema => _parent.Schema;
+
+ public Cursor(BatchDataViewMapperBase<TInput, TBatch> parent, DataViewRowCursor input, DataViewRowCursor lookAheadCursor, bool[] active)
+ : base(parent.Host)
+ {
+ _parent = parent;
+ _input = input;
+ _lookAheadCursor = lookAheadCursor;
+ _active = active;
+
+ _currentBatch = _parent.CreateBatch(_input);
+
+ _getters = _parent.CreateGetters(_input, _currentBatch, _active);
+
+ _lastInBatchInLookAheadCursorDel = _parent.GetLastInBatchDelegate(_lookAheadCursor);
+ _firstInBatchInInputCursorDel = _parent.GetIsNewBatchDelegate(_input);
+ _inputGetterInLookAheadCursor = _parent.GetLookAheadGetter(_lookAheadCursor);
+ }
+
+ public override ValueGetter<TValue> GetGetter<TValue>(DataViewSchema.Column column)
+ {
+ Contracts.CheckParam(IsColumnActive(column), nameof(column), "requested column is not active");
+
+ var col = _parent.SchemaBindings.MapColumnIndex(out bool isSrc, column.Index);
+ if (isSrc)
+ {
+ Contracts.AssertValue(_input);
+ return _input.GetGetter<TValue>(_input.Schema[col]);
+ }
+
+ Ch.AssertValue(_getters);
+ var getter = _getters[col];
+ Ch.Assert(getter != null);
+ var fn = getter as ValueGetter<TValue>;
+ if (fn == null)
+ throw Ch.Except("Invalid TValue in GetGetter: '{0}'", typeof(TValue));
+ return fn;
+ }
+
+ public override ValueGetter<DataViewRowId> GetIdGetter()
+ {
+ return
+ (ref DataViewRowId val) =>
+ {
+ Ch.Check(IsGood, "Cannot call ID getter in current state");
+ val = new DataViewRowId((ulong)Position, 0);
+ };
+ }
+
+ public override bool IsColumnActive(DataViewSchema.Column column)
+ {
+ Ch.Check(column.Index < _parent.SchemaBindings.AsSchema.Count);
+ return _active[column.Index];
+ }
+
+ protected override bool MoveNextCore()
+ {
+ if (!_input.MoveNext())
+ return false;
+ if (!_firstInBatchInInputCursorDel())
+ return true;
+
+ // If we are here, this means that _input.MoveNext() has gotten us to the beginning of the next batch,
+ // so now we need to look ahead at the entire next batch in the _lookAheadCursor.
+ // The _lookAheadCursor's position should be on the last row of the previous batch (or -1).
+ Ch.Assert(_lastInBatchInLookAheadCursorDel());
+
+ var good = _lookAheadCursor.MoveNext();
+ // The two cursors should have the same number of elements, so if _input.MoveNext() returned true,
+ // then it must return true here too.
+ Ch.Assert(good);
+
+ do
+ {
+ _inputGetterInLookAheadCursor(ref _currentInput);
+ _parent.ProcessExample(_currentBatch, _currentInput);
+ } while (!_lastInBatchInLookAheadCursorDel() && _lookAheadCursor.MoveNext());
+
+ _parent.ProcessBatch(_currentBatch);
+ return true;
+ }
+ }
+ }
+}

src/Microsoft.ML.TimeSeries/ExtensionsCatalog.cs

@@ -2,8 +2,6 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 // See the LICENSE file in the project root for more information.
 
-using System;
-using System.Reflection;
 using Microsoft.ML.Data;
 using Microsoft.ML.Runtime;
 using Microsoft.ML.TimeSeries;
@@ -150,6 +148,35 @@ public static SrCnnAnomalyEstimator DetectAnomalyBySrCnn(this TransformsCatalog
  int windowSize = 64, int backAddWindowSize = 5, int lookaheadWindowSize = 5, int averageingWindowSize = 3, int judgementWindowSize = 21, double threshold = 0.3)
  => new SrCnnAnomalyEstimator(CatalogUtils.GetEnvironment(catalog), outputColumnName, windowSize, backAddWindowSize, lookaheadWindowSize, averageingWindowSize, judgementWindowSize, threshold, inputColumnName);
 
+ /// <summary>
+ /// Create <see cref="SrCnnEntireAnomalyDetector"/>, which detects timeseries anomalies for entire input using SRCNN algorithm.
+ /// </summary>
+ /// <param name="catalog">The AnomalyDetectionCatalog.</param>
+ /// <param name="input">Input DataView.</param>
+ /// <param name="outputColumnName">Name of the column resulting from data processing of <paramref name="inputColumnName"/>.
+ /// The column data is a vector of <see cref="System.Double"/>. The length of this vector varies depending on <paramref name="detectMode"/>.</param>
+ /// <param name="inputColumnName">Name of column to process. The column data must be <see cref="System.Double"/>.</param>
+ /// <param name="threshold">The threshold to determine anomaly, score larger than the threshold is considered as anomaly. Must be in [0,1]. Default value is 0.3.</param>
+ /// <param name="batchSize">Divide the input data into batches to fit srcnn model.
+ /// When set to -1, use the whole input to fit model instead of batch by batch, when set to a positive integer, use this number as batch size.
+ /// Must be -1 or a positive integer no less than 12. Default value is 1024.</param>
+ /// <param name="sensitivity">Sensitivity of boundaries, only useful when srCnnDetectMode is AnomalyAndMargin. Must be in [0,100]. Default value is 99.</param>
+ /// <param name="detectMode">An enum type of <see cref="SrCnnDetectMode"/>.
+ /// When set to AnomalyOnly, the output vector would be a 3-element Double vector of (IsAnomaly, RawScore, Mag).
+ /// When set to AnomalyAndExpectedValue, the output vector would be a 4-element Double vector of (IsAnomaly, RawScore, Mag, ExpectedValue).
+ /// When set to AnomalyAndMargin, the output vector would be a 7-element Double vector of (IsAnomaly, AnomalyScore, Mag, ExpectedValue, BoundaryUnit, UpperBoundary, LowerBoundary).
+ /// Default value is AnomalyOnly.</param>
+ /// <example>
+ /// <format type="text/markdown">
+ /// <![CDATA[
+ /// [!code-csharp[DetectEntireAnomalyBySrCnn](~/../docs/samples/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectEntireAnomalyBySrCnn.cs)]
+ /// ]]>
+ /// </format>
+ /// </example>
+ public static IDataView DetectEntireAnomalyBySrCnn(this AnomalyDetectionCatalog catalog, IDataView input, string outputColumnName, string inputColumnName,
+ double threshold = 0.3, int batchSize = 1024, double sensitivity = 99, SrCnnDetectMode detectMode = SrCnnDetectMode.AnomalyOnly)
+ => new SrCnnEntireAnomalyDetector(CatalogUtils.GetEnvironment(catalog), input, inputColumnName, outputColumnName, threshold, batchSize, sensitivity, detectMode);
+
  /// <summary>
  /// Create <see cref="RootCause"/>, which localizes root causes using decision tree algorithm.
  /// </summary>

docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectEntireAnomalyBySrCnn.cs

@@ -0,0 +1,99 @@
+using System;
+using System.Collections.Generic;
+using Microsoft.ML;
+using Microsoft.ML.Data;
+using Microsoft.ML.TimeSeries;
+
+namespace Samples.Dynamic
+{
+ public static class DetectEntireAnomalyBySrCnn
+ {
+ public static void Example()
+ {
+ // Create a new ML context, for ML.NET operations. It can be used for
+ // exception tracking and logging, 
+ // as well as the source of randomness.
+ var ml = new MLContext();
+
+ // Generate sample series data with an anomaly
+ var data = new List<TimeSeriesData>();
+ for (int index = 0; index < 20; index++)
+ {
+ data.Add(new TimeSeriesData { Value = 5 });
+ }
+ data.Add(new TimeSeriesData { Value = 10 });
+ for (int index = 0; index < 5; index++)
+ {
+ data.Add(new TimeSeriesData { Value = 5 });
+ }
+
+ // Convert data to IDataView.
+ var dataView = ml.Data.LoadFromEnumerable(data);
+
+ // Setup the detection arguments
+ string outputColumnName = nameof(SrCnnAnomalyDetection.Prediction);
+ string inputColumnName = nameof(TimeSeriesData.Value);
+
+ // Do batch anomaly detection
+ var outputDataView = ml.AnomalyDetection.DetectEntireAnomalyBySrCnn(dataView, outputColumnName, inputColumnName,
+ threshold: 0.35, batchSize: 512, sensitivity: 90.0, detectMode: SrCnnDetectMode.AnomalyAndMargin);
+
+ // Getting the data of the newly created column as an IEnumerable of
+ // SrCnnAnomalyDetection.
+ var predictionColumn = ml.Data.CreateEnumerable<SrCnnAnomalyDetection>(
+ outputDataView, reuseRowObject: false);
+
+ Console.WriteLine("Index\tData\tAnomaly\tAnomalyScore\tMag\tExpectedValue\tBoundaryUnit\tUpperBoundary\tLowerBoundary");
+
+ int k = 0;
+ foreach (var prediction in predictionColumn)
+ {
+ PrintPrediction(k, data[k].Value, prediction);
+ k++;
+ }
+ //Index Data Anomaly AnomalyScore Mag ExpectedValue BoundaryUnit UpperBoundary LowerBoundary
+ //0 5.00 0 0.00 0.21 5.00 5.00 5.01 4.99
+ //1 5.00 0 0.00 0.11 5.00 5.00 5.01 4.99
+ //2 5.00 0 0.00 0.03 5.00 5.00 5.01 4.99
+ //3 5.00 0 0.00 0.01 5.00 5.00 5.01 4.99
+ //4 5.00 0 0.00 0.03 5.00 5.00 5.01 4.99
+ //5 5.00 0 0.00 0.06 5.00 5.00 5.01 4.99
+ //6 5.00 0 0.00 0.02 5.00 5.00 5.01 4.99
+ //7 5.00 0 0.00 0.01 5.00 5.00 5.01 4.99
+ //8 5.00 0 0.00 0.01 5.00 5.00 5.01 4.99
+ //9 5.00 0 0.00 0.01 5.00 5.00 5.01 4.99
+ //10 5.00 0 0.00 0.00 5.00 5.00 5.01 4.99
+ //11 5.00 0 0.00 0.01 5.00 5.00 5.01 4.99
+ //12 5.00 0 0.00 0.01 5.00 5.00 5.01 4.99
+ //13 5.00 0 0.00 0.02 5.00 5.00 5.01 4.99
+ //14 5.00 0 0.00 0.07 5.00 5.00 5.01 4.99
+ //15 5.00 0 0.00 0.08 5.00 5.00 5.01 4.99
+ //16 5.00 0 0.00 0.02 5.00 5.00 5.01 4.99
+ //17 5.00 0 0.00 0.05 5.00 5.00 5.01 4.99
+ //18 5.00 0 0.00 0.12 5.00 5.00 5.01 4.99
+ //19 5.00 0 0.00 0.17 5.00 5.00 5.01 4.99
+ //20 10.00 1 0.50 0.80 5.00 5.00 5.01 4.99
+ //21 5.00 0 0.00 0.16 5.00 5.00 5.01 4.99
+ //22 5.00 0 0.00 0.11 5.00 5.00 5.01 4.99
+ //23 5.00 0 0.00 0.05 5.00 5.00 5.01 4.99
+ //24 5.00 0 0.00 0.11 5.00 5.00 5.01 4.99
+ //25 5.00 0 0.00 0.19 5.00 5.00 5.01 4.99
+ }
+
+ private static void PrintPrediction(int idx, double value, SrCnnAnomalyDetection prediction) =>
+ Console.WriteLine("{0}\t{1:0.00}\t{2}\t\t{3:0.00}\t{4:0.00}\t\t{5:0.00}\t\t{6:0.00}\t\t{7:0.00}\t\t{8:0.00}",
+ idx, value, prediction.Prediction[0], prediction.Prediction[1], prediction.Prediction[2],
+ prediction.Prediction[3], prediction.Prediction[4], prediction.Prediction[5], prediction.Prediction[6]);
+
+ private class TimeSeriesData
+ {
+ public double Value { get; set; }
+ }
+
+ private class SrCnnAnomalyDetection
+ {
+ [VectorType]
+ public double[] Prediction { get; set; }
+ }
+ }
+}

src/Microsoft.ML.Data/DataView/BatchDataViewMapperBase.cs

@@ -0,0 +1,173 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using Microsoft.ML.Runtime;
+
+namespace Microsoft.ML.Data.DataView
+{
+ internal abstract class BatchDataViewMapperBase<TInput, TBatch> : IDataView
+ {
+ public bool CanShuffle => false;
+
+ public DataViewSchema Schema => SchemaBindings.AsSchema;
+
+ private readonly IDataView _source;
+ protected readonly IHost Host;
+
+ protected BatchDataViewMapperBase(IHostEnvironment env, string registrationName, IDataView input)
+ {
+ Contracts.CheckValue(env, nameof(env));
+ Host = env.Register(registrationName);
+ _source = input;
+ }
+
+ public long? GetRowCount() => _source.GetRowCount();
+
+ public DataViewRowCursor GetRowCursor(IEnumerable<DataViewSchema.Column> columnsNeeded, Random rand = null)
+ {
+ Host.CheckValue(columnsNeeded, nameof(columnsNeeded));
+ Host.CheckValueOrNull(rand);
+
+ var predicate = RowCursorUtils.FromColumnsToPredicate(columnsNeeded, SchemaBindings.AsSchema);
+
+ // If we aren't selecting any of the output columns, don't construct our cursor.
+ // Note that because we cannot support random due to the inherently
+ // stratified nature, neither can we allow the base data to be shuffled,
+ // even if it supports shuffling.
+ if (!SchemaBindings.AnyNewColumnsActive(predicate))
+ {
+ var activeInput = SchemaBindings.GetActiveInput(predicate);
+ var inputCursor = _source.GetRowCursor(_source.Schema.Where(c => activeInput[c.Index]), null);
+ return new BindingsWrappedRowCursor(Host, inputCursor, SchemaBindings);
+ }
+ var active = SchemaBindings.GetActive(predicate);
+ Contracts.Assert(active.Length == SchemaBindings.ColumnCount);
+
+ // REVIEW: We can get a different input predicate for the input cursor and for the lookahead cursor. The lookahead
+ // cursor is only used for getting the values from the input column, so it only needs that column activated. The
+ // other cursor is used to get source columns, so it needs the rest of them activated.
+ var predInput = GetSchemaBindingDependencies(predicate);
+ var inputCols = _source.Schema.Where(c => predInput(c.Index));
+ return new Cursor(this, _source.GetRowCursor(inputCols), _source.GetRowCursor(inputCols), active);
+ }
+
+ public DataViewRowCursor[] GetRowCursorSet(IEnumerable<DataViewSchema.Column> columnsNeeded, int n, Random rand = null)
+ {
+ return new[] { GetRowCursor(columnsNeeded, rand) };
+ }
+
+ protected abstract ColumnBindingsBase SchemaBindings { get; }
+ protected abstract TBatch CreateBatch(DataViewRowCursor input);
+ protected abstract void ProcessBatch(TBatch currentBatch);
+ protected abstract void ProcessExample(TBatch currentBatch, TInput currentInput);
+ protected abstract Func<bool> GetLastInBatchDelegate(DataViewRowCursor lookAheadCursor);
+ protected abstract Func<bool> GetIsNewBatchDelegate(DataViewRowCursor lookAheadCursor);
+ protected abstract ValueGetter<TInput> GetLookAheadGetter(DataViewRowCursor lookAheadCursor);
+ protected abstract Delegate[] CreateGetters(DataViewRowCursor input, TBatch currentBatch, bool[] active);
+ protected abstract Func<int, bool> GetSchemaBindingDependencies(Func<int, bool> predicate);
+
+ private sealed class Cursor : RootCursorBase
+ {
+ private readonly BatchDataViewMapperBase<TInput, TBatch> _parent;
+ private readonly DataViewRowCursor _lookAheadCursor;
+ private readonly DataViewRowCursor _input;
+
+ private readonly bool[] _active;
+ private readonly Delegate[] _getters;
+
+ private readonly TBatch _currentBatch;
+ private readonly Func<bool> _lastInBatchInLookAheadCursorDel;
+ private readonly Func<bool> _firstInBatchInInputCursorDel;
+ private readonly ValueGetter<TInput> _inputGetterInLookAheadCursor;
+ private TInput _currentInput;
+
+ public override long Batch => 0;
+
+ public override DataViewSchema Schema => _parent.Schema;
+
+ public Cursor(BatchDataViewMapperBase<TInput, TBatch> parent, DataViewRowCursor input, DataViewRowCursor lookAheadCursor, bool[] active)
+ : base(parent.Host)
+ {
+ _parent = parent;
+ _input = input;
+ _lookAheadCursor = lookAheadCursor;
+ _active = active;
+
+ _currentBatch = _parent.CreateBatch(_input);
+
+ _getters = _parent.CreateGetters(_input, _currentBatch, _active);
+
+ _lastInBatchInLookAheadCursorDel = _parent.GetLastInBatchDelegate(_lookAheadCursor);
+ _firstInBatchInInputCursorDel = _parent.GetIsNewBatchDelegate(_input);
+ _inputGetterInLookAheadCursor = _parent.GetLookAheadGetter(_lookAheadCursor);
+ }
+
+ public override ValueGetter<TValue> GetGetter<TValue>(DataViewSchema.Column column)
+ {
+ Contracts.CheckParam(IsColumnActive(column), nameof(column), "requested column is not active");
+
+ var col = _parent.SchemaBindings.MapColumnIndex(out bool isSrc, column.Index);
+ if (isSrc)
+ {
+ Contracts.AssertValue(_input);
+ return _input.GetGetter<TValue>(_input.Schema[col]);
+ }
+
+ Ch.AssertValue(_getters);
+ var getter = _getters[col];
+ Ch.Assert(getter != null);
+ var fn = getter as ValueGetter<TValue>;
+ if (fn == null)
+ throw Ch.Except("Invalid TValue in GetGetter: '{0}'", typeof(TValue));
+ return fn;
+ }
+
+ public override ValueGetter<DataViewRowId> GetIdGetter()
+ {
+ return
+ (ref DataViewRowId val) =>
+ {
+ Ch.Check(IsGood, "Cannot call ID getter in current state");
+ val = new DataViewRowId((ulong)Position, 0);
+ };
+ }
+
+ public override bool IsColumnActive(DataViewSchema.Column column)
+ {
+ Ch.Check(column.Index < _parent.SchemaBindings.AsSchema.Count);
+ return _active[column.Index];
+ }
+
+ protected override bool MoveNextCore()
+ {
+ if (!_input.MoveNext())
+ return false;
+ if (!_firstInBatchInInputCursorDel())
+ return true;
+
+ // If we are here, this means that _input.MoveNext() has gotten us to the beginning of the next batch,
+ // so now we need to look ahead at the entire next batch in the _lookAheadCursor.
+ // The _lookAheadCursor's position should be on the last row of the previous batch (or -1).
+ Ch.Assert(_lastInBatchInLookAheadCursorDel());
+
+ var good = _lookAheadCursor.MoveNext();
+ // The two cursors should have the same number of elements, so if _input.MoveNext() returned true,
+ // then it must return true here too.
+ Ch.Assert(good);
+
+ do
+ {
+ _inputGetterInLookAheadCursor(ref _currentInput);
+ _parent.ProcessExample(_currentBatch, _currentInput);
+ } while (!_lastInBatchInLookAheadCursorDel() && _lookAheadCursor.MoveNext());
+
+ _parent.ProcessBatch(_currentBatch);
+ return true;
+ }
+ }
+ }
+}

src/Microsoft.ML.TimeSeries/ExtensionsCatalog.cs

@@ -2,8 +2,6 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 // See the LICENSE file in the project root for more information.
 
-using System;
-using System.Reflection;
 using Microsoft.ML.Data;
 using Microsoft.ML.Runtime;
 using Microsoft.ML.TimeSeries;
@@ -150,6 +148,35 @@ public static SrCnnAnomalyEstimator DetectAnomalyBySrCnn(this TransformsCatalog
  int windowSize = 64, int backAddWindowSize = 5, int lookaheadWindowSize = 5, int averageingWindowSize = 3, int judgementWindowSize = 21, double threshold = 0.3)
  => new SrCnnAnomalyEstimator(CatalogUtils.GetEnvironment(catalog), outputColumnName, windowSize, backAddWindowSize, lookaheadWindowSize, averageingWindowSize, judgementWindowSize, threshold, inputColumnName);
 
+ /// <summary>
+ /// Create <see cref="SrCnnEntireAnomalyDetector"/>, which detects timeseries anomalies for entire input using SRCNN algorithm.
+ /// </summary>
+ /// <param name="catalog">The AnomalyDetectionCatalog.</param>
+ /// <param name="input">Input DataView.</param>
+ /// <param name="outputColumnName">Name of the column resulting from data processing of <paramref name="inputColumnName"/>.
+ /// The column data is a vector of <see cref="System.Double"/>. The length of this vector varies depending on <paramref name="detectMode"/>.</param>
+ /// <param name="inputColumnName">Name of column to process. The column data must be <see cref="System.Double"/>.</param>
+ /// <param name="threshold">The threshold to determine anomaly, score larger than the threshold is considered as anomaly. Must be in [0,1]. Default value is 0.3.</param>
+ /// <param name="batchSize">Divide the input data into batches to fit srcnn model.
+ /// When set to -1, use the whole input to fit model instead of batch by batch, when set to a positive integer, use this number as batch size.
+ /// Must be -1 or a positive integer no less than 12. Default value is 1024.</param>
+ /// <param name="sensitivity">Sensitivity of boundaries, only useful when srCnnDetectMode is AnomalyAndMargin. Must be in [0,100]. Default value is 99.</param>
+ /// <param name="detectMode">An enum type of <see cref="SrCnnDetectMode"/>.
+ /// When set to AnomalyOnly, the output vector would be a 3-element Double vector of (IsAnomaly, RawScore, Mag).
+ /// When set to AnomalyAndExpectedValue, the output vector would be a 4-element Double vector of (IsAnomaly, RawScore, Mag, ExpectedValue).
+ /// When set to AnomalyAndMargin, the output vector would be a 7-element Double vector of (IsAnomaly, AnomalyScore, Mag, ExpectedValue, BoundaryUnit, UpperBoundary, LowerBoundary).
+ /// Default value is AnomalyOnly.</param>
+ /// <example>
+ /// <format type="text/markdown">
+ /// <![CDATA[
+ /// [!code-csharp[DetectEntireAnomalyBySrCnn](~/../docs/samples/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectEntireAnomalyBySrCnn.cs)]
+ /// ]]>
+ /// </format>
+ /// </example>
+ public static IDataView DetectEntireAnomalyBySrCnn(this AnomalyDetectionCatalog catalog, IDataView input, string outputColumnName, string inputColumnName,
+ double threshold = 0.3, int batchSize = 1024, double sensitivity = 99, SrCnnDetectMode detectMode = SrCnnDetectMode.AnomalyOnly)
+ => new SrCnnEntireAnomalyDetector(CatalogUtils.GetEnvironment(catalog), input, inputColumnName, outputColumnName, threshold, batchSize, sensitivity, detectMode);
+
  /// <summary>
  /// Create <see cref="RootCause"/>, which localizes root causes using decision tree algorithm.
  /// </summary>

docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectEntireAnomalyBySrCnn.cs

@@ -0,0 +1,99 @@
+using System;
+using System.Collections.Generic;
+using Microsoft.ML;
+using Microsoft.ML.Data;
+using Microsoft.ML.TimeSeries;
+
+namespace Samples.Dynamic
+{
+ public static class DetectEntireAnomalyBySrCnn
+ {
+ public static void Example()
+ {
+ // Create a new ML context, for ML.NET operations. It can be used for
+ // exception tracking and logging, 
+ // as well as the source of randomness.
+ var ml = new MLContext();
+
+ // Generate sample series data with an anomaly
+ var data = new List<TimeSeriesData>();
+ for (int index = 0; index < 20; index++)
+ {
+ data.Add(new TimeSeriesData { Value = 5 });
+ }
+ data.Add(new TimeSeriesData { Value = 10 });
+ for (int index = 0; index < 5; index++)
+ {
+ data.Add(new TimeSeriesData { Value = 5 });
+ }
+
+ // Convert data to IDataView.
+ var dataView = ml.Data.LoadFromEnumerable(data);
+
+ // Setup the detection arguments
+ string outputColumnName = nameof(SrCnnAnomalyDetection.Prediction);
+ string inputColumnName = nameof(TimeSeriesData.Value);
+
+ // Do batch anomaly detection
+ var outputDataView = ml.AnomalyDetection.DetectEntireAnomalyBySrCnn(dataView, outputColumnName, inputColumnName,
+ threshold: 0.35, batchSize: 512, sensitivity: 90.0, detectMode: SrCnnDetectMode.AnomalyAndMargin);
+
+ // Getting the data of the newly created column as an IEnumerable of
+ // SrCnnAnomalyDetection.
+ var predictionColumn = ml.Data.CreateEnumerable<SrCnnAnomalyDetection>(
+ outputDataView, reuseRowObject: false);
+
+ Console.WriteLine("Index\tData\tAnomaly\tAnomalyScore\tMag\tExpectedValue\tBoundaryUnit\tUpperBoundary\tLowerBoundary");
+
+ int k = 0;
+ foreach (var prediction in predictionColumn)
+ {
+ PrintPrediction(k, data[k].Value, prediction);
+ k++;
+ }
+ //Index Data Anomaly AnomalyScore Mag ExpectedValue BoundaryUnit UpperBoundary LowerBoundary
+ //0 5.00 0 0.00 0.21 5.00 5.00 5.01 4.99
+ //1 5.00 0 0.00 0.11 5.00 5.00 5.01 4.99
+ //2 5.00 0 0.00 0.03 5.00 5.00 5.01 4.99
+ //3 5.00 0 0.00 0.01 5.00 5.00 5.01 4.99
+ //4 5.00 0 0.00 0.03 5.00 5.00 5.01 4.99
+ //5 5.00 0 0.00 0.06 5.00 5.00 5.01 4.99
+ //6 5.00 0 0.00 0.02 5.00 5.00 5.01 4.99
+ //7 5.00 0 0.00 0.01 5.00 5.00 5.01 4.99
+ //8 5.00 0 0.00 0.01 5.00 5.00 5.01 4.99
+ //9 5.00 0 0.00 0.01 5.00 5.00 5.01 4.99
+ //10 5.00 0 0.00 0.00 5.00 5.00 5.01 4.99
+ //11 5.00 0 0.00 0.01 5.00 5.00 5.01 4.99
+ //12 5.00 0 0.00 0.01 5.00 5.00 5.01 4.99
+ //13 5.00 0 0.00 0.02 5.00 5.00 5.01 4.99
+ //14 5.00 0 0.00 0.07 5.00 5.00 5.01 4.99
+ //15 5.00 0 0.00 0.08 5.00 5.00 5.01 4.99
+ //16 5.00 0 0.00 0.02 5.00 5.00 5.01 4.99
+ //17 5.00 0 0.00 0.05 5.00 5.00 5.01 4.99
+ //18 5.00 0 0.00 0.12 5.00 5.00 5.01 4.99
+ //19 5.00 0 0.00 0.17 5.00 5.00 5.01 4.99
+ //20 10.00 1 0.50 0.80 5.00 5.00 5.01 4.99
+ //21 5.00 0 0.00 0.16 5.00 5.00 5.01 4.99
+ //22 5.00 0 0.00 0.11 5.00 5.00 5.01 4.99
+ //23 5.00 0 0.00 0.05 5.00 5.00 5.01 4.99
+ //24 5.00 0 0.00 0.11 5.00 5.00 5.01 4.99
+ //25 5.00 0 0.00 0.19 5.00 5.00 5.01 4.99
+ }
+
+ private static void PrintPrediction(int idx, double value, SrCnnAnomalyDetection prediction) =>
+ Console.WriteLine("{0}\t{1:0.00}\t{2}\t\t{3:0.00}\t{4:0.00}\t\t{5:0.00}\t\t{6:0.00}\t\t{7:0.00}\t\t{8:0.00}",
+ idx, value, prediction.Prediction[0], prediction.Prediction[1], prediction.Prediction[2],
+ prediction.Prediction[3], prediction.Prediction[4], prediction.Prediction[5], prediction.Prediction[6]);
+
+ private class TimeSeriesData
+ {
+ public double Value { get; set; }
+ }
+
+ private class SrCnnAnomalyDetection
+ {
+ [VectorType]
+ public double[] Prediction { get; set; }
+ }
+ }
+}

src/Microsoft.ML.Data/DataView/BatchDataViewMapperBase.cs

@@ -0,0 +1,173 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using Microsoft.ML.Runtime;
+
+namespace Microsoft.ML.Data.DataView
+{
+ internal abstract class BatchDataViewMapperBase<TInput, TBatch> : IDataView
+ {
+ public bool CanShuffle => false;
+
+ public DataViewSchema Schema => SchemaBindings.AsSchema;
+
+ private readonly IDataView _source;
+ protected readonly IHost Host;
+
+ protected BatchDataViewMapperBase(IHostEnvironment env, string registrationName, IDataView input)
+ {
+ Contracts.CheckValue(env, nameof(env));
+ Host = env.Register(registrationName);
+ _source = input;
+ }
+
+ public long? GetRowCount() => _source.GetRowCount();
+
+ public DataViewRowCursor GetRowCursor(IEnumerable<DataViewSchema.Column> columnsNeeded, Random rand = null)
+ {
+ Host.CheckValue(columnsNeeded, nameof(columnsNeeded));
+ Host.CheckValueOrNull(rand);
+
+ var predicate = RowCursorUtils.FromColumnsToPredicate(columnsNeeded, SchemaBindings.AsSchema);
+
+ // If we aren't selecting any of the output columns, don't construct our cursor.
+ // Note that because we cannot support random due to the inherently
+ // stratified nature, neither can we allow the base data to be shuffled,
+ // even if it supports shuffling.
+ if (!SchemaBindings.AnyNewColumnsActive(predicate))
+ {
+ var activeInput = SchemaBindings.GetActiveInput(predicate);
+ var inputCursor = _source.GetRowCursor(_source.Schema.Where(c => activeInput[c.Index]), null);
+ return new BindingsWrappedRowCursor(Host, inputCursor, SchemaBindings);
+ }
+ var active = SchemaBindings.GetActive(predicate);
+ Contracts.Assert(active.Length == SchemaBindings.ColumnCount);
+
+ // REVIEW: We can get a different input predicate for the input cursor and for the lookahead cursor. The lookahead
+ // cursor is only used for getting the values from the input column, so it only needs that column activated. The
+ // other cursor is used to get source columns, so it needs the rest of them activated.
+ var predInput = GetSchemaBindingDependencies(predicate);
+ var inputCols = _source.Schema.Where(c => predInput(c.Index));
+ return new Cursor(this, _source.GetRowCursor(inputCols), _source.GetRowCursor(inputCols), active);
+ }
+
+ public DataViewRowCursor[] GetRowCursorSet(IEnumerable<DataViewSchema.Column> columnsNeeded, int n, Random rand = null)
+ {
+ return new[] { GetRowCursor(columnsNeeded, rand) };
+ }
+
+ protected abstract ColumnBindingsBase SchemaBindings { get; }
+ protected abstract TBatch CreateBatch(DataViewRowCursor input);
+ protected abstract void ProcessBatch(TBatch currentBatch);
+ protected abstract void ProcessExample(TBatch currentBatch, TInput currentInput);
+ protected abstract Func<bool> GetLastInBatchDelegate(DataViewRowCursor lookAheadCursor);
+ protected abstract Func<bool> GetIsNewBatchDelegate(DataViewRowCursor lookAheadCursor);
+ protected abstract ValueGetter<TInput> GetLookAheadGetter(DataViewRowCursor lookAheadCursor);
+ protected abstract Delegate[] CreateGetters(DataViewRowCursor input, TBatch currentBatch, bool[] active);
+ protected abstract Func<int, bool> GetSchemaBindingDependencies(Func<int, bool> predicate);
+
+ private sealed class Cursor : RootCursorBase
+ {
+ private readonly BatchDataViewMapperBase<TInput, TBatch> _parent;
+ private readonly DataViewRowCursor _lookAheadCursor;
+ private readonly DataViewRowCursor _input;
+
+ private readonly bool[] _active;
+ private readonly Delegate[] _getters;
+
+ private readonly TBatch _currentBatch;
+ private readonly Func<bool> _lastInBatchInLookAheadCursorDel;
+ private readonly Func<bool> _firstInBatchInInputCursorDel;
+ private readonly ValueGetter<TInput> _inputGetterInLookAheadCursor;
+ private TInput _currentInput;
+
+ public override long Batch => 0;
+
+ public override DataViewSchema Schema => _parent.Schema;
+
+ public Cursor(BatchDataViewMapperBase<TInput, TBatch> parent, DataViewRowCursor input, DataViewRowCursor lookAheadCursor, bool[] active)
+ : base(parent.Host)
+ {
+ _parent = parent;
+ _input = input;
+ _lookAheadCursor = lookAheadCursor;
+ _active = active;
+
+ _currentBatch = _parent.CreateBatch(_input);
+
+ _getters = _parent.CreateGetters(_input, _currentBatch, _active);
+
+ _lastInBatchInLookAheadCursorDel = _parent.GetLastInBatchDelegate(_lookAheadCursor);
+ _firstInBatchInInputCursorDel = _parent.GetIsNewBatchDelegate(_input);
+ _inputGetterInLookAheadCursor = _parent.GetLookAheadGetter(_lookAheadCursor);
+ }
+
+ public override ValueGetter<TValue> GetGetter<TValue>(DataViewSchema.Column column)
+ {
+ Contracts.CheckParam(IsColumnActive(column), nameof(column), "requested column is not active");
+
+ var col = _parent.SchemaBindings.MapColumnIndex(out bool isSrc, column.Index);
+ if (isSrc)
+ {
+ Contracts.AssertValue(_input);
+ return _input.GetGetter<TValue>(_input.Schema[col]);
+ }
+
+ Ch.AssertValue(_getters);
+ var getter = _getters[col];
+ Ch.Assert(getter != null);
+ var fn = getter as ValueGetter<TValue>;
+ if (fn == null)
+ throw Ch.Except("Invalid TValue in GetGetter: '{0}'", typeof(TValue));
+ return fn;
+ }
+
+ public override ValueGetter<DataViewRowId> GetIdGetter()
+ {
+ return
+ (ref DataViewRowId val) =>
+ {
+ Ch.Check(IsGood, "Cannot call ID getter in current state");
+ val = new DataViewRowId((ulong)Position, 0);
+ };
+ }
+
+ public override bool IsColumnActive(DataViewSchema.Column column)
+ {
+ Ch.Check(column.Index < _parent.SchemaBindings.AsSchema.Count);
+ return _active[column.Index];
+ }
+
+ protected override bool MoveNextCore()
+ {
+ if (!_input.MoveNext())
+ return false;
+ if (!_firstInBatchInInputCursorDel())
+ return true;
+
+ // If we are here, this means that _input.MoveNext() has gotten us to the beginning of the next batch,
+ // so now we need to look ahead at the entire next batch in the _lookAheadCursor.
+ // The _lookAheadCursor's position should be on the last row of the previous batch (or -1).
+ Ch.Assert(_lastInBatchInLookAheadCursorDel());
+
+ var good = _lookAheadCursor.MoveNext();
+ // The two cursors should have the same number of elements, so if _input.MoveNext() returned true,
+ // then it must return true here too.
+ Ch.Assert(good);
+
+ do
+ {
+ _inputGetterInLookAheadCursor(ref _currentInput);
+ _parent.ProcessExample(_currentBatch, _currentInput);
+ } while (!_lastInBatchInLookAheadCursorDel() && _lookAheadCursor.MoveNext());
+
+ _parent.ProcessBatch(_currentBatch);
+ return true;
+ }
+ }
+ }
+}

src/Microsoft.ML.TimeSeries/ExtensionsCatalog.cs

@@ -2,8 +2,6 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 // See the LICENSE file in the project root for more information.
 
-using System;
-using System.Reflection;
 using Microsoft.ML.Data;
 using Microsoft.ML.Runtime;
 using Microsoft.ML.TimeSeries;
@@ -150,6 +148,35 @@ public static SrCnnAnomalyEstimator DetectAnomalyBySrCnn(this TransformsCatalog
  int windowSize = 64, int backAddWindowSize = 5, int lookaheadWindowSize = 5, int averageingWindowSize = 3, int judgementWindowSize = 21, double threshold = 0.3)
  => new SrCnnAnomalyEstimator(CatalogUtils.GetEnvironment(catalog), outputColumnName, windowSize, backAddWindowSize, lookaheadWindowSize, averageingWindowSize, judgementWindowSize, threshold, inputColumnName);
 
+ /// <summary>
+ /// Create <see cref="SrCnnEntireAnomalyDetector"/>, which detects timeseries anomalies for entire input using SRCNN algorithm.
+ /// </summary>
+ /// <param name="catalog">The AnomalyDetectionCatalog.</param>
+ /// <param name="input">Input DataView.</param>
+ /// <param name="outputColumnName">Name of the column resulting from data processing of <paramref name="inputColumnName"/>.
+ /// The column data is a vector of <see cref="System.Double"/>. The length of this vector varies depending on <paramref name="detectMode"/>.</param>
+ /// <param name="inputColumnName">Name of column to process. The column data must be <see cref="System.Double"/>.</param>
+ /// <param name="threshold">The threshold to determine anomaly, score larger than the threshold is considered as anomaly. Must be in [0,1]. Default value is 0.3.</param>
+ /// <param name="batchSize">Divide the input data into batches to fit srcnn model.
+ /// When set to -1, use the whole input to fit model instead of batch by batch, when set to a positive integer, use this number as batch size.
+ /// Must be -1 or a positive integer no less than 12. Default value is 1024.</param>
+ /// <param name="sensitivity">Sensitivity of boundaries, only useful when srCnnDetectMode is AnomalyAndMargin. Must be in [0,100]. Default value is 99.</param>
+ /// <param name="detectMode">An enum type of <see cref="SrCnnDetectMode"/>.
+ /// When set to AnomalyOnly, the output vector would be a 3-element Double vector of (IsAnomaly, RawScore, Mag).
+ /// When set to AnomalyAndExpectedValue, the output vector would be a 4-element Double vector of (IsAnomaly, RawScore, Mag, ExpectedValue).
+ /// When set to AnomalyAndMargin, the output vector would be a 7-element Double vector of (IsAnomaly, AnomalyScore, Mag, ExpectedValue, BoundaryUnit, UpperBoundary, LowerBoundary).
+ /// Default value is AnomalyOnly.</param>
+ /// <example>
+ /// <format type="text/markdown">
+ /// <![CDATA[
+ /// [!code-csharp[DetectEntireAnomalyBySrCnn](~/../docs/samples/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectEntireAnomalyBySrCnn.cs)]
+ /// ]]>
+ /// </format>
+ /// </example>
+ public static IDataView DetectEntireAnomalyBySrCnn(this AnomalyDetectionCatalog catalog, IDataView input, string outputColumnName, string inputColumnName,
+ double threshold = 0.3, int batchSize = 1024, double sensitivity = 99, SrCnnDetectMode detectMode = SrCnnDetectMode.AnomalyOnly)
+ => new SrCnnEntireAnomalyDetector(CatalogUtils.GetEnvironment(catalog), input, inputColumnName, outputColumnName, threshold, batchSize, sensitivity, detectMode);
+
  /// <summary>
  /// Create <see cref="RootCause"/>, which localizes root causes using decision tree algorithm.
  /// </summary>

docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectEntireAnomalyBySrCnn.cs

@@ -0,0 +1,99 @@
+using System;
+using System.Collections.Generic;
+using Microsoft.ML;
+using Microsoft.ML.Data;
+using Microsoft.ML.TimeSeries;
+
+namespace Samples.Dynamic
+{
+ public static class DetectEntireAnomalyBySrCnn
+ {
+ public static void Example()
+ {
+ // Create a new ML context, for ML.NET operations. It can be used for
+ // exception tracking and logging, 
+ // as well as the source of randomness.
+ var ml = new MLContext();
+
+ // Generate sample series data with an anomaly
+ var data = new List<TimeSeriesData>();
+ for (int index = 0; index < 20; index++)
+ {
+ data.Add(new TimeSeriesData { Value = 5 });
+ }
+ data.Add(new TimeSeriesData { Value = 10 });
+ for (int index = 0; index < 5; index++)
+ {
+ data.Add(new TimeSeriesData { Value = 5 });
+ }
+
+ // Convert data to IDataView.
+ var dataView = ml.Data.LoadFromEnumerable(data);
+
+ // Setup the detection arguments
+ string outputColumnName = nameof(SrCnnAnomalyDetection.Prediction);
+ string inputColumnName = nameof(TimeSeriesData.Value);
+
+ // Do batch anomaly detection
+ var outputDataView = ml.AnomalyDetection.DetectEntireAnomalyBySrCnn(dataView, outputColumnName, inputColumnName,
+ threshold: 0.35, batchSize: 512, sensitivity: 90.0, detectMode: SrCnnDetectMode.AnomalyAndMargin);
+
+ // Getting the data of the newly created column as an IEnumerable of
+ // SrCnnAnomalyDetection.
+ var predictionColumn = ml.Data.CreateEnumerable<SrCnnAnomalyDetection>(
+ outputDataView, reuseRowObject: false);
+
+ Console.WriteLine("Index\tData\tAnomaly\tAnomalyScore\tMag\tExpectedValue\tBoundaryUnit\tUpperBoundary\tLowerBoundary");
+
+ int k = 0;
+ foreach (var prediction in predictionColumn)
+ {
+ PrintPrediction(k, data[k].Value, prediction);
+ k++;
+ }
+ //Index Data Anomaly AnomalyScore Mag ExpectedValue BoundaryUnit UpperBoundary LowerBoundary
+ //0 5.00 0 0.00 0.21 5.00 5.00 5.01 4.99
+ //1 5.00 0 0.00 0.11 5.00 5.00 5.01 4.99
+ //2 5.00 0 0.00 0.03 5.00 5.00 5.01 4.99
+ //3 5.00 0 0.00 0.01 5.00 5.00 5.01 4.99
+ //4 5.00 0 0.00 0.03 5.00 5.00 5.01 4.99
+ //5 5.00 0 0.00 0.06 5.00 5.00 5.01 4.99
+ //6 5.00 0 0.00 0.02 5.00 5.00 5.01 4.99
+ //7 5.00 0 0.00 0.01 5.00 5.00 5.01 4.99
+ //8 5.00 0 0.00 0.01 5.00 5.00 5.01 4.99
+ //9 5.00 0 0.00 0.01 5.00 5.00 5.01 4.99
+ //10 5.00 0 0.00 0.00 5.00 5.00 5.01 4.99
+ //11 5.00 0 0.00 0.01 5.00 5.00 5.01 4.99
+ //12 5.00 0 0.00 0.01 5.00 5.00 5.01 4.99
+ //13 5.00 0 0.00 0.02 5.00 5.00 5.01 4.99
+ //14 5.00 0 0.00 0.07 5.00 5.00 5.01 4.99
+ //15 5.00 0 0.00 0.08 5.00 5.00 5.01 4.99
+ //16 5.00 0 0.00 0.02 5.00 5.00 5.01 4.99
+ //17 5.00 0 0.00 0.05 5.00 5.00 5.01 4.99
+ //18 5.00 0 0.00 0.12 5.00 5.00 5.01 4.99
+ //19 5.00 0 0.00 0.17 5.00 5.00 5.01 4.99
+ //20 10.00 1 0.50 0.80 5.00 5.00 5.01 4.99
+ //21 5.00 0 0.00 0.16 5.00 5.00 5.01 4.99
+ //22 5.00 0 0.00 0.11 5.00 5.00 5.01 4.99
+ //23 5.00 0 0.00 0.05 5.00 5.00 5.01 4.99
+ //24 5.00 0 0.00 0.11 5.00 5.00 5.01 4.99
+ //25 5.00 0 0.00 0.19 5.00 5.00 5.01 4.99
+ }
+
+ private static void PrintPrediction(int idx, double value, SrCnnAnomalyDetection prediction) =>
+ Console.WriteLine("{0}\t{1:0.00}\t{2}\t\t{3:0.00}\t{4:0.00}\t\t{5:0.00}\t\t{6:0.00}\t\t{7:0.00}\t\t{8:0.00}",
+ idx, value, prediction.Prediction[0], prediction.Prediction[1], prediction.Prediction[2],
+ prediction.Prediction[3], prediction.Prediction[4], prediction.Prediction[5], prediction.Prediction[6]);
+
+ private class TimeSeriesData
+ {
+ public double Value { get; set; }
+ }
+
+ private class SrCnnAnomalyDetection
+ {
+ [VectorType]
+ public double[] Prediction { get; set; }
+ }
+ }
+}

src/Microsoft.ML.Data/DataView/BatchDataViewMapperBase.cs

@@ -0,0 +1,173 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using Microsoft.ML.Runtime;
+
+namespace Microsoft.ML.Data.DataView
+{
+ internal abstract class BatchDataViewMapperBase<TInput, TBatch> : IDataView
+ {
+ public bool CanShuffle => false;
+
+ public DataViewSchema Schema => SchemaBindings.AsSchema;
+
+ private readonly IDataView _source;
+ protected readonly IHost Host;
+
+ protected BatchDataViewMapperBase(IHostEnvironment env, string registrationName, IDataView input)
+ {
+ Contracts.CheckValue(env, nameof(env));
+ Host = env.Register(registrationName);
+ _source = input;
+ }
+
+ public long? GetRowCount() => _source.GetRowCount();
+
+ public DataViewRowCursor GetRowCursor(IEnumerable<DataViewSchema.Column> columnsNeeded, Random rand = null)
+ {
+ Host.CheckValue(columnsNeeded, nameof(columnsNeeded));
+ Host.CheckValueOrNull(rand);
+
+ var predicate = RowCursorUtils.FromColumnsToPredicate(columnsNeeded, SchemaBindings.AsSchema);
+
+ // If we aren't selecting any of the output columns, don't construct our cursor.
+ // Note that because we cannot support random due to the inherently
+ // stratified nature, neither can we allow the base data to be shuffled,
+ // even if it supports shuffling.
+ if (!SchemaBindings.AnyNewColumnsActive(predicate))
+ {
+ var activeInput = SchemaBindings.GetActiveInput(predicate);
+ var inputCursor = _source.GetRowCursor(_source.Schema.Where(c => activeInput[c.Index]), null);
+ return new BindingsWrappedRowCursor(Host, inputCursor, SchemaBindings);
+ }
+ var active = SchemaBindings.GetActive(predicate);
+ Contracts.Assert(active.Length == SchemaBindings.ColumnCount);
+
+ // REVIEW: We can get a different input predicate for the input cursor and for the lookahead cursor. The lookahead
+ // cursor is only used for getting the values from the input column, so it only needs that column activated. The
+ // other cursor is used to get source columns, so it needs the rest of them activated.
+ var predInput = GetSchemaBindingDependencies(predicate);
+ var inputCols = _source.Schema.Where(c => predInput(c.Index));
+ return new Cursor(this, _source.GetRowCursor(inputCols), _source.GetRowCursor(inputCols), active);
+ }
+
+ public DataViewRowCursor[] GetRowCursorSet(IEnumerable<DataViewSchema.Column> columnsNeeded, int n, Random rand = null)
+ {
+ return new[] { GetRowCursor(columnsNeeded, rand) };
+ }
+
+ protected abstract ColumnBindingsBase SchemaBindings { get; }
+ protected abstract TBatch CreateBatch(DataViewRowCursor input);
+ protected abstract void ProcessBatch(TBatch currentBatch);
+ protected abstract void ProcessExample(TBatch currentBatch, TInput currentInput);
+ protected abstract Func<bool> GetLastInBatchDelegate(DataViewRowCursor lookAheadCursor);
+ protected abstract Func<bool> GetIsNewBatchDelegate(DataViewRowCursor lookAheadCursor);
+ protected abstract ValueGetter<TInput> GetLookAheadGetter(DataViewRowCursor lookAheadCursor);
+ protected abstract Delegate[] CreateGetters(DataViewRowCursor input, TBatch currentBatch, bool[] active);
+ protected abstract Func<int, bool> GetSchemaBindingDependencies(Func<int, bool> predicate);
+
+ private sealed class Cursor : RootCursorBase
+ {
+ private readonly BatchDataViewMapperBase<TInput, TBatch> _parent;
+ private readonly DataViewRowCursor _lookAheadCursor;
+ private readonly DataViewRowCursor _input;
+
+ private readonly bool[] _active;
+ private readonly Delegate[] _getters;
+
+ private readonly TBatch _currentBatch;
+ private readonly Func<bool> _lastInBatchInLookAheadCursorDel;
+ private readonly Func<bool> _firstInBatchInInputCursorDel;
+ private readonly ValueGetter<TInput> _inputGetterInLookAheadCursor;
+ private TInput _currentInput;
+
+ public override long Batch => 0;
+
+ public override DataViewSchema Schema => _parent.Schema;
+
+ public Cursor(BatchDataViewMapperBase<TInput, TBatch> parent, DataViewRowCursor input, DataViewRowCursor lookAheadCursor, bool[] active)
+ : base(parent.Host)
+ {
+ _parent = parent;
+ _input = input;
+ _lookAheadCursor = lookAheadCursor;
+ _active = active;
+
+ _currentBatch = _parent.CreateBatch(_input);
+
+ _getters = _parent.CreateGetters(_input, _currentBatch, _active);
+
+ _lastInBatchInLookAheadCursorDel = _parent.GetLastInBatchDelegate(_lookAheadCursor);
+ _firstInBatchInInputCursorDel = _parent.GetIsNewBatchDelegate(_input);
+ _inputGetterInLookAheadCursor = _parent.GetLookAheadGetter(_lookAheadCursor);
+ }
+
+ public override ValueGetter<TValue> GetGetter<TValue>(DataViewSchema.Column column)
+ {
+ Contracts.CheckParam(IsColumnActive(column), nameof(column), "requested column is not active");
+
+ var col = _parent.SchemaBindings.MapColumnIndex(out bool isSrc, column.Index);
+ if (isSrc)
+ {
+ Contracts.AssertValue(_input);
+ return _input.GetGetter<TValue>(_input.Schema[col]);
+ }
+
+ Ch.AssertValue(_getters);
+ var getter = _getters[col];
+ Ch.Assert(getter != null);
+ var fn = getter as ValueGetter<TValue>;
+ if (fn == null)
+ throw Ch.Except("Invalid TValue in GetGetter: '{0}'", typeof(TValue));
+ return fn;
+ }
+
+ public override ValueGetter<DataViewRowId> GetIdGetter()
+ {
+ return
+ (ref DataViewRowId val) =>
+ {
+ Ch.Check(IsGood, "Cannot call ID getter in current state");
+ val = new DataViewRowId((ulong)Position, 0);
+ };
+ }
+
+ public override bool IsColumnActive(DataViewSchema.Column column)
+ {
+ Ch.Check(column.Index < _parent.SchemaBindings.AsSchema.Count);
+ return _active[column.Index];
+ }
+
+ protected override bool MoveNextCore()
+ {
+ if (!_input.MoveNext())
+ return false;
+ if (!_firstInBatchInInputCursorDel())
+ return true;
+
+ // If we are here, this means that _input.MoveNext() has gotten us to the beginning of the next batch,
+ // so now we need to look ahead at the entire next batch in the _lookAheadCursor.
+ // The _lookAheadCursor's position should be on the last row of the previous batch (or -1).
+ Ch.Assert(_lastInBatchInLookAheadCursorDel());
+
+ var good = _lookAheadCursor.MoveNext();
+ // The two cursors should have the same number of elements, so if _input.MoveNext() returned true,
+ // then it must return true here too.
+ Ch.Assert(good);
+
+ do
+ {
+ _inputGetterInLookAheadCursor(ref _currentInput);
+ _parent.ProcessExample(_currentBatch, _currentInput);
+ } while (!_lastInBatchInLookAheadCursorDel() && _lookAheadCursor.MoveNext());
+
+ _parent.ProcessBatch(_currentBatch);
+ return true;
+ }
+ }
+ }
+}

src/Microsoft.ML.TimeSeries/ExtensionsCatalog.cs

@@ -2,8 +2,6 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 // See the LICENSE file in the project root for more information.
 
-using System;
-using System.Reflection;
 using Microsoft.ML.Data;
 using Microsoft.ML.Runtime;
 using Microsoft.ML.TimeSeries;
@@ -150,6 +148,35 @@ public static SrCnnAnomalyEstimator DetectAnomalyBySrCnn(this TransformsCatalog
  int windowSize = 64, int backAddWindowSize = 5, int lookaheadWindowSize = 5, int averageingWindowSize = 3, int judgementWindowSize = 21, double threshold = 0.3)
  => new SrCnnAnomalyEstimator(CatalogUtils.GetEnvironment(catalog), outputColumnName, windowSize, backAddWindowSize, lookaheadWindowSize, averageingWindowSize, judgementWindowSize, threshold, inputColumnName);
 
+ /// <summary>
+ /// Create <see cref="SrCnnEntireAnomalyDetector"/>, which detects timeseries anomalies for entire input using SRCNN algorithm.
+ /// </summary>
+ /// <param name="catalog">The AnomalyDetectionCatalog.</param>
+ /// <param name="input">Input DataView.</param>
+ /// <param name="outputColumnName">Name of the column resulting from data processing of <paramref name="inputColumnName"/>.
+ /// The column data is a vector of <see cref="System.Double"/>. The length of this vector varies depending on <paramref name="detectMode"/>.</param>
+ /// <param name="inputColumnName">Name of column to process. The column data must be <see cref="System.Double"/>.</param>
+ /// <param name="threshold">The threshold to determine anomaly, score larger than the threshold is considered as anomaly. Must be in [0,1]. Default value is 0.3.</param>
+ /// <param name="batchSize">Divide the input data into batches to fit srcnn model.
+ /// When set to -1, use the whole input to fit model instead of batch by batch, when set to a positive integer, use this number as batch size.
+ /// Must be -1 or a positive integer no less than 12. Default value is 1024.</param>
+ /// <param name="sensitivity">Sensitivity of boundaries, only useful when srCnnDetectMode is AnomalyAndMargin. Must be in [0,100]. Default value is 99.</param>
+ /// <param name="detectMode">An enum type of <see cref="SrCnnDetectMode"/>.
+ /// When set to AnomalyOnly, the output vector would be a 3-element Double vector of (IsAnomaly, RawScore, Mag).
+ /// When set to AnomalyAndExpectedValue, the output vector would be a 4-element Double vector of (IsAnomaly, RawScore, Mag, ExpectedValue).
+ /// When set to AnomalyAndMargin, the output vector would be a 7-element Double vector of (IsAnomaly, AnomalyScore, Mag, ExpectedValue, BoundaryUnit, UpperBoundary, LowerBoundary).
+ /// Default value is AnomalyOnly.</param>
+ /// <example>
+ /// <format type="text/markdown">
+ /// <![CDATA[
+ /// [!code-csharp[DetectEntireAnomalyBySrCnn](~/../docs/samples/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectEntireAnomalyBySrCnn.cs)]
+ /// ]]>
+ /// </format>
+ /// </example>
+ public static IDataView DetectEntireAnomalyBySrCnn(this AnomalyDetectionCatalog catalog, IDataView input, string outputColumnName, string inputColumnName,
+ double threshold = 0.3, int batchSize = 1024, double sensitivity = 99, SrCnnDetectMode detectMode = SrCnnDetectMode.AnomalyOnly)
+ => new SrCnnEntireAnomalyDetector(CatalogUtils.GetEnvironment(catalog), input, inputColumnName, outputColumnName, threshold, batchSize, sensitivity, detectMode);
+
  /// <summary>
  /// Create <see cref="RootCause"/>, which localizes root causes using decision tree algorithm.
  /// </summary>