Combine multiple tree ensemble models into a single tree ensemble

src/Microsoft.ML.Core/Prediction/ITrainer.cs

@@ -74,6 +74,8 @@ public interface IModelCombiner<TModel, TPredictor>
  TPredictor CombineModels(IEnumerable<TModel> models);
  }
 
+ public delegate void SignatureModelCombiner(PredictionKind kind);
+
  /// <summary>
  /// Weakly typed interface for a trainer "session" that produces a predictor.
  /// </summary>

src/Microsoft.ML.FastTree/Microsoft.ML.FastTree.csproj

@@ -57,6 +57,7 @@
  <Compile Include="TreeEnsemble\Ensemble.cs" />
  <Compile Include="TreeEnsemble\QuantileRegressionTree.cs" />
  <Compile Include="TreeEnsemble\RegressionTree.cs" />
+ <Compile Include="TreeEnsemble\TreeEnsembleCombiner.cs" />
  <Compile Include="Training\Applications\GradientWrappers.cs" />
  <Compile Include="Training\Applications\ObjectiveFunction.cs" />
  <Compile Include="Training\BaggingProvider.cs" />

src/Microsoft.ML.FastTree/TreeEnsemble/RegressionTree.cs

@@ -105,22 +105,21 @@ public RegressionTree(byte[] buffer, ref int position)
  LteChild = buffer.ToIntArray(ref position);
  GtChild = buffer.ToIntArray(ref position);
  SplitFeatures = buffer.ToIntArray(ref position);
-int[] categoricalNodeIndices = buffer.ToIntArray(ref position);
- CategoricalSplit = GetCategoricalSplitFromIndices(categoricalNodeIndices);
- if (categoricalNodeIndices?.Length > 0)
+byte[] categoricalSplitAsBytes = buffer.ToByteArray(ref position);
+ CategoricalSplit = categoricalSplitAsBytes.Select(b => b > 0).ToArray();
+ if (CategoricalSplit.Any(b => b))
  {
  CategoricalSplitFeatures = new int[NumNodes][];
  CategoricalSplitFeatureRanges = new int[NumNodes][];
-foreach (var index in categoricalNodeIndices)
+for (int index = 0; index < NumNodes; index++)
  {
- Contracts.Assert(CategoricalSplit[index]);
-
  CategoricalSplitFeatures[index] = buffer.ToIntArray(ref position);
- CategoricalSplitFeatureRanges[index] = buffer.ToIntArray(ref position, 2);
+ CategoricalSplitFeatureRanges[index] = buffer.ToIntArray(ref position);
  }
  }
 
  Thresholds = buffer.ToUIntArray(ref position);
+ RawThresholds = buffer.ToFloatArray(ref position);
  _splitGain = buffer.ToDoubleArray(ref position);
  _gainPValue = buffer.ToDoubleArray(ref position);
  _previousLeafValue = buffer.ToDoubleArray(ref position);
@@ -144,6 +143,23 @@ private bool[] GetCategoricalSplitFromIndices(int[] indices)
  return categoricalSplit;
  }
 
+ private bool[] GetCategoricalSplitFromBytes(byte[] indices)
+ {
+ bool[] categoricalSplit = new bool[NumNodes];
+ if (indices == null)
+ return categoricalSplit;
+
+ Contracts.Assert(indices.Length <= NumNodes);
+
+ foreach (int index in indices)
+ {
+ Contracts.Assert(index >= 0 && index < NumNodes);
+ categoricalSplit[index] = true;
+ }
+
+ return categoricalSplit;
+ }
+
  /// <summary>
  /// Create a Regression Tree object from raw tree contents.
  /// </summary>
@@ -192,7 +208,7 @@ internal RegressionTree(int[] splitFeatures, Double[] splitGain, Double[] gainPV
  LeafValues = leafValues;
  CategoricalSplitFeatures = categoricalSplitFeatures;
  CategoricalSplitFeatureRanges = new int[CategoricalSplitFeatures.Length][];
- for(int i= 0; i < CategoricalSplitFeatures.Length; ++i)
+ for(int i= 0; i < CategoricalSplitFeatures.Length; ++i)
  {
  if (CategoricalSplitFeatures[i] != null && CategoricalSplitFeatures[i].Length > 0)
  {
@@ -500,6 +516,7 @@ public virtual int SizeInBytes()
  NumNodes * sizeof(int) +
  CategoricalSplit.Length * sizeof(bool) +
  Thresholds.SizeInBytes() +
+ RawThresholds.SizeInBytes() +
  _splitGain.SizeInBytes() +
  _gainPValue.SizeInBytes() +
  _previousLeafValue.SizeInBytes() +
@@ -514,22 +531,22 @@ public virtual void ToByteArray(byte[] buffer, ref int position)
  LteChild.ToByteArray(buffer, ref position);
  GtChild.ToByteArray(buffer, ref position);
  SplitFeatures.ToByteArray(buffer, ref position);
+ CategoricalSplit.Length.ToByteArray(buffer, ref position);
  foreach (var split in CategoricalSplit)
  Convert.ToByte(split).ToByteArray(buffer, ref position);
 
  if (CategoricalSplitFeatures != null)
  {
- foreach (var splits in CategoricalSplitFeatures)
- splits.ToByteArray(buffer, ref position);
- }
-
- if (CategoricalSplitFeatureRanges != null)
- {
- foreach (var ranges in CategoricalSplitFeatureRanges)
- ranges.ToByteArray(buffer, ref position);
+ Contracts.AssertValue(CategoricalSplitFeatureRanges);
+ for (int i = 0; i < CategoricalSplitFeatures.Length; i++)
+ {
+ CategoricalSplitFeatures[i].ToByteArray(buffer, ref position);
+ CategoricalSplitFeatureRanges[i].ToByteArray(buffer, ref position);
+ }
  }
 
  Thresholds.ToByteArray(buffer, ref position);
+ RawThresholds.ToByteArray(buffer, ref position);
  _splitGain.ToByteArray(buffer, ref position);
  _gainPValue.ToByteArray(buffer, ref position);
  _previousLeafValue.ToByteArray(buffer, ref position);

src/Microsoft.ML.FastTree/TreeEnsemble/TreeEnsembleCombiner.cs

@@ -0,0 +1,115 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System.Collections.Generic;
+using Microsoft.ML.Runtime;
+using Microsoft.ML.Runtime.FastTree.Internal;
+using Microsoft.ML.Runtime.Internal.Calibration;
+
+[assembly: LoadableClass(typeof(TreeEnsembleCombiner), null, typeof(SignatureModelCombiner), "Fast Tree Model Combiner", "FastTreeCombiner")]
+
+namespace Microsoft.ML.Runtime.FastTree.Internal
+{
+ public sealed class TreeEnsembleCombiner : IModelCombiner<IPredictorProducing<float>, IPredictorProducing<float>>
+ {
+ private readonly IHost _host;
+ private readonly PredictionKind _kind;
+
+ public TreeEnsembleCombiner(IHostEnvironment env, PredictionKind kind)
+ {
+ _host = env.Register("TreeEnsembleCombiner");
+ switch (kind)
+ {
+ case PredictionKind.BinaryClassification:
+ case PredictionKind.Regression:
+ case PredictionKind.Ranking:
+ _kind = kind;
+ break;
+ default:
+ throw _host.ExceptUserArg(nameof(kind), $"Tree ensembles can be either of type {nameof(PredictionKind.BinaryClassification)}, " +
+ $"{nameof(PredictionKind.Regression)} or {nameof(PredictionKind.Ranking)}");
+ }
+ }
+
+ public IPredictorProducing<float> CombineModels(IEnumerable<IPredictorProducing<float>> models)
+ {
+ _host.CheckValue(models, nameof(models));
+
+ var ensemble = new Ensemble();
+ int modelCount = 0;
+ int featureCount = -1;
+ bool binaryClassifier = false;
+ foreach (var model in models)
+ {
+ modelCount++;
+
+ var predictor = model;
+ _host.CheckValue(predictor, nameof(models), "One of the models is null");
+
+ var calibrated = predictor as CalibratedPredictorBase;
+ double paramA = 1;
+ if (calibrated != null)
+ {
+ _host.Check(calibrated.Calibrator is PlattCalibrator,
+ "Combining FastTree models can only be done when the models are calibrated with Platt calibrator");
+ predictor = calibrated.SubPredictor;
+ paramA = -(calibrated.Calibrator as PlattCalibrator).ParamA;
+ }
+ var tree = predictor as FastTreePredictionWrapper;
+ if (tree == null)
+ throw _host.Except("Model is not a tree ensemble");
+ foreach (var t in tree.TrainedEnsemble.Trees)
+ {
+ var bytes = new byte[t.SizeInBytes()];
+ int position = -1;
+ t.ToByteArray(bytes, ref position);
+ position = -1;
+ var tNew = new RegressionTree(bytes, ref position);
+ if (paramA != 1)
+ {
+ for (int i = 0; i < tNew.NumLeaves; i++)
+ tNew.SetOutput(i, tNew.LeafValues[i] * paramA);
+ }
+ ensemble.AddTree(tNew);
+ }
+
+ if (modelCount == 1)
+ {
+ binaryClassifier = calibrated != null;
+ featureCount = tree.InputType.ValueCount;
+ }
+ else
+ {
+ _host.Check((calibrated != null) == binaryClassifier, "Ensemble contains both calibrated and uncalibrated models");
+ _host.Check(featureCount == tree.InputType.ValueCount, "Found models with different number of features");
+ }
+ }
+
+ var scale = 1 / (double)modelCount;
+
+ foreach (var t in ensemble.Trees)
+ {
+ for (int i = 0; i < t.NumLeaves; i++)
+ t.SetOutput(i, t.LeafValues[i] * scale);
+ }
+
+ switch (_kind)
+ {
+ case PredictionKind.BinaryClassification:
+ if (!binaryClassifier)
+ return new FastTreeBinaryPredictor(_host, ensemble, featureCount, null);
+
+ var cali = new PlattCalibrator(_host, -1, 0);
+ return new FeatureWeightsCalibratedPredictor(_host, new FastTreeBinaryPredictor(_host, ensemble, featureCount, null), cali);
+ case PredictionKind.Regression:
+ return new FastTreeRegressionPredictor(_host, ensemble, featureCount, null);
+ case PredictionKind.Ranking:
+ return new FastTreeRankingPredictor(_host, ensemble, featureCount, null);
+ default:
+ _host.Assert(false);
+ throw _host.ExceptNotSupp();
+ }
+ }
+ }
+}

src/Microsoft.ML.FastTree/Utils/ToByteArrayExtensions.cs

@@ -5,6 +5,7 @@
 using System;
 using System.Linq;
 using System.Text;
+using Microsoft.ML.Runtime.Internal.Utilities;
 
 namespace Microsoft.ML.Runtime.FastTree.Internal
 {
@@ -290,7 +291,7 @@ public static string ToString(this byte[] buffer, ref int position)
 
  public static int SizeInBytes(this byte[] a)
  {
- return sizeof(int) + a.Length * sizeof(byte);
+ return sizeof(int) + Utils.Size(a) * sizeof(byte);
  }
 
  public static void ToByteArray(this byte[] a, byte[] buffer, ref int position)
@@ -314,7 +315,7 @@ public static byte[] ToByteArray(this byte[] buffer, ref int position)
 
  public static int SizeInBytes(this short[] a)
  {
- return sizeof(int) + a.Length * sizeof(short);
+ return sizeof(int) + Utils.Size(a) * sizeof(short);
  }
 
  public unsafe static void ToByteArray(this short[] a, byte[] buffer, ref int position)
@@ -353,7 +354,7 @@ public unsafe static short[] ToShortArray(this byte[] buffer, ref int position)
 
  public static int SizeInBytes(this ushort[] a)
  {
- return sizeof(int) + a.Length * sizeof(ushort);
+ return sizeof(int) + Utils.Size(a) * sizeof(ushort);
  }
 
  public unsafe static void ToByteArray(this ushort[] a, byte[] buffer, ref int position)
@@ -392,12 +393,12 @@ public unsafe static ushort[] ToUShortArray(this byte[] buffer, ref int position
 
  public static int SizeInBytes(this int[] array)
  {
- return sizeof(int) + array.Length * sizeof(int);
+ return sizeof(int) + Utils.Size(array) * sizeof(int);
  }
 
  public unsafe static void ToByteArray(this int[] a, byte[] buffer, ref int position)
  {
- int length = a.Length;
+ int length = Utils.Size(a);
  length.ToByteArray(buffer, ref position);
 
  fixed (byte* tmpBuffer = buffer)
@@ -415,6 +416,9 @@ public unsafe static int[] ToIntArray(this byte[] buffer, ref int position)
 
  public unsafe static int[] ToIntArray(this byte[] buffer, ref int position, int length)
  {
+ if (length == 0)
+ return null;
+
  int[] a = new int[length];
 
  fixed (byte* tmpBuffer = buffer)
@@ -433,7 +437,7 @@ public unsafe static int[] ToIntArray(this byte[] buffer, ref int position, int
 
  public static int SizeInBytes(this uint[] array)
  {
- return sizeof(int) + array.Length * sizeof(uint);
+ return sizeof(int) + Utils.Size(array) * sizeof(uint);
  }
 
  public unsafe static void ToByteArray(this uint[] a, byte[] buffer, ref int position)
@@ -472,7 +476,7 @@ public unsafe static uint[] ToUIntArray(this byte[] buffer, ref int position)
 
  public static int SizeInBytes(this long[] array)
  {
- return sizeof(int) + array.Length * sizeof(long);
+ return sizeof(int) + Utils.Size(array) * sizeof(long);
  }
 
  public unsafe static void ToByteArray(this long[] a, byte[] buffer, ref int position)
@@ -511,7 +515,7 @@ public unsafe static long[] ToLongArray(this byte[] buffer, ref int position)
 
  public static int SizeInBytes(this ulong[] array)
  {
- return sizeof(int) + array.Length * sizeof(ulong);
+ return sizeof(int) + Utils.Size(array) * sizeof(ulong);
  }
 
  public unsafe static void ToByteArray(this ulong[] a, byte[] buffer, ref int position)
@@ -550,7 +554,7 @@ public unsafe static ulong[] ToULongArray(this byte[] buffer, ref int position)
 
  public static int SizeInBytes(this MD5Hash[] array)
  {
- return sizeof(int) + array.Length * MD5Hash.SizeInBytes();
+ return sizeof(int) + Utils.Size(array) * MD5Hash.SizeInBytes();
  }
 
  public static void ToByteArray(this MD5Hash[] a, byte[] buffer, ref int position)
@@ -577,7 +581,7 @@ public unsafe static MD5Hash[] ToUInt128Array(this byte[] buffer, ref int positi
 
  public static int SizeInBytes(this float[] array)
  {
- return sizeof(int) + array.Length * sizeof(float);
+ return sizeof(int) + Utils.Size(array) * sizeof(float);
  }
 
  public unsafe static void ToByteArray(this float[] a, byte[] buffer, ref int position)
@@ -616,7 +620,7 @@ public unsafe static float[] ToFloatArray(this byte[] buffer, ref int position)
 
  public static int SizeInBytes(this double[] array)
  {
- return sizeof(int) + array.Length * sizeof(double);
+ return sizeof(int) + Utils.Size(array) * sizeof(double);
  }
 
  public unsafe static void ToByteArray(this double[] a, byte[] buffer, ref int position)
@@ -655,6 +659,8 @@ public unsafe static double[] ToDoubleArray(this byte[] buffer, ref int position
 
  public static int SizeInBytes(this double[][] array)
  {
+ if (Utils.Size(array) == 0)
+ return sizeof(int);
  return sizeof(int) + array.Sum(x => x.SizeInBytes());
  }
 
@@ -683,7 +689,7 @@ public static double[][] ToDoubleJaggedArray(this byte[] buffer, ref int positio
  public static long SizeInBytes(this string[] array)
  {
  long length = sizeof(int);
- for (int i = 0; i < array.Length; ++i)
+ for (int i = 0; i < Utils.Size(array); ++i)
  {
  length += array[i].SizeInBytes();
  }
@@ -692,8 +698,8 @@ public static long SizeInBytes(this string[] array)
 
  public static void ToByteArray(this string[] a, byte[] buffer, ref int position)
  {
-a.Length.ToByteArray(buffer, ref position);
- for (int i = 0; i < a.Length; ++i)
+Utils.Size(a).ToByteArray(buffer, ref position);
+ for (int i = 0; i < Utils.Size(a); ++i)
  {
  a[i].ToByteArray(buffer, ref position);
  }

src/Microsoft.ML.Core/Prediction/ITrainer.cs

@@ -74,6 +74,8 @@ public interface IModelCombiner<TModel, TPredictor>
  TPredictor CombineModels(IEnumerable<TModel> models);
  }
 
+ public delegate void SignatureModelCombiner(PredictionKind kind);
+
  /// <summary>
  /// Weakly typed interface for a trainer "session" that produces a predictor.
  /// </summary>

src/Microsoft.ML.FastTree/Microsoft.ML.FastTree.csproj

@@ -57,6 +57,7 @@
  <Compile Include="TreeEnsemble\Ensemble.cs" />
  <Compile Include="TreeEnsemble\QuantileRegressionTree.cs" />
  <Compile Include="TreeEnsemble\RegressionTree.cs" />
+ <Compile Include="TreeEnsemble\TreeEnsembleCombiner.cs" />
  <Compile Include="Training\Applications\GradientWrappers.cs" />
  <Compile Include="Training\Applications\ObjectiveFunction.cs" />
  <Compile Include="Training\BaggingProvider.cs" />

src/Microsoft.ML.FastTree/TreeEnsemble/RegressionTree.cs

@@ -105,22 +105,21 @@ public RegressionTree(byte[] buffer, ref int position)
  LteChild = buffer.ToIntArray(ref position);
  GtChild = buffer.ToIntArray(ref position);
  SplitFeatures = buffer.ToIntArray(ref position);
-int[] categoricalNodeIndices = buffer.ToIntArray(ref position);
- CategoricalSplit = GetCategoricalSplitFromIndices(categoricalNodeIndices);
- if (categoricalNodeIndices?.Length > 0)
+byte[] categoricalSplitAsBytes = buffer.ToByteArray(ref position);
+ CategoricalSplit = categoricalSplitAsBytes.Select(b => b > 0).ToArray();
+ if (CategoricalSplit.Any(b => b))
  {
  CategoricalSplitFeatures = new int[NumNodes][];
  CategoricalSplitFeatureRanges = new int[NumNodes][];
-foreach (var index in categoricalNodeIndices)
+for (int index = 0; index < NumNodes; index++)
  {
- Contracts.Assert(CategoricalSplit[index]);
-
  CategoricalSplitFeatures[index] = buffer.ToIntArray(ref position);
- CategoricalSplitFeatureRanges[index] = buffer.ToIntArray(ref position, 2);
+ CategoricalSplitFeatureRanges[index] = buffer.ToIntArray(ref position);
  }
  }
 
  Thresholds = buffer.ToUIntArray(ref position);
+ RawThresholds = buffer.ToFloatArray(ref position);
  _splitGain = buffer.ToDoubleArray(ref position);
  _gainPValue = buffer.ToDoubleArray(ref position);
  _previousLeafValue = buffer.ToDoubleArray(ref position);
@@ -144,6 +143,23 @@ private bool[] GetCategoricalSplitFromIndices(int[] indices)
  return categoricalSplit;
  }
 
+ private bool[] GetCategoricalSplitFromBytes(byte[] indices)
+ {
+ bool[] categoricalSplit = new bool[NumNodes];
+ if (indices == null)
+ return categoricalSplit;
+
+ Contracts.Assert(indices.Length <= NumNodes);
+
+ foreach (int index in indices)
+ {
+ Contracts.Assert(index >= 0 && index < NumNodes);
+ categoricalSplit[index] = true;
+ }
+
+ return categoricalSplit;
+ }
+
  /// <summary>
  /// Create a Regression Tree object from raw tree contents.
  /// </summary>
@@ -192,7 +208,7 @@ internal RegressionTree(int[] splitFeatures, Double[] splitGain, Double[] gainPV
  LeafValues = leafValues;
  CategoricalSplitFeatures = categoricalSplitFeatures;
  CategoricalSplitFeatureRanges = new int[CategoricalSplitFeatures.Length][];
- for(int i= 0; i < CategoricalSplitFeatures.Length; ++i)
+ for(int i= 0; i < CategoricalSplitFeatures.Length; ++i)
  {
  if (CategoricalSplitFeatures[i] != null && CategoricalSplitFeatures[i].Length > 0)
  {
@@ -500,6 +516,7 @@ public virtual int SizeInBytes()
  NumNodes * sizeof(int) +
  CategoricalSplit.Length * sizeof(bool) +
  Thresholds.SizeInBytes() +
+ RawThresholds.SizeInBytes() +
  _splitGain.SizeInBytes() +
  _gainPValue.SizeInBytes() +
  _previousLeafValue.SizeInBytes() +
@@ -514,22 +531,22 @@ public virtual void ToByteArray(byte[] buffer, ref int position)
  LteChild.ToByteArray(buffer, ref position);
  GtChild.ToByteArray(buffer, ref position);
  SplitFeatures.ToByteArray(buffer, ref position);
+ CategoricalSplit.Length.ToByteArray(buffer, ref position);
  foreach (var split in CategoricalSplit)
  Convert.ToByte(split).ToByteArray(buffer, ref position);
 
  if (CategoricalSplitFeatures != null)
  {
- foreach (var splits in CategoricalSplitFeatures)
- splits.ToByteArray(buffer, ref position);
- }
-
- if (CategoricalSplitFeatureRanges != null)
- {
- foreach (var ranges in CategoricalSplitFeatureRanges)
- ranges.ToByteArray(buffer, ref position);
+ Contracts.AssertValue(CategoricalSplitFeatureRanges);
+ for (int i = 0; i < CategoricalSplitFeatures.Length; i++)
+ {
+ CategoricalSplitFeatures[i].ToByteArray(buffer, ref position);
+ CategoricalSplitFeatureRanges[i].ToByteArray(buffer, ref position);
+ }
  }
 
  Thresholds.ToByteArray(buffer, ref position);
+ RawThresholds.ToByteArray(buffer, ref position);
  _splitGain.ToByteArray(buffer, ref position);
  _gainPValue.ToByteArray(buffer, ref position);
  _previousLeafValue.ToByteArray(buffer, ref position);

src/Microsoft.ML.FastTree/TreeEnsemble/TreeEnsembleCombiner.cs

@@ -0,0 +1,115 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System.Collections.Generic;
+using Microsoft.ML.Runtime;
+using Microsoft.ML.Runtime.FastTree.Internal;
+using Microsoft.ML.Runtime.Internal.Calibration;
+
+[assembly: LoadableClass(typeof(TreeEnsembleCombiner), null, typeof(SignatureModelCombiner), "Fast Tree Model Combiner", "FastTreeCombiner")]
+
+namespace Microsoft.ML.Runtime.FastTree.Internal
+{
+ public sealed class TreeEnsembleCombiner : IModelCombiner<IPredictorProducing<float>, IPredictorProducing<float>>
+ {
+ private readonly IHost _host;
+ private readonly PredictionKind _kind;
+
+ public TreeEnsembleCombiner(IHostEnvironment env, PredictionKind kind)
+ {
+ _host = env.Register("TreeEnsembleCombiner");
+ switch (kind)
+ {
+ case PredictionKind.BinaryClassification:
+ case PredictionKind.Regression:
+ case PredictionKind.Ranking:
+ _kind = kind;
+ break;
+ default:
+ throw _host.ExceptUserArg(nameof(kind), $"Tree ensembles can be either of type {nameof(PredictionKind.BinaryClassification)}, " +
+ $"{nameof(PredictionKind.Regression)} or {nameof(PredictionKind.Ranking)}");
+ }
+ }
+
+ public IPredictorProducing<float> CombineModels(IEnumerable<IPredictorProducing<float>> models)
+ {
+ _host.CheckValue(models, nameof(models));
+
+ var ensemble = new Ensemble();
+ int modelCount = 0;
+ int featureCount = -1;
+ bool binaryClassifier = false;
+ foreach (var model in models)
+ {
+ modelCount++;
+
+ var predictor = model;
+ _host.CheckValue(predictor, nameof(models), "One of the models is null");
+
+ var calibrated = predictor as CalibratedPredictorBase;
+ double paramA = 1;
+ if (calibrated != null)
+ {
+ _host.Check(calibrated.Calibrator is PlattCalibrator,
+ "Combining FastTree models can only be done when the models are calibrated with Platt calibrator");
+ predictor = calibrated.SubPredictor;
+ paramA = -(calibrated.Calibrator as PlattCalibrator).ParamA;
+ }
+ var tree = predictor as FastTreePredictionWrapper;
+ if (tree == null)
+ throw _host.Except("Model is not a tree ensemble");
+ foreach (var t in tree.TrainedEnsemble.Trees)
+ {
+ var bytes = new byte[t.SizeInBytes()];
+ int position = -1;
+ t.ToByteArray(bytes, ref position);
+ position = -1;
+ var tNew = new RegressionTree(bytes, ref position);
+ if (paramA != 1)
+ {
+ for (int i = 0; i < tNew.NumLeaves; i++)
+ tNew.SetOutput(i, tNew.LeafValues[i] * paramA);
+ }
+ ensemble.AddTree(tNew);
+ }
+
+ if (modelCount == 1)
+ {
+ binaryClassifier = calibrated != null;
+ featureCount = tree.InputType.ValueCount;
+ }
+ else
+ {
+ _host.Check((calibrated != null) == binaryClassifier, "Ensemble contains both calibrated and uncalibrated models");
+ _host.Check(featureCount == tree.InputType.ValueCount, "Found models with different number of features");
+ }
+ }
+
+ var scale = 1 / (double)modelCount;
+
+ foreach (var t in ensemble.Trees)
+ {
+ for (int i = 0; i < t.NumLeaves; i++)
+ t.SetOutput(i, t.LeafValues[i] * scale);
+ }
+
+ switch (_kind)
+ {
+ case PredictionKind.BinaryClassification:
+ if (!binaryClassifier)
+ return new FastTreeBinaryPredictor(_host, ensemble, featureCount, null);
+
+ var cali = new PlattCalibrator(_host, -1, 0);
+ return new FeatureWeightsCalibratedPredictor(_host, new FastTreeBinaryPredictor(_host, ensemble, featureCount, null), cali);
+ case PredictionKind.Regression:
+ return new FastTreeRegressionPredictor(_host, ensemble, featureCount, null);
+ case PredictionKind.Ranking:
+ return new FastTreeRankingPredictor(_host, ensemble, featureCount, null);
+ default:
+ _host.Assert(false);
+ throw _host.ExceptNotSupp();
+ }
+ }
+ }
+}

src/Microsoft.ML.FastTree/Utils/ToByteArrayExtensions.cs

@@ -5,6 +5,7 @@
 using System;
 using System.Linq;
 using System.Text;
+using Microsoft.ML.Runtime.Internal.Utilities;
 
 namespace Microsoft.ML.Runtime.FastTree.Internal
 {
@@ -290,7 +291,7 @@ public static string ToString(this byte[] buffer, ref int position)
 
  public static int SizeInBytes(this byte[] a)
  {
- return sizeof(int) + a.Length * sizeof(byte);
+ return sizeof(int) + Utils.Size(a) * sizeof(byte);
  }
 
  public static void ToByteArray(this byte[] a, byte[] buffer, ref int position)
@@ -314,7 +315,7 @@ public static byte[] ToByteArray(this byte[] buffer, ref int position)
 
  public static int SizeInBytes(this short[] a)
  {
- return sizeof(int) + a.Length * sizeof(short);
+ return sizeof(int) + Utils.Size(a) * sizeof(short);
  }
 
  public unsafe static void ToByteArray(this short[] a, byte[] buffer, ref int position)
@@ -353,7 +354,7 @@ public unsafe static short[] ToShortArray(this byte[] buffer, ref int position)
 
  public static int SizeInBytes(this ushort[] a)
  {
- return sizeof(int) + a.Length * sizeof(ushort);
+ return sizeof(int) + Utils.Size(a) * sizeof(ushort);
  }
 
  public unsafe static void ToByteArray(this ushort[] a, byte[] buffer, ref int position)
@@ -392,12 +393,12 @@ public unsafe static ushort[] ToUShortArray(this byte[] buffer, ref int position
 
  public static int SizeInBytes(this int[] array)
  {
- return sizeof(int) + array.Length * sizeof(int);
+ return sizeof(int) + Utils.Size(array) * sizeof(int);
  }
 
  public unsafe static void ToByteArray(this int[] a, byte[] buffer, ref int position)
  {
- int length = a.Length;
+ int length = Utils.Size(a);
  length.ToByteArray(buffer, ref position);
 
  fixed (byte* tmpBuffer = buffer)
@@ -415,6 +416,9 @@ public unsafe static int[] ToIntArray(this byte[] buffer, ref int position)
 
  public unsafe static int[] ToIntArray(this byte[] buffer, ref int position, int length)
  {
+ if (length == 0)
+ return null;
+
  int[] a = new int[length];
 
  fixed (byte* tmpBuffer = buffer)
@@ -433,7 +437,7 @@ public unsafe static int[] ToIntArray(this byte[] buffer, ref int position, int
 
  public static int SizeInBytes(this uint[] array)
  {
- return sizeof(int) + array.Length * sizeof(uint);
+ return sizeof(int) + Utils.Size(array) * sizeof(uint);
  }
 
  public unsafe static void ToByteArray(this uint[] a, byte[] buffer, ref int position)
@@ -472,7 +476,7 @@ public unsafe static uint[] ToUIntArray(this byte[] buffer, ref int position)
 
  public static int SizeInBytes(this long[] array)
  {
- return sizeof(int) + array.Length * sizeof(long);
+ return sizeof(int) + Utils.Size(array) * sizeof(long);
  }
 
  public unsafe static void ToByteArray(this long[] a, byte[] buffer, ref int position)
@@ -511,7 +515,7 @@ public unsafe static long[] ToLongArray(this byte[] buffer, ref int position)
 
  public static int SizeInBytes(this ulong[] array)
  {
- return sizeof(int) + array.Length * sizeof(ulong);
+ return sizeof(int) + Utils.Size(array) * sizeof(ulong);
  }
 
  public unsafe static void ToByteArray(this ulong[] a, byte[] buffer, ref int position)
@@ -550,7 +554,7 @@ public unsafe static ulong[] ToULongArray(this byte[] buffer, ref int position)
 
  public static int SizeInBytes(this MD5Hash[] array)
  {
- return sizeof(int) + array.Length * MD5Hash.SizeInBytes();
+ return sizeof(int) + Utils.Size(array) * MD5Hash.SizeInBytes();
  }
 
  public static void ToByteArray(this MD5Hash[] a, byte[] buffer, ref int position)
@@ -577,7 +581,7 @@ public unsafe static MD5Hash[] ToUInt128Array(this byte[] buffer, ref int positi
 
  public static int SizeInBytes(this float[] array)
  {
- return sizeof(int) + array.Length * sizeof(float);
+ return sizeof(int) + Utils.Size(array) * sizeof(float);
  }
 
  public unsafe static void ToByteArray(this float[] a, byte[] buffer, ref int position)
@@ -616,7 +620,7 @@ public unsafe static float[] ToFloatArray(this byte[] buffer, ref int position)
 
  public static int SizeInBytes(this double[] array)
  {
- return sizeof(int) + array.Length * sizeof(double);
+ return sizeof(int) + Utils.Size(array) * sizeof(double);
  }
 
  public unsafe static void ToByteArray(this double[] a, byte[] buffer, ref int position)
@@ -655,6 +659,8 @@ public unsafe static double[] ToDoubleArray(this byte[] buffer, ref int position
 
  public static int SizeInBytes(this double[][] array)
  {
+ if (Utils.Size(array) == 0)
+ return sizeof(int);
  return sizeof(int) + array.Sum(x => x.SizeInBytes());
  }
 
@@ -683,7 +689,7 @@ public static double[][] ToDoubleJaggedArray(this byte[] buffer, ref int positio
  public static long SizeInBytes(this string[] array)
  {
  long length = sizeof(int);
- for (int i = 0; i < array.Length; ++i)
+ for (int i = 0; i < Utils.Size(array); ++i)
  {
  length += array[i].SizeInBytes();
  }
@@ -692,8 +698,8 @@ public static long SizeInBytes(this string[] array)
 
  public static void ToByteArray(this string[] a, byte[] buffer, ref int position)
  {
-a.Length.ToByteArray(buffer, ref position);
- for (int i = 0; i < a.Length; ++i)
+Utils.Size(a).ToByteArray(buffer, ref position);
+ for (int i = 0; i < Utils.Size(a); ++i)
  {
  a[i].ToByteArray(buffer, ref position);
  }

src/Microsoft.ML.Core/Prediction/ITrainer.cs

@@ -74,6 +74,8 @@ public interface IModelCombiner<TModel, TPredictor>
  TPredictor CombineModels(IEnumerable<TModel> models);
  }
 
+ public delegate void SignatureModelCombiner(PredictionKind kind);
+
  /// <summary>
  /// Weakly typed interface for a trainer "session" that produces a predictor.
  /// </summary>

src/Microsoft.ML.FastTree/Microsoft.ML.FastTree.csproj

@@ -57,6 +57,7 @@
  <Compile Include="TreeEnsemble\Ensemble.cs" />
  <Compile Include="TreeEnsemble\QuantileRegressionTree.cs" />
  <Compile Include="TreeEnsemble\RegressionTree.cs" />
+ <Compile Include="TreeEnsemble\TreeEnsembleCombiner.cs" />
  <Compile Include="Training\Applications\GradientWrappers.cs" />
  <Compile Include="Training\Applications\ObjectiveFunction.cs" />
  <Compile Include="Training\BaggingProvider.cs" />

src/Microsoft.ML.FastTree/TreeEnsemble/RegressionTree.cs

@@ -105,22 +105,21 @@ public RegressionTree(byte[] buffer, ref int position)
  LteChild = buffer.ToIntArray(ref position);
  GtChild = buffer.ToIntArray(ref position);
  SplitFeatures = buffer.ToIntArray(ref position);
-int[] categoricalNodeIndices = buffer.ToIntArray(ref position);
- CategoricalSplit = GetCategoricalSplitFromIndices(categoricalNodeIndices);
- if (categoricalNodeIndices?.Length > 0)
+byte[] categoricalSplitAsBytes = buffer.ToByteArray(ref position);
+ CategoricalSplit = categoricalSplitAsBytes.Select(b => b > 0).ToArray();
+ if (CategoricalSplit.Any(b => b))
  {
  CategoricalSplitFeatures = new int[NumNodes][];
  CategoricalSplitFeatureRanges = new int[NumNodes][];
-foreach (var index in categoricalNodeIndices)
+for (int index = 0; index < NumNodes; index++)
  {
- Contracts.Assert(CategoricalSplit[index]);
-
  CategoricalSplitFeatures[index] = buffer.ToIntArray(ref position);
- CategoricalSplitFeatureRanges[index] = buffer.ToIntArray(ref position, 2);
+ CategoricalSplitFeatureRanges[index] = buffer.ToIntArray(ref position);
  }
  }
 
  Thresholds = buffer.ToUIntArray(ref position);
+ RawThresholds = buffer.ToFloatArray(ref position);
  _splitGain = buffer.ToDoubleArray(ref position);
  _gainPValue = buffer.ToDoubleArray(ref position);
  _previousLeafValue = buffer.ToDoubleArray(ref position);
@@ -144,6 +143,23 @@ private bool[] GetCategoricalSplitFromIndices(int[] indices)
  return categoricalSplit;
  }
 
+ private bool[] GetCategoricalSplitFromBytes(byte[] indices)
+ {
+ bool[] categoricalSplit = new bool[NumNodes];
+ if (indices == null)
+ return categoricalSplit;
+
+ Contracts.Assert(indices.Length <= NumNodes);
+
+ foreach (int index in indices)
+ {
+ Contracts.Assert(index >= 0 && index < NumNodes);
+ categoricalSplit[index] = true;
+ }
+
+ return categoricalSplit;
+ }
+
  /// <summary>
  /// Create a Regression Tree object from raw tree contents.
  /// </summary>
@@ -192,7 +208,7 @@ internal RegressionTree(int[] splitFeatures, Double[] splitGain, Double[] gainPV
  LeafValues = leafValues;
  CategoricalSplitFeatures = categoricalSplitFeatures;
  CategoricalSplitFeatureRanges = new int[CategoricalSplitFeatures.Length][];
- for(int i= 0; i < CategoricalSplitFeatures.Length; ++i)
+ for(int i= 0; i < CategoricalSplitFeatures.Length; ++i)
  {
  if (CategoricalSplitFeatures[i] != null && CategoricalSplitFeatures[i].Length > 0)
  {
@@ -500,6 +516,7 @@ public virtual int SizeInBytes()
  NumNodes * sizeof(int) +
  CategoricalSplit.Length * sizeof(bool) +
  Thresholds.SizeInBytes() +
+ RawThresholds.SizeInBytes() +
  _splitGain.SizeInBytes() +
  _gainPValue.SizeInBytes() +
  _previousLeafValue.SizeInBytes() +
@@ -514,22 +531,22 @@ public virtual void ToByteArray(byte[] buffer, ref int position)
  LteChild.ToByteArray(buffer, ref position);
  GtChild.ToByteArray(buffer, ref position);
  SplitFeatures.ToByteArray(buffer, ref position);
+ CategoricalSplit.Length.ToByteArray(buffer, ref position);
  foreach (var split in CategoricalSplit)
  Convert.ToByte(split).ToByteArray(buffer, ref position);
 
  if (CategoricalSplitFeatures != null)
  {
- foreach (var splits in CategoricalSplitFeatures)
- splits.ToByteArray(buffer, ref position);
- }
-
- if (CategoricalSplitFeatureRanges != null)
- {
- foreach (var ranges in CategoricalSplitFeatureRanges)
- ranges.ToByteArray(buffer, ref position);
+ Contracts.AssertValue(CategoricalSplitFeatureRanges);
+ for (int i = 0; i < CategoricalSplitFeatures.Length; i++)
+ {
+ CategoricalSplitFeatures[i].ToByteArray(buffer, ref position);
+ CategoricalSplitFeatureRanges[i].ToByteArray(buffer, ref position);
+ }
  }
 
  Thresholds.ToByteArray(buffer, ref position);
+ RawThresholds.ToByteArray(buffer, ref position);
  _splitGain.ToByteArray(buffer, ref position);
  _gainPValue.ToByteArray(buffer, ref position);
  _previousLeafValue.ToByteArray(buffer, ref position);

src/Microsoft.ML.FastTree/TreeEnsemble/TreeEnsembleCombiner.cs

@@ -0,0 +1,115 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System.Collections.Generic;
+using Microsoft.ML.Runtime;
+using Microsoft.ML.Runtime.FastTree.Internal;
+using Microsoft.ML.Runtime.Internal.Calibration;
+
+[assembly: LoadableClass(typeof(TreeEnsembleCombiner), null, typeof(SignatureModelCombiner), "Fast Tree Model Combiner", "FastTreeCombiner")]
+
+namespace Microsoft.ML.Runtime.FastTree.Internal
+{
+ public sealed class TreeEnsembleCombiner : IModelCombiner<IPredictorProducing<float>, IPredictorProducing<float>>
+ {
+ private readonly IHost _host;
+ private readonly PredictionKind _kind;
+
+ public TreeEnsembleCombiner(IHostEnvironment env, PredictionKind kind)
+ {
+ _host = env.Register("TreeEnsembleCombiner");
+ switch (kind)
+ {
+ case PredictionKind.BinaryClassification:
+ case PredictionKind.Regression:
+ case PredictionKind.Ranking:
+ _kind = kind;
+ break;
+ default:
+ throw _host.ExceptUserArg(nameof(kind), $"Tree ensembles can be either of type {nameof(PredictionKind.BinaryClassification)}, " +
+ $"{nameof(PredictionKind.Regression)} or {nameof(PredictionKind.Ranking)}");
+ }
+ }
+
+ public IPredictorProducing<float> CombineModels(IEnumerable<IPredictorProducing<float>> models)
+ {
+ _host.CheckValue(models, nameof(models));
+
+ var ensemble = new Ensemble();
+ int modelCount = 0;
+ int featureCount = -1;
+ bool binaryClassifier = false;
+ foreach (var model in models)
+ {
+ modelCount++;
+
+ var predictor = model;
+ _host.CheckValue(predictor, nameof(models), "One of the models is null");
+
+ var calibrated = predictor as CalibratedPredictorBase;
+ double paramA = 1;
+ if (calibrated != null)
+ {
+ _host.Check(calibrated.Calibrator is PlattCalibrator,
+ "Combining FastTree models can only be done when the models are calibrated with Platt calibrator");
+ predictor = calibrated.SubPredictor;
+ paramA = -(calibrated.Calibrator as PlattCalibrator).ParamA;
+ }
+ var tree = predictor as FastTreePredictionWrapper;
+ if (tree == null)
+ throw _host.Except("Model is not a tree ensemble");
+ foreach (var t in tree.TrainedEnsemble.Trees)
+ {
+ var bytes = new byte[t.SizeInBytes()];
+ int position = -1;
+ t.ToByteArray(bytes, ref position);
+ position = -1;
+ var tNew = new RegressionTree(bytes, ref position);
+ if (paramA != 1)
+ {
+ for (int i = 0; i < tNew.NumLeaves; i++)
+ tNew.SetOutput(i, tNew.LeafValues[i] * paramA);
+ }
+ ensemble.AddTree(tNew);
+ }
+
+ if (modelCount == 1)
+ {
+ binaryClassifier = calibrated != null;
+ featureCount = tree.InputType.ValueCount;
+ }
+ else
+ {
+ _host.Check((calibrated != null) == binaryClassifier, "Ensemble contains both calibrated and uncalibrated models");
+ _host.Check(featureCount == tree.InputType.ValueCount, "Found models with different number of features");
+ }
+ }
+
+ var scale = 1 / (double)modelCount;
+
+ foreach (var t in ensemble.Trees)
+ {
+ for (int i = 0; i < t.NumLeaves; i++)
+ t.SetOutput(i, t.LeafValues[i] * scale);
+ }
+
+ switch (_kind)
+ {
+ case PredictionKind.BinaryClassification:
+ if (!binaryClassifier)
+ return new FastTreeBinaryPredictor(_host, ensemble, featureCount, null);
+
+ var cali = new PlattCalibrator(_host, -1, 0);
+ return new FeatureWeightsCalibratedPredictor(_host, new FastTreeBinaryPredictor(_host, ensemble, featureCount, null), cali);
+ case PredictionKind.Regression:
+ return new FastTreeRegressionPredictor(_host, ensemble, featureCount, null);
+ case PredictionKind.Ranking:
+ return new FastTreeRankingPredictor(_host, ensemble, featureCount, null);
+ default:
+ _host.Assert(false);
+ throw _host.ExceptNotSupp();
+ }
+ }
+ }
+}

src/Microsoft.ML.FastTree/Utils/ToByteArrayExtensions.cs

@@ -5,6 +5,7 @@
 using System;
 using System.Linq;
 using System.Text;
+using Microsoft.ML.Runtime.Internal.Utilities;
 
 namespace Microsoft.ML.Runtime.FastTree.Internal
 {
@@ -290,7 +291,7 @@ public static string ToString(this byte[] buffer, ref int position)
 
  public static int SizeInBytes(this byte[] a)
  {
- return sizeof(int) + a.Length * sizeof(byte);
+ return sizeof(int) + Utils.Size(a) * sizeof(byte);
  }
 
  public static void ToByteArray(this byte[] a, byte[] buffer, ref int position)
@@ -314,7 +315,7 @@ public static byte[] ToByteArray(this byte[] buffer, ref int position)
 
  public static int SizeInBytes(this short[] a)
  {
- return sizeof(int) + a.Length * sizeof(short);
+ return sizeof(int) + Utils.Size(a) * sizeof(short);
  }
 
  public unsafe static void ToByteArray(this short[] a, byte[] buffer, ref int position)
@@ -353,7 +354,7 @@ public unsafe static short[] ToShortArray(this byte[] buffer, ref int position)
 
  public static int SizeInBytes(this ushort[] a)
  {
- return sizeof(int) + a.Length * sizeof(ushort);
+ return sizeof(int) + Utils.Size(a) * sizeof(ushort);
  }
 
  public unsafe static void ToByteArray(this ushort[] a, byte[] buffer, ref int position)
@@ -392,12 +393,12 @@ public unsafe static ushort[] ToUShortArray(this byte[] buffer, ref int position
 
  public static int SizeInBytes(this int[] array)
  {
- return sizeof(int) + array.Length * sizeof(int);
+ return sizeof(int) + Utils.Size(array) * sizeof(int);
  }
 
  public unsafe static void ToByteArray(this int[] a, byte[] buffer, ref int position)
  {
- int length = a.Length;
+ int length = Utils.Size(a);
  length.ToByteArray(buffer, ref position);
 
  fixed (byte* tmpBuffer = buffer)
@@ -415,6 +416,9 @@ public unsafe static int[] ToIntArray(this byte[] buffer, ref int position)
 
  public unsafe static int[] ToIntArray(this byte[] buffer, ref int position, int length)
  {
+ if (length == 0)
+ return null;
+
  int[] a = new int[length];
 
  fixed (byte* tmpBuffer = buffer)
@@ -433,7 +437,7 @@ public unsafe static int[] ToIntArray(this byte[] buffer, ref int position, int
 
  public static int SizeInBytes(this uint[] array)
  {
- return sizeof(int) + array.Length * sizeof(uint);
+ return sizeof(int) + Utils.Size(array) * sizeof(uint);
  }
 
  public unsafe static void ToByteArray(this uint[] a, byte[] buffer, ref int position)
@@ -472,7 +476,7 @@ public unsafe static uint[] ToUIntArray(this byte[] buffer, ref int position)
 
  public static int SizeInBytes(this long[] array)
  {
- return sizeof(int) + array.Length * sizeof(long);
+ return sizeof(int) + Utils.Size(array) * sizeof(long);
  }
 
  public unsafe static void ToByteArray(this long[] a, byte[] buffer, ref int position)
@@ -511,7 +515,7 @@ public unsafe static long[] ToLongArray(this byte[] buffer, ref int position)
 
  public static int SizeInBytes(this ulong[] array)
  {
- return sizeof(int) + array.Length * sizeof(ulong);
+ return sizeof(int) + Utils.Size(array) * sizeof(ulong);
  }
 
  public unsafe static void ToByteArray(this ulong[] a, byte[] buffer, ref int position)
@@ -550,7 +554,7 @@ public unsafe static ulong[] ToULongArray(this byte[] buffer, ref int position)
 
  public static int SizeInBytes(this MD5Hash[] array)
  {
- return sizeof(int) + array.Length * MD5Hash.SizeInBytes();
+ return sizeof(int) + Utils.Size(array) * MD5Hash.SizeInBytes();
  }
 
  public static void ToByteArray(this MD5Hash[] a, byte[] buffer, ref int position)
@@ -577,7 +581,7 @@ public unsafe static MD5Hash[] ToUInt128Array(this byte[] buffer, ref int positi
 
  public static int SizeInBytes(this float[] array)
  {
- return sizeof(int) + array.Length * sizeof(float);
+ return sizeof(int) + Utils.Size(array) * sizeof(float);
  }
 
  public unsafe static void ToByteArray(this float[] a, byte[] buffer, ref int position)
@@ -616,7 +620,7 @@ public unsafe static float[] ToFloatArray(this byte[] buffer, ref int position)
 
  public static int SizeInBytes(this double[] array)
  {
- return sizeof(int) + array.Length * sizeof(double);
+ return sizeof(int) + Utils.Size(array) * sizeof(double);
  }
 
  public unsafe static void ToByteArray(this double[] a, byte[] buffer, ref int position)
@@ -655,6 +659,8 @@ public unsafe static double[] ToDoubleArray(this byte[] buffer, ref int position
 
  public static int SizeInBytes(this double[][] array)
  {
+ if (Utils.Size(array) == 0)
+ return sizeof(int);
  return sizeof(int) + array.Sum(x => x.SizeInBytes());
  }
 
@@ -683,7 +689,7 @@ public static double[][] ToDoubleJaggedArray(this byte[] buffer, ref int positio
  public static long SizeInBytes(this string[] array)
  {
  long length = sizeof(int);
- for (int i = 0; i < array.Length; ++i)
+ for (int i = 0; i < Utils.Size(array); ++i)
  {
  length += array[i].SizeInBytes();
  }
@@ -692,8 +698,8 @@ public static long SizeInBytes(this string[] array)
 
  public static void ToByteArray(this string[] a, byte[] buffer, ref int position)
  {
-a.Length.ToByteArray(buffer, ref position);
- for (int i = 0; i < a.Length; ++i)
+Utils.Size(a).ToByteArray(buffer, ref position);
+ for (int i = 0; i < Utils.Size(a); ++i)
  {
  a[i].ToByteArray(buffer, ref position);
  }

src/Microsoft.ML.Core/Prediction/ITrainer.cs

@@ -74,6 +74,8 @@ public interface IModelCombiner<TModel, TPredictor>
  TPredictor CombineModels(IEnumerable<TModel> models);
  }
 
+ public delegate void SignatureModelCombiner(PredictionKind kind);
+
  /// <summary>
  /// Weakly typed interface for a trainer "session" that produces a predictor.
  /// </summary>

src/Microsoft.ML.FastTree/Microsoft.ML.FastTree.csproj

@@ -57,6 +57,7 @@
  <Compile Include="TreeEnsemble\Ensemble.cs" />
  <Compile Include="TreeEnsemble\QuantileRegressionTree.cs" />
  <Compile Include="TreeEnsemble\RegressionTree.cs" />
+ <Compile Include="TreeEnsemble\TreeEnsembleCombiner.cs" />
  <Compile Include="Training\Applications\GradientWrappers.cs" />
  <Compile Include="Training\Applications\ObjectiveFunction.cs" />
  <Compile Include="Training\BaggingProvider.cs" />

src/Microsoft.ML.FastTree/TreeEnsemble/RegressionTree.cs

@@ -105,22 +105,21 @@ public RegressionTree(byte[] buffer, ref int position)
  LteChild = buffer.ToIntArray(ref position);
  GtChild = buffer.ToIntArray(ref position);
  SplitFeatures = buffer.ToIntArray(ref position);
-int[] categoricalNodeIndices = buffer.ToIntArray(ref position);
- CategoricalSplit = GetCategoricalSplitFromIndices(categoricalNodeIndices);
- if (categoricalNodeIndices?.Length > 0)
+byte[] categoricalSplitAsBytes = buffer.ToByteArray(ref position);
+ CategoricalSplit = categoricalSplitAsBytes.Select(b => b > 0).ToArray();
+ if (CategoricalSplit.Any(b => b))
  {
  CategoricalSplitFeatures = new int[NumNodes][];
  CategoricalSplitFeatureRanges = new int[NumNodes][];
-foreach (var index in categoricalNodeIndices)
+for (int index = 0; index < NumNodes; index++)
  {
- Contracts.Assert(CategoricalSplit[index]);
-
  CategoricalSplitFeatures[index] = buffer.ToIntArray(ref position);
- CategoricalSplitFeatureRanges[index] = buffer.ToIntArray(ref position, 2);
+ CategoricalSplitFeatureRanges[index] = buffer.ToIntArray(ref position);
  }
  }
 
  Thresholds = buffer.ToUIntArray(ref position);
+ RawThresholds = buffer.ToFloatArray(ref position);
  _splitGain = buffer.ToDoubleArray(ref position);
  _gainPValue = buffer.ToDoubleArray(ref position);
  _previousLeafValue = buffer.ToDoubleArray(ref position);
@@ -144,6 +143,23 @@ private bool[] GetCategoricalSplitFromIndices(int[] indices)
  return categoricalSplit;
  }
 
+ private bool[] GetCategoricalSplitFromBytes(byte[] indices)
+ {
+ bool[] categoricalSplit = new bool[NumNodes];
+ if (indices == null)
+ return categoricalSplit;
+
+ Contracts.Assert(indices.Length <= NumNodes);
+
+ foreach (int index in indices)
+ {
+ Contracts.Assert(index >= 0 && index < NumNodes);
+ categoricalSplit[index] = true;
+ }
+
+ return categoricalSplit;
+ }
+
  /// <summary>
  /// Create a Regression Tree object from raw tree contents.
  /// </summary>
@@ -192,7 +208,7 @@ internal RegressionTree(int[] splitFeatures, Double[] splitGain, Double[] gainPV
  LeafValues = leafValues;
  CategoricalSplitFeatures = categoricalSplitFeatures;
  CategoricalSplitFeatureRanges = new int[CategoricalSplitFeatures.Length][];
- for(int i= 0; i < CategoricalSplitFeatures.Length; ++i)
+ for(int i= 0; i < CategoricalSplitFeatures.Length; ++i)
  {
  if (CategoricalSplitFeatures[i] != null && CategoricalSplitFeatures[i].Length > 0)
  {
@@ -500,6 +516,7 @@ public virtual int SizeInBytes()
  NumNodes * sizeof(int) +
  CategoricalSplit.Length * sizeof(bool) +
  Thresholds.SizeInBytes() +
+ RawThresholds.SizeInBytes() +
  _splitGain.SizeInBytes() +
  _gainPValue.SizeInBytes() +
  _previousLeafValue.SizeInBytes() +
@@ -514,22 +531,22 @@ public virtual void ToByteArray(byte[] buffer, ref int position)
  LteChild.ToByteArray(buffer, ref position);
  GtChild.ToByteArray(buffer, ref position);
  SplitFeatures.ToByteArray(buffer, ref position);
+ CategoricalSplit.Length.ToByteArray(buffer, ref position);
  foreach (var split in CategoricalSplit)
  Convert.ToByte(split).ToByteArray(buffer, ref position);
 
  if (CategoricalSplitFeatures != null)
  {
- foreach (var splits in CategoricalSplitFeatures)
- splits.ToByteArray(buffer, ref position);
- }
-
- if (CategoricalSplitFeatureRanges != null)
- {
- foreach (var ranges in CategoricalSplitFeatureRanges)
- ranges.ToByteArray(buffer, ref position);
+ Contracts.AssertValue(CategoricalSplitFeatureRanges);
+ for (int i = 0; i < CategoricalSplitFeatures.Length; i++)
+ {
+ CategoricalSplitFeatures[i].ToByteArray(buffer, ref position);
+ CategoricalSplitFeatureRanges[i].ToByteArray(buffer, ref position);
+ }
  }
 
  Thresholds.ToByteArray(buffer, ref position);
+ RawThresholds.ToByteArray(buffer, ref position);
  _splitGain.ToByteArray(buffer, ref position);
  _gainPValue.ToByteArray(buffer, ref position);
  _previousLeafValue.ToByteArray(buffer, ref position);

src/Microsoft.ML.FastTree/TreeEnsemble/TreeEnsembleCombiner.cs

@@ -0,0 +1,115 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System.Collections.Generic;
+using Microsoft.ML.Runtime;
+using Microsoft.ML.Runtime.FastTree.Internal;
+using Microsoft.ML.Runtime.Internal.Calibration;
+
+[assembly: LoadableClass(typeof(TreeEnsembleCombiner), null, typeof(SignatureModelCombiner), "Fast Tree Model Combiner", "FastTreeCombiner")]
+
+namespace Microsoft.ML.Runtime.FastTree.Internal
+{
+ public sealed class TreeEnsembleCombiner : IModelCombiner<IPredictorProducing<float>, IPredictorProducing<float>>
+ {
+ private readonly IHost _host;
+ private readonly PredictionKind _kind;
+
+ public TreeEnsembleCombiner(IHostEnvironment env, PredictionKind kind)
+ {
+ _host = env.Register("TreeEnsembleCombiner");
+ switch (kind)
+ {
+ case PredictionKind.BinaryClassification:
+ case PredictionKind.Regression:
+ case PredictionKind.Ranking:
+ _kind = kind;
+ break;
+ default:
+ throw _host.ExceptUserArg(nameof(kind), $"Tree ensembles can be either of type {nameof(PredictionKind.BinaryClassification)}, " +
+ $"{nameof(PredictionKind.Regression)} or {nameof(PredictionKind.Ranking)}");
+ }
+ }
+
+ public IPredictorProducing<float> CombineModels(IEnumerable<IPredictorProducing<float>> models)
+ {
+ _host.CheckValue(models, nameof(models));
+
+ var ensemble = new Ensemble();
+ int modelCount = 0;
+ int featureCount = -1;
+ bool binaryClassifier = false;
+ foreach (var model in models)
+ {
+ modelCount++;
+
+ var predictor = model;
+ _host.CheckValue(predictor, nameof(models), "One of the models is null");
+
+ var calibrated = predictor as CalibratedPredictorBase;
+ double paramA = 1;
+ if (calibrated != null)
+ {
+ _host.Check(calibrated.Calibrator is PlattCalibrator,
+ "Combining FastTree models can only be done when the models are calibrated with Platt calibrator");
+ predictor = calibrated.SubPredictor;
+ paramA = -(calibrated.Calibrator as PlattCalibrator).ParamA;
+ }
+ var tree = predictor as FastTreePredictionWrapper;
+ if (tree == null)
+ throw _host.Except("Model is not a tree ensemble");
+ foreach (var t in tree.TrainedEnsemble.Trees)
+ {
+ var bytes = new byte[t.SizeInBytes()];
+ int position = -1;
+ t.ToByteArray(bytes, ref position);
+ position = -1;
+ var tNew = new RegressionTree(bytes, ref position);
+ if (paramA != 1)
+ {
+ for (int i = 0; i < tNew.NumLeaves; i++)
+ tNew.SetOutput(i, tNew.LeafValues[i] * paramA);
+ }
+ ensemble.AddTree(tNew);
+ }
+
+ if (modelCount == 1)
+ {
+ binaryClassifier = calibrated != null;
+ featureCount = tree.InputType.ValueCount;
+ }
+ else
+ {
+ _host.Check((calibrated != null) == binaryClassifier, "Ensemble contains both calibrated and uncalibrated models");
+ _host.Check(featureCount == tree.InputType.ValueCount, "Found models with different number of features");
+ }
+ }
+
+ var scale = 1 / (double)modelCount;
+
+ foreach (var t in ensemble.Trees)
+ {
+ for (int i = 0; i < t.NumLeaves; i++)
+ t.SetOutput(i, t.LeafValues[i] * scale);
+ }
+
+ switch (_kind)
+ {
+ case PredictionKind.BinaryClassification:
+ if (!binaryClassifier)
+ return new FastTreeBinaryPredictor(_host, ensemble, featureCount, null);
+
+ var cali = new PlattCalibrator(_host, -1, 0);
+ return new FeatureWeightsCalibratedPredictor(_host, new FastTreeBinaryPredictor(_host, ensemble, featureCount, null), cali);
+ case PredictionKind.Regression:
+ return new FastTreeRegressionPredictor(_host, ensemble, featureCount, null);
+ case PredictionKind.Ranking:
+ return new FastTreeRankingPredictor(_host, ensemble, featureCount, null);
+ default:
+ _host.Assert(false);
+ throw _host.ExceptNotSupp();
+ }
+ }
+ }
+}

src/Microsoft.ML.FastTree/Utils/ToByteArrayExtensions.cs

@@ -5,6 +5,7 @@
 using System;
 using System.Linq;
 using System.Text;
+using Microsoft.ML.Runtime.Internal.Utilities;
 
 namespace Microsoft.ML.Runtime.FastTree.Internal
 {
@@ -290,7 +291,7 @@ public static string ToString(this byte[] buffer, ref int position)
 
  public static int SizeInBytes(this byte[] a)
  {
- return sizeof(int) + a.Length * sizeof(byte);
+ return sizeof(int) + Utils.Size(a) * sizeof(byte);
  }
 
  public static void ToByteArray(this byte[] a, byte[] buffer, ref int position)
@@ -314,7 +315,7 @@ public static byte[] ToByteArray(this byte[] buffer, ref int position)
 
  public static int SizeInBytes(this short[] a)
  {
- return sizeof(int) + a.Length * sizeof(short);
+ return sizeof(int) + Utils.Size(a) * sizeof(short);
  }
 
  public unsafe static void ToByteArray(this short[] a, byte[] buffer, ref int position)
@@ -353,7 +354,7 @@ public unsafe static short[] ToShortArray(this byte[] buffer, ref int position)
 
  public static int SizeInBytes(this ushort[] a)
  {
- return sizeof(int) + a.Length * sizeof(ushort);
+ return sizeof(int) + Utils.Size(a) * sizeof(ushort);
  }
 
  public unsafe static void ToByteArray(this ushort[] a, byte[] buffer, ref int position)
@@ -392,12 +393,12 @@ public unsafe static ushort[] ToUShortArray(this byte[] buffer, ref int position
 
  public static int SizeInBytes(this int[] array)
  {
- return sizeof(int) + array.Length * sizeof(int);
+ return sizeof(int) + Utils.Size(array) * sizeof(int);
  }
 
  public unsafe static void ToByteArray(this int[] a, byte[] buffer, ref int position)
  {
- int length = a.Length;
+ int length = Utils.Size(a);
  length.ToByteArray(buffer, ref position);
 
  fixed (byte* tmpBuffer = buffer)
@@ -415,6 +416,9 @@ public unsafe static int[] ToIntArray(this byte[] buffer, ref int position)
 
  public unsafe static int[] ToIntArray(this byte[] buffer, ref int position, int length)
  {
+ if (length == 0)
+ return null;
+
  int[] a = new int[length];
 
  fixed (byte* tmpBuffer = buffer)
@@ -433,7 +437,7 @@ public unsafe static int[] ToIntArray(this byte[] buffer, ref int position, int
 
  public static int SizeInBytes(this uint[] array)
  {
- return sizeof(int) + array.Length * sizeof(uint);
+ return sizeof(int) + Utils.Size(array) * sizeof(uint);
  }
 
  public unsafe static void ToByteArray(this uint[] a, byte[] buffer, ref int position)
@@ -472,7 +476,7 @@ public unsafe static uint[] ToUIntArray(this byte[] buffer, ref int position)
 
  public static int SizeInBytes(this long[] array)
  {
- return sizeof(int) + array.Length * sizeof(long);
+ return sizeof(int) + Utils.Size(array) * sizeof(long);
  }
 
  public unsafe static void ToByteArray(this long[] a, byte[] buffer, ref int position)
@@ -511,7 +515,7 @@ public unsafe static long[] ToLongArray(this byte[] buffer, ref int position)
 
  public static int SizeInBytes(this ulong[] array)
  {
- return sizeof(int) + array.Length * sizeof(ulong);
+ return sizeof(int) + Utils.Size(array) * sizeof(ulong);
  }
 
  public unsafe static void ToByteArray(this ulong[] a, byte[] buffer, ref int position)
@@ -550,7 +554,7 @@ public unsafe static ulong[] ToULongArray(this byte[] buffer, ref int position)
 
  public static int SizeInBytes(this MD5Hash[] array)
  {
- return sizeof(int) + array.Length * MD5Hash.SizeInBytes();
+ return sizeof(int) + Utils.Size(array) * MD5Hash.SizeInBytes();
  }
 
  public static void ToByteArray(this MD5Hash[] a, byte[] buffer, ref int position)
@@ -577,7 +581,7 @@ public unsafe static MD5Hash[] ToUInt128Array(this byte[] buffer, ref int positi
 
  public static int SizeInBytes(this float[] array)
  {
- return sizeof(int) + array.Length * sizeof(float);
+ return sizeof(int) + Utils.Size(array) * sizeof(float);
  }
 
  public unsafe static void ToByteArray(this float[] a, byte[] buffer, ref int position)
@@ -616,7 +620,7 @@ public unsafe static float[] ToFloatArray(this byte[] buffer, ref int position)
 
  public static int SizeInBytes(this double[] array)
  {
- return sizeof(int) + array.Length * sizeof(double);
+ return sizeof(int) + Utils.Size(array) * sizeof(double);
  }
 
  public unsafe static void ToByteArray(this double[] a, byte[] buffer, ref int position)
@@ -655,6 +659,8 @@ public unsafe static double[] ToDoubleArray(this byte[] buffer, ref int position
 
  public static int SizeInBytes(this double[][] array)
  {
+ if (Utils.Size(array) == 0)
+ return sizeof(int);
  return sizeof(int) + array.Sum(x => x.SizeInBytes());
  }
 
@@ -683,7 +689,7 @@ public static double[][] ToDoubleJaggedArray(this byte[] buffer, ref int positio
  public static long SizeInBytes(this string[] array)
  {
  long length = sizeof(int);
- for (int i = 0; i < array.Length; ++i)
+ for (int i = 0; i < Utils.Size(array); ++i)
  {
  length += array[i].SizeInBytes();
  }
@@ -692,8 +698,8 @@ public static long SizeInBytes(this string[] array)
 
  public static void ToByteArray(this string[] a, byte[] buffer, ref int position)
  {
-a.Length.ToByteArray(buffer, ref position);
- for (int i = 0; i < a.Length; ++i)
+Utils.Size(a).ToByteArray(buffer, ref position);
+ for (int i = 0; i < Utils.Size(a); ++i)
  {
  a[i].ToByteArray(buffer, ref position);
  }

src/Microsoft.ML.Core/Prediction/ITrainer.cs

@@ -74,6 +74,8 @@ public interface IModelCombiner<TModel, TPredictor>
  TPredictor CombineModels(IEnumerable<TModel> models);
  }
 
+ public delegate void SignatureModelCombiner(PredictionKind kind);
+
  /// <summary>
  /// Weakly typed interface for a trainer "session" that produces a predictor.
  /// </summary>

src/Microsoft.ML.FastTree/Microsoft.ML.FastTree.csproj

@@ -57,6 +57,7 @@
  <Compile Include="TreeEnsemble\Ensemble.cs" />
  <Compile Include="TreeEnsemble\QuantileRegressionTree.cs" />
  <Compile Include="TreeEnsemble\RegressionTree.cs" />
+ <Compile Include="TreeEnsemble\TreeEnsembleCombiner.cs" />
  <Compile Include="Training\Applications\GradientWrappers.cs" />
  <Compile Include="Training\Applications\ObjectiveFunction.cs" />
  <Compile Include="Training\BaggingProvider.cs" />

src/Microsoft.ML.FastTree/TreeEnsemble/RegressionTree.cs

@@ -105,22 +105,21 @@ public RegressionTree(byte[] buffer, ref int position)
  LteChild = buffer.ToIntArray(ref position);
  GtChild = buffer.ToIntArray(ref position);
  SplitFeatures = buffer.ToIntArray(ref position);
-int[] categoricalNodeIndices = buffer.ToIntArray(ref position);
- CategoricalSplit = GetCategoricalSplitFromIndices(categoricalNodeIndices);
- if (categoricalNodeIndices?.Length > 0)
+byte[] categoricalSplitAsBytes = buffer.ToByteArray(ref position);
+ CategoricalSplit = categoricalSplitAsBytes.Select(b => b > 0).ToArray();
+ if (CategoricalSplit.Any(b => b))
  {
  CategoricalSplitFeatures = new int[NumNodes][];
  CategoricalSplitFeatureRanges = new int[NumNodes][];
-foreach (var index in categoricalNodeIndices)
+for (int index = 0; index < NumNodes; index++)
  {
- Contracts.Assert(CategoricalSplit[index]);
-
  CategoricalSplitFeatures[index] = buffer.ToIntArray(ref position);
- CategoricalSplitFeatureRanges[index] = buffer.ToIntArray(ref position, 2);
+ CategoricalSplitFeatureRanges[index] = buffer.ToIntArray(ref position);
  }
  }
 
  Thresholds = buffer.ToUIntArray(ref position);
+ RawThresholds = buffer.ToFloatArray(ref position);
  _splitGain = buffer.ToDoubleArray(ref position);
  _gainPValue = buffer.ToDoubleArray(ref position);
  _previousLeafValue = buffer.ToDoubleArray(ref position);
@@ -144,6 +143,23 @@ private bool[] GetCategoricalSplitFromIndices(int[] indices)
  return categoricalSplit;
  }
 
+ private bool[] GetCategoricalSplitFromBytes(byte[] indices)
+ {
+ bool[] categoricalSplit = new bool[NumNodes];
+ if (indices == null)
+ return categoricalSplit;
+
+ Contracts.Assert(indices.Length <= NumNodes);
+
+ foreach (int index in indices)
+ {
+ Contracts.Assert(index >= 0 && index < NumNodes);
+ categoricalSplit[index] = true;
+ }
+
+ return categoricalSplit;
+ }
+
  /// <summary>
  /// Create a Regression Tree object from raw tree contents.
  /// </summary>
@@ -192,7 +208,7 @@ internal RegressionTree(int[] splitFeatures, Double[] splitGain, Double[] gainPV
  LeafValues = leafValues;
  CategoricalSplitFeatures = categoricalSplitFeatures;
  CategoricalSplitFeatureRanges = new int[CategoricalSplitFeatures.Length][];
- for(int i= 0; i < CategoricalSplitFeatures.Length; ++i)
+ for(int i= 0; i < CategoricalSplitFeatures.Length; ++i)
  {
  if (CategoricalSplitFeatures[i] != null && CategoricalSplitFeatures[i].Length > 0)
  {
@@ -500,6 +516,7 @@ public virtual int SizeInBytes()
  NumNodes * sizeof(int) +
  CategoricalSplit.Length * sizeof(bool) +
  Thresholds.SizeInBytes() +
+ RawThresholds.SizeInBytes() +
  _splitGain.SizeInBytes() +
  _gainPValue.SizeInBytes() +
  _previousLeafValue.SizeInBytes() +
@@ -514,22 +531,22 @@ public virtual void ToByteArray(byte[] buffer, ref int position)
  LteChild.ToByteArray(buffer, ref position);
  GtChild.ToByteArray(buffer, ref position);
  SplitFeatures.ToByteArray(buffer, ref position);
+ CategoricalSplit.Length.ToByteArray(buffer, ref position);
  foreach (var split in CategoricalSplit)
  Convert.ToByte(split).ToByteArray(buffer, ref position);
 
  if (CategoricalSplitFeatures != null)
  {
- foreach (var splits in CategoricalSplitFeatures)
- splits.ToByteArray(buffer, ref position);
- }
-
- if (CategoricalSplitFeatureRanges != null)
- {
- foreach (var ranges in CategoricalSplitFeatureRanges)
- ranges.ToByteArray(buffer, ref position);
+ Contracts.AssertValue(CategoricalSplitFeatureRanges);
+ for (int i = 0; i < CategoricalSplitFeatures.Length; i++)
+ {
+ CategoricalSplitFeatures[i].ToByteArray(buffer, ref position);
+ CategoricalSplitFeatureRanges[i].ToByteArray(buffer, ref position);
+ }
  }
 
  Thresholds.ToByteArray(buffer, ref position);
+ RawThresholds.ToByteArray(buffer, ref position);
  _splitGain.ToByteArray(buffer, ref position);
  _gainPValue.ToByteArray(buffer, ref position);
  _previousLeafValue.ToByteArray(buffer, ref position);

src/Microsoft.ML.FastTree/TreeEnsemble/TreeEnsembleCombiner.cs

@@ -0,0 +1,115 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System.Collections.Generic;
+using Microsoft.ML.Runtime;
+using Microsoft.ML.Runtime.FastTree.Internal;
+using Microsoft.ML.Runtime.Internal.Calibration;
+
+[assembly: LoadableClass(typeof(TreeEnsembleCombiner), null, typeof(SignatureModelCombiner), "Fast Tree Model Combiner", "FastTreeCombiner")]
+
+namespace Microsoft.ML.Runtime.FastTree.Internal
+{
+ public sealed class TreeEnsembleCombiner : IModelCombiner<IPredictorProducing<float>, IPredictorProducing<float>>
+ {
+ private readonly IHost _host;
+ private readonly PredictionKind _kind;
+
+ public TreeEnsembleCombiner(IHostEnvironment env, PredictionKind kind)
+ {
+ _host = env.Register("TreeEnsembleCombiner");
+ switch (kind)
+ {
+ case PredictionKind.BinaryClassification:
+ case PredictionKind.Regression:
+ case PredictionKind.Ranking:
+ _kind = kind;
+ break;
+ default:
+ throw _host.ExceptUserArg(nameof(kind), $"Tree ensembles can be either of type {nameof(PredictionKind.BinaryClassification)}, " +
+ $"{nameof(PredictionKind.Regression)} or {nameof(PredictionKind.Ranking)}");
+ }
+ }
+
+ public IPredictorProducing<float> CombineModels(IEnumerable<IPredictorProducing<float>> models)
+ {
+ _host.CheckValue(models, nameof(models));
+
+ var ensemble = new Ensemble();
+ int modelCount = 0;
+ int featureCount = -1;
+ bool binaryClassifier = false;
+ foreach (var model in models)
+ {
+ modelCount++;
+
+ var predictor = model;
+ _host.CheckValue(predictor, nameof(models), "One of the models is null");
+
+ var calibrated = predictor as CalibratedPredictorBase;
+ double paramA = 1;
+ if (calibrated != null)
+ {
+ _host.Check(calibrated.Calibrator is PlattCalibrator,
+ "Combining FastTree models can only be done when the models are calibrated with Platt calibrator");
+ predictor = calibrated.SubPredictor;
+ paramA = -(calibrated.Calibrator as PlattCalibrator).ParamA;
+ }
+ var tree = predictor as FastTreePredictionWrapper;
+ if (tree == null)
+ throw _host.Except("Model is not a tree ensemble");
+ foreach (var t in tree.TrainedEnsemble.Trees)
+ {
+ var bytes = new byte[t.SizeInBytes()];
+ int position = -1;
+ t.ToByteArray(bytes, ref position);
+ position = -1;
+ var tNew = new RegressionTree(bytes, ref position);
+ if (paramA != 1)
+ {
+ for (int i = 0; i < tNew.NumLeaves; i++)
+ tNew.SetOutput(i, tNew.LeafValues[i] * paramA);
+ }
+ ensemble.AddTree(tNew);
+ }
+
+ if (modelCount == 1)
+ {
+ binaryClassifier = calibrated != null;
+ featureCount = tree.InputType.ValueCount;
+ }
+ else
+ {
+ _host.Check((calibrated != null) == binaryClassifier, "Ensemble contains both calibrated and uncalibrated models");
+ _host.Check(featureCount == tree.InputType.ValueCount, "Found models with different number of features");
+ }
+ }
+
+ var scale = 1 / (double)modelCount;
+
+ foreach (var t in ensemble.Trees)
+ {
+ for (int i = 0; i < t.NumLeaves; i++)
+ t.SetOutput(i, t.LeafValues[i] * scale);
+ }
+
+ switch (_kind)
+ {
+ case PredictionKind.BinaryClassification:
+ if (!binaryClassifier)
+ return new FastTreeBinaryPredictor(_host, ensemble, featureCount, null);
+
+ var cali = new PlattCalibrator(_host, -1, 0);
+ return new FeatureWeightsCalibratedPredictor(_host, new FastTreeBinaryPredictor(_host, ensemble, featureCount, null), cali);
+ case PredictionKind.Regression:
+ return new FastTreeRegressionPredictor(_host, ensemble, featureCount, null);
+ case PredictionKind.Ranking:
+ return new FastTreeRankingPredictor(_host, ensemble, featureCount, null);
+ default:
+ _host.Assert(false);
+ throw _host.ExceptNotSupp();
+ }
+ }
+ }
+}

src/Microsoft.ML.FastTree/Utils/ToByteArrayExtensions.cs

@@ -5,6 +5,7 @@
 using System;
 using System.Linq;
 using System.Text;
+using Microsoft.ML.Runtime.Internal.Utilities;
 
 namespace Microsoft.ML.Runtime.FastTree.Internal
 {
@@ -290,7 +291,7 @@ public static string ToString(this byte[] buffer, ref int position)
 
  public static int SizeInBytes(this byte[] a)
  {
- return sizeof(int) + a.Length * sizeof(byte);
+ return sizeof(int) + Utils.Size(a) * sizeof(byte);
  }
 
  public static void ToByteArray(this byte[] a, byte[] buffer, ref int position)
@@ -314,7 +315,7 @@ public static byte[] ToByteArray(this byte[] buffer, ref int position)
 
  public static int SizeInBytes(this short[] a)
  {
- return sizeof(int) + a.Length * sizeof(short);
+ return sizeof(int) + Utils.Size(a) * sizeof(short);
  }
 
  public unsafe static void ToByteArray(this short[] a, byte[] buffer, ref int position)
@@ -353,7 +354,7 @@ public unsafe static short[] ToShortArray(this byte[] buffer, ref int position)
 
  public static int SizeInBytes(this ushort[] a)
  {
- return sizeof(int) + a.Length * sizeof(ushort);
+ return sizeof(int) + Utils.Size(a) * sizeof(ushort);
  }
 
  public unsafe static void ToByteArray(this ushort[] a, byte[] buffer, ref int position)
@@ -392,12 +393,12 @@ public unsafe static ushort[] ToUShortArray(this byte[] buffer, ref int position
 
  public static int SizeInBytes(this int[] array)
  {
- return sizeof(int) + array.Length * sizeof(int);
+ return sizeof(int) + Utils.Size(array) * sizeof(int);
  }
 
  public unsafe static void ToByteArray(this int[] a, byte[] buffer, ref int position)
  {
- int length = a.Length;
+ int length = Utils.Size(a);
  length.ToByteArray(buffer, ref position);
 
  fixed (byte* tmpBuffer = buffer)
@@ -415,6 +416,9 @@ public unsafe static int[] ToIntArray(this byte[] buffer, ref int position)
 
  public unsafe static int[] ToIntArray(this byte[] buffer, ref int position, int length)
  {
+ if (length == 0)
+ return null;
+
  int[] a = new int[length];
 
  fixed (byte* tmpBuffer = buffer)
@@ -433,7 +437,7 @@ public unsafe static int[] ToIntArray(this byte[] buffer, ref int position, int
 
  public static int SizeInBytes(this uint[] array)
  {
- return sizeof(int) + array.Length * sizeof(uint);
+ return sizeof(int) + Utils.Size(array) * sizeof(uint);
  }
 
  public unsafe static void ToByteArray(this uint[] a, byte[] buffer, ref int position)
@@ -472,7 +476,7 @@ public unsafe static uint[] ToUIntArray(this byte[] buffer, ref int position)
 
  public static int SizeInBytes(this long[] array)
  {
- return sizeof(int) + array.Length * sizeof(long);
+ return sizeof(int) + Utils.Size(array) * sizeof(long);
  }
 
  public unsafe static void ToByteArray(this long[] a, byte[] buffer, ref int position)
@@ -511,7 +515,7 @@ public unsafe static long[] ToLongArray(this byte[] buffer, ref int position)
 
  public static int SizeInBytes(this ulong[] array)
  {
- return sizeof(int) + array.Length * sizeof(ulong);
+ return sizeof(int) + Utils.Size(array) * sizeof(ulong);
  }
 
  public unsafe static void ToByteArray(this ulong[] a, byte[] buffer, ref int position)
@@ -550,7 +554,7 @@ public unsafe static ulong[] ToULongArray(this byte[] buffer, ref int position)
 
  public static int SizeInBytes(this MD5Hash[] array)
  {
- return sizeof(int) + array.Length * MD5Hash.SizeInBytes();
+ return sizeof(int) + Utils.Size(array) * MD5Hash.SizeInBytes();
  }
 
  public static void ToByteArray(this MD5Hash[] a, byte[] buffer, ref int position)
@@ -577,7 +581,7 @@ public unsafe static MD5Hash[] ToUInt128Array(this byte[] buffer, ref int positi
 
  public static int SizeInBytes(this float[] array)
  {
- return sizeof(int) + array.Length * sizeof(float);
+ return sizeof(int) + Utils.Size(array) * sizeof(float);
  }
 
  public unsafe static void ToByteArray(this float[] a, byte[] buffer, ref int position)
@@ -616,7 +620,7 @@ public unsafe static float[] ToFloatArray(this byte[] buffer, ref int position)
 
  public static int SizeInBytes(this double[] array)
  {
- return sizeof(int) + array.Length * sizeof(double);
+ return sizeof(int) + Utils.Size(array) * sizeof(double);
  }
 
  public unsafe static void ToByteArray(this double[] a, byte[] buffer, ref int position)
@@ -655,6 +659,8 @@ public unsafe static double[] ToDoubleArray(this byte[] buffer, ref int position
 
  public static int SizeInBytes(this double[][] array)
  {
+ if (Utils.Size(array) == 0)
+ return sizeof(int);
  return sizeof(int) + array.Sum(x => x.SizeInBytes());
  }
 
@@ -683,7 +689,7 @@ public static double[][] ToDoubleJaggedArray(this byte[] buffer, ref int positio
  public static long SizeInBytes(this string[] array)
  {
  long length = sizeof(int);
- for (int i = 0; i < array.Length; ++i)
+ for (int i = 0; i < Utils.Size(array); ++i)
  {
  length += array[i].SizeInBytes();
  }
@@ -692,8 +698,8 @@ public static long SizeInBytes(this string[] array)
 
  public static void ToByteArray(this string[] a, byte[] buffer, ref int position)
  {
-a.Length.ToByteArray(buffer, ref position);
- for (int i = 0; i < a.Length; ++i)
+Utils.Size(a).ToByteArray(buffer, ref position);
+ for (int i = 0; i < Utils.Size(a); ++i)
  {
  a[i].ToByteArray(buffer, ref position);
  }

src/Microsoft.ML.Core/Prediction/ITrainer.cs

@@ -74,6 +74,8 @@ public interface IModelCombiner<TModel, TPredictor>
  TPredictor CombineModels(IEnumerable<TModel> models);
  }
 
+ public delegate void SignatureModelCombiner(PredictionKind kind);
+
  /// <summary>
  /// Weakly typed interface for a trainer "session" that produces a predictor.
  /// </summary>

src/Microsoft.ML.FastTree/Microsoft.ML.FastTree.csproj

@@ -57,6 +57,7 @@
  <Compile Include="TreeEnsemble\Ensemble.cs" />
  <Compile Include="TreeEnsemble\QuantileRegressionTree.cs" />
  <Compile Include="TreeEnsemble\RegressionTree.cs" />
+ <Compile Include="TreeEnsemble\TreeEnsembleCombiner.cs" />
  <Compile Include="Training\Applications\GradientWrappers.cs" />
  <Compile Include="Training\Applications\ObjectiveFunction.cs" />
  <Compile Include="Training\BaggingProvider.cs" />

src/Microsoft.ML.FastTree/TreeEnsemble/RegressionTree.cs

@@ -105,22 +105,21 @@ public RegressionTree(byte[] buffer, ref int position)
  LteChild = buffer.ToIntArray(ref position);
  GtChild = buffer.ToIntArray(ref position);
  SplitFeatures = buffer.ToIntArray(ref position);
-int[] categoricalNodeIndices = buffer.ToIntArray(ref position);
- CategoricalSplit = GetCategoricalSplitFromIndices(categoricalNodeIndices);
- if (categoricalNodeIndices?.Length > 0)
+byte[] categoricalSplitAsBytes = buffer.ToByteArray(ref position);
+ CategoricalSplit = categoricalSplitAsBytes.Select(b => b > 0).ToArray();
+ if (CategoricalSplit.Any(b => b))
  {
  CategoricalSplitFeatures = new int[NumNodes][];
  CategoricalSplitFeatureRanges = new int[NumNodes][];
-foreach (var index in categoricalNodeIndices)
+for (int index = 0; index < NumNodes; index++)
  {
- Contracts.Assert(CategoricalSplit[index]);
-
  CategoricalSplitFeatures[index] = buffer.ToIntArray(ref position);
- CategoricalSplitFeatureRanges[index] = buffer.ToIntArray(ref position, 2);
+ CategoricalSplitFeatureRanges[index] = buffer.ToIntArray(ref position);
  }
  }
 
  Thresholds = buffer.ToUIntArray(ref position);
+ RawThresholds = buffer.ToFloatArray(ref position);
  _splitGain = buffer.ToDoubleArray(ref position);
  _gainPValue = buffer.ToDoubleArray(ref position);
  _previousLeafValue = buffer.ToDoubleArray(ref position);
@@ -144,6 +143,23 @@ private bool[] GetCategoricalSplitFromIndices(int[] indices)
  return categoricalSplit;
  }
 
+ private bool[] GetCategoricalSplitFromBytes(byte[] indices)
+ {
+ bool[] categoricalSplit = new bool[NumNodes];
+ if (indices == null)
+ return categoricalSplit;
+
+ Contracts.Assert(indices.Length <= NumNodes);
+
+ foreach (int index in indices)
+ {
+ Contracts.Assert(index >= 0 && index < NumNodes);
+ categoricalSplit[index] = true;
+ }
+
+ return categoricalSplit;
+ }
+
  /// <summary>
  /// Create a Regression Tree object from raw tree contents.
  /// </summary>
@@ -192,7 +208,7 @@ internal RegressionTree(int[] splitFeatures, Double[] splitGain, Double[] gainPV
  LeafValues = leafValues;
  CategoricalSplitFeatures = categoricalSplitFeatures;
  CategoricalSplitFeatureRanges = new int[CategoricalSplitFeatures.Length][];
- for(int i= 0; i < CategoricalSplitFeatures.Length; ++i)
+ for(int i= 0; i < CategoricalSplitFeatures.Length; ++i)
  {
  if (CategoricalSplitFeatures[i] != null && CategoricalSplitFeatures[i].Length > 0)
  {
@@ -500,6 +516,7 @@ public virtual int SizeInBytes()
  NumNodes * sizeof(int) +
  CategoricalSplit.Length * sizeof(bool) +
  Thresholds.SizeInBytes() +
+ RawThresholds.SizeInBytes() +
  _splitGain.SizeInBytes() +
  _gainPValue.SizeInBytes() +
  _previousLeafValue.SizeInBytes() +
@@ -514,22 +531,22 @@ public virtual void ToByteArray(byte[] buffer, ref int position)
  LteChild.ToByteArray(buffer, ref position);
  GtChild.ToByteArray(buffer, ref position);
  SplitFeatures.ToByteArray(buffer, ref position);
+ CategoricalSplit.Length.ToByteArray(buffer, ref position);
  foreach (var split in CategoricalSplit)
  Convert.ToByte(split).ToByteArray(buffer, ref position);
 
  if (CategoricalSplitFeatures != null)
  {
- foreach (var splits in CategoricalSplitFeatures)
- splits.ToByteArray(buffer, ref position);
- }
-
- if (CategoricalSplitFeatureRanges != null)
- {
- foreach (var ranges in CategoricalSplitFeatureRanges)
- ranges.ToByteArray(buffer, ref position);
+ Contracts.AssertValue(CategoricalSplitFeatureRanges);
+ for (int i = 0; i < CategoricalSplitFeatures.Length; i++)
+ {
+ CategoricalSplitFeatures[i].ToByteArray(buffer, ref position);
+ CategoricalSplitFeatureRanges[i].ToByteArray(buffer, ref position);
+ }
  }
 
  Thresholds.ToByteArray(buffer, ref position);
+ RawThresholds.ToByteArray(buffer, ref position);
  _splitGain.ToByteArray(buffer, ref position);
  _gainPValue.ToByteArray(buffer, ref position);
  _previousLeafValue.ToByteArray(buffer, ref position);

src/Microsoft.ML.FastTree/TreeEnsemble/TreeEnsembleCombiner.cs

@@ -0,0 +1,115 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System.Collections.Generic;
+using Microsoft.ML.Runtime;
+using Microsoft.ML.Runtime.FastTree.Internal;
+using Microsoft.ML.Runtime.Internal.Calibration;
+
+[assembly: LoadableClass(typeof(TreeEnsembleCombiner), null, typeof(SignatureModelCombiner), "Fast Tree Model Combiner", "FastTreeCombiner")]
+
+namespace Microsoft.ML.Runtime.FastTree.Internal
+{
+ public sealed class TreeEnsembleCombiner : IModelCombiner<IPredictorProducing<float>, IPredictorProducing<float>>
+ {
+ private readonly IHost _host;
+ private readonly PredictionKind _kind;
+
+ public TreeEnsembleCombiner(IHostEnvironment env, PredictionKind kind)
+ {
+ _host = env.Register("TreeEnsembleCombiner");
+ switch (kind)
+ {
+ case PredictionKind.BinaryClassification:
+ case PredictionKind.Regression:
+ case PredictionKind.Ranking:
+ _kind = kind;
+ break;
+ default:
+ throw _host.ExceptUserArg(nameof(kind), $"Tree ensembles can be either of type {nameof(PredictionKind.BinaryClassification)}, " +
+ $"{nameof(PredictionKind.Regression)} or {nameof(PredictionKind.Ranking)}");
+ }
+ }
+
+ public IPredictorProducing<float> CombineModels(IEnumerable<IPredictorProducing<float>> models)
+ {
+ _host.CheckValue(models, nameof(models));
+
+ var ensemble = new Ensemble();
+ int modelCount = 0;
+ int featureCount = -1;
+ bool binaryClassifier = false;
+ foreach (var model in models)
+ {
+ modelCount++;
+
+ var predictor = model;
+ _host.CheckValue(predictor, nameof(models), "One of the models is null");
+
+ var calibrated = predictor as CalibratedPredictorBase;
+ double paramA = 1;
+ if (calibrated != null)
+ {
+ _host.Check(calibrated.Calibrator is PlattCalibrator,
+ "Combining FastTree models can only be done when the models are calibrated with Platt calibrator");
+ predictor = calibrated.SubPredictor;
+ paramA = -(calibrated.Calibrator as PlattCalibrator).ParamA;
+ }
+ var tree = predictor as FastTreePredictionWrapper;
+ if (tree == null)
+ throw _host.Except("Model is not a tree ensemble");
+ foreach (var t in tree.TrainedEnsemble.Trees)
+ {
+ var bytes = new byte[t.SizeInBytes()];
+ int position = -1;
+ t.ToByteArray(bytes, ref position);
+ position = -1;
+ var tNew = new RegressionTree(bytes, ref position);
+ if (paramA != 1)
+ {
+ for (int i = 0; i < tNew.NumLeaves; i++)
+ tNew.SetOutput(i, tNew.LeafValues[i] * paramA);
+ }
+ ensemble.AddTree(tNew);
+ }
+
+ if (modelCount == 1)
+ {
+ binaryClassifier = calibrated != null;
+ featureCount = tree.InputType.ValueCount;
+ }
+ else
+ {
+ _host.Check((calibrated != null) == binaryClassifier, "Ensemble contains both calibrated and uncalibrated models");
+ _host.Check(featureCount == tree.InputType.ValueCount, "Found models with different number of features");
+ }
+ }
+
+ var scale = 1 / (double)modelCount;
+
+ foreach (var t in ensemble.Trees)
+ {
+ for (int i = 0; i < t.NumLeaves; i++)
+ t.SetOutput(i, t.LeafValues[i] * scale);
+ }
+
+ switch (_kind)
+ {
+ case PredictionKind.BinaryClassification:
+ if (!binaryClassifier)
+ return new FastTreeBinaryPredictor(_host, ensemble, featureCount, null);
+
+ var cali = new PlattCalibrator(_host, -1, 0);
+ return new FeatureWeightsCalibratedPredictor(_host, new FastTreeBinaryPredictor(_host, ensemble, featureCount, null), cali);
+ case PredictionKind.Regression:
+ return new FastTreeRegressionPredictor(_host, ensemble, featureCount, null);
+ case PredictionKind.Ranking:
+ return new FastTreeRankingPredictor(_host, ensemble, featureCount, null);
+ default:
+ _host.Assert(false);
+ throw _host.ExceptNotSupp();
+ }
+ }
+ }
+}

src/Microsoft.ML.FastTree/Utils/ToByteArrayExtensions.cs

@@ -5,6 +5,7 @@
 using System;
 using System.Linq;
 using System.Text;
+using Microsoft.ML.Runtime.Internal.Utilities;
 
 namespace Microsoft.ML.Runtime.FastTree.Internal
 {
@@ -290,7 +291,7 @@ public static string ToString(this byte[] buffer, ref int position)
 
  public static int SizeInBytes(this byte[] a)
  {
- return sizeof(int) + a.Length * sizeof(byte);
+ return sizeof(int) + Utils.Size(a) * sizeof(byte);
  }
 
  public static void ToByteArray(this byte[] a, byte[] buffer, ref int position)
@@ -314,7 +315,7 @@ public static byte[] ToByteArray(this byte[] buffer, ref int position)
 
  public static int SizeInBytes(this short[] a)
  {
- return sizeof(int) + a.Length * sizeof(short);
+ return sizeof(int) + Utils.Size(a) * sizeof(short);
  }
 
  public unsafe static void ToByteArray(this short[] a, byte[] buffer, ref int position)
@@ -353,7 +354,7 @@ public unsafe static short[] ToShortArray(this byte[] buffer, ref int position)
 
  public static int SizeInBytes(this ushort[] a)
  {
- return sizeof(int) + a.Length * sizeof(ushort);
+ return sizeof(int) + Utils.Size(a) * sizeof(ushort);
  }
 
  public unsafe static void ToByteArray(this ushort[] a, byte[] buffer, ref int position)
@@ -392,12 +393,12 @@ public unsafe static ushort[] ToUShortArray(this byte[] buffer, ref int position
 
  public static int SizeInBytes(this int[] array)
  {
- return sizeof(int) + array.Length * sizeof(int);
+ return sizeof(int) + Utils.Size(array) * sizeof(int);
  }
 
  public unsafe static void ToByteArray(this int[] a, byte[] buffer, ref int position)
  {
- int length = a.Length;
+ int length = Utils.Size(a);
  length.ToByteArray(buffer, ref position);
 
  fixed (byte* tmpBuffer = buffer)
@@ -415,6 +416,9 @@ public unsafe static int[] ToIntArray(this byte[] buffer, ref int position)
 
  public unsafe static int[] ToIntArray(this byte[] buffer, ref int position, int length)
  {
+ if (length == 0)
+ return null;
+
  int[] a = new int[length];
 
  fixed (byte* tmpBuffer = buffer)
@@ -433,7 +437,7 @@ public unsafe static int[] ToIntArray(this byte[] buffer, ref int position, int
 
  public static int SizeInBytes(this uint[] array)
  {
- return sizeof(int) + array.Length * sizeof(uint);
+ return sizeof(int) + Utils.Size(array) * sizeof(uint);
  }
 
  public unsafe static void ToByteArray(this uint[] a, byte[] buffer, ref int position)
@@ -472,7 +476,7 @@ public unsafe static uint[] ToUIntArray(this byte[] buffer, ref int position)
 
  public static int SizeInBytes(this long[] array)
  {
- return sizeof(int) + array.Length * sizeof(long);
+ return sizeof(int) + Utils.Size(array) * sizeof(long);
  }
 
  public unsafe static void ToByteArray(this long[] a, byte[] buffer, ref int position)
@@ -511,7 +515,7 @@ public unsafe static long[] ToLongArray(this byte[] buffer, ref int position)
 
  public static int SizeInBytes(this ulong[] array)
  {
- return sizeof(int) + array.Length * sizeof(ulong);
+ return sizeof(int) + Utils.Size(array) * sizeof(ulong);
  }
 
  public unsafe static void ToByteArray(this ulong[] a, byte[] buffer, ref int position)
@@ -550,7 +554,7 @@ public unsafe static ulong[] ToULongArray(this byte[] buffer, ref int position)
 
  public static int SizeInBytes(this MD5Hash[] array)
  {
- return sizeof(int) + array.Length * MD5Hash.SizeInBytes();
+ return sizeof(int) + Utils.Size(array) * MD5Hash.SizeInBytes();
  }
 
  public static void ToByteArray(this MD5Hash[] a, byte[] buffer, ref int position)
@@ -577,7 +581,7 @@ public unsafe static MD5Hash[] ToUInt128Array(this byte[] buffer, ref int positi
 
  public static int SizeInBytes(this float[] array)
  {
- return sizeof(int) + array.Length * sizeof(float);
+ return sizeof(int) + Utils.Size(array) * sizeof(float);
  }
 
  public unsafe static void ToByteArray(this float[] a, byte[] buffer, ref int position)
@@ -616,7 +620,7 @@ public unsafe static float[] ToFloatArray(this byte[] buffer, ref int position)
 
  public static int SizeInBytes(this double[] array)
  {
- return sizeof(int) + array.Length * sizeof(double);
+ return sizeof(int) + Utils.Size(array) * sizeof(double);
  }
 
  public unsafe static void ToByteArray(this double[] a, byte[] buffer, ref int position)
@@ -655,6 +659,8 @@ public unsafe static double[] ToDoubleArray(this byte[] buffer, ref int position
 
  public static int SizeInBytes(this double[][] array)
  {
+ if (Utils.Size(array) == 0)
+ return sizeof(int);
  return sizeof(int) + array.Sum(x => x.SizeInBytes());
  }
 
@@ -683,7 +689,7 @@ public static double[][] ToDoubleJaggedArray(this byte[] buffer, ref int positio
  public static long SizeInBytes(this string[] array)
  {
  long length = sizeof(int);
- for (int i = 0; i < array.Length; ++i)
+ for (int i = 0; i < Utils.Size(array); ++i)
  {
  length += array[i].SizeInBytes();
  }
@@ -692,8 +698,8 @@ public static long SizeInBytes(this string[] array)
 
  public static void ToByteArray(this string[] a, byte[] buffer, ref int position)
  {
-a.Length.ToByteArray(buffer, ref position);
- for (int i = 0; i < a.Length; ++i)
+Utils.Size(a).ToByteArray(buffer, ref position);
+ for (int i = 0; i < Utils.Size(a); ++i)
  {
  a[i].ToByteArray(buffer, ref position);
  }

src/Microsoft.ML.Core/Prediction/ITrainer.cs

@@ -74,6 +74,8 @@ public interface IModelCombiner<TModel, TPredictor>
  TPredictor CombineModels(IEnumerable<TModel> models);
  }
 
+ public delegate void SignatureModelCombiner(PredictionKind kind);
+
  /// <summary>
  /// Weakly typed interface for a trainer "session" that produces a predictor.
  /// </summary>

src/Microsoft.ML.FastTree/Microsoft.ML.FastTree.csproj

@@ -57,6 +57,7 @@
  <Compile Include="TreeEnsemble\Ensemble.cs" />
  <Compile Include="TreeEnsemble\QuantileRegressionTree.cs" />
  <Compile Include="TreeEnsemble\RegressionTree.cs" />
+ <Compile Include="TreeEnsemble\TreeEnsembleCombiner.cs" />
  <Compile Include="Training\Applications\GradientWrappers.cs" />
  <Compile Include="Training\Applications\ObjectiveFunction.cs" />
  <Compile Include="Training\BaggingProvider.cs" />

src/Microsoft.ML.FastTree/TreeEnsemble/RegressionTree.cs

@@ -105,22 +105,21 @@ public RegressionTree(byte[] buffer, ref int position)
  LteChild = buffer.ToIntArray(ref position);
  GtChild = buffer.ToIntArray(ref position);
  SplitFeatures = buffer.ToIntArray(ref position);
-int[] categoricalNodeIndices = buffer.ToIntArray(ref position);
- CategoricalSplit = GetCategoricalSplitFromIndices(categoricalNodeIndices);
- if (categoricalNodeIndices?.Length > 0)
+byte[] categoricalSplitAsBytes = buffer.ToByteArray(ref position);
+ CategoricalSplit = categoricalSplitAsBytes.Select(b => b > 0).ToArray();
+ if (CategoricalSplit.Any(b => b))
  {
  CategoricalSplitFeatures = new int[NumNodes][];
  CategoricalSplitFeatureRanges = new int[NumNodes][];
-foreach (var index in categoricalNodeIndices)
+for (int index = 0; index < NumNodes; index++)
  {
- Contracts.Assert(CategoricalSplit[index]);
-
  CategoricalSplitFeatures[index] = buffer.ToIntArray(ref position);
- CategoricalSplitFeatureRanges[index] = buffer.ToIntArray(ref position, 2);
+ CategoricalSplitFeatureRanges[index] = buffer.ToIntArray(ref position);
  }
  }
 
  Thresholds = buffer.ToUIntArray(ref position);
+ RawThresholds = buffer.ToFloatArray(ref position);
  _splitGain = buffer.ToDoubleArray(ref position);
  _gainPValue = buffer.ToDoubleArray(ref position);
  _previousLeafValue = buffer.ToDoubleArray(ref position);
@@ -144,6 +143,23 @@ private bool[] GetCategoricalSplitFromIndices(int[] indices)
  return categoricalSplit;
  }
 
+ private bool[] GetCategoricalSplitFromBytes(byte[] indices)
+ {
+ bool[] categoricalSplit = new bool[NumNodes];
+ if (indices == null)
+ return categoricalSplit;
+
+ Contracts.Assert(indices.Length <= NumNodes);
+
+ foreach (int index in indices)
+ {
+ Contracts.Assert(index >= 0 && index < NumNodes);
+ categoricalSplit[index] = true;
+ }
+
+ return categoricalSplit;
+ }
+
  /// <summary>
  /// Create a Regression Tree object from raw tree contents.
  /// </summary>
@@ -192,7 +208,7 @@ internal RegressionTree(int[] splitFeatures, Double[] splitGain, Double[] gainPV
  LeafValues = leafValues;
  CategoricalSplitFeatures = categoricalSplitFeatures;
  CategoricalSplitFeatureRanges = new int[CategoricalSplitFeatures.Length][];
- for(int i= 0; i < CategoricalSplitFeatures.Length; ++i)
+ for(int i= 0; i < CategoricalSplitFeatures.Length; ++i)
  {
  if (CategoricalSplitFeatures[i] != null && CategoricalSplitFeatures[i].Length > 0)
  {
@@ -500,6 +516,7 @@ public virtual int SizeInBytes()
  NumNodes * sizeof(int) +
  CategoricalSplit.Length * sizeof(bool) +
  Thresholds.SizeInBytes() +
+ RawThresholds.SizeInBytes() +
  _splitGain.SizeInBytes() +
  _gainPValue.SizeInBytes() +
  _previousLeafValue.SizeInBytes() +
@@ -514,22 +531,22 @@ public virtual void ToByteArray(byte[] buffer, ref int position)
  LteChild.ToByteArray(buffer, ref position);
  GtChild.ToByteArray(buffer, ref position);
  SplitFeatures.ToByteArray(buffer, ref position);
+ CategoricalSplit.Length.ToByteArray(buffer, ref position);
  foreach (var split in CategoricalSplit)
  Convert.ToByte(split).ToByteArray(buffer, ref position);
 
  if (CategoricalSplitFeatures != null)
  {
- foreach (var splits in CategoricalSplitFeatures)
- splits.ToByteArray(buffer, ref position);
- }
-
- if (CategoricalSplitFeatureRanges != null)
- {
- foreach (var ranges in CategoricalSplitFeatureRanges)
- ranges.ToByteArray(buffer, ref position);
+ Contracts.AssertValue(CategoricalSplitFeatureRanges);
+ for (int i = 0; i < CategoricalSplitFeatures.Length; i++)
+ {
+ CategoricalSplitFeatures[i].ToByteArray(buffer, ref position);
+ CategoricalSplitFeatureRanges[i].ToByteArray(buffer, ref position);
+ }
  }
 
  Thresholds.ToByteArray(buffer, ref position);
+ RawThresholds.ToByteArray(buffer, ref position);
  _splitGain.ToByteArray(buffer, ref position);
  _gainPValue.ToByteArray(buffer, ref position);
  _previousLeafValue.ToByteArray(buffer, ref position);

src/Microsoft.ML.FastTree/TreeEnsemble/TreeEnsembleCombiner.cs

@@ -0,0 +1,115 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System.Collections.Generic;
+using Microsoft.ML.Runtime;
+using Microsoft.ML.Runtime.FastTree.Internal;
+using Microsoft.ML.Runtime.Internal.Calibration;
+
+[assembly: LoadableClass(typeof(TreeEnsembleCombiner), null, typeof(SignatureModelCombiner), "Fast Tree Model Combiner", "FastTreeCombiner")]
+
+namespace Microsoft.ML.Runtime.FastTree.Internal
+{
+ public sealed class TreeEnsembleCombiner : IModelCombiner<IPredictorProducing<float>, IPredictorProducing<float>>
+ {
+ private readonly IHost _host;
+ private readonly PredictionKind _kind;
+
+ public TreeEnsembleCombiner(IHostEnvironment env, PredictionKind kind)
+ {
+ _host = env.Register("TreeEnsembleCombiner");
+ switch (kind)
+ {
+ case PredictionKind.BinaryClassification:
+ case PredictionKind.Regression:
+ case PredictionKind.Ranking:
+ _kind = kind;
+ break;
+ default:
+ throw _host.ExceptUserArg(nameof(kind), $"Tree ensembles can be either of type {nameof(PredictionKind.BinaryClassification)}, " +
+ $"{nameof(PredictionKind.Regression)} or {nameof(PredictionKind.Ranking)}");
+ }
+ }
+
+ public IPredictorProducing<float> CombineModels(IEnumerable<IPredictorProducing<float>> models)
+ {
+ _host.CheckValue(models, nameof(models));
+
+ var ensemble = new Ensemble();
+ int modelCount = 0;
+ int featureCount = -1;
+ bool binaryClassifier = false;
+ foreach (var model in models)
+ {
+ modelCount++;
+
+ var predictor = model;
+ _host.CheckValue(predictor, nameof(models), "One of the models is null");
+
+ var calibrated = predictor as CalibratedPredictorBase;
+ double paramA = 1;
+ if (calibrated != null)
+ {
+ _host.Check(calibrated.Calibrator is PlattCalibrator,
+ "Combining FastTree models can only be done when the models are calibrated with Platt calibrator");
+ predictor = calibrated.SubPredictor;
+ paramA = -(calibrated.Calibrator as PlattCalibrator).ParamA;
+ }
+ var tree = predictor as FastTreePredictionWrapper;
+ if (tree == null)
+ throw _host.Except("Model is not a tree ensemble");
+ foreach (var t in tree.TrainedEnsemble.Trees)
+ {
+ var bytes = new byte[t.SizeInBytes()];
+ int position = -1;
+ t.ToByteArray(bytes, ref position);
+ position = -1;
+ var tNew = new RegressionTree(bytes, ref position);
+ if (paramA != 1)
+ {
+ for (int i = 0; i < tNew.NumLeaves; i++)
+ tNew.SetOutput(i, tNew.LeafValues[i] * paramA);
+ }
+ ensemble.AddTree(tNew);
+ }
+
+ if (modelCount == 1)
+ {
+ binaryClassifier = calibrated != null;
+ featureCount = tree.InputType.ValueCount;
+ }
+ else
+ {
+ _host.Check((calibrated != null) == binaryClassifier, "Ensemble contains both calibrated and uncalibrated models");
+ _host.Check(featureCount == tree.InputType.ValueCount, "Found models with different number of features");
+ }
+ }
+
+ var scale = 1 / (double)modelCount;
+
+ foreach (var t in ensemble.Trees)
+ {
+ for (int i = 0; i < t.NumLeaves; i++)
+ t.SetOutput(i, t.LeafValues[i] * scale);
+ }
+
+ switch (_kind)
+ {
+ case PredictionKind.BinaryClassification:
+ if (!binaryClassifier)
+ return new FastTreeBinaryPredictor(_host, ensemble, featureCount, null);
+
+ var cali = new PlattCalibrator(_host, -1, 0);
+ return new FeatureWeightsCalibratedPredictor(_host, new FastTreeBinaryPredictor(_host, ensemble, featureCount, null), cali);
+ case PredictionKind.Regression:
+ return new FastTreeRegressionPredictor(_host, ensemble, featureCount, null);
+ case PredictionKind.Ranking:
+ return new FastTreeRankingPredictor(_host, ensemble, featureCount, null);
+ default:
+ _host.Assert(false);
+ throw _host.ExceptNotSupp();
+ }
+ }
+ }
+}

src/Microsoft.ML.FastTree/Utils/ToByteArrayExtensions.cs

@@ -5,6 +5,7 @@
 using System;
 using System.Linq;
 using System.Text;
+using Microsoft.ML.Runtime.Internal.Utilities;
 
 namespace Microsoft.ML.Runtime.FastTree.Internal
 {
@@ -290,7 +291,7 @@ public static string ToString(this byte[] buffer, ref int position)
 
  public static int SizeInBytes(this byte[] a)
  {
- return sizeof(int) + a.Length * sizeof(byte);
+ return sizeof(int) + Utils.Size(a) * sizeof(byte);
  }
 
  public static void ToByteArray(this byte[] a, byte[] buffer, ref int position)
@@ -314,7 +315,7 @@ public static byte[] ToByteArray(this byte[] buffer, ref int position)
 
  public static int SizeInBytes(this short[] a)
  {
- return sizeof(int) + a.Length * sizeof(short);
+ return sizeof(int) + Utils.Size(a) * sizeof(short);
  }
 
  public unsafe static void ToByteArray(this short[] a, byte[] buffer, ref int position)
@@ -353,7 +354,7 @@ public unsafe static short[] ToShortArray(this byte[] buffer, ref int position)
 
  public static int SizeInBytes(this ushort[] a)
  {
- return sizeof(int) + a.Length * sizeof(ushort);
+ return sizeof(int) + Utils.Size(a) * sizeof(ushort);
  }
 
  public unsafe static void ToByteArray(this ushort[] a, byte[] buffer, ref int position)
@@ -392,12 +393,12 @@ public unsafe static ushort[] ToUShortArray(this byte[] buffer, ref int position
 
  public static int SizeInBytes(this int[] array)
  {
- return sizeof(int) + array.Length * sizeof(int);
+ return sizeof(int) + Utils.Size(array) * sizeof(int);
  }
 
  public unsafe static void ToByteArray(this int[] a, byte[] buffer, ref int position)
  {
- int length = a.Length;
+ int length = Utils.Size(a);
  length.ToByteArray(buffer, ref position);
 
  fixed (byte* tmpBuffer = buffer)
@@ -415,6 +416,9 @@ public unsafe static int[] ToIntArray(this byte[] buffer, ref int position)
 
  public unsafe static int[] ToIntArray(this byte[] buffer, ref int position, int length)
  {
+ if (length == 0)
+ return null;
+
  int[] a = new int[length];
 
  fixed (byte* tmpBuffer = buffer)
@@ -433,7 +437,7 @@ public unsafe static int[] ToIntArray(this byte[] buffer, ref int position, int
 
  public static int SizeInBytes(this uint[] array)
  {
- return sizeof(int) + array.Length * sizeof(uint);
+ return sizeof(int) + Utils.Size(array) * sizeof(uint);
  }
 
  public unsafe static void ToByteArray(this uint[] a, byte[] buffer, ref int position)
@@ -472,7 +476,7 @@ public unsafe static uint[] ToUIntArray(this byte[] buffer, ref int position)
 
  public static int SizeInBytes(this long[] array)
  {
- return sizeof(int) + array.Length * sizeof(long);
+ return sizeof(int) + Utils.Size(array) * sizeof(long);
  }
 
  public unsafe static void ToByteArray(this long[] a, byte[] buffer, ref int position)
@@ -511,7 +515,7 @@ public unsafe static long[] ToLongArray(this byte[] buffer, ref int position)
 
  public static int SizeInBytes(this ulong[] array)
  {
- return sizeof(int) + array.Length * sizeof(ulong);
+ return sizeof(int) + Utils.Size(array) * sizeof(ulong);
  }
 
  public unsafe static void ToByteArray(this ulong[] a, byte[] buffer, ref int position)
@@ -550,7 +554,7 @@ public unsafe static ulong[] ToULongArray(this byte[] buffer, ref int position)
 
  public static int SizeInBytes(this MD5Hash[] array)
  {
- return sizeof(int) + array.Length * MD5Hash.SizeInBytes();
+ return sizeof(int) + Utils.Size(array) * MD5Hash.SizeInBytes();
  }
 
  public static void ToByteArray(this MD5Hash[] a, byte[] buffer, ref int position)
@@ -577,7 +581,7 @@ public unsafe static MD5Hash[] ToUInt128Array(this byte[] buffer, ref int positi
 
  public static int SizeInBytes(this float[] array)
  {
- return sizeof(int) + array.Length * sizeof(float);
+ return sizeof(int) + Utils.Size(array) * sizeof(float);
  }
 
  public unsafe static void ToByteArray(this float[] a, byte[] buffer, ref int position)
@@ -616,7 +620,7 @@ public unsafe static float[] ToFloatArray(this byte[] buffer, ref int position)
 
  public static int SizeInBytes(this double[] array)
  {
- return sizeof(int) + array.Length * sizeof(double);
+ return sizeof(int) + Utils.Size(array) * sizeof(double);
  }
 
  public unsafe static void ToByteArray(this double[] a, byte[] buffer, ref int position)
@@ -655,6 +659,8 @@ public unsafe static double[] ToDoubleArray(this byte[] buffer, ref int position
 
  public static int SizeInBytes(this double[][] array)
  {
+ if (Utils.Size(array) == 0)
+ return sizeof(int);
  return sizeof(int) + array.Sum(x => x.SizeInBytes());
  }
 
@@ -683,7 +689,7 @@ public static double[][] ToDoubleJaggedArray(this byte[] buffer, ref int positio
  public static long SizeInBytes(this string[] array)
  {
  long length = sizeof(int);
- for (int i = 0; i < array.Length; ++i)
+ for (int i = 0; i < Utils.Size(array); ++i)
  {
  length += array[i].SizeInBytes();
  }
@@ -692,8 +698,8 @@ public static long SizeInBytes(this string[] array)
 
  public static void ToByteArray(this string[] a, byte[] buffer, ref int position)
  {
-a.Length.ToByteArray(buffer, ref position);
- for (int i = 0; i < a.Length; ++i)
+Utils.Size(a).ToByteArray(buffer, ref position);
+ for (int i = 0; i < Utils.Size(a); ++i)
  {
  a[i].ToByteArray(buffer, ref position);
  }

src/Microsoft.ML.Core/Prediction/ITrainer.cs

@@ -74,6 +74,8 @@ public interface IModelCombiner<TModel, TPredictor>
  TPredictor CombineModels(IEnumerable<TModel> models);
  }
 
+ public delegate void SignatureModelCombiner(PredictionKind kind);
+
  /// <summary>
  /// Weakly typed interface for a trainer "session" that produces a predictor.
  /// </summary>

src/Microsoft.ML.FastTree/Microsoft.ML.FastTree.csproj

@@ -57,6 +57,7 @@
  <Compile Include="TreeEnsemble\Ensemble.cs" />
  <Compile Include="TreeEnsemble\QuantileRegressionTree.cs" />
  <Compile Include="TreeEnsemble\RegressionTree.cs" />
+ <Compile Include="TreeEnsemble\TreeEnsembleCombiner.cs" />
  <Compile Include="Training\Applications\GradientWrappers.cs" />
  <Compile Include="Training\Applications\ObjectiveFunction.cs" />
  <Compile Include="Training\BaggingProvider.cs" />

src/Microsoft.ML.FastTree/TreeEnsemble/RegressionTree.cs

@@ -105,22 +105,21 @@ public RegressionTree(byte[] buffer, ref int position)
  LteChild = buffer.ToIntArray(ref position);
  GtChild = buffer.ToIntArray(ref position);
  SplitFeatures = buffer.ToIntArray(ref position);
-int[] categoricalNodeIndices = buffer.ToIntArray(ref position);
- CategoricalSplit = GetCategoricalSplitFromIndices(categoricalNodeIndices);
- if (categoricalNodeIndices?.Length > 0)
+byte[] categoricalSplitAsBytes = buffer.ToByteArray(ref position);
+ CategoricalSplit = categoricalSplitAsBytes.Select(b => b > 0).ToArray();
+ if (CategoricalSplit.Any(b => b))
  {
  CategoricalSplitFeatures = new int[NumNodes][];
  CategoricalSplitFeatureRanges = new int[NumNodes][];
-foreach (var index in categoricalNodeIndices)
+for (int index = 0; index < NumNodes; index++)
  {
- Contracts.Assert(CategoricalSplit[index]);
-
  CategoricalSplitFeatures[index] = buffer.ToIntArray(ref position);
- CategoricalSplitFeatureRanges[index] = buffer.ToIntArray(ref position, 2);
+ CategoricalSplitFeatureRanges[index] = buffer.ToIntArray(ref position);
  }
  }
 
  Thresholds = buffer.ToUIntArray(ref position);
+ RawThresholds = buffer.ToFloatArray(ref position);
  _splitGain = buffer.ToDoubleArray(ref position);
  _gainPValue = buffer.ToDoubleArray(ref position);
  _previousLeafValue = buffer.ToDoubleArray(ref position);
@@ -144,6 +143,23 @@ private bool[] GetCategoricalSplitFromIndices(int[] indices)
  return categoricalSplit;
  }
 
+ private bool[] GetCategoricalSplitFromBytes(byte[] indices)
+ {
+ bool[] categoricalSplit = new bool[NumNodes];
+ if (indices == null)
+ return categoricalSplit;
+
+ Contracts.Assert(indices.Length <= NumNodes);
+
+ foreach (int index in indices)
+ {
+ Contracts.Assert(index >= 0 && index < NumNodes);
+ categoricalSplit[index] = true;
+ }
+
+ return categoricalSplit;
+ }
+
  /// <summary>
  /// Create a Regression Tree object from raw tree contents.
  /// </summary>
@@ -192,7 +208,7 @@ internal RegressionTree(int[] splitFeatures, Double[] splitGain, Double[] gainPV
  LeafValues = leafValues;
  CategoricalSplitFeatures = categoricalSplitFeatures;
  CategoricalSplitFeatureRanges = new int[CategoricalSplitFeatures.Length][];
- for(int i= 0; i < CategoricalSplitFeatures.Length; ++i)
+ for(int i= 0; i < CategoricalSplitFeatures.Length; ++i)
  {
  if (CategoricalSplitFeatures[i] != null && CategoricalSplitFeatures[i].Length > 0)
  {
@@ -500,6 +516,7 @@ public virtual int SizeInBytes()
  NumNodes * sizeof(int) +
  CategoricalSplit.Length * sizeof(bool) +
  Thresholds.SizeInBytes() +
+ RawThresholds.SizeInBytes() +
  _splitGain.SizeInBytes() +
  _gainPValue.SizeInBytes() +
  _previousLeafValue.SizeInBytes() +
@@ -514,22 +531,22 @@ public virtual void ToByteArray(byte[] buffer, ref int position)
  LteChild.ToByteArray(buffer, ref position);
  GtChild.ToByteArray(buffer, ref position);
  SplitFeatures.ToByteArray(buffer, ref position);
+ CategoricalSplit.Length.ToByteArray(buffer, ref position);
  foreach (var split in CategoricalSplit)
  Convert.ToByte(split).ToByteArray(buffer, ref position);
 
  if (CategoricalSplitFeatures != null)
  {
- foreach (var splits in CategoricalSplitFeatures)
- splits.ToByteArray(buffer, ref position);
- }
-
- if (CategoricalSplitFeatureRanges != null)
- {
- foreach (var ranges in CategoricalSplitFeatureRanges)
- ranges.ToByteArray(buffer, ref position);
+ Contracts.AssertValue(CategoricalSplitFeatureRanges);
+ for (int i = 0; i < CategoricalSplitFeatures.Length; i++)
+ {
+ CategoricalSplitFeatures[i].ToByteArray(buffer, ref position);
+ CategoricalSplitFeatureRanges[i].ToByteArray(buffer, ref position);
+ }
  }
 
  Thresholds.ToByteArray(buffer, ref position);
+ RawThresholds.ToByteArray(buffer, ref position);
  _splitGain.ToByteArray(buffer, ref position);
  _gainPValue.ToByteArray(buffer, ref position);
  _previousLeafValue.ToByteArray(buffer, ref position);

src/Microsoft.ML.FastTree/TreeEnsemble/TreeEnsembleCombiner.cs

@@ -0,0 +1,115 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System.Collections.Generic;
+using Microsoft.ML.Runtime;
+using Microsoft.ML.Runtime.FastTree.Internal;
+using Microsoft.ML.Runtime.Internal.Calibration;
+
+[assembly: LoadableClass(typeof(TreeEnsembleCombiner), null, typeof(SignatureModelCombiner), "Fast Tree Model Combiner", "FastTreeCombiner")]
+
+namespace Microsoft.ML.Runtime.FastTree.Internal
+{
+ public sealed class TreeEnsembleCombiner : IModelCombiner<IPredictorProducing<float>, IPredictorProducing<float>>
+ {
+ private readonly IHost _host;
+ private readonly PredictionKind _kind;
+
+ public TreeEnsembleCombiner(IHostEnvironment env, PredictionKind kind)
+ {
+ _host = env.Register("TreeEnsembleCombiner");
+ switch (kind)
+ {
+ case PredictionKind.BinaryClassification:
+ case PredictionKind.Regression:
+ case PredictionKind.Ranking:
+ _kind = kind;
+ break;
+ default:
+ throw _host.ExceptUserArg(nameof(kind), $"Tree ensembles can be either of type {nameof(PredictionKind.BinaryClassification)}, " +
+ $"{nameof(PredictionKind.Regression)} or {nameof(PredictionKind.Ranking)}");
+ }
+ }
+
+ public IPredictorProducing<float> CombineModels(IEnumerable<IPredictorProducing<float>> models)
+ {
+ _host.CheckValue(models, nameof(models));
+
+ var ensemble = new Ensemble();
+ int modelCount = 0;
+ int featureCount = -1;
+ bool binaryClassifier = false;
+ foreach (var model in models)
+ {
+ modelCount++;
+
+ var predictor = model;
+ _host.CheckValue(predictor, nameof(models), "One of the models is null");
+
+ var calibrated = predictor as CalibratedPredictorBase;
+ double paramA = 1;
+ if (calibrated != null)
+ {
+ _host.Check(calibrated.Calibrator is PlattCalibrator,
+ "Combining FastTree models can only be done when the models are calibrated with Platt calibrator");
+ predictor = calibrated.SubPredictor;
+ paramA = -(calibrated.Calibrator as PlattCalibrator).ParamA;
+ }
+ var tree = predictor as FastTreePredictionWrapper;
+ if (tree == null)
+ throw _host.Except("Model is not a tree ensemble");
+ foreach (var t in tree.TrainedEnsemble.Trees)
+ {
+ var bytes = new byte[t.SizeInBytes()];
+ int position = -1;
+ t.ToByteArray(bytes, ref position);
+ position = -1;
+ var tNew = new RegressionTree(bytes, ref position);
+ if (paramA != 1)
+ {
+ for (int i = 0; i < tNew.NumLeaves; i++)
+ tNew.SetOutput(i, tNew.LeafValues[i] * paramA);
+ }
+ ensemble.AddTree(tNew);
+ }
+
+ if (modelCount == 1)
+ {
+ binaryClassifier = calibrated != null;
+ featureCount = tree.InputType.ValueCount;
+ }
+ else
+ {
+ _host.Check((calibrated != null) == binaryClassifier, "Ensemble contains both calibrated and uncalibrated models");
+ _host.Check(featureCount == tree.InputType.ValueCount, "Found models with different number of features");
+ }
+ }
+
+ var scale = 1 / (double)modelCount;
+
+ foreach (var t in ensemble.Trees)
+ {
+ for (int i = 0; i < t.NumLeaves; i++)
+ t.SetOutput(i, t.LeafValues[i] * scale);
+ }
+
+ switch (_kind)
+ {
+ case PredictionKind.BinaryClassification:
+ if (!binaryClassifier)
+ return new FastTreeBinaryPredictor(_host, ensemble, featureCount, null);
+
+ var cali = new PlattCalibrator(_host, -1, 0);
+ return new FeatureWeightsCalibratedPredictor(_host, new FastTreeBinaryPredictor(_host, ensemble, featureCount, null), cali);
+ case PredictionKind.Regression:
+ return new FastTreeRegressionPredictor(_host, ensemble, featureCount, null);
+ case PredictionKind.Ranking:
+ return new FastTreeRankingPredictor(_host, ensemble, featureCount, null);
+ default:
+ _host.Assert(false);
+ throw _host.ExceptNotSupp();
+ }
+ }
+ }
+}

src/Microsoft.ML.FastTree/Utils/ToByteArrayExtensions.cs

@@ -5,6 +5,7 @@
 using System;
 using System.Linq;
 using System.Text;
+using Microsoft.ML.Runtime.Internal.Utilities;
 
 namespace Microsoft.ML.Runtime.FastTree.Internal
 {
@@ -290,7 +291,7 @@ public static string ToString(this byte[] buffer, ref int position)
 
  public static int SizeInBytes(this byte[] a)
  {
- return sizeof(int) + a.Length * sizeof(byte);
+ return sizeof(int) + Utils.Size(a) * sizeof(byte);
  }
 
  public static void ToByteArray(this byte[] a, byte[] buffer, ref int position)
@@ -314,7 +315,7 @@ public static byte[] ToByteArray(this byte[] buffer, ref int position)
 
  public static int SizeInBytes(this short[] a)
  {
- return sizeof(int) + a.Length * sizeof(short);
+ return sizeof(int) + Utils.Size(a) * sizeof(short);
  }
 
  public unsafe static void ToByteArray(this short[] a, byte[] buffer, ref int position)
@@ -353,7 +354,7 @@ public unsafe static short[] ToShortArray(this byte[] buffer, ref int position)
 
  public static int SizeInBytes(this ushort[] a)
  {
- return sizeof(int) + a.Length * sizeof(ushort);
+ return sizeof(int) + Utils.Size(a) * sizeof(ushort);
  }
 
  public unsafe static void ToByteArray(this ushort[] a, byte[] buffer, ref int position)
@@ -392,12 +393,12 @@ public unsafe static ushort[] ToUShortArray(this byte[] buffer, ref int position
 
  public static int SizeInBytes(this int[] array)
  {
- return sizeof(int) + array.Length * sizeof(int);
+ return sizeof(int) + Utils.Size(array) * sizeof(int);
  }
 
  public unsafe static void ToByteArray(this int[] a, byte[] buffer, ref int position)
  {
- int length = a.Length;
+ int length = Utils.Size(a);
  length.ToByteArray(buffer, ref position);
 
  fixed (byte* tmpBuffer = buffer)
@@ -415,6 +416,9 @@ public unsafe static int[] ToIntArray(this byte[] buffer, ref int position)
 
  public unsafe static int[] ToIntArray(this byte[] buffer, ref int position, int length)
  {
+ if (length == 0)
+ return null;
+
  int[] a = new int[length];
 
  fixed (byte* tmpBuffer = buffer)
@@ -433,7 +437,7 @@ public unsafe static int[] ToIntArray(this byte[] buffer, ref int position, int
 
  public static int SizeInBytes(this uint[] array)
  {
- return sizeof(int) + array.Length * sizeof(uint);
+ return sizeof(int) + Utils.Size(array) * sizeof(uint);
  }
 
  public unsafe static void ToByteArray(this uint[] a, byte[] buffer, ref int position)
@@ -472,7 +476,7 @@ public unsafe static uint[] ToUIntArray(this byte[] buffer, ref int position)
 
  public static int SizeInBytes(this long[] array)
  {
- return sizeof(int) + array.Length * sizeof(long);
+ return sizeof(int) + Utils.Size(array) * sizeof(long);
  }
 
  public unsafe static void ToByteArray(this long[] a, byte[] buffer, ref int position)
@@ -511,7 +515,7 @@ public unsafe static long[] ToLongArray(this byte[] buffer, ref int position)
 
  public static int SizeInBytes(this ulong[] array)
  {
- return sizeof(int) + array.Length * sizeof(ulong);
+ return sizeof(int) + Utils.Size(array) * sizeof(ulong);
  }
 
  public unsafe static void ToByteArray(this ulong[] a, byte[] buffer, ref int position)
@@ -550,7 +554,7 @@ public unsafe static ulong[] ToULongArray(this byte[] buffer, ref int position)
 
  public static int SizeInBytes(this MD5Hash[] array)
  {
- return sizeof(int) + array.Length * MD5Hash.SizeInBytes();
+ return sizeof(int) + Utils.Size(array) * MD5Hash.SizeInBytes();
  }
 
  public static void ToByteArray(this MD5Hash[] a, byte[] buffer, ref int position)
@@ -577,7 +581,7 @@ public unsafe static MD5Hash[] ToUInt128Array(this byte[] buffer, ref int positi
 
  public static int SizeInBytes(this float[] array)
  {
- return sizeof(int) + array.Length * sizeof(float);
+ return sizeof(int) + Utils.Size(array) * sizeof(float);
  }
 
  public unsafe static void ToByteArray(this float[] a, byte[] buffer, ref int position)
@@ -616,7 +620,7 @@ public unsafe static float[] ToFloatArray(this byte[] buffer, ref int position)
 
  public static int SizeInBytes(this double[] array)
  {
- return sizeof(int) + array.Length * sizeof(double);
+ return sizeof(int) + Utils.Size(array) * sizeof(double);
  }
 
  public unsafe static void ToByteArray(this double[] a, byte[] buffer, ref int position)
@@ -655,6 +659,8 @@ public unsafe static double[] ToDoubleArray(this byte[] buffer, ref int position
 
  public static int SizeInBytes(this double[][] array)
  {
+ if (Utils.Size(array) == 0)
+ return sizeof(int);
  return sizeof(int) + array.Sum(x => x.SizeInBytes());
  }
 
@@ -683,7 +689,7 @@ public static double[][] ToDoubleJaggedArray(this byte[] buffer, ref int positio
  public static long SizeInBytes(this string[] array)
  {
  long length = sizeof(int);
- for (int i = 0; i < array.Length; ++i)
+ for (int i = 0; i < Utils.Size(array); ++i)
  {
  length += array[i].SizeInBytes();
  }
@@ -692,8 +698,8 @@ public static long SizeInBytes(this string[] array)
 
  public static void ToByteArray(this string[] a, byte[] buffer, ref int position)
  {
-a.Length.ToByteArray(buffer, ref position);
- for (int i = 0; i < a.Length; ++i)
+Utils.Size(a).ToByteArray(buffer, ref position);
+ for (int i = 0; i < Utils.Size(a); ++i)
  {
  a[i].ToByteArray(buffer, ref position);
  }