Accept TDigest bytes with buffered values

.editorconfig

@@ -0,0 +1,28 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+root = true
+
+[*]
+end_of_line = lf
+indent_style = space
+insert_final_newline = true
+trim_trailing_whitespace = true
+
+[*.java]
+indent_size = tab
+tab_width = 2

src/main/java/org/apache/datasketches/tdigest/TDigestDouble.java

@@ -426,7 +426,7 @@ public static TDigestDouble heapify(final MemorySegment seg, final boolean isFlo
  return new TDigestDouble(reverseMerge, k, value, value, new double[] {value}, new long[] {1}, 1, null);
  }
  final int numCentroids = posSeg.getInt();
- posSeg.getInt(); // unused
+final int numBuffered = posSeg.getInt();
  final double min;
  final double max;
  if (isFloat) {
@@ -444,7 +444,11 @@ public static TDigestDouble heapify(final MemorySegment seg, final boolean isFlo
  weights[i] = isFloat ? posSeg.getInt() : posSeg.getLong();
  totalWeight += weights[i];
  }
- return new TDigestDouble(reverseMerge, k, min, max, means, weights, totalWeight, null);
+ final double[] buffered = new double[numBuffered];
+ for (int i = 0; i < numBuffered; i++) {
+ buffered[i] = isFloat ? posSeg.getFloat() : posSeg.getDouble();
+ }
+ return new TDigestDouble(reverseMerge, k, min, max, means, weights, totalWeight, buffered);
  }
 
  // compatibility with the format of the reference implementation

src/test/java/org/apache/datasketches/tdigest/TDigestCrossLanguageTest.java

@@ -25,54 +25,68 @@
 import static org.apache.datasketches.common.TestUtil.javaPath;
 import static org.testng.Assert.assertEquals;
 import static org.testng.Assert.assertTrue;
-
-import java.lang.foreign.MemorySegment;
 import java.io.IOException;
+import java.lang.foreign.MemorySegment;
 import java.nio.file.Files;
-
 import org.testng.annotations.Test;
 
 public class TDigestCrossLanguageTest {
 
  @Test(groups = {CHECK_CPP_FILES})
  public void deserializeFromCppDouble() throws IOException {
- final int[] nArr = {0, 1, 10, 100, 1000, 10_000, 100_000, 1_000_000};
- for (final int n: nArr) {
- final byte[] bytes = Files.readAllBytes(cppPath.resolve("tdigest_double_n" + n + "_cpp.sk"));
- final TDigestDouble td = TDigestDouble.heapify(MemorySegment.ofArray(bytes));
- assertTrue(n == 0 ? td.isEmpty() : !td.isEmpty());
- assertEquals(td.getTotalWeight(), n);
- if (n > 0) {
- assertEquals(td.getMinValue(), 1);
- assertEquals(td.getMaxValue(), n);
- assertEquals(td.getRank(0), 0);
- assertEquals(td.getRank(n + 1), 1);
- if (n == 1) {
- assertEquals(td.getRank(n), 0.5);
+ final boolean[] with_buffer = {false, true};
+ for (final boolean buffered : with_buffer) {
+ final int[] nArr = {0, 1, 10, 100, 1000, 10_000, 100_000, 1_000_000};
+ for (final int n : nArr) {
+ final byte[] bytes;
+ if (buffered) {
+ bytes = Files.readAllBytes(cppPath.resolve("tdigest_double_buf_n" + n + "_cpp.sk"));
  } else {
- assertEquals(td.getRank(n / 2), 0.5, 0.05);
+ bytes = Files.readAllBytes(cppPath.resolve("tdigest_double_n" + n + "_cpp.sk"));
+ }
+ final TDigestDouble td = TDigestDouble.heapify(MemorySegment.ofArray(bytes));
+ assertTrue(n == 0 ? td.isEmpty() : !td.isEmpty());
+ assertEquals(td.getTotalWeight(), n);
+ if (n > 0) {
+ assertEquals(td.getMinValue(), 1);
+ assertEquals(td.getMaxValue(), n);
+ assertEquals(td.getRank(0), 0);
+ assertEquals(td.getRank(n + 1), 1);
+ if (n == 1) {
+ assertEquals(td.getRank(n), 0.5);
+ } else {
+ assertEquals(td.getRank(n / 2), 0.5, 0.05);
+ }
  }
  }
  }
  }
 
  @Test(groups = {CHECK_CPP_FILES})
  public void deserializeFromCppFloat() throws IOException {
+ final boolean[] with_buffer = {false, true};
  final int[] nArr = {0, 1, 10, 100, 1000, 10_000, 100_000, 1_000_000};
- for (final int n: nArr) {
- final byte[] bytes = Files.readAllBytes(cppPath.resolve("tdigest_float_n" + n + "_cpp.sk"));
- final TDigestDouble td = TDigestDouble.heapify(MemorySegment.ofArray(bytes), true);
- assertTrue(n == 0 ? td.isEmpty() : !td.isEmpty());
- assertEquals(td.getTotalWeight(), n);
- if (n > 0) {
- assertEquals(td.getMinValue(), 1);
- assertEquals(td.getMaxValue(), n);
- assertEquals(td.getRank(0), 0);
- assertEquals(td.getRank(n + 1), 1);
- if (n == 1) {
- assertEquals(td.getRank(n), 0.5);
+ for (final boolean buffered : with_buffer) {
+ for (final int n : nArr) {
+ final byte[] bytes;
+ if (buffered) {
+ bytes = Files.readAllBytes(cppPath.resolve("tdigest_float_buf_n" + n + "_cpp.sk"));
  } else {
- assertEquals(td.getRank(n / 2), 0.5, 0.05);
+ bytes = Files.readAllBytes(cppPath.resolve("tdigest_float_n" + n + "_cpp.sk"));
+ }
+ final TDigestDouble td = TDigestDouble.heapify(MemorySegment.ofArray(bytes), true);
+ assertTrue(n == 0 ? td.isEmpty() : !td.isEmpty());
+ assertEquals(td.getTotalWeight(), n);
+ if (n > 0) {
+ assertEquals(td.getMinValue(), 1);
+ assertEquals(td.getMaxValue(), n);
+ assertEquals(td.getRank(0), 0);
+ assertEquals(td.getRank(n + 1), 1);
+ if (n == 1) {
+ assertEquals(td.getRank(n), 0.5);
+ } else {
+ assertEquals(td.getRank(n / 2), 0.5, 0.05);
+ }
  }
  }
  }
@@ -81,7 +95,7 @@ public void deserializeFromCppFloat() throws IOException {
  @Test(groups = {GENERATE_JAVA_FILES})
  public void generateForCppDouble() throws IOException {
  final int[] nArr = {0, 1, 10, 100, 1000, 10_000, 100_000, 1_000_000};
- for (final int n: nArr) {
+ for (final int n: nArr) {
  final TDigestDouble td = new TDigestDouble((short) 100);
  for (int i = 1; i <= n; i++) {
  td.update(i);