From ed1f29cef051a29c65b46911db506140b8b7782d Mon Sep 17 00:00:00 2001 From: CritasWang Date: Wed, 26 Aug 2026 09:56:18 +0800 Subject: [PATCH] feat: support OBJECT data type in table-model tablet writes Add TSDataType.OBJECT (12) and table-model OBJECT write support: - Tablet.BuildObjectValue / SetObjectValueAt with the Java-compatible 9-byte segment header (isEOF + big-endian offset + content), including null-bit clearing after a previously null cell is overwritten - Serialize OBJECT columns like BLOB in EstimateBufferSize/GetBinaryValues - Accept OBJECT (and STRING/BLOB) in BinaryArrayColumnDecoder - Return "(Object) ..." summaries from RpcDataSet GetObject/GetString/GetRow and reject GetBinary for OBJECT columns - Add unit tests for segment framing, request bytes, null bitmap, read semantics, and size formatting Refs apache/iotdb-client-go#175 --- src/Apache.IoTDB/Client.cs | 1 + .../DataStructure/ColumnDecoder.cs | 5 +- src/Apache.IoTDB/DataStructure/RpcDataSet.cs | 13 ++ src/Apache.IoTDB/DataStructure/Tablet.cs | 54 +++++++ src/Apache.IoTDB/IoTDBConstants.cs | 1 + src/Apache.IoTDB/Utils.cs | 35 +++++ tests/Apache.IoTDB.Tests/RpcDataSetTests.cs | 96 ++++++++++++ tests/Apache.IoTDB.Tests/TabletObjectTests.cs | 139 ++++++++++++++++++ tests/Apache.IoTDB.Tests/UtilsTests.cs | 43 ++++++ 9 files changed, 386 insertions(+), 1 deletion(-) create mode 100644 tests/Apache.IoTDB.Tests/TabletObjectTests.cs diff --git a/src/Apache.IoTDB/Client.cs b/src/Apache.IoTDB/Client.cs index 943cfa1..d6879bf 100644 --- a/src/Apache.IoTDB/Client.cs +++ b/src/Apache.IoTDB/Client.cs @@ -50,6 +50,7 @@ static public TSDataType GetDataTypeByStr(string typeStr) "TEXT" => TSDataType.TEXT, "STRING" => TSDataType.STRING, "BLOB" => TSDataType.BLOB, + "OBJECT" => TSDataType.OBJECT, "TIMESTAMP" => TSDataType.TIMESTAMP, "DATE" => TSDataType.DATE, _ => TSDataType.NONE diff --git a/src/Apache.IoTDB/DataStructure/ColumnDecoder.cs b/src/Apache.IoTDB/DataStructure/ColumnDecoder.cs index f1267b5..219ba69 100644 --- a/src/Apache.IoTDB/DataStructure/ColumnDecoder.cs +++ b/src/Apache.IoTDB/DataStructure/ColumnDecoder.cs @@ -188,7 +188,10 @@ public class BinaryArrayColumnDecoder : ColumnDecoder { public Column ReadColumn(ByteBuffer reader, TSDataType dataType, int positionCount) { - if (dataType != TSDataType.TEXT) + if (dataType != TSDataType.TEXT + && dataType != TSDataType.STRING + && dataType != TSDataType.BLOB + && dataType != TSDataType.OBJECT) throw new ArgumentException($"Invalid data type: {dataType}"); bool[] nullIndicators = ColumnDeserializer.DeserializeNullIndicators(reader, positionCount); diff --git a/src/Apache.IoTDB/DataStructure/RpcDataSet.cs b/src/Apache.IoTDB/DataStructure/RpcDataSet.cs index 1cc7c79..369572c 100644 --- a/src/Apache.IoTDB/DataStructure/RpcDataSet.cs +++ b/src/Apache.IoTDB/DataStructure/RpcDataSet.cs @@ -495,6 +495,8 @@ public Binary GetBinary(string columnName) private Binary GetBinaryByTsBlockColumnIndex(int tsBlockColumnIndex) { CheckRecord(); + if (GetDataTypeByTsBlockColumnIndex(tsBlockColumnIndex) == TSDataType.OBJECT) + throw new InvalidOperationException("OBJECT type does not support GetBlob"); if (!IsNull(tsBlockColumnIndex, _tsBlockIndex)) { _lastReadWasNull = false; @@ -554,6 +556,10 @@ private object GetObjectByTsBlockIndex(int tsBlockColumnIndex) case TSDataType.BLOB: return _curTsBlock.GetColumn(tsBlockColumnIndex).GetBinary(_tsBlockIndex); + case TSDataType.OBJECT: + Binary objectBytes = _curTsBlock.GetColumn(tsBlockColumnIndex).GetBinary(_tsBlockIndex); + return Utils.ObjectBytesToString(objectBytes.Data); + case TSDataType.DATE: int value = _curTsBlock.GetColumn(tsBlockColumnIndex).GetInt(_tsBlockIndex); return Int32ToDate(value); @@ -635,6 +641,10 @@ private string GetStringByTsBlockColumnIndexAndDataType(int index, TSDataType ts Binary blobBytes = _curTsBlock.GetColumn(index).GetBinary(_tsBlockIndex); return blobBytes.ToString().Replace("-", ""); + case TSDataType.OBJECT: + Binary objectBytes = _curTsBlock.GetColumn(index).GetBinary(_tsBlockIndex); + return Utils.ObjectBytesToString(objectBytes.Data); + case TSDataType.DATE: int dateValue = _curTsBlock.GetColumn(index).GetInt(_tsBlockIndex); DateTime date = Int32ToDate(dateValue); @@ -701,6 +711,9 @@ public RowRecord GetRow() var binary = GetBinary(columnName); localfield = binary?.Data; break; + case TSDataType.OBJECT: + localfield = GetString(columnName); + break; case TSDataType.DATE: localfield = GetDate(columnName); break; diff --git a/src/Apache.IoTDB/DataStructure/Tablet.cs b/src/Apache.IoTDB/DataStructure/Tablet.cs index 471bff5..97b815b 100644 --- a/src/Apache.IoTDB/DataStructure/Tablet.cs +++ b/src/Apache.IoTDB/DataStructure/Tablet.cs @@ -297,6 +297,57 @@ public List GetColumnColumnCategories() return columnCategories; } + /// + /// Builds the wire representation of one OBJECT segment: a 1-byte isEOF + /// flag, an 8-byte big-endian offset, then the raw segment content. + /// This matches Java's + /// Tablet.addValue(rowIndex, columnIndex, isEOF, offset, content). + /// + public static byte[] BuildObjectValue(bool isEOF, long offset, byte[] content) + { + if (content == null) + throw new ArgumentNullException(nameof(content)); + if (offset < 0) + throw new ArgumentOutOfRangeException(nameof(offset), offset, "OBJECT segment offset must be non-negative."); + + var value = new byte[9 + content.Length]; + value[0] = isEOF ? (byte)1 : (byte)0; + value[1] = (byte)(offset >> 56); + value[2] = (byte)(offset >> 48); + value[3] = (byte)(offset >> 40); + value[4] = (byte)(offset >> 32); + value[5] = (byte)(offset >> 24); + value[6] = (byte)(offset >> 16); + value[7] = (byte)(offset >> 8); + value[8] = (byte)offset; + Array.Copy(content, 0, value, 9, content.Length); + return value; + } + + /// + /// Writes one segment of an OBJECT column value at an existing row. + /// + /// An OBJECT value can be written in multiple segments so that a large + /// object does not need to be fully loaded into memory. Segments must be + /// written with ascending offsets and the last segment must set isEOF + /// to true. + /// + public void SetObjectValueAt(bool isEOF, long offset, byte[] content, int columnIndex, int rowIndex) + { + if (columnIndex < 0 || columnIndex >= ColNumber) + throw new ArgumentOutOfRangeException(nameof(columnIndex), columnIndex, "Column index is out of range."); + if (rowIndex < 0 || rowIndex >= RowNumber) + throw new ArgumentOutOfRangeException(nameof(rowIndex), rowIndex, "Row index is out of range."); + if (DataTypes[columnIndex] != TSDataType.OBJECT) + throw new ArgumentException($"Column {columnIndex} must be of type OBJECT.", nameof(columnIndex)); + + _values[rowIndex][columnIndex] = BuildObjectValue(isEOF, offset, content); + if (BitMaps != null && BitMaps[columnIndex] != null) + { + BitMaps[columnIndex].unmark(rowIndex); + } + } + private int EstimateBufferSize() { var estimateSize = 0; @@ -326,6 +377,7 @@ private int EstimateBufferSize() case TSDataType.TEXT: case TSDataType.BLOB: case TSDataType.STRING: + case TSDataType.OBJECT: estimateSize += 8; break; default: @@ -441,7 +493,9 @@ public byte[] GetBinaryValues() break; } case TSDataType.BLOB: + case TSDataType.OBJECT: { + // OBJECT uses the same length-prefixed binary encoding as BLOB. for (int j = 0; j < RowNumber; j++) { var value = _values[j][i]; diff --git a/src/Apache.IoTDB/IoTDBConstants.cs b/src/Apache.IoTDB/IoTDBConstants.cs index 13fe80e..3c2bcbd 100644 --- a/src/Apache.IoTDB/IoTDBConstants.cs +++ b/src/Apache.IoTDB/IoTDBConstants.cs @@ -33,6 +33,7 @@ public enum TSDataType DATE = 9, BLOB = 10, STRING = 11, + OBJECT = 12, } public enum TSEncoding diff --git a/src/Apache.IoTDB/Utils.cs b/src/Apache.IoTDB/Utils.cs index 627314c..b728a20 100644 --- a/src/Apache.IoTDB/Utils.cs +++ b/src/Apache.IoTDB/Utils.cs @@ -19,6 +19,7 @@ using System; using System.Collections.Generic; +using System.Globalization; using System.Linq; using System.Text; @@ -121,5 +122,39 @@ public static string ByteArrayToHexString(byte[] bytes) { return "0x" + BitConverter.ToString(bytes).Replace("-", "").ToLowerInvariant(); } + + /// + /// Formats the wire representation of a stored OBJECT value for display. + /// The server stores OBJECT cells as an 8-byte big-endian file size + /// followed by the internal object path; this renders the size in + /// human-readable units (mirrors the Go client's objectBytesToString). + /// + public static string ObjectBytesToString(byte[] input) + { + if (input == null) + throw new ArgumentNullException(nameof(input)); + if (input.Length < 8) + throw new ArgumentException( + $"Invalid OBJECT value: expected at least 8 bytes, got {input.Length}.", + nameof(input)); + + ulong size = 0; + for (int i = 0; i < 8; i++) + { + size = (size << 8) | input[i]; + } + + const ulong kilobyte = 1024; + const ulong megabyte = kilobyte * 1024; + const ulong gigabyte = megabyte * 1024; + + if (size < kilobyte) + return $"(Object) {size} B"; + if (size < megabyte) + return string.Format(CultureInfo.InvariantCulture, "(Object) {0:F2} KB", (double)size / kilobyte); + if (size < gigabyte) + return string.Format(CultureInfo.InvariantCulture, "(Object) {0:F2} MB", (double)size / megabyte); + return string.Format(CultureInfo.InvariantCulture, "(Object) {0:F2} GB", (double)size / gigabyte); + } } } diff --git a/tests/Apache.IoTDB.Tests/RpcDataSetTests.cs b/tests/Apache.IoTDB.Tests/RpcDataSetTests.cs index cce3bb4..fc2edc5 100644 --- a/tests/Apache.IoTDB.Tests/RpcDataSetTests.cs +++ b/tests/Apache.IoTDB.Tests/RpcDataSetTests.cs @@ -184,5 +184,101 @@ public void GetRow_DataTypesMatchMeasurements() Assert.That(row0.DataTypes.Count, Is.EqualTo(row0.Values.Count), "DataTypes count should match Values count."); } + + private static byte[] BuildObjectTsBlockBytes() + { + var payload = new List(); + payload.AddRange(new byte[] { 0, 0, 0, 0, 0, 0, 0x04, 0x00 }); // size 1024 BE + payload.AddRange(System.Text.Encoding.UTF8.GetBytes("internal/path/1.bin")); + + var buf = new ByteBuffer(256); + buf.AddInt(1); // value column count + buf.AddByte((byte)TSDataType.OBJECT); // value column type + buf.AddInt(1); // position count + buf.AddByte((byte)ColumnEncoding.Int64Array); // time encoding + buf.AddByte((byte)ColumnEncoding.BinaryArray); // value encoding + + buf.AddByte(0); // time mayHaveNull + buf.AddLong(1000L); // timestamp + + buf.AddByte(0); // object mayHaveNull + buf.AddInt(payload.Count); + foreach (var b in payload) + { + buf.AddByte(b); + } + + return buf.GetBuffer(); + } + + private RpcDataSet CreateObjectDataSet() + { + var columnNames = new List { "file" }; + var columnTypes = new List { "OBJECT" }; + var columnNameIndex = new Dictionary { { "file", 0 } }; + var columnIndex2TsBlockColumnIndexList = new List { 0 }; + + return new RpcDataSet( + sql: "select file from object_table", + columnNameList: columnNames, + columnTypeList: columnTypes, + columnNameIndex: columnNameIndex, + ignoreTimestamp: false, + moreData: false, + queryId: 1, + statementId: 1, + client: null, + sessionId: 1, + queryResult: new List { BuildObjectTsBlockBytes() }, + fetchSize: 1024, + timeout: 10000, + zoneId: "UTC", + columnIndex2TsBlockColumnIndexList: columnIndex2TsBlockColumnIndexList + ); + } + + [Test] + public void GetObject_ObjectColumn_ReturnsFormattedSizeString() + { + var dataSet = CreateObjectDataSet(); + dataSet.Next(); + + Assert.That(dataSet.GetObject("file"), Is.EqualTo("(Object) 1.00 KB")); + // By-index APIs are 1-based with the implicit Time column at index 1, + // so the OBJECT column is index 2. + Assert.That(dataSet.GetObjectByIndex(2), Is.EqualTo("(Object) 1.00 KB")); + } + + [Test] + public void GetString_ObjectColumn_ReturnsFormattedSizeString() + { + var dataSet = CreateObjectDataSet(); + dataSet.Next(); + + Assert.That(dataSet.GetString("file"), Is.EqualTo("(Object) 1.00 KB")); + Assert.That(dataSet.GetStringByIndex(2), Is.EqualTo("(Object) 1.00 KB")); + } + + [Test] + public void GetRow_ObjectColumn_KeepsObjectTypeAndValue() + { + var dataSet = CreateObjectDataSet(); + dataSet.Next(); + var row = dataSet.GetRow(); + + Assert.That(row.Measurements, Does.Contain("file")); + Assert.That(row.DataTypes, Does.Contain(TSDataType.OBJECT)); + Assert.That(row.Values, Does.Contain("(Object) 1.00 KB")); + } + + [Test] + public void GetBinary_ObjectColumn_Throws() + { + var dataSet = CreateObjectDataSet(); + dataSet.Next(); + + Assert.Throws(() => dataSet.GetBinary("file")); + Assert.Throws(() => dataSet.GetBinaryByIndex(2)); + } } } diff --git a/tests/Apache.IoTDB.Tests/TabletObjectTests.cs b/tests/Apache.IoTDB.Tests/TabletObjectTests.cs new file mode 100644 index 0000000..8cd509b --- /dev/null +++ b/tests/Apache.IoTDB.Tests/TabletObjectTests.cs @@ -0,0 +1,139 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +using System; +using System.Collections.Generic; +using Apache.IoTDB.DataStructure; +using NUnit.Framework; + +namespace Apache.IoTDB.Tests +{ + [TestFixture] + public class TabletObjectTests + { + private static Tablet NewObjectTablet(int rows = 2) + { + var values = new List>(); + var timestamps = new List(); + for (int i = 0; i < rows; i++) + { + values.Add(new List { $"r{i}", null }); + timestamps.Add(i + 1); + } + + return new Tablet( + "object_table", + new List { "region_id", "file" }, + new List { ColumnCategory.TAG, ColumnCategory.FIELD }, + new List { TSDataType.STRING, TSDataType.OBJECT }, + values, + timestamps); + } + + [Test] + public void BuildObjectValue_PrependsIsEOFAndBigEndianOffset() + { + var whole = Tablet.BuildObjectValue(true, 0, new byte[] { 0x11, 0x22 }); + Assert.That(whole, Is.EqualTo(new byte[] { 1, 0, 0, 0, 0, 0, 0, 0, 0, 0x11, 0x22 })); + + var segment = Tablet.BuildObjectValue(false, 512, new byte[] { 0x33 }); + Assert.That(segment, Is.EqualTo(new byte[] { 0, 0, 0, 0, 0, 0, 0, 2, 0, 0x33 })); + } + + [Test] + public void BuildObjectValue_RejectsInvalidInputs() + { + Assert.Throws(() => Tablet.BuildObjectValue(true, 0, null)); + Assert.Throws(() => Tablet.BuildObjectValue(true, -1, new byte[] { 0x01 })); + } + + [Test] + public void SetObjectValueAt_WritesWholeAndSegmentedValues() + { + var tablet = NewObjectTablet(); + + tablet.SetObjectValueAt(true, 0, new byte[] { 0x11, 0x22 }, 1, 0); + tablet.SetObjectValueAt(false, 512, new byte[] { 0x33 }, 1, 1); + + var bytes = tablet.GetBinaryValues(); + var expected = new List(); + expected.AddRange(new byte[] { 0, 0, 0, 2, (byte)'r', (byte)'0' }); // "r0" + expected.AddRange(new byte[] { 0, 0, 0, 2, (byte)'r', (byte)'1' }); // "r1" + expected.AddRange(new byte[] { 0, 0, 0, 11 }); // object row 0 length + expected.AddRange(new byte[] { 1, 0, 0, 0, 0, 0, 0, 0, 0, 0x11, 0x22 }); + expected.AddRange(new byte[] { 0, 0, 0, 10 }); // object row 1 length + expected.AddRange(new byte[] { 0, 0, 0, 0, 0, 0, 0, 2, 0, 0x33 }); + // No trailing bitmap section on the first serialization: BitMaps is + // only allocated once a null cell is encountered. + + Assert.That(bytes, Is.EqualTo(expected.ToArray())); + } + + [Test] + public void SetObjectValueAt_ExposesObjectTypeCodeInRequest() + { + var tablet = NewObjectTablet(1); + tablet.SetObjectValueAt(true, 0, new byte[] { 0x01, 0x02, 0x03 }, 1, 0); + + var pool = new SessionPool("localhost", 6667); + var req = pool.GenInsertTabletReq(tablet, 1); + + Assert.That(req.Types, Is.EqualTo(new List { (int)TSDataType.STRING, (int)TSDataType.OBJECT })); + Assert.That(req.Size, Is.EqualTo(1)); + Assert.That(req.PrefixPath, Is.EqualTo("object_table")); + Assert.That(req.Values, Is.EqualTo(tablet.GetBinaryValues())); + } + + [Test] + public void SetObjectValueAt_ClearsPreviouslyMarkedNullBit() + { + var tablet = NewObjectTablet(1); + // First serialization marks the null OBJECT cell. + var withNull = tablet.GetBinaryValues(); + // Bitmap section: [STRING hasNull=0][OBJECT hasNull=1][bitmap 0x01]. + Assert.That(withNull[withNull.Length - 2], Is.EqualTo(1), "OBJECT column has one null"); + Assert.That(withNull[withNull.Length - 1], Is.EqualTo(0x01), "null bitmap marks row 0"); + + tablet.SetObjectValueAt(true, 0, new byte[] { 0x44 }, 1, 0); + var withoutNull = tablet.GetBinaryValues(); + Assert.That(withoutNull[withoutNull.Length - 1], Is.EqualTo(0), "OBJECT null bit cleared"); + } + + [Test] + public void SetObjectValueAt_RejectsNonObjectColumnAndOutOfRangeIndexes() + { + var tablet = NewObjectTablet(1); + + Assert.Throws( + () => tablet.SetObjectValueAt(true, 0, new byte[] { 0x01 }, 0, 0)); + Assert.Throws( + () => tablet.SetObjectValueAt(true, 0, new byte[] { 0x01 }, 1, -1)); + Assert.Throws( + () => tablet.SetObjectValueAt(true, 0, new byte[] { 0x01 }, -1, 0)); + Assert.Throws( + () => tablet.SetObjectValueAt(true, 0, new byte[] { 0x01 }, 1, 1)); + } + + [Test] + public void GetDataTypeByStr_ResolvesObject() + { + Assert.That(Client.GetDataTypeByStr("OBJECT"), Is.EqualTo(TSDataType.OBJECT)); + } + } +} diff --git a/tests/Apache.IoTDB.Tests/UtilsTests.cs b/tests/Apache.IoTDB.Tests/UtilsTests.cs index 8efe423..e78d96f 100644 --- a/tests/Apache.IoTDB.Tests/UtilsTests.cs +++ b/tests/Apache.IoTDB.Tests/UtilsTests.cs @@ -201,6 +201,49 @@ public void IsSorted_SingleElementList_ReturnsTrue() } } + [TestFixture] + public class ObjectBytesToStringTests : UtilsTests + { + private static byte[] ObjectValue(ulong size, byte[] suffix = null) + { + var bytes = new byte[8 + (suffix?.Length ?? 0)]; + for (int i = 0; i < 8; i++) + { + bytes[i] = (byte)(size >> (56 - 8 * i)); + } + suffix?.CopyTo(bytes, 8); + return bytes; + } + + [Test] + public void ObjectBytesToString_FormatsBytesKilobytesMegabytesAndGigabytes() + { + Assert.That(Utils.ObjectBytesToString(ObjectValue(1023)), Is.EqualTo("(Object) 1023 B")); + Assert.That(Utils.ObjectBytesToString(ObjectValue(1024)), Is.EqualTo("(Object) 1.00 KB")); + Assert.That(Utils.ObjectBytesToString(ObjectValue(1024 * 1024)), Is.EqualTo("(Object) 1.00 MB")); + Assert.That(Utils.ObjectBytesToString(ObjectValue(1024UL * 1024 * 1024)), Is.EqualTo("(Object) 1.00 GB")); + } + + [Test] + public void ObjectBytesToString_IgnoresPathSuffix() + { + var value = ObjectValue(1024, System.Text.Encoding.UTF8.GetBytes("internal/path/1.bin")); + Assert.That(Utils.ObjectBytesToString(value), Is.EqualTo("(Object) 1.00 KB")); + } + + [Test] + public void ObjectBytesToString_ShortValueThrows() + { + Assert.Throws(() => Utils.ObjectBytesToString(new byte[7])); + } + + [Test] + public void ObjectBytesToString_NullThrows() + { + Assert.Throws(() => Utils.ObjectBytesToString(null)); + } + } + [TestFixture] public class DateUtilsTests : UtilsTests {