From c4be0ecd11328a6f66f4c3676501e55d3767f6e7 Mon Sep 17 00:00:00 2001 From: Chenyang Sun Date: Wed, 13 May 2026 15:00:34 +0800 Subject: [PATCH 1/5] [refactor](storage) replace TypeInfo polymorphic hierarchy with FieldType (#63153) Replace TypeInfo* fields and parameters with FieldType throughout the storage layer, and add a static field_type_size(FieldType) helper that maps each type to sizeof(CppTypeTraits::CppType) so size lookups stay in sync with the existing traits machinery. (cherry picked from commit 7e5c941524b5b3c38748a22780c86e26838b5c16) --- .../delete/delete_bitmap_calculator.cpp | 6 +- be/src/storage/field.h | 11 +- .../bloom_filter_index_reader.cpp | 3 +- .../bloom_filter/bloom_filter_index_reader.h | 6 +- .../bloom_filter_index_writer.cpp | 34 +-- .../bloom_filter/bloom_filter_index_writer.h | 12 +- be/src/storage/index/index_writer.cpp | 13 +- .../storage/index/indexed_column_reader.cpp | 8 +- be/src/storage/index/indexed_column_reader.h | 5 +- .../storage/index/indexed_column_writer.cpp | 12 +- be/src/storage/index/indexed_column_writer.h | 8 +- .../index/inverted/inverted_index_reader.cpp | 38 ++- .../index/inverted/inverted_index_reader.h | 3 +- be/src/storage/index/primary_key_index.cpp | 8 +- be/src/storage/index/primary_key_index.h | 5 +- .../storage/index/zone_map/zone_map_index.cpp | 6 +- be/src/storage/merger.cpp | 2 +- be/src/storage/segment/column_reader.cpp | 18 +- be/src/storage/segment/column_reader.h | 12 +- be/src/storage/segment/column_writer.cpp | 18 +- be/src/storage/segment/encoding_info.h | 1 - be/src/storage/segment/segment.cpp | 12 +- be/src/storage/segment/segment_iterator.cpp | 2 +- be/src/storage/segment/segment_writer.cpp | 3 +- .../variant/variant_ext_meta_writer.cpp | 4 +- .../segment/vertical_segment_writer.cpp | 3 +- be/src/storage/tablet/base_tablet.cpp | 13 +- be/src/storage/tablet/tablet_schema.cpp | 2 +- be/src/storage/types.cpp | 203 ------------- be/src/storage/types.h | 286 +++--------------- .../index/ann/ann_index_smoke_test.cpp | 4 - .../util/index_compaction_utils.cpp | 6 +- .../storage/index/primary_key_index_test.cpp | 4 +- .../bloom_filter_index_reader_writer_test.cpp | 30 +- .../storage/segment/encoding_info_test.cpp | 36 +-- be/test/storage/storage_types_test.cpp | 12 +- 36 files changed, 195 insertions(+), 654 deletions(-) diff --git a/be/src/storage/delete/delete_bitmap_calculator.cpp b/be/src/storage/delete/delete_bitmap_calculator.cpp index 0dee5f8d4012f5..d3965d41f7bcfd 100644 --- a/be/src/storage/delete/delete_bitmap_calculator.cpp +++ b/be/src/storage/delete/delete_bitmap_calculator.cpp @@ -155,14 +155,12 @@ Status MergeIndexDeleteBitmapCalculator::init(RowsetId rowset_id, auto pk_idx = segment->get_primary_key_index(); std::unique_ptr index; RETURN_IF_ERROR(pk_idx->new_iterator(&index, nullptr)); - auto index_type = - DataTypeFactory::instance().create_data_type(pk_idx->type_info()->type(), 1, 0); + auto index_type = DataTypeFactory::instance().create_data_type(pk_idx->type(), 1, 0); _contexts.emplace_back(std::move(index), index_type, segment->id(), pk_idx->num_rows()); _heap->push(&_contexts.back()); } if (_rowid_length > 0) { - _rowid_coder = get_key_coder( - get_scalar_type_info()->type()); + _rowid_coder = get_key_coder(FieldType::OLAP_FIELD_TYPE_UNSIGNED_INT); } }); return Status::OK(); diff --git a/be/src/storage/field.h b/be/src/storage/field.h index 2b738e47a33b32..c1aedb777f3793 100644 --- a/be/src/storage/field.h +++ b/be/src/storage/field.h @@ -41,7 +41,7 @@ namespace doris { class StorageField { public: StorageField(const TabletColumn& column) - : _type_info(get_type_info(&column)), + : _type(column.type()), _desc(column), _length(column.length()), _key_coder(get_key_coder(column.type())), @@ -55,7 +55,7 @@ class StorageField { virtual ~StorageField() = default; - size_t size() const { return _type_info->size(); } + size_t size() const { return field_type_size(_type); } size_t length() const { return _length; } size_t field_size() const { return size() + 1; } size_t index_size() const { return _index_size; } @@ -71,8 +71,7 @@ class StorageField { return local; } - FieldType type() const { return _type_info->type(); } - const TypeInfo* type_info() const { return _type_info.get(); } + FieldType type() const { return _type; } bool is_nullable() const { return _is_nullable; } // similar to `full_encode_ascending`, but only encode part (the first `index_size` bytes) of the value. @@ -104,7 +103,7 @@ class StorageField { } protected: - TypeInfoPtr _type_info; + FieldType _type; TabletColumn _desc; // unit : byte // except for strings, other types have fixed lengths @@ -114,7 +113,7 @@ class StorageField { size_t _length; void clone(StorageField* other) const { - other->_type_info = clone_type_info(this->_type_info.get()); + other->_type = this->_type; other->_key_coder = this->_key_coder; other->_name = this->_name; other->_index_size = this->_index_size; diff --git a/be/src/storage/index/bloom_filter/bloom_filter_index_reader.cpp b/be/src/storage/index/bloom_filter/bloom_filter_index_reader.cpp index be029263067eda..393844f53ab26c 100644 --- a/be/src/storage/index/bloom_filter/bloom_filter_index_reader.cpp +++ b/be/src/storage/index/bloom_filter/bloom_filter_index_reader.cpp @@ -67,8 +67,7 @@ Status BloomFilterIndexReader::new_iterator(std::unique_ptr* bf) { size_t num_to_read = 1; - auto data_type = - DataTypeFactory::instance().create_data_type(_reader->type_info()->type(), 1, 0); + auto data_type = DataTypeFactory::instance().create_data_type(_reader->type(), 1, 0); auto column = data_type->create_column(); RETURN_IF_ERROR(_bloom_filter_iter.seek_to_ordinal(ordinal)); diff --git a/be/src/storage/index/bloom_filter/bloom_filter_index_reader.h b/be/src/storage/index/bloom_filter/bloom_filter_index_reader.h index a04ad8fdff8139..dc0b78f16c3c60 100644 --- a/be/src/storage/index/bloom_filter/bloom_filter_index_reader.h +++ b/be/src/storage/index/bloom_filter/bloom_filter_index_reader.h @@ -42,8 +42,7 @@ class BloomFilterIndexReader : public MetadataAdder { public: explicit BloomFilterIndexReader(io::FileReaderSPtr file_reader, const BloomFilterIndexPB& bloom_filter_index_meta) - : _file_reader(std::move(file_reader)), - _type_info(get_scalar_type_info()) { + : _file_reader(std::move(file_reader)) { _bloom_filter_index_meta.reset(new BloomFilterIndexPB(bloom_filter_index_meta)); } @@ -56,7 +55,7 @@ class BloomFilterIndexReader : public MetadataAdder { Status new_iterator(std::unique_ptr* iterator, OlapReaderStatistics* index_load_stats); - const TypeInfo* type_info() const { return _type_info; } + FieldType type() const { return FieldType::OLAP_FIELD_TYPE_VARCHAR; } private: Status _load(bool use_page_cache, bool kept_in_memory, OlapReaderStatistics* index_load_stats); @@ -68,7 +67,6 @@ class BloomFilterIndexReader : public MetadataAdder { io::FileReaderSPtr _file_reader; DorisCallOnce _load_once; - const TypeInfo* _type_info = nullptr; std::unique_ptr _bloom_filter_index_meta = nullptr; std::unique_ptr _bloom_filter_reader; }; diff --git a/be/src/storage/index/bloom_filter/bloom_filter_index_writer.cpp b/be/src/storage/index/bloom_filter/bloom_filter_index_writer.cpp index f169334ef02898..1c2d0fa908964c 100644 --- a/be/src/storage/index/bloom_filter/bloom_filter_index_writer.cpp +++ b/be/src/storage/index/bloom_filter/bloom_filter_index_writer.cpp @@ -67,9 +67,8 @@ class BloomFilterIndexWriterImpl : public BloomFilterIndexWriter { using CppType = typename CppTypeTraits::CppType; using ValueDict = typename BloomFilterTraits::ValueDict; - explicit BloomFilterIndexWriterImpl(const BloomFilterOptions& bf_options, - const TypeInfo* type_info) - : _bf_options(bf_options), _type_info(type_info) {} + explicit BloomFilterIndexWriterImpl(const BloomFilterOptions& bf_options) + : _bf_options(bf_options) {} ~BloomFilterIndexWriterImpl() override = default; @@ -130,12 +129,11 @@ class BloomFilterIndexWriterImpl : public BloomFilterIndexWriter { meta->set_algorithm(BLOCK_BLOOM_FILTER); // write bloom filters - const auto* bf_type_info = get_scalar_type_info(); IndexedColumnWriterOptions options; options.write_ordinal_index = true; options.write_value_index = false; options.encoding = PLAIN_ENCODING; - IndexedColumnWriter bf_writer(options, bf_type_info, file_writer); + IndexedColumnWriter bf_writer(options, FieldType::OLAP_FIELD_TYPE_VARCHAR, file_writer); RETURN_IF_ERROR(bf_writer.init()); for (auto& bf : _bfs) { Slice data(bf->data(), bf->size()); @@ -163,7 +161,6 @@ class BloomFilterIndexWriterImpl : public BloomFilterIndexWriter { private: BloomFilterOptions _bf_options {}; - const TypeInfo* _type_info = nullptr; Arena _arena; bool _has_null = false; uint64_t _bf_buffer_size = 0; @@ -223,12 +220,11 @@ Status PrimaryKeyBloomFilterIndexWriterImpl::finish(io::FileWriter* file_writer, meta->set_algorithm(BLOCK_BLOOM_FILTER); // write bloom filters - const auto* bf_type_info = get_scalar_type_info(); IndexedColumnWriterOptions options; options.write_ordinal_index = true; options.write_value_index = false; options.encoding = PLAIN_ENCODING; - IndexedColumnWriter bf_writer(options, bf_type_info, file_writer); + IndexedColumnWriter bf_writer(options, FieldType::OLAP_FIELD_TYPE_VARCHAR, file_writer); RETURN_IF_ERROR(bf_writer.init()); for (auto& bf : _bfs) { Slice data(bf->data(), bf->size()); @@ -281,12 +277,11 @@ Status NGramBloomFilterIndexWriterImpl::finish(io::FileWriter* file_writer, meta->set_algorithm(NGRAM_BLOOM_FILTER); // write bloom filters - const TypeInfo* bf_typeinfo = get_scalar_type_info(FieldType::OLAP_FIELD_TYPE_VARCHAR); IndexedColumnWriterOptions options; options.write_ordinal_index = true; options.write_value_index = false; options.encoding = PLAIN_ENCODING; - IndexedColumnWriter bf_writer(options, bf_typeinfo, file_writer); + IndexedColumnWriter bf_writer(options, FieldType::OLAP_FIELD_TYPE_VARCHAR, file_writer); RETURN_IF_ERROR(bf_writer.init()); for (auto& bf : _bfs) { Slice data(bf->data(), bf->size()); @@ -303,8 +298,7 @@ uint64_t NGramBloomFilterIndexWriterImpl::size() { } // TODO currently we don't support bloom filter index for tinyint/hll/float/double -Status BloomFilterIndexWriter::create(const BloomFilterOptions& bf_options, - const TypeInfo* type_info, +Status BloomFilterIndexWriter::create(const BloomFilterOptions& bf_options, FieldType type, std::unique_ptr* res) { DBUG_EXECUTE_IF("BloomFilterIndexWriter::create", { auto fpp = DebugPoints::instance()->get_debug_param_or_default( @@ -317,11 +311,10 @@ Status BloomFilterIndexWriter::create(const BloomFilterOptions& bf_options, } } }) - FieldType type = type_info->type(); switch (type) { -#define M(TYPE) \ - case TYPE: \ - res->reset(new BloomFilterIndexWriterImpl(bf_options, type_info)); \ +#define M(TYPE) \ + case TYPE: \ + res->reset(new BloomFilterIndexWriterImpl(bf_options)); \ break; M(FieldType::OLAP_FIELD_TYPE_BOOL) M(FieldType::OLAP_FIELD_TYPE_TINYINT) @@ -353,11 +346,9 @@ Status BloomFilterIndexWriter::create(const BloomFilterOptions& bf_options, return Status::OK(); } -Status NGramBloomFilterIndexWriterImpl::create(const BloomFilterOptions& bf_options, - const TypeInfo* typeinfo, uint8_t gram_size, - uint16_t gram_bf_size, +Status NGramBloomFilterIndexWriterImpl::create(const BloomFilterOptions& bf_options, FieldType type, + uint8_t gram_size, uint16_t gram_bf_size, std::unique_ptr* res) { - FieldType type = typeinfo->type(); switch (type) { case FieldType::OLAP_FIELD_TYPE_CHAR: case FieldType::OLAP_FIELD_TYPE_VARCHAR: @@ -373,9 +364,8 @@ Status NGramBloomFilterIndexWriterImpl::create(const BloomFilterOptions& bf_opti } Status PrimaryKeyBloomFilterIndexWriterImpl::create(const BloomFilterOptions& bf_options, - const TypeInfo* typeinfo, + FieldType type, std::unique_ptr* res) { - FieldType type = typeinfo->type(); switch (type) { case FieldType::OLAP_FIELD_TYPE_CHAR: case FieldType::OLAP_FIELD_TYPE_VARCHAR: diff --git a/be/src/storage/index/bloom_filter/bloom_filter_index_writer.h b/be/src/storage/index/bloom_filter/bloom_filter_index_writer.h index 0d4aecee850521..d6865b57a6eca5 100644 --- a/be/src/storage/index/bloom_filter/bloom_filter_index_writer.h +++ b/be/src/storage/index/bloom_filter/bloom_filter_index_writer.h @@ -28,12 +28,11 @@ #include "core/arena.h" #include "storage/index/bloom_filter/bloom_filter.h" #include "storage/itoken_extractor.h" +#include "storage/olap_common.h" #include "util/slice.h" namespace doris { -class TypeInfo; - namespace io { class FileWriter; } @@ -44,7 +43,7 @@ class ColumnIndexMetaPB; class BloomFilterIndexWriter { public: - static Status create(const BloomFilterOptions& bf_options, const TypeInfo* typeinfo, + static Status create(const BloomFilterOptions& bf_options, FieldType type, std::unique_ptr* res); BloomFilterIndexWriter() = default; @@ -81,7 +80,7 @@ class PrimaryKeyBloomFilterIndexWriterImpl : public BloomFilterIndexWriter { } }; - static Status create(const BloomFilterOptions& bf_options, const TypeInfo* typeinfo, + static Status create(const BloomFilterOptions& bf_options, FieldType type, std::unique_ptr* res); // This method may allocate large memory for bf, will return error // when memory is exhaused to prevent oom. @@ -107,9 +106,8 @@ class PrimaryKeyBloomFilterIndexWriterImpl : public BloomFilterIndexWriter { class NGramBloomFilterIndexWriterImpl : public BloomFilterIndexWriter { public: - static Status create(const BloomFilterOptions& bf_options, const TypeInfo* typeinfo, - uint8_t gram_size, uint16_t gram_bf_size, - std::unique_ptr* res); + static Status create(const BloomFilterOptions& bf_options, FieldType type, uint8_t gram_size, + uint16_t gram_bf_size, std::unique_ptr* res); NGramBloomFilterIndexWriterImpl(const BloomFilterOptions& bf_options, uint8_t gram_size, uint16_t bf_size); diff --git a/be/src/storage/index/index_writer.cpp b/be/src/storage/index/index_writer.cpp index 02afc229518556..e209caece4f6e4 100644 --- a/be/src/storage/index/index_writer.cpp +++ b/be/src/storage/index/index_writer.cpp @@ -49,8 +49,7 @@ bool IndexColumnWriter::check_support_ann_index(const TabletColumn& column) { Status IndexColumnWriter::create(const StorageField* field, std::unique_ptr* res, IndexFileWriter* index_file_writer, const TabletIndex* index_meta) { - const auto* typeinfo = field->type_info(); - FieldType type = typeinfo->type(); + FieldType type = field->type(); std::string field_name; auto storage_format = index_file_writer->get_storage_format(); if (storage_format == InvertedIndexStorageFormatPB::V1) { @@ -68,12 +67,12 @@ Status IndexColumnWriter::create(const StorageField* field, std::unique_ptris_inverted_index()) { bool single_field = true; if (type == FieldType::OLAP_FIELD_TYPE_ARRAY) { - const auto* array_typeinfo = dynamic_cast(typeinfo); + const auto& column = field->get_desc(); + bool has_item_subcolumn = column.get_subtype_count() > 0; DBUG_EXECUTE_IF("InvertedIndexColumnWriter::create_array_typeinfo_is_nullptr", - { array_typeinfo = nullptr; }) - if (array_typeinfo != nullptr) { - typeinfo = array_typeinfo->item_type_info(); - type = typeinfo->type(); + { has_item_subcolumn = false; }) + if (has_item_subcolumn) { + type = column.get_sub_column(0).type(); single_field = false; } else { return Status::NotSupported("unsupported array type for inverted index: " + diff --git a/be/src/storage/index/indexed_column_reader.cpp b/be/src/storage/index/indexed_column_reader.cpp index 04bcfc1a0f930c..b8fe9a57541a2e 100644 --- a/be/src/storage/index/indexed_column_reader.cpp +++ b/be/src/storage/index/indexed_column_reader.cpp @@ -64,12 +64,12 @@ Status IndexedColumnReader::load(bool use_page_cache, bool kept_in_memory, _use_page_cache = use_page_cache; _kept_in_memory = kept_in_memory; - _type_info = get_scalar_type_info((FieldType)_meta.data_type()); - if (_type_info == nullptr) { + _type = (FieldType)_meta.data_type(); + if (!is_scalar_type(_type)) { return Status::NotSupported("unsupported typeinfo, type={}", _meta.data_type()); } - RETURN_IF_ERROR(EncodingInfo::get(_type_info->type(), _meta.encoding(), {}, &_encoding_info)); - _value_key_coder = get_key_coder(_type_info->type()); + RETURN_IF_ERROR(EncodingInfo::get(_type, _meta.encoding(), {}, &_encoding_info)); + _value_key_coder = get_key_coder(_type); // read and parse ordinal index page when exists if (_meta.has_ordinal_index_meta()) { diff --git a/be/src/storage/index/indexed_column_reader.h b/be/src/storage/index/indexed_column_reader.h index 7d649b4f24179e..1cea4641595dc8 100644 --- a/be/src/storage/index/indexed_column_reader.h +++ b/be/src/storage/index/indexed_column_reader.h @@ -39,7 +39,6 @@ namespace doris { class KeyCoder; -class TypeInfo; class BlockCompressionCodec; namespace segment_v2 { @@ -67,7 +66,7 @@ class IndexedColumnReader : public MetadataAdder { int64_t num_values() const { return _num_values; } const EncodingInfo* encoding_info() const { return _encoding_info; } - const TypeInfo* type_info() const { return _type_info; } + FieldType type() const { return _type; } bool support_ordinal_seek() const { return _meta.has_ordinal_index_meta(); } bool support_value_seek() const { return _meta.has_value_index_meta(); } @@ -99,7 +98,7 @@ class IndexedColumnReader : public MetadataAdder { PageHandle _ordinal_index_page_handle; PageHandle _value_index_page_handle; - const TypeInfo* _type_info = nullptr; + FieldType _type = FieldType::OLAP_FIELD_TYPE_NONE; const EncodingInfo* _encoding_info = nullptr; const KeyCoder* _value_key_coder = nullptr; uint64_t _mem_size = 0; diff --git a/be/src/storage/index/indexed_column_writer.cpp b/be/src/storage/index/indexed_column_writer.cpp index 928164602211c6..0b0da38f812605 100644 --- a/be/src/storage/index/indexed_column_writer.cpp +++ b/be/src/storage/index/indexed_column_writer.cpp @@ -40,10 +40,10 @@ namespace doris { namespace segment_v2 { #include "common/compile_check_begin.h" -IndexedColumnWriter::IndexedColumnWriter(const IndexedColumnWriterOptions& options, - const TypeInfo* type_info, io::FileWriter* file_writer) +IndexedColumnWriter::IndexedColumnWriter(const IndexedColumnWriterOptions& options, FieldType type, + io::FileWriter* file_writer) : _options(options), - _type_info(type_info), + _type(type), _file_writer(file_writer), _num_values(0), _num_data_pages(0), @@ -55,7 +55,7 @@ IndexedColumnWriter::~IndexedColumnWriter() = default; Status IndexedColumnWriter::init() { const EncodingInfo* encoding_info; - RETURN_IF_ERROR(EncodingInfo::get(_type_info->type(), _options.encoding, {}, &encoding_info)); + RETURN_IF_ERROR(EncodingInfo::get(_type, _options.encoding, {}, &encoding_info)); _options.encoding = encoding_info->encoding(); // should store more concrete encoding type instead of DEFAULT_ENCODING // because the default encoding of a data type can be changed in the future @@ -73,7 +73,7 @@ Status IndexedColumnWriter::init() { } if (_options.write_value_index) { _value_index_builder.reset(new IndexPageBuilder(_options.index_page_size, true)); - _value_key_coder = get_key_coder(_type_info->type()); + _value_key_coder = get_key_coder(_type); } if (_options.compression != NO_COMPRESSION) { @@ -160,7 +160,7 @@ Status IndexedColumnWriter::finish(IndexedColumnMetaPB* meta) { if (_options.write_value_index) { RETURN_IF_ERROR(_flush_index(_value_index_builder.get(), meta->mutable_value_index_meta())); } - meta->set_data_type(int(_type_info->type())); + meta->set_data_type(int(_type)); meta->set_encoding(_options.encoding); meta->set_num_values(_num_values); meta->set_compression(_options.compression); diff --git a/be/src/storage/index/indexed_column_writer.h b/be/src/storage/index/indexed_column_writer.h index d201cf8fc897a8..98b45ac4c8c771 100644 --- a/be/src/storage/index/indexed_column_writer.h +++ b/be/src/storage/index/indexed_column_writer.h @@ -27,6 +27,7 @@ #include #include "common/status.h" +#include "storage/olap_common.h" #include "storage/segment/common.h" #include "storage/segment/page_pointer.h" @@ -34,7 +35,6 @@ namespace doris { class BlockCompressionCodec; class KeyCoder; -class TypeInfo; namespace io { class FileWriter; @@ -71,8 +71,8 @@ struct IndexedColumnWriterOptions { // TODO test with empty input class IndexedColumnWriter { public: - explicit IndexedColumnWriter(const IndexedColumnWriterOptions& options, - const TypeInfo* type_info, io::FileWriter* file_writer); + explicit IndexedColumnWriter(const IndexedColumnWriterOptions& options, FieldType type, + io::FileWriter* file_writer); ~IndexedColumnWriter(); @@ -93,7 +93,7 @@ class IndexedColumnWriter { Status _flush_index(IndexPageBuilder* index_builder, BTreeMetaPB* meta); IndexedColumnWriterOptions _options; - const TypeInfo* _type_info = nullptr; + FieldType _type; io::FileWriter* _file_writer = nullptr; ordinal_t _num_values; diff --git a/be/src/storage/index/inverted/inverted_index_reader.cpp b/be/src/storage/index/inverted/inverted_index_reader.cpp index b69dde206ca97a..bef44b4a78a72f 100644 --- a/be/src/storage/index/inverted/inverted_index_reader.cpp +++ b/be/src/storage/index/inverted/inverted_index_reader.cpp @@ -638,22 +638,20 @@ Status BkdIndexReader::construct_bkd_query_value(const Field& query_value, std::shared_ptr r, InvertedIndexVisitor* visitor) { if constexpr (QT == InvertedIndexQueryType::EQUAL_QUERY) { - RETURN_IF_ERROR(encode_bkd_field_ascending(_type_info->type(), query_value, - _value_key_coder, &visitor->query_max)); - RETURN_IF_ERROR(encode_bkd_field_ascending(_type_info->type(), query_value, - _value_key_coder, &visitor->query_min)); + RETURN_IF_ERROR(encode_bkd_field_ascending(_type, query_value, _value_key_coder, + &visitor->query_max)); + RETURN_IF_ERROR(encode_bkd_field_ascending(_type, query_value, _value_key_coder, + &visitor->query_min)); } else if constexpr (QT == InvertedIndexQueryType::LESS_THAN_QUERY || QT == InvertedIndexQueryType::LESS_EQUAL_QUERY) { - RETURN_IF_ERROR(encode_bkd_field_ascending(_type_info->type(), query_value, - _value_key_coder, &visitor->query_max)); - RETURN_IF_ERROR(encode_bkd_min_ascending(_type_info->type(), _value_key_coder, - &visitor->query_min)); + RETURN_IF_ERROR(encode_bkd_field_ascending(_type, query_value, _value_key_coder, + &visitor->query_max)); + RETURN_IF_ERROR(encode_bkd_min_ascending(_type, _value_key_coder, &visitor->query_min)); } else if constexpr (QT == InvertedIndexQueryType::GREATER_THAN_QUERY || QT == InvertedIndexQueryType::GREATER_EQUAL_QUERY) { - RETURN_IF_ERROR(encode_bkd_field_ascending(_type_info->type(), query_value, - _value_key_coder, &visitor->query_min)); - RETURN_IF_ERROR(encode_bkd_max_ascending(_type_info->type(), _value_key_coder, - &visitor->query_max)); + RETURN_IF_ERROR(encode_bkd_field_ascending(_type, query_value, _value_key_coder, + &visitor->query_min)); + RETURN_IF_ERROR(encode_bkd_max_ascending(_type, _value_key_coder, &visitor->query_max)); } else { return Status::Error( "invalid query type when query bkd index"); @@ -776,8 +774,8 @@ Status BkdIndexReader::try_query(const IndexQueryContextPtr& context, return st; } std::string query_str; - RETURN_IF_ERROR(encode_bkd_field_ascending(_type_info->type(), query_value, - _value_key_coder, &query_str)); + RETURN_IF_ERROR( + encode_bkd_field_ascending(_type, query_value, _value_key_coder, &query_str)); auto index_file_key = _index_file_reader->get_index_file_cache_key(&_index_meta); InvertedIndexQueryCache::CacheKey cache_key {index_file_key, column_name, query_type, @@ -816,8 +814,8 @@ Status BkdIndexReader::query(const IndexQueryContextPtr& context, const std::str return st; } std::string query_str; - RETURN_IF_ERROR(encode_bkd_field_ascending(_type_info->type(), query_value, - _value_key_coder, &query_str)); + RETURN_IF_ERROR( + encode_bkd_field_ascending(_type, query_value, _value_key_coder, &query_str)); auto index_file_key = _index_file_reader->get_index_file_cache_key(&_index_meta); InvertedIndexQueryCache::CacheKey cache_key {index_file_key, column_name, query_type, @@ -852,15 +850,15 @@ Status BkdIndexReader::get_bkd_reader(const IndexQueryContextPtr& context, auto searcher_variant = inverted_index_cache_handle.get_index_searcher(); bkd_searcher = std::get_if(&searcher_variant); if (bkd_searcher) { - _type_info = get_scalar_type_info((FieldType)(*bkd_searcher)->type); - if (_type_info == nullptr) { + _type = (FieldType)(*bkd_searcher)->type; + if (!is_scalar_type(_type)) { return Status::Error( "unsupported typeinfo, type={}", (*bkd_searcher)->type); } - _value_key_coder = get_key_coder(_type_info->type()); + _value_key_coder = get_key_coder(_type); bkd_reader = *bkd_searcher; if (bkd_reader->bytes_per_dim_ == 0) { - bkd_reader->bytes_per_dim_ = cast_set(_type_info->size()); + bkd_reader->bytes_per_dim_ = cast_set(field_type_size(_type)); } return Status::OK(); } diff --git a/be/src/storage/index/inverted/inverted_index_reader.h b/be/src/storage/index/inverted/inverted_index_reader.h index 38fd2e7cda40d6..0e2f6a120d41e3 100644 --- a/be/src/storage/index/inverted/inverted_index_reader.h +++ b/be/src/storage/index/inverted/inverted_index_reader.h @@ -66,7 +66,6 @@ class Roaring; namespace doris { class KeyCoder; -class TypeInfo; struct OlapReaderStatistics; class RuntimeState; @@ -393,7 +392,7 @@ class BkdIndexReader : public InvertedIndexReader { Status get_bkd_reader(const IndexQueryContextPtr& context, BKDIndexSearcherPtr& reader); private: - const TypeInfo* _type_info {}; + FieldType _type = FieldType::OLAP_FIELD_TYPE_NONE; const KeyCoder* _value_key_coder {}; }; diff --git a/be/src/storage/index/primary_key_index.cpp b/be/src/storage/index/primary_key_index.cpp index 6cda43f01d313d..d91f9e8f586ea6 100644 --- a/be/src/storage/index/primary_key_index.cpp +++ b/be/src/storage/index/primary_key_index.cpp @@ -37,21 +37,21 @@ static bvar::Adder g_primary_key_index_memory_bytes("doris_primary_key_i Status PrimaryKeyIndexBuilder::init() { // TODO(liaoxin) using the column type directly if there's only one column in unique key columns - const auto* type_info = get_scalar_type_info(); + constexpr FieldType type = FieldType::OLAP_FIELD_TYPE_VARCHAR; segment_v2::IndexedColumnWriterOptions options; options.write_ordinal_index = true; options.write_value_index = true; options.data_page_size = config::primary_key_data_page_size; - options.encoding = segment_v2::EncodingInfo::get_default_encoding(type_info->type(), {}, true); + options.encoding = segment_v2::EncodingInfo::get_default_encoding(type, {}, true); options.compression = segment_v2::ZSTD; _primary_key_index_builder.reset( - new segment_v2::IndexedColumnWriter(options, type_info, _file_writer)); + new segment_v2::IndexedColumnWriter(options, type, _file_writer)); RETURN_IF_ERROR(_primary_key_index_builder->init()); auto opt = segment_v2::BloomFilterOptions(); opt.fpp = 0.01; RETURN_IF_ERROR(segment_v2::PrimaryKeyBloomFilterIndexWriterImpl::create( - opt, type_info, &_bloom_filter_index_builder)); + opt, type, &_bloom_filter_index_builder)); return Status::OK(); } diff --git a/be/src/storage/index/primary_key_index.h b/be/src/storage/index/primary_key_index.h index 0bcb36c00f1dd3..89d8a0564932bc 100644 --- a/be/src/storage/index/primary_key_index.h +++ b/be/src/storage/index/primary_key_index.h @@ -35,7 +35,6 @@ namespace doris { #include "common/compile_check_begin.h" -class TypeInfo; namespace io { class FileWriter; @@ -122,9 +121,9 @@ class PrimaryKeyIndexReader { return Status::OK(); } - const TypeInfo* type_info() const { + FieldType type() const { DCHECK(_index_parsed); - return _index_reader->type_info(); + return _index_reader->type(); } // verify whether exist in BloomFilter diff --git a/be/src/storage/index/zone_map/zone_map_index.cpp b/be/src/storage/index/zone_map/zone_map_index.cpp index 3c3a7ba6ed3e5a..33f39bfb5b5ca7 100644 --- a/be/src/storage/index/zone_map/zone_map_index.cpp +++ b/be/src/storage/index/zone_map/zone_map_index.cpp @@ -279,14 +279,14 @@ Status TypedZoneMapIndexWriter::finish(io::FileWriter* file_writer, _segment_zone_map.to_proto(meta->mutable_segment_zone_map(), _data_type); // write out zone map for each data pages - const auto* type_info = get_scalar_type_info(); + constexpr FieldType type = FieldType::OLAP_FIELD_TYPE_BITMAP; IndexedColumnWriterOptions options; options.write_ordinal_index = true; options.write_value_index = false; - options.encoding = EncodingInfo::get_default_encoding(type_info->type(), {}, false); + options.encoding = EncodingInfo::get_default_encoding(type, {}, false); options.compression = NO_COMPRESSION; // currently not compressed - IndexedColumnWriter writer(options, type_info, file_writer); + IndexedColumnWriter writer(options, type, file_writer); RETURN_IF_ERROR(writer.init()); for (auto& value : _values) { diff --git a/be/src/storage/merger.cpp b/be/src/storage/merger.cpp index 5cd9ee5c20f7fa..f574f63f056f44 100644 --- a/be/src/storage/merger.cpp +++ b/be/src/storage/merger.cpp @@ -669,7 +669,7 @@ Status Merger::vertical_merge_rowsets(BaseTabletSPtr tablet, ReaderType reader_t // still calls ColumnNullable::insert_many_defaults() for null runs, // which grows the nested PODArray by N * type_size. So the runtime // per-row footprint is at least type_size, no matter how sparse. - int64_t type_size = get_type_info(&col)->size(); + int64_t type_size = field_type_size(col.type()); col_per_row = std::max(raw_per_row, type_size); if (col.is_nullable()) { col_per_row += 1; // null map diff --git a/be/src/storage/segment/column_reader.cpp b/be/src/storage/segment/column_reader.cpp index a5a98675e48b41..b3884609db4682 100644 --- a/be/src/storage/segment/column_reader.cpp +++ b/be/src/storage/segment/column_reader.cpp @@ -311,7 +311,7 @@ void ColumnReader::check_data_by_zone_map_for_test(const MutableColumnPtr& dst) return; } - FieldType type = _type_info->type(); + FieldType type = _type; if (type != FieldType::OLAP_FIELD_TYPE_INT) { return; @@ -347,16 +347,16 @@ void ColumnReader::check_data_by_zone_map_for_test(const MutableColumnPtr& dst) #endif Status ColumnReader::init(const ColumnMetaPB* meta) { - _type_info = get_type_info(meta); + _type = (FieldType)meta->type(); if (meta->has_be_exec_version()) { _be_exec_version = meta->be_exec_version(); } - if (_type_info == nullptr) { + if (_type == FieldType::OLAP_FIELD_TYPE_NONE || _type == FieldType::OLAP_FIELD_TYPE_UNKNOWN) { return Status::NotSupported("unsupported typeinfo, type={}", meta->type()); } - RETURN_IF_ERROR(EncodingInfo::get(_type_info->type(), meta->encoding(), {}, &_encoding_info)); + RETURN_IF_ERROR(EncodingInfo::get(_type, meta->encoding(), {}, &_encoding_info)); for (int i = 0; i < meta->indexes_size(); i++) { const auto& index_meta = meta->indexes(i); @@ -636,7 +636,7 @@ Status ColumnReader::_load_index(const std::shared_ptr& index_f if (_meta_type == FieldType::OLAP_FIELD_TYPE_ARRAY) { type = _meta_children_column_type; } else { - type = _type_info->type(); + type = _type; } if (index_meta->index_type() == IndexType::ANN) { @@ -2120,19 +2120,19 @@ Status DefaultValueColumnIterator::init(const ColumnIteratorOptions& opts) { if (_default_value == "NULL") { _default_value_field = Field::create_field(Null {}); } else { - if (_type_info->type() == FieldType::OLAP_FIELD_TYPE_ARRAY) { + if (_type == FieldType::OLAP_FIELD_TYPE_ARRAY) { if (_default_value != "[]") { return Status::NotSupported("Array default {} is unsupported", _default_value); } else { _default_value_field = Field::create_field(Array {}); return Status::OK(); } - } else if (_type_info->type() == FieldType::OLAP_FIELD_TYPE_STRUCT) { + } else if (_type == FieldType::OLAP_FIELD_TYPE_STRUCT) { return Status::NotSupported("STRUCT default type is unsupported"); - } else if (_type_info->type() == FieldType::OLAP_FIELD_TYPE_MAP) { + } else if (_type == FieldType::OLAP_FIELD_TYPE_MAP) { return Status::NotSupported("MAP default type is unsupported"); } - const auto t = _type_info->type(); + const auto t = _type; const auto serde = DataTypeFactory::instance() .create_data_type(t, _precision, _scale, _len) ->get_serde(); diff --git a/be/src/storage/segment/column_reader.h b/be/src/storage/segment/column_reader.h index f51f98504df272..0c33ed91de4046 100644 --- a/be/src/storage/segment/column_reader.h +++ b/be/src/storage/segment/column_reader.h @@ -287,9 +287,8 @@ class ColumnReader : public MetadataAdder, DataTypePtr _data_type; - TypeInfoPtr _type_info = - TypeInfoPtr(nullptr, - nullptr); // initialized in init(), may changed by subclasses. + FieldType _type = + FieldType::OLAP_FIELD_TYPE_NONE; // initialized in init(), may changed by subclasses. const EncodingInfo* _encoding_info = nullptr; // initialized in init(), used for create PageDecoder @@ -755,12 +754,11 @@ class RowIdColumnIteratorV2 : public ColumnIterator { class DefaultValueColumnIterator : public ColumnIterator { public: DefaultValueColumnIterator(bool has_default_value, const std::string& default_value, - bool is_nullable, TypeInfoPtr type_info, int precision, int scale, - int len) + bool is_nullable, FieldType type, int precision, int scale, int len) : _has_default_value(has_default_value), _default_value(default_value), _is_nullable(is_nullable), - _type_info(std::move(type_info)), + _type(type), _precision(precision), _scale(scale), _len(len) {} @@ -794,7 +792,7 @@ class DefaultValueColumnIterator : public ColumnIterator { bool _has_default_value; std::string _default_value; bool _is_nullable; - TypeInfoPtr _type_info; + FieldType _type; int _precision; int _scale; const int _len; diff --git a/be/src/storage/segment/column_writer.cpp b/be/src/storage/segment/column_writer.cpp index 7643ba8a48ab3b..f09ccc294b1cd2 100644 --- a/be/src/storage/segment/column_writer.cpp +++ b/be/src/storage/segment/column_writer.cpp @@ -132,7 +132,7 @@ inline ScalarColumnWriter* get_null_writer(const ColumnWriterOptions& opts, null_options.meta->set_type(int(null_type)); null_options.meta->set_is_nullable(false); null_options.meta->set_length( - cast_set(get_scalar_type_info()->size())); + cast_set(field_type_size(FieldType::OLAP_FIELD_TYPE_TINYINT))); null_options.meta->set_encoding(DEFAULT_ENCODING); null_options.meta->set_compression(opts.meta->compression()); @@ -211,8 +211,8 @@ Status ColumnWriter::create_array_writer(const ColumnWriterOptions& opts, length_options.meta->set_unique_id(2); length_options.meta->set_type(int(length_type)); length_options.meta->set_is_nullable(false); - length_options.meta->set_length(cast_set( - get_scalar_type_info()->size())); + length_options.meta->set_length( + cast_set(field_type_size(FieldType::OLAP_FIELD_TYPE_UNSIGNED_BIGINT))); length_options.meta->set_encoding(DEFAULT_ENCODING); length_options.meta->set_compression(opts.meta->compression()); @@ -275,8 +275,8 @@ Status ColumnWriter::create_map_writer(const ColumnWriterOptions& opts, const Ta length_options.meta->set_unique_id(column->get_subtype_count() + 1); length_options.meta->set_type(int(length_type)); length_options.meta->set_is_nullable(false); - length_options.meta->set_length(cast_set( - get_scalar_type_info()->size())); + length_options.meta->set_length( + cast_set(field_type_size(FieldType::OLAP_FIELD_TYPE_UNSIGNED_BIGINT))); length_options.meta->set_encoding(DEFAULT_ENCODING); length_options.meta->set_compression(opts.meta->compression()); @@ -577,11 +577,11 @@ Status ScalarColumnWriter::init() { if (_opts.need_bloom_filter) { if (_opts.is_ngram_bf_index) { RETURN_IF_ERROR(NGramBloomFilterIndexWriterImpl::create( - BloomFilterOptions(), get_field()->type_info(), _opts.gram_size, - _opts.gram_bf_size, &_bloom_filter_index_builder)); + BloomFilterOptions(), get_field()->type(), _opts.gram_size, _opts.gram_bf_size, + &_bloom_filter_index_builder)); } else { - RETURN_IF_ERROR(BloomFilterIndexWriter::create( - _opts.bf_options, get_field()->type_info(), &_bloom_filter_index_builder)); + RETURN_IF_ERROR(BloomFilterIndexWriter::create(_opts.bf_options, get_field()->type(), + &_bloom_filter_index_builder)); } } return Status::OK(); diff --git a/be/src/storage/segment/encoding_info.h b/be/src/storage/segment/encoding_info.h index 1f3a5372922560..3ecf817a42ca62 100644 --- a/be/src/storage/segment/encoding_info.h +++ b/be/src/storage/segment/encoding_info.h @@ -30,7 +30,6 @@ namespace doris { -class TypeInfo; enum class FieldType; namespace segment_v2 { diff --git a/be/src/storage/segment/segment.cpp b/be/src/storage/segment/segment.cpp index ef4c34b4f95613..574248725e581f 100644 --- a/be/src/storage/segment/segment.cpp +++ b/be/src/storage/segment/segment.cpp @@ -652,10 +652,9 @@ Status Segment::new_default_iterator(const TabletColumn& tablet_column, "column_type={}", tablet_column.unique_id(), tablet_column.name(), tablet_column.type()); } - auto type_info = get_type_info(&tablet_column); std::unique_ptr default_value_iter(new DefaultValueColumnIterator( tablet_column.has_default_value(), tablet_column.default_value(), - tablet_column.is_nullable(), std::move(type_info), tablet_column.precision(), + tablet_column.is_nullable(), tablet_column.type(), tablet_column.precision(), tablet_column.frac(), tablet_column.length())); ColumnIteratorOptions iter_opts; @@ -889,8 +888,7 @@ Status Segment::lookup_row_key(const Slice& key, const TabletSchema* latest_sche row_location->rowset_id = _rowset_id; size_t num_to_read = 1; - auto index_type = DataTypeFactory::instance().create_data_type( - _pk_index_reader->type_info()->type(), 1, 0); + auto index_type = DataTypeFactory::instance().create_data_type(_pk_index_reader->type(), 1, 0); auto index_column = index_type->create_column(); size_t num_read = num_to_read; RETURN_IF_ERROR(index_iterator->next_batch(&num_read, index_column)); @@ -937,8 +935,7 @@ Status Segment::lookup_row_key(const Slice& key, const TabletSchema* latest_sche Slice rowid_slice = Slice(sought_key.get_data() + sought_key_without_seq.get_size() + (segment_has_seq_col ? seq_col_length : 0) + 1, rowid_length - 1); - const auto* type_info = get_scalar_type_info(); - const auto* rowid_coder = get_key_coder(type_info->type()); + const auto* rowid_coder = get_key_coder(FieldType::OLAP_FIELD_TYPE_UNSIGNED_INT); RETURN_IF_ERROR(rowid_coder->decode_ascending(&rowid_slice, rowid_length, (uint8_t*)&row_location->row_id)); } @@ -962,8 +959,7 @@ Status Segment::read_key_by_rowid(uint32_t row_id, std::string* key) { std::unique_ptr iter; RETURN_IF_ERROR(_pk_index_reader->new_iterator(&iter, null_stat)); - auto index_type = DataTypeFactory::instance().create_data_type( - _pk_index_reader->type_info()->type(), 1, 0); + auto index_type = DataTypeFactory::instance().create_data_type(_pk_index_reader->type(), 1, 0); auto index_column = index_type->create_column(); RETURN_IF_ERROR(iter->seek_to_ordinal(row_id)); size_t num_read = 1; diff --git a/be/src/storage/segment/segment_iterator.cpp b/be/src/storage/segment/segment_iterator.cpp index fb336f8b25beb3..95a0850e90b7a1 100644 --- a/be/src/storage/segment/segment_iterator.cpp +++ b/be/src/storage/segment/segment_iterator.cpp @@ -1793,7 +1793,7 @@ Status SegmentIterator::_lookup_ordinal_from_pk_index(const RowCursor& key, bool .length() + 1; auto index_type = DataTypeFactory::instance().create_data_type( - _segment->_pk_index_reader->type_info()->type(), 1, 0); + _segment->_pk_index_reader->type(), 1, 0); auto index_column = index_type->create_column(); size_t num_to_read = 1; size_t num_read = num_to_read; diff --git a/be/src/storage/segment/segment_writer.cpp b/be/src/storage/segment/segment_writer.cpp index 81540c1dd03529..82c0894fa4a5bb 100644 --- a/be/src/storage/segment/segment_writer.cpp +++ b/be/src/storage/segment/segment_writer.cpp @@ -125,8 +125,7 @@ SegmentWriter::SegmentWriter(io::FileWriter* file_writer, uint32_t segment_id, } // encode the rowid into the primary key index if (_is_mow_with_cluster_key()) { - const auto* type_info = get_scalar_type_info(); - _rowid_coder = get_key_coder(type_info->type()); + _rowid_coder = get_key_coder(FieldType::OLAP_FIELD_TYPE_UNSIGNED_INT); // primary keys _primary_key_coders.swap(_key_coders); // cluster keys diff --git a/be/src/storage/segment/variant/variant_ext_meta_writer.cpp b/be/src/storage/segment/variant/variant_ext_meta_writer.cpp index 1a1a0619fcc743..1509ac6ef2a710 100644 --- a/be/src/storage/segment/variant/variant_ext_meta_writer.cpp +++ b/be/src/storage/segment/variant/variant_ext_meta_writer.cpp @@ -39,8 +39,8 @@ Status VariantExtMetaWriter::_ensure_inited(Writers* w) { dict_opts.write_ordinal_index = true; dict_opts.encoding = PREFIX_ENCODING; dict_opts.compression = _comp; - const TypeInfo* dict_type = get_scalar_type_info(); - w->key_writer = std::make_unique(dict_opts, dict_type, _fw); + w->key_writer = std::make_unique(dict_opts, + FieldType::OLAP_FIELD_TYPE_VARCHAR, _fw); RETURN_IF_ERROR(w->key_writer->init()); w->inited = true; diff --git a/be/src/storage/segment/vertical_segment_writer.cpp b/be/src/storage/segment/vertical_segment_writer.cpp index 670119dcba9e30..4de6a79d232888 100644 --- a/be/src/storage/segment/vertical_segment_writer.cpp +++ b/be/src/storage/segment/vertical_segment_writer.cpp @@ -134,8 +134,7 @@ VerticalSegmentWriter::VerticalSegmentWriter(io::FileWriter* file_writer, uint32 } // encode the rowid into the primary key index if (_is_mow_with_cluster_key()) { - const auto* type_info = get_scalar_type_info(); - _rowid_coder = get_key_coder(type_info->type()); + _rowid_coder = get_key_coder(FieldType::OLAP_FIELD_TYPE_UNSIGNED_INT); // primary keys _primary_key_coders.swap(_key_coders); // cluster keys diff --git a/be/src/storage/tablet/base_tablet.cpp b/be/src/storage/tablet/base_tablet.cpp index fbdc5544044c2a..28724b4d7f4820 100644 --- a/be/src/storage/tablet/base_tablet.cpp +++ b/be/src/storage/tablet/base_tablet.cpp @@ -129,8 +129,7 @@ Status parse_compaction_output_pk_entry( Slice rowid_slice(encoded_key.get_data() + unique_key_length + seq_col_length + 1, rowid_length - 1); - const auto* type_info = get_scalar_type_info(); - const auto* rowid_coder = get_key_coder(type_info->type()); + const auto* rowid_coder = get_key_coder(FieldType::OLAP_FIELD_TYPE_UNSIGNED_INT); uint32_t row_id = 0; RETURN_IF_ERROR(rowid_coder->decode_ascending(&rowid_slice, rowid_length, reinterpret_cast(&row_id))); @@ -726,8 +725,7 @@ Status BaseTablet::calc_segment_delete_bitmap(RowsetSharedPtr rowset, RETURN_IF_ERROR(pk_idx->new_iterator(&iter, nullptr)); size_t num_to_read = std::min(batch_size, remaining); - auto index_type = - DataTypeFactory::instance().create_data_type(pk_idx->type_info()->type(), 1, 0); + auto index_type = DataTypeFactory::instance().create_data_type(pk_idx->type(), 1, 0); auto index_column = index_type->create_column(); Slice last_key_slice(last_key); RETURN_IF_ERROR(iter->seek_at_or_after(&last_key_slice, &exact_match)); @@ -765,9 +763,7 @@ Status BaseTablet::calc_segment_delete_bitmap(RowsetSharedPtr rowset, Slice rowid_slice = Slice(key.get_data() + key_without_seq.get_size() + seq_col_length + 1, rowid_length - 1); - const auto* type_info = - get_scalar_type_info(); - const auto* rowid_coder = get_key_coder(type_info->type()); + const auto* rowid_coder = get_key_coder(FieldType::OLAP_FIELD_TYPE_UNSIGNED_INT); RETURN_IF_ERROR(rowid_coder->decode_ascending(&rowid_slice, rowid_length, (uint8_t*)&row_id)); } @@ -1821,8 +1817,7 @@ Status BaseTablet::calc_compaction_output_rowset_internal_delete_bitmap( auto scanner = std::make_unique(); scanner->segment_id = segment_id; scanner->remaining = pk_index->num_rows(); - scanner->index_type = - DataTypeFactory::instance().create_data_type(pk_index->type_info()->type(), 1, 0); + scanner->index_type = DataTypeFactory::instance().create_data_type(pk_index->type(), 1, 0); RETURN_IF_ERROR(pk_index->new_iterator(&scanner->iter, nullptr)); RETURN_IF_ERROR(load_next_compaction_output_pk_entry( output_rowset->rowset_id(), seq_col_length, output_row_sources, scanner.get())); diff --git a/be/src/storage/tablet/tablet_schema.cpp b/be/src/storage/tablet/tablet_schema.cpp index 1bfc89ace185db..5984458d563526 100644 --- a/be/src/storage/tablet/tablet_schema.cpp +++ b/be/src/storage/tablet/tablet_schema.cpp @@ -552,7 +552,7 @@ TabletColumn::TabletColumn(FieldAggregationMethod agg, FieldType type) { TabletColumn::TabletColumn(FieldAggregationMethod agg, FieldType filed_type, bool is_nullable) { _aggregation = agg; _type = filed_type; - _length = cast_set(get_scalar_type_info(filed_type)->size()); + _length = cast_set(field_type_size(filed_type)); _is_nullable = is_nullable; } diff --git a/be/src/storage/types.cpp b/be/src/storage/types.cpp index e2137efc3825c3..eb2ccd3c96bfe1 100644 --- a/be/src/storage/types.cpp +++ b/be/src/storage/types.cpp @@ -17,17 +17,8 @@ #include "storage/types.h" -#include - -#include - -#include "common/compiler_util.h" // IWYU pragma: keep -#include "storage/tablet/tablet_schema.h" - namespace doris { -static TypeInfoPtr create_type_info_ptr(const TypeInfo* type_info, bool should_reclaim_memory); - bool is_scalar_type(FieldType field_type) { switch (field_type) { case FieldType::OLAP_FIELD_TYPE_STRUCT: @@ -41,198 +32,4 @@ bool is_scalar_type(FieldType field_type) { } } -bool is_olap_string_type(FieldType field_type) { - switch (field_type) { - case FieldType::OLAP_FIELD_TYPE_CHAR: - case FieldType::OLAP_FIELD_TYPE_VARCHAR: - case FieldType::OLAP_FIELD_TYPE_HLL: - case FieldType::OLAP_FIELD_TYPE_BITMAP: - case FieldType::OLAP_FIELD_TYPE_STRING: - case FieldType::OLAP_FIELD_TYPE_JSONB: - return true; - default: - return false; - } -} - -const TypeInfo* get_scalar_type_info(FieldType field_type) { - // nullptr means that there is no TypeInfo implementation for the corresponding field_type - static const TypeInfo* field_type_array[] = { - nullptr, - get_scalar_type_info(), - nullptr, - get_scalar_type_info(), - nullptr, - get_scalar_type_info(), - get_scalar_type_info(), - get_scalar_type_info(), - get_scalar_type_info(), - get_scalar_type_info(), - get_scalar_type_info(), - get_scalar_type_info(), - nullptr, - get_scalar_type_info(), - get_scalar_type_info(), - get_scalar_type_info(), - get_scalar_type_info(), - get_scalar_type_info(), - nullptr, - nullptr, - nullptr, - nullptr, - nullptr, - get_scalar_type_info(), - get_scalar_type_info(), - get_scalar_type_info(), - get_scalar_type_info(), - get_scalar_type_info(), - get_scalar_type_info(), - get_scalar_type_info(), - get_scalar_type_info(), - get_scalar_type_info(), - get_scalar_type_info(), - get_scalar_type_info(), - get_scalar_type_info(), - get_scalar_type_info(), - get_scalar_type_info(), - get_scalar_type_info(), - get_scalar_type_info(), - get_scalar_type_info(), - get_scalar_type_info(), - nullptr}; - return field_type_array[int(field_type)]; -} - -template -const ArrayTypeInfo* get_init_array_type_info(int32_t iterations) { - static ArrayTypeInfo nested_type_info_0( - create_static_type_info_ptr(get_scalar_type_info())); - static ArrayTypeInfo nested_type_info_1(create_static_type_info_ptr(&nested_type_info_0)); - static ArrayTypeInfo nested_type_info_2(create_static_type_info_ptr(&nested_type_info_1)); - static ArrayTypeInfo nested_type_info_3(create_static_type_info_ptr(&nested_type_info_2)); - static ArrayTypeInfo nested_type_info_4(create_static_type_info_ptr(&nested_type_info_3)); - static ArrayTypeInfo nested_type_info_5(create_static_type_info_ptr(&nested_type_info_4)); - static ArrayTypeInfo nested_type_info_6(create_static_type_info_ptr(&nested_type_info_5)); - static ArrayTypeInfo nested_type_info_7(create_static_type_info_ptr(&nested_type_info_6)); - static ArrayTypeInfo nested_type_info_8(create_static_type_info_ptr(&nested_type_info_7)); - static ArrayTypeInfo* nested_type_info_array[] = { - &nested_type_info_0, &nested_type_info_1, &nested_type_info_2, - &nested_type_info_3, &nested_type_info_4, &nested_type_info_5, - &nested_type_info_6, &nested_type_info_7, &nested_type_info_8}; - return nested_type_info_array[iterations]; -} - -// Produce a struct type info -// TODO(xy): Need refactor to this produce method -const TypeInfo* get_struct_type_info(std::vector field_types) { - std::vector type_infos; - type_infos.reserve(field_types.size()); - for (FieldType& type : field_types) { - if (is_scalar_type(type)) { - type_infos.push_back(create_static_type_info_ptr(get_scalar_type_info(type))); - } else { - // TODO(xy): Not supported nested complex type now - } - } - return new StructTypeInfo(type_infos); -} - -// TODO: Support the type info of the nested array with more than 9 depths. -// TODO(xy): Support the type info of the nested struct -TypeInfoPtr get_type_info(const segment_v2::ColumnMetaPB* column_meta_pb) { - FieldType type = (FieldType)column_meta_pb->type(); - if (UNLIKELY(type == FieldType::OLAP_FIELD_TYPE_STRUCT)) { - std::vector field_types; - for (uint32_t i = 0; i < column_meta_pb->children_columns_size(); i++) { - const auto* child_column = &column_meta_pb->children_columns(i); - field_types.push_back((FieldType)child_column->type()); - } - return create_dynamic_type_info_ptr(get_struct_type_info(field_types)); - } else if (UNLIKELY(type == FieldType::OLAP_FIELD_TYPE_ARRAY)) { - segment_v2::ColumnMetaPB child_column = column_meta_pb->children_columns(0); - TypeInfoPtr child_info = get_type_info(&child_column); - ArrayTypeInfo* array_type_info = new ArrayTypeInfo(std::move(child_info)); - return create_dynamic_type_info_ptr(array_type_info); - } else if (UNLIKELY(type == FieldType::OLAP_FIELD_TYPE_MAP)) { - segment_v2::ColumnMetaPB key_meta = column_meta_pb->children_columns(0); - TypeInfoPtr key_type_info = get_type_info(&key_meta); - segment_v2::ColumnMetaPB value_meta = column_meta_pb->children_columns(1); - TypeInfoPtr value_type_info = get_type_info(&value_meta); - - MapTypeInfo* map_type_info = - new MapTypeInfo(std::move(key_type_info), std::move(value_type_info)); - return create_dynamic_type_info_ptr(map_type_info); - } else { - return create_static_type_info_ptr(get_scalar_type_info(type)); - } -} - -TypeInfoPtr create_static_type_info_ptr(const TypeInfo* type_info) { - return create_type_info_ptr(type_info, false); -} - -TypeInfoPtr create_dynamic_type_info_ptr(const TypeInfo* type_info) { - return create_type_info_ptr(type_info, true); -} - -TypeInfoPtr create_type_info_ptr(const TypeInfo* type_info, bool should_reclaim_memory) { - if (!should_reclaim_memory) { - return TypeInfoPtr(type_info, [](const TypeInfo*) {}); - } else { - return TypeInfoPtr(type_info, [](const TypeInfo* type_info) { delete type_info; }); - } -} - -// TODO: Support the type info of the nested array with more than 9 depths. -TypeInfoPtr get_type_info(const TabletColumn* col) { - auto type = col->type(); - if (UNLIKELY(type == FieldType::OLAP_FIELD_TYPE_STRUCT)) { - std::vector field_types; - for (uint32_t i = 0; i < col->get_subtype_count(); i++) { - const auto* child_column = &col->get_sub_column(i); - field_types.push_back(child_column->type()); - } - return create_dynamic_type_info_ptr(get_struct_type_info(field_types)); - } else if (UNLIKELY(type == FieldType::OLAP_FIELD_TYPE_ARRAY)) { - const auto* child_column = &col->get_sub_column(0); - TypeInfoPtr item_type = get_type_info(child_column); - ArrayTypeInfo* array_type_info = new ArrayTypeInfo(std::move(item_type)); - return create_dynamic_type_info_ptr(array_type_info); - } else if (UNLIKELY(type == FieldType::OLAP_FIELD_TYPE_MAP)) { - const auto* key_column = &col->get_sub_column(0); - TypeInfoPtr key_type = get_type_info(key_column); - const auto* val_column = &col->get_sub_column(1); - TypeInfoPtr value_type = get_type_info(val_column); - MapTypeInfo* map_type_info = new MapTypeInfo(std::move(key_type), std::move(value_type)); - return create_dynamic_type_info_ptr(map_type_info); - } else { - return create_static_type_info_ptr(get_scalar_type_info(type)); - } -} - -TypeInfoPtr clone_type_info(const TypeInfo* type_info) { - auto type = type_info->type(); - if (UNLIKELY(type == FieldType::OLAP_FIELD_TYPE_MAP)) { - const auto map_type_info = dynamic_cast(type_info); - return create_dynamic_type_info_ptr( - new MapTypeInfo(clone_type_info(map_type_info->get_key_type_info()), - clone_type_info(map_type_info->get_value_type_info()))); - } else if (UNLIKELY(type == FieldType::OLAP_FIELD_TYPE_STRUCT)) { - const auto struct_type_info = dynamic_cast(type_info); - std::vector clone_type_infos; - const std::vector* sub_type_infos = struct_type_info->type_infos(); - clone_type_infos.reserve(sub_type_infos->size()); - for (size_t i = 0; i < sub_type_infos->size(); i++) { - clone_type_infos.push_back(clone_type_info((*sub_type_infos)[i].get())); - } - return create_dynamic_type_info_ptr(new StructTypeInfo(clone_type_infos)); - } else if (UNLIKELY(type == FieldType::OLAP_FIELD_TYPE_ARRAY)) { - const auto array_type_info = dynamic_cast(type_info); - return create_dynamic_type_info_ptr( - new ArrayTypeInfo(clone_type_info(array_type_info->item_type_info()))); - } else { - return create_static_type_info_ptr(type_info); - } -} - } // namespace doris diff --git a/be/src/storage/types.h b/be/src/storage/types.h index 19dc5a5aad3303..89510f4ff6b79e 100644 --- a/be/src/storage/types.h +++ b/be/src/storage/types.h @@ -52,231 +52,8 @@ namespace doris { #include "common/compile_check_begin.h" -namespace segment_v2 { -class ColumnMetaPB; -} - -class TabletColumn; - -class TypeInfo; - -using TypeInfoPtr = std::unique_ptr; - -TypeInfoPtr create_static_type_info_ptr(const TypeInfo* type_info); -TypeInfoPtr create_dynamic_type_info_ptr(const TypeInfo* type_info); - -class TypeInfo { -public: - virtual ~TypeInfo() = default; - virtual int cmp(const void* left, const void* right) const = 0; - - virtual size_t size() const = 0; - - virtual FieldType type() const = 0; -}; - -class ScalarTypeInfo : public TypeInfo { -public: - int cmp(const void* left, const void* right) const override { return _cmp(left, right); } - - size_t size() const override { return _size; } - - FieldType type() const override { return _field_type; } - - template - ScalarTypeInfo(TypeTraitsClass t) - : _cmp(TypeTraitsClass::cmp), - _size(TypeTraitsClass::size), - _field_type(TypeTraitsClass::type) {} - -private: - int (*_cmp)(const void* left, const void* right); - - const size_t _size; - const FieldType _field_type; - - friend class ScalarTypeInfoResolver; -}; - -class ArrayTypeInfo : public TypeInfo { -public: - explicit ArrayTypeInfo(TypeInfoPtr item_type_info) - : _item_type_info(std::move(item_type_info)), _item_size(_item_type_info->size()) {} - ~ArrayTypeInfo() override = default; - - int cmp(const void* left, const void* right) const override { - auto l_value = reinterpret_cast(left); - auto r_value = reinterpret_cast(right); - size_t l_length = l_value->length(); - size_t r_length = r_value->length(); - size_t cur = 0; - - if (!l_value->has_null() && !r_value->has_null()) { - while (cur < l_length && cur < r_length) { - int result = _item_type_info->cmp((uint8_t*)(l_value->data()) + cur * _item_size, - (uint8_t*)(r_value->data()) + cur * _item_size); - if (result != 0) { - return result; - } - ++cur; - } - } else { - while (cur < l_length && cur < r_length) { - if (l_value->is_null_at(cur)) { - if (!r_value->is_null_at(cur)) { // left is null & right is not null - return -1; - } - } else if (r_value->is_null_at(cur)) { // left is not null & right is null - return 1; - } else { // both are not null - int result = - _item_type_info->cmp((uint8_t*)(l_value->data()) + cur * _item_size, - (uint8_t*)(r_value->data()) + cur * _item_size); - if (result != 0) { - return result; - } - } - ++cur; - } - } - - if (l_length < r_length) { - return -1; - } else if (l_length > r_length) { - return 1; - } else { - return 0; - } - } - - size_t size() const override { return sizeof(CollectionValue); } - - FieldType type() const override { return FieldType::OLAP_FIELD_TYPE_ARRAY; } - - inline const TypeInfo* item_type_info() const { return _item_type_info.get(); } - -private: - TypeInfoPtr _item_type_info; - const size_t _item_size; -}; -///====================== MapType Info ==========================/// -class MapTypeInfo : public TypeInfo { -public: - explicit MapTypeInfo(TypeInfoPtr key_type_info, TypeInfoPtr value_type_info) - : _key_type_info(std::move(key_type_info)), - _value_type_info(std::move(value_type_info)) {} - ~MapTypeInfo() override = default; - - int cmp(const void* left, const void* right) const override { - auto l_value = reinterpret_cast(left); - auto r_value = reinterpret_cast(right); - uint32_t l_size = l_value->size(); - uint32_t r_size = r_value->size(); - if (l_size < r_size) { - return -1; - } else if (l_size > r_size) { - return 1; - } else { - // now we use collection value in array to pack map k-v - auto l_k = reinterpret_cast(l_value->key_data()); - auto l_v = reinterpret_cast(l_value->value_data()); - auto r_k = reinterpret_cast(r_value->key_data()); - auto r_v = reinterpret_cast(r_value->value_data()); - auto key_arr = new ArrayTypeInfo(create_static_type_info_ptr(_key_type_info.get())); - auto val_arr = new ArrayTypeInfo(create_static_type_info_ptr(_value_type_info.get())); - if (int kc = key_arr->cmp(l_k, r_k) != 0) { - return kc; - } else { - return val_arr->cmp(l_v, r_v); - } - } - } - - size_t size() const override { return sizeof(MapValue); } - - FieldType type() const override { return FieldType::OLAP_FIELD_TYPE_MAP; } - - inline const TypeInfo* get_key_type_info() const { return _key_type_info.get(); } - inline const TypeInfo* get_value_type_info() const { return _value_type_info.get(); } - -private: - TypeInfoPtr _key_type_info; - TypeInfoPtr _value_type_info; -}; - -class StructTypeInfo : public TypeInfo { -public: - explicit StructTypeInfo(std::vector& type_infos) { - for (TypeInfoPtr& type_info : type_infos) { - _type_infos.push_back(std::move(type_info)); - } - } - ~StructTypeInfo() override = default; - - int cmp(const void* left, const void* right) const override { - auto l_value = reinterpret_cast(left); - auto r_value = reinterpret_cast(right); - uint32_t l_size = l_value->size(); - uint32_t r_size = r_value->size(); - uint32_t cur = 0; - - if (!l_value->has_null() && !r_value->has_null()) { - while (cur < l_size && cur < r_size) { - int result = - _type_infos[cur]->cmp(l_value->child_value(cur), r_value->child_value(cur)); - if (result != 0) { - return result; - } - ++cur; - } - } else { - while (cur < l_size && cur < r_size) { - if (l_value->is_null_at(cur)) { - if (!r_value->is_null_at(cur)) { // left is null & right is not null - return -1; - } - } else if (r_value->is_null_at(cur)) { // left is not null & right is null - return 1; - } else { // both are not null - int result = _type_infos[cur]->cmp(l_value->child_value(cur), - r_value->child_value(cur)); - if (result != 0) { - return result; - } - } - ++cur; - } - } - - if (l_size < r_size) { - return -1; - } else if (l_size > r_size) { - return 1; - } else { - return 0; - } - } - - size_t size() const override { return sizeof(StructValue); } - - FieldType type() const override { return FieldType::OLAP_FIELD_TYPE_STRUCT; } - - inline const std::vector* type_infos() const { return &_type_infos; } - -private: - std::vector _type_infos; -}; - bool is_scalar_type(FieldType field_type); -const TypeInfo* get_scalar_type_info(FieldType field_type); - -TypeInfoPtr get_type_info(const segment_v2::ColumnMetaPB* column_meta_pb); - -TypeInfoPtr get_type_info(const TabletColumn* col); - -TypeInfoPtr clone_type_info(const TypeInfo* type_info); - // support following formats when convert varchar to date static const std::vector DATE_FORMATS { "%Y-%m-%d", "%y-%m-%d", "%Y%m%d", "%y%m%d", "%Y/%m/%d", "%y/%m/%d", @@ -596,24 +373,51 @@ struct TypeTraits : public FieldTypeTraits { static const int32_t size = sizeof(CppType); }; -template -const TypeInfo* get_scalar_type_info() { - static constexpr TypeTraits traits; - static ScalarTypeInfo scalar_type_info(traits); - return &scalar_type_info; -} - -template -inline const TypeInfo* get_collection_type_info() { - static ArrayTypeInfo collection_type_info( - create_static_type_info_ptr(get_scalar_type_info())); - return &collection_type_info; -} - -// nested array type is unsupported for sub_type of collection -template <> -inline const TypeInfo* get_collection_type_info() { - return nullptr; +inline size_t field_type_size(FieldType field_type) { + switch (field_type) { +#define DORIS_FIELD_TYPE_SIZE_CASE(ft) \ + case FieldType::ft: \ + return sizeof(typename CppTypeTraits::CppType); + DORIS_FIELD_TYPE_SIZE_CASE(OLAP_FIELD_TYPE_BOOL) + DORIS_FIELD_TYPE_SIZE_CASE(OLAP_FIELD_TYPE_TINYINT) + DORIS_FIELD_TYPE_SIZE_CASE(OLAP_FIELD_TYPE_SMALLINT) + DORIS_FIELD_TYPE_SIZE_CASE(OLAP_FIELD_TYPE_INT) + DORIS_FIELD_TYPE_SIZE_CASE(OLAP_FIELD_TYPE_UNSIGNED_INT) + DORIS_FIELD_TYPE_SIZE_CASE(OLAP_FIELD_TYPE_BIGINT) + DORIS_FIELD_TYPE_SIZE_CASE(OLAP_FIELD_TYPE_UNSIGNED_BIGINT) + DORIS_FIELD_TYPE_SIZE_CASE(OLAP_FIELD_TYPE_LARGEINT) + DORIS_FIELD_TYPE_SIZE_CASE(OLAP_FIELD_TYPE_FLOAT) + DORIS_FIELD_TYPE_SIZE_CASE(OLAP_FIELD_TYPE_DOUBLE) + DORIS_FIELD_TYPE_SIZE_CASE(OLAP_FIELD_TYPE_DECIMAL) + DORIS_FIELD_TYPE_SIZE_CASE(OLAP_FIELD_TYPE_DECIMAL32) + DORIS_FIELD_TYPE_SIZE_CASE(OLAP_FIELD_TYPE_DECIMAL64) + DORIS_FIELD_TYPE_SIZE_CASE(OLAP_FIELD_TYPE_DECIMAL128I) + DORIS_FIELD_TYPE_SIZE_CASE(OLAP_FIELD_TYPE_DECIMAL256) + DORIS_FIELD_TYPE_SIZE_CASE(OLAP_FIELD_TYPE_DATE) + DORIS_FIELD_TYPE_SIZE_CASE(OLAP_FIELD_TYPE_DATETIME) + DORIS_FIELD_TYPE_SIZE_CASE(OLAP_FIELD_TYPE_DATEV2) + DORIS_FIELD_TYPE_SIZE_CASE(OLAP_FIELD_TYPE_DATETIMEV2) + DORIS_FIELD_TYPE_SIZE_CASE(OLAP_FIELD_TYPE_TIMEV2) + DORIS_FIELD_TYPE_SIZE_CASE(OLAP_FIELD_TYPE_TIMESTAMPTZ) + DORIS_FIELD_TYPE_SIZE_CASE(OLAP_FIELD_TYPE_IPV4) + DORIS_FIELD_TYPE_SIZE_CASE(OLAP_FIELD_TYPE_IPV6) + DORIS_FIELD_TYPE_SIZE_CASE(OLAP_FIELD_TYPE_CHAR) + DORIS_FIELD_TYPE_SIZE_CASE(OLAP_FIELD_TYPE_VARCHAR) + DORIS_FIELD_TYPE_SIZE_CASE(OLAP_FIELD_TYPE_STRING) + DORIS_FIELD_TYPE_SIZE_CASE(OLAP_FIELD_TYPE_JSONB) + DORIS_FIELD_TYPE_SIZE_CASE(OLAP_FIELD_TYPE_VARIANT) + DORIS_FIELD_TYPE_SIZE_CASE(OLAP_FIELD_TYPE_HLL) + DORIS_FIELD_TYPE_SIZE_CASE(OLAP_FIELD_TYPE_BITMAP) + DORIS_FIELD_TYPE_SIZE_CASE(OLAP_FIELD_TYPE_QUANTILE_STATE) + DORIS_FIELD_TYPE_SIZE_CASE(OLAP_FIELD_TYPE_AGG_STATE) + DORIS_FIELD_TYPE_SIZE_CASE(OLAP_FIELD_TYPE_STRUCT) + DORIS_FIELD_TYPE_SIZE_CASE(OLAP_FIELD_TYPE_ARRAY) + DORIS_FIELD_TYPE_SIZE_CASE(OLAP_FIELD_TYPE_MAP) +#undef DORIS_FIELD_TYPE_SIZE_CASE + default: + LOG(FATAL) << "field_type_size: unsupported FieldType " << int(field_type); + return 0; + } } #include "common/compile_check_end.h" diff --git a/be/test/storage/index/ann/ann_index_smoke_test.cpp b/be/test/storage/index/ann/ann_index_smoke_test.cpp index 1672bc3985fe43..32de8d0626bd61 100644 --- a/be/test/storage/index/ann/ann_index_smoke_test.cpp +++ b/be/test/storage/index/ann/ann_index_smoke_test.cpp @@ -57,10 +57,6 @@ class AnnIndexTest : public testing::Test { EXPECT_CALL(*_tablet_column_array, type()) .WillRepeatedly(testing::Return(FieldType::OLAP_FIELD_TYPE_ARRAY)); - EXPECT_CALL(*_tablet_column_array, get_sub_column(0)) - .WillOnce(testing::ReturnRef(*_tablet_column_float)); - EXPECT_CALL(*_tablet_column_float, type()) - .WillOnce(testing::Return(FieldType::OLAP_FIELD_TYPE_FLOAT)); StorageField field(*_tablet_column_array); diff --git a/be/test/storage/index/inverted/compaction/util/index_compaction_utils.cpp b/be/test/storage/index/inverted/compaction/util/index_compaction_utils.cpp index b8eada222697aa..162543c3033dc8 100644 --- a/be/test/storage/index/inverted/compaction/util/index_compaction_utils.cpp +++ b/be/test/storage/index/inverted/compaction/util/index_compaction_utils.cpp @@ -158,9 +158,9 @@ class IndexCompactionUtils { EXPECT_TRUE(searcher_result.has_value()); auto bkd_searcher = std::get_if(&searcher_result.value()); EXPECT_TRUE(bkd_searcher != nullptr); - idx_reader->_type_info = get_scalar_type_info((FieldType)(*bkd_searcher)->type); - EXPECT_TRUE(idx_reader->_type_info != nullptr); - idx_reader->_value_key_coder = get_key_coder(idx_reader->_type_info->type()); + idx_reader->_type = (FieldType)(*bkd_searcher)->type; + EXPECT_TRUE(is_scalar_type(idx_reader->_type)); + idx_reader->_value_key_coder = get_key_coder(idx_reader->_type); for (int i = 0; i < query_data.size(); i++) { Field param_value = Field::create_field(int32_t(query_data[i])); diff --git a/be/test/storage/index/primary_key_index_test.cpp b/be/test/storage/index/primary_key_index_test.cpp index 7f4c47693f5816..83a56beb27420b 100644 --- a/be/test/storage/index/primary_key_index_test.cpp +++ b/be/test/storage/index/primary_key_index_test.cpp @@ -144,8 +144,8 @@ TEST_F(PrimaryKeyIndexTest, builder) { EXPECT_TRUE(index_reader.new_iterator(&iter, nullptr).ok()); size_t num_to_read = std::min(batch_size, remaining); - auto index_type = DataTypeFactory::instance().create_data_type( - index_reader.type_info()->type(), 1, 0); + auto index_type = + DataTypeFactory::instance().create_data_type(index_reader.type(), 1, 0); auto index_column = index_type->create_column(); Slice last_key_slice(last_key); EXPECT_TRUE(iter->seek_at_or_after(&last_key_slice, &exact_match).ok()); diff --git a/be/test/storage/segment/bloom_filter_index_reader_writer_test.cpp b/be/test/storage/segment/bloom_filter_index_reader_writer_test.cpp index 05635aaf407a89..17eeac11a7db15 100644 --- a/be/test/storage/segment/bloom_filter_index_reader_writer_test.cpp +++ b/be/test/storage/segment/bloom_filter_index_reader_writer_test.cpp @@ -66,7 +66,6 @@ Status write_bloom_filter_index_file(const std::string& file_name, const void* v size_t value_count, size_t null_count, ColumnIndexMetaPB* index_meta, bool use_primary_key_bloom_filter = false, double fpp = 0.05) { - const auto* type_info = get_scalar_type_info(); using CppType = typename CppTypeTraits::CppType; std::string fname = dname + "/" + file_name; auto fs = io::global_local_filesystem(); @@ -80,10 +79,10 @@ Status write_bloom_filter_index_file(const std::string& file_name, const void* v bf_options.fpp = fpp; // Set the expected FPP if (use_primary_key_bloom_filter) { RETURN_IF_ERROR(PrimaryKeyBloomFilterIndexWriterImpl::create( - bf_options, type_info, &bloom_filter_index_writer)); + bf_options, type, &bloom_filter_index_writer)); } else { - RETURN_IF_ERROR(BloomFilterIndexWriter::create(bf_options, type_info, - &bloom_filter_index_writer)); + RETURN_IF_ERROR( + BloomFilterIndexWriter::create(bf_options, type, &bloom_filter_index_writer)); } const CppType* vals = (const CppType*)values; @@ -616,7 +615,7 @@ TEST_F(BloomFilterIndexReaderWriterTest, test_ipv6) { template Status write_ngram_bloom_filter_index_file(const std::string& file_name, Slice* values, - size_t num_values, const TypeInfo* type_info, + size_t num_values, BloomFilterIndexWriter* bf_index_writer, ColumnIndexMetaPB* meta) { auto fs = io::global_local_filesystem(); @@ -685,16 +684,15 @@ template Status test_ngram_bloom_filter_index_reader_writer(const std::string& file_name, Slice* values, size_t num_values, uint8_t gram_size, uint16_t bf_size) { - const auto* type_info = get_scalar_type_info(); ColumnIndexMetaPB meta; BloomFilterOptions bf_options; std::unique_ptr bf_index_writer; - RETURN_IF_ERROR(NGramBloomFilterIndexWriterImpl::create(bf_options, type_info, gram_size, - bf_size, &bf_index_writer)); + RETURN_IF_ERROR(NGramBloomFilterIndexWriterImpl::create(bf_options, type, gram_size, bf_size, + &bf_index_writer)); - RETURN_IF_ERROR(write_ngram_bloom_filter_index_file( - file_name, values, num_values, type_info, bf_index_writer.get(), &meta)); + RETURN_IF_ERROR(write_ngram_bloom_filter_index_file(file_name, values, num_values, + bf_index_writer.get(), &meta)); std::vector test_patterns = {"ngram15", "ngram1000", "ngram1499", "non-existent-string"}; @@ -734,7 +732,7 @@ TEST_F(BloomFilterIndexReaderWriterTest, test_ngram_bloom_filter) { EXPECT_EQ(st.code(), TStatusCode::NOT_IMPLEMENTED_ERROR); } void test_ngram_bloom_filter_with_size(uint16_t bf_size) { - const auto* type_info = get_scalar_type_info(); + constexpr FieldType type = FieldType::OLAP_FIELD_TYPE_VARCHAR; ColumnIndexMetaPB meta; BloomFilterOptions bf_options; @@ -751,13 +749,13 @@ void test_ngram_bloom_filter_with_size(uint16_t bf_size) { uint8_t gram_size = 5; std::unique_ptr bf_index_writer; - auto st = NGramBloomFilterIndexWriterImpl::create(bf_options, type_info, gram_size, bf_size, + auto st = NGramBloomFilterIndexWriterImpl::create(bf_options, type, gram_size, bf_size, &bf_index_writer); EXPECT_TRUE(st.ok()); std::string file_name = "bloom_filter_ngram_varchar_size_" + std::to_string(bf_size); - st = write_ngram_bloom_filter_index_file( - file_name, slices.data(), num, type_info, bf_index_writer.get(), &meta); + st = write_ngram_bloom_filter_index_file(file_name, slices.data(), num, + bf_index_writer.get(), &meta); EXPECT_TRUE(st.ok()); EXPECT_EQ(bf_index_writer->size(), static_cast(bf_size) * total_pages); } @@ -770,10 +768,10 @@ TEST_F(BloomFilterIndexReaderWriterTest, test_ngram_bloom_filter_size) { } TEST_F(BloomFilterIndexReaderWriterTest, test_unsupported_type) { - auto type_info = get_scalar_type_info(); BloomFilterOptions bf_options; std::unique_ptr bloom_filter_index_writer; - auto st = BloomFilterIndexWriter::create(bf_options, type_info, &bloom_filter_index_writer); + auto st = BloomFilterIndexWriter::create(bf_options, FieldType::OLAP_FIELD_TYPE_FLOAT, + &bloom_filter_index_writer); EXPECT_FALSE(st.ok()); EXPECT_EQ(st.code(), TStatusCode::NOT_IMPLEMENTED_ERROR); } diff --git a/be/test/storage/segment/encoding_info_test.cpp b/be/test/storage/segment/encoding_info_test.cpp index 5583a18df8a290..666363c9566c83 100644 --- a/be/test/storage/segment/encoding_info_test.cpp +++ b/be/test/storage/segment/encoding_info_test.cpp @@ -42,36 +42,31 @@ class EncodingInfoTest : public testing::Test { }; TEST_F(EncodingInfoTest, normal) { - const auto* type_info = get_scalar_type_info(); + constexpr FieldType type = FieldType::OLAP_FIELD_TYPE_BIGINT; const EncodingInfo* encoding_info = nullptr; EncodingPreference encoding_preference; - auto status = EncodingInfo::get(type_info->type(), PLAIN_ENCODING, encoding_preference, - &encoding_info); + auto status = EncodingInfo::get(type, PLAIN_ENCODING, encoding_preference, &encoding_info); EXPECT_TRUE(status.ok()); EXPECT_NE(nullptr, encoding_info); } TEST_F(EncodingInfoTest, no_encoding) { - const auto* type_info = get_scalar_type_info(); + constexpr FieldType type = FieldType::OLAP_FIELD_TYPE_BIGINT; const EncodingInfo* encoding_info = nullptr; EncodingPreference encoding_preference; - auto status = EncodingInfo::get(type_info->type(), DICT_ENCODING, encoding_preference, - &encoding_info); + auto status = EncodingInfo::get(type, DICT_ENCODING, encoding_preference, &encoding_info); EXPECT_FALSE(status.ok()); } TEST_F(EncodingInfoTest, test_use_plain_binary_v2_config) { // Helper lambda to test string/JSON types with DICT_ENCODING as default auto test_dict_type_encoding = [](FieldType type, const std::string& type_name) { - const auto* type_info = get_scalar_type_info(type); - // Test with BINARY_PLAIN_ENCODING_V1 (default) // String and JSON types default to DICT_ENCODING EncodingPreference pref_v1; pref_v1.binary_plain_encoding_default_impl = BinaryPlainEncodingTypePB::BINARY_PLAIN_ENCODING_V1; - EncodingTypePB encoding_type = - EncodingInfo::get_default_encoding(type_info->type(), pref_v1, false); + EncodingTypePB encoding_type = EncodingInfo::get_default_encoding(type, pref_v1, false); EXPECT_EQ(DICT_ENCODING, encoding_type) << "Type " << type_name << " should use DICT_ENCODING with V1 preference"; @@ -80,21 +75,18 @@ TEST_F(EncodingInfoTest, test_use_plain_binary_v2_config) { EncodingPreference pref_v2; pref_v2.binary_plain_encoding_default_impl = BinaryPlainEncodingTypePB::BINARY_PLAIN_ENCODING_V2; - encoding_type = EncodingInfo::get_default_encoding(type_info->type(), pref_v2, false); + encoding_type = EncodingInfo::get_default_encoding(type, pref_v2, false); EXPECT_EQ(DICT_ENCODING, encoding_type) << "Type " << type_name << " should still use DICT_ENCODING with V2 preference"; }; // Helper lambda to test aggregate state types with PLAIN_ENCODING as default auto test_plain_type_encoding = [](FieldType type, const std::string& type_name) { - const auto* type_info = get_scalar_type_info(type); - // Test with BINARY_PLAIN_ENCODING_V1 (default) EncodingPreference pref_v1; pref_v1.binary_plain_encoding_default_impl = BinaryPlainEncodingTypePB::BINARY_PLAIN_ENCODING_V1; - EncodingTypePB encoding_type = - EncodingInfo::get_default_encoding(type_info->type(), pref_v1, false); + EncodingTypePB encoding_type = EncodingInfo::get_default_encoding(type, pref_v1, false); EXPECT_EQ(PLAIN_ENCODING, encoding_type) << "Type " << type_name << " should use PLAIN_ENCODING with V1 preference"; @@ -102,7 +94,7 @@ TEST_F(EncodingInfoTest, test_use_plain_binary_v2_config) { EncodingPreference pref_v2; pref_v2.binary_plain_encoding_default_impl = BinaryPlainEncodingTypePB::BINARY_PLAIN_ENCODING_V2; - encoding_type = EncodingInfo::get_default_encoding(type_info->type(), pref_v2, false); + encoding_type = EncodingInfo::get_default_encoding(type, pref_v2, false); EXPECT_EQ(PLAIN_ENCODING_V2, encoding_type) << "Type " << type_name << " should use PLAIN_ENCODING_V2 with V2 preference"; }; @@ -123,15 +115,15 @@ TEST_F(EncodingInfoTest, test_use_plain_binary_v2_config) { test_plain_type_encoding(FieldType::OLAP_FIELD_TYPE_AGG_STATE, "AGG_STATE"); // Test non-binary type (BIGINT) - should not be affected by binary preference - const auto* bigint_type_info = get_scalar_type_info(); + constexpr FieldType bigint_type = FieldType::OLAP_FIELD_TYPE_BIGINT; // Test with plain encoding disabled for integers (default) EncodingPreference pref_plain_disabled; pref_plain_disabled.integer_type_default_use_plain_encoding = false; pref_plain_disabled.binary_plain_encoding_default_impl = BinaryPlainEncodingTypePB::BINARY_PLAIN_ENCODING_V1; - EncodingTypePB encoding_type = EncodingInfo::get_default_encoding(bigint_type_info->type(), - pref_plain_disabled, false); + EncodingTypePB encoding_type = + EncodingInfo::get_default_encoding(bigint_type, pref_plain_disabled, false); EXPECT_EQ(BIT_SHUFFLE, encoding_type); // Test with plain encoding enabled for integers @@ -139,15 +131,13 @@ TEST_F(EncodingInfoTest, test_use_plain_binary_v2_config) { pref_plain_enabled.integer_type_default_use_plain_encoding = true; pref_plain_enabled.binary_plain_encoding_default_impl = BinaryPlainEncodingTypePB::BINARY_PLAIN_ENCODING_V1; - encoding_type = - EncodingInfo::get_default_encoding(bigint_type_info->type(), pref_plain_enabled, false); + encoding_type = EncodingInfo::get_default_encoding(bigint_type, pref_plain_enabled, false); EXPECT_EQ(PLAIN_ENCODING, encoding_type); // Verify binary preference doesn't affect integer types pref_plain_enabled.binary_plain_encoding_default_impl = BinaryPlainEncodingTypePB::BINARY_PLAIN_ENCODING_V2; - encoding_type = - EncodingInfo::get_default_encoding(bigint_type_info->type(), pref_plain_enabled, false); + encoding_type = EncodingInfo::get_default_encoding(bigint_type, pref_plain_enabled, false); EXPECT_EQ(PLAIN_ENCODING, encoding_type); // Should still be PLAIN_ENCODING } diff --git a/be/test/storage/storage_types_test.cpp b/be/test/storage/storage_types_test.cpp index 5d79e4f56ec320..f0d285b5861dc2 100644 --- a/be/test/storage/storage_types_test.cpp +++ b/be/test/storage/storage_types_test.cpp @@ -41,20 +41,16 @@ class TypesTest : public testing::Test { template void common_test(typename TypeTraits::CppType src_val) { - const auto* type = get_scalar_type_info(); - - EXPECT_EQ(field_type, type->type()); - EXPECT_EQ(sizeof(src_val), type->size()); + EXPECT_EQ(sizeof(src_val), field_type_size(field_type)); } template void test_char(Slice src_val) { StorageField* field = StorageFieldFactory::create_by_type(fieldType); field->_length = src_val.size; - const auto* type = field->type_info(); EXPECT_EQ(field->type(), fieldType); - EXPECT_EQ(sizeof(src_val), type->size()); + EXPECT_EQ(sizeof(src_val), field->size()); delete field; } @@ -105,9 +101,7 @@ void common_test_array(CollectionValue src_val) { 0, item_length); list_column.add_sub_column(item_column); - auto array_type = get_type_info(&list_column); - ASSERT_EQ(item_type, - dynamic_cast(array_type.get())->item_type_info()->type()); + ASSERT_EQ(item_type, list_column.get_sub_column(0).type()); } TEST(ArrayTypeTest, copy_and_equal) { From ff4644b7c88095221cf2c044cb783aad2b708813 Mon Sep 17 00:00:00 2001 From: Chenyang Sun Date: Thu, 21 May 2026 09:28:53 +0800 Subject: [PATCH 2/5] [refactor](storage) drop StorageField wrapper and clean up related dead code (#63233) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Drop the `StorageField` wrapper and related dead code. `StorageField` was a thin layer over `TabletColumn` — every accessor just forwarded, all 11 subclasses were empty stubs with no caller distinguishing them via dynamic_cast/typeid. After removing it, several dead pieces fell out. Issue Number: close #xxx Related PR: #xxx (cherry picked from commit 8a8999cf3a5b7d8e753ef1bab9c9e223ce72cd52) --- be/benchmark/benchmark_zone_map_index.hpp | 4 +- be/src/core/data_type/data_type_factory.cpp | 5 - be/src/core/data_type/data_type_factory.hpp | 2 - be/src/core/value/map_value.h | 57 --- be/src/core/value/struct_value.h | 61 --- be/src/io/cache/cache_block_meta_store.cpp | 2 - be/src/runtime/collection_value.cpp | 39 -- be/src/runtime/collection_value.h | 80 ---- be/src/storage/field.h | 384 ------------------ be/src/storage/index/ann/ann_index_writer.cpp | 5 - be/src/storage/index/ann/ann_index_writer.h | 3 - be/src/storage/index/index_writer.cpp | 21 +- be/src/storage/index/index_writer.h | 8 +- .../index/inverted/inverted_index_reader.cpp | 1 - .../index/inverted/inverted_index_writer.cpp | 54 --- .../index/inverted/inverted_index_writer.h | 2 - .../storage/index/zone_map/zone_map_index.cpp | 6 +- .../storage/index/zone_map/zone_map_index.h | 4 +- .../storage/iterator/olap_data_convertor.cpp | 1 - be/src/storage/iterator/olap_data_convertor.h | 1 - .../iterator/vertical_merge_iterator.cpp | 1 - .../storage/iterator/vgeneric_iterators.cpp | 1 - be/src/storage/olap_common.h | 12 +- be/src/storage/row_cursor.cpp | 42 +- be/src/storage/row_cursor.h | 9 +- be/src/storage/schema.cpp | 24 +- be/src/storage/schema.h | 11 +- .../storage/schema_change/schema_change.cpp | 9 - be/src/storage/schema_change/schema_change.h | 1 - be/src/storage/segment/column_writer.cpp | 170 ++++---- be/src/storage/segment/column_writer.h | 36 +- be/src/storage/segment/segment.cpp | 1 + be/src/storage/segment/segment.h | 5 +- be/src/storage/segment/segment_iterator.cpp | 71 ++-- be/src/storage/segment/segment_iterator.h | 6 +- .../variant/binary_column_extract_iterator.h | 1 - .../variant/hierarchical_data_iterator.h | 1 - .../variant/sparse_column_merge_iterator.h | 1 - .../variant/variant_column_writer_impl.cpp | 31 +- .../variant/variant_column_writer_impl.h | 4 +- .../variant_streaming_compaction_writer.cpp | 3 +- be/src/storage/task/index_builder.cpp | 34 +- be/src/storage/task/index_builder.h | 12 +- be/src/storage/types.h | 60 ++- be/test/exec/scan/vgeneric_iterators_test.cpp | 1 - .../cast/function_variant_cast_test.cpp | 1 - .../memtable/memtable_flush_executor_test.cpp | 2 +- .../ordered_data_compaction_test.cpp | 1 - .../compaction/vertical_compaction_test.cpp | 1 - .../index/ann/ann_index_smoke_test.cpp | 6 - .../index/ann/ann_index_writer_test.cpp | 24 +- .../util/index_compaction_utils.cpp | 1 + .../inverted/query/phrase_edge_query_test.cpp | 11 +- .../query/phrase_prefix_query_test.cpp | 11 +- .../inverted/query/phrase_query_test.cpp | 11 +- be/test/storage/metadata_adder_test.cpp | 8 +- .../segment/column_reader_writer_test.cpp | 227 ----------- .../segment/inverted_index_array_test.cpp | 46 +-- .../segment/inverted_index_reader_test.cpp | 56 +-- .../segment/inverted_index_writer_test.cpp | 285 +++++++------ .../segment/segment_corruption_test.cpp | 1 - .../storage/segment/zone_map_index_test.cpp | 52 +-- be/test/storage/storage_types_test.cpp | 87 +--- 63 files changed, 536 insertions(+), 1582 deletions(-) delete mode 100644 be/src/core/value/map_value.h delete mode 100644 be/src/core/value/struct_value.h delete mode 100644 be/src/runtime/collection_value.cpp delete mode 100644 be/src/runtime/collection_value.h delete mode 100644 be/src/storage/field.h diff --git a/be/benchmark/benchmark_zone_map_index.hpp b/be/benchmark/benchmark_zone_map_index.hpp index 2fe0d41733b9a2..a93ccb010a17b4 100644 --- a/be/benchmark/benchmark_zone_map_index.hpp +++ b/be/benchmark/benchmark_zone_map_index.hpp @@ -34,7 +34,6 @@ #include "core/data_type/data_type_factory.hpp" #include "core/string_ref.h" -#include "storage/field.h" #include "storage/index/zone_map/zone_map_index.h" #include "storage/tablet/tablet_schema.h" #include "util/slice.h" @@ -116,9 +115,8 @@ std::unique_ptr make_writer() { col = make_column(FieldType::OLAP_FIELD_TYPE_VARCHAR, 64, 1); dtype = DataTypeFactory::instance().create_data_type(TYPE_VARCHAR, false, 0, 0, 64); } - std::unique_ptr field(StorageFieldFactory::create(*col)); std::unique_ptr w; - (void)ZoneMapIndexWriter::create(dtype, field.get(), w); + (void)ZoneMapIndexWriter::create(dtype, col.get(), w); return w; } diff --git a/be/src/core/data_type/data_type_factory.cpp b/be/src/core/data_type/data_type_factory.cpp index 755ef5a25ffe67..6d9ed4f139bf63 100644 --- a/be/src/core/data_type/data_type_factory.cpp +++ b/be/src/core/data_type/data_type_factory.cpp @@ -65,15 +65,10 @@ #include "core/data_type/define_primitive_type.h" #include "core/types.h" #include "core/uint128.h" -#include "storage/field.h" #include "storage/olap_common.h" namespace doris { #include "common/compile_check_begin.h" -DataTypePtr DataTypeFactory::create_data_type(const doris::StorageField& col_desc) { - return create_data_type(col_desc.get_desc(), col_desc.is_nullable()); -} - DataTypePtr DataTypeFactory::create_data_type(const TabletColumn& col_desc, bool is_nullable) { DataTypePtr nested = nullptr; if (col_desc.type() == FieldType::OLAP_FIELD_TYPE_AGG_STATE) { diff --git a/be/src/core/data_type/data_type_factory.hpp b/be/src/core/data_type/data_type_factory.hpp index 15c55a99450063..e169375e3fec8c 100644 --- a/be/src/core/data_type/data_type_factory.hpp +++ b/be/src/core/data_type/data_type_factory.hpp @@ -33,7 +33,6 @@ namespace arrow { class DataType; } // namespace arrow namespace doris { -class StorageField; class PColumnMeta; enum class FieldType; @@ -52,7 +51,6 @@ class DataTypeFactory { return instance; } - DataTypePtr create_data_type(const doris::StorageField& col_desc); DataTypePtr create_data_type(const TabletColumn& col_desc, bool is_nullable = false); DataTypePtr create_data_type(const PColumnMeta& pcolumn); diff --git a/be/src/core/value/map_value.h b/be/src/core/value/map_value.h deleted file mode 100644 index 68480cdbcb5e26..00000000000000 --- a/be/src/core/value/map_value.h +++ /dev/null @@ -1,57 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include - -namespace doris { - -/** - * MapValue is for map type in memory - */ -class MapValue { -public: - MapValue() = default; - - explicit MapValue(int32_t length) : _key_data(nullptr), _value_data(nullptr), _length(length) {} - - MapValue(void* k_data, void* v_data, int32_t length) - : _key_data(k_data), _value_data(v_data), _length(length) {} - - int32_t size() const { return _length; } - - int32_t length() const { return _length; } - - const void* key_data() const { return _key_data; } - void* mutable_key_data() const { return _key_data; } - const void* value_data() const { return _value_data; } - void* mutable_value_data() const { return _value_data; } - - void set_length(int32_t length) { _length = length; } - void set_key(void* data) { _key_data = data; } - void set_value(void* data) { _value_data = data; } - -private: - // child column data pointer - void* _key_data = nullptr; - void* _value_data = nullptr; - // length for map size - int32_t _length; - -}; //map-value -} // namespace doris diff --git a/be/src/core/value/struct_value.h b/be/src/core/value/struct_value.h deleted file mode 100644 index fa79a3e0a9bd87..00000000000000 --- a/be/src/core/value/struct_value.h +++ /dev/null @@ -1,61 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include - -namespace doris { - -class StructValue { -public: - StructValue() = default; - - explicit StructValue(uint32_t size) : _values(nullptr), _size(size), _has_null(false) {} - StructValue(void** values, uint32_t size) : _values(values), _size(size), _has_null(false) {} - StructValue(void** values, uint32_t size, bool has_null) - : _values(values), _size(size), _has_null(has_null) {} - - //void to_struct_val(StructVal* val) const; - //static StructValue from_struct_val(const StructVal& val); - - uint32_t size() const { return _size; } - void set_size(uint32_t size) { _size = size; } - bool has_null() const { return _has_null; } - void set_has_null(bool has_null) { _has_null = has_null; } - bool is_null_at(uint32_t index) const { - return this->_has_null && this->_values[index] == nullptr; - } - - const void** values() const { return const_cast(_values); } - void** mutable_values() { return _values; } - void set_values(void** values) { _values = values; } - const void* child_value(uint32_t index) const { return _values[index]; } - void* mutable_child_value(uint32_t index) { return _values[index]; } - void set_child_value(void* value, uint32_t index) { _values[index] = value; } - -private: - // pointer to the start of the vector of children pointers. These pointers are - // point to children values where a null pointer means that this child is NULL. - void** _values = nullptr; - // the number of values in this struct value. - uint32_t _size; - // child has no null value if has_null is false. - // child may has null value if has_null is true. - bool _has_null; -}; -} // namespace doris \ No newline at end of file diff --git a/be/src/io/cache/cache_block_meta_store.cpp b/be/src/io/cache/cache_block_meta_store.cpp index 369d81537969e0..fa82cee3ce99e8 100644 --- a/be/src/io/cache/cache_block_meta_store.cpp +++ b/be/src/io/cache/cache_block_meta_store.cpp @@ -33,8 +33,6 @@ #include "common/status.h" #include "exec/common/hex.h" -#include "storage/field.h" -#include "storage/field.h" // For OLAP_FIELD_TYPE_BIGINT #include "storage/key_coder.h" #include "storage/olap_common.h" #include "util/threadpool.h" diff --git a/be/src/runtime/collection_value.cpp b/be/src/runtime/collection_value.cpp deleted file mode 100644 index 1d501720695f38..00000000000000 --- a/be/src/runtime/collection_value.cpp +++ /dev/null @@ -1,39 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "runtime/collection_value.h" - -#include - -namespace doris { - -void CollectionValue::shallow_copy(const CollectionValue* value) { - _length = value->_length; - _null_signs = value->_null_signs; - _data = value->_data; - _has_null = value->_has_null; -} - -void CollectionValue::copy_null_signs(const CollectionValue* other) { - if (other->_has_null) { - memcpy(_null_signs, other->_null_signs, other->size()); - } else { - _null_signs = nullptr; - } -} - -} // namespace doris diff --git a/be/src/runtime/collection_value.h b/be/src/runtime/collection_value.h deleted file mode 100644 index da916a9a1ae357..00000000000000 --- a/be/src/runtime/collection_value.h +++ /dev/null @@ -1,80 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include - -#include -#include - -namespace doris { - -using MemFootprint = std::pair; -using GenMemFootprintFunc = std::function; - -/** - * The format of array-typed slot. - * A new array needs to be initialized before using it. - */ -class CollectionValue { -public: - CollectionValue() = default; - - explicit CollectionValue(uint64_t length) - : _data(nullptr), _length(length), _has_null(false), _null_signs(nullptr) {} - - CollectionValue(void* data, uint64_t length) - : _data(data), _length(length), _has_null(false), _null_signs(nullptr) {} - - CollectionValue(void* data, uint64_t length, bool* null_signs) - : _data(data), _length(length), _has_null(true), _null_signs(null_signs) {} - - CollectionValue(void* data, uint64_t length, bool has_null, bool* null_signs) - : _data(data), _length(length), _has_null(has_null), _null_signs(null_signs) {} - - bool is_null_at(uint64_t index) const { return this->_has_null && this->_null_signs[index]; } - - uint64_t size() const { return _length; } - - uint64_t length() const { return _length; } - - void shallow_copy(const CollectionValue* other); - - void copy_null_signs(const CollectionValue* other); - - const void* data() const { return _data; } - bool has_null() const { return _has_null; } - const bool* null_signs() const { return _null_signs; } - void* mutable_data() { return _data; } - bool* mutable_null_signs() { return _null_signs; } - void set_length(uint64_t length) { _length = length; } - void set_has_null(bool has_null) { _has_null = has_null; } - void set_data(void* data) { _data = data; } - void set_null_signs(bool* null_signs) { _null_signs = null_signs; } - -private: - // child column data - void* _data = nullptr; - uint64_t _length = 0; - // item has no null value if has_null is false. - // item ```may``` has null value if has_null is true. - bool _has_null = false; - // null bitmap - bool* _null_signs = nullptr; -}; -} // namespace doris diff --git a/be/src/storage/field.h b/be/src/storage/field.h deleted file mode 100644 index c1aedb777f3793..00000000000000 --- a/be/src/storage/field.h +++ /dev/null @@ -1,384 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include -#include -#include - -#include "core/arena.h" -#include "core/value/map_value.h" -#include "runtime/collection_value.h" -#include "storage/key_coder.h" -#include "storage/olap_common.h" -#include "storage/olap_define.h" -#include "storage/tablet/tablet_schema.h" -#include "storage/types.h" -#include "storage/utils.h" -#include "util/hash_util.hpp" -#include "util/json/path_in_data.h" -#include "util/slice.h" - -namespace doris { -#include "common/compile_check_begin.h" -// A Field is used to represent a column in memory format. -// User can use this class to access or deal with column data in memory. -class StorageField { -public: - StorageField(const TabletColumn& column) - : _type(column.type()), - _desc(column), - _length(column.length()), - _key_coder(get_key_coder(column.type())), - _name(column.name()), - _index_size(column.index_length()), - _is_nullable(column.is_nullable()), - _unique_id(column.unique_id()), - _parent_unique_id(column.parent_unique_id()), - _is_extracted_column(column.is_extracted_column()), - _path(column.path_info_ptr()) {} - - virtual ~StorageField() = default; - - size_t size() const { return field_type_size(_type); } - size_t length() const { return _length; } - size_t field_size() const { return size() + 1; } - size_t index_size() const { return _index_size; } - int32_t unique_id() const { return _unique_id; } - int32_t parent_unique_id() const { return _parent_unique_id; } - bool is_extracted_column() const { return _is_extracted_column; } - const std::string& name() const { return _name; } - const PathInDataPtr& path() const { return _path; } - - virtual StorageField* clone() const { - auto* local = new StorageField(_desc); - this->clone(local); - return local; - } - - FieldType type() const { return _type; } - bool is_nullable() const { return _is_nullable; } - - // similar to `full_encode_ascending`, but only encode part (the first `index_size` bytes) of the value. - // only applicable to string type - void encode_ascending(const void* value, std::string* buf) const { - _key_coder->encode_ascending(value, _index_size, buf); - } - - // encode the provided `value` into `buf`. - void full_encode_ascending(const void* value, std::string* buf) const { - _key_coder->full_encode_ascending(value, buf); - } - - const KeyCoder* key_coder() const { return _key_coder; } - void add_sub_field(std::unique_ptr sub_field) { - _sub_fields.emplace_back(std::move(sub_field)); - } - StorageField* get_sub_field(size_t i) const { return _sub_fields[i].get(); } - size_t get_sub_field_count() const { return _sub_fields.size(); } - - void set_precision(int32_t precision) { _precision = precision; } - void set_scale(int32_t scale) { _scale = scale; } - int32_t get_precision() const { return _precision; } - int32_t get_scale() const { return _scale; } - const TabletColumn& get_desc() const { return _desc; } - - int32_t get_unique_id() const { - return is_extracted_column() ? parent_unique_id() : unique_id(); - } - -protected: - FieldType _type; - TabletColumn _desc; - // unit : byte - // except for strings, other types have fixed lengths - // Note that, the struct type itself has fixed length, but due to - // its number of subfields is a variable, so the actual length of - // a struct field is not fixed. - size_t _length; - - void clone(StorageField* other) const { - other->_type = this->_type; - other->_key_coder = this->_key_coder; - other->_name = this->_name; - other->_index_size = this->_index_size; - other->_is_nullable = this->_is_nullable; - other->_sub_fields.clear(); - other->_precision = this->_precision; - other->_scale = this->_scale; - other->_unique_id = this->_unique_id; - other->_parent_unique_id = this->_parent_unique_id; - other->_is_extracted_column = this->_is_extracted_column; - for (const auto& f : _sub_fields) { - StorageField* item = f->clone(); - other->add_sub_field(std::unique_ptr(item)); - } - } - -private: - // maximum length of Field, unit : bytes - // usually equal to length, except for variable-length strings - const KeyCoder* _key_coder; - std::string _name; - size_t _index_size; - bool _is_nullable; - std::vector> _sub_fields; - int32_t _precision; - int32_t _scale; - int32_t _unique_id; - int32_t _parent_unique_id; - bool _is_extracted_column = false; - PathInDataPtr _path; -}; - -class MapField : public StorageField { -public: - MapField(const TabletColumn& column) : StorageField(column) {} -}; - -class StructField : public StorageField { -public: - StructField(const TabletColumn& column) : StorageField(column) {} -}; - -class ArrayField : public StorageField { -public: - ArrayField(const TabletColumn& column) : StorageField(column) {} -}; - -class CharField : public StorageField { -public: - CharField(const TabletColumn& column) : StorageField(column) {} - - CharField* clone() const override { - auto* local = new CharField(_desc); - StorageField::clone(local); - return local; - } -}; - -class VarcharField : public StorageField { -public: - VarcharField(const TabletColumn& column) : StorageField(column) {} - - VarcharField* clone() const override { - auto* local = new VarcharField(_desc); - StorageField::clone(local); - return local; - } -}; -class StringField : public StorageField { -public: - StringField(const TabletColumn& column) : StorageField(column) {} - - StringField* clone() const override { - auto* local = new StringField(_desc); - StorageField::clone(local); - return local; - } -}; - -class BitmapAggField : public StorageField { -public: - BitmapAggField(const TabletColumn& column) : StorageField(column) {} - - BitmapAggField* clone() const override { - auto* local = new BitmapAggField(_desc); - StorageField::clone(local); - return local; - } -}; - -class QuantileStateAggField : public StorageField { -public: - QuantileStateAggField(const TabletColumn& column) : StorageField(column) {} - - QuantileStateAggField* clone() const override { - auto* local = new QuantileStateAggField(_desc); - StorageField::clone(local); - return local; - } -}; - -class AggStateField : public StorageField { -public: - AggStateField(const TabletColumn& column) : StorageField(column) {} - - AggStateField* clone() const override { - auto* local = new AggStateField(_desc); - StorageField::clone(local); - return local; - } -}; - -class HllAggField : public StorageField { -public: - HllAggField(const TabletColumn& column) : StorageField(column) {} - - HllAggField* clone() const override { - auto* local = new HllAggField(_desc); - StorageField::clone(local); - return local; - } -}; - -class StorageFieldFactory { -public: - static StorageField* create(const TabletColumn& column) { - // for key column - if (column.is_key()) { - switch (column.type()) { - case FieldType::OLAP_FIELD_TYPE_CHAR: - return new CharField(column); - case FieldType::OLAP_FIELD_TYPE_VARCHAR: - case FieldType::OLAP_FIELD_TYPE_STRING: - return new StringField(column); - case FieldType::OLAP_FIELD_TYPE_STRUCT: { - auto* local = new StructField(column); - for (uint32_t i = 0; i < column.get_subtype_count(); i++) { - std::unique_ptr sub_field( - StorageFieldFactory::create(column.get_sub_column(i))); - local->add_sub_field(std::move(sub_field)); - } - return local; - } - case FieldType::OLAP_FIELD_TYPE_ARRAY: { - std::unique_ptr item_field( - StorageFieldFactory::create(column.get_sub_column(0))); - auto* local = new ArrayField(column); - local->add_sub_field(std::move(item_field)); - return local; - } - case FieldType::OLAP_FIELD_TYPE_MAP: { - std::unique_ptr key_field( - StorageFieldFactory::create(column.get_sub_column(0))); - std::unique_ptr val_field( - StorageFieldFactory::create(column.get_sub_column(1))); - auto* local = new MapField(column); - local->add_sub_field(std::move(key_field)); - local->add_sub_field(std::move(val_field)); - return local; - } - case FieldType::OLAP_FIELD_TYPE_DECIMAL: - [[fallthrough]]; - case FieldType::OLAP_FIELD_TYPE_DECIMAL32: - [[fallthrough]]; - case FieldType::OLAP_FIELD_TYPE_DECIMAL64: - [[fallthrough]]; - case FieldType::OLAP_FIELD_TYPE_DECIMAL128I: - [[fallthrough]]; - case FieldType::OLAP_FIELD_TYPE_DECIMAL256: - [[fallthrough]]; - case FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ: - [[fallthrough]]; - case FieldType::OLAP_FIELD_TYPE_DATETIMEV2: { - StorageField* field = new StorageField(column); - field->set_precision(column.precision()); - field->set_scale(column.frac()); - return field; - } - default: - return new StorageField(column); - } - } - - // for value column - switch (column.aggregation()) { - case FieldAggregationMethod::OLAP_FIELD_AGGREGATION_NONE: - case FieldAggregationMethod::OLAP_FIELD_AGGREGATION_SUM: - case FieldAggregationMethod::OLAP_FIELD_AGGREGATION_MIN: - case FieldAggregationMethod::OLAP_FIELD_AGGREGATION_MAX: - case FieldAggregationMethod::OLAP_FIELD_AGGREGATION_REPLACE: - case FieldAggregationMethod::OLAP_FIELD_AGGREGATION_REPLACE_IF_NOT_NULL: - switch (column.type()) { - case FieldType::OLAP_FIELD_TYPE_CHAR: - return new CharField(column); - case FieldType::OLAP_FIELD_TYPE_VARCHAR: - return new VarcharField(column); - case FieldType::OLAP_FIELD_TYPE_STRING: - return new StringField(column); - case FieldType::OLAP_FIELD_TYPE_STRUCT: { - auto* local = new StructField(column); - for (uint32_t i = 0; i < column.get_subtype_count(); i++) { - std::unique_ptr sub_field( - StorageFieldFactory::create(column.get_sub_column(i))); - local->add_sub_field(std::move(sub_field)); - } - return local; - } - case FieldType::OLAP_FIELD_TYPE_ARRAY: { - std::unique_ptr item_field( - StorageFieldFactory::create(column.get_sub_column(0))); - auto* local = new ArrayField(column); - local->add_sub_field(std::move(item_field)); - return local; - } - case FieldType::OLAP_FIELD_TYPE_MAP: { - DCHECK(column.get_subtype_count() == 2); - auto* local = new MapField(column); - std::unique_ptr key_field( - StorageFieldFactory::create(column.get_sub_column(0))); - std::unique_ptr value_field( - StorageFieldFactory::create(column.get_sub_column(1))); - local->add_sub_field(std::move(key_field)); - local->add_sub_field(std::move(value_field)); - return local; - } - case FieldType::OLAP_FIELD_TYPE_DECIMAL: - [[fallthrough]]; - case FieldType::OLAP_FIELD_TYPE_DECIMAL32: - [[fallthrough]]; - case FieldType::OLAP_FIELD_TYPE_DECIMAL64: - [[fallthrough]]; - case FieldType::OLAP_FIELD_TYPE_DECIMAL128I: - [[fallthrough]]; - case FieldType::OLAP_FIELD_TYPE_DECIMAL256: - [[fallthrough]]; - case FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ: - [[fallthrough]]; - case FieldType::OLAP_FIELD_TYPE_DATETIMEV2: { - StorageField* field = new StorageField(column); - field->set_precision(column.precision()); - field->set_scale(column.frac()); - return field; - } - default: - return new StorageField(column); - } - case FieldAggregationMethod::OLAP_FIELD_AGGREGATION_HLL_UNION: - return new HllAggField(column); - case FieldAggregationMethod::OLAP_FIELD_AGGREGATION_BITMAP_UNION: - return new BitmapAggField(column); - case FieldAggregationMethod::OLAP_FIELD_AGGREGATION_QUANTILE_UNION: - return new QuantileStateAggField(column); - case FieldAggregationMethod::OLAP_FIELD_AGGREGATION_GENERIC: - return new AggStateField(column); - case FieldAggregationMethod::OLAP_FIELD_AGGREGATION_UNKNOWN: - CHECK(false) << ", value column no agg type"; - return nullptr; - } - return nullptr; - } - - static StorageField* create_by_type(const FieldType& type) { - TabletColumn column(FieldAggregationMethod::OLAP_FIELD_AGGREGATION_NONE, type); - return create(column); - } -}; -#include "common/compile_check_end.h" -} // namespace doris diff --git a/be/src/storage/index/ann/ann_index_writer.cpp b/be/src/storage/index/ann/ann_index_writer.cpp index fc4966fb7a93ec..c93e96fa5e07f9 100644 --- a/be/src/storage/index/ann/ann_index_writer.cpp +++ b/be/src/storage/index/ann/ann_index_writer.cpp @@ -120,11 +120,6 @@ Status AnnIndexColumnWriter::add_array_values(size_t field_size, const void* val return Status::OK(); } -Status AnnIndexColumnWriter::add_array_values(size_t field_size, const CollectionValue* values, - size_t count) { - return Status::InternalError("Ann index should not be used on nullable column"); -} - Status AnnIndexColumnWriter::add_nulls(uint32_t count) { return Status::InternalError("Ann index should not be used on nullable column"); } diff --git a/be/src/storage/index/ann/ann_index_writer.h b/be/src/storage/index/ann/ann_index_writer.h index d749564a7bf793..1e7ab85fb3a48a 100644 --- a/be/src/storage/index/ann/ann_index_writer.h +++ b/be/src/storage/index/ann/ann_index_writer.h @@ -28,7 +28,6 @@ #include #include "core/pod_array.h" -#include "runtime/collection_value.h" #include "storage/index/ann/ann_index.h" #include "storage/index/index_file_writer.h" #include "storage/index/index_writer.h" @@ -61,8 +60,6 @@ class AnnIndexColumnWriter : public IndexColumnWriter { Status add_values(const std::string fn, const void* values, size_t count) override; Status add_array_values(size_t field_size, const void* value_ptr, const uint8_t* null_map, const uint8_t* offsets_ptr, size_t count) override; - Status add_array_values(size_t field_size, const CollectionValue* values, - size_t count) override; int64_t size() const override; Status finish() override; diff --git a/be/src/storage/index/index_writer.cpp b/be/src/storage/index/index_writer.cpp index e209caece4f6e4..edb13cd1aca2a0 100644 --- a/be/src/storage/index/index_writer.cpp +++ b/be/src/storage/index/index_writer.cpp @@ -16,9 +16,10 @@ // under the License. #include "common/exception.h" -#include "storage/field.h" #include "storage/index/ann/ann_index_writer.h" #include "storage/index/inverted/inverted_index_writer.h" +#include "storage/tablet/tablet_schema.h" +#include "storage/types.h" namespace doris::segment_v2 { #include "common/compile_check_begin.h" @@ -46,33 +47,33 @@ bool IndexColumnWriter::check_support_ann_index(const TabletColumn& column) { } // create index writer -Status IndexColumnWriter::create(const StorageField* field, std::unique_ptr* res, +Status IndexColumnWriter::create(const TabletColumn* column, + std::unique_ptr* res, IndexFileWriter* index_file_writer, const TabletIndex* index_meta) { - FieldType type = field->type(); + FieldType type = column->type(); std::string field_name; auto storage_format = index_file_writer->get_storage_format(); if (storage_format == InvertedIndexStorageFormatPB::V1) { - field_name = field->name(); + field_name = column->name(); } else { - if (field->is_extracted_column()) { + if (column->is_extracted_column()) { // variant sub col // field_name format: parent_unique_id.sub_col_name - field_name = std::to_string(field->parent_unique_id()) + "." + field->name(); + field_name = std::to_string(column->parent_unique_id()) + "." + column->name(); } else { - field_name = std::to_string(field->unique_id()); + field_name = std::to_string(column->unique_id()); } } if (index_meta->is_inverted_index()) { bool single_field = true; if (type == FieldType::OLAP_FIELD_TYPE_ARRAY) { - const auto& column = field->get_desc(); - bool has_item_subcolumn = column.get_subtype_count() > 0; + bool has_item_subcolumn = column->get_subtype_count() > 0; DBUG_EXECUTE_IF("InvertedIndexColumnWriter::create_array_typeinfo_is_nullptr", { has_item_subcolumn = false; }) if (has_item_subcolumn) { - type = column.get_sub_column(0).type(); + type = column->get_sub_column(0).type(); single_field = false; } else { return Status::NotSupported("unsupported array type for inverted index: " + diff --git a/be/src/storage/index/index_writer.h b/be/src/storage/index/index_writer.h index 62e2cc18a25980..a2538c9778f59d 100644 --- a/be/src/storage/index/index_writer.h +++ b/be/src/storage/index/index_writer.h @@ -36,10 +36,6 @@ namespace doris { #include "common/compile_check_begin.h" -class CollectionValue; - -class StorageField; - class TabletIndex; class TabletColumn; @@ -48,7 +44,7 @@ class IndexFileWriter; class IndexColumnWriter { public: - static Status create(const StorageField* field, std::unique_ptr* res, + static Status create(const TabletColumn* column, std::unique_ptr* res, IndexFileWriter* index_file_writer, const TabletIndex* inverted_index); virtual Status init() = 0; @@ -56,8 +52,6 @@ class IndexColumnWriter { virtual ~IndexColumnWriter() = default; virtual Status add_values(const std::string name, const void* values, size_t count) = 0; - virtual Status add_array_values(size_t field_size, const CollectionValue* values, - size_t count) = 0; virtual Status add_array_values(size_t field_size, const void* value_ptr, const uint8_t* null_map, const uint8_t* offsets_ptr, diff --git a/be/src/storage/index/inverted/inverted_index_reader.cpp b/be/src/storage/index/inverted/inverted_index_reader.cpp index bef44b4a78a72f..64d32115041b48 100644 --- a/be/src/storage/index/inverted/inverted_index_reader.cpp +++ b/be/src/storage/index/inverted/inverted_index_reader.cpp @@ -44,7 +44,6 @@ #include "core/type_limit.h" #include "runtime/runtime_profile.h" #include "runtime/runtime_state.h" -#include "storage/field.h" #include "storage/index/index_file_reader.h" #include "storage/index/index_reader_helper.h" #include "storage/index/inverted/analyzer/analyzer.h" diff --git a/be/src/storage/index/inverted/inverted_index_writer.cpp b/be/src/storage/index/inverted/inverted_index_writer.cpp index f0b01bbcc9d63e..8e4730cc063a73 100644 --- a/be/src/storage/index/inverted/inverted_index_writer.cpp +++ b/be/src/storage/index/inverted/inverted_index_writer.cpp @@ -526,60 +526,6 @@ Status InvertedIndexColumnWriter::add_array_values(size_t field_size return Status::OK(); } -template -Status InvertedIndexColumnWriter::add_array_values(size_t field_size, - const CollectionValue* values, - size_t count) { - if constexpr (field_is_slice_type(field_type)) { - DBUG_EXECUTE_IF("InvertedIndexColumnWriter::add_array_values_field_is_nullptr", - { _field = nullptr; }) - DBUG_EXECUTE_IF( - "InvertedIndexColumnWriter::add_array_values_index_writer_is_" - "nullptr", - { _index_writer = nullptr; }) - if (_field == nullptr || _index_writer == nullptr) { - LOG(ERROR) << "field or index writer is null in inverted index writer."; - return Status::InternalError("field or index writer is null in inverted index writer"); - } - for (int i = 0; i < count; ++i) { - const auto* item_data_ptr = values->data(); - std::vector strings; - - for (size_t j = 0; j < values->length(); ++j) { - auto* v = (Slice*)item_data_ptr; - - if (!values->is_null_at(j)) { - strings.emplace_back(v->get_data(), v->get_size()); - } - item_data_ptr = (uint8_t*)item_data_ptr + field_size; - } - auto value = join(strings, " "); - RETURN_IF_ERROR(new_inverted_index_field(value.c_str(), value.length())); - _rid++; - RETURN_IF_ERROR(add_document()); - values++; - } - } else if constexpr (field_is_numeric_type(field_type)) { - for (int i = 0; i < count; ++i) { - const auto* item_data_ptr = values->data(); - - for (size_t j = 0; j < values->length(); ++j) { - const auto* p = reinterpret_cast(item_data_ptr); - if (values->is_null_at(j)) { - // bkd do not index null values, so we do nothing here. - } else { - RETURN_IF_ERROR(add_value(*p)); - } - item_data_ptr = (uint8_t*)item_data_ptr + field_size; - } - _row_ids_seen_for_bkd++; - _rid++; - values++; - } - } - return Status::OK(); -} - template Status InvertedIndexColumnWriter::add_numeric_values(const void* values, size_t count) { auto p = reinterpret_cast(values); diff --git a/be/src/storage/index/inverted/inverted_index_writer.h b/be/src/storage/index/inverted/inverted_index_writer.h index de0b370acbe268..3ef7b4d9319e39 100644 --- a/be/src/storage/index/inverted/inverted_index_writer.h +++ b/be/src/storage/index/inverted/inverted_index_writer.h @@ -72,8 +72,6 @@ class InvertedIndexColumnWriter : public IndexColumnWriter { Status add_array_values(size_t field_size, const void* value_ptr, const uint8_t* nested_null_map, const uint8_t* offsets_ptr, size_t count) override; - Status add_array_values(size_t field_size, const CollectionValue* values, - size_t count) override; Status add_numeric_values(const void* values, size_t count); Status add_value(const CppType& value); int64_t size() const override; diff --git a/be/src/storage/index/zone_map/zone_map_index.cpp b/be/src/storage/index/zone_map/zone_map_index.cpp index 33f39bfb5b5ca7..cc59b8d02df73b 100644 --- a/be/src/storage/index/zone_map/zone_map_index.cpp +++ b/be/src/storage/index/zone_map/zone_map_index.cpp @@ -32,11 +32,11 @@ #include "core/string_ref.h" #include "core/value/decimalv2_value.h" #include "core/value/vdatetime_value.h" -#include "storage/field.h" #include "storage/index/indexed_column_reader.h" #include "storage/index/indexed_column_writer.h" #include "storage/olap_common.h" #include "storage/segment/encoding_info.h" +#include "storage/tablet/tablet_schema.h" #include "storage/types.h" #include "util/slice.h" #include "util/unaligned.h" @@ -366,9 +366,9 @@ ZoneMapIndexReader::~ZoneMapIndexReader() = default; M(TYPE_DECIMAL128I) \ M(TYPE_DECIMAL256) -Status ZoneMapIndexWriter::create(DataTypePtr data_type, StorageField* field, +Status ZoneMapIndexWriter::create(DataTypePtr data_type, const TabletColumn* column, std::unique_ptr& res) { - switch (field->type()) { + switch (column->type()) { #define M(NAME) \ case FieldType::OLAP_FIELD_##NAME: { \ res.reset(new TypedZoneMapIndexWriter(std::move(data_type))); \ diff --git a/be/src/storage/index/zone_map/zone_map_index.h b/be/src/storage/index/zone_map/zone_map_index.h index fe671f07fb3341..1330c5bfe231c0 100644 --- a/be/src/storage/index/zone_map/zone_map_index.h +++ b/be/src/storage/index/zone_map/zone_map_index.h @@ -31,8 +31,8 @@ #include "core/data_type/define_primitive_type.h" #include "core/string_ref.h" #include "io/fs/file_reader_writer_fwd.h" -#include "storage/field.h" #include "storage/metadata_adder.h" +#include "storage/tablet/tablet_schema.h" #include "util/once.h" namespace doris { @@ -88,7 +88,7 @@ struct ZoneMap { class ZoneMapIndexWriter { public: - static Status create(DataTypePtr data_type, StorageField* field, + static Status create(DataTypePtr data_type, const TabletColumn* column, std::unique_ptr& res); ZoneMapIndexWriter() = default; diff --git a/be/src/storage/iterator/olap_data_convertor.cpp b/be/src/storage/iterator/olap_data_convertor.cpp index 3f56e91e6f3bee..42b792ab1c7f18 100644 --- a/be/src/storage/iterator/olap_data_convertor.cpp +++ b/be/src/storage/iterator/olap_data_convertor.cpp @@ -994,7 +994,6 @@ Status OlapBlockDataConvertor::OlapColumnDataConvertorMap::convert_to_olap( _value_convertor->set_source_column(value_typed_column, start_offset, elem_size); RETURN_IF_ERROR(_value_convertor->convert_to_olap()); - // todo (Amory). put this value into MapValue _results[0] = (void*)elem_size; _results[1] = _offsets.data(); _results[2] = _key_convertor->get_data(); diff --git a/be/src/storage/iterator/olap_data_convertor.h b/be/src/storage/iterator/olap_data_convertor.h index 3de07ad1e48874..5d600003d68416 100644 --- a/be/src/storage/iterator/olap_data_convertor.h +++ b/be/src/storage/iterator/olap_data_convertor.h @@ -46,7 +46,6 @@ #include "core/string_ref.h" #include "core/types.h" #include "core/uint24.h" -#include "runtime/collection_value.h" #include "util/slice.h" namespace doris { diff --git a/be/src/storage/iterator/vertical_merge_iterator.cpp b/be/src/storage/iterator/vertical_merge_iterator.cpp index 731686f7218d95..d4d96f274dcaf2 100644 --- a/be/src/storage/iterator/vertical_merge_iterator.cpp +++ b/be/src/storage/iterator/vertical_merge_iterator.cpp @@ -34,7 +34,6 @@ #include "core/string_ref.h" #include "core/types.h" #include "io/cache/block_file_cache_factory.h" -#include "storage/field.h" #include "storage/iterators.h" #include "storage/olap_common.h" diff --git a/be/src/storage/iterator/vgeneric_iterators.cpp b/be/src/storage/iterator/vgeneric_iterators.cpp index eb4ab0b1b6c49a..2458373b34f082 100644 --- a/be/src/storage/iterator/vgeneric_iterators.cpp +++ b/be/src/storage/iterator/vgeneric_iterators.cpp @@ -26,7 +26,6 @@ #include "core/block/column_with_type_and_name.h" #include "core/column/column.h" #include "core/data_type/data_type.h" -#include "storage/field.h" #include "storage/iterators.h" #include "storage/olap_common.h" #include "storage/schema.h" diff --git a/be/src/storage/olap_common.h b/be/src/storage/olap_common.h index 9185ec262699bd..8219acf28b8d0a 100644 --- a/be/src/storage/olap_common.h +++ b/be/src/storage/olap_common.h @@ -116,9 +116,9 @@ struct TabletSize { size_t tablet_size; }; -// Define all data types supported by StorageField. -// If new filed_type is defined, not only new TypeInfo may need be defined, -// but also some functions like get_type_info in types.cpp need to be changed. +// Storage-engine cell types, used by TabletColumn / KeyCoder and the +// data_type traits chain. When adding a new value, also extend CppTypeTraits, +// FieldTypeTraits and the field_type_size() switch in storage/types.h. enum class FieldType { OLAP_FIELD_TYPE_TINYINT = 1, // MYSQL_TYPE_TINY OLAP_FIELD_TYPE_UNSIGNED_TINYINT = 2, @@ -163,10 +163,10 @@ enum class FieldType { OLAP_FIELD_TYPE_TIMESTAMPTZ = 40, }; -// Define all aggregation methods supported by StorageField +// Define all aggregation methods supported by TabletColumn // Note that in practice, not all types can use all the following aggregation methods // For example, it is meaningless to use SUM for the string type (but it will not cause the program to crash) -// The implementation of the StorageField class does not perform such checks, and should be constrained when creating the table +// The implementation of the TabletColumn class does not perform such checks, and should be constrained when creating the table enum class FieldAggregationMethod { OLAP_FIELD_AGGREGATION_NONE = 0, OLAP_FIELD_AGGREGATION_SUM = 1, @@ -281,8 +281,6 @@ struct Vertex { Vertex(int64_t v) : value(v) {} }; -class StorageField; - // ReaderStatistics used to collect statistics when scan data from storage struct OlapReaderStatistics { int64_t io_ns = 0; diff --git a/be/src/storage/row_cursor.cpp b/be/src/storage/row_cursor.cpp index c562e1388ffe84..f5b99f670c7967 100644 --- a/be/src/storage/row_cursor.cpp +++ b/be/src/storage/row_cursor.cpp @@ -27,7 +27,7 @@ #include "common/consts.h" #include "core/data_type/primitive_type.h" #include "core/field.h" -#include "storage/field.h" +#include "storage/key_coder.h" #include "storage/olap_common.h" #include "storage/olap_define.h" #include "storage/tablet/tablet_schema.h" @@ -126,7 +126,7 @@ RowCursor RowCursor::clone() const { void RowCursor::pad_char_fields() { for (size_t i = 0; i < _fields.size(); ++i) { - const StorageField* col = _schema->column(cast_set(i)); + const TabletColumn* col = _schema->column(cast_set(i)); if (col->type() == FieldType::OLAP_FIELD_TYPE_CHAR && !_fields[i].is_null()) { String padded = _fields[i].get(); padded.resize(col->length(), '\0'); @@ -145,40 +145,41 @@ std::string RowCursor::to_string() const { result.append("1&NULL"); } else { result.append("0&"); - result.append(_fields[i].to_debug_string( - _schema->column(cast_set(i))->get_scale())); + result.append( + _fields[i].to_debug_string(_schema->column(cast_set(i))->frac())); } } return result; } -void RowCursor::_encode_field(const StorageField* storage_field, const Field& f, bool full_encode, - std::string* buf) const { - FieldType ft = storage_field->type(); +void RowCursor::_encode_column_value(const TabletColumn* column, const Field& value, + bool full_encode, std::string* buf) const { + FieldType ft = column->type(); + const KeyCoder* coder = get_key_coder(ft); if (field_is_slice_type(ft)) { // String types: CHAR, VARCHAR, STRING — all stored as String in Field. - const String& str = f.get(); + const String& str = value.get(); if (ft == FieldType::OLAP_FIELD_TYPE_CHAR) { // CHAR type: must pad with \0 to the declared column length - size_t col_len = storage_field->length(); + size_t col_len = column->length(); String padded(col_len, '\0'); memcpy(padded.data(), str.data(), std::min(str.size(), col_len)); Slice slice(padded.data(), col_len); if (full_encode) { - storage_field->full_encode_ascending(&slice, buf); + coder->full_encode_ascending(&slice, buf); } else { - storage_field->encode_ascending(&slice, buf); + coder->encode_ascending(&slice, column->index_length(), buf); } } else { // VARCHAR / STRING: use actual length Slice slice(str.data(), str.size()); if (full_encode) { - storage_field->full_encode_ascending(&slice, buf); + coder->full_encode_ascending(&slice, buf); } else { - storage_field->encode_ascending(&slice, buf); + coder->encode_ascending(&slice, column->index_length(), buf); } } return; @@ -187,11 +188,10 @@ void RowCursor::_encode_field(const StorageField* storage_field, const Field& f, // Non-string scalar keys are fixed-width; their KeyCoder::encode_ascending // ignores `index_size` and delegates to full_encode_ascending, so the // `full_encode` flag here is a no-op and we always call the full helper. - const KeyCoder* coder = storage_field->key_coder(); switch (ft) { -#define CASE(FT, PT) \ - case FieldType::FT: \ - full_encode_field_as_key(f, coder, buf); \ +#define CASE(FT, PT) \ + case FieldType::FT: \ + full_encode_field_as_key(value, coder, buf); \ break; DORIS_APPLY_FOR_KEY_ENCODABLE_NON_STRING_TYPES(CASE) #undef CASE @@ -216,8 +216,8 @@ template void RowCursor::encode_key_with_padding(std::string* buf, size_t num_keys, bool padding_minimal) const { for (uint32_t cid = 0; cid < num_keys; cid++) { - auto* storage_field = _schema->column(cid); - if (storage_field == nullptr) { + auto* column = _schema->column(cid); + if (column == nullptr) { if (padding_minimal) { buf->push_back(KeyConsts::KEY_MINIMAL_MARKER); } else { @@ -236,7 +236,7 @@ void RowCursor::encode_key_with_padding(std::string* buf, size_t num_keys, } buf->push_back(KeyConsts::KEY_NORMAL_MARKER); - _encode_field(storage_field, _fields[cid], is_mow, buf); + _encode_column_value(column, _fields[cid], is_mow, buf); } } @@ -252,7 +252,7 @@ void RowCursor::encode_key(std::string* buf, size_t num_keys) const { continue; } buf->push_back(KeyConsts::KEY_NORMAL_MARKER); - _encode_field(_schema->column(cid), _fields[cid], full_encode, buf); + _encode_column_value(_schema->column(cid), _fields[cid], full_encode, buf); } } diff --git a/be/src/storage/row_cursor.h b/be/src/storage/row_cursor.h index 8e8a70daf587fb..7baa029bcd2929 100644 --- a/be/src/storage/row_cursor.h +++ b/be/src/storage/row_cursor.h @@ -35,7 +35,6 @@ namespace doris { #include "common/compile_check_begin.h" -class StorageField; // Delegate the operation of a row of data. // Stores values as core::Field objects instead of raw byte buffers. @@ -66,7 +65,7 @@ class RowCursor { size_t field_count() const { return _fields.size(); } - const StorageField* column_schema(uint32_t cid) const { return _schema->column(cid); } + const TabletColumn* column(uint32_t cid) const { return _schema->column(cid); } const Schema* schema() const { return _schema.get(); } // Returns a deep copy of this RowCursor with the same schema and field values. @@ -96,7 +95,7 @@ class RowCursor { void encode_single_field(uint32_t cid, std::string* buf, bool full_encode) const { const auto& f = _fields[cid]; DCHECK(!f.is_null()); - _encode_field(_schema->column(cid), f, full_encode, buf); + _encode_column_value(_schema->column(cid), f, full_encode, buf); } private: @@ -108,8 +107,8 @@ class RowCursor { // Helper: encode a single non-null field for the given column. // Converts the core::Field to storage format and calls KeyCoder. - void _encode_field(const StorageField* storage_field, const Field& f, bool full_encode, - std::string* buf) const; + void _encode_column_value(const TabletColumn* column, const Field& value, bool full_encode, + std::string* buf) const; std::unique_ptr _schema; std::vector _fields; diff --git a/be/src/storage/schema.cpp b/be/src/storage/schema.cpp index 9a7c59a24d6ab9..4f6cada1e8847c 100644 --- a/be/src/storage/schema.cpp +++ b/be/src/storage/schema.cpp @@ -55,11 +55,9 @@ void Schema::_copy_from(const Schema& other) { _col_ids = other._col_ids; _num_key_columns = other._num_key_columns; - // Deep copy _cols - // TODO(lingbin): really need clone? - _cols.resize(other._cols.size(), nullptr); + _cols.resize(other._cols.size()); for (auto cid : _col_ids) { - _cols[cid] = other._cols[cid]->clone(); + _cols[cid] = other._cols[cid]; } } @@ -68,29 +66,21 @@ void Schema::_init(const std::vector& cols, const std::vector col_id_set(col_ids.begin(), col_ids.end()); for (int cid = 0; cid < cols.size(); ++cid) { if (col_id_set.find(cid) == col_id_set.end()) { continue; } - _cols[cid] = StorageFieldFactory::create(*cols[cid]); + _cols[cid] = cols[cid]; } } -Schema::~Schema() { - for (auto col : _cols) { - delete col; - } -} - -DataTypePtr Schema::get_data_type_ptr(const StorageField& field) { - return DataTypeFactory::instance().create_data_type(field); -} +Schema::~Schema() = default; -IColumn::MutablePtr Schema::get_column_by_field(const StorageField& field) { - return get_data_type_ptr(field)->create_column(); +DataTypePtr Schema::get_data_type_ptr(const TabletColumn& column) { + return DataTypeFactory::instance().create_data_type(column); } IColumn::MutablePtr Schema::get_predicate_column_ptr(const FieldType& type, bool is_nullable, diff --git a/be/src/storage/schema.h b/be/src/storage/schema.h index b036def75dfff6..226bd8ebb2aca1 100644 --- a/be/src/storage/schema.h +++ b/be/src/storage/schema.h @@ -31,7 +31,6 @@ #include "exprs/aggregate/aggregate_function.h" #include "io/io_common.h" #include "runtime/thread_context.h" -#include "storage/field.h" #include "storage/olap_common.h" #include "storage/tablet/tablet_schema.h" #include "storage/utils.h" @@ -126,16 +125,14 @@ class Schema { ~Schema(); - static DataTypePtr get_data_type_ptr(const doris::StorageField& field); - - static IColumn::MutablePtr get_column_by_field(const doris::StorageField& field); + static DataTypePtr get_data_type_ptr(const TabletColumn& column); static IColumn::MutablePtr get_predicate_column_ptr(const FieldType& type, bool is_nullable, const ReaderType reader_type); - const std::vector& columns() const { return _cols; } + const std::vector& columns() const { return _cols; } - const doris::StorageField* column(ColumnId cid) const { return _cols[cid]; } + const TabletColumn* column(ColumnId cid) const { return _cols[cid].get(); } size_t num_key_columns() const { return _num_key_columns; } @@ -165,7 +162,7 @@ class Schema { std::vector _unique_ids; // NOTE: _cols[cid] can only be accessed when the cid is // contained in _col_ids - std::vector _cols; + std::vector _cols; size_t _num_key_columns; int32_t _delete_sign_idx = -1; diff --git a/be/src/storage/schema_change/schema_change.cpp b/be/src/storage/schema_change/schema_change.cpp index 64db6ced4c2f96..2438f5d822b005 100644 --- a/be/src/storage/schema_change/schema_change.cpp +++ b/be/src/storage/schema_change/schema_change.cpp @@ -56,7 +56,6 @@ #include "runtime/runtime_state.h" #include "storage/data_dir.h" #include "storage/delete/delete_handler.h" -#include "storage/field.h" #include "storage/index/inverted/inverted_index_desc.h" #include "storage/index/inverted/inverted_index_writer.h" #include "storage/iterator/olap_data_convertor.h" @@ -90,8 +89,6 @@ namespace doris { #include "common/compile_check_begin.h" -class CollectionValue; - using namespace ErrorCode; constexpr int ALTER_TABLE_BATCH_SIZE = 4064; @@ -1516,12 +1513,6 @@ Status SchemaChangeJob::parse_request(const SchemaChangeParams& sc_params, Status SchemaChangeJob::_init_column_mapping(ColumnMapping* column_mapping, const TabletColumn& column_schema, const std::string& value) { - auto t = StorageFieldFactory::create(column_schema); - Defer defer([t]() { delete t; }); - if (t == nullptr) { - return Status::Uninitialized("Unsupport field creation of {}", column_schema.name()); - } - if (!column_schema.is_nullable() || value.length() != 0) { RETURN_IF_ERROR(column_schema.get_vec_type()->get_serde()->from_fe_string( value, column_mapping->default_value)); diff --git a/be/src/storage/schema_change/schema_change.h b/be/src/storage/schema_change/schema_change.h index c76d2f25f36fd4..fbd302f3a176e8 100644 --- a/be/src/storage/schema_change/schema_change.h +++ b/be/src/storage/schema_change/schema_change.h @@ -54,7 +54,6 @@ namespace doris { class DeleteHandler; -class StorageField; class TAlterInvertedIndexReq; class TAlterTabletReqV2; class TExpr; diff --git a/be/src/storage/segment/column_writer.cpp b/be/src/storage/segment/column_writer.cpp index f09ccc294b1cd2..3f7b0687dd7582 100644 --- a/be/src/storage/segment/column_writer.cpp +++ b/be/src/storage/segment/column_writer.cpp @@ -30,8 +30,6 @@ #include "core/data_type/data_type_factory.hpp" #include "core/types.h" #include "io/fs/file_writer.h" -#include "runtime/collection_value.h" -#include "storage/field.h" #include "storage/index/bloom_filter/bloom_filter_index_writer.h" #include "storage/index/inverted/inverted_index_writer.h" #include "storage/index/ordinal_page_index.h" @@ -140,18 +138,16 @@ inline ScalarColumnWriter* get_null_writer(const ColumnWriterOptions& opts, null_options.need_bloom_filter = false; null_options.encoding_preference = opts.encoding_preference; - TabletColumn null_column = - TabletColumn(FieldAggregationMethod::OLAP_FIELD_AGGREGATION_NONE, null_type, false, - null_options.meta->unique_id(), null_options.meta->length()); - null_column.set_name("nullable"); - null_column.set_index_length(-1); // no short key index - std::unique_ptr null_field(StorageFieldFactory::create(null_column)); - return new ScalarColumnWriter(null_options, std::move(null_field), file_writer); + auto null_column_ptr = std::make_shared( + FieldAggregationMethod::OLAP_FIELD_AGGREGATION_NONE, null_type, false, + null_options.meta->unique_id(), null_options.meta->length()); + null_column_ptr->set_name("nullable"); + null_column_ptr->set_index_length(-1); // no short key index + return new ScalarColumnWriter(null_options, std::move(null_column_ptr), file_writer); } -ColumnWriter::ColumnWriter(std::unique_ptr field, bool is_nullable, - ColumnMetaPB* meta) - : _field(std::move(field)), _is_nullable(is_nullable), _column_meta(meta) { +ColumnWriter::ColumnWriter(TabletColumnPtr column, bool is_nullable, ColumnMetaPB* meta) + : _column(std::move(column)), _is_nullable(is_nullable), _column_meta(meta) { _data_type = DataTypeFactory::instance().create_data_type(*_column_meta); } Status ColumnWriter::create_struct_writer(const ColumnWriterOptions& opts, @@ -181,8 +177,7 @@ Status ColumnWriter::create_struct_writer(const ColumnWriterOptions& opts, get_null_writer(opts, file_writer, column->get_subtype_count() + 1); *writer = std::unique_ptr(new StructColumnWriter( - opts, std::unique_ptr(StorageFieldFactory::create(*column)), null_writer, - sub_column_writers)); + opts, std::make_shared(*column), null_writer, sub_column_writers)); return Status::OK(); } @@ -220,21 +215,20 @@ Status ColumnWriter::create_array_writer(const ColumnWriterOptions& opts, length_options.need_bloom_filter = false; length_options.encoding_preference = opts.encoding_preference; - TabletColumn length_column = - TabletColumn(FieldAggregationMethod::OLAP_FIELD_AGGREGATION_NONE, length_type, - length_options.meta->is_nullable(), length_options.meta->unique_id(), - length_options.meta->length()); - length_column.set_name("length"); - length_column.set_index_length(-1); // no short key index - std::unique_ptr bigint_field(StorageFieldFactory::create(length_column)); + auto length_column_ptr = std::make_shared( + FieldAggregationMethod::OLAP_FIELD_AGGREGATION_NONE, length_type, + length_options.meta->is_nullable(), length_options.meta->unique_id(), + length_options.meta->length()); + length_column_ptr->set_name("length"); + length_column_ptr->set_index_length(-1); // no short key index auto* length_writer = - new OffsetColumnWriter(length_options, std::move(bigint_field), file_writer); + new OffsetColumnWriter(length_options, std::move(length_column_ptr), file_writer); ScalarColumnWriter* null_writer = get_null_writer(opts, file_writer, 3); - *writer = std::unique_ptr(new ArrayColumnWriter( - opts, std::unique_ptr(StorageFieldFactory::create(*column)), - length_writer, null_writer, std::move(item_writer))); + *writer = std::unique_ptr( + new ArrayColumnWriter(opts, std::make_shared(*column), length_writer, + null_writer, std::move(item_writer))); return Status::OK(); } @@ -284,22 +278,21 @@ Status ColumnWriter::create_map_writer(const ColumnWriterOptions& opts, const Ta length_options.need_bloom_filter = false; length_options.encoding_preference = opts.encoding_preference; - TabletColumn length_column = - TabletColumn(FieldAggregationMethod::OLAP_FIELD_AGGREGATION_NONE, length_type, - length_options.meta->is_nullable(), length_options.meta->unique_id(), - length_options.meta->length()); - length_column.set_name("length"); - length_column.set_index_length(-1); // no short key index - std::unique_ptr bigint_field(StorageFieldFactory::create(length_column)); + auto length_column_ptr = std::make_shared( + FieldAggregationMethod::OLAP_FIELD_AGGREGATION_NONE, length_type, + length_options.meta->is_nullable(), length_options.meta->unique_id(), + length_options.meta->length()); + length_column_ptr->set_name("length"); + length_column_ptr->set_index_length(-1); // no short key index auto* length_writer = - new OffsetColumnWriter(length_options, std::move(bigint_field), file_writer); + new OffsetColumnWriter(length_options, std::move(length_column_ptr), file_writer); ScalarColumnWriter* null_writer = get_null_writer(opts, file_writer, column->get_subtype_count() + 2); - *writer = std::unique_ptr(new MapColumnWriter( - opts, std::unique_ptr(StorageFieldFactory::create(*column)), null_writer, - length_writer, inner_writer_list)); + *writer = std::unique_ptr( + new MapColumnWriter(opts, std::make_shared(*column), null_writer, + length_writer, inner_writer_list)); return Status::OK(); } @@ -313,9 +306,8 @@ Status ColumnWriter::create_agg_state_writer(const ColumnWriterOptions& opts, auto type = agg_state_type->get_serialized_type()->get_primitive_type(); if (type == PrimitiveType::TYPE_STRING || type == PrimitiveType::INVALID_TYPE || type == PrimitiveType::TYPE_FIXED_LENGTH_OBJECT || type == PrimitiveType::TYPE_BITMAP) { - *writer = std::unique_ptr(new ScalarColumnWriter( - opts, std::unique_ptr(StorageFieldFactory::create(*column)), - file_writer)); + *writer = std::unique_ptr( + new ScalarColumnWriter(opts, std::make_shared(*column), file_writer)); } else if (type == PrimitiveType::TYPE_ARRAY) { RETURN_IF_ERROR(create_array_writer(opts, column, file_writer, writer)); } else if (type == PrimitiveType::TYPE_MAP) { @@ -339,28 +331,25 @@ Status ColumnWriter::create_variant_writer(const ColumnWriterOptions& opts, if (column->is_extracted_column()) { if (column->name().find(DOC_VALUE_COLUMN_PATH) != std::string::npos) { *writer = std::make_unique( - opts, column, - std::unique_ptr(StorageFieldFactory::create(*column))); + opts, std::make_shared(*column)); return Status::OK(); } VLOG_DEBUG << "gen subwriter for " << column->path_info_ptr()->get_path(); - *writer = std::make_unique( - opts, column, std::unique_ptr(StorageFieldFactory::create(*column))); + *writer = std::make_unique(opts, + std::make_shared(*column)); return Status::OK(); } - *writer = std::make_unique( - opts, column, std::unique_ptr(StorageFieldFactory::create(*column))); + *writer = std::make_unique(opts, std::make_shared(*column)); return Status::OK(); } //Todo(Amory): here should according nullable and offset and need sub to simply this function Status ColumnWriter::create(const ColumnWriterOptions& opts, const TabletColumn* column, io::FileWriter* file_writer, std::unique_ptr* writer) { - std::unique_ptr field(StorageFieldFactory::create(*column)); - DCHECK(field.get() != nullptr); + auto column_ptr = std::make_shared(*column); if (is_scalar_type(column->type())) { *writer = std::unique_ptr( - new ScalarColumnWriter(opts, std::move(field), file_writer)); + new ScalarColumnWriter(opts, std::move(column_ptr), file_writer)); return Status::OK(); } else { switch (column->type()) { @@ -387,7 +376,7 @@ Status ColumnWriter::create(const ColumnWriterOptions& opts, const TabletColumn* } default: return Status::NotSupported("unsupported type for ColumnWriter: {}", - std::to_string(int(field->type()))); + std::to_string(int(column_ptr->type()))); } } } @@ -418,7 +407,7 @@ Status ColumnWriter::append_nullable(const uint8_t* null_map, const uint8_t** pt if (non_null_count == 0) { // All NULL: skip run-length iteration, directly append all nulls RETURN_IF_ERROR(append_nulls(num_rows)); - *ptr += get_field()->size() * num_rows; + *ptr += cell_size() * num_rows; return Status::OK(); } @@ -446,10 +435,10 @@ Status ColumnWriter::append_nullable(const uint8_t* null_map, const uint8_t** pt auto step = next_run_step(); if (null_map[offset]) { RETURN_IF_ERROR(append_nulls(step)); - *ptr += get_field()->size() * step; + *ptr += cell_size() * step; } else { // TODO: - // 1. `*ptr += get_field()->size() * step;` should do in this function, not append_data; + // 1. `*ptr += cell_size() * step;` should do in this function, not append_data; // 2. support array vectorized load and ptr offset add RETURN_IF_ERROR(append_data(ptr, step)); } @@ -471,10 +460,9 @@ Status ColumnWriter::append(const uint8_t* nullmap, const void* data, size_t num /////////////////////////////////////////////////////////////////////////////////// -ScalarColumnWriter::ScalarColumnWriter(const ColumnWriterOptions& opts, - std::unique_ptr field, +ScalarColumnWriter::ScalarColumnWriter(const ColumnWriterOptions& opts, TabletColumnPtr column, io::FileWriter* file_writer) - : ColumnWriter(std::move(field), opts.meta->is_nullable(), opts.meta), + : ColumnWriter(std::move(column), opts.meta->is_nullable(), opts.meta), _opts(opts), _file_writer(file_writer), _data_size(0) { @@ -500,7 +488,7 @@ Status ScalarColumnWriter::init() { PageBuilder* page_builder = nullptr; - RETURN_IF_ERROR(EncodingInfo::get(get_field()->type(), _opts.meta->encoding(), + RETURN_IF_ERROR(EncodingInfo::get(get_column()->type(), _opts.meta->encoding(), _opts.encoding_preference, &_encoding_info)); _opts.meta->set_encoding(_encoding_info->encoding()); // create page builder @@ -511,7 +499,7 @@ Status ScalarColumnWriter::init() { RETURN_IF_ERROR(_encoding_info->create_page_builder(opts, &page_builder)); if (page_builder == nullptr) { return Status::NotSupported("Failed to create page builder for type {} and encoding {}", - get_field()->type(), _opts.meta->encoding()); + get_column()->type(), _opts.meta->encoding()); } // should store more concrete encoding type instead of DEFAULT_ENCODING // because the default encoding of a data type can be changed in the future @@ -519,7 +507,7 @@ Status ScalarColumnWriter::init() { VLOG_DEBUG << fmt::format( "[verbose] scalar column writer init, column_id={}, type={}, encoding={}, " "is_nullable={}", - _opts.meta->column_id(), get_field()->type(), + _opts.meta->column_id(), get_column()->type(), EncodingTypePB_Name(_opts.meta->encoding()), _opts.meta->is_nullable()); _page_builder.reset(page_builder); // create ordinal builder @@ -530,7 +518,7 @@ Status ScalarColumnWriter::init() { } if (_opts.need_zone_map) { RETURN_IF_ERROR( - ZoneMapIndexWriter::create(_data_type, get_field(), _zone_map_index_builder)); + ZoneMapIndexWriter::create(_data_type, get_column(), _zone_map_index_builder)); } if (_opts.need_inverted_index) { @@ -544,10 +532,6 @@ Status ScalarColumnWriter::init() { size_t count) override { return Status::OK(); } - Status add_array_values(size_t field_size, const CollectionValue* values, - size_t count) override { - return Status::OK(); - } Status add_array_values(size_t field_size, const void* value_ptr, const uint8_t* null_map, const uint8_t* offsets_ptr, size_t count) override { @@ -568,19 +552,19 @@ Status ScalarColumnWriter::init() { break; }); - RETURN_IF_ERROR(IndexColumnWriter::create(get_field(), &_inverted_index_builders[i], - _opts.index_file_writer, - _opts.inverted_indexes[i])); + RETURN_IF_ERROR(IndexColumnWriter::create( + get_column(), &_inverted_index_builders[i], _opts.index_file_writer, + _opts.inverted_indexes[i])); } } while (false); } if (_opts.need_bloom_filter) { if (_opts.is_ngram_bf_index) { RETURN_IF_ERROR(NGramBloomFilterIndexWriterImpl::create( - BloomFilterOptions(), get_field()->type(), _opts.gram_size, _opts.gram_bf_size, + BloomFilterOptions(), get_column()->type(), _opts.gram_size, _opts.gram_bf_size, &_bloom_filter_index_builder)); } else { - RETURN_IF_ERROR(BloomFilterIndexWriter::create(_opts.bf_options, get_field()->type(), + RETURN_IF_ERROR(BloomFilterIndexWriter::create(_opts.bf_options, get_column()->type(), &_bloom_filter_index_builder)); } } @@ -630,7 +614,7 @@ Status ScalarColumnWriter::_internal_append_data_in_current_page(const uint8_t* } if (_opts.need_inverted_index) { for (const auto& builder : _inverted_index_builders) { - RETURN_IF_ERROR(builder->add_values(get_field()->name(), data, *num_written)); + RETURN_IF_ERROR(builder->add_values(get_column()->name(), data, *num_written)); } } if (_opts.need_bloom_filter) { @@ -649,7 +633,7 @@ Status ScalarColumnWriter::_internal_append_data_in_current_page(const uint8_t* Status ScalarColumnWriter::append_data_in_current_page(const uint8_t** data, size_t* num_written) { RETURN_IF_ERROR(append_data_in_current_page(*data, num_written)); - *data += get_field()->size() * (*num_written); + *data += cell_size() * (*num_written); return Status::OK(); } @@ -699,7 +683,7 @@ Status ScalarColumnWriter::append_nullable(const uint8_t* null_map, const uint8_ if (non_null_count == 0) { // All NULL: skip data writing, only update null bitmap and indexes RETURN_IF_ERROR(append_nulls(num_rows)); - *ptr += get_field()->size() * num_rows; + *ptr += cell_size() * num_rows; return Status::OK(); } @@ -713,10 +697,10 @@ Status ScalarColumnWriter::append_nullable(const uint8_t* null_map, const uint8_ size_t run_length = run.len; if (run.is_null) { RETURN_IF_ERROR(append_nulls(run_length)); - *ptr += get_field()->size() * run_length; + *ptr += cell_size() * run_length; } else { // TODO: - // 1. `*ptr += get_field()->size() * step;` should do in this function, not append_data; + // 1. `*ptr += cell_size() * step;` should do in this function, not append_data; // 2. support array vectorized load and ptr offset add RETURN_IF_ERROR(append_data(ptr, run_length)); } @@ -889,12 +873,11 @@ Status ScalarColumnWriter::finish_current_page() { // offset column writer //////////////////////////////////////////////////////////////////////////////// -OffsetColumnWriter::OffsetColumnWriter(const ColumnWriterOptions& opts, - std::unique_ptr field, +OffsetColumnWriter::OffsetColumnWriter(const ColumnWriterOptions& opts, TabletColumnPtr column, io::FileWriter* file_writer) - : ScalarColumnWriter(opts, std::move(field), file_writer) { + : ScalarColumnWriter(opts, std::move(column), file_writer) { // now we only explain data in offset column as uint64 - DCHECK(get_field()->type() == FieldType::OLAP_FIELD_TYPE_UNSIGNED_BIGINT); + DCHECK(get_column()->type() == FieldType::OLAP_FIELD_TYPE_UNSIGNED_BIGINT); } OffsetColumnWriter::~OffsetColumnWriter() = default; @@ -929,10 +912,9 @@ void OffsetColumnWriter::put_extra_info_in_page(DataPageFooterPB* footer) { } StructColumnWriter::StructColumnWriter( - const ColumnWriterOptions& opts, std::unique_ptr field, - ScalarColumnWriter* null_writer, + const ColumnWriterOptions& opts, TabletColumnPtr column, ScalarColumnWriter* null_writer, std::vector>& sub_column_writers) - : ColumnWriter(std::move(field), opts.meta->is_nullable(), opts.meta), _opts(opts) { + : ColumnWriter(std::move(column), opts.meta->is_nullable(), opts.meta), _opts(opts) { for (auto& sub_column_writer : sub_column_writers) { _sub_column_writers.push_back(std::move(sub_column_writer)); } @@ -1037,12 +1019,11 @@ Status StructColumnWriter::finish_current_page() { return Status::NotSupported("struct writer has no data, can not finish_current_page"); } -ArrayColumnWriter::ArrayColumnWriter(const ColumnWriterOptions& opts, - std::unique_ptr field, +ArrayColumnWriter::ArrayColumnWriter(const ColumnWriterOptions& opts, TabletColumnPtr column, OffsetColumnWriter* offset_writer, ScalarColumnWriter* null_writer, std::unique_ptr item_writer) - : ColumnWriter(std::move(field), opts.meta->is_nullable(), opts.meta), + : ColumnWriter(std::move(column), opts.meta->is_nullable(), opts.meta), _item_writer(std::move(item_writer)), _opts(opts) { _offset_writer.reset(offset_writer); @@ -1060,7 +1041,7 @@ Status ArrayColumnWriter::init() { if (_opts.need_inverted_index) { auto* writer = dynamic_cast(_item_writer.get()); if (writer != nullptr) { - RETURN_IF_ERROR(IndexColumnWriter::create(get_field(), &_inverted_index_writer, + RETURN_IF_ERROR(IndexColumnWriter::create(get_column(), &_inverted_index_writer, _opts.index_file_writer, _opts.inverted_indexes[0])); } @@ -1111,7 +1092,8 @@ Status ArrayColumnWriter::append_data(const uint8_t** ptr, size_t num_rows) { if (writer != nullptr) { //NOTE: use array field name as index field, but item_writer size should be used when moving item_data_ptr RETURN_IF_ERROR(_inverted_index_writer->add_array_values( - _item_writer->get_field()->size(), reinterpret_cast(data), + field_type_size(_item_writer->get_column()->type()), + reinterpret_cast(data), reinterpret_cast(nested_null_map), offsets_ptr, num_rows)); } } @@ -1122,13 +1104,14 @@ Status ArrayColumnWriter::append_data(const uint8_t** ptr, size_t num_rows) { if (writer != nullptr) { //NOTE: use array field name as index field, but item_writer size should be used when moving item_data_ptr RETURN_IF_ERROR(_ann_index_writer->add_array_values( - _item_writer->get_field()->size(), reinterpret_cast(data), + field_type_size(_item_writer->get_column()->type()), + reinterpret_cast(data), reinterpret_cast(nested_null_map), offsets_ptr, num_rows)); } else { return Status::NotSupported( "Ann index can only be build on array with scalar type. but got {} as " "nested", - _item_writer->get_field()->type()); + _item_writer->get_column()->type()); } } @@ -1208,11 +1191,10 @@ Status ArrayColumnWriter::finish_current_page() { } /// ============================= MapColumnWriter =====================//// -MapColumnWriter::MapColumnWriter(const ColumnWriterOptions& opts, - std::unique_ptr field, +MapColumnWriter::MapColumnWriter(const ColumnWriterOptions& opts, TabletColumnPtr column, ScalarColumnWriter* null_writer, OffsetColumnWriter* offset_writer, std::vector>& kv_writers) - : ColumnWriter(std::move(field), opts.meta->is_nullable(), opts.meta), _opts(opts) { + : ColumnWriter(std::move(column), opts.meta->is_nullable(), opts.meta), _opts(opts) { CHECK_EQ(kv_writers.size(), 2); _offsets_writer.reset(offset_writer); if (is_nullable()) { @@ -1347,11 +1329,9 @@ Status MapColumnWriter::write_inverted_index() { return Status::OK(); } -VariantColumnWriter::VariantColumnWriter(const ColumnWriterOptions& opts, - const TabletColumn* column, - std::unique_ptr field) - : ColumnWriter(std::move(field), opts.meta->is_nullable(), opts.meta) { - _impl = std::make_unique(opts, column); +VariantColumnWriter::VariantColumnWriter(const ColumnWriterOptions& opts, TabletColumnPtr column) + : ColumnWriter(std::move(column), opts.meta->is_nullable(), opts.meta) { + _impl = std::make_unique(opts, get_column()); } Status VariantColumnWriter::init() { diff --git a/be/src/storage/segment/column_writer.h b/be/src/storage/segment/column_writer.h index 8a87be44ebe0a1..44567c2d8d3f6c 100644 --- a/be/src/storage/segment/column_writer.h +++ b/be/src/storage/segment/column_writer.h @@ -31,7 +31,6 @@ #include "common/status.h" // for Status #include "core/column/column_variant.h" -#include "storage/field.h" // for StorageField #include "storage/index/ann/ann_index_writer.h" #include "storage/index/bloom_filter/bloom_filter.h" #include "storage/index/inverted/inverted_index_writer.h" @@ -39,8 +38,10 @@ #include "storage/segment/options.h" #include "storage/segment/variant/nested_group_provider.h" #include "storage/segment/variant/variant_statistics.h" -#include "util/bitmap.h" // for BitmapChange -#include "util/slice.h" // for OwnedSlice +#include "storage/tablet/tablet_schema.h" // for TabletColumnPtr +#include "storage/types.h" // for field_type_size +#include "util/bitmap.h" // for BitmapChange +#include "util/slice.h" // for OwnedSlice namespace doris { @@ -127,8 +128,7 @@ class ColumnWriter { const TabletColumn* column, io::FileWriter* file_writer, std::unique_ptr* writer); - explicit ColumnWriter(std::unique_ptr field, bool is_nullable, - ColumnMetaPB* meta); + explicit ColumnWriter(TabletColumnPtr column, bool is_nullable, ColumnMetaPB* meta); virtual ~ColumnWriter() = default; @@ -194,7 +194,11 @@ class ColumnWriter { bool is_nullable() const { return _is_nullable; } - StorageField* get_field() const { return _field.get(); } + const TabletColumn* get_column() const { return _column.get(); } + + // Per-row in-memory cell footprint of this writer's column, used to step + // the input pointer across rows in append_*/null-run loops. + size_t cell_size() const { return field_type_size(_column->type()); } ColumnMetaPB* get_column_meta() const { return _column_meta; } @@ -202,7 +206,7 @@ class ColumnWriter { DataTypePtr _data_type; private: - std::unique_ptr _field; + TabletColumnPtr _column; bool _is_nullable; ColumnMetaPB* _column_meta; std::vector _null_bitmap; @@ -220,7 +224,7 @@ class FlushPageCallback { // to file class ScalarColumnWriter : public ColumnWriter { public: - ScalarColumnWriter(const ColumnWriterOptions& opts, std::unique_ptr field, + ScalarColumnWriter(const ColumnWriterOptions& opts, TabletColumnPtr column, io::FileWriter* file_writer); ~ScalarColumnWriter() override; @@ -341,7 +345,7 @@ class ScalarColumnWriter : public ColumnWriter { // in footer.next_array_item_ordinal which in finish_cur_page() callback put_extra_info_in_page() class OffsetColumnWriter final : public ScalarColumnWriter, FlushPageCallback { public: - OffsetColumnWriter(const ColumnWriterOptions& opts, std::unique_ptr field, + OffsetColumnWriter(const ColumnWriterOptions& opts, TabletColumnPtr column, io::FileWriter* file_writer); ~OffsetColumnWriter() override; @@ -358,8 +362,7 @@ class OffsetColumnWriter final : public ScalarColumnWriter, FlushPageCallback { class StructColumnWriter final : public ColumnWriter { public: - explicit StructColumnWriter(const ColumnWriterOptions& opts, - std::unique_ptr field, + explicit StructColumnWriter(const ColumnWriterOptions& opts, TabletColumnPtr column, ScalarColumnWriter* null_writer, std::vector>& sub_column_writers); ~StructColumnWriter() override = default; @@ -426,7 +429,7 @@ class StructColumnWriter final : public ColumnWriter { class ArrayColumnWriter final : public ColumnWriter { public: - explicit ArrayColumnWriter(const ColumnWriterOptions& opts, std::unique_ptr field, + explicit ArrayColumnWriter(const ColumnWriterOptions& opts, TabletColumnPtr column, OffsetColumnWriter* offset_writer, ScalarColumnWriter* null_writer, std::unique_ptr item_writer); ~ArrayColumnWriter() override = default; @@ -500,7 +503,7 @@ class ArrayColumnWriter final : public ColumnWriter { class MapColumnWriter final : public ColumnWriter { public: - explicit MapColumnWriter(const ColumnWriterOptions& opts, std::unique_ptr field, + explicit MapColumnWriter(const ColumnWriterOptions& opts, TabletColumnPtr column, ScalarColumnWriter* null_writer, OffsetColumnWriter* offsets_writer, std::vector>& _kv_writers); @@ -574,8 +577,7 @@ class MapColumnWriter final : public ColumnWriter { // used for compaction to write sub variant column class VariantSubcolumnWriter : public ColumnWriter { public: - explicit VariantSubcolumnWriter(const ColumnWriterOptions& opts, const TabletColumn* column, - std::unique_ptr field); + explicit VariantSubcolumnWriter(const ColumnWriterOptions& opts, TabletColumnPtr column); ~VariantSubcolumnWriter() override = default; @@ -626,7 +628,6 @@ class VariantSubcolumnWriter : public ColumnWriter { ordinal_t _next_rowid = 0; size_t none_null_size = 0; ColumnVariant::MutablePtr _column; - const TabletColumn* _tablet_column = nullptr; ColumnWriterOptions _opts; std::unique_ptr _writer; TabletIndexes _indexes; @@ -637,8 +638,7 @@ class VariantSubcolumnWriter : public ColumnWriter { class VariantColumnWriter : public ColumnWriter { public: - explicit VariantColumnWriter(const ColumnWriterOptions& opts, const TabletColumn* column, - std::unique_ptr field); + explicit VariantColumnWriter(const ColumnWriterOptions& opts, TabletColumnPtr column); ~VariantColumnWriter() override = default; diff --git a/be/src/storage/segment/segment.cpp b/be/src/storage/segment/segment.cpp index 574248725e581f..70888a67834b58 100644 --- a/be/src/storage/segment/segment.cpp +++ b/be/src/storage/segment/segment.cpp @@ -57,6 +57,7 @@ #include "storage/index/short_key_index.h" #include "storage/iterator/vgeneric_iterators.h" #include "storage/iterators.h" +#include "storage/key_coder.h" #include "storage/olap_common.h" #include "storage/predicate/block_column_predicate.h" #include "storage/predicate/column_predicate.h" diff --git a/be/src/storage/segment/segment.h b/be/src/storage/segment/segment.h index 5b1870979d83df..66dcc891735db7 100644 --- a/be/src/storage/segment/segment.h +++ b/be/src/storage/segment/segment.h @@ -39,7 +39,6 @@ #include "io/fs/file_system.h" #include "runtime/descriptors.h" #include "storage/cache/page_cache.h" -#include "storage/field.h" #include "storage/olap_common.h" #include "storage/schema.h" #include "storage/segment/column_reader.h" @@ -187,9 +186,9 @@ class Segment : public std::enable_shared_from_this, public MetadataAdd int cid, const Schema& schema, const std::map& target_cast_type_for_variants, const StorageReadOptions& read_options) { - const doris::StorageField* col = schema.column(cid); + const TabletColumn* col = schema.column(cid); DCHECK(col != nullptr) << "Column not found in schema for cid=" << cid; - DataTypePtr storage_column_type = get_data_type_of(col->get_desc(), read_options); + DataTypePtr storage_column_type = get_data_type_of(*col, read_options); if (storage_column_type == nullptr || col->type() != FieldType::OLAP_FIELD_TYPE_VARIANT || !target_cast_type_for_variants.contains(col->name())) { // Default column iterator or not variant column diff --git a/be/src/storage/segment/segment_iterator.cpp b/be/src/storage/segment/segment_iterator.cpp index 95a0850e90b7a1..87a4ce53e9f4fd 100644 --- a/be/src/storage/segment/segment_iterator.cpp +++ b/be/src/storage/segment/segment_iterator.cpp @@ -73,7 +73,6 @@ #include "runtime/runtime_state.h" #include "runtime/thread_context.h" #include "storage/compaction/collection_similarity.h" -#include "storage/field.h" #include "storage/id_manager.h" #include "storage/index/ann/ann_index.h" #include "storage/index/ann/ann_index_iterator.h" @@ -422,12 +421,12 @@ Status SegmentIterator::_init_impl(const StorageReadOptions& opts) { _storage_name_and_type.resize(_schema->columns().size()); auto storage_format = _opts.tablet_schema->get_inverted_index_storage_format(); for (int i = 0; i < _schema->columns().size(); ++i) { - const StorageField* col = _schema->column(i); + const TabletColumn* col = _schema->column(i); if (col) { - auto storage_type = _segment->get_data_type_of(col->get_desc(), _opts); + auto storage_type = _segment->get_data_type_of(*col, _opts); if (storage_type == nullptr) { - storage_type = DataTypeFactory::instance().create_data_type(col->get_desc(), - col->is_nullable()); + storage_type = + DataTypeFactory::instance().create_data_type(*col, col->is_nullable()); } // Currently, when writing a lucene index, the field of the document is column_name, and the column name is // bound to the index field. Since version 1.2, the data file storage has been changed from column_name to @@ -449,7 +448,9 @@ Status SegmentIterator::_init_impl(const StorageReadOptions& opts) { } } _storage_name_and_type[i] = std::make_pair(field_name, storage_type); - if (int32_t uid = col->get_unique_id(); !_variant_sparse_column_cache.contains(uid)) { + if (int32_t uid = + col->is_extracted_column() ? col->parent_unique_id() : col->unique_id(); + !_variant_sparse_column_cache.contains(uid)) { DCHECK(uid >= 0); _variant_sparse_column_cache.emplace(uid, std::make_unique()); @@ -684,10 +685,11 @@ Status SegmentIterator::_get_row_ranges_by_keys() { } // Read & seek key columns is a waste of time when no key column in _schema - if (std::none_of( - _schema->columns().begin(), _schema->columns().end(), [&](const StorageField* col) { - return col && _opts.tablet_schema->column_by_uid(col->unique_id()).is_key(); - })) { + if (std::none_of(_schema->columns().begin(), _schema->columns().end(), + [&](const TabletColumnPtr& col) { + return col && + _opts.tablet_schema->column_by_uid(col->unique_id()).is_key(); + })) { return Status::OK(); } @@ -719,29 +721,27 @@ Status SegmentIterator::_get_row_ranges_by_keys() { // Set up environment for the following seek. Status SegmentIterator::_prepare_seek(const StorageReadOptions::KeyRange& key_range) { - std::vector key_fields; + std::vector key_columns; std::set column_set; if (key_range.lower_key != nullptr) { for (auto cid : key_range.lower_key->schema()->column_ids()) { column_set.emplace(cid); - key_fields.emplace_back(key_range.lower_key->column_schema(cid)); + key_columns.emplace_back(key_range.lower_key->column(cid)); } } if (key_range.upper_key != nullptr) { for (auto cid : key_range.upper_key->schema()->column_ids()) { if (column_set.count(cid) == 0) { - key_fields.emplace_back(key_range.upper_key->column_schema(cid)); + key_columns.emplace_back(key_range.upper_key->column(cid)); column_set.emplace(cid); } } } if (!_seek_schema) { - // Schema constructors accept a vector of TabletColumnPtr. Convert - // StorageField pointers to TabletColumnPtr by copying their descriptors. std::vector cols; - cols.reserve(key_fields.size()); - for (const StorageField* f : key_fields) { - cols.emplace_back(std::make_shared(f->get_desc())); + cols.reserve(key_columns.size()); + for (const TabletColumn* col : key_columns) { + cols.emplace_back(std::make_shared(*col)); } _seek_schema = std::make_unique(cols, cols.size()); } @@ -751,7 +751,7 @@ Status SegmentIterator::_prepare_seek(const StorageReadOptions::KeyRange& key_ra int i = 0; for (auto cid : _seek_schema->column_ids()) { auto column_desc = _seek_schema->column(cid); - _seek_block[i] = Schema::get_column_by_field(*column_desc); + _seek_block[i] = Schema::get_data_type_ptr(*column_desc)->create_column(); i++; } } @@ -2085,18 +2085,18 @@ bool SegmentIterator::_can_evaluated_by_vectorized(std::shared_ptrcolumns().size(), false); for (size_t i = 0; i < _schema->num_column_ids(); i++) { auto cid = _schema->column_id(i); - const StorageField* column_desc = _schema->column(cid); + const TabletColumn* column_desc = _schema->column(cid); // The additional deleted filter condition will be in the materialized column at the end of the block. // After _output_column_by_sel_idx, it will be erased, so we do not need to shrink it. @@ -2184,7 +2184,9 @@ Status SegmentIterator::_init_current_block(Block* block, "col_path {}", block->get_by_position(i).type->get_name(), file_column_type->get_name(), column_desc->name(), - column_desc->path() == nullptr ? "" : column_desc->path()->get_path()); + column_desc->path_info_ptr() == nullptr + ? "" + : column_desc->path_info_ptr()->get_path()); // TODO reuse current_columns[cid] = file_column_type->create_column(); current_columns[cid]->reserve(nrows_read_limit); @@ -2656,8 +2658,8 @@ Status SegmentIterator::_convert_to_expected_type(const std::vector& c if (!_current_return_columns[i] || _converted_column_ids[i] || _is_pred_column[i]) { continue; } - const StorageField* field_type = _schema->column(i); - DataTypePtr expected_type = Schema::get_data_type_ptr(*field_type); + const TabletColumn* column_desc = _schema->column(i); + DataTypePtr expected_type = Schema::get_data_type_ptr(*column_desc); DataTypePtr file_column_type = _storage_name_and_type[i].second; if (!file_column_type->equals(*expected_type)) { ColumnPtr expected; @@ -2666,11 +2668,12 @@ Status SegmentIterator::_convert_to_expected_type(const std::vector& c expected_type, &expected)); _current_return_columns[i] = expected->assume_mutable(); _converted_column_ids[i] = true; - VLOG_DEBUG << fmt::format( - "Convert {} fom file column type {} to {}, num_rows {}", - field_type->path() == nullptr ? "" : field_type->path()->get_path(), - file_column_type->get_name(), expected_type->get_name(), - _current_return_columns[i]->size()); + VLOG_DEBUG << fmt::format("Convert {} fom file column type {} to {}, num_rows {}", + column_desc->path_info_ptr() == nullptr + ? "" + : column_desc->path_info_ptr()->get_path(), + file_column_type->get_name(), expected_type->get_name(), + _current_return_columns[i]->size()); } } return Status::OK(); diff --git a/be/src/storage/segment/segment_iterator.h b/be/src/storage/segment/segment_iterator.h index f6fed55df2f077..e828e6495ddea0 100644 --- a/be/src/storage/segment/segment_iterator.h +++ b/be/src/storage/segment/segment_iterator.h @@ -44,7 +44,6 @@ #include "exprs/vexpr_fwd.h" #include "io/fs/file_reader_writer_fwd.h" #include "runtime/runtime_profile.h" -#include "storage/field.h" #include "storage/index/ann/ann_topn_runtime.h" #include "storage/index/index_iterator.h" #include "storage/iterators.h" @@ -208,7 +207,7 @@ class SegmentIterator : public RowwiseIterator { // CHAR type in storage layer padding the 0 in length. But query engine need ignore the padding 0. // so segment iterator need to shrink char column before output it. only use in vec query engine. void _vec_init_char_column_id(Block* block); - bool _has_char_type(const StorageField& column_desc); + bool _has_char_type(const TabletColumn& column_desc); uint32_t segment_id() const { return _segment->id(); } uint32_t num_rows() const { return _segment->num_rows(); } @@ -252,8 +251,7 @@ class SegmentIterator : public RowwiseIterator { if (block_cid >= block->columns()) { continue; } - DataTypePtr storage_type = - _segment->get_data_type_of(_schema->column(cid)->get_desc(), _opts); + DataTypePtr storage_type = _segment->get_data_type_of(*_schema->column(cid), _opts); if (storage_type && !storage_type->equals(*block->get_by_position(block_cid).type)) { // Do additional cast MutableColumnPtr tmp = storage_type->create_column(); diff --git a/be/src/storage/segment/variant/binary_column_extract_iterator.h b/be/src/storage/segment/variant/binary_column_extract_iterator.h index 655005f611ecd7..ef09d40b30bbda 100644 --- a/be/src/storage/segment/variant/binary_column_extract_iterator.h +++ b/be/src/storage/segment/variant/binary_column_extract_iterator.h @@ -41,7 +41,6 @@ #include "core/types.h" #include "exprs/function/function_helpers.h" #include "io/io_common.h" -#include "storage/field.h" #include "storage/iterators.h" #include "storage/schema.h" #include "storage/segment/column_reader.h" diff --git a/be/src/storage/segment/variant/hierarchical_data_iterator.h b/be/src/storage/segment/variant/hierarchical_data_iterator.h index 8c0e3366b1e006..60198d09c422e3 100644 --- a/be/src/storage/segment/variant/hierarchical_data_iterator.h +++ b/be/src/storage/segment/variant/hierarchical_data_iterator.h @@ -41,7 +41,6 @@ #include "core/types.h" #include "exprs/function/function_helpers.h" #include "io/io_common.h" -#include "storage/field.h" #include "storage/iterators.h" #include "storage/schema.h" #include "storage/segment/column_reader.h" diff --git a/be/src/storage/segment/variant/sparse_column_merge_iterator.h b/be/src/storage/segment/variant/sparse_column_merge_iterator.h index eb0babbf642164..4b96ca1f39b078 100644 --- a/be/src/storage/segment/variant/sparse_column_merge_iterator.h +++ b/be/src/storage/segment/variant/sparse_column_merge_iterator.h @@ -41,7 +41,6 @@ #include "core/types.h" #include "exprs/function/function_helpers.h" #include "io/io_common.h" -#include "storage/field.h" #include "storage/iterators.h" #include "storage/schema.h" #include "storage/segment/column_reader.h" diff --git a/be/src/storage/segment/variant/variant_column_writer_impl.cpp b/be/src/storage/segment/variant/variant_column_writer_impl.cpp index 1dccec87807eba..edb2ecca44e1d8 100644 --- a/be/src/storage/segment/variant/variant_column_writer_impl.cpp +++ b/be/src/storage/segment/variant/variant_column_writer_impl.cpp @@ -535,7 +535,6 @@ Status append_sparse_converted_column(const TabletColumn& tablet_column, ColumnW const DataTypePtr& type, const ColumnPtr& values_column, const std::vector& rowids, size_t total_rows) { DCHECK_EQ(values_column->size(), rowids.size()); - const size_t cell_size = writer->get_field()->size(); auto base_type = type; if (base_type->is_nullable()) { @@ -595,6 +594,9 @@ Status append_sparse_converted_column(const TabletColumn& tablet_column, ColumnW return writer->append_nulls(total_rows); } + // Non-ARRAY scalar path: writer cell is strided by sizeof(CppType). + const size_t cell_size = field_type_size(writer->get_column()->type()); + converter->add_column_data_convertor(tablet_column); RETURN_IF_ERROR(converter->set_source_content_with_specifid_column({values_column, type, ""}, 0, rowids.size(), cid)); @@ -1554,8 +1556,7 @@ Status VariantColumnWriterImpl::_process_root_column(ColumnVariant* ptr, size_t num_rows, int& column_id) { // root column _root_writer = std::make_unique( - _opts, std::unique_ptr(StorageFieldFactory::create(*_tablet_column)), - _opts.file_writer); + _opts, std::make_shared(*_tablet_column), _opts.file_writer); RETURN_IF_ERROR(_root_writer->init()); // make sure the root type @@ -1987,10 +1988,8 @@ Status VariantColumnWriterImpl::append_nullable(const uint8_t* null_map, const u } VariantSubcolumnWriter::VariantSubcolumnWriter(const ColumnWriterOptions& opts, - const TabletColumn* column, - std::unique_ptr field) - : ColumnWriter(std::move(field), opts.meta->is_nullable(), opts.meta) { - _tablet_column = column; + TabletColumnPtr column) + : ColumnWriter(std::move(column), opts.meta->is_nullable(), opts.meta) { _opts = opts; _column = ColumnVariant::create(0, false); } @@ -2024,7 +2023,7 @@ Status VariantSubcolumnWriter::finalize() { DCHECK(ptr->is_finalized()); const auto& parent_column = - _opts.rowset_ctx->tablet_schema->column_by_uid(_tablet_column->parent_unique_id()); + _opts.rowset_ctx->tablet_schema->column_by_uid(get_column()->parent_unique_id()); TabletColumn flush_column; if (ptr->get_subcolumns().get_root()->data.get_least_common_base_type_id() == @@ -2034,10 +2033,10 @@ Status VariantSubcolumnWriter::finalize() { ptr->ensure_root_node_type(flush_type); } flush_column = variant_util::get_column_by_type( - ptr->get_root_type(), _tablet_column->name(), + ptr->get_root_type(), get_column()->name(), variant_util::ExtraInfo {.unique_id = -1, - .parent_unique_id = _tablet_column->parent_unique_id(), - .path_info = *_tablet_column->path_info_ptr()}); + .parent_unique_id = get_column()->parent_unique_id(), + .path_info = *get_column()->path_info_ptr()}); int64_t none_null_value_size = ptr->get_subcolumns().get_root()->data.get_non_null_value_size(); bool need_record_none_null_value_size = (!flush_column.path_info_ptr()->get_is_typed()) && @@ -2113,11 +2112,9 @@ Status VariantSubcolumnWriter::append_nullable(const uint8_t* null_map, const ui } VariantDocCompactWriter::VariantDocCompactWriter(const ColumnWriterOptions& opts, - const TabletColumn* column, - std::unique_ptr field) - : ColumnWriter(std::move(field), opts.meta->is_nullable(), opts.meta) { + TabletColumnPtr column) + : ColumnWriter(std::move(column), opts.meta->is_nullable(), opts.meta) { _opts = opts; - _tablet_column = column; _column = ColumnVariant::create(0, false); } @@ -2220,7 +2217,7 @@ Status VariantDocCompactWriter::_write_doc_value_column(const TabletColumn& pare ColumnVariant* variant_column, OlapBlockDataConvertor* converter, int column_id, size_t num_rows) { - std::string doc_value_column_path = _tablet_column->path_info_ptr()->get_path(); + std::string doc_value_column_path = get_column()->path_info_ptr()->get_path(); size_t pos = doc_value_column_path.rfind("b"); int bucket_value = std::stoi(doc_value_column_path.substr(pos + 1)); TabletColumn doc_value_column = @@ -2246,7 +2243,7 @@ Status VariantDocCompactWriter::finalize() { auto* variant_column = assert_cast(_column.get()); const auto& parent_column = - _opts.rowset_ctx->tablet_schema->column_by_uid(_tablet_column->parent_unique_id()); + _opts.rowset_ctx->tablet_schema->column_by_uid(get_column()->parent_unique_id()); size_t num_rows = variant_column->size(); auto converter = std::make_unique(); diff --git a/be/src/storage/segment/variant/variant_column_writer_impl.h b/be/src/storage/segment/variant/variant_column_writer_impl.h index 5fc41b54b23015..ebcc683571039d 100644 --- a/be/src/storage/segment/variant/variant_column_writer_impl.h +++ b/be/src/storage/segment/variant/variant_column_writer_impl.h @@ -231,8 +231,7 @@ class VariantColumnWriterImpl { class VariantDocCompactWriter : public ColumnWriter { public: - explicit VariantDocCompactWriter(const ColumnWriterOptions& opts, const TabletColumn* column, - std::unique_ptr field); + explicit VariantDocCompactWriter(const ColumnWriterOptions& opts, TabletColumnPtr column); ~VariantDocCompactWriter() override = default; @@ -287,7 +286,6 @@ class VariantDocCompactWriter : public ColumnWriter { ordinal_t _next_rowid = 0; MutableColumnPtr _column; - const TabletColumn* _tablet_column = nullptr; ColumnWriterOptions _opts; bool _is_finalized = false; bool _data_written = false; diff --git a/be/src/storage/segment/variant/variant_streaming_compaction_writer.cpp b/be/src/storage/segment/variant/variant_streaming_compaction_writer.cpp index 2ca6ceec0f811e..ff569576274feb 100644 --- a/be/src/storage/segment/variant/variant_streaming_compaction_writer.cpp +++ b/be/src/storage/segment/variant/variant_streaming_compaction_writer.cpp @@ -57,8 +57,7 @@ Status VariantStreamingCompactionWriter::init() { Status VariantStreamingCompactionWriter::_init_root_writer() { _root_writer = std::make_unique( - _opts, std::unique_ptr(StorageFieldFactory::create(*_tablet_column)), - _opts.file_writer); + _opts, std::make_shared(*_tablet_column), _opts.file_writer); RETURN_IF_ERROR(_root_writer->init()); _opts.meta->set_num_rows(0); return Status::OK(); diff --git a/be/src/storage/task/index_builder.cpp b/be/src/storage/task/index_builder.cpp index 2b5b6a7469c640..f213963b6de6d7 100644 --- a/be/src/storage/task/index_builder.cpp +++ b/be/src/storage/task/index_builder.cpp @@ -21,7 +21,6 @@ #include "common/logging.h" #include "common/status.h" -#include "storage/field.h" #include "storage/index/index_file_reader.h" #include "storage/index/index_file_writer.h" #include "storage/index/inverted/inverted_index_desc.h" @@ -487,7 +486,6 @@ Status IndexBuilder::handle_single_rowset(RowsetMetaSharedPtr output_rowset_meta DCHECK(output_rowset_schema->has_inverted_index_with_index_id(index_id)); _olap_data_convertor->add_column_data_convertor(column); return_columns.emplace_back(column_idx); - std::unique_ptr field(StorageFieldFactory::create(column)); if (inverted_index.index_type == TIndexType::INVERTED) { // inverted index @@ -499,7 +497,7 @@ Status IndexBuilder::handle_single_rowset(RowsetMetaSharedPtr output_rowset_meta std::unique_ptr inverted_index_builder; try { RETURN_IF_ERROR(segment_v2::IndexColumnWriter::create( - field.get(), &inverted_index_builder, index_file_writer.get(), + &column, &inverted_index_builder, index_file_writer.get(), index_meta)); DBUG_EXECUTE_IF( "IndexBuilder::handle_single_rowset_index_column_writer_create_" @@ -529,8 +527,7 @@ Status IndexBuilder::handle_single_rowset(RowsetMetaSharedPtr output_rowset_meta std::unique_ptr index_writer; try { RETURN_IF_ERROR(segment_v2::IndexColumnWriter::create( - field.get(), &index_writer, index_file_writer.get(), - index_meta)); + &column, &index_writer, index_file_writer.get(), index_meta)); DBUG_EXECUTE_IF( "IndexBuilder::handle_single_rowset_index_column_writer_create_" "error", @@ -687,9 +684,8 @@ Status IndexBuilder::_write_inverted_index_data(TabletSchemaSPtr tablet_schema, continue; } } - auto column = tablet_schema->column(column_idx); + const auto& column = tablet_schema->column(column_idx); auto writer_sign = std::make_pair(segment_idx, index_id); - std::unique_ptr field(StorageFieldFactory::create(column)); auto converted_result = _olap_data_convertor->convert_column_data(i); DBUG_EXECUTE_IF("IndexBuilder::_write_inverted_index_data_convert_column_data_error", { converted_result.first = Status::Error( @@ -702,10 +698,10 @@ Status IndexBuilder::_write_inverted_index_data(TabletSchemaSPtr tablet_schema, const auto* ptr = (const uint8_t*)converted_result.second->get_data(); const auto* null_map = converted_result.second->get_nullmap(); if (null_map) { - RETURN_IF_ERROR(_add_nullable(column_name, writer_sign, field.get(), null_map, &ptr, + RETURN_IF_ERROR(_add_nullable(column_name, writer_sign, &column, null_map, &ptr, block->rows())); } else { - RETURN_IF_ERROR(_add_data(column_name, writer_sign, field.get(), &ptr, block->rows())); + RETURN_IF_ERROR(_add_data(column_name, writer_sign, &column, &ptr, block->rows())); } } _olap_data_convertor->clear_source_content(); @@ -715,11 +711,11 @@ Status IndexBuilder::_write_inverted_index_data(TabletSchemaSPtr tablet_schema, Status IndexBuilder::_add_nullable(const std::string& column_name, const std::pair& index_writer_sign, - StorageField* field, const uint8_t* null_map, + const TabletColumn* column, const uint8_t* null_map, const uint8_t** ptr, size_t num_rows) { // TODO: need to process null data for inverted index - if (field->type() == FieldType::OLAP_FIELD_TYPE_ARRAY) { - DCHECK(field->get_sub_field_count() == 1); + if (column->type() == FieldType::OLAP_FIELD_TYPE_ARRAY) { + DCHECK(column->get_subtype_count() == 1); // [size, offset_ptr, item_data_ptr, item_nullmap_ptr] const auto* data_ptr = reinterpret_cast(*ptr); // total number length @@ -729,7 +725,8 @@ Status IndexBuilder::_add_nullable(const std::string& column_name, auto data = *(data_ptr + 2); auto nested_null_map = *(data_ptr + 3); RETURN_IF_ERROR(_index_column_writers[index_writer_sign]->add_array_values( - field->get_sub_field(0)->size(), reinterpret_cast(data), + field_type_size(column->get_sub_column(0).type()), + reinterpret_cast(data), reinterpret_cast(nested_null_map), offsets_ptr, num_rows)); DBUG_EXECUTE_IF("IndexBuilder::_add_nullable_add_array_values_error", { _CLTHROWA(CL_ERR_IO, "debug point: _add_nullable_add_array_values_error"); @@ -765,7 +762,7 @@ Status IndexBuilder::_add_nullable(const std::string& column_name, RETURN_IF_ERROR(_index_column_writers[index_writer_sign]->add_values(column_name, *ptr, step)); } - *ptr += field->size() * step; + *ptr += field_type_size(column->type()) * step; offset += step; DBUG_EXECUTE_IF("IndexBuilder::_add_nullable_throw_exception", { _CLTHROWA(CL_ERR_IO, "debug point: _add_nullable_throw_exception"); }) @@ -780,10 +777,10 @@ Status IndexBuilder::_add_nullable(const std::string& column_name, Status IndexBuilder::_add_data(const std::string& column_name, const std::pair& index_writer_sign, - StorageField* field, const uint8_t** ptr, size_t num_rows) { + const TabletColumn* column, const uint8_t** ptr, size_t num_rows) { try { - if (field->type() == FieldType::OLAP_FIELD_TYPE_ARRAY) { - DCHECK(field->get_sub_field_count() == 1); + if (column->type() == FieldType::OLAP_FIELD_TYPE_ARRAY) { + DCHECK(column->get_subtype_count() == 1); // [size, offset_ptr, item_data_ptr, item_nullmap_ptr] const auto* data_ptr = reinterpret_cast(*ptr); // total number length @@ -794,7 +791,8 @@ Status IndexBuilder::_add_data(const std::string& column_name, auto data = *(data_ptr + 2); auto nested_null_map = *(data_ptr + 3); RETURN_IF_ERROR(_index_column_writers[index_writer_sign]->add_array_values( - field->get_sub_field(0)->size(), reinterpret_cast(data), + field_type_size(column->get_sub_column(0).type()), + reinterpret_cast(data), reinterpret_cast(nested_null_map), offsets_ptr, num_rows)); } } else { diff --git a/be/src/storage/task/index_builder.h b/be/src/storage/task/index_builder.h index e3b536f54614a7..bf417182b7ff3c 100644 --- a/be/src/storage/task/index_builder.h +++ b/be/src/storage/task/index_builder.h @@ -17,7 +17,6 @@ #pragma once -#include "storage/field.h" #include "storage/index/index_file_writer.h" #include "storage/index/inverted/inverted_index_desc.h" #include "storage/iterator/olap_data_convertor.h" @@ -36,8 +35,6 @@ class IndexFileWriter; } // namespace segment_v2 class OlapBlockDataConvertor; -class StorageField; - class StorageEngine; class RowsetWriter; @@ -63,11 +60,12 @@ class IndexBuilder { Status _write_inverted_index_data(TabletSchemaSPtr tablet_schema, int64_t segment_idx, Block* block); Status _add_data(const std::string& column_name, - const std::pair& index_writer_sign, StorageField* field, - const uint8_t** ptr, size_t num_rows); + const std::pair& index_writer_sign, + const TabletColumn* column, const uint8_t** ptr, size_t num_rows); Status _add_nullable(const std::string& column_name, - const std::pair& index_writer_sign, StorageField* field, - const uint8_t* null_map, const uint8_t** ptr, size_t num_rows); + const std::pair& index_writer_sign, + const TabletColumn* column, const uint8_t* null_map, const uint8_t** ptr, + size_t num_rows); private: StorageEngine& _engine; diff --git a/be/src/storage/types.h b/be/src/storage/types.h index 89510f4ff6b79e..730a89db8a9bb4 100644 --- a/be/src/storage/types.h +++ b/be/src/storage/types.h @@ -40,11 +40,8 @@ #include "core/uint24.h" #include "core/value/ipv4_value.h" #include "core/value/ipv6_value.h" -#include "core/value/map_value.h" -#include "core/value/struct_value.h" #include "core/value/vdatetime_value.h" #include "exprs/function/cast/cast_to_timestamptz.h" -#include "runtime/collection_value.h" #include "storage/olap_common.h" #include "storage/olap_define.h" #include "util/slice.h" @@ -59,8 +56,27 @@ static const std::vector DATE_FORMATS { "%Y-%m-%d", "%y-%m-%d", "%Y%m%d", "%y%m%d", "%Y/%m/%d", "%y/%m/%d", }; +// Maps a storage FieldType to its in-memory cell representation. +// +// ARRAY / MAP / STRUCT are intentionally NOT specialized here: they are +// containers of other types, so only their element types have a storage-layer +// cell representation. The primary template below uses a deferred +// static_assert so that instantiating `CppTypeTraits` for any +// unspecialized FieldType — most importantly ARRAY/MAP/STRUCT — fails at +// build time with a clear message, rather than at runtime. +namespace detail { +template +inline constexpr bool cpp_type_traits_unspecialized = false; +} // namespace detail + template -struct CppTypeTraits {}; +struct CppTypeTraits { + static_assert(detail::cpp_type_traits_unspecialized, + "CppTypeTraits not specialized for this FieldType. " + "ARRAY / MAP / STRUCT and similar container types have no " + "storage-layer cell representation — operate on the element " + "type instead."); +}; template <> struct CppTypeTraits { @@ -215,18 +231,6 @@ struct CppTypeTraits { using CppType = Slice; }; -template <> -struct CppTypeTraits { - using CppType = StructValue; -}; -template <> -struct CppTypeTraits { - using CppType = CollectionValue; -}; -template <> -struct CppTypeTraits { - using CppType = MapValue; -}; template struct BaseFieldTypeTraits : public CppTypeTraits { using CppType = typename CppTypeTraits::CppType; @@ -373,6 +377,27 @@ struct TypeTraits : public FieldTypeTraits { static const int32_t size = sizeof(CppType); }; +// In-memory storage cell footprint for one value of `field_type`, +// i.e. sizeof(CppTypeTraits::CppType). +// +// This is NOT the schema-declared length: +// - CHAR(N) / VARCHAR(N) / STRING / JSONB / VARIANT / HLL / BITMAP / +// QUANTILE_STATE / AGG_STATE all return sizeof(Slice) == 16 (the ptr+len +// descriptor in a row buffer); for the user-declared N see +// TabletColumn::get_field_length_by_type. +// +// ARRAY / MAP / STRUCT are containers of other types — only their element +// types have a storage-layer cell size. The container itself has no such size +// at this layer, so it is not handled here. Passing one in is a programming +// error and trips the default LOG(FATAL) below. +// +// VARIANT root data is still routed through ColumnReader/EncodingInfo at read +// time, so VARIANT keeps its full traits chain even though the column-writer +// step path doesn't reach it. +// +// Used for cell-level pointer arithmetic on row buffers, BKD bytes_per_dim +// fallback when the index file has no header, and per-row footprint estimation +// during compaction. inline size_t field_type_size(FieldType field_type) { switch (field_type) { #define DORIS_FIELD_TYPE_SIZE_CASE(ft) \ @@ -410,9 +435,6 @@ inline size_t field_type_size(FieldType field_type) { DORIS_FIELD_TYPE_SIZE_CASE(OLAP_FIELD_TYPE_BITMAP) DORIS_FIELD_TYPE_SIZE_CASE(OLAP_FIELD_TYPE_QUANTILE_STATE) DORIS_FIELD_TYPE_SIZE_CASE(OLAP_FIELD_TYPE_AGG_STATE) - DORIS_FIELD_TYPE_SIZE_CASE(OLAP_FIELD_TYPE_STRUCT) - DORIS_FIELD_TYPE_SIZE_CASE(OLAP_FIELD_TYPE_ARRAY) - DORIS_FIELD_TYPE_SIZE_CASE(OLAP_FIELD_TYPE_MAP) #undef DORIS_FIELD_TYPE_SIZE_CASE default: LOG(FATAL) << "field_type_size: unsupported FieldType " << int(field_type); diff --git a/be/test/exec/scan/vgeneric_iterators_test.cpp b/be/test/exec/scan/vgeneric_iterators_test.cpp index d02cc8dd34dfab..34349b8bd36038 100644 --- a/be/test/exec/scan/vgeneric_iterators_test.cpp +++ b/be/test/exec/scan/vgeneric_iterators_test.cpp @@ -28,7 +28,6 @@ #include "core/data_type/data_type.h" #include "core/field.h" #include "gtest/gtest_pred_impl.h" -#include "storage/field.h" #include "storage/olap_common.h" #include "storage/schema.h" #include "storage/segment/column_reader.h" diff --git a/be/test/exprs/function/cast/function_variant_cast_test.cpp b/be/test/exprs/function/cast/function_variant_cast_test.cpp index 9fd12fc2576575..f3b3e21388e536 100644 --- a/be/test/exprs/function/cast/function_variant_cast_test.cpp +++ b/be/test/exprs/function/cast/function_variant_cast_test.cpp @@ -33,7 +33,6 @@ #include "exprs/function/simple_function_factory.h" #include "gtest/gtest_pred_impl.h" #include "runtime/runtime_state.h" -#include "storage/field.h" namespace doris { static doris::Field construct_variant_map( diff --git a/be/test/load/memtable/memtable_flush_executor_test.cpp b/be/test/load/memtable/memtable_flush_executor_test.cpp index 9c03f2c0fb7057..d89a803d9d0683 100644 --- a/be/test/load/memtable/memtable_flush_executor_test.cpp +++ b/be/test/load/memtable/memtable_flush_executor_test.cpp @@ -31,7 +31,7 @@ #include "load/memtable/memtable.h" #include "runtime/descriptor_helper.h" #include "runtime/exec_env.h" -#include "storage/field.h" +#include "runtime/thread_context.h" #include "storage/options.h" #include "storage/schema.h" #include "storage/storage_engine.h" diff --git a/be/test/storage/compaction/ordered_data_compaction_test.cpp b/be/test/storage/compaction/ordered_data_compaction_test.cpp index 006d48358c467e..fa050f6a68b40e 100644 --- a/be/test/storage/compaction/ordered_data_compaction_test.cpp +++ b/be/test/storage/compaction/ordered_data_compaction_test.cpp @@ -49,7 +49,6 @@ #include "storage/compaction/cumulative_compaction.h" #include "storage/data_dir.h" #include "storage/delete/delete_handler.h" -#include "storage/field.h" #include "storage/olap_common.h" #include "storage/options.h" #include "storage/rowset/beta_rowset.h" diff --git a/be/test/storage/compaction/vertical_compaction_test.cpp b/be/test/storage/compaction/vertical_compaction_test.cpp index 37eedcca319759..ec320222f05ea2 100644 --- a/be/test/storage/compaction/vertical_compaction_test.cpp +++ b/be/test/storage/compaction/vertical_compaction_test.cpp @@ -50,7 +50,6 @@ #include "runtime/exec_env.h" #include "storage/compaction/compaction.h" #include "storage/delete/delete_handler.h" -#include "storage/field.h" #include "storage/iterator/vertical_merge_iterator.h" #include "storage/merger.h" #include "storage/olap_common.h" diff --git a/be/test/storage/index/ann/ann_index_smoke_test.cpp b/be/test/storage/index/ann/ann_index_smoke_test.cpp index 32de8d0626bd61..3b5886cdffc456 100644 --- a/be/test/storage/index/ann/ann_index_smoke_test.cpp +++ b/be/test/storage/index/ann/ann_index_smoke_test.cpp @@ -22,7 +22,6 @@ #include #include -#include "storage/field.h" #include "storage/index/ann/ann_index.h" #include "storage/index/ann/ann_index_writer.h" #include "storage/index/ann/ann_search_params.h" @@ -55,11 +54,6 @@ class AnnIndexTest : public testing::Test { _tablet_column_array = std::make_unique(); _tablet_column_float = std::make_unique(); - EXPECT_CALL(*_tablet_column_array, type()) - .WillRepeatedly(testing::Return(FieldType::OLAP_FIELD_TYPE_ARRAY)); - - StorageField field(*_tablet_column_array); - EXPECT_CALL(*_index_file_writer, open(_index_meta.get())) .WillOnce(testing::Return(_ram_dir)); diff --git a/be/test/storage/index/ann/ann_index_writer_test.cpp b/be/test/storage/index/ann/ann_index_writer_test.cpp index fc3a73929629a9..20107c90779501 100644 --- a/be/test/storage/index/ann/ann_index_writer_test.cpp +++ b/be/test/storage/index/ann/ann_index_writer_test.cpp @@ -27,8 +27,6 @@ #include #include "common/config.h" -#include "runtime/collection_value.h" -#include "storage/field.h" #include "storage/index/ann/faiss_ann_index.h" #include "storage/index/ann/vector_search_utils.h" #include "storage/index/index_file_writer.h" @@ -251,22 +249,6 @@ TEST_F(AnnIndexWriterTest, TestAddArrayValuesWrongDimension) { EXPECT_TRUE(status.is()); } -TEST_F(AnnIndexWriterTest, TestAddArrayValuesWithCollectionValue) { - auto writer = - std::make_unique(_index_file_writer.get(), _tablet_index.get()); - - auto fs_dir = std::make_shared(); - fs_dir->init(doris::io::global_local_filesystem(), "./ut_dir/tmp_vector_search", nullptr); - EXPECT_CALL(*_index_file_writer, open(testing::_)).WillOnce(testing::Return(fs_dir)); - - ASSERT_TRUE(writer->init().ok()); - - // This should return an error as ANN index doesn't support nullable columns - Status status = writer->add_array_values(sizeof(float), nullptr, 1); - EXPECT_FALSE(status.ok()); - EXPECT_TRUE(status.is()); -} - TEST_F(AnnIndexWriterTest, TestAddValues) { auto writer = std::make_unique(_index_file_writer.get(), _tablet_index.get()); @@ -611,8 +593,8 @@ TEST_F(AnnIndexWriterTest, TestCreateFromIndexColumnWriter) { tablet_schema->append_column(array_column); // Get field for array column - std::unique_ptr field(StorageFieldFactory::create(array_column)); - ASSERT_NE(field.get(), nullptr); + const TabletColumn* field = &(array_column); + ASSERT_NE(field, nullptr); auto fs_dir = std::make_shared(); fs_dir->init(doris::io::global_local_filesystem(), "./ut_dir/tmp_vector_search", nullptr); @@ -620,7 +602,7 @@ TEST_F(AnnIndexWriterTest, TestCreateFromIndexColumnWriter) { // Create column writer std::unique_ptr column_writer; - auto status = IndexColumnWriter::create(field.get(), &column_writer, _index_file_writer.get(), + auto status = IndexColumnWriter::create(field, &column_writer, _index_file_writer.get(), _tablet_index.get()); EXPECT_TRUE(status.ok()); diff --git a/be/test/storage/index/inverted/compaction/util/index_compaction_utils.cpp b/be/test/storage/index/inverted/compaction/util/index_compaction_utils.cpp index 162543c3033dc8..2a59fb86acc5e8 100644 --- a/be/test/storage/index/inverted/compaction/util/index_compaction_utils.cpp +++ b/be/test/storage/index/inverted/compaction/util/index_compaction_utils.cpp @@ -33,6 +33,7 @@ #include "storage/compaction/base_compaction.h" #include "storage/index/index_file_reader.h" #include "storage/index/inverted/query/query_factory.h" +#include "storage/key_coder.h" #include "storage/rowset/beta_rowset.h" #include "storage/rowset/beta_rowset_writer.h" #include "storage/rowset/rowset_factory.h" diff --git a/be/test/storage/index/inverted/query/phrase_edge_query_test.cpp b/be/test/storage/index/inverted/query/phrase_edge_query_test.cpp index 3fc08841df774b..2da72758470a27 100644 --- a/be/test/storage/index/inverted/query/phrase_edge_query_test.cpp +++ b/be/test/storage/index/inverted/query/phrase_edge_query_test.cpp @@ -24,7 +24,6 @@ #include "io/fs/local_file_system.h" #include "runtime/exec_env.h" #include "runtime/runtime_state.h" -#include "storage/field.h" #include "storage/index/index_file_reader.h" #include "storage/index/index_file_writer.h" #include "storage/index/inverted/inverted_index_cache.h" @@ -131,16 +130,16 @@ class PhraseEdgeQueryTest : public testing::Test { std::make_unique(fs, *index_path_prefix, std::string {rowset_id}, seg_id, format, std::move(file_writer)); - // Get c2 column StorageField + // Get c2 column descriptor const TabletColumn& column = tablet_schema->column(1); ASSERT_NE(&column, nullptr); - std::unique_ptr field(StorageFieldFactory::create(column)); - ASSERT_NE(field.get(), nullptr); + const TabletColumn* field = &(column); + ASSERT_NE(field, nullptr); // Create column writer std::unique_ptr column_writer; - auto status = IndexColumnWriter::create(field.get(), &column_writer, - index_file_writer.get(), idx_meta); + auto status = + IndexColumnWriter::create(field, &column_writer, index_file_writer.get(), idx_meta); EXPECT_TRUE(status.ok()) << status; // Write string values diff --git a/be/test/storage/index/inverted/query/phrase_prefix_query_test.cpp b/be/test/storage/index/inverted/query/phrase_prefix_query_test.cpp index cd9c8eb214d9e3..739720d61bd9d2 100644 --- a/be/test/storage/index/inverted/query/phrase_prefix_query_test.cpp +++ b/be/test/storage/index/inverted/query/phrase_prefix_query_test.cpp @@ -23,7 +23,6 @@ #include "io/fs/local_file_system.h" #include "runtime/exec_env.h" -#include "storage/field.h" #include "storage/index/index_file_reader.h" #include "storage/index/index_file_writer.h" #include "storage/index/inverted/inverted_index_cache.h" @@ -130,16 +129,16 @@ class PhrasePrefixQueryTest : public testing::Test { std::make_unique(fs, *index_path_prefix, std::string {rowset_id}, seg_id, format, std::move(file_writer)); - // Get c2 column StorageField + // Get c2 column descriptor const TabletColumn& column = tablet_schema->column(1); ASSERT_NE(&column, nullptr); - std::unique_ptr field(StorageFieldFactory::create(column)); - ASSERT_NE(field.get(), nullptr); + const TabletColumn* field = &(column); + ASSERT_NE(field, nullptr); // Create column writer std::unique_ptr column_writer; - auto status = IndexColumnWriter::create(field.get(), &column_writer, - index_file_writer.get(), idx_meta); + auto status = + IndexColumnWriter::create(field, &column_writer, index_file_writer.get(), idx_meta); EXPECT_TRUE(status.ok()) << status; // Write string values diff --git a/be/test/storage/index/inverted/query/phrase_query_test.cpp b/be/test/storage/index/inverted/query/phrase_query_test.cpp index 877a91d1f20571..40f7f59349dbb5 100644 --- a/be/test/storage/index/inverted/query/phrase_query_test.cpp +++ b/be/test/storage/index/inverted/query/phrase_query_test.cpp @@ -23,7 +23,6 @@ #include "io/fs/local_file_system.h" #include "runtime/exec_env.h" -#include "storage/field.h" #include "storage/index/index_file_reader.h" #include "storage/index/index_file_writer.h" #include "storage/index/inverted/inverted_index_cache.h" @@ -131,16 +130,16 @@ class PhraseQueryTest : public testing::Test { std::make_unique(fs, *index_path_prefix, std::string {rowset_id}, seg_id, format, std::move(file_writer)); - // Get c2 column StorageField + // Get c2 column descriptor const TabletColumn& column = tablet_schema->column(1); ASSERT_NE(&column, nullptr); - std::unique_ptr field(StorageFieldFactory::create(column)); - ASSERT_NE(field.get(), nullptr); + const TabletColumn* field = &(column); + ASSERT_NE(field, nullptr); // Create column writer std::unique_ptr column_writer; - auto status = IndexColumnWriter::create(field.get(), &column_writer, - index_file_writer.get(), idx_meta); + auto status = + IndexColumnWriter::create(field, &column_writer, index_file_writer.get(), idx_meta); EXPECT_TRUE(status.ok()) << status; // Write string values diff --git a/be/test/storage/metadata_adder_test.cpp b/be/test/storage/metadata_adder_test.cpp index 8a2d8ec13e5277..b436a6c670cb2d 100644 --- a/be/test/storage/metadata_adder_test.cpp +++ b/be/test/storage/metadata_adder_test.cpp @@ -21,7 +21,6 @@ #include "core/data_type/data_type_factory.hpp" #include "core/data_type/define_primitive_type.h" -#include "storage/field.h" #include "storage/index/zone_map/zone_map_index.h" #include "storage/tablet/tablet_schema.h" #include "storage/tablet/tablet_schema_helper.h" @@ -123,7 +122,7 @@ TEST_F(MetadataAdderTest, meta_load_with_pb_test) { { auto fs = io::global_local_filesystem(); TabletColumnPtr int_column = create_int_key(0); - StorageField* int_field = StorageFieldFactory::create(*int_column); + const TabletColumn* int_field = int_column.get(); auto int_data_type_ptr = DataTypeFactory::instance().create_data_type(TYPE_INT, false); // 1 load first column @@ -161,7 +160,7 @@ TEST_F(MetadataAdderTest, meta_load_with_pb_test) { // load second column segment_v2::ColumnIndexMetaPB index_meta2; TabletColumnPtr varchar_column = create_varchar_key(0); - StorageField* str_field = StorageFieldFactory::create(*varchar_column); + const TabletColumn* str_field = varchar_column.get(); auto str_data_type_ptr = DataTypeFactory::instance().create_data_type(TYPE_VARCHAR, false); std::string file2 = kTestDir + "/copy_obj2"; @@ -193,9 +192,6 @@ TEST_F(MetadataAdderTest, meta_load_with_pb_test) { ASSERT_TRUE(MetadataAdder::get_all_segments_size() == mem_size2 + mem_size); - - delete int_field; - delete str_field; } ASSERT_TRUE(MetadataAdder::get_all_segments_size() == 0); diff --git a/be/test/storage/segment/column_reader_writer_test.cpp b/be/test/storage/segment/column_reader_writer_test.cpp index 60dfa0cba2bfc2..e49655951506ef 100644 --- a/be/test/storage/segment/column_reader_writer_test.cpp +++ b/be/test/storage/segment/column_reader_writer_test.cpp @@ -233,209 +233,6 @@ void test_nullable_data(uint8_t* src_data, uint8_t* src_is_null, int num_rows, } } -template -void test_array_nullable_data(CollectionValue* src_data, uint8_t* src_is_null, int num_rows, - std::string test_name) { - CollectionValue* src = src_data; - ColumnMetaPB meta; - TabletColumn list_column(OLAP_FIELD_AGGREGATION_NONE, FieldType::OLAP_FIELD_TYPE_ARRAY); - int32 item_length = 0; - if (item_type == FieldType::OLAP_FIELD_TYPE_CHAR || - item_type == FieldType::OLAP_FIELD_TYPE_VARCHAR) { - item_length = 10; - } - TabletColumn item_column(OLAP_FIELD_AGGREGATION_NONE, item_type, true, 0, item_length); - list_column.add_sub_column(item_column); - Field* field = StorageFieldFactory::create(list_column); - - // write data - std::string fname = TEST_DIR + "/" + test_name; - auto fs = io::global_local_filesystem(); - { - io::FileWriterPtr file_writer; - Status st = fs->create_file(fname, &file_writer); - EXPECT_TRUE(st.ok()) << st; - - ColumnWriterOptions writer_opts; - writer_opts.meta = &meta; - writer_opts.meta->set_column_id(0); - writer_opts.meta->set_unique_id(0); - writer_opts.meta->set_type(FieldType::OLAP_FIELD_TYPE_ARRAY); - writer_opts.meta->set_length(0); - writer_opts.meta->set_encoding(array_encoding); - writer_opts.meta->set_compression(segment_v2::CompressionTypePB::LZ4F); - writer_opts.meta->set_is_nullable(true); - writer_opts.data_page_size = 5 * 8; - - ColumnMetaPB* child_meta = meta.add_children_columns(); - - child_meta->set_column_id(1); - child_meta->set_unique_id(1); - child_meta->set_type(item_type); - child_meta->set_length(item_length); - child_meta->set_encoding(item_encoding); - child_meta->set_compression(segment_v2::CompressionTypePB::LZ4F); - child_meta->set_is_nullable(true); - - std::unique_ptr writer; - ColumnWriter::create(writer_opts, &list_column, file_writer.get(), &writer); - st = writer->init(); - EXPECT_TRUE(st.ok()) << st.to_string(); - - for (int i = 0; i < num_rows; ++i) { - st = writer->append(BitmapTest(src_is_null, i), src + i); - EXPECT_TRUE(st.ok()); - } - - st = writer->finish(); - EXPECT_TRUE(st.ok()); - - st = writer->write_data(); - EXPECT_TRUE(st.ok()); - st = writer->write_ordinal_index(); - EXPECT_TRUE(st.ok()); - - // close the file - EXPECT_TRUE(file_writer->close().ok()); - } - auto type_info = get_type_info(&meta); - io::FileReaderSPtr file_reader; - ASSERT_EQ(fs->open_file(fname, &file_reader), Status::OK()); - // read and check - { - ColumnReaderOptions reader_opts; - std::shared_ptr reader; - auto st = ColumnReader::create(reader_opts, meta, num_rows, file_reader, &reader); - EXPECT_TRUE(st.ok()); - - ColumnIteratorUPtr iter; - st = reader->new_iterator(&iter); - EXPECT_TRUE(st.ok()); - - ColumnIteratorOptions iter_opts; - OlapReaderStatistics stats; - iter_opts.stats = &stats; - iter_opts.file_reader = file_reader.get(); - st = iter->init(iter_opts); - EXPECT_TRUE(st.ok()); - // sequence read - { - Arena pool; - std::unique_ptr cvb; - ColumnVectorBatch::create(0, true, type_info.get(), field, &cvb); - cvb->resize(1024); - ColumnBlock col(cvb.get(), &pool); - - int idx = 0; - while (true) { - size_t rows_read = 1024; - ColumnBlockView dst(&col); - st = iter->next_batch(&rows_read, &dst); - EXPECT_TRUE(st.ok()); - for (int j = 0; j < rows_read; ++j) { - EXPECT_EQ(BitmapTest(src_is_null, idx), col.is_null(j)); - if (!col.is_null(j)) { - EXPECT_TRUE(type_info->equal(&src[idx], col.cell_ptr(j))); - } - ++idx; - } - if (rows_read < 1024) { - break; - } - } - } - // seek read - { - Arena pool; - std::unique_ptr cvb; - ColumnVectorBatch::create(0, true, type_info.get(), field, &cvb); - cvb->resize(1024); - ColumnBlock col(cvb.get(), &pool); - - for (int rowid = 0; rowid < num_rows; rowid += 4025) { - st = iter->seek_to_ordinal(rowid); - EXPECT_TRUE(st.ok()); - - int idx = rowid; - size_t rows_read = 1024; - ColumnBlockView dst(&col); - - st = iter->next_batch(&rows_read, &dst); - EXPECT_TRUE(st.ok()); - for (int j = 0; j < rows_read; ++j) { - EXPECT_EQ(BitmapTest(src_is_null, idx), col.is_null(j)); - if (!col.is_null(j)) { - EXPECT_TRUE(type_info->equal(&src[idx], col.cell_ptr(j))); - } - ++idx; - } - } - } - delete iter; - } - delete field; -} - -TEST_F(ColumnReaderWriterTest, test_array_type) { - size_t num_array = LOOP_LESS_OR_MORE(1024, 24 * 1024); - size_t num_item = num_array * 3; - - uint8_t* array_is_null = new uint8_t[BitmapSize(num_array)]; - CollectionValue* array_val = new CollectionValue[num_array]; - bool* item_is_null = new bool[num_item]; - uint8_t* item_val = new uint8_t[num_item]; - for (int i = 0; i < num_item; ++i) { - item_val[i] = i; - item_is_null[i] = (i % 4) == 0; - if (i % 3 == 0) { - size_t array_index = i / 3; - bool is_null = (array_index % 4) == 1; - BitmapChange(array_is_null, array_index, is_null); - if (is_null) { - continue; - } - array_val[array_index].set_data(&item_val[i]); - array_val[array_index].set_null_signs(&item_is_null[i]); - array_val[array_index].set_length(3); - } - } - test_array_nullable_data( - array_val, array_is_null, num_array, "null_array_bs"); - - delete[] array_val; - delete[] item_val; - delete[] item_is_null; - - array_val = new CollectionValue[num_array]; - Slice* varchar_vals = new Slice[3]; - item_is_null = new bool[3]; - for (int i = 0; i < 3; ++i) { - item_is_null[i] = i == 1; - if (i != 1) { - set_column_value_by_type(FieldType::OLAP_FIELD_TYPE_VARCHAR, i, (char*)&varchar_vals[i], - &_pool); - } - } - for (int i = 0; i < num_array; ++i) { - bool is_null = (i % 4) == 1; - BitmapChange(array_is_null, i, is_null); - if (is_null) { - continue; - } - array_val[i].set_data(varchar_vals); - array_val[i].set_null_signs(item_is_null); - array_val[i].set_length(3); - } - test_array_nullable_data( - array_val, array_is_null, num_array, "null_array_chars"); - - delete[] array_val; - delete[] varchar_vals; - delete[] item_is_null; - - delete[] array_is_null; -} - TEST_F(ColumnReaderWriterTest, test_array_append_nulls) { ColumnMetaPB meta; TabletColumn list_column(OLAP_FIELD_AGGREGATION_NONE, FieldType::OLAP_FIELD_TYPE_ARRAY); @@ -837,29 +634,5 @@ TEST_F(ColumnReaderWriterTest, test_v_default_value) { test_v_read_default_value(v_decimal, &decimal); } -TEST_F(ColumnReaderWriterTest, test_single_empty_array) { - size_t num_array = 1; - std::unique_ptr array_is_null(new uint8_t[BitmapSize(num_array)]()); - CollectionValue array(0); - test_array_nullable_data( - &array, array_is_null.get(), num_array, "test_single_empty_array"); -} - -TEST_F(ColumnReaderWriterTest, test_mixed_empty_arrays) { - size_t num_array = 3; - std::unique_ptr array_is_null(new uint8_t[BitmapSize(num_array)]()); - std::unique_ptr collection_values(new CollectionValue[num_array]); - int data[] = {1, 2, 3}; - for (int i = 0; i < num_array; ++i) { - if (i % 2 == 1) { - new (&collection_values[i]) CollectionValue(0); - } else { - new (&collection_values[i]) CollectionValue(&data, 3, false, nullptr); - } - } - test_array_nullable_data( - collection_values.get(), array_is_null.get(), num_array, "test_mixed_empty_arrays"); -} - } // namespace segment_v2 } // namespace doris diff --git a/be/test/storage/segment/inverted_index_array_test.cpp b/be/test/storage/segment/inverted_index_array_test.cpp index 5eb93a75bb578a..7fc63e6fcd0b78 100644 --- a/be/test/storage/segment/inverted_index_array_test.cpp +++ b/be/test/storage/segment/inverted_index_array_test.cpp @@ -42,7 +42,6 @@ #include "io/fs/file_writer.h" #include "io/fs/local_file_system.h" #include "runtime/exec_env.h" -#include "storage/field.h" #include "storage/index/index_file_reader.h" #include "storage/index/index_file_writer.h" #include "storage/index/inverted/inverted_index_compound_reader.h" @@ -53,6 +52,7 @@ #include "storage/iterator/olap_data_convertor.h" #include "storage/tablet/tablet_schema.h" #include "storage/tablet/tablet_schema_helper.h" +#include "storage/types.h" #include "util/faststring.h" #include "util/slice.h" @@ -203,7 +203,7 @@ class InvertedIndexArrayTest : public testing::Test { return tablet_schema; } - void test_non_null_string(std::string_view rowset_id, int seg_id, StorageField* field) { + void test_non_null_string(std::string_view rowset_id, int seg_id, const TabletColumn* field) { EXPECT_TRUE(field->type() == FieldType::OLAP_FIELD_TYPE_ARRAY); std::string index_path_prefix {InvertedIndexDescriptor::get_index_file_path_prefix( local_segment_path(kTestDir, rowset_id, seg_id))}; @@ -270,7 +270,7 @@ class InvertedIndexArrayTest : public testing::Test { const auto* item_nullmap = reinterpret_cast(data_ptr[3]); // Get the length of the subfield, used for inverted index writing - auto field_size = field->get_sub_field(0)->size(); + auto field_size = field_type_size(field->get_sub_column(0).type()); // Call the inverted index writing interface, passing in item_data, item_nullmap, offsets_ptr, and the number of rows (the number of array rows in the Block) st = _inverted_index_builder->add_array_values(field_size, item_data, item_nullmap, offsets_ptr, block.rows()); @@ -289,7 +289,7 @@ class InvertedIndexArrayTest : public testing::Test { &idx_meta); } - void test_string(std::string_view rowset_id, int seg_id, StorageField* field) { + void test_string(std::string_view rowset_id, int seg_id, const TabletColumn* field) { EXPECT_TRUE(field->type() == FieldType::OLAP_FIELD_TYPE_ARRAY); std::string index_path_prefix {InvertedIndexDescriptor::get_index_file_path_prefix( local_segment_path(kTestDir, rowset_id, seg_id))}; @@ -357,7 +357,7 @@ class InvertedIndexArrayTest : public testing::Test { const auto* item_nullmap = reinterpret_cast(data_ptr[3]); // Get the length of the subfield, used for inverted index writing - auto field_size = field->get_sub_field(0)->size(); + auto field_size = field_type_size(field->get_sub_column(0).type()); // Call the inverted index writing interface, passing in item_data, item_nullmap, offsets_ptr, and the number of rows (the number of array rows in the Block) st = _inverted_index_builder->add_array_values(field_size, item_data, item_nullmap, offsets_ptr, block.rows()); @@ -375,7 +375,7 @@ class InvertedIndexArrayTest : public testing::Test { &idx_meta); } - void test_null_write_v2(std::string_view rowset_id, int seg_id, StorageField* field) { + void test_null_write_v2(std::string_view rowset_id, int seg_id, const TabletColumn* field) { EXPECT_TRUE(field->type() == FieldType::OLAP_FIELD_TYPE_ARRAY); std::string index_path_prefix {InvertedIndexDescriptor::get_index_file_path_prefix( local_segment_path(kTestDir, rowset_id, seg_id))}; @@ -469,7 +469,7 @@ class InvertedIndexArrayTest : public testing::Test { const auto* item_nullmap = reinterpret_cast(data_ptr[3]); // Call the inverted index writing interface, passing in the converted nested data, nullmap, and offsets - auto field_size = field->get_sub_field(0)->size(); + auto field_size = field_type_size(field->get_sub_column(0).type()); st = _inverted_index_builder->add_array_values(field_size, item_data, item_nullmap, offsets_ptr, block.rows()); EXPECT_EQ(st, Status::OK()); @@ -491,7 +491,7 @@ class InvertedIndexArrayTest : public testing::Test { InvertedIndexStorageFormatPB::V2, &idx_meta); } - void test_null_write(std::string_view rowset_id, int seg_id, StorageField* field) { + void test_null_write(std::string_view rowset_id, int seg_id, const TabletColumn* field) { EXPECT_TRUE(field->type() == FieldType::OLAP_FIELD_TYPE_ARRAY); std::string index_path_prefix {InvertedIndexDescriptor::get_index_file_path_prefix( local_segment_path(kTestDir, rowset_id, seg_id))}; @@ -582,7 +582,7 @@ class InvertedIndexArrayTest : public testing::Test { const auto* item_nullmap = reinterpret_cast(data_ptr[3]); // Call the inverted index writing interface, passing in the converted nested data, nullmap, and offsets - auto field_size = field->get_sub_field(0)->size(); + auto field_size = field_type_size(field->get_sub_column(0).type()); st = _inverted_index_builder->add_array_values(field_size, item_data, item_nullmap, offsets_ptr, block.rows()); EXPECT_EQ(st, Status::OK()); @@ -604,7 +604,7 @@ class InvertedIndexArrayTest : public testing::Test { InvertedIndexStorageFormatPB::V1, &idx_meta); } - void test_multi_block_write(std::string_view rowset_id, int seg_id, StorageField* field) { + void test_multi_block_write(std::string_view rowset_id, int seg_id, const TabletColumn* field) { EXPECT_TRUE(field->type() == FieldType::OLAP_FIELD_TYPE_ARRAY); std::string index_path_prefix {InvertedIndexDescriptor::get_index_file_path_prefix( local_segment_path(kTestDir, rowset_id, seg_id))}; @@ -678,7 +678,7 @@ class InvertedIndexArrayTest : public testing::Test { const auto* offsets_ptr = reinterpret_cast(data_ptr[1]); const void* item_data = reinterpret_cast(data_ptr[2]); const auto* item_nullmap = reinterpret_cast(data_ptr[3]); - auto field_size = field->get_sub_field(0)->size(); + auto field_size = field_type_size(field->get_sub_column(0).type()); st = _inverted_index_builder->add_array_values(field_size, item_data, item_nullmap, offsets_ptr, row_num); EXPECT_EQ(st, Status::OK()); @@ -727,7 +727,7 @@ class InvertedIndexArrayTest : public testing::Test { const void* item_data = reinterpret_cast(data_ptr[2]); const auto* item_nullmap = reinterpret_cast(data_ptr[3]); - auto field_size = field->get_sub_field(0)->size(); + auto field_size = field_type_size(field->get_sub_column(0).type()); st = _inverted_index_builder->add_array_values(field_size, item_data, item_nullmap, offsets_ptr, row_num); EXPECT_EQ(st, Status::OK()); @@ -774,7 +774,7 @@ class InvertedIndexArrayTest : public testing::Test { const auto* offsets_ptr = reinterpret_cast(data_ptr[1]); const void* item_data = reinterpret_cast(data_ptr[2]); const auto* item_nullmap = reinterpret_cast(data_ptr[3]); - auto field_size = field->get_sub_field(0)->size(); + auto field_size = field_type_size(field->get_sub_column(0).type()); st = _inverted_index_builder->add_array_values(field_size, item_data, item_nullmap, offsets_ptr, row_num); EXPECT_EQ(st, Status::OK()); @@ -796,7 +796,7 @@ class InvertedIndexArrayTest : public testing::Test { InvertedIndexStorageFormatPB::V1, &idx_meta); } - void test_array_numeric(std::string_view rowset_id, int seg_id, StorageField* field) { + void test_array_numeric(std::string_view rowset_id, int seg_id, const TabletColumn* field) { EXPECT_TRUE(field->type() == FieldType::OLAP_FIELD_TYPE_ARRAY); std::string index_path_prefix {InvertedIndexDescriptor::get_index_file_path_prefix( local_segment_path(kTestDir, rowset_id, seg_id))}; @@ -882,7 +882,7 @@ class InvertedIndexArrayTest : public testing::Test { const auto* item_nullmap = reinterpret_cast(data_ptr[3]); // get the size of the sub field (4 bytes for INT type) - auto field_size = field->get_sub_field(0)->size(); + auto field_size = field_type_size(field->get_sub_column(0).type()); st = _inverted_index_builder->add_array_values(field_size, item_data, item_nullmap, offsets_ptr, block.rows()); EXPECT_EQ(st, Status::OK()); @@ -938,7 +938,7 @@ class InvertedIndexArrayTest : public testing::Test { } } - void test_array_all_null(std::string_view rowset_id, int seg_id, StorageField* field) { + void test_array_all_null(std::string_view rowset_id, int seg_id, const TabletColumn* field) { EXPECT_TRUE(field->type() == FieldType::OLAP_FIELD_TYPE_ARRAY); std::string index_path_prefix {InvertedIndexDescriptor::get_index_file_path_prefix( local_segment_path(kTestDir, rowset_id, seg_id))}; @@ -993,7 +993,7 @@ class InvertedIndexArrayTest : public testing::Test { const auto* item_nullmap = reinterpret_cast(data_ptr[3]); const auto* null_map = accessor->get_nullmap(); - auto field_size = field->get_sub_field(0)->size(); + auto field_size = field_type_size(field->get_sub_column(0).type()); st = _inverted_index_builder->add_array_values(field_size, item_data, item_nullmap, offsets_ptr, block.rows()); EXPECT_EQ(st, Status::OK()); @@ -1055,10 +1055,9 @@ TEST_F(InvertedIndexArrayTest, ArrayString) { arraySubColumn.set_name("arr_sub_string"); arraySubColumn.set_type(FieldType::OLAP_FIELD_TYPE_STRING); arrayTabletColumn.add_sub_column(arraySubColumn); - StorageField* field = StorageFieldFactory::create(arrayTabletColumn); + const TabletColumn* field = &(arrayTabletColumn); test_string("rowset_id", 0, field); test_non_null_string("rowset_id_non_null", 0, field); - delete field; } TEST_F(InvertedIndexArrayTest, ComplexNullCases) { @@ -1071,11 +1070,10 @@ TEST_F(InvertedIndexArrayTest, ComplexNullCases) { arraySubColumn.set_name("arr_sub_string"); arraySubColumn.set_type(FieldType::OLAP_FIELD_TYPE_STRING); arrayTabletColumn.add_sub_column(arraySubColumn); - StorageField* field = StorageFieldFactory::create(arrayTabletColumn); + const TabletColumn* field = &(arrayTabletColumn); test_null_write("complex_null", 0, field); test_null_write_v2("complex_null_v2", 0, field); test_array_all_null("complex_null_all_null", 0, field); - delete field; } TEST_F(InvertedIndexArrayTest, MultiBlockWrite) { @@ -1088,9 +1086,8 @@ TEST_F(InvertedIndexArrayTest, MultiBlockWrite) { arraySubColumn.set_name("arr_sub_string"); arraySubColumn.set_type(FieldType::OLAP_FIELD_TYPE_STRING); arrayTabletColumn.add_sub_column(arraySubColumn); - StorageField* field = StorageFieldFactory::create(arrayTabletColumn); + const TabletColumn* field = &(arrayTabletColumn); test_multi_block_write("multi_block", 0, field); - delete field; } TEST_F(InvertedIndexArrayTest, ArrayInt) { @@ -1103,8 +1100,7 @@ TEST_F(InvertedIndexArrayTest, ArrayInt) { arraySubColumn.set_name("arr_sub_int"); arraySubColumn.set_type(FieldType::OLAP_FIELD_TYPE_INT); arrayTabletColumn.add_sub_column(arraySubColumn); - StorageField* field = StorageFieldFactory::create(arrayTabletColumn); + const TabletColumn* field = &(arrayTabletColumn); test_array_numeric("int_test", 0, field); - delete field; } } // namespace doris::segment_v2 diff --git a/be/test/storage/segment/inverted_index_reader_test.cpp b/be/test/storage/segment/inverted_index_reader_test.cpp index 27a1c3d2c01c2c..184b679dd3d76e 100644 --- a/be/test/storage/segment/inverted_index_reader_test.cpp +++ b/be/test/storage/segment/inverted_index_reader_test.cpp @@ -32,12 +32,12 @@ #include "core/field.h" #include "core/value/vdatetime_value.h" #include "runtime/runtime_state.h" -#include "storage/field.h" #include "storage/index/index_file_reader.h" #include "storage/index/index_file_writer.h" #include "storage/index/inverted/inverted_index_desc.h" #include "storage/index/inverted/inverted_index_iterator.h" #include "storage/index/inverted/inverted_index_writer.h" +#include "storage/key_coder.h" #include "storage/tablet/tablet_schema.h" #include "storage/tablet/tablet_schema_helper.h" #include "util/slice.h" @@ -136,16 +136,16 @@ class InvertedIndexReaderTest : public testing::Test { std::make_unique(fs, *index_path_prefix, std::string {rowset_id}, seg_id, format, std::move(file_writer)); - // Get c2 column StorageField + // Get c2 column descriptor const TabletColumn& column = tablet_schema->column(1); ASSERT_NE(&column, nullptr); - std::unique_ptr field(StorageFieldFactory::create(column)); - ASSERT_NE(field.get(), nullptr); + const TabletColumn* field = &(column); + ASSERT_NE(field, nullptr); // Create column writer std::unique_ptr column_writer; - auto status = IndexColumnWriter::create(field.get(), &column_writer, - index_file_writer.get(), idx_meta); + auto status = + IndexColumnWriter::create(field, &column_writer, index_file_writer.get(), idx_meta); EXPECT_TRUE(status.ok()) << status; // Write string values @@ -190,16 +190,16 @@ class InvertedIndexReaderTest : public testing::Test { fs, *index_path_prefix, std::string {rowset_id}, seg_id, InvertedIndexStorageFormatPB::V2, std::move(file_writer)); - // Get c2 column StorageField + // Get c2 column descriptor const TabletColumn& column = tablet_schema->column(1); ASSERT_NE(&column, nullptr); - std::unique_ptr field(StorageFieldFactory::create(column)); - ASSERT_NE(field.get(), nullptr); + const TabletColumn* field = &(column); + ASSERT_NE(field, nullptr); // Create column writer std::unique_ptr column_writer; - auto status = IndexColumnWriter::create(field.get(), &column_writer, - index_file_writer.get(), idx_meta); + auto status = + IndexColumnWriter::create(field, &column_writer, index_file_writer.get(), idx_meta); EXPECT_TRUE(status.ok()) << status; // Add NULL values @@ -258,16 +258,16 @@ class InvertedIndexReaderTest : public testing::Test { fs, *index_path_prefix, std::string {rowset_id}, seg_id, InvertedIndexStorageFormatPB::V2, std::move(file_writer)); - // Get c1 column StorageField + // Get c1 column descriptor const TabletColumn& column = tablet_schema->column(0); ASSERT_NE(&column, nullptr); - std::unique_ptr field(StorageFieldFactory::create(column)); - ASSERT_NE(field.get(), nullptr); + const TabletColumn* field = &(column); + ASSERT_NE(field, nullptr); // Create column writer std::unique_ptr column_writer; - auto status = IndexColumnWriter::create(field.get(), &column_writer, - index_file_writer.get(), idx_meta); + auto status = + IndexColumnWriter::create(field, &column_writer, index_file_writer.get(), idx_meta); EXPECT_TRUE(status.ok()) << status; // Add integer values @@ -2712,12 +2712,12 @@ class InvertedIndexReaderTest : public testing::Test { double_schema->append_column(double_column); const TabletColumn& column = double_schema->column(0); - std::unique_ptr field(StorageFieldFactory::create(column)); - ASSERT_NE(field.get(), nullptr); + const TabletColumn* field = &(column); + ASSERT_NE(field, nullptr); std::unique_ptr column_writer; - auto status = IndexColumnWriter::create(field.get(), &column_writer, - index_file_writer.get(), idx_meta); + auto status = + IndexColumnWriter::create(field, &column_writer, index_file_writer.get(), idx_meta); EXPECT_TRUE(status.ok()) << status; for (const auto& value : values) { @@ -3491,12 +3491,12 @@ class InvertedIndexReaderTest : public testing::Test { seg_id, format, std::move(file_writer)); const TabletColumn& column = tablet_schema->column(col_id); - std::unique_ptr field(StorageFieldFactory::create(column)); - ASSERT_NE(field.get(), nullptr); + const TabletColumn* field = &(column); + ASSERT_NE(field, nullptr); std::unique_ptr column_writer; - auto status = IndexColumnWriter::create(field.get(), &column_writer, - index_file_writer.get(), idx_meta); + auto status = + IndexColumnWriter::create(field, &column_writer, index_file_writer.get(), idx_meta); EXPECT_TRUE(status.ok()) << status; for (const auto& value : values) { @@ -3996,12 +3996,12 @@ class InvertedIndexReaderTest : public testing::Test { InvertedIndexStorageFormatPB::V2, std::move(file_writer)); const TabletColumn& test_column = tablet_schema->column(0); - std::unique_ptr field(StorageFieldFactory::create(test_column)); - ASSERT_NE(field.get(), nullptr); + const TabletColumn* field = &(test_column); + ASSERT_NE(field, nullptr); std::unique_ptr column_writer; - auto status = IndexColumnWriter::create(field.get(), &column_writer, - index_file_writer.get(), &idx_meta); + auto status = IndexColumnWriter::create(field, &column_writer, index_file_writer.get(), + &idx_meta); // This should fail for unsupported types, demonstrating the default case // If it succeeds, we can still test with invalid query parameters diff --git a/be/test/storage/segment/inverted_index_writer_test.cpp b/be/test/storage/segment/inverted_index_writer_test.cpp index 85bff7c66ed1fb..c5bdf4c5547391 100644 --- a/be/test/storage/segment/inverted_index_writer_test.cpp +++ b/be/test/storage/segment/inverted_index_writer_test.cpp @@ -31,12 +31,12 @@ #include #include +#include "core/block/block.h" #include "core/data_type/data_type_factory.hpp" #include "core/data_type/data_type_number.h" #include "core/field.h" #include "io/fs/local_file_system.h" #include "runtime/runtime_state.h" -#include "storage/field.h" #include "storage/index/index_file_reader.h" #include "storage/index/index_file_writer.h" #include "storage/index/inverted/inverted_index_desc.h" @@ -44,6 +44,7 @@ #include "storage/index/inverted/inverted_index_reader.h" #include "storage/iterator/olap_data_convertor.h" #include "storage/tablet/tablet_schema.h" +#include "storage/types.h" #include "util/faststring.h" #include "util/slice.h" @@ -309,6 +310,125 @@ class InvertedIndexWriterTest : public testing::Test { return fmt::format("{}/{}_{}.dat", base, rowset_id, seg_id); } + // Check if .nrm file exists in the inverted index + // Norms files store normalization factors for scoring, typically created when field is tokenized + bool check_norms_file_exists(const std::string& index_prefix, const TabletIndex* index_meta) { + try { + std::unique_ptr reader = std::make_unique( + io::global_local_filesystem(), index_prefix, InvertedIndexStorageFormatPB::V2); + auto st = reader->init(); + EXPECT_TRUE(st.ok()); + auto result = reader->open(index_meta); + EXPECT_TRUE(result.has_value()); + auto compound_reader = std::move(result.value()); + + CLuceneError err; + CL_NS(store)::IndexInput* index_input = nullptr; + std::string file_str = InvertedIndexDescriptor::get_index_file_path_v2(index_prefix); + auto ok = DorisFSDirectory::FSIndexInput::open( + io::global_local_filesystem(), file_str.c_str(), index_input, err, 4096); + EXPECT_TRUE(ok); + + // Try to open the index reader to list all files + lucene::store::Directory* dir = compound_reader.get(); + lucene::index::IndexReader* r = lucene::index::IndexReader::open(dir); + + // Get the list of files in the directory + std::vector files; + dir->list(&files); + bool norms_found = false; + + for (const auto& file_name : files) { + // .nrm files are the norms data files in Lucene + // They have pattern: _N.nrm where N is a number (field number) + if (file_name.find(".nrm") != std::string::npos) { + norms_found = true; + } + } + + r->close(); + _CLLDELETE(r); + index_input->close(); + _CLLDELETE(index_input); + + return norms_found; + } catch (const CLuceneError& e) { + std::cout << "Error checking norms file: " << e.what() << std::endl; + return false; + } catch (const std::exception& e) { + std::cout << "Exception checking norms file: " << e.what() << std::endl; + return false; + } + } + + // Helper method to create an inverted index with tokenization enabled + void create_tokenized_index(std::string_view rowset_id, int seg_id, bool enable_analyzer) { + auto tablet_schema = create_schema(); + + // Create index meta with tokenization setting + auto index_meta_pb = std::make_unique(); + index_meta_pb->set_index_type(IndexType::INVERTED); + index_meta_pb->set_index_id(1); + index_meta_pb->set_index_name("test"); + index_meta_pb->clear_col_unique_id(); + index_meta_pb->add_col_unique_id(1); // c2 column id + + // Add parser type property to control tokenization + // should_analyzer returns true if: + // 1. analyzer or normalizer property is not empty, OR + // 2. parser type is not UNKNOWN and not NONE + auto* properties = index_meta_pb->mutable_properties(); + if (enable_analyzer) { + // Enable tokenization by setting parser to standard + // This will make should_analyzer() return true + (*properties)["parser"] = "standard"; + } + + TabletIndex idx_meta; + idx_meta.init_from_pb(*index_meta_pb.get()); + + std::string index_path_prefix {InvertedIndexDescriptor::get_index_file_path_prefix( + local_segment_path(kTestDir, rowset_id, seg_id))}; + std::string index_path = InvertedIndexDescriptor::get_index_file_path_v2(index_path_prefix); + + io::FileWriterPtr file_writer; + io::FileWriterOptions opts; + auto fs = io::global_local_filesystem(); + Status sts = fs->create_file(index_path, &file_writer, &opts); + ASSERT_TRUE(sts.ok()) << sts; + auto index_file_writer = std::make_unique( + fs, index_path_prefix, std::string {rowset_id}, seg_id, + InvertedIndexStorageFormatPB::V2, std::move(file_writer)); + + // Get field for column c2 + const TabletColumn& column = tablet_schema->column(1); // c2 is the second column + ASSERT_NE(&column, nullptr); + const TabletColumn* field = &(column); + ASSERT_NE(field, nullptr); + + // Create column writer + std::unique_ptr column_writer; + auto status = IndexColumnWriter::create(field, &column_writer, index_file_writer.get(), + &idx_meta); + EXPECT_TRUE(status.ok()) << status; + + // Add some string values + std::vector values = {Slice("hello world"), Slice("testing value"), + Slice("sample data")}; + + status = column_writer->add_values("c2", values.data(), values.size()); + EXPECT_TRUE(status.ok()) << status; + + // Finish and close + status = column_writer->finish(); + EXPECT_TRUE(status.ok()) << status; + + status = index_file_writer->begin_close(); + EXPECT_TRUE(status.ok()) << status; + status = index_file_writer->finish_close(); + EXPECT_TRUE(status.ok()) << status; + } + void test_string_write(std::string_view rowset_id, int seg_id) { auto tablet_schema = create_schema(); @@ -339,13 +459,13 @@ class InvertedIndexWriterTest : public testing::Test { // Get field for column c2 const TabletColumn& column = tablet_schema->column(1); // c2 is the second column ASSERT_NE(&column, nullptr); - std::unique_ptr field(StorageFieldFactory::create(column)); - ASSERT_NE(field.get(), nullptr); + const TabletColumn* field = &(column); + ASSERT_NE(field, nullptr); // Create column writer std::unique_ptr column_writer; - auto status = IndexColumnWriter::create(field.get(), &column_writer, - index_file_writer.get(), &idx_meta); + auto status = IndexColumnWriter::create(field, &column_writer, index_file_writer.get(), + &idx_meta); EXPECT_TRUE(status.ok()) << status; // Add string values @@ -403,13 +523,13 @@ class InvertedIndexWriterTest : public testing::Test { // Get field for column c2 const TabletColumn& column = tablet_schema->column(1); // c2 is the second column ASSERT_NE(&column, nullptr); - std::unique_ptr field(StorageFieldFactory::create(column)); - ASSERT_NE(field.get(), nullptr); + const TabletColumn* field = &(column); + ASSERT_NE(field, nullptr); // Create column writer std::unique_ptr column_writer; - auto status = IndexColumnWriter::create(field.get(), &column_writer, - index_file_writer.get(), &idx_meta); + auto status = IndexColumnWriter::create(field, &column_writer, index_file_writer.get(), + &idx_meta); EXPECT_TRUE(status.ok()) << status; // Add null values @@ -479,13 +599,13 @@ class InvertedIndexWriterTest : public testing::Test { // Get field for column c1 const TabletColumn& column = tablet_schema->column(0); ASSERT_NE(&column, nullptr); - std::unique_ptr field(StorageFieldFactory::create(column)); - ASSERT_NE(field.get(), nullptr); + const TabletColumn* field = &(column); + ASSERT_NE(field, nullptr); // Create column writer std::unique_ptr column_writer; - auto status = IndexColumnWriter::create(field.get(), &column_writer, - index_file_writer.get(), &idx_meta); + auto status = IndexColumnWriter::create(field, &column_writer, index_file_writer.get(), + &idx_meta); EXPECT_TRUE(status.ok()) << status; // Add integer values @@ -542,8 +662,8 @@ class InvertedIndexWriterTest : public testing::Test { // Get field for column c2 const TabletColumn& column = tablet_schema->column(1); // c2 is the second column ASSERT_NE(&column, nullptr); - std::unique_ptr field(StorageFieldFactory::create(column)); - ASSERT_NE(field.get(), nullptr); + const TabletColumn* field = &(column); + ASSERT_NE(field, nullptr); // Save original config value bool original_config_value = config::enable_inverted_index_correct_term_write; @@ -553,8 +673,8 @@ class InvertedIndexWriterTest : public testing::Test { // Create column writer std::unique_ptr column_writer; - auto status = IndexColumnWriter::create(field.get(), &column_writer, - index_file_writer.get(), &idx_meta); + auto status = IndexColumnWriter::create(field, &column_writer, index_file_writer.get(), + &idx_meta); EXPECT_TRUE(status.ok()) << status; // Add string values with Unicode characters above 0xFFFF @@ -727,8 +847,8 @@ TEST_F(InvertedIndexWriterTest, CompareUnicodeStringWriteResults) { // Get field for column c2 const TabletColumn& column = tablet_schema->column(1); // c2 is the second column ASSERT_NE(&column, nullptr); - std::unique_ptr field(StorageFieldFactory::create(column)); - ASSERT_NE(field.get(), nullptr); + const TabletColumn* field = &(column); + ASSERT_NE(field, nullptr); // Save original config value bool original_config_value = config::enable_inverted_index_correct_term_write; @@ -738,13 +858,13 @@ TEST_F(InvertedIndexWriterTest, CompareUnicodeStringWriteResults) { // Set config to enabled for first writer config::enable_inverted_index_correct_term_write = true; - auto status = IndexColumnWriter::create(field.get(), &column_writer_enabled, + auto status = IndexColumnWriter::create(field, &column_writer_enabled, index_file_writer_enabled.get(), &idx_meta); EXPECT_TRUE(status.ok()) << status; // Set config to disabled for second writer config::enable_inverted_index_correct_term_write = false; - status = IndexColumnWriter::create(field.get(), &column_writer_disabled, + status = IndexColumnWriter::create(field, &column_writer_disabled, index_file_writer_disabled.get(), &idx_meta); EXPECT_TRUE(status.ok()) << status; @@ -890,13 +1010,13 @@ TEST_F(InvertedIndexWriterTest, ErrorHandlingInFileWriter) { // Get field for column c2 const TabletColumn& column = tablet_schema->column(1); // c2 is the second column ASSERT_NE(&column, nullptr); - std::unique_ptr field(StorageFieldFactory::create(column)); - ASSERT_NE(field.get(), nullptr); + const TabletColumn* field = &(column); + ASSERT_NE(field, nullptr); // Create column writer std::unique_ptr column_writer; - auto status = IndexColumnWriter::create(field.get(), &column_writer, index_file_writer.get(), - &idx_meta); + auto status = + IndexColumnWriter::create(field, &column_writer, index_file_writer.get(), &idx_meta); EXPECT_TRUE(status.ok()) << status; // Test with empty values array to trigger certain error paths @@ -969,13 +1089,13 @@ TEST_F(InvertedIndexWriterTest, ArrayValuesWithNulls) { std::move(file_writer)); // Get field for array column - std::unique_ptr field(StorageFieldFactory::create(array_column)); - ASSERT_NE(field.get(), nullptr); + const TabletColumn* field = &(array_column); + ASSERT_NE(field, nullptr); // Create column writer std::unique_ptr column_writer; - auto status = IndexColumnWriter::create(field.get(), &column_writer, index_file_writer.get(), - &idx_meta); + auto status = + IndexColumnWriter::create(field, &column_writer, index_file_writer.get(), &idx_meta); EXPECT_TRUE(status.ok()) << status; // Construct arrays with mixed null and non-null elements (reference inverted_index_array_test.cpp) @@ -1024,7 +1144,7 @@ TEST_F(InvertedIndexWriterTest, ArrayValuesWithNulls) { const auto* item_nullmap = reinterpret_cast(data_ptr[3]); // Get the length of the subfield - auto field_size = field->get_sub_field(0)->size(); + auto field_size = field_type_size(field->get_sub_column(0).type()); // Call the inverted index writing interface status = column_writer->add_array_values(field_size, item_data, item_nullmap, offsets_ptr, @@ -1098,13 +1218,13 @@ TEST_F(InvertedIndexWriterTest, NumericArrayWithErrorConditions) { std::move(file_writer)); // Get field for array column - std::unique_ptr field(StorageFieldFactory::create(array_column)); - ASSERT_NE(field.get(), nullptr); + const TabletColumn* field = &(array_column); + ASSERT_NE(field, nullptr); // Create column writer std::unique_ptr column_writer; - auto status = IndexColumnWriter::create(field.get(), &column_writer, index_file_writer.get(), - &idx_meta); + auto status = + IndexColumnWriter::create(field, &column_writer, index_file_writer.get(), &idx_meta); EXPECT_TRUE(status.ok()) << status; // Construct numeric arrays (reference inverted_index_array_test.cpp) @@ -1155,7 +1275,7 @@ TEST_F(InvertedIndexWriterTest, NumericArrayWithErrorConditions) { const auto* item_nullmap = reinterpret_cast(data_ptr[3]); // Get the length of the subfield - auto field_size = field->get_sub_field(0)->size(); + auto field_size = field_type_size(field->get_sub_column(0).type()); // Call the inverted index writing interface status = column_writer->add_array_values(field_size, item_data, item_nullmap, offsets_ptr, @@ -1213,13 +1333,13 @@ TEST_F(InvertedIndexWriterTest, CopyFileErrorHandling) { // Get field for column c2 const TabletColumn& column = tablet_schema->column(1); // c2 is the second column ASSERT_NE(&column, nullptr); - std::unique_ptr field(StorageFieldFactory::create(column)); - ASSERT_NE(field.get(), nullptr); + const TabletColumn* field = &(column); + ASSERT_NE(field, nullptr); // Create column writer std::unique_ptr column_writer; - auto status = IndexColumnWriter::create(field.get(), &column_writer, index_file_writer.get(), - &idx_meta); + auto status = + IndexColumnWriter::create(field, &column_writer, index_file_writer.get(), &idx_meta); EXPECT_TRUE(status.ok()) << status; // Add some values to create index files @@ -1237,77 +1357,6 @@ TEST_F(InvertedIndexWriterTest, CopyFileErrorHandling) { EXPECT_TRUE(status.ok()) << status; } -// Test case for Collection value processing -TEST_F(InvertedIndexWriterTest, CollectionValueProcessing) { - auto tablet_schema = create_schema(); - - // Create index meta - auto index_meta_pb = std::make_unique(); - index_meta_pb->set_index_type(IndexType::INVERTED); - index_meta_pb->set_index_id(1); - index_meta_pb->set_index_name("test"); - index_meta_pb->clear_col_unique_id(); - index_meta_pb->add_col_unique_id(1); // c2 column id - - TabletIndex idx_meta; - idx_meta.init_from_pb(*index_meta_pb.get()); - - std::string index_path_prefix {InvertedIndexDescriptor::get_index_file_path_prefix( - local_segment_path(kTestDir, "test_collection", 0))}; - std::string index_path = InvertedIndexDescriptor::get_index_file_path_v2(index_path_prefix); - - io::FileWriterPtr file_writer; - io::FileWriterOptions opts; - auto fs = io::global_local_filesystem(); - Status sts = fs->create_file(index_path, &file_writer, &opts); - ASSERT_TRUE(sts.ok()) << sts; - - auto index_file_writer = std::make_unique( - fs, index_path_prefix, "test_collection", 0, InvertedIndexStorageFormatPB::V2, - std::move(file_writer)); - - // Get field for column c2 - const TabletColumn& column = tablet_schema->column(1); // c2 is the second column - ASSERT_NE(&column, nullptr); - std::unique_ptr field(StorageFieldFactory::create(column)); - ASSERT_NE(field.get(), nullptr); - - // Create column writer - std::unique_ptr column_writer; - auto status = IndexColumnWriter::create(field.get(), &column_writer, index_file_writer.get(), - &idx_meta); - EXPECT_TRUE(status.ok()) << status; - - // Create collection values for testing - std::vector test_strings = {"apple", "banana", "cherry"}; - std::vector slices; - for (const auto& s : test_strings) { - slices.emplace_back(s); - } - - // Create CollectionValue instances - std::vector collections; - CollectionValue collection1; - collection1.set_data(reinterpret_cast(slices.data())); - collection1.set_length(3); - bool null_signs[] = {false, false, false}; - collection1.set_null_signs(null_signs); - collections.push_back(collection1); - - // Test add_array_values with CollectionValue - status = column_writer->add_array_values(sizeof(Slice), collections.data(), 1); - EXPECT_TRUE(status.ok()) << status; - - // Finish and write - status = column_writer->finish(); - EXPECT_TRUE(status.ok()) << status; - - status = index_file_writer->begin_close(); - EXPECT_TRUE(status.ok()) << status; - status = index_file_writer->finish_close(); - EXPECT_TRUE(status.ok()) << status; -} - // Test case for BKD writer error conditions TEST_F(InvertedIndexWriterTest, BKDWriterErrorConditions) { auto tablet_schema = create_schema(); @@ -1344,13 +1393,13 @@ TEST_F(InvertedIndexWriterTest, BKDWriterErrorConditions) { // Get field for column c1 const TabletColumn& column = tablet_schema->column(0); ASSERT_NE(&column, nullptr); - std::unique_ptr field(StorageFieldFactory::create(column)); - ASSERT_NE(field.get(), nullptr); + const TabletColumn* field = &(column); + ASSERT_NE(field, nullptr); // Create column writer std::unique_ptr column_writer; - auto status = IndexColumnWriter::create(field.get(), &column_writer, index_file_writer.get(), - &idx_meta); + auto status = + IndexColumnWriter::create(field, &column_writer, index_file_writer.get(), &idx_meta); EXPECT_TRUE(status.ok()) << status; // Add some numeric values with edge cases @@ -1406,13 +1455,13 @@ TEST_F(InvertedIndexWriterTest, FileCreationAndOutputErrorHandling) { // Get field for column c2 const TabletColumn& column = tablet_schema->column(1); // c2 is the second column ASSERT_NE(&column, nullptr); - std::unique_ptr field(StorageFieldFactory::create(column)); - ASSERT_NE(field.get(), nullptr); + const TabletColumn* field = &(column); + ASSERT_NE(field, nullptr); // Create column writer std::unique_ptr column_writer; - auto status = IndexColumnWriter::create(field.get(), &column_writer, index_file_writer.get(), - &idx_meta); + auto status = + IndexColumnWriter::create(field, &column_writer, index_file_writer.get(), &idx_meta); EXPECT_TRUE(status.ok()) << status; // Add some values to ensure files are created diff --git a/be/test/storage/segment/segment_corruption_test.cpp b/be/test/storage/segment/segment_corruption_test.cpp index affeac8fc3770a..0212fd90d6252c 100644 --- a/be/test/storage/segment/segment_corruption_test.cpp +++ b/be/test/storage/segment/segment_corruption_test.cpp @@ -30,7 +30,6 @@ #include "io/cache/block_file_cache_factory.h" #include "io/fs/local_file_system.h" #include "runtime/exec_env.h" -#include "storage/field.h" #include "storage/index/index_file_reader.h" #include "storage/index/index_file_writer.h" #include "storage/index/inverted/inverted_index_cache.h" diff --git a/be/test/storage/segment/zone_map_index_test.cpp b/be/test/storage/segment/zone_map_index_test.cpp index bc9d5f352a7d8f..6d0f59da648be8 100644 --- a/be/test/storage/segment/zone_map_index_test.cpp +++ b/be/test/storage/segment/zone_map_index_test.cpp @@ -37,7 +37,6 @@ #include "exprs/function/cast/cast_to_string.h" #include "io/fs/file_writer.h" #include "io/fs/local_file_system.h" -#include "storage/field.h" #include "storage/olap_common.h" #include "storage/predicate/comparison_predicate.h" #include "storage/tablet/tablet_schema.h" @@ -60,7 +59,7 @@ class ColumnZoneMapTest : public testing::Test { } void TearDown() override { EXPECT_TRUE(_fs->delete_directory(kTestDir).ok()); } - void test_string(std::string testname, StorageField* field, DataTypePtr data_type_ptr) { + void test_string(std::string testname, const TabletColumn* field, DataTypePtr data_type_ptr) { std::string filename = kTestDir + "/" + testname; auto fs = io::global_local_filesystem(); @@ -129,10 +128,10 @@ class ColumnZoneMapTest : public testing::Test { } else { tab_col = create_string_key(0); } - auto field = std::unique_ptr(StorageFieldFactory::create(*tab_col)); + const TabletColumn* field = tab_col.get(); std::unique_ptr writer; - ASSERT_TRUE(ZoneMapIndexWriter::create(data_type, field.get(), writer).ok()); + ASSERT_TRUE(ZoneMapIndexWriter::create(data_type, field, writer).ok()); // Create a string longer than MAX_ZONE_MAP_INDEX_SIZE (512) std::string short_string = "mmmm"; @@ -249,7 +248,7 @@ class ColumnZoneMapTest : public testing::Test { auto data_type = DataTypeFactory::instance().create_data_type(TYPE_CHAR, true, 0, 0, length); auto tab_col = create_char_key(0, true, length); - auto field = std::unique_ptr(StorageFieldFactory::create(*tab_col)); + const TabletColumn* field = tab_col.get(); std::string s_less_than_schema_length1(length - 1, 'a'); std::string s_less_than_schema_length1_expect(length, 'a'); s_less_than_schema_length1_expect[length - 1] = '\0'; @@ -258,7 +257,7 @@ class ColumnZoneMapTest : public testing::Test { s_less_than_schema_length2_expect[length - 1] = '\0'; s_less_than_schema_length2_expect[length - 2] = '\0'; std::unique_ptr writer; - ASSERT_TRUE(ZoneMapIndexWriter::create(data_type, field.get(), writer).ok()); + ASSERT_TRUE(ZoneMapIndexWriter::create(data_type, field, writer).ok()); Slice slices[] = {Slice(s_less_than_schema_length1), Slice(s_less_than_schema_length2)}; writer->add_values(&slices, 2); if (pass_all) { @@ -343,10 +342,10 @@ class ColumnZoneMapTest : public testing::Test { precision, scale); TabletColumnPtr tab_col; tab_col = create_decimalv2_key(0, true); - auto field = std::unique_ptr(StorageFieldFactory::create(*tab_col)); + const TabletColumn* field = tab_col.get(); std::unique_ptr writer; - ASSERT_TRUE(ZoneMapIndexWriter::create(data_type, field.get(), writer).ok()); + ASSERT_TRUE(ZoneMapIndexWriter::create(data_type, field, writer).ok()); decimal12_t decimal1 {.integer = 123, .fraction = 456}; decimal12_t decimal2 {.integer = 223, .fraction = 4567}; @@ -431,10 +430,10 @@ class ColumnZoneMapTest : public testing::Test { auto data_type = DataTypeFactory::instance().create_data_type(TYPE_DATE, true); TabletColumnPtr tab_col; tab_col = create_datev1_key(0, true); - auto field = std::unique_ptr(StorageFieldFactory::create(*tab_col)); + const TabletColumn* field = tab_col.get(); std::unique_ptr writer; - ASSERT_TRUE(ZoneMapIndexWriter::create(data_type, field.get(), writer).ok()); + ASSERT_TRUE(ZoneMapIndexWriter::create(data_type, field, writer).ok()); VecDateTimeValue value1(false, TIME_DATE, 0, 0, 0, 2026, 2, 1); VecDateTimeValue value2(false, TIME_DATE, 0, 0, 0, 2026, 2, 2); @@ -512,10 +511,10 @@ class ColumnZoneMapTest : public testing::Test { auto data_type = DataTypeFactory::instance().create_data_type(TYPE_DATETIME, true); TabletColumnPtr tab_col; tab_col = create_datetimev1_key(0, true); - auto field = std::unique_ptr(StorageFieldFactory::create(*tab_col)); + const TabletColumn* field = tab_col.get(); std::unique_ptr writer; - ASSERT_TRUE(ZoneMapIndexWriter::create(data_type, field.get(), writer).ok()); + ASSERT_TRUE(ZoneMapIndexWriter::create(data_type, field, writer).ok()); VecDateTimeValue value1(false, TIME_DATETIME, 18, 12, 10, 2026, 2, 1); VecDateTimeValue value2(false, TIME_DATETIME, 18, 13, 0, 2026, 2, 2); @@ -586,7 +585,7 @@ TEST_F(ColumnZoneMapTest, NormalTestIntPage) { auto fs = io::global_local_filesystem(); TabletColumnPtr int_column = create_int_key(0); - StorageField* field = StorageFieldFactory::create(*int_column); + const TabletColumn* field = &(*int_column); auto data_type_ptr = DataTypeFactory::instance().create_data_type(TYPE_INT, false); std::unique_ptr builder(nullptr); @@ -635,35 +634,31 @@ TEST_F(ColumnZoneMapTest, NormalTestIntPage) { EXPECT_EQ(true, zone_maps[2].has_null()); EXPECT_EQ(false, zone_maps[2].has_not_null()); - delete field; } // Test for string TEST_F(ColumnZoneMapTest, NormalTestVarcharPage) { TabletColumnPtr varchar_column = create_varchar_key(0); - StorageField* field = StorageFieldFactory::create(*varchar_column); + const TabletColumn* field = &(*varchar_column); auto str_data_type_ptr = DataTypeFactory::instance().create_data_type(TYPE_VARCHAR, false); test_string("NormalTestVarcharPage", field, str_data_type_ptr); - delete field; } // Test for string TEST_F(ColumnZoneMapTest, NormalTestCharPage) { TabletColumnPtr char_column = create_char_key(0); - StorageField* field = StorageFieldFactory::create(*char_column); + const TabletColumn* field = &(*char_column); auto char_data_type_ptr = DataTypeFactory::instance().create_data_type(TYPE_CHAR, false); test_string("NormalTestCharPage", field, char_data_type_ptr); - delete field; } // Test for zone map limit TEST_F(ColumnZoneMapTest, ZoneMapCut) { TabletColumnPtr varchar_column = create_varchar_key(0); varchar_column->set_index_length(1024); - StorageField* field = StorageFieldFactory::create(*varchar_column); + const TabletColumn* field = &(*varchar_column); auto data_type_ptr = DataTypeFactory::instance().create_data_type(TYPE_VARCHAR, false); test_string("ZoneMapCut", field, data_type_ptr); - delete field; } TEST_F(ColumnZoneMapTest, StringColumnTruncation) { @@ -716,7 +711,7 @@ TEST_F(ColumnZoneMapTest, NormalTestFloatPage) { auto fs = io::global_local_filesystem(); auto column = create_float_column(0, true); - StorageField* field = StorageFieldFactory::create(*column); + const TabletColumn* field = &(*column); auto data_type_ptr = DataTypeFactory::instance().create_data_type(TYPE_FLOAT, false); std::unique_ptr builder(nullptr); @@ -798,7 +793,6 @@ TEST_F(ColumnZoneMapTest, NormalTestFloatPage) { EXPECT_EQ(true, zone_maps[2].has_null()); EXPECT_EQ(false, zone_maps[2].has_not_null()); - delete field; } TEST_F(ColumnZoneMapTest, NormalTestDoublePage) { @@ -806,7 +800,7 @@ TEST_F(ColumnZoneMapTest, NormalTestDoublePage) { auto fs = io::global_local_filesystem(); auto column = create_float_column(0, true); - StorageField* field = StorageFieldFactory::create(*column); + const TabletColumn* field = &(*column); auto data_type_ptr = DataTypeFactory::instance().create_data_type(TYPE_DOUBLE, false); std::unique_ptr builder(nullptr); @@ -889,7 +883,6 @@ TEST_F(ColumnZoneMapTest, NormalTestDoublePage) { EXPECT_EQ(true, zone_maps[2].has_null()); EXPECT_EQ(false, zone_maps[2].has_not_null()); - delete field; } TabletColumnPtr create_timestamptz_column(int32_t id, bool is_nullable) { @@ -910,7 +903,7 @@ TEST_F(ColumnZoneMapTest, TimestamptzPage) { auto fs = io::global_local_filesystem(); auto column = create_timestamptz_column(0, true); - StorageField* field = StorageFieldFactory::create(*column); + const TabletColumn* field = &(*column); auto data_type_ptr = DataTypeFactory::instance().create_data_type(TYPE_TIMESTAMPTZ, false); std::unique_ptr builder(nullptr); @@ -1066,7 +1059,6 @@ TEST_F(ColumnZoneMapTest, TimestamptzPage) { // page 5 EXPECT_EQ(true, zone_maps[4].has_null()); EXPECT_EQ(false, zone_maps[4].has_not_null()); - delete field; } // Regression test for "all-null page after a value page" — int variant. @@ -1078,11 +1070,11 @@ TEST_F(ColumnZoneMapTest, TimestamptzPage) { // a no-op. This test pins that behavior. TEST_F(ColumnZoneMapTest, AllNullPageAfterIntValues_SegmentMinMaxPreserved) { TabletColumnPtr int_column = create_int_key(0); - std::unique_ptr field(StorageFieldFactory::create(*int_column)); + const TabletColumn* field = &(*int_column); auto data_type_ptr = DataTypeFactory::instance().create_data_type(TYPE_INT, false); std::unique_ptr writer; - ASSERT_TRUE(ZoneMapIndexWriter::create(data_type_ptr, field.get(), writer).ok()); + ASSERT_TRUE(ZoneMapIndexWriter::create(data_type_ptr, field, writer).ok()); // Page 1: integers spanning [100, 200]. std::vector values = {100, 150, 200}; @@ -1133,10 +1125,10 @@ TEST_F(ColumnZoneMapTest, AllNullPageAfterIntValues_SegmentMinMaxPreserved) { TEST_F(ColumnZoneMapTest, AllNullPageAfterMaxLenStringPage_NoSegmentMaxDoubleIncrement) { auto data_type = DataTypeFactory::instance().create_data_type(TYPE_STRING, true); auto tab_col = create_string_key(0); - std::unique_ptr field(StorageFieldFactory::create(*tab_col)); + const TabletColumn* field = &(*tab_col); std::unique_ptr writer; - ASSERT_TRUE(ZoneMapIndexWriter::create(data_type, field.get(), writer).ok()); + ASSERT_TRUE(ZoneMapIndexWriter::create(data_type, field, writer).ok()); // Page 1: one string of exactly MAX_ZONE_MAP_INDEX_SIZE bytes, all 'x'. std::string long_x(MAX_ZONE_MAP_INDEX_SIZE, 'x'); diff --git a/be/test/storage/storage_types_test.cpp b/be/test/storage/storage_types_test.cpp index f0d285b5861dc2..45a89abe87e013 100644 --- a/be/test/storage/storage_types_test.cpp +++ b/be/test/storage/storage_types_test.cpp @@ -24,8 +24,6 @@ #include "core/decimal12.h" #include "core/uint24.h" #include "gtest/gtest_pred_impl.h" -#include "runtime/collection_value.h" -#include "storage/field.h" #include "storage/olap_common.h" #include "storage/tablet/tablet_schema.h" #include "storage/types.h" @@ -46,12 +44,10 @@ void common_test(typename TypeTraits::CppType src_val) { template void test_char(Slice src_val) { - StorageField* field = StorageFieldFactory::create_by_type(fieldType); - field->_length = src_val.size; - + auto field = std::make_unique(FieldAggregationMethod::OLAP_FIELD_AGGREGATION_NONE, + fieldType, false, 0, src_val.size); EXPECT_EQ(field->type(), fieldType); - EXPECT_EQ(sizeof(src_val), field->size()); - delete field; + EXPECT_EQ(sizeof(src_val), field_type_size(field->type())); } template <> @@ -64,7 +60,7 @@ void common_test(Slice src_val) { test_char(src_val); } -TEST(TypesTest, cmp_and_minmax) { +TEST(TypesTest, field_type_size_matches_cpp_type) { common_test(true); common_test(112); common_test(static_cast(54321)); @@ -88,81 +84,6 @@ TEST(TypesTest, cmp_and_minmax) { common_test(slice); } -template -void common_test_array(CollectionValue src_val) { - TabletColumn list_column(FieldAggregationMethod::OLAP_FIELD_AGGREGATION_NONE, - FieldType::OLAP_FIELD_TYPE_ARRAY); - int32_t item_length = 0; - if (item_type == FieldType::OLAP_FIELD_TYPE_CHAR || - item_type == FieldType::OLAP_FIELD_TYPE_VARCHAR) { - item_length = 10; - } - TabletColumn item_column(FieldAggregationMethod::OLAP_FIELD_AGGREGATION_NONE, item_type, true, - 0, item_length); - list_column.add_sub_column(item_column); - - ASSERT_EQ(item_type, list_column.get_sub_column(0).type()); -} - -TEST(ArrayTypeTest, copy_and_equal) { - bool bool_array[3] = {true, false, true}; - bool null_signs[3] = {true, true, true}; - common_test_array(CollectionValue(bool_array, 3, null_signs)); - - uint8_t tiny_int_array[3] = {3, 4, 5}; - common_test_array( - CollectionValue(tiny_int_array, 3, null_signs)); - - int16_t small_int_array[3] = {123, 234, 345}; - common_test_array( - CollectionValue(small_int_array, 3, null_signs)); - - int32_t int_array[3] = {-123454321, 123454321, 323412343}; - common_test_array(CollectionValue(int_array, 3, null_signs)); - - uint32_t uint_array[3] = {123454321, 2342341, 52435234}; - common_test_array( - CollectionValue(uint_array, 3, null_signs)); - - int64_t bigint_array[3] = {123454321123456789L, 23534543234L, -123454321123456789L}; - common_test_array( - CollectionValue(bigint_array, 3, null_signs)); - - __int128 large_int_array[3] = {1234567899L, 1234567899L, -12345631899L}; - common_test_array( - CollectionValue(large_int_array, 3, null_signs)); - - float float_array[3] = {1.11, 2.22, -3.33}; - common_test_array( - CollectionValue(float_array, 3, null_signs)); - - double double_array[3] = {12221.11, 12221.11, -12221.11}; - common_test_array( - CollectionValue(double_array, 3, null_signs)); - - decimal12_t decimal_array[3] = {{123, 234}, {345, 453}, {4524, 2123}}; - common_test_array( - CollectionValue(decimal_array, 3, null_signs)); - - uint24_t date_array[3] = {(1988 << 9) | (2 << 5) | 1, (1998 << 9) | (2 << 5) | 1, - (2008 << 9) | (2 << 5) | 1}; - common_test_array(CollectionValue(date_array, 3, null_signs)); - - uint32_t date_v2_array[3] = {(1988 << 9) | (2 << 5) | 1, (1998 << 9) | (2 << 5) | 1, - (2008 << 9) | (2 << 5) | 1}; - common_test_array( - CollectionValue(date_v2_array, 3, null_signs)); - - int64_t datetime_array[3] = {19880201010203L, 19980201010203L, 20080204010203L}; - common_test_array( - CollectionValue(datetime_array, 3, null_signs)); - - Slice char_array[3] = {"12345abcde", "12345abcde", "asdf322"}; - common_test_array(CollectionValue(char_array, 3, null_signs)); - common_test_array( - CollectionValue(char_array, 3, null_signs)); -} - TEST(TypesTest, has_char_type) { // Test basic types TabletColumn char_column(FieldAggregationMethod::OLAP_FIELD_AGGREGATION_NONE, From b3f7ac4ca6b10424d59d0536e52098bc68658187 Mon Sep 17 00:00:00 2001 From: Chenyang Sun Date: Fri, 29 May 2026 10:48:33 +0800 Subject: [PATCH 3/5] [refactor](BE) split EncodingInfo defaults into 4 explicit maps (#63622) Replace the EncodingPreference + runtime hook machinery in EncodingInfoResolver with four explicit maps and four matching get methods: - _v2_default_map -> get_v2_default_encoding(type) - _v3_default_map -> get_v3_default_encoding(type) - _index_column_default_map -> get_index_column_encoding(type) - _encoding_map -> get(type, encoding, out) No on-disk format change; the resolved encodings written into ColumnMetaPB match the pre-refactor outputs for both v2 and V3 tablets. Issue Number: close #xxx Related PR: #xxx Problem Summary: None - Test - [ ] Regression test - [x] Unit Test - [ ] Manual test (add detailed scripts or steps below) - [ ] No need to test or manual test. Explain why: - [x] This is a refactor/code format and no logic has been changed. - [ ] Previous test can cover this change. - [ ] No code files have been changed. - [ ] Other reason - Behavior changed: - [ ] No. - [ ] Yes. - Does this need documentation? - [ ] No. - [ ] Yes. - [ ] Confirm the release note - [ ] Confirm test cases - [ ] Confirm document - [ ] Add branch pick label --------- Co-authored-by: Claude Opus 4.7 (1M context) (cherry picked from commit 4f1dcdf33956a8adaf956ebca46da2820abf3ffb) --- be/src/cloud/pb_convert.cpp | 12 + .../storage/index/indexed_column_reader.cpp | 2 +- .../storage/index/indexed_column_writer.cpp | 13 +- be/src/storage/index/primary_key_index.cpp | 2 +- .../storage/index/zone_map/zone_map_index.cpp | 3 +- be/src/storage/segment/binary_dict_page.cpp | 19 +- be/src/storage/segment/binary_dict_page.h | 7 +- be/src/storage/segment/column_reader.cpp | 10 +- be/src/storage/segment/column_reader.h | 2 +- be/src/storage/segment/column_writer.cpp | 41 +- be/src/storage/segment/column_writer.h | 9 +- be/src/storage/segment/encoding_info.cpp | 443 ++++++++++-------- be/src/storage/segment/encoding_info.h | 129 +++-- be/src/storage/segment/options.h | 11 +- be/src/storage/segment/page_io.cpp | 8 +- be/src/storage/segment/page_io.h | 5 - be/src/storage/segment/segment_writer.cpp | 26 +- be/src/storage/segment/segment_writer.h | 2 +- .../variant/variant_column_writer_impl.cpp | 24 +- .../variant/variant_column_writer_impl.h | 2 +- .../segment/vertical_segment_writer.cpp | 26 +- .../storage/segment/vertical_segment_writer.h | 3 +- be/src/storage/tablet/tablet_meta.cpp | 11 +- be/src/storage/tablet/tablet_schema.cpp | 43 +- be/src/storage/tablet/tablet_schema.h | 36 +- .../exec/common/schema_util_rowset_test.cpp | 28 +- .../storage/segment/binary_dict_page_test.cpp | 68 +-- .../segment/column_meta_accessor_test.cpp | 17 +- .../segment/column_reader_cache_test.cpp | 39 +- .../storage/segment/encoding_info_test.cpp | 433 +++++++++++++---- .../segment/external_col_meta_util_test.cpp | 2 +- .../variant_column_writer_reader_test.cpp | 294 ++++++++++-- be/test/storage/tablet/tablet_schema_test.cpp | 66 +++ .../datasource/CloudInternalCatalog.java | 7 +- gensrc/proto/olap_file.proto | 20 + ...st_storage_format_controls_encoding.groovy | 19 +- 36 files changed, 1239 insertions(+), 643 deletions(-) diff --git a/be/src/cloud/pb_convert.cpp b/be/src/cloud/pb_convert.cpp index d7b34574e4242e..a90f93feeda77f 100644 --- a/be/src/cloud/pb_convert.cpp +++ b/be/src/cloud/pb_convert.cpp @@ -432,6 +432,9 @@ void doris_tablet_schema_to_cloud(TabletSchemaCloudPB* out, const TabletSchemaPB if (in.has_binary_plain_encoding_default_impl()) { out->set_binary_plain_encoding_default_impl(in.binary_plain_encoding_default_impl()); } + if (in.has_storage_format()) { + out->set_storage_format(in.storage_format()); + } if (in.has___split_schema()) { out->mutable___split_schema()->CopyFrom(in.__split_schema()); @@ -475,6 +478,9 @@ void doris_tablet_schema_to_cloud(TabletSchemaCloudPB* out, TabletSchemaPB&& in) if (in.has_binary_plain_encoding_default_impl()) { out->set_binary_plain_encoding_default_impl(in.binary_plain_encoding_default_impl()); } + if (in.has_storage_format()) { + out->set_storage_format(in.storage_format()); + } if (in.has___split_schema()) { out->mutable___split_schema()->CopyFrom(in.__split_schema()); @@ -531,6 +537,9 @@ void cloud_tablet_schema_to_doris(TabletSchemaPB* out, const TabletSchemaCloudPB if (in.has_binary_plain_encoding_default_impl()) { out->set_binary_plain_encoding_default_impl(in.binary_plain_encoding_default_impl()); } + if (in.has_storage_format()) { + out->set_storage_format(in.storage_format()); + } if (in.has___split_schema()) { out->mutable___split_schema()->CopyFrom(in.__split_schema()); @@ -575,6 +584,9 @@ void cloud_tablet_schema_to_doris(TabletSchemaPB* out, TabletSchemaCloudPB&& in) if (in.has_binary_plain_encoding_default_impl()) { out->set_binary_plain_encoding_default_impl(in.binary_plain_encoding_default_impl()); } + if (in.has_storage_format()) { + out->set_storage_format(in.storage_format()); + } if (in.has___split_schema()) { out->mutable___split_schema()->CopyFrom(in.__split_schema()); diff --git a/be/src/storage/index/indexed_column_reader.cpp b/be/src/storage/index/indexed_column_reader.cpp index b8fe9a57541a2e..d30c2c82dbc41b 100644 --- a/be/src/storage/index/indexed_column_reader.cpp +++ b/be/src/storage/index/indexed_column_reader.cpp @@ -68,7 +68,7 @@ Status IndexedColumnReader::load(bool use_page_cache, bool kept_in_memory, if (!is_scalar_type(_type)) { return Status::NotSupported("unsupported typeinfo, type={}", _meta.data_type()); } - RETURN_IF_ERROR(EncodingInfo::get(_type, _meta.encoding(), {}, &_encoding_info)); + RETURN_IF_ERROR(EncodingInfo::get(_type, _meta.encoding(), &_encoding_info)); _value_key_coder = get_key_coder(_type); // read and parse ordinal index page when exists diff --git a/be/src/storage/index/indexed_column_writer.cpp b/be/src/storage/index/indexed_column_writer.cpp index 0b0da38f812605..f148df1a6dcdfd 100644 --- a/be/src/storage/index/indexed_column_writer.cpp +++ b/be/src/storage/index/indexed_column_writer.cpp @@ -54,12 +54,15 @@ IndexedColumnWriter::IndexedColumnWriter(const IndexedColumnWriterOptions& optio IndexedColumnWriter::~IndexedColumnWriter() = default; Status IndexedColumnWriter::init() { + // Caller must set _options.encoding to a concrete value before calling init. + if (_options.encoding == DEFAULT_ENCODING) { + return Status::InternalError( + "IndexedColumnWriterOptions::encoding is DEFAULT_ENCODING for type={}; caller must " + "resolve to a concrete encoding before IndexedColumnWriter::init", + _type); + } const EncodingInfo* encoding_info; - RETURN_IF_ERROR(EncodingInfo::get(_type, _options.encoding, {}, &encoding_info)); - _options.encoding = encoding_info->encoding(); - // should store more concrete encoding type instead of DEFAULT_ENCODING - // because the default encoding of a data type can be changed in the future - DCHECK_NE(_options.encoding, DEFAULT_ENCODING); + RETURN_IF_ERROR(EncodingInfo::get(_type, _options.encoding, &encoding_info)); PageBuilder* data_page_builder = nullptr; PageBuilderOptions builder_option; diff --git a/be/src/storage/index/primary_key_index.cpp b/be/src/storage/index/primary_key_index.cpp index d91f9e8f586ea6..654584948e7724 100644 --- a/be/src/storage/index/primary_key_index.cpp +++ b/be/src/storage/index/primary_key_index.cpp @@ -42,7 +42,7 @@ Status PrimaryKeyIndexBuilder::init() { options.write_ordinal_index = true; options.write_value_index = true; options.data_page_size = config::primary_key_data_page_size; - options.encoding = segment_v2::EncodingInfo::get_default_encoding(type, {}, true); + options.encoding = segment_v2::EncodingInfo::get_index_column_encoding(type); options.compression = segment_v2::ZSTD; _primary_key_index_builder.reset( new segment_v2::IndexedColumnWriter(options, type, _file_writer)); diff --git a/be/src/storage/index/zone_map/zone_map_index.cpp b/be/src/storage/index/zone_map/zone_map_index.cpp index cc59b8d02df73b..3c5cb175616c53 100644 --- a/be/src/storage/index/zone_map/zone_map_index.cpp +++ b/be/src/storage/index/zone_map/zone_map_index.cpp @@ -283,7 +283,8 @@ Status TypedZoneMapIndexWriter::finish(io::FileWriter* file_writer, IndexedColumnWriterOptions options; options.write_ordinal_index = true; options.write_value_index = false; - options.encoding = EncodingInfo::get_default_encoding(type, {}, false); + // Zone map page always uses PLAIN_ENCODING. Do not change. + options.encoding = PLAIN_ENCODING; options.compression = NO_COMPRESSION; // currently not compressed IndexedColumnWriter writer(options, type, file_writer); diff --git a/be/src/storage/segment/binary_dict_page.cpp b/be/src/storage/segment/binary_dict_page.cpp index 7b89a91c008cb1..ebfedcb4afe522 100644 --- a/be/src/storage/segment/binary_dict_page.cpp +++ b/be/src/storage/segment/binary_dict_page.cpp @@ -47,13 +47,8 @@ BinaryDictPageBuilder::BinaryDictPageBuilder(const PageBuilderOptions& options) _data_page_builder(nullptr), _dict_builder(nullptr), _encoding_type(DICT_ENCODING), - _dict_word_page_encoding_type( - options.encoding_preference.binary_plain_encoding_default_impl == - BinaryPlainEncodingTypePB::BINARY_PLAIN_ENCODING_V2 - ? PLAIN_ENCODING_V2 - : PLAIN_ENCODING), - _fallback_binary_encoding_type( - options.encoding_preference.binary_plain_encoding_default_impl == + _binary_plain_encoding_type( + options.dict_binary_plain_encoding == BinaryPlainEncodingTypePB::BINARY_PLAIN_ENCODING_V2 ? PLAIN_ENCODING_V2 : PLAIN_ENCODING) {} @@ -74,7 +69,7 @@ Status BinaryDictPageBuilder::init() { const EncodingInfo* encoding_info; RETURN_IF_ERROR(EncodingInfo::get(FieldType::OLAP_FIELD_TYPE_VARCHAR, - _dict_word_page_encoding_type, {}, &encoding_info)); + _binary_plain_encoding_type, &encoding_info)); RETURN_IF_ERROR(encoding_info->create_page_builder(dict_builder_options, _dict_builder)); return reset(); } @@ -183,13 +178,11 @@ Status BinaryDictPageBuilder::reset() { _buffer.resize(BINARY_DICT_PAGE_HEADER_SIZE); if (_encoding_type == DICT_ENCODING && _dict_builder->is_page_full()) { - DCHECK(_fallback_binary_encoding_type == PLAIN_ENCODING || - _fallback_binary_encoding_type == PLAIN_ENCODING_V2); const EncodingInfo* encoding_info; RETURN_IF_ERROR(EncodingInfo::get(FieldType::OLAP_FIELD_TYPE_VARCHAR, - _fallback_binary_encoding_type, {}, &encoding_info)); + _binary_plain_encoding_type, &encoding_info)); RETURN_IF_ERROR(encoding_info->create_page_builder(_options, _data_page_builder)); - _encoding_type = _fallback_binary_encoding_type; + _encoding_type = _binary_plain_encoding_type; } else { RETURN_IF_ERROR(_data_page_builder->reset()); } @@ -210,7 +203,7 @@ Status BinaryDictPageBuilder::get_dictionary_page(OwnedSlice* dictionary_page) { } Status BinaryDictPageBuilder::get_dictionary_page_encoding(EncodingTypePB* encoding) const { - *encoding = _dict_word_page_encoding_type; + *encoding = _binary_plain_encoding_type; return Status::OK(); } diff --git a/be/src/storage/segment/binary_dict_page.h b/be/src/storage/segment/binary_dict_page.h index 861b57f776353f..c53676285612d1 100644 --- a/be/src/storage/segment/binary_dict_page.h +++ b/be/src/storage/segment/binary_dict_page.h @@ -106,10 +106,9 @@ class BinaryDictPageBuilder : public PageBuilderHelper { EncodingTypePB _encoding_type; - EncodingTypePB - _dict_word_page_encoding_type; // currently only support PLAIN_ENCODING and PLAIN_ENCODING_V2 - EncodingTypePB - _fallback_binary_encoding_type; // currently only support PLAIN_ENCODING and PLAIN_ENCODING_V2 + // Binary-plain flavor (V1 or V2) used both for the dictionary word page and for the data + // page when the dictionary overflows. Resolved from PageBuilderOptions::dict_binary_plain_encoding. + const EncodingTypePB _binary_plain_encoding_type; struct HashOfSlice { size_t operator()(const Slice& slice) const { return crc32_hash(slice.data, slice.size); } diff --git a/be/src/storage/segment/column_reader.cpp b/be/src/storage/segment/column_reader.cpp index b3884609db4682..89b3b2e3673e6a 100644 --- a/be/src/storage/segment/column_reader.cpp +++ b/be/src/storage/segment/column_reader.cpp @@ -356,7 +356,7 @@ Status ColumnReader::init(const ColumnMetaPB* meta) { if (_type == FieldType::OLAP_FIELD_TYPE_NONE || _type == FieldType::OLAP_FIELD_TYPE_UNKNOWN) { return Status::NotSupported("unsupported typeinfo, type={}", meta->type()); } - RETURN_IF_ERROR(EncodingInfo::get(_type, meta->encoding(), {}, &_encoding_info)); + RETURN_IF_ERROR(EncodingInfo::get(_type, meta->encoding(), &_encoding_info)); for (int i = 0; i < meta->indexes_size(); i++) { const auto& index_meta = meta->indexes(i); @@ -414,7 +414,7 @@ Status ColumnReader::new_index_iterator(const std::shared_ptr& Status ColumnReader::read_page(const ColumnIteratorOptions& iter_opts, const PagePointer& pp, PageHandle* handle, Slice* page_body, PageFooterPB* footer, - BlockCompressionCodec* codec, bool is_dict_page) const { + BlockCompressionCodec* codec) const { SCOPED_CONCURRENCY_COUNT(ConcurrencyStatsManager::instance().column_reader_read_page); iter_opts.sanity_check(); PageReadOptions opts(iter_opts.io_ctx); @@ -427,7 +427,6 @@ Status ColumnReader::read_page(const ColumnIteratorOptions& iter_opts, const Pag opts.codec = codec; opts.stats = iter_opts.stats; opts.encoding_info = _encoding_info; - opts.is_dict_page = is_dict_page; return PageIO::read_and_decompress_page(opts, handle, page_body, footer); } @@ -2041,11 +2040,10 @@ Status FileColumnIterator::_read_dict_data() { _opts.type = INDEX_PAGE; RETURN_IF_ERROR(_reader->read_page(_opts, _reader->get_dict_page_pointer(), &_dict_page_handle, - &dict_data, &dict_footer, _compress_codec, true)); + &dict_data, &dict_footer, _compress_codec)); const EncodingInfo* encoding_info; RETURN_IF_ERROR(EncodingInfo::get(FieldType::OLAP_FIELD_TYPE_VARCHAR, - dict_footer.dict_page_footer().encoding(), {}, - &encoding_info)); + dict_footer.dict_page_footer().encoding(), &encoding_info)); RETURN_IF_ERROR(encoding_info->create_page_decoder(dict_data, {}, _dict_decoder)); RETURN_IF_ERROR(_dict_decoder->init()); diff --git a/be/src/storage/segment/column_reader.h b/be/src/storage/segment/column_reader.h index 0c33ed91de4046..c939d617346f87 100644 --- a/be/src/storage/segment/column_reader.h +++ b/be/src/storage/segment/column_reader.h @@ -178,7 +178,7 @@ class ColumnReader : public MetadataAdder, // read a page from file into a page handle Status read_page(const ColumnIteratorOptions& iter_opts, const PagePointer& pp, PageHandle* handle, Slice* page_body, PageFooterPB* footer, - BlockCompressionCodec* codec, bool is_dict_page = false) const; + BlockCompressionCodec* codec) const; bool is_nullable() const { return _meta_is_nullable; } diff --git a/be/src/storage/segment/column_writer.cpp b/be/src/storage/segment/column_writer.cpp index 3f7b0687dd7582..d11e6bbb93c822 100644 --- a/be/src/storage/segment/column_writer.cpp +++ b/be/src/storage/segment/column_writer.cpp @@ -131,18 +131,19 @@ inline ScalarColumnWriter* get_null_writer(const ColumnWriterOptions& opts, null_options.meta->set_is_nullable(false); null_options.meta->set_length( cast_set(field_type_size(FieldType::OLAP_FIELD_TYPE_TINYINT))); - null_options.meta->set_encoding(DEFAULT_ENCODING); null_options.meta->set_compression(opts.meta->compression()); null_options.need_zone_map = false; null_options.need_bloom_filter = false; - null_options.encoding_preference = opts.encoding_preference; + null_options.storage_format = opts.storage_format; auto null_column_ptr = std::make_shared( FieldAggregationMethod::OLAP_FIELD_AGGREGATION_NONE, null_type, false, null_options.meta->unique_id(), null_options.meta->length()); null_column_ptr->set_name("nullable"); null_column_ptr->set_index_length(-1); // no short key index + null_options.meta->set_encoding( + EncodingInfo::resolve_default_encoding(opts.storage_format, *null_column_ptr)); return new ScalarColumnWriter(null_options, std::move(null_column_ptr), file_writer); } @@ -166,7 +167,7 @@ Status ColumnWriter::create_struct_writer(const ColumnWriterOptions& opts, column_options.meta = opts.meta->mutable_children_columns(i); column_options.need_zone_map = false; column_options.need_bloom_filter = sub_column.is_bf_column(); - column_options.encoding_preference = opts.encoding_preference; + column_options.storage_format = opts.storage_format; std::unique_ptr sub_column_writer; RETURN_IF_ERROR( ColumnWriter::create(column_options, &sub_column, file_writer, &sub_column_writer)); @@ -193,7 +194,7 @@ Status ColumnWriter::create_array_writer(const ColumnWriterOptions& opts, item_options.meta = opts.meta->mutable_children_columns(0); item_options.need_zone_map = false; item_options.need_bloom_filter = item_column.is_bf_column(); - item_options.encoding_preference = opts.encoding_preference; + item_options.storage_format = opts.storage_format; std::unique_ptr item_writer; RETURN_IF_ERROR(ColumnWriter::create(item_options, &item_column, file_writer, &item_writer)); @@ -208,12 +209,11 @@ Status ColumnWriter::create_array_writer(const ColumnWriterOptions& opts, length_options.meta->set_is_nullable(false); length_options.meta->set_length( cast_set(field_type_size(FieldType::OLAP_FIELD_TYPE_UNSIGNED_BIGINT))); - length_options.meta->set_encoding(DEFAULT_ENCODING); length_options.meta->set_compression(opts.meta->compression()); length_options.need_zone_map = false; length_options.need_bloom_filter = false; - length_options.encoding_preference = opts.encoding_preference; + length_options.storage_format = opts.storage_format; auto length_column_ptr = std::make_shared( FieldAggregationMethod::OLAP_FIELD_AGGREGATION_NONE, length_type, @@ -221,6 +221,8 @@ Status ColumnWriter::create_array_writer(const ColumnWriterOptions& opts, length_options.meta->length()); length_column_ptr->set_name("length"); length_column_ptr->set_index_length(-1); // no short key index + length_options.meta->set_encoding( + EncodingInfo::resolve_default_encoding(opts.storage_format, *length_column_ptr)); auto* length_writer = new OffsetColumnWriter(length_options, std::move(length_column_ptr), file_writer); @@ -252,7 +254,7 @@ Status ColumnWriter::create_map_writer(const ColumnWriterOptions& opts, const Ta item_options.meta = opts.meta->mutable_children_columns(i); item_options.need_zone_map = false; item_options.need_bloom_filter = item_column.is_bf_column(); - item_options.encoding_preference = opts.encoding_preference; + item_options.storage_format = opts.storage_format; std::unique_ptr item_writer; RETURN_IF_ERROR( ColumnWriter::create(item_options, &item_column, file_writer, &item_writer)); @@ -271,12 +273,11 @@ Status ColumnWriter::create_map_writer(const ColumnWriterOptions& opts, const Ta length_options.meta->set_is_nullable(false); length_options.meta->set_length( cast_set(field_type_size(FieldType::OLAP_FIELD_TYPE_UNSIGNED_BIGINT))); - length_options.meta->set_encoding(DEFAULT_ENCODING); length_options.meta->set_compression(opts.meta->compression()); length_options.need_zone_map = false; length_options.need_bloom_filter = false; - length_options.encoding_preference = opts.encoding_preference; + length_options.storage_format = opts.storage_format; auto length_column_ptr = std::make_shared( FieldAggregationMethod::OLAP_FIELD_AGGREGATION_NONE, length_type, @@ -284,6 +285,8 @@ Status ColumnWriter::create_map_writer(const ColumnWriterOptions& opts, const Ta length_options.meta->length()); length_column_ptr->set_name("length"); length_column_ptr->set_index_length(-1); // no short key index + length_options.meta->set_encoding( + EncodingInfo::resolve_default_encoding(opts.storage_format, *length_column_ptr)); auto* length_writer = new OffsetColumnWriter(length_options, std::move(length_column_ptr), file_writer); @@ -488,22 +491,28 @@ Status ScalarColumnWriter::init() { PageBuilder* page_builder = nullptr; - RETURN_IF_ERROR(EncodingInfo::get(get_column()->type(), _opts.meta->encoding(), - _opts.encoding_preference, &_encoding_info)); - _opts.meta->set_encoding(_encoding_info->encoding()); + // Caller must set a concrete (non-DEFAULT) encoding on the meta before init. + if (_opts.meta->encoding() == DEFAULT_ENCODING) { + return Status::InternalError( + "ColumnMetaPB encoding is DEFAULT_ENCODING for column_id={}, type={}; caller must " + "resolve to a concrete encoding before ScalarColumnWriter::init", + _opts.meta->column_id(), get_column()->type()); + } + RETURN_IF_ERROR( + EncodingInfo::get(get_column()->type(), _opts.meta->encoding(), &_encoding_info)); // create page builder PageBuilderOptions opts; opts.data_page_size = _opts.data_page_size; opts.dict_page_size = _opts.dict_page_size; - opts.encoding_preference = _opts.encoding_preference; + opts.dict_binary_plain_encoding = + (_opts.storage_format == TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V3) + ? BinaryPlainEncodingTypePB::BINARY_PLAIN_ENCODING_V2 + : BinaryPlainEncodingTypePB::BINARY_PLAIN_ENCODING_V1; RETURN_IF_ERROR(_encoding_info->create_page_builder(opts, &page_builder)); if (page_builder == nullptr) { return Status::NotSupported("Failed to create page builder for type {} and encoding {}", get_column()->type(), _opts.meta->encoding()); } - // should store more concrete encoding type instead of DEFAULT_ENCODING - // because the default encoding of a data type can be changed in the future - DCHECK_NE(_opts.meta->encoding(), DEFAULT_ENCODING); VLOG_DEBUG << fmt::format( "[verbose] scalar column writer init, column_id={}, type={}, encoding={}, " "is_nullable={}", diff --git a/be/src/storage/segment/column_writer.h b/be/src/storage/segment/column_writer.h index 44567c2d8d3f6c..50822c945c1bb7 100644 --- a/be/src/storage/segment/column_writer.h +++ b/be/src/storage/segment/column_writer.h @@ -17,6 +17,8 @@ #pragma once +#include +#include #include #include #include @@ -85,7 +87,12 @@ struct ColumnWriterOptions { std::vector input_rs_readers; const TabletIndex* ann_index = nullptr; - EncodingPreference encoding_preference {}; + // Storage format of the owning tablet (V2 or V3). Set once by the segment writer + // (from TabletMeta::storage_format()) and propagated down to aux child writers + // (null / array-length / map-length), struct subcolumn writers and variant subcolumn + // writers. All encoding-default decisions consult this via resolve_default_encoding(). + // Also forwarded to BinaryDictPageBuilder via PageBuilderOptions::binary_plain_encoding. + TabletStorageFormatPB storage_format = TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2; std::string to_string() const { std::stringstream ss; diff --git a/be/src/storage/segment/encoding_info.cpp b/be/src/storage/segment/encoding_info.cpp index 9fdad8504063aa..30700fe0f41fd0 100644 --- a/be/src/storage/segment/encoding_info.cpp +++ b/be/src/storage/segment/encoding_info.cpp @@ -27,6 +27,7 @@ #include #include "common/config.h" +#include "common/exception.h" #include "runtime/exec_env.h" #include "storage/olap_common.h" #include "storage/segment/binary_dict_page.h" @@ -41,6 +42,7 @@ #include "storage/segment/options.h" #include "storage/segment/plain_page.h" #include "storage/segment/rle_page.h" +#include "storage/tablet/tablet_schema.h" #include "storage/types.h" namespace doris { @@ -214,124 +216,181 @@ struct TypeEncodingTraits { }; EncodingInfoResolver::EncodingInfoResolver() { - _add_map(); - _add_map(); - _add_map(); - - _add_map(); - _add_map(); - _add_map(); - - _add_map(); - _add_map(); - _add_map(); - - _add_map(); - _add_map(); - _add_map(); - - _add_map(); - _add_map(); - - _add_map(); - _add_map(); - _add_map(); - - _add_map(); - _add_map(); - - _add_map(); - _add_map(); - - _add_map(); - _add_map(); - _add_map(); - _add_map(); - - _add_map(); - _add_map(); - _add_map(); - _add_map(); - - _add_map(); - _add_map(); - _add_map(); - _add_map(); - - _add_map(); - _add_map(); - _add_map(); - _add_map(); - - _add_map(); - _add_map(); - _add_map(); - _add_map(); - - _add_map(); - _add_map(); - _add_map(); - _add_map(); - - _add_map(); - _add_map(); - _add_map(); - - _add_map(); - _add_map(); - _add_map(); - - _add_map(); - _add_map(); - _add_map(); - - _add_map(); - _add_map(); - _add_map(); - - _add_map(); - _add_map(); - _add_map(); - - _add_map(); - _add_map(); - _add_map(); - - _add_map(); - _add_map(); - _add_map(); - - _add_map(); - _add_map(); - _add_map(); - - _add_map(); - _add_map(); - _add_map(); - - _add_map(); - _add_map(); - _add_map(); - - _add_map(); - _add_map(); - _add_map(); - - _add_map(); - _add_map(); - _add_map(); - - _add_map(); - _add_map(); - - _add_map(); - _add_map(); - - _add_map(); - _add_map(); - - _add_map(); - _add_map(); + // ===== Phase 1: register every supported (type, encoding) combination exactly once ===== + // _register_supported_encoding CHECKs against duplicates; the Phase 2 calls below do not insert into _encoding_map, + // so every (type, encoding) used as a default must appear here first. + + // signed integers + _register_supported_encoding(); + _register_supported_encoding(); + _register_supported_encoding(); + _register_supported_encoding(); + _register_supported_encoding(); + _register_supported_encoding(); + _register_supported_encoding(); + _register_supported_encoding(); + _register_supported_encoding(); + _register_supported_encoding(); + _register_supported_encoding(); + _register_supported_encoding(); + _register_supported_encoding(); + _register_supported_encoding(); + _register_supported_encoding(); + + // unsigned integers + _register_supported_encoding(); + _register_supported_encoding(); + + // FLOAT / DOUBLE + _register_supported_encoding(); + _register_supported_encoding(); + _register_supported_encoding(); + _register_supported_encoding(); + + // binary types (CHAR/VARCHAR/STRING/JSONB/VARIANT) + _register_supported_encoding(); + _register_supported_encoding(); + _register_supported_encoding(); + _register_supported_encoding(); + _register_supported_encoding(); + _register_supported_encoding(); + _register_supported_encoding(); + _register_supported_encoding(); + _register_supported_encoding(); + _register_supported_encoding(); + _register_supported_encoding(); + _register_supported_encoding(); + _register_supported_encoding(); + _register_supported_encoding(); + _register_supported_encoding(); + _register_supported_encoding(); + _register_supported_encoding(); + _register_supported_encoding(); + _register_supported_encoding(); + _register_supported_encoding(); + + // BOOL + _register_supported_encoding(); + _register_supported_encoding(); + _register_supported_encoding(); + + // date / datetime + _register_supported_encoding(); + _register_supported_encoding(); + _register_supported_encoding(); + _register_supported_encoding(); + _register_supported_encoding(); + _register_supported_encoding(); + _register_supported_encoding(); + _register_supported_encoding(); + _register_supported_encoding(); + _register_supported_encoding(); + _register_supported_encoding(); + _register_supported_encoding(); + _register_supported_encoding(); + _register_supported_encoding(); + _register_supported_encoding(); + + // decimal + _register_supported_encoding(); + _register_supported_encoding(); + _register_supported_encoding(); + _register_supported_encoding(); + _register_supported_encoding(); + _register_supported_encoding(); + _register_supported_encoding(); + _register_supported_encoding(); + _register_supported_encoding(); + _register_supported_encoding(); + + // ip + _register_supported_encoding(); + _register_supported_encoding(); + _register_supported_encoding(); + _register_supported_encoding(); + + // aggregate / binary-flavored types + _register_supported_encoding(); + _register_supported_encoding(); + _register_supported_encoding(); + _register_supported_encoding(); + _register_supported_encoding(); + _register_supported_encoding(); + _register_supported_encoding(); + _register_supported_encoding(); + + // ===== Phase 2a: V2 defaults (write path, V1/V2 segments) ===== + _set_v2_default(); + _set_v2_default(); + _set_v2_default(); + _set_v2_default(); + _set_v2_default(); + _set_v2_default(); + _set_v2_default(); + _set_v2_default(); + _set_v2_default(); + _set_v2_default(); + _set_v2_default(); + _set_v2_default(); + _set_v2_default(); + _set_v2_default(); + _set_v2_default(); + _set_v2_default(); + _set_v2_default(); + _set_v2_default(); + _set_v2_default(); + _set_v2_default(); + _set_v2_default(); + _set_v2_default(); + _set_v2_default(); + _set_v2_default(); + _set_v2_default(); + _set_v2_default(); + _set_v2_default(); + _set_v2_default(); + _set_v2_default(); + _set_v2_default(); + _set_v2_default(); + + // ===== Phase 2b: V3 defaults (write path, V3 segments) ===== + _set_v3_default(); + _set_v3_default(); + _set_v3_default(); + _set_v3_default(); + _set_v3_default(); + _set_v3_default(); + _set_v3_default(); + _set_v3_default(); + _set_v3_default(); + _set_v3_default(); + _set_v3_default(); + _set_v3_default(); + _set_v3_default(); + _set_v3_default(); + _set_v3_default(); + _set_v3_default(); + _set_v3_default(); + _set_v3_default(); + _set_v3_default(); + _set_v3_default(); + _set_v3_default(); + _set_v3_default(); + _set_v3_default(); + _set_v3_default(); + _set_v3_default(); + _set_v3_default(); + _set_v3_default(); + _set_v3_default(); + _set_v3_default(); + _set_v3_default(); + _set_v3_default(); + + // ===== Phase 2c: IndexedColumn (value-seek) defaults ===== + // Only the PrimaryKeyIndexBuilder consults this map, and it hardcodes VARCHAR. + // Other types were registered historically (since #2308, 2019) for a generic + // "any IndexedColumn value-seek caller" use case that never materialized; they + // were removed to keep this map honest about what production actually needs. + _set_index_column_encoding(); } EncodingInfoResolver::~EncodingInfoResolver() { @@ -341,90 +400,20 @@ EncodingInfoResolver::~EncodingInfoResolver() { _encoding_map.clear(); } -namespace { -bool is_integer_type(FieldType type) { - return type == FieldType::OLAP_FIELD_TYPE_TINYINT || - type == FieldType::OLAP_FIELD_TYPE_SMALLINT || type == FieldType::OLAP_FIELD_TYPE_INT || - type == FieldType::OLAP_FIELD_TYPE_BIGINT || type == FieldType::OLAP_FIELD_TYPE_LARGEINT; +EncodingTypePB EncodingInfoResolver::get_v2_default_encoding(FieldType type) const { + return _lookup(_v2_default_map, type); } -bool is_binary_type(FieldType type) { - return type == FieldType::OLAP_FIELD_TYPE_CHAR || type == FieldType::OLAP_FIELD_TYPE_VARCHAR || - type == FieldType::OLAP_FIELD_TYPE_STRING || type == FieldType::OLAP_FIELD_TYPE_JSONB || - type == FieldType::OLAP_FIELD_TYPE_VARIANT || type == FieldType::OLAP_FIELD_TYPE_HLL || - type == FieldType::OLAP_FIELD_TYPE_BITMAP || - type == FieldType::OLAP_FIELD_TYPE_QUANTILE_STATE || - type == FieldType::OLAP_FIELD_TYPE_AGG_STATE; +EncodingTypePB EncodingInfoResolver::get_v3_default_encoding(FieldType type) const { + return _lookup(_v3_default_map, type); } -} // namespace - -EncodingTypePB EncodingInfoResolver::get_default_encoding(FieldType type, - EncodingPreference encoding_preference, - bool optimize_value_seek) const { - // Predicate for default encoding transformation - // Parameters: (type, current_default_encoding, optimize_value_seek) - // Returns: true if the transformation should be applied - using Predicate = std::function; - - // Hook for transforming default encoding: predicate -> target encoding - struct EncodingTransform { - Predicate predicate; - EncodingTypePB target_encoding; - }; - - // Static array of hooks for default encoding transformations - static const std::vector hooks = { - // Hook 1: Binary types - PLAIN_ENCODING -> PLAIN_ENCODING_V2 - // Applies when: type is binary, encoding is PLAIN_ENCODING, and config enables v2 - EncodingTransform { - .predicate = - [](FieldType type, EncodingTypePB encoding, - EncodingPreference encoding_preference, bool optimize_value_seek) { - return encoding == PLAIN_ENCODING && is_binary_type(type) && - encoding_preference.binary_plain_encoding_default_impl == - BinaryPlainEncodingTypePB::BINARY_PLAIN_ENCODING_V2; - }, - .target_encoding = PLAIN_ENCODING_V2}, - - // Hook 2: Integer types - any encoding -> PLAIN_ENCODING - // Applies when: type is integer and config enables plain encoding for integers - EncodingTransform { - .predicate = - [](FieldType type, EncodingTypePB encoding, - EncodingPreference encoding_preference, bool optimize_value_seek) { - return is_integer_type(type) && - encoding_preference.integer_type_default_use_plain_encoding; - }, - .target_encoding = PLAIN_ENCODING}}; - - auto& encoding_map = - optimize_value_seek ? _value_seek_encoding_map : _default_encoding_type_map; - auto it = encoding_map.find(type); - if (it != encoding_map.end()) { - EncodingTypePB encoding = it->second; - - // Apply hooks in order to transform the default encoding - for (const auto& hook : hooks) { - if (hook.predicate(type, encoding, encoding_preference, optimize_value_seek)) { - // Verify target encoding is available for this type - if (_encoding_map.contains(std::make_pair(type, hook.target_encoding))) { - encoding = hook.target_encoding; - break; // Apply only the first matching hook - } - } - } - return encoding; - } - return UNKNOWN_ENCODING; +EncodingTypePB EncodingInfoResolver::get_index_column_encoding(FieldType type) const { + return _lookup(_index_column_encoding_map, type); } Status EncodingInfoResolver::get(FieldType data_type, EncodingTypePB encoding_type, - EncodingPreference encoding_preference, const EncodingInfo** out) { - if (encoding_type == DEFAULT_ENCODING) { - encoding_type = get_default_encoding(data_type, encoding_preference, false); - } + const EncodingInfo** out) { auto key = std::make_pair(data_type, encoding_type); auto it = _encoding_map.find(key); if (it == std::end(_encoding_map)) { @@ -446,10 +435,18 @@ EncodingInfo::EncodingInfo(TraitsClass traits) } else if (_encoding == DICT_ENCODING) { _data_page_pre_decoder = std::make_unique(); } else if (_encoding == PLAIN_ENCODING_V2) { - // Only binary types (Slice) need the predecoder for PLAIN_ENCODING_V2 - // to convert varint-encoded lengths to offset array format + // Only binary types (Slice) need the predecoder for PLAIN_ENCODING_V2 — it converts + // varint-encoded lengths to an offset-array format that downstream Slice decoders expect. + // All current (type, PLAIN_ENCODING_V2) registrations are Slice (CHAR/VARCHAR/STRING/ + // JSONB/VARIANT/HLL/BITMAP/QUANTILE_STATE/AGG_STATE per storage/types.h). The else throws + // at construction time to fail loudly if a future non-Slice registration is added. if constexpr (std::is_same_v) { _data_page_pre_decoder = std::make_unique(); + } else { + throw Exception(Status::FatalError( + "PLAIN_ENCODING_V2 is only supported for Slice (binary) types, but got " + "non-Slice type {}", + int(TraitsClass::type))); } } } @@ -458,34 +455,66 @@ EncodingInfo::EncodingInfo(TraitsClass traits) static EncodingInfoResolver s_encoding_info_resolver; #endif -Status EncodingInfo::get(FieldType type, EncodingTypePB encoding_type, - EncodingPreference encoding_preference, const EncodingInfo** out) { +Status EncodingInfo::get(FieldType type, EncodingTypePB encoding_type, const EncodingInfo** out) { #ifdef BE_TEST - return s_encoding_info_resolver.get(type, encoding_type, encoding_preference, out); + return s_encoding_info_resolver.get(type, encoding_type, out); #else auto* resolver = ExecEnv::GetInstance()->get_encoding_info_resolver(); if (resolver == nullptr) { return Status::InternalError("EncodingInfoResolver not initialized"); } - return resolver->get(type, encoding_type, encoding_preference, out); + return resolver->get(type, encoding_type, out); #endif } -EncodingTypePB EncodingInfo::get_default_encoding(FieldType type, - EncodingPreference encoding_preference, - bool optimize_value_seek) { +EncodingTypePB EncodingInfo::get_v2_default_encoding(FieldType type) { #ifdef BE_TEST - return s_encoding_info_resolver.get_default_encoding(type, encoding_preference, - optimize_value_seek); + return s_encoding_info_resolver.get_v2_default_encoding(type); #else auto* resolver = ExecEnv::GetInstance()->get_encoding_info_resolver(); if (resolver == nullptr) { return UNKNOWN_ENCODING; } - return resolver->get_default_encoding(type, encoding_preference, optimize_value_seek); + return resolver->get_v2_default_encoding(type); #endif } +EncodingTypePB EncodingInfo::get_v3_default_encoding(FieldType type) { +#ifdef BE_TEST + return s_encoding_info_resolver.get_v3_default_encoding(type); +#else + auto* resolver = ExecEnv::GetInstance()->get_encoding_info_resolver(); + if (resolver == nullptr) { + return UNKNOWN_ENCODING; + } + return resolver->get_v3_default_encoding(type); +#endif +} + +EncodingTypePB EncodingInfo::get_index_column_encoding(FieldType type) { +#ifdef BE_TEST + return s_encoding_info_resolver.get_index_column_encoding(type); +#else + auto* resolver = ExecEnv::GetInstance()->get_encoding_info_resolver(); + if (resolver == nullptr) { + return UNKNOWN_ENCODING; + } + return resolver->get_index_column_encoding(type); +#endif +} + +EncodingTypePB EncodingInfo::resolve_default_encoding(TabletStorageFormatPB storage_format, + const TabletColumn& column) { + const bool is_v3 = (storage_format == TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V3); + + // Row store data is already serialized as a single blob. Keep it on plain pages to + // avoid introducing dictionary pages for the hidden row store column. + if (column.is_row_store_column()) { + return is_v3 ? PLAIN_ENCODING_V2 : PLAIN_ENCODING; + } + return is_v3 ? get_v3_default_encoding(column.type()) : get_v2_default_encoding(column.type()); +} + Status EncodingInfo::create_page_builder(const PageBuilderOptions& opts, std::unique_ptr& builder) const { PageBuilder* raw_builder = nullptr; diff --git a/be/src/storage/segment/encoding_info.h b/be/src/storage/segment/encoding_info.h index 3ecf817a42ca62..de670da0964237 100644 --- a/be/src/storage/segment/encoding_info.h +++ b/be/src/storage/segment/encoding_info.h @@ -17,6 +17,7 @@ #pragma once +#include #include #include @@ -31,6 +32,7 @@ namespace doris { enum class FieldType; +class TabletColumn; namespace segment_v2 { @@ -51,15 +53,22 @@ class DataPagePreDecoder { class EncodingInfo { public: - // Get EncodingInfo for TypeInfo and EncodingTypePB - static Status get(FieldType type, EncodingTypePB encoding_type, - EncodingPreference encoding_preference, const EncodingInfo** encoding); - - // optimize_value_search: whether the encoding scheme should optimize for ordered data - // and support fast value seek operation - static EncodingTypePB get_default_encoding(FieldType type, - EncodingPreference encoding_preference, - bool optimize_value_seek); + // Look up the EncodingInfo for an already-resolved (type, encoding) pair. + // Read paths use this directly; write paths first resolve a default via the + // get_*_default_encoding helpers below. + static Status get(FieldType type, EncodingTypePB encoding_type, const EncodingInfo** encoding); + + // Default encoding for IndexedColumn writers (PK index, variant ext meta key writer) + // that need fast value-seek via BinaryPrefixPage. + static EncodingTypePB get_index_column_encoding(FieldType type); + + // Resolve the default encoding for one column when populating a fresh ColumnMetaPB. + // All write paths (top-level segment writer, aux child writers for null bitmap / + // array length / map length, struct subcolumns, variant subcolumns) should route the + // encoding decision through this helper. Centralizing it here means future + // per-storage-format or per-column encoding policy lives in one place. + static EncodingTypePB resolve_default_encoding(TabletStorageFormatPB storage_format, + const TabletColumn& column); Status create_page_builder(const PageBuilderOptions& opts, PageBuilder** builder) const { return _create_builder_func(opts, builder); @@ -79,6 +88,14 @@ class EncodingInfo { private: friend class EncodingInfoResolver; + friend class EncodingInfoTest; + friend class ColumnReaderCacheTest; + + // Per-storage-format defaults are an internal lookup table. Production write paths + // go through resolve_default_encoding(); other callers (e.g., zone map index) hardcode + // the encoding they want. Tests use the friend declarations above to read these tables. + static EncodingTypePB get_v2_default_encoding(FieldType type); + static EncodingTypePB get_v3_default_encoding(FieldType type); template explicit EncodingInfo(TypeEncodingTraits traits); @@ -107,21 +124,43 @@ class EncodingInfoResolver { EncodingInfoResolver(); ~EncodingInfoResolver(); - EncodingTypePB get_default_encoding(FieldType type, EncodingPreference encoding_preference, - bool optimize_value_seek) const; + EncodingTypePB get_v2_default_encoding(FieldType type) const; + EncodingTypePB get_v3_default_encoding(FieldType type) const; + EncodingTypePB get_index_column_encoding(FieldType type) const; - Status get(FieldType data_type, EncodingTypePB encoding_type, - EncodingPreference encoding_preference, const EncodingInfo** out); + Status get(FieldType data_type, EncodingTypePB encoding_type, const EncodingInfo** out); private: - // Not thread-safe - template - void _add_map(); - - std::unordered_map _default_encoding_type_map; + // Registration helpers used by the constructor. Not thread-safe. + // + // _register_supported_encoding: declare that this (type, encoding) is a supported combination, + // inserting one EncodingInfo* into _encoding_map. CHECK-fails on + // duplicate registration of the same key. + // _set_v2_default: mark this (type, encoding) as the default for the V2 (V1/V2) + // write path. Only writes _v2_default_map; the caller must have + // already _register_supported_encoding'd the same key, otherwise + // EncodingInfo::get will return InternalError when first asked + // about this combination. + // _set_v3_default: same, for the V3 write path. + // _set_index_column_encoding: same, for IndexedColumn value-seek writers (PK index). + template + void _register_supported_encoding(); + template + void _set_v2_default(); + template + void _set_v3_default(); + template + void _set_index_column_encoding(); + + static EncodingTypePB _lookup( + const std::unordered_map& m, FieldType t) { + auto it = m.find(t); + return it != m.end() ? it->second : UNKNOWN_ENCODING; + } - // default encoding for each type which optimizes value seek - std::unordered_map _value_seek_encoding_map; + std::unordered_map _v2_default_map; + std::unordered_map _v3_default_map; + std::unordered_map _index_column_encoding_map; std::unordered_map, EncodingInfo*, EncodingMapHash> _encoding_map; @@ -138,23 +177,39 @@ struct EncodingTraits : TypeEncodingTraits -void EncodingInfoResolver::_add_map() { - EncodingTraits traits; - std::unique_ptr encoding(new EncodingInfo(traits)); - if (_default_encoding_type_map.find(type) == std::end(_default_encoding_type_map)) { - _default_encoding_type_map[type] = encoding_type; - } - if (optimize_value_seek && - _value_seek_encoding_map.find(type) == _value_seek_encoding_map.end()) { - _value_seek_encoding_map[type] = encoding_type; - } - auto key = std::make_pair(type, encoding_type); - auto it = _encoding_map.find(key); - if (it != _encoding_map.end()) { - return; - } - _encoding_map.emplace(key, encoding.release()); +template +void EncodingInfoResolver::_register_supported_encoding() { + auto key = std::make_pair(type, encoding); + CHECK(_encoding_map.find(key) == _encoding_map.end()) + << "duplicate _register_supported_encoding for (type=" << int(type) + << ", encoding=" << encoding << ")"; + EncodingTraits traits; + _encoding_map.emplace(key, new EncodingInfo(traits)); +} + +// The _set_*_default helpers only write to the corresponding default map. The caller must +// _register_supported_encoding the same (type, encoding) separately; if not, EncodingInfo::get +// will return InternalError when that combination is first looked up. + +template +void EncodingInfoResolver::_set_v2_default() { + DCHECK(_v2_default_map.find(type) == _v2_default_map.end()) + << "duplicate v2 default for type " << int(type); + _v2_default_map[type] = encoding; +} + +template +void EncodingInfoResolver::_set_v3_default() { + DCHECK(_v3_default_map.find(type) == _v3_default_map.end()) + << "duplicate v3 default for type " << int(type); + _v3_default_map[type] = encoding; +} + +template +void EncodingInfoResolver::_set_index_column_encoding() { + DCHECK(_index_column_encoding_map.find(type) == _index_column_encoding_map.end()) + << "duplicate index_column_encoding for type " << int(type); + _index_column_encoding_map[type] = encoding; } } // namespace segment_v2 diff --git a/be/src/storage/segment/options.h b/be/src/storage/segment/options.h index 72509cb4602eaa..3941be17716bd3 100644 --- a/be/src/storage/segment/options.h +++ b/be/src/storage/segment/options.h @@ -29,12 +29,6 @@ static constexpr size_t STORAGE_DICT_PAGE_SIZE_DEFAULT_VALUE = 256 * 1024l; constexpr long ROW_STORE_PAGE_SIZE_DEFAULT_VALUE = 16384; // default row store page size: 16KB -struct EncodingPreference { - bool integer_type_default_use_plain_encoding {false}; - BinaryPlainEncodingTypePB binary_plain_encoding_default_impl { - BinaryPlainEncodingTypePB::BINARY_PLAIN_ENCODING_V1}; -}; - struct PageBuilderOptions { size_t data_page_size = STORAGE_PAGE_SIZE_DEFAULT_VALUE; @@ -44,7 +38,10 @@ struct PageBuilderOptions { bool is_dict_page = false; // page used for saving dictionary - EncodingPreference encoding_preference {}; + // BinaryPlain variant used by BinaryDictPageBuilder for its dict word page and + // dict-overflow fallback. Consumed only by BinaryDictPageBuilder. + BinaryPlainEncodingTypePB dict_binary_plain_encoding = + BinaryPlainEncodingTypePB::BINARY_PLAIN_ENCODING_V1; }; struct PageDecoderOptions { diff --git a/be/src/storage/segment/page_io.cpp b/be/src/storage/segment/page_io.cpp index f8dec23998ac35..72d33105e52823 100644 --- a/be/src/storage/segment/page_io.cpp +++ b/be/src/storage/segment/page_io.cpp @@ -232,11 +232,11 @@ Status PageIO::read_and_decompress_page_(const PageReadOptions& opts, PageHandle if (opts.pre_decode) { const auto* encoding_info = opts.encoding_info; - if (opts.is_dict_page) { - // for dict page, we need to use encoding_info based on footer->dict_page_footer().encoding() - // to get its pre_decoder + if (footer->type() == DICTIONARY_PAGE) { + // dict page uses its own encoding from footer->dict_page_footer().encoding() + // to look up the pre_decoder RETURN_IF_ERROR(EncodingInfo::get(FieldType::OLAP_FIELD_TYPE_VARCHAR, - footer->dict_page_footer().encoding(), {}, + footer->dict_page_footer().encoding(), &encoding_info)); } if (encoding_info) { diff --git a/be/src/storage/segment/page_io.h b/be/src/storage/segment/page_io.h index 2a1b4539d8d73c..4da3f53530d88e 100644 --- a/be/src/storage/segment/page_io.h +++ b/be/src/storage/segment/page_io.h @@ -72,10 +72,6 @@ struct PageReadOptions { const io::IOContext io_ctx; - // for dict page, we need to use encoding_info based on footer->dict_page_footer().encoding() - // to get its pre_decoder - bool is_dict_page {false}; - void sanity_check() const { CHECK_NOTNULL(file_reader); CHECK_NOTNULL(stats); @@ -93,7 +89,6 @@ struct PageReadOptions { type = old.type; encoding_info = old.encoding_info; pre_decode = old.pre_decode; - is_dict_page = old.is_dict_page; } }; diff --git a/be/src/storage/segment/segment_writer.cpp b/be/src/storage/segment/segment_writer.cpp index 82c0894fa4a5bb..68f0ec8277f586 100644 --- a/be/src/storage/segment/segment_writer.cpp +++ b/be/src/storage/segment/segment_writer.cpp @@ -62,6 +62,7 @@ #include "storage/rowset/rowset_writer_context.h" // RowsetWriterContext #include "storage/rowset/segment_creator.h" #include "storage/segment/column_writer.h" // ColumnWriter +#include "storage/segment/encoding_info.h" #include "storage/segment/external_col_meta_util.h" #include "storage/segment/page_io.h" #include "storage/segment/page_pointer.h" @@ -146,11 +147,11 @@ SegmentWriter::~SegmentWriter() { } void SegmentWriter::init_column_meta(ColumnMetaPB* meta, uint32_t column_id, - const TabletColumn& column, TabletSchemaSPtr tablet_schema) { + const TabletColumn& column, const ColumnWriterOptions& opts) { meta->set_column_id(column_id); meta->set_type(int(column.type())); meta->set_length(column.length()); - meta->set_encoding(DEFAULT_ENCODING); + meta->set_encoding(EncodingInfo::resolve_default_encoding(opts.storage_format, column)); meta->set_compression(_opts.compression_type); meta->set_is_nullable(column.is_nullable()); meta->set_default_value(column.default_value()); @@ -162,8 +163,7 @@ void SegmentWriter::init_column_meta(ColumnMetaPB* meta, uint32_t column_id, } meta->set_unique_id(column.unique_id()); for (uint32_t i = 0; i < column.get_subtype_count(); ++i) { - init_column_meta(meta->add_children_columns(), column_id, column.get_sub_column(i), - tablet_schema); + init_column_meta(meta->add_children_columns(), column_id, column.get_sub_column(i), opts); } meta->set_result_is_nullable(column.get_result_is_nullable()); meta->set_function_name(column.get_aggregation_name()); @@ -187,8 +187,9 @@ Status SegmentWriter::_create_column_writer(uint32_t cid, const TabletColumn& co const TabletSchemaSPtr& schema) { ColumnWriterOptions opts; opts.meta = _footer.add_columns(); + opts.storage_format = schema->storage_format(); - init_column_meta(opts.meta, cid, column, schema); + init_column_meta(opts.meta, cid, column, opts); // now we create zone map for key columns in AGG_KEYS or all column in UNIQUE_KEYS or DUP_KEYS // except for columns whose type don't support zone map. @@ -291,16 +292,11 @@ Status SegmentWriter::_create_column_writer(uint32_t cid, const TabletColumn& co } }) if (column.is_row_store_column()) { - // smaller page size for row store column + // smaller page size for row store column; encoding is already set to PLAIN / + // PLAIN_V2 by init_column_meta via resolve_default_encoding(). auto page_size = _tablet_schema->row_store_page_size(); opts.data_page_size = (page_size > 0) ? page_size : segment_v2::ROW_STORE_PAGE_SIZE_DEFAULT_VALUE; - // Row store data is already serialized as a single blob. Keep it on plain pages - // to avoid introducing dictionary pages for the hidden row store column. - opts.meta->set_encoding(_tablet_schema->binary_plain_encoding_default_impl() == - BinaryPlainEncodingTypePB::BINARY_PLAIN_ENCODING_V2 - ? PLAIN_ENCODING_V2 - : PLAIN_ENCODING); } opts.rowset_ctx = _opts.rowset_ctx; @@ -310,10 +306,6 @@ Status SegmentWriter::_create_column_writer(uint32_t cid, const TabletColumn& co if (_opts.rowset_ctx != nullptr) { opts.input_rs_readers = _opts.rowset_ctx->input_rs_readers; } - opts.encoding_preference = {.integer_type_default_use_plain_encoding = - _tablet_schema->integer_type_default_use_plain_encoding(), - .binary_plain_encoding_default_impl = - _tablet_schema->binary_plain_encoding_default_impl()}; std::unique_ptr writer; RETURN_IF_ERROR(ColumnWriter::create(opts, &column, _file_writer, &writer)); @@ -1127,7 +1119,7 @@ Status SegmentWriter::_write_primary_key_index() { Status SegmentWriter::_write_footer() { _footer.set_num_rows(_row_count); // Decide whether to externalize ColumnMetaPB by tablet default, and stamp footer version - if (_tablet_schema->is_external_segment_column_meta_used()) { + if (_tablet_schema->storage_format() == TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V3) { _footer.set_version(SEGMENT_FOOTER_VERSION_V3_EXT_COL_META); VLOG_DEBUG << "use external column meta"; // External ColumnMetaPB writing (optional) diff --git a/be/src/storage/segment/segment_writer.h b/be/src/storage/segment/segment_writer.h index ebda86e3c13872..d381d06afdf83d 100644 --- a/be/src/storage/segment/segment_writer.h +++ b/be/src/storage/segment/segment_writer.h @@ -125,7 +125,7 @@ class SegmentWriter { Status finalize_footer(uint64_t* segment_file_size); void init_column_meta(ColumnMetaPB* meta, uint32_t column_id, const TabletColumn& column, - TabletSchemaSPtr tablet_schema); + const ColumnWriterOptions& opts); Slice min_encoded_key(); Slice max_encoded_key(); diff --git a/be/src/storage/segment/variant/variant_column_writer_impl.cpp b/be/src/storage/segment/variant/variant_column_writer_impl.cpp index edb2ecca44e1d8..1f501adf88e67b 100644 --- a/be/src/storage/segment/variant/variant_column_writer_impl.cpp +++ b/be/src/storage/segment/variant/variant_column_writer_impl.cpp @@ -51,6 +51,7 @@ #include "storage/olap_define.h" #include "storage/rowset/rowset_writer_context.h" #include "storage/segment/column_writer.h" +#include "storage/segment/encoding_info.h" #include "storage/segment/variant/nested_group_path.h" #include "storage/segment/variant/nested_group_routing_plan.h" #include "storage/segment/variant/variant_writer_helpers.h" @@ -63,13 +64,13 @@ namespace doris::segment_v2 { #include "common/compile_check_begin.h" void _init_column_meta(ColumnMetaPB* meta, uint32_t column_id, const TabletColumn& column, - CompressionTypePB compression_type) { + const ColumnWriterOptions& opts) { meta->Clear(); meta->set_column_id(column_id); meta->set_type(int(column.type())); meta->set_length(column.length()); - meta->set_encoding(DEFAULT_ENCODING); - meta->set_compression(compression_type); + meta->set_encoding(EncodingInfo::resolve_default_encoding(opts.storage_format, column)); + meta->set_compression(opts.compression_type); meta->set_is_nullable(column.is_nullable()); meta->set_default_value(column.default_value()); meta->set_precision(column.precision()); @@ -80,8 +81,7 @@ void _init_column_meta(ColumnMetaPB* meta, uint32_t column_id, const TabletColum } meta->set_unique_id(column.unique_id()); for (uint32_t i = 0; i < column.get_subtype_count(); ++i) { - _init_column_meta(meta->add_children_columns(), column_id, column.get_sub_column(i), - compression_type); + _init_column_meta(meta->add_children_columns(), column_id, column.get_sub_column(i), opts); } if (column.is_variant_type()) { meta->set_variant_max_subcolumns_count(column.variant_max_subcolumns_count()); @@ -95,7 +95,7 @@ Status _create_column_writer(uint32_t cid, const TabletColumn& column, std::unique_ptr* writer, TabletIndexes& subcolumn_indexes, ColumnWriterOptions* opt, int64_t none_null_value_size, bool need_record_none_null_value_size) { - _init_column_meta(opt->meta, cid, column, opt->compression_type); + _init_column_meta(opt->meta, cid, column, *opt); // no need to record none null value size for typed column or nested column, since it's compaction stage // will directly pick it as sub column if (need_record_none_null_value_size) { @@ -456,6 +456,7 @@ Status prepare_materialized_subcolumn_writer( opts.compression_type = base_opts.compression_type; opts.rowset_ctx = base_opts.rowset_ctx; opts.file_writer = base_opts.file_writer; + opts.storage_format = base_opts.storage_format; std::unique_ptr writer; variant_util::inherit_column_attributes(parent_column, tablet_column); @@ -939,7 +940,7 @@ Status UnifiedSparseColumnWriter::init_single(const TabletColumn& sparse_column, SegmentFooterPB* footer) { _single_opts = base_opts; _single_opts.meta = footer->add_columns(); - _init_column_meta(_single_opts.meta, column_id, sparse_column, base_opts.compression_type); + _init_column_meta(_single_opts.meta, column_id, sparse_column, base_opts); RETURN_IF_ERROR(ColumnWriter::create_map_writer(_single_opts, &sparse_column, base_opts.file_writer, &_single_writer)); RETURN_IF_ERROR(_single_writer->init()); @@ -959,7 +960,7 @@ Status UnifiedSparseColumnWriter::init_buckets(int bucket_num, const TabletColum TabletColumn bucket_col = variant_util::create_sparse_shard_column(parent_column, b); _bucket_opts[b] = base_opts; _bucket_opts[b].meta = footer->add_columns(); - _init_column_meta(_bucket_opts[b].meta, column_id, bucket_col, base_opts.compression_type); + _init_column_meta(_bucket_opts[b].meta, column_id, bucket_col, base_opts); RETURN_IF_ERROR(ColumnWriter::create_map_writer( _bucket_opts[b], &bucket_col, base_opts.file_writer, &_bucket_writers[b])); RETURN_IF_ERROR(_bucket_writers[b]->init()); @@ -1184,8 +1185,7 @@ Status VariantDocWriter::init(const TabletColumn* parent_column, int bucket_num, variant_util::create_doc_value_column(*parent_column, b); _doc_value_column_opts[b] = opts; _doc_value_column_opts[b].meta = footer->add_columns(); - _init_column_meta(_doc_value_column_opts[b].meta, column_id, bucket_column, - opts.compression_type); + _init_column_meta(_doc_value_column_opts[b].meta, column_id, bucket_column, opts); RETURN_IF_ERROR(ColumnWriter::create_map_writer(_doc_value_column_opts[b], &bucket_column, opts.file_writer, &_doc_value_column_writers[b])); @@ -1483,7 +1483,7 @@ Status prepare_subcolumn_writer_target( opts.compression_type = base_opts.compression_type; opts.rowset_ctx = base_opts.rowset_ctx; opts.file_writer = base_opts.file_writer; - opts.encoding_preference = base_opts.encoding_preference; + opts.storage_format = base_opts.storage_format; variant_util::inherit_column_attributes(parent_column, tablet_column); bool need_record_none_null_value_size = @@ -2222,7 +2222,7 @@ Status VariantDocCompactWriter::_write_doc_value_column(const TabletColumn& pare int bucket_value = std::stoi(doc_value_column_path.substr(pos + 1)); TabletColumn doc_value_column = variant_util::create_doc_value_column(parent_column, bucket_value); - _init_column_meta(_opts.meta, column_id, doc_value_column, _opts.compression_type); + _init_column_meta(_opts.meta, column_id, doc_value_column, _opts); RETURN_IF_ERROR(ColumnWriter::create_map_writer(_opts, &doc_value_column, _opts.file_writer, &_doc_value_column_writer)); RETURN_IF_ERROR(_doc_value_column_writer->init()); diff --git a/be/src/storage/segment/variant/variant_column_writer_impl.h b/be/src/storage/segment/variant/variant_column_writer_impl.h index ebcc683571039d..e70bda00f83b56 100644 --- a/be/src/storage/segment/variant/variant_column_writer_impl.h +++ b/be/src/storage/segment/variant/variant_column_writer_impl.h @@ -296,7 +296,7 @@ class VariantDocCompactWriter : public ColumnWriter { }; void _init_column_meta(ColumnMetaPB* meta, uint32_t column_id, const TabletColumn& column, - CompressionTypePB compression_type); + const ColumnWriterOptions& opts); #include "common/compile_check_end.h" diff --git a/be/src/storage/segment/vertical_segment_writer.cpp b/be/src/storage/segment/vertical_segment_writer.cpp index 4de6a79d232888..e432e7ae507fd6 100644 --- a/be/src/storage/segment/vertical_segment_writer.cpp +++ b/be/src/storage/segment/vertical_segment_writer.cpp @@ -68,6 +68,7 @@ #include "storage/rowset/rowset_writer_context.h" // RowsetWriterContext #include "storage/rowset/segment_creator.h" #include "storage/segment/column_writer.h" // ColumnWriter +#include "storage/segment/encoding_info.h" #include "storage/segment/external_col_meta_util.h" #include "storage/segment/page_io.h" #include "storage/segment/page_pointer.h" @@ -155,11 +156,12 @@ VerticalSegmentWriter::~VerticalSegmentWriter() { } void VerticalSegmentWriter::_init_column_meta(ColumnMetaPB* meta, uint32_t column_id, - const TabletColumn& column) { + const TabletColumn& column, + const ColumnWriterOptions& opts) { meta->set_column_id(column_id); meta->set_type(int(column.type())); meta->set_length(cast_set(column.length())); - meta->set_encoding(DEFAULT_ENCODING); + meta->set_encoding(EncodingInfo::resolve_default_encoding(opts.storage_format, column)); meta->set_compression(_opts.compression_type); meta->set_is_nullable(column.is_nullable()); meta->set_default_value(column.default_value()); @@ -171,7 +173,7 @@ void VerticalSegmentWriter::_init_column_meta(ColumnMetaPB* meta, uint32_t colum } meta->set_unique_id(column.unique_id()); for (uint32_t i = 0; i < column.get_subtype_count(); ++i) { - _init_column_meta(meta->add_children_columns(), column_id, column.get_sub_column(i)); + _init_column_meta(meta->add_children_columns(), column_id, column.get_sub_column(i), opts); } if (column.is_variant_type()) { meta->set_variant_max_subcolumns_count(column.variant_max_subcolumns_count()); @@ -186,8 +188,9 @@ Status VerticalSegmentWriter::_create_column_writer(uint32_t cid, const TabletCo const TabletSchemaSPtr& tablet_schema) { ColumnWriterOptions opts; opts.meta = _footer.add_columns(); + opts.storage_format = tablet_schema->storage_format(); - _init_column_meta(opts.meta, cid, column); + _init_column_meta(opts.meta, cid, column, opts); // now we create zone map for key columns in AGG_KEYS or all column in UNIQUE_KEYS or DUP_KEYS // except for columns whose type don't support zone map. @@ -293,16 +296,11 @@ Status VerticalSegmentWriter::_create_column_writer(uint32_t cid, const TabletCo } }) if (column.is_row_store_column()) { - // smaller page size for row store column + // smaller page size for row store column; encoding is already set to PLAIN / + // PLAIN_V2 by _init_column_meta via resolve_default_encoding(). auto page_size = _tablet_schema->row_store_page_size(); opts.data_page_size = (page_size > 0) ? page_size : segment_v2::ROW_STORE_PAGE_SIZE_DEFAULT_VALUE; - // Row store data is already serialized as a single blob. Keep it on plain pages - // to avoid introducing dictionary pages for the hidden row store column. - opts.meta->set_encoding(_tablet_schema->binary_plain_encoding_default_impl() == - BinaryPlainEncodingTypePB::BINARY_PLAIN_ENCODING_V2 - ? PLAIN_ENCODING_V2 - : PLAIN_ENCODING); } opts.rowset_ctx = _opts.rowset_ctx; @@ -311,10 +309,6 @@ Status VerticalSegmentWriter::_create_column_writer(uint32_t cid, const TabletCo opts.footer = &_footer; opts.input_rs_readers = _opts.rowset_ctx->input_rs_readers; - opts.encoding_preference = {.integer_type_default_use_plain_encoding = - _tablet_schema->integer_type_default_use_plain_encoding(), - .binary_plain_encoding_default_impl = - _tablet_schema->binary_plain_encoding_default_impl()}; std::unique_ptr writer; RETURN_IF_ERROR(ColumnWriter::create(opts, &column, _file_writer, &writer)); RETURN_IF_ERROR(writer->init()); @@ -1448,7 +1442,7 @@ Status VerticalSegmentWriter::_write_footer() { // Decide whether to externalize ColumnMetaPB by tablet default, and stamp footer version - if (_tablet_schema->is_external_segment_column_meta_used()) { + if (_tablet_schema->storage_format() == TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V3) { _footer.set_version(SEGMENT_FOOTER_VERSION_V3_EXT_COL_META); VLOG_DEBUG << "use external column meta"; // External ColumnMetaPB writing (optional) diff --git a/be/src/storage/segment/vertical_segment_writer.h b/be/src/storage/segment/vertical_segment_writer.h index 8aa8a24a7c44d3..e6ee4933dd6d19 100644 --- a/be/src/storage/segment/vertical_segment_writer.h +++ b/be/src/storage/segment/vertical_segment_writer.h @@ -129,7 +129,8 @@ class VerticalSegmentWriter { } private: - void _init_column_meta(ColumnMetaPB* meta, uint32_t column_id, const TabletColumn& column); + void _init_column_meta(ColumnMetaPB* meta, uint32_t column_id, const TabletColumn& column, + const ColumnWriterOptions& opts); Status _create_column_writer(uint32_t cid, const TabletColumn& column, const TabletSchemaSPtr& schema); uint64_t _estimated_remaining_size(); diff --git a/be/src/storage/tablet/tablet_meta.cpp b/be/src/storage/tablet/tablet_meta.cpp index 2058674c9e7804..5292e9fc0ad5ec 100644 --- a/be/src/storage/tablet/tablet_meta.cpp +++ b/be/src/storage/tablet/tablet_meta.cpp @@ -407,15 +407,8 @@ TabletMeta::TabletMeta(int64_t table_id, int64_t partition_id, int64_t tablet_id case TStorageFormat::V1: break; case TStorageFormat::V3: - schema->set_is_external_segment_column_meta_used(true); - _schema->set_external_segment_meta_used_default(true); - - schema->set_integer_type_default_use_plain_encoding(true); - _schema->set_integer_type_default_use_plain_encoding(true); - schema->set_binary_plain_encoding_default_impl( - BinaryPlainEncodingTypePB::BINARY_PLAIN_ENCODING_V2); - _schema->set_binary_plain_encoding_default_impl( - BinaryPlainEncodingTypePB::BINARY_PLAIN_ENCODING_V2); + schema->set_storage_format(TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V3); + _schema->set_storage_format(TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V3); break; default: break; diff --git a/be/src/storage/tablet/tablet_schema.cpp b/be/src/storage/tablet/tablet_schema.cpp index 5984458d563526..bce66484aa2af3 100644 --- a/be/src/storage/tablet/tablet_schema.cpp +++ b/be/src/storage/tablet/tablet_schema.cpp @@ -1200,16 +1200,16 @@ void TabletSchema::init_from_pb(const TabletSchemaPB& schema, bool ignore_extrac _row_store_column_unique_ids.assign(schema.row_store_column_unique_ids().begin(), schema.row_store_column_unique_ids().end()); _deprecated_enable_variant_flatten_nested = schema.enable_variant_flatten_nested(); - if (schema.has_is_external_segment_column_meta_used()) { - _is_external_segment_column_meta_used = schema.is_external_segment_column_meta_used(); + if (schema.has_storage_format()) { + _storage_format = schema.storage_format(); + } else if (schema.is_external_segment_column_meta_used() || + schema.integer_type_default_use_plain_encoding() || + schema.binary_plain_encoding_default_impl() == + BinaryPlainEncodingTypePB::BINARY_PLAIN_ENCODING_V2) { + // Old PB without storage_format: any of the three legacy V3-flavor flags implies V3. + _storage_format = TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V3; } else { - _is_external_segment_column_meta_used = false; - } - if (schema.has_integer_type_default_use_plain_encoding()) { - _integer_type_default_use_plain_encoding = schema.integer_type_default_use_plain_encoding(); - } - if (schema.has_binary_plain_encoding_default_impl()) { - _binary_plain_encoding_default_impl = schema.binary_plain_encoding_default_impl(); + _storage_format = TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2; } update_metadata_size(); } @@ -1480,11 +1480,19 @@ void TabletSchema::to_schema_pb(TabletSchemaPB* tablet_schema_pb) const { tablet_schema_pb->mutable_row_store_column_unique_ids()->Assign( _row_store_column_unique_ids.begin(), _row_store_column_unique_ids.end()); tablet_schema_pb->set_enable_variant_flatten_nested(_deprecated_enable_variant_flatten_nested); - tablet_schema_pb->set_is_external_segment_column_meta_used( - _is_external_segment_column_meta_used); - tablet_schema_pb->set_integer_type_default_use_plain_encoding( - _integer_type_default_use_plain_encoding); - tablet_schema_pb->set_binary_plain_encoding_default_impl(_binary_plain_encoding_default_impl); + tablet_schema_pb->set_storage_format(_storage_format); + // Backward downgrade safety: if a new BE rewrites tablet_meta.json carrying only + // storage_format and the deployment is then rolled back to an old BE, the old BE + // does not know the new field and would default-derive V2 for a V3 tablet, causing + // it to write V2-encoded segments into a V3 tablet. Redundantly emit the three + // legacy V3-flavor flags so old BEs can recover the format via the prior "any of + // these implies V3" rule. ~3 bytes per schema PB; only paid for V3 tablets. + if (_storage_format == TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V3) { + tablet_schema_pb->set_is_external_segment_column_meta_used(true); + tablet_schema_pb->set_integer_type_default_use_plain_encoding(true); + tablet_schema_pb->set_binary_plain_encoding_default_impl( + BinaryPlainEncodingTypePB::BINARY_PLAIN_ENCODING_V2); + } } size_t TabletSchema::row_size() const { @@ -1867,12 +1875,7 @@ bool operator==(const TabletSchema& a, const TabletSchema& b) { b._deprecated_enable_variant_flatten_nested) { return false; } - if (a._is_external_segment_column_meta_used != b._is_external_segment_column_meta_used) - return false; - if (a._integer_type_default_use_plain_encoding != b._integer_type_default_use_plain_encoding) - return false; - if (a._binary_plain_encoding_default_impl != b._binary_plain_encoding_default_impl) - return false; + if (a._storage_format != b._storage_format) return false; return true; } diff --git a/be/src/storage/tablet/tablet_schema.h b/be/src/storage/tablet/tablet_schema.h index fb86c34741b535..12279f6a5443b0 100644 --- a/be/src/storage/tablet/tablet_schema.h +++ b/be/src/storage/tablet/tablet_schema.h @@ -17,6 +17,7 @@ #pragma once +#include #include #include #include @@ -724,30 +725,8 @@ class TabletSchema : public MetadataAdder { return 0; } - // Whether new segments use externalized ColumnMetaPB layout (CMO) by default - bool is_external_segment_column_meta_used() const { - return _is_external_segment_column_meta_used; - } - - void set_external_segment_meta_used_default(bool v) { - _is_external_segment_column_meta_used = v; - } - - bool integer_type_default_use_plain_encoding() const { - return _integer_type_default_use_plain_encoding; - } - - void set_integer_type_default_use_plain_encoding(bool v) { - _integer_type_default_use_plain_encoding = v; - } - - BinaryPlainEncodingTypePB binary_plain_encoding_default_impl() const { - return _binary_plain_encoding_default_impl; - } - - void set_binary_plain_encoding_default_impl(BinaryPlainEncodingTypePB impl) { - _binary_plain_encoding_default_impl = impl; - } + TabletStorageFormatPB storage_format() const { return _storage_format; } + void set_storage_format(TabletStorageFormatPB v) { _storage_format = v; } void add_pruned_columns_data_type(int32_t col_unique_id, DataTypePtr data_type) { _pruned_columns_data_type[col_unique_id] = std::move(data_type); @@ -836,11 +815,10 @@ class TabletSchema : public MetadataAdder { std::unordered_map _index_by_unique_id_with_pattern; // Default behavior for new segments: use external ColumnMeta region + CMO table if true - bool _is_external_segment_column_meta_used = false; - - bool _integer_type_default_use_plain_encoding {false}; - BinaryPlainEncodingTypePB _binary_plain_encoding_default_impl { - BinaryPlainEncodingTypePB::BINARY_PLAIN_ENCODING_V1}; + // Persisted tablet storage format. Authoritative source for "is this tablet V3?" + // decisions in the segment write paths. Old PBs without this field are upgraded in + // init_from_pb() by deriving V3 from any of the three legacy V3-flavor flags. + TabletStorageFormatPB _storage_format {TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2}; }; bool operator==(const TabletSchema& a, const TabletSchema& b); diff --git a/be/test/exec/common/schema_util_rowset_test.cpp b/be/test/exec/common/schema_util_rowset_test.cpp index cf99c9824956c5..51d545ff5cb6b0 100644 --- a/be/test/exec/common/schema_util_rowset_test.cpp +++ b/be/test/exec/common/schema_util_rowset_test.cpp @@ -228,7 +228,9 @@ TEST_F(SchemaUtilRowsetTest, check_path_stats_agg_key) { bool external_segment_meta_used_default = rand() % 2 == 0; std::cout << "external_segment_meta_used_default: " << external_segment_meta_used_default << std::endl; - tablet_schema->set_external_segment_meta_used_default(external_segment_meta_used_default); + tablet_schema->set_storage_format(external_segment_meta_used_default + ? TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V3 + : TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2); std::string absolute_dir = _curreent_dir + std::string("/ut_dir/schema_util_rows"); EXPECT_TRUE(io::global_local_filesystem()->delete_directory(absolute_dir).ok()); EXPECT_TRUE(io::global_local_filesystem()->create_directory(absolute_dir).ok()); @@ -278,7 +280,9 @@ TEST_F(SchemaUtilRowsetTest, check_path_stats_agg_delete) { bool external_segment_meta_used_default = rand() % 2 == 0; std::cout << "external_segment_meta_used_default: " << external_segment_meta_used_default << std::endl; - tablet_schema->set_external_segment_meta_used_default(external_segment_meta_used_default); + tablet_schema->set_storage_format(external_segment_meta_used_default + ? TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V3 + : TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2); std::string absolute_dir = _curreent_dir + std::string("/ut_dir/schema_util_rows1"); EXPECT_TRUE(io::global_local_filesystem()->delete_directory(absolute_dir).ok()); EXPECT_TRUE(io::global_local_filesystem()->create_directory(absolute_dir).ok()); @@ -328,7 +332,7 @@ TEST_F(SchemaUtilRowsetTest, mixed_external_segment_meta_old_new) { // 2. create tablet and data dir TabletMetaSharedPtr tablet_meta(new TabletMeta(tablet_schema)); // First write a few rowsets with external_segment_meta_used_default = false (old format) - tablet_schema->set_external_segment_meta_used_default(false); + tablet_schema->set_storage_format(TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2); std::string absolute_dir = _curreent_dir + std::string("/ut_dir/schema_util_rows_mixed"); EXPECT_TRUE(io::global_local_filesystem()->delete_directory(absolute_dir).ok()); EXPECT_TRUE(io::global_local_filesystem()->create_directory(absolute_dir).ok()); @@ -361,7 +365,7 @@ TEST_F(SchemaUtilRowsetTest, mixed_external_segment_meta_old_new) { } // 3.2 flip tablet default to enable external column meta for subsequent segments - tablet_schema->set_external_segment_meta_used_default(true); + tablet_schema->set_storage_format(TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V3); // 3.3 write a few new-format rowsets (external meta enabled) for (int i = 0; i < 3; ++i) { @@ -394,7 +398,9 @@ TEST_F(SchemaUtilRowsetTest, collect_path_stats_and_get_extended_compaction_sche bool external_segment_meta_used_default = rand() % 2 == 0; std::cout << "external_segment_meta_used_default: " << external_segment_meta_used_default << std::endl; - tablet_schema->set_external_segment_meta_used_default(external_segment_meta_used_default); + tablet_schema->set_storage_format(external_segment_meta_used_default + ? TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V3 + : TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2); tablet_meta->_tablet_id = 12345; _tablet = std::make_shared(*_engine_ref, tablet_meta, _data_dir.get()); EXPECT_TRUE(_tablet->init().ok()); @@ -601,7 +607,9 @@ TabletSchemaSPtr create_compaction_schema_common(StorageEngine* _engine_ref, bool external_segment_meta_used_default = rand() % 2 == 0; std::cout << "external_segment_meta_used_default: " << external_segment_meta_used_default << std::endl; - tablet_schema->set_external_segment_meta_used_default(external_segment_meta_used_default); + tablet_schema->set_storage_format(external_segment_meta_used_default + ? TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V3 + : TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2); EXPECT_TRUE(io::global_local_filesystem()->delete_directory(_absolute_dir).ok()); EXPECT_TRUE(io::global_local_filesystem()->create_directory(_absolute_dir).ok()); std::unique_ptr _data_dir = std::make_unique(*_engine_ref, _absolute_dir); @@ -694,7 +702,9 @@ TEST_F(SchemaUtilRowsetTest, some_test_for_subcolumn_writer) { std::cout << compaction_schema->dump_structure() << std::endl; // this is v1.key1 TabletColumn column = compaction_schema->column(2); - _init_column_meta(opts.meta, 0, column, CompressionTypePB::LZ4); + opts.compression_type = CompressionTypePB::LZ4; + opts.storage_format = TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2; + _init_column_meta(opts.meta, 0, column, opts); std::unique_ptr writer; EXPECT_TRUE( ColumnWriter::create_variant_writer(opts, &column, file_writer.get(), &writer).ok()); @@ -735,7 +745,9 @@ TEST_F(SchemaUtilRowsetTest, typed_path_to_sparse_column) { bool external_segment_meta_used_default = rand() % 2 == 0; std::cout << "external_segment_meta_used_default: " << external_segment_meta_used_default << std::endl; - tablet_schema->set_external_segment_meta_used_default(external_segment_meta_used_default); + tablet_schema->set_storage_format(external_segment_meta_used_default + ? TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V3 + : TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2); _tablet = std::make_shared(*_engine_ref, tablet_meta, _data_dir.get()); EXPECT_TRUE(_tablet->init().ok()); EXPECT_TRUE(io::global_local_filesystem()->create_directory(_tablet->tablet_path()).ok()); diff --git a/be/test/storage/segment/binary_dict_page_test.cpp b/be/test/storage/segment/binary_dict_page_test.cpp index 70f3b1ada154f3..36d455c72a5000 100644 --- a/be/test/storage/segment/binary_dict_page_test.cpp +++ b/be/test/storage/segment/binary_dict_page_test.cpp @@ -143,7 +143,7 @@ class BinaryDictPageTest : public testing::Test { PageBuilderOptions options; options.data_page_size = 256 * 1024; options.dict_page_size = 256 * 1024; - options.encoding_preference.binary_plain_encoding_default_impl = + options.dict_binary_plain_encoding = use_v2 ? BinaryPlainEncodingTypePB::BINARY_PLAIN_ENCODING_V2 : BinaryPlainEncodingTypePB::BINARY_PLAIN_ENCODING_V1; @@ -163,13 +163,14 @@ class BinaryDictPageTest : public testing::Test { << "Expected encoding type does not match when use_v2=" << use_v2; } - void test_by_small_data_size(const std::vector& slices, - EncodingPreference encoding_preference = EncodingPreference()) { + void test_by_small_data_size(const std::vector& slices, bool use_plain_v2 = false) { // Encode PageBuilderOptions options; options.data_page_size = 256 * 1024; options.dict_page_size = 256 * 1024; - options.encoding_preference = encoding_preference; + options.dict_binary_plain_encoding = + use_plain_v2 ? BinaryPlainEncodingTypePB::BINARY_PLAIN_ENCODING_V2 + : BinaryPlainEncodingTypePB::BINARY_PLAIN_ENCODING_V1; PageBuilder* builder_ptr = nullptr; Status ret0 = BinaryDictPageBuilder::create(&builder_ptr, options); @@ -285,15 +286,16 @@ class BinaryDictPageTest : public testing::Test { } } - void test_with_large_data_size(const std::vector& contents, - EncodingPreference encoding_preference = EncodingPreference()) { + void test_with_large_data_size(const std::vector& contents, bool use_plain_v2 = false) { // Encode PageBuilderOptions options; // Use smaller page sizes to ensure we trigger fallback scenario // where dictionary gets full and we switch to plain encoding options.data_page_size = 64 * 1024; // 64KB data page options.dict_page_size = 1024; // 1KB dict page to trigger fallback - options.encoding_preference = encoding_preference; + options.dict_binary_plain_encoding = + use_plain_v2 ? BinaryPlainEncodingTypePB::BINARY_PLAIN_ENCODING_V2 + : BinaryPlainEncodingTypePB::BINARY_PLAIN_ENCODING_V1; PageBuilder* builder_ptr = nullptr; Status ret0 = BinaryDictPageBuilder::create(&builder_ptr, options); @@ -603,8 +605,7 @@ TEST_F(BinaryDictPageTest, TestConfigAffectsDictionaryPageEncoding) { PageBuilderOptions options; options.data_page_size = 256 * 1024; options.dict_page_size = 256 * 1024; - options.encoding_preference.binary_plain_encoding_default_impl = - BinaryPlainEncodingTypePB::BINARY_PLAIN_ENCODING_V1; + options.dict_binary_plain_encoding = BinaryPlainEncodingTypePB::BINARY_PLAIN_ENCODING_V1; PageBuilder* builder_ptr = nullptr; Status status = BinaryDictPageBuilder::create(&builder_ptr, options); @@ -644,8 +645,7 @@ TEST_F(BinaryDictPageTest, TestConfigAffectsDictionaryPageEncoding) { PageBuilderOptions options; options.data_page_size = 256 * 1024; options.dict_page_size = 256 * 1024; - options.encoding_preference.binary_plain_encoding_default_impl = - BinaryPlainEncodingTypePB::BINARY_PLAIN_ENCODING_V2; + options.dict_binary_plain_encoding = BinaryPlainEncodingTypePB::BINARY_PLAIN_ENCODING_V2; PageBuilder* builder_ptr = nullptr; Status status = BinaryDictPageBuilder::create(&builder_ptr, options); @@ -707,8 +707,7 @@ TEST_F(BinaryDictPageTest, TestConfigAffectsFallbackEncoding) { PageBuilderOptions options; options.data_page_size = 256 * 1024; options.dict_page_size = 128; // Small dict size to force fallback - options.encoding_preference.binary_plain_encoding_default_impl = - BinaryPlainEncodingTypePB::BINARY_PLAIN_ENCODING_V1; + options.dict_binary_plain_encoding = BinaryPlainEncodingTypePB::BINARY_PLAIN_ENCODING_V1; PageBuilder* builder_ptr = nullptr; Status status = BinaryDictPageBuilder::create(&builder_ptr, options); @@ -739,13 +738,9 @@ TEST_F(BinaryDictPageTest, TestConfigAffectsFallbackEncoding) { status = page_builder->reset(); EXPECT_TRUE(status.ok()); - // Access private member _fallback_binary_encoding_type to verify - EXPECT_EQ(PLAIN_ENCODING, page_builder->_fallback_binary_encoding_type) - << "Fallback encoding should be PLAIN_ENCODING with V1 preference"; - - // Also check the dict word page encoding type - EXPECT_EQ(PLAIN_ENCODING, page_builder->_dict_word_page_encoding_type) - << "Dict word page encoding should be PLAIN_ENCODING with V1 preference"; + // Verify the binary-plain flavor (used for both dict word page and fallback data page). + EXPECT_EQ(PLAIN_ENCODING, page_builder->_binary_plain_encoding_type) + << "Binary plain encoding should be PLAIN_ENCODING with V1 preference"; // Check the actual encoding type used (should have fallen back) EXPECT_EQ(PLAIN_ENCODING, page_builder->_encoding_type) @@ -757,8 +752,7 @@ TEST_F(BinaryDictPageTest, TestConfigAffectsFallbackEncoding) { PageBuilderOptions options; options.data_page_size = 256 * 1024; options.dict_page_size = 128; // Small dict size to force fallback - options.encoding_preference.binary_plain_encoding_default_impl = - BinaryPlainEncodingTypePB::BINARY_PLAIN_ENCODING_V2; + options.dict_binary_plain_encoding = BinaryPlainEncodingTypePB::BINARY_PLAIN_ENCODING_V2; PageBuilder* builder_ptr = nullptr; Status status = BinaryDictPageBuilder::create(&builder_ptr, options); @@ -789,13 +783,9 @@ TEST_F(BinaryDictPageTest, TestConfigAffectsFallbackEncoding) { status = page_builder->reset(); EXPECT_TRUE(status.ok()); - // Access private member _fallback_binary_encoding_type to verify - EXPECT_EQ(PLAIN_ENCODING_V2, page_builder->_fallback_binary_encoding_type) - << "Fallback encoding should be PLAIN_ENCODING_V2 with V2 preference"; - - // Also check the dict word page encoding type - EXPECT_EQ(PLAIN_ENCODING_V2, page_builder->_dict_word_page_encoding_type) - << "Dict word page encoding should be PLAIN_ENCODING_V2 with V2 preference"; + // Verify the binary-plain flavor (used for both dict word page and fallback data page). + EXPECT_EQ(PLAIN_ENCODING_V2, page_builder->_binary_plain_encoding_type) + << "Binary plain encoding should be PLAIN_ENCODING_V2 with V2 preference"; // Check the actual encoding type used (should have fallen back) EXPECT_EQ(PLAIN_ENCODING_V2, page_builder->_encoding_type) @@ -824,10 +814,7 @@ TEST_F(BinaryDictPageTest, TestSmallDataWithConfigFalse) { slices.emplace_back(str); } - EncodingPreference encoding_preference; - encoding_preference.binary_plain_encoding_default_impl = - BinaryPlainEncodingTypePB::BINARY_PLAIN_ENCODING_V1; - test_by_small_data_size(slices, encoding_preference); + test_by_small_data_size(slices, /*use_plain_v2=*/false); } TEST_F(BinaryDictPageTest, TestSmallDataWithConfigTrue) { @@ -837,10 +824,7 @@ TEST_F(BinaryDictPageTest, TestSmallDataWithConfigTrue) { slices.emplace_back(str); } - EncodingPreference encoding_preference; - encoding_preference.binary_plain_encoding_default_impl = - BinaryPlainEncodingTypePB::BINARY_PLAIN_ENCODING_V2; - test_by_small_data_size(slices, encoding_preference); + test_by_small_data_size(slices, /*use_plain_v2=*/true); } TEST_F(BinaryDictPageTest, TestLargeDataWithConfigFalse) { @@ -860,11 +844,8 @@ TEST_F(BinaryDictPageTest, TestLargeDataWithConfigFalse) { slices.push_back(str); } - EncodingPreference encoding_preference; - encoding_preference.binary_plain_encoding_default_impl = - BinaryPlainEncodingTypePB::BINARY_PLAIN_ENCODING_V1; LOG(INFO) << "Testing large data with V1 preference, entry count: " << slices.size(); - test_with_large_data_size(slices, encoding_preference); + test_with_large_data_size(slices, /*use_plain_v2=*/false); } TEST_F(BinaryDictPageTest, TestLargeDataWithConfigTrue) { @@ -884,11 +865,8 @@ TEST_F(BinaryDictPageTest, TestLargeDataWithConfigTrue) { slices.push_back(str); } - EncodingPreference encoding_preference; - encoding_preference.binary_plain_encoding_default_impl = - BinaryPlainEncodingTypePB::BINARY_PLAIN_ENCODING_V2; LOG(INFO) << "Testing large data with V2 preference, entry count: " << slices.size(); - test_with_large_data_size(slices, encoding_preference); + test_with_large_data_size(slices, /*use_plain_v2=*/true); } } // namespace segment_v2 diff --git a/be/test/storage/segment/column_meta_accessor_test.cpp b/be/test/storage/segment/column_meta_accessor_test.cpp index bdcdd39d8c85b9..48c2581b841bbc 100644 --- a/be/test/storage/segment/column_meta_accessor_test.cpp +++ b/be/test/storage/segment/column_meta_accessor_test.cpp @@ -27,6 +27,7 @@ #include "common/consts.h" #include "core/field.h" #include "io/fs/local_file_system.h" +#include "storage/segment/external_col_meta_util.h" #include "storage/segment/segment.h" #include "storage/segment/segment_writer.h" #include "util/coding.h" @@ -639,7 +640,7 @@ TEST(ColumnMetaAccessorTest, FooterSizeWithManyColumnsExternalVsInline) { TabletSchemaSPtr external_schema = create_schema(columns, UNIQUE_KEYS); // Enable external ColumnMetaPB for the second schema so that SegmentWriter // produces a V3 footer with externalized column meta region. - external_schema->set_external_segment_meta_used_default(true); + external_schema->set_storage_format(TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V3); // 2. Common SegmentWriter options and row generator. SegmentWriterOptions opts; @@ -712,8 +713,7 @@ TEST(ColumnMetaAccessorTest, RowStoreColumnDoesNotUseDictEncoding) { columns.emplace_back(create_row_store_test_column(kRowStoreUid)); auto tablet_schema = create_schema(columns, UNIQUE_KEYS); - tablet_schema->set_binary_plain_encoding_default_impl( - BinaryPlainEncodingTypePB::BINARY_PLAIN_ENCODING_V2); + tablet_schema->set_storage_format(TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V3); SegmentWriterOptions opts; opts.enable_unique_key_merge_on_write = false; @@ -739,9 +739,14 @@ TEST(ColumnMetaAccessorTest, RowStoreColumnDoesNotUseDictEncoding) { SegmentFooterPB footer; ASSERT_TRUE(read_footer_from_file(reader, &footer).ok()); - ASSERT_EQ(2, footer.columns_size()); - - const auto& row_store_meta = footer.columns(1); + // V3 schemas externalize column meta -- read row_store col (col_id=1) from the + // external region instead of the inline footer.columns(). + ExternalColMetaUtil::ExternalMetaPointers ptrs; + ASSERT_TRUE(ExternalColMetaUtil::parse_external_meta_pointers(footer, &ptrs).ok()); + ColumnMetaPB row_store_meta; + ASSERT_TRUE( + ExternalColMetaUtil::read_col_meta(reader, footer, ptrs, /*col_id=*/1, &row_store_meta) + .ok()); EXPECT_EQ(kRowStoreUid, row_store_meta.unique_id()); EXPECT_EQ(static_cast(FieldType::OLAP_FIELD_TYPE_STRING), row_store_meta.type()); EXPECT_EQ(PLAIN_ENCODING_V2, row_store_meta.encoding()); diff --git a/be/test/storage/segment/column_reader_cache_test.cpp b/be/test/storage/segment/column_reader_cache_test.cpp index 6a11b77f32fa1a..da8aa13d3bfff8 100644 --- a/be/test/storage/segment/column_reader_cache_test.cpp +++ b/be/test/storage/segment/column_reader_cache_test.cpp @@ -29,6 +29,7 @@ #include "io/fs/file_reader.h" #include "storage/segment/column_meta_accessor.h" #include "storage/segment/column_reader.h" +#include "storage/segment/encoding_info.h" #include "storage/segment/mock/mock_segment.h" #include "storage/segment/segment.h" #include "storage/segment/variant/variant_column_reader.h" @@ -57,6 +58,14 @@ class MockColumnReader : public ColumnReader { class ColumnReaderCacheTest : public ::testing::Test { protected: + // EncodingInfo friended ColumnReaderCacheTest so the fixture can pick a valid V2 + // default encoding for the synthetic ColumnMetaPBs constructed by these tests. + // TEST_F bodies subclass this fixture and call the helper (friendship doesn't carry + // to subclasses). + static segment_v2::EncodingTypePB get_v2_default_encoding(FieldType t) { + return segment_v2::EncodingInfo::get_v2_default_encoding(t); + } + void SetUp() override { // Set up test configuration config::max_segment_partial_column_cache_size = 3; @@ -141,7 +150,7 @@ TEST_F(ColumnReaderCacheTest, BasicCacheOperations) { ColumnMetaPB col_meta; col_meta.set_type(static_cast(FieldType::OLAP_FIELD_TYPE_INT)); col_meta.set_unique_id(1); - col_meta.set_encoding(EncodingTypePB::DEFAULT_ENCODING); + col_meta.set_encoding(get_v2_default_encoding(static_cast(col_meta.type()))); col_meta.mutable_indexes()->Add()->set_type(ORDINAL_INDEX); setup_segment_footer({col_meta}); @@ -174,7 +183,7 @@ TEST_F(ColumnReaderCacheTest, LRUEviction) { ColumnMetaPB col_meta; col_meta.set_type(static_cast(FieldType::OLAP_FIELD_TYPE_INT)); col_meta.set_unique_id(i); - col_meta.set_encoding(EncodingTypePB::DEFAULT_ENCODING); + col_meta.set_encoding(get_v2_default_encoding(static_cast(col_meta.type()))); col_meta.mutable_indexes()->Add()->set_type(ORDINAL_INDEX); setup_segment_footer({col_meta}); @@ -202,11 +211,11 @@ TEST_F(ColumnReaderCacheTest, LRUOrderMaintenance) { ColumnMetaPB col_meta1, col_meta2; col_meta1.set_type(static_cast(FieldType::OLAP_FIELD_TYPE_INT)); col_meta1.set_unique_id(1); - col_meta1.set_encoding(EncodingTypePB::DEFAULT_ENCODING); + col_meta1.set_encoding(get_v2_default_encoding(static_cast(col_meta1.type()))); col_meta1.mutable_indexes()->Add()->set_type(ORDINAL_INDEX); col_meta2.set_type(static_cast(FieldType::OLAP_FIELD_TYPE_INT)); col_meta2.set_unique_id(2); - col_meta2.set_encoding(EncodingTypePB::DEFAULT_ENCODING); + col_meta2.set_encoding(get_v2_default_encoding(static_cast(col_meta2.type()))); col_meta2.mutable_indexes()->Add()->set_type(ORDINAL_INDEX); setup_segment_footer({col_meta1, col_meta2}); @@ -230,7 +239,7 @@ TEST_F(ColumnReaderCacheTest, LRUOrderMaintenance) { ColumnMetaPB col_meta3; col_meta3.set_type(static_cast(FieldType::OLAP_FIELD_TYPE_INT)); col_meta3.set_unique_id(3); - col_meta3.set_encoding(EncodingTypePB::DEFAULT_ENCODING); + col_meta3.set_encoding(get_v2_default_encoding(static_cast(col_meta3.type()))); col_meta3.mutable_indexes()->Add()->set_type(ORDINAL_INDEX); setup_segment_footer({col_meta1, col_meta2, col_meta3}); @@ -254,14 +263,14 @@ TEST_F(ColumnReaderCacheTest, VariantColumnPathReading) { ColumnMetaPB variant_meta; variant_meta.set_type(static_cast(FieldType::OLAP_FIELD_TYPE_VARIANT)); variant_meta.set_unique_id(1); - variant_meta.set_encoding(EncodingTypePB::DEFAULT_ENCODING); + variant_meta.set_encoding(get_v2_default_encoding(static_cast(variant_meta.type()))); variant_meta.mutable_indexes()->Add()->set_type(ORDINAL_INDEX); // Create subcolumn meta ColumnMetaPB subcol_meta; subcol_meta.set_type(static_cast(FieldType::OLAP_FIELD_TYPE_STRING)); subcol_meta.set_unique_id(2); - subcol_meta.set_encoding(EncodingTypePB::DEFAULT_ENCODING); + subcol_meta.set_encoding(get_v2_default_encoding(static_cast(subcol_meta.type()))); subcol_meta.mutable_indexes()->Add()->set_type(ORDINAL_INDEX); setup_segment_footer({variant_meta, subcol_meta}); @@ -289,7 +298,7 @@ TEST_F(ColumnReaderCacheTest, NonExistentVariantPath) { ColumnMetaPB variant_meta; variant_meta.set_type(static_cast(FieldType::OLAP_FIELD_TYPE_VARIANT)); variant_meta.set_unique_id(1); - variant_meta.set_encoding(EncodingTypePB::DEFAULT_ENCODING); + variant_meta.set_encoding(get_v2_default_encoding(static_cast(variant_meta.type()))); variant_meta.mutable_indexes()->Add()->set_type(ORDINAL_INDEX); setup_segment_footer({variant_meta}); @@ -307,7 +316,7 @@ TEST_F(ColumnReaderCacheTest, ConcurrentAccess) { ColumnMetaPB col_meta; col_meta.set_type(static_cast(FieldType::OLAP_FIELD_TYPE_INT)); col_meta.set_unique_id(1); - col_meta.set_encoding(EncodingTypePB::DEFAULT_ENCODING); + col_meta.set_encoding(get_v2_default_encoding(static_cast(col_meta.type()))); col_meta.mutable_indexes()->Add()->set_type(ORDINAL_INDEX); setup_segment_footer({col_meta}); @@ -342,7 +351,7 @@ TEST_F(ColumnReaderCacheTest, CacheStatistics) { ColumnMetaPB col_meta; col_meta.set_type(static_cast(FieldType::OLAP_FIELD_TYPE_INT)); col_meta.set_unique_id(1); - col_meta.set_encoding(EncodingTypePB::DEFAULT_ENCODING); + col_meta.set_encoding(get_v2_default_encoding(static_cast(col_meta.type()))); col_meta.mutable_indexes()->Add()->set_type(ORDINAL_INDEX); setup_segment_footer({col_meta}); @@ -372,7 +381,7 @@ TEST_F(ColumnReaderCacheTest, DifferentColumnTypes) { ColumnMetaPB col_meta; col_meta.set_type(static_cast(types[i])); col_meta.set_unique_id(static_cast(i + 1)); - col_meta.set_encoding(EncodingTypePB::DEFAULT_ENCODING); + col_meta.set_encoding(get_v2_default_encoding(static_cast(col_meta.type()))); col_meta.mutable_indexes()->Add()->set_type(ORDINAL_INDEX); setup_segment_footer({col_meta}); @@ -393,11 +402,11 @@ TEST_F(ColumnReaderCacheTest, NodeHintParameter) { ColumnMetaPB variant_meta, subcol_meta; variant_meta.set_type(static_cast(FieldType::OLAP_FIELD_TYPE_VARIANT)); variant_meta.set_unique_id(1); - variant_meta.set_encoding(EncodingTypePB::DEFAULT_ENCODING); + variant_meta.set_encoding(get_v2_default_encoding(static_cast(variant_meta.type()))); variant_meta.mutable_indexes()->Add()->set_type(ORDINAL_INDEX); subcol_meta.set_type(static_cast(FieldType::OLAP_FIELD_TYPE_STRING)); subcol_meta.set_unique_id(2); - subcol_meta.set_encoding(EncodingTypePB::DEFAULT_ENCODING); + subcol_meta.set_encoding(get_v2_default_encoding(static_cast(subcol_meta.type()))); subcol_meta.mutable_indexes()->Add()->set_type(ORDINAL_INDEX); setup_segment_footer({variant_meta, subcol_meta}); @@ -431,7 +440,7 @@ TEST_F(ColumnReaderCacheTest, PerformanceUnderLoad) { ColumnMetaPB col_meta; col_meta.set_type(static_cast(FieldType::OLAP_FIELD_TYPE_INT)); col_meta.set_unique_id(i); - col_meta.set_encoding(EncodingTypePB::DEFAULT_ENCODING); + col_meta.set_encoding(get_v2_default_encoding(static_cast(col_meta.type()))); col_meta.mutable_indexes()->Add()->set_type(ORDINAL_INDEX); all_columns.push_back(col_meta); } @@ -471,7 +480,7 @@ TEST_F(ColumnReaderCacheTest, EmptyPath) { ColumnMetaPB col_meta; col_meta.set_type(static_cast(FieldType::OLAP_FIELD_TYPE_VARIANT)); col_meta.set_unique_id(1); - col_meta.set_encoding(EncodingTypePB::DEFAULT_ENCODING); + col_meta.set_encoding(get_v2_default_encoding(static_cast(col_meta.type()))); col_meta.mutable_indexes()->Add()->set_type(ORDINAL_INDEX); setup_segment_footer({col_meta}); diff --git a/be/test/storage/segment/encoding_info_test.cpp b/be/test/storage/segment/encoding_info_test.cpp index 666363c9566c83..641e84e0759fcf 100644 --- a/be/test/storage/segment/encoding_info_test.cpp +++ b/be/test/storage/segment/encoding_info_test.cpp @@ -39,13 +39,23 @@ class EncodingInfoTest : public testing::Test { public: EncodingInfoTest() {} virtual ~EncodingInfoTest() {} + +protected: + // EncodingInfo has friended EncodingInfoTest so this fixture can poke the private + // get_v2/v3_default_encoding lookup tables. TEST_F bodies subclass this fixture and + // call these helpers (they don't get the friend permission directly). + static EncodingTypePB get_v2_default_encoding(FieldType t) { + return EncodingInfo::get_v2_default_encoding(t); + } + static EncodingTypePB get_v3_default_encoding(FieldType t) { + return EncodingInfo::get_v3_default_encoding(t); + } }; TEST_F(EncodingInfoTest, normal) { constexpr FieldType type = FieldType::OLAP_FIELD_TYPE_BIGINT; const EncodingInfo* encoding_info = nullptr; - EncodingPreference encoding_preference; - auto status = EncodingInfo::get(type, PLAIN_ENCODING, encoding_preference, &encoding_info); + auto status = EncodingInfo::get(type, PLAIN_ENCODING, &encoding_info); EXPECT_TRUE(status.ok()); EXPECT_NE(nullptr, encoding_info); } @@ -53,98 +63,45 @@ TEST_F(EncodingInfoTest, normal) { TEST_F(EncodingInfoTest, no_encoding) { constexpr FieldType type = FieldType::OLAP_FIELD_TYPE_BIGINT; const EncodingInfo* encoding_info = nullptr; - EncodingPreference encoding_preference; - auto status = EncodingInfo::get(type, DICT_ENCODING, encoding_preference, &encoding_info); + auto status = EncodingInfo::get(type, DICT_ENCODING, &encoding_info); EXPECT_FALSE(status.ok()); } -TEST_F(EncodingInfoTest, test_use_plain_binary_v2_config) { - // Helper lambda to test string/JSON types with DICT_ENCODING as default - auto test_dict_type_encoding = [](FieldType type, const std::string& type_name) { - // Test with BINARY_PLAIN_ENCODING_V1 (default) - // String and JSON types default to DICT_ENCODING - EncodingPreference pref_v1; - pref_v1.binary_plain_encoding_default_impl = - BinaryPlainEncodingTypePB::BINARY_PLAIN_ENCODING_V1; - EncodingTypePB encoding_type = EncodingInfo::get_default_encoding(type, pref_v1, false); - EXPECT_EQ(DICT_ENCODING, encoding_type) - << "Type " << type_name << " should use DICT_ENCODING with V1 preference"; - - // Test with BINARY_PLAIN_ENCODING_V2 - // Preference doesn't affect DICT_ENCODING types - EncodingPreference pref_v2; - pref_v2.binary_plain_encoding_default_impl = - BinaryPlainEncodingTypePB::BINARY_PLAIN_ENCODING_V2; - encoding_type = EncodingInfo::get_default_encoding(type, pref_v2, false); - EXPECT_EQ(DICT_ENCODING, encoding_type) - << "Type " << type_name << " should still use DICT_ENCODING with V2 preference"; +TEST_F(EncodingInfoTest, v2_vs_v3_defaults) { + // String / JSON / variant: default is DICT_ENCODING for both V2 and V3. + auto check_same = [](FieldType type, const std::string& name, EncodingTypePB expected) { + EXPECT_EQ(expected, get_v2_default_encoding(type)) << name << " v2 default"; + EXPECT_EQ(expected, get_v3_default_encoding(type)) << name << " v3 default"; }; - - // Helper lambda to test aggregate state types with PLAIN_ENCODING as default - auto test_plain_type_encoding = [](FieldType type, const std::string& type_name) { - // Test with BINARY_PLAIN_ENCODING_V1 (default) - EncodingPreference pref_v1; - pref_v1.binary_plain_encoding_default_impl = - BinaryPlainEncodingTypePB::BINARY_PLAIN_ENCODING_V1; - EncodingTypePB encoding_type = EncodingInfo::get_default_encoding(type, pref_v1, false); - EXPECT_EQ(PLAIN_ENCODING, encoding_type) - << "Type " << type_name << " should use PLAIN_ENCODING with V1 preference"; - - // Test with BINARY_PLAIN_ENCODING_V2 - EncodingPreference pref_v2; - pref_v2.binary_plain_encoding_default_impl = - BinaryPlainEncodingTypePB::BINARY_PLAIN_ENCODING_V2; - encoding_type = EncodingInfo::get_default_encoding(type, pref_v2, false); - EXPECT_EQ(PLAIN_ENCODING_V2, encoding_type) - << "Type " << type_name << " should use PLAIN_ENCODING_V2 with V2 preference"; + check_same(FieldType::OLAP_FIELD_TYPE_VARCHAR, "VARCHAR", DICT_ENCODING); + check_same(FieldType::OLAP_FIELD_TYPE_STRING, "STRING", DICT_ENCODING); + check_same(FieldType::OLAP_FIELD_TYPE_CHAR, "CHAR", DICT_ENCODING); + check_same(FieldType::OLAP_FIELD_TYPE_JSONB, "JSONB", DICT_ENCODING); + check_same(FieldType::OLAP_FIELD_TYPE_VARIANT, "VARIANT", DICT_ENCODING); + + // Aggregate/binary-flavored types: V2=PLAIN, V3=PLAIN_V2. + auto check_split = [](FieldType type, const std::string& name) { + EXPECT_EQ(PLAIN_ENCODING, get_v2_default_encoding(type)) << name << " v2 default"; + EXPECT_EQ(PLAIN_ENCODING_V2, get_v3_default_encoding(type)) << name << " v3 default"; }; + check_split(FieldType::OLAP_FIELD_TYPE_HLL, "HLL"); + check_split(FieldType::OLAP_FIELD_TYPE_BITMAP, "BITMAP"); + check_split(FieldType::OLAP_FIELD_TYPE_QUANTILE_STATE, "QUANTILE_STATE"); + check_split(FieldType::OLAP_FIELD_TYPE_AGG_STATE, "AGG_STATE"); - // Test string types (default to DICT_ENCODING, not affected by preference) - test_dict_type_encoding(FieldType::OLAP_FIELD_TYPE_VARCHAR, "VARCHAR"); - test_dict_type_encoding(FieldType::OLAP_FIELD_TYPE_STRING, "STRING"); - test_dict_type_encoding(FieldType::OLAP_FIELD_TYPE_CHAR, "CHAR"); - - // Test JSON/variant types (default to DICT_ENCODING, not affected by preference) - test_dict_type_encoding(FieldType::OLAP_FIELD_TYPE_JSONB, "JSONB"); - test_dict_type_encoding(FieldType::OLAP_FIELD_TYPE_VARIANT, "VARIANT"); - - // Test aggregate state types (default to PLAIN_ENCODING, affected by preference) - test_plain_type_encoding(FieldType::OLAP_FIELD_TYPE_HLL, "HLL"); - test_plain_type_encoding(FieldType::OLAP_FIELD_TYPE_BITMAP, "BITMAP"); - test_plain_type_encoding(FieldType::OLAP_FIELD_TYPE_QUANTILE_STATE, "QUANTILE_STATE"); - test_plain_type_encoding(FieldType::OLAP_FIELD_TYPE_AGG_STATE, "AGG_STATE"); - - // Test non-binary type (BIGINT) - should not be affected by binary preference + // Signed integers: V2=BIT_SHUFFLE, V3=PLAIN. constexpr FieldType bigint_type = FieldType::OLAP_FIELD_TYPE_BIGINT; + EXPECT_EQ(BIT_SHUFFLE, get_v2_default_encoding(bigint_type)); + EXPECT_EQ(PLAIN_ENCODING, get_v3_default_encoding(bigint_type)); - // Test with plain encoding disabled for integers (default) - EncodingPreference pref_plain_disabled; - pref_plain_disabled.integer_type_default_use_plain_encoding = false; - pref_plain_disabled.binary_plain_encoding_default_impl = - BinaryPlainEncodingTypePB::BINARY_PLAIN_ENCODING_V1; - EncodingTypePB encoding_type = - EncodingInfo::get_default_encoding(bigint_type, pref_plain_disabled, false); - EXPECT_EQ(BIT_SHUFFLE, encoding_type); - - // Test with plain encoding enabled for integers - EncodingPreference pref_plain_enabled; - pref_plain_enabled.integer_type_default_use_plain_encoding = true; - pref_plain_enabled.binary_plain_encoding_default_impl = - BinaryPlainEncodingTypePB::BINARY_PLAIN_ENCODING_V1; - encoding_type = EncodingInfo::get_default_encoding(bigint_type, pref_plain_enabled, false); - EXPECT_EQ(PLAIN_ENCODING, encoding_type); - - // Verify binary preference doesn't affect integer types - pref_plain_enabled.binary_plain_encoding_default_impl = - BinaryPlainEncodingTypePB::BINARY_PLAIN_ENCODING_V2; - encoding_type = EncodingInfo::get_default_encoding(bigint_type, pref_plain_enabled, false); - EXPECT_EQ(PLAIN_ENCODING, encoding_type); // Should still be PLAIN_ENCODING + // Value-seek default is only registered for VARCHAR (the only production caller). + EXPECT_EQ(PREFIX_ENCODING, + EncodingInfo::get_index_column_encoding(FieldType::OLAP_FIELD_TYPE_VARCHAR)); + EXPECT_EQ(UNKNOWN_ENCODING, EncodingInfo::get_index_column_encoding(bigint_type)); } // Comprehensive test for _data_page_pre_decoder for all encoding types TEST_F(EncodingInfoTest, test_all_pre_decoders) { - EncodingPreference encoding_preference; - // Test BIT_SHUFFLE encoding - should have BitShufflePagePreDecoder // Test various integer types std::vector bitshuffle_types = { @@ -163,7 +120,7 @@ TEST_F(EncodingInfoTest, test_all_pre_decoders) { for (auto type : bitshuffle_types) { const EncodingInfo* encoding_info = nullptr; - auto status = EncodingInfo::get(type, BIT_SHUFFLE, encoding_preference, &encoding_info); + auto status = EncodingInfo::get(type, BIT_SHUFFLE, &encoding_info); if (status.ok()) { ASSERT_NE(nullptr, encoding_info); auto* pre_decoder = encoding_info->get_data_page_pre_decoder(); @@ -185,7 +142,7 @@ TEST_F(EncodingInfoTest, test_all_pre_decoders) { for (auto type : dict_types) { const EncodingInfo* encoding_info = nullptr; - auto status = EncodingInfo::get(type, DICT_ENCODING, encoding_preference, &encoding_info); + auto status = EncodingInfo::get(type, DICT_ENCODING, &encoding_info); ASSERT_TRUE(status.ok()) << "Type " << static_cast(type) << " should support DICT_ENCODING"; ASSERT_NE(nullptr, encoding_info); @@ -209,8 +166,7 @@ TEST_F(EncodingInfoTest, test_all_pre_decoders) { for (auto type : plain_v2_types) { const EncodingInfo* encoding_info = nullptr; - auto status = - EncodingInfo::get(type, PLAIN_ENCODING_V2, encoding_preference, &encoding_info); + auto status = EncodingInfo::get(type, PLAIN_ENCODING_V2, &encoding_info); ASSERT_TRUE(status.ok()) << "Type " << static_cast(type) << " should support PLAIN_ENCODING_V2"; ASSERT_NE(nullptr, encoding_info); @@ -257,7 +213,7 @@ TEST_F(EncodingInfoTest, test_all_pre_decoders) { for (auto type : plain_encoding_types) { const EncodingInfo* encoding_info = nullptr; - auto status = EncodingInfo::get(type, PLAIN_ENCODING, encoding_preference, &encoding_info); + auto status = EncodingInfo::get(type, PLAIN_ENCODING, &encoding_info); if (status.ok() && encoding_info != nullptr) { auto* pre_decoder = encoding_info->get_data_page_pre_decoder(); EXPECT_EQ(nullptr, pre_decoder) << "Type " << static_cast(type) @@ -276,7 +232,7 @@ TEST_F(EncodingInfoTest, test_all_pre_decoders) { for (auto type : for_encoding_types) { const EncodingInfo* encoding_info = nullptr; - auto status = EncodingInfo::get(type, FOR_ENCODING, encoding_preference, &encoding_info); + auto status = EncodingInfo::get(type, FOR_ENCODING, &encoding_info); if (status.ok() && encoding_info != nullptr) { auto* pre_decoder = encoding_info->get_data_page_pre_decoder(); EXPECT_EQ(nullptr, pre_decoder) << "Type " << static_cast(type) @@ -293,7 +249,7 @@ TEST_F(EncodingInfoTest, test_all_pre_decoders) { for (auto type : prefix_encoding_types) { const EncodingInfo* encoding_info = nullptr; - auto status = EncodingInfo::get(type, PREFIX_ENCODING, encoding_preference, &encoding_info); + auto status = EncodingInfo::get(type, PREFIX_ENCODING, &encoding_info); if (status.ok() && encoding_info != nullptr) { auto* pre_decoder = encoding_info->get_data_page_pre_decoder(); EXPECT_EQ(nullptr, pre_decoder) << "Type " << static_cast(type) @@ -304,8 +260,7 @@ TEST_F(EncodingInfoTest, test_all_pre_decoders) { // Test RLE - should NOT have pre_decoder (only for BOOL) { const EncodingInfo* encoding_info = nullptr; - auto status = EncodingInfo::get(FieldType::OLAP_FIELD_TYPE_BOOL, RLE, encoding_preference, - &encoding_info); + auto status = EncodingInfo::get(FieldType::OLAP_FIELD_TYPE_BOOL, RLE, &encoding_info); if (status.ok() && encoding_info != nullptr) { auto* pre_decoder = encoding_info->get_data_page_pre_decoder(); EXPECT_EQ(nullptr, pre_decoder) << "BOOL with RLE should NOT have pre_decoder"; @@ -313,5 +268,301 @@ TEST_F(EncodingInfoTest, test_all_pre_decoders) { } } +// ============================================================================ +// Behavior-locking tests. The expectation tables below are the authoritative +// contract for what each get_*_encoding API returns per FieldType. Changing +// a value here is a deliberate behavior change and must be justified. +// ============================================================================ + +namespace { + +// (type, expected_encoding) row for a default-encoding test. +struct DefaultExpectation { + FieldType type; + EncodingTypePB expected; + const char* name; +}; + +// (type, encoding, should_exist) row for the global encoding_map completeness test. +struct EncodingMapEntry { + FieldType type; + EncodingTypePB encoding; + bool should_exist; + const char* name; +}; + +// Expected V3 default per type. Differs from V2 default in two type families: +// - binary blobs (HLL/BITMAP/QUANTILE_STATE/AGG_STATE): PLAIN_ENCODING_V2 (vs V2's PLAIN) +// - signed integers (TINYINT..LARGEINT): PLAIN_ENCODING (vs V2's BIT_SHUFFLE) +const std::vector kV3DefaultExpect = { + {FieldType::OLAP_FIELD_TYPE_TINYINT, PLAIN_ENCODING, "TINYINT"}, + {FieldType::OLAP_FIELD_TYPE_SMALLINT, PLAIN_ENCODING, "SMALLINT"}, + {FieldType::OLAP_FIELD_TYPE_INT, PLAIN_ENCODING, "INT"}, + {FieldType::OLAP_FIELD_TYPE_BIGINT, PLAIN_ENCODING, "BIGINT"}, + {FieldType::OLAP_FIELD_TYPE_LARGEINT, PLAIN_ENCODING, "LARGEINT"}, + {FieldType::OLAP_FIELD_TYPE_UNSIGNED_BIGINT, BIT_SHUFFLE, "UNSIGNED_BIGINT"}, + {FieldType::OLAP_FIELD_TYPE_UNSIGNED_INT, BIT_SHUFFLE, "UNSIGNED_INT"}, + {FieldType::OLAP_FIELD_TYPE_FLOAT, BIT_SHUFFLE, "FLOAT"}, + {FieldType::OLAP_FIELD_TYPE_DOUBLE, BIT_SHUFFLE, "DOUBLE"}, + {FieldType::OLAP_FIELD_TYPE_CHAR, DICT_ENCODING, "CHAR"}, + {FieldType::OLAP_FIELD_TYPE_VARCHAR, DICT_ENCODING, "VARCHAR"}, + {FieldType::OLAP_FIELD_TYPE_STRING, DICT_ENCODING, "STRING"}, + {FieldType::OLAP_FIELD_TYPE_JSONB, DICT_ENCODING, "JSONB"}, + {FieldType::OLAP_FIELD_TYPE_VARIANT, DICT_ENCODING, "VARIANT"}, + {FieldType::OLAP_FIELD_TYPE_BOOL, RLE, "BOOL"}, + {FieldType::OLAP_FIELD_TYPE_DATE, BIT_SHUFFLE, "DATE"}, + {FieldType::OLAP_FIELD_TYPE_DATEV2, BIT_SHUFFLE, "DATEV2"}, + {FieldType::OLAP_FIELD_TYPE_DATETIMEV2, BIT_SHUFFLE, "DATETIMEV2"}, + {FieldType::OLAP_FIELD_TYPE_DATETIME, BIT_SHUFFLE, "DATETIME"}, + {FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ, BIT_SHUFFLE, "TIMESTAMPTZ"}, + {FieldType::OLAP_FIELD_TYPE_DECIMAL, BIT_SHUFFLE, "DECIMAL"}, + {FieldType::OLAP_FIELD_TYPE_DECIMAL32, BIT_SHUFFLE, "DECIMAL32"}, + {FieldType::OLAP_FIELD_TYPE_DECIMAL64, BIT_SHUFFLE, "DECIMAL64"}, + {FieldType::OLAP_FIELD_TYPE_DECIMAL128I, BIT_SHUFFLE, "DECIMAL128I"}, + {FieldType::OLAP_FIELD_TYPE_DECIMAL256, BIT_SHUFFLE, "DECIMAL256"}, + {FieldType::OLAP_FIELD_TYPE_IPV4, BIT_SHUFFLE, "IPV4"}, + {FieldType::OLAP_FIELD_TYPE_IPV6, BIT_SHUFFLE, "IPV6"}, + {FieldType::OLAP_FIELD_TYPE_HLL, PLAIN_ENCODING_V2, "HLL"}, + {FieldType::OLAP_FIELD_TYPE_BITMAP, PLAIN_ENCODING_V2, "BITMAP"}, + {FieldType::OLAP_FIELD_TYPE_QUANTILE_STATE, PLAIN_ENCODING_V2, "QUANTILE_STATE"}, + {FieldType::OLAP_FIELD_TYPE_AGG_STATE, PLAIN_ENCODING_V2, "AGG_STATE"}, +}; + +// Expected V2 (non-V3) default per type. +const std::vector kV2DefaultExpect = { + {FieldType::OLAP_FIELD_TYPE_TINYINT, BIT_SHUFFLE, "TINYINT"}, + {FieldType::OLAP_FIELD_TYPE_SMALLINT, BIT_SHUFFLE, "SMALLINT"}, + {FieldType::OLAP_FIELD_TYPE_INT, BIT_SHUFFLE, "INT"}, + {FieldType::OLAP_FIELD_TYPE_BIGINT, BIT_SHUFFLE, "BIGINT"}, + {FieldType::OLAP_FIELD_TYPE_LARGEINT, BIT_SHUFFLE, "LARGEINT"}, + {FieldType::OLAP_FIELD_TYPE_UNSIGNED_BIGINT, BIT_SHUFFLE, "UNSIGNED_BIGINT"}, + {FieldType::OLAP_FIELD_TYPE_UNSIGNED_INT, BIT_SHUFFLE, "UNSIGNED_INT"}, + {FieldType::OLAP_FIELD_TYPE_FLOAT, BIT_SHUFFLE, "FLOAT"}, + {FieldType::OLAP_FIELD_TYPE_DOUBLE, BIT_SHUFFLE, "DOUBLE"}, + {FieldType::OLAP_FIELD_TYPE_CHAR, DICT_ENCODING, "CHAR"}, + {FieldType::OLAP_FIELD_TYPE_VARCHAR, DICT_ENCODING, "VARCHAR"}, + {FieldType::OLAP_FIELD_TYPE_STRING, DICT_ENCODING, "STRING"}, + {FieldType::OLAP_FIELD_TYPE_JSONB, DICT_ENCODING, "JSONB"}, + {FieldType::OLAP_FIELD_TYPE_VARIANT, DICT_ENCODING, "VARIANT"}, + {FieldType::OLAP_FIELD_TYPE_BOOL, RLE, "BOOL"}, + {FieldType::OLAP_FIELD_TYPE_DATE, BIT_SHUFFLE, "DATE"}, + {FieldType::OLAP_FIELD_TYPE_DATEV2, BIT_SHUFFLE, "DATEV2"}, + {FieldType::OLAP_FIELD_TYPE_DATETIMEV2, BIT_SHUFFLE, "DATETIMEV2"}, + {FieldType::OLAP_FIELD_TYPE_DATETIME, BIT_SHUFFLE, "DATETIME"}, + {FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ, BIT_SHUFFLE, "TIMESTAMPTZ"}, + {FieldType::OLAP_FIELD_TYPE_DECIMAL, BIT_SHUFFLE, "DECIMAL"}, + {FieldType::OLAP_FIELD_TYPE_DECIMAL32, BIT_SHUFFLE, "DECIMAL32"}, + {FieldType::OLAP_FIELD_TYPE_DECIMAL64, BIT_SHUFFLE, "DECIMAL64"}, + {FieldType::OLAP_FIELD_TYPE_DECIMAL128I, BIT_SHUFFLE, "DECIMAL128I"}, + {FieldType::OLAP_FIELD_TYPE_DECIMAL256, BIT_SHUFFLE, "DECIMAL256"}, + {FieldType::OLAP_FIELD_TYPE_IPV4, BIT_SHUFFLE, "IPV4"}, + {FieldType::OLAP_FIELD_TYPE_IPV6, BIT_SHUFFLE, "IPV6"}, + {FieldType::OLAP_FIELD_TYPE_HLL, PLAIN_ENCODING, "HLL"}, + {FieldType::OLAP_FIELD_TYPE_BITMAP, PLAIN_ENCODING, "BITMAP"}, + {FieldType::OLAP_FIELD_TYPE_QUANTILE_STATE, PLAIN_ENCODING, "QUANTILE_STATE"}, + {FieldType::OLAP_FIELD_TYPE_AGG_STATE, PLAIN_ENCODING, "AGG_STATE"}, +}; + +// Expected index_column_encoding per type. Only VARCHAR is consulted in production +// (PrimaryKeyIndexBuilder::init hardcodes VARCHAR); all other entries return +// UNKNOWN_ENCODING because they were never queried and have been removed. +const std::vector kIndexColumnEncodingExpect = { + {FieldType::OLAP_FIELD_TYPE_TINYINT, UNKNOWN_ENCODING, "TINYINT"}, + {FieldType::OLAP_FIELD_TYPE_SMALLINT, UNKNOWN_ENCODING, "SMALLINT"}, + {FieldType::OLAP_FIELD_TYPE_INT, UNKNOWN_ENCODING, "INT"}, + {FieldType::OLAP_FIELD_TYPE_BIGINT, UNKNOWN_ENCODING, "BIGINT"}, + {FieldType::OLAP_FIELD_TYPE_LARGEINT, UNKNOWN_ENCODING, "LARGEINT"}, + {FieldType::OLAP_FIELD_TYPE_UNSIGNED_BIGINT, UNKNOWN_ENCODING, "UNSIGNED_BIGINT"}, + {FieldType::OLAP_FIELD_TYPE_UNSIGNED_INT, UNKNOWN_ENCODING, "UNSIGNED_INT"}, + {FieldType::OLAP_FIELD_TYPE_FLOAT, UNKNOWN_ENCODING, "FLOAT"}, + {FieldType::OLAP_FIELD_TYPE_DOUBLE, UNKNOWN_ENCODING, "DOUBLE"}, + {FieldType::OLAP_FIELD_TYPE_CHAR, UNKNOWN_ENCODING, "CHAR"}, + {FieldType::OLAP_FIELD_TYPE_VARCHAR, PREFIX_ENCODING, "VARCHAR"}, + {FieldType::OLAP_FIELD_TYPE_STRING, UNKNOWN_ENCODING, "STRING"}, + {FieldType::OLAP_FIELD_TYPE_JSONB, UNKNOWN_ENCODING, "JSONB"}, + {FieldType::OLAP_FIELD_TYPE_VARIANT, UNKNOWN_ENCODING, "VARIANT"}, + {FieldType::OLAP_FIELD_TYPE_BOOL, UNKNOWN_ENCODING, "BOOL"}, + {FieldType::OLAP_FIELD_TYPE_DATE, UNKNOWN_ENCODING, "DATE"}, + {FieldType::OLAP_FIELD_TYPE_DATEV2, UNKNOWN_ENCODING, "DATEV2"}, + {FieldType::OLAP_FIELD_TYPE_DATETIMEV2, UNKNOWN_ENCODING, "DATETIMEV2"}, + {FieldType::OLAP_FIELD_TYPE_DATETIME, UNKNOWN_ENCODING, "DATETIME"}, + {FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ, UNKNOWN_ENCODING, "TIMESTAMPTZ"}, + {FieldType::OLAP_FIELD_TYPE_DECIMAL, UNKNOWN_ENCODING, "DECIMAL"}, + {FieldType::OLAP_FIELD_TYPE_DECIMAL32, UNKNOWN_ENCODING, "DECIMAL32"}, + {FieldType::OLAP_FIELD_TYPE_DECIMAL64, UNKNOWN_ENCODING, "DECIMAL64"}, + {FieldType::OLAP_FIELD_TYPE_DECIMAL128I, UNKNOWN_ENCODING, "DECIMAL128I"}, + {FieldType::OLAP_FIELD_TYPE_DECIMAL256, UNKNOWN_ENCODING, "DECIMAL256"}, + {FieldType::OLAP_FIELD_TYPE_IPV4, UNKNOWN_ENCODING, "IPV4"}, + {FieldType::OLAP_FIELD_TYPE_IPV6, UNKNOWN_ENCODING, "IPV6"}, + {FieldType::OLAP_FIELD_TYPE_HLL, UNKNOWN_ENCODING, "HLL"}, + {FieldType::OLAP_FIELD_TYPE_BITMAP, UNKNOWN_ENCODING, "BITMAP"}, + {FieldType::OLAP_FIELD_TYPE_QUANTILE_STATE, UNKNOWN_ENCODING, "QUANTILE_STATE"}, + {FieldType::OLAP_FIELD_TYPE_AGG_STATE, UNKNOWN_ENCODING, "AGG_STATE"}, +}; + +// Full enumeration of (type, encoding) combinations that EncodingInfo::get should +// find or reject. should_exist=true means EncodingInfo::get must succeed. +const std::vector kEncodingMapEntries = { + // signed integers: BIT_SHUFFLE, PLAIN_ENCODING, FOR_ENCODING + {FieldType::OLAP_FIELD_TYPE_TINYINT, BIT_SHUFFLE, true, "TINYINT+BIT_SHUFFLE"}, + {FieldType::OLAP_FIELD_TYPE_TINYINT, PLAIN_ENCODING, true, "TINYINT+PLAIN"}, + {FieldType::OLAP_FIELD_TYPE_TINYINT, FOR_ENCODING, true, "TINYINT+FOR"}, + {FieldType::OLAP_FIELD_TYPE_SMALLINT, BIT_SHUFFLE, true, "SMALLINT+BIT_SHUFFLE"}, + {FieldType::OLAP_FIELD_TYPE_SMALLINT, PLAIN_ENCODING, true, "SMALLINT+PLAIN"}, + {FieldType::OLAP_FIELD_TYPE_SMALLINT, FOR_ENCODING, true, "SMALLINT+FOR"}, + {FieldType::OLAP_FIELD_TYPE_INT, BIT_SHUFFLE, true, "INT+BIT_SHUFFLE"}, + {FieldType::OLAP_FIELD_TYPE_INT, PLAIN_ENCODING, true, "INT+PLAIN"}, + {FieldType::OLAP_FIELD_TYPE_INT, FOR_ENCODING, true, "INT+FOR"}, + {FieldType::OLAP_FIELD_TYPE_BIGINT, BIT_SHUFFLE, true, "BIGINT+BIT_SHUFFLE"}, + {FieldType::OLAP_FIELD_TYPE_BIGINT, PLAIN_ENCODING, true, "BIGINT+PLAIN"}, + {FieldType::OLAP_FIELD_TYPE_BIGINT, FOR_ENCODING, true, "BIGINT+FOR"}, + {FieldType::OLAP_FIELD_TYPE_LARGEINT, BIT_SHUFFLE, true, "LARGEINT+BIT_SHUFFLE"}, + {FieldType::OLAP_FIELD_TYPE_LARGEINT, PLAIN_ENCODING, true, "LARGEINT+PLAIN"}, + {FieldType::OLAP_FIELD_TYPE_LARGEINT, FOR_ENCODING, true, "LARGEINT+FOR"}, + // unsigned integers: only BIT_SHUFFLE + {FieldType::OLAP_FIELD_TYPE_UNSIGNED_BIGINT, BIT_SHUFFLE, true, + "UNSIGNED_BIGINT+BIT_SHUFFLE"}, + {FieldType::OLAP_FIELD_TYPE_UNSIGNED_INT, BIT_SHUFFLE, true, "UNSIGNED_INT+BIT_SHUFFLE"}, + // FLOAT/DOUBLE: BIT_SHUFFLE, PLAIN_ENCODING + {FieldType::OLAP_FIELD_TYPE_FLOAT, BIT_SHUFFLE, true, "FLOAT+BIT_SHUFFLE"}, + {FieldType::OLAP_FIELD_TYPE_FLOAT, PLAIN_ENCODING, true, "FLOAT+PLAIN"}, + {FieldType::OLAP_FIELD_TYPE_DOUBLE, BIT_SHUFFLE, true, "DOUBLE+BIT_SHUFFLE"}, + {FieldType::OLAP_FIELD_TYPE_DOUBLE, PLAIN_ENCODING, true, "DOUBLE+PLAIN"}, + // binary types: DICT, PLAIN, PLAIN_V2, PREFIX + {FieldType::OLAP_FIELD_TYPE_CHAR, DICT_ENCODING, true, "CHAR+DICT"}, + {FieldType::OLAP_FIELD_TYPE_CHAR, PLAIN_ENCODING, true, "CHAR+PLAIN"}, + {FieldType::OLAP_FIELD_TYPE_CHAR, PLAIN_ENCODING_V2, true, "CHAR+PLAIN_V2"}, + {FieldType::OLAP_FIELD_TYPE_CHAR, PREFIX_ENCODING, true, "CHAR+PREFIX"}, + {FieldType::OLAP_FIELD_TYPE_VARCHAR, DICT_ENCODING, true, "VARCHAR+DICT"}, + {FieldType::OLAP_FIELD_TYPE_VARCHAR, PLAIN_ENCODING, true, "VARCHAR+PLAIN"}, + {FieldType::OLAP_FIELD_TYPE_VARCHAR, PLAIN_ENCODING_V2, true, "VARCHAR+PLAIN_V2"}, + {FieldType::OLAP_FIELD_TYPE_VARCHAR, PREFIX_ENCODING, true, "VARCHAR+PREFIX"}, + {FieldType::OLAP_FIELD_TYPE_STRING, DICT_ENCODING, true, "STRING+DICT"}, + {FieldType::OLAP_FIELD_TYPE_STRING, PLAIN_ENCODING, true, "STRING+PLAIN"}, + {FieldType::OLAP_FIELD_TYPE_STRING, PLAIN_ENCODING_V2, true, "STRING+PLAIN_V2"}, + {FieldType::OLAP_FIELD_TYPE_STRING, PREFIX_ENCODING, true, "STRING+PREFIX"}, + {FieldType::OLAP_FIELD_TYPE_JSONB, DICT_ENCODING, true, "JSONB+DICT"}, + {FieldType::OLAP_FIELD_TYPE_JSONB, PLAIN_ENCODING, true, "JSONB+PLAIN"}, + {FieldType::OLAP_FIELD_TYPE_JSONB, PLAIN_ENCODING_V2, true, "JSONB+PLAIN_V2"}, + {FieldType::OLAP_FIELD_TYPE_JSONB, PREFIX_ENCODING, true, "JSONB+PREFIX"}, + {FieldType::OLAP_FIELD_TYPE_VARIANT, DICT_ENCODING, true, "VARIANT+DICT"}, + {FieldType::OLAP_FIELD_TYPE_VARIANT, PLAIN_ENCODING, true, "VARIANT+PLAIN"}, + {FieldType::OLAP_FIELD_TYPE_VARIANT, PLAIN_ENCODING_V2, true, "VARIANT+PLAIN_V2"}, + {FieldType::OLAP_FIELD_TYPE_VARIANT, PREFIX_ENCODING, true, "VARIANT+PREFIX"}, + // BOOL: RLE, BIT_SHUFFLE, PLAIN + {FieldType::OLAP_FIELD_TYPE_BOOL, RLE, true, "BOOL+RLE"}, + {FieldType::OLAP_FIELD_TYPE_BOOL, BIT_SHUFFLE, true, "BOOL+BIT_SHUFFLE"}, + {FieldType::OLAP_FIELD_TYPE_BOOL, PLAIN_ENCODING, true, "BOOL+PLAIN"}, + // date / datetime: BIT_SHUFFLE, PLAIN, FOR + {FieldType::OLAP_FIELD_TYPE_DATE, BIT_SHUFFLE, true, "DATE+BIT_SHUFFLE"}, + {FieldType::OLAP_FIELD_TYPE_DATE, PLAIN_ENCODING, true, "DATE+PLAIN"}, + {FieldType::OLAP_FIELD_TYPE_DATE, FOR_ENCODING, true, "DATE+FOR"}, + {FieldType::OLAP_FIELD_TYPE_DATEV2, BIT_SHUFFLE, true, "DATEV2+BIT_SHUFFLE"}, + {FieldType::OLAP_FIELD_TYPE_DATEV2, PLAIN_ENCODING, true, "DATEV2+PLAIN"}, + {FieldType::OLAP_FIELD_TYPE_DATEV2, FOR_ENCODING, true, "DATEV2+FOR"}, + {FieldType::OLAP_FIELD_TYPE_DATETIMEV2, BIT_SHUFFLE, true, "DATETIMEV2+BIT_SHUFFLE"}, + {FieldType::OLAP_FIELD_TYPE_DATETIMEV2, PLAIN_ENCODING, true, "DATETIMEV2+PLAIN"}, + {FieldType::OLAP_FIELD_TYPE_DATETIMEV2, FOR_ENCODING, true, "DATETIMEV2+FOR"}, + {FieldType::OLAP_FIELD_TYPE_DATETIME, BIT_SHUFFLE, true, "DATETIME+BIT_SHUFFLE"}, + {FieldType::OLAP_FIELD_TYPE_DATETIME, PLAIN_ENCODING, true, "DATETIME+PLAIN"}, + {FieldType::OLAP_FIELD_TYPE_DATETIME, FOR_ENCODING, true, "DATETIME+FOR"}, + {FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ, BIT_SHUFFLE, true, "TIMESTAMPTZ+BIT_SHUFFLE"}, + {FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ, PLAIN_ENCODING, true, "TIMESTAMPTZ+PLAIN"}, + {FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ, FOR_ENCODING, true, "TIMESTAMPTZ+FOR"}, + // decimal: BIT_SHUFFLE, PLAIN + {FieldType::OLAP_FIELD_TYPE_DECIMAL, BIT_SHUFFLE, true, "DECIMAL+BIT_SHUFFLE"}, + {FieldType::OLAP_FIELD_TYPE_DECIMAL, PLAIN_ENCODING, true, "DECIMAL+PLAIN"}, + {FieldType::OLAP_FIELD_TYPE_DECIMAL32, BIT_SHUFFLE, true, "DECIMAL32+BIT_SHUFFLE"}, + {FieldType::OLAP_FIELD_TYPE_DECIMAL32, PLAIN_ENCODING, true, "DECIMAL32+PLAIN"}, + {FieldType::OLAP_FIELD_TYPE_DECIMAL64, BIT_SHUFFLE, true, "DECIMAL64+BIT_SHUFFLE"}, + {FieldType::OLAP_FIELD_TYPE_DECIMAL64, PLAIN_ENCODING, true, "DECIMAL64+PLAIN"}, + {FieldType::OLAP_FIELD_TYPE_DECIMAL128I, BIT_SHUFFLE, true, "DECIMAL128I+BIT_SHUFFLE"}, + {FieldType::OLAP_FIELD_TYPE_DECIMAL128I, PLAIN_ENCODING, true, "DECIMAL128I+PLAIN"}, + {FieldType::OLAP_FIELD_TYPE_DECIMAL256, BIT_SHUFFLE, true, "DECIMAL256+BIT_SHUFFLE"}, + {FieldType::OLAP_FIELD_TYPE_DECIMAL256, PLAIN_ENCODING, true, "DECIMAL256+PLAIN"}, + // ip + {FieldType::OLAP_FIELD_TYPE_IPV4, BIT_SHUFFLE, true, "IPV4+BIT_SHUFFLE"}, + {FieldType::OLAP_FIELD_TYPE_IPV4, PLAIN_ENCODING, true, "IPV4+PLAIN"}, + {FieldType::OLAP_FIELD_TYPE_IPV6, BIT_SHUFFLE, true, "IPV6+BIT_SHUFFLE"}, + {FieldType::OLAP_FIELD_TYPE_IPV6, PLAIN_ENCODING, true, "IPV6+PLAIN"}, + // aggregate-flavored binary + {FieldType::OLAP_FIELD_TYPE_HLL, PLAIN_ENCODING, true, "HLL+PLAIN"}, + {FieldType::OLAP_FIELD_TYPE_HLL, PLAIN_ENCODING_V2, true, "HLL+PLAIN_V2"}, + {FieldType::OLAP_FIELD_TYPE_BITMAP, PLAIN_ENCODING, true, "BITMAP+PLAIN"}, + {FieldType::OLAP_FIELD_TYPE_BITMAP, PLAIN_ENCODING_V2, true, "BITMAP+PLAIN_V2"}, + {FieldType::OLAP_FIELD_TYPE_QUANTILE_STATE, PLAIN_ENCODING, true, "QUANTILE_STATE+PLAIN"}, + {FieldType::OLAP_FIELD_TYPE_QUANTILE_STATE, PLAIN_ENCODING_V2, true, + "QUANTILE_STATE+PLAIN_V2"}, + {FieldType::OLAP_FIELD_TYPE_AGG_STATE, PLAIN_ENCODING, true, "AGG_STATE+PLAIN"}, + {FieldType::OLAP_FIELD_TYPE_AGG_STATE, PLAIN_ENCODING_V2, true, "AGG_STATE+PLAIN_V2"}, + // Negative samples: combinations that should NOT be registered. + {FieldType::OLAP_FIELD_TYPE_INT, DICT_ENCODING, false, "INT+DICT"}, + {FieldType::OLAP_FIELD_TYPE_INT, RLE, false, "INT+RLE"}, + {FieldType::OLAP_FIELD_TYPE_INT, PREFIX_ENCODING, false, "INT+PREFIX"}, + {FieldType::OLAP_FIELD_TYPE_INT, PLAIN_ENCODING_V2, false, "INT+PLAIN_V2"}, + {FieldType::OLAP_FIELD_TYPE_BIGINT, RLE, false, "BIGINT+RLE"}, + {FieldType::OLAP_FIELD_TYPE_FLOAT, DICT_ENCODING, false, "FLOAT+DICT"}, + {FieldType::OLAP_FIELD_TYPE_FLOAT, FOR_ENCODING, false, "FLOAT+FOR"}, + {FieldType::OLAP_FIELD_TYPE_VARCHAR, BIT_SHUFFLE, false, "VARCHAR+BIT_SHUFFLE"}, + {FieldType::OLAP_FIELD_TYPE_VARCHAR, RLE, false, "VARCHAR+RLE"}, + {FieldType::OLAP_FIELD_TYPE_BOOL, DICT_ENCODING, false, "BOOL+DICT"}, + {FieldType::OLAP_FIELD_TYPE_BOOL, FOR_ENCODING, false, "BOOL+FOR"}, + {FieldType::OLAP_FIELD_TYPE_DATE, DICT_ENCODING, false, "DATE+DICT"}, + {FieldType::OLAP_FIELD_TYPE_DECIMAL, DICT_ENCODING, false, "DECIMAL+DICT"}, + {FieldType::OLAP_FIELD_TYPE_HLL, DICT_ENCODING, false, "HLL+DICT"}, + {FieldType::OLAP_FIELD_TYPE_HLL, BIT_SHUFFLE, false, "HLL+BIT_SHUFFLE"}, +}; + +} // namespace + +// case 1: V3 default encoding for every type. +TEST_F(EncodingInfoTest, locked_v3_default_per_type) { + for (const auto& row : kV3DefaultExpect) { + auto got = get_v3_default_encoding(row.type); + EXPECT_EQ(row.expected, got) + << "V3 default mismatch for " << row.name << " (type=" << int(row.type) + << "): expected " << row.expected << ", got " << got; + } +} + +// case 2: V2 (non-V3) default encoding for every type. +TEST_F(EncodingInfoTest, locked_v2_default_per_type) { + for (const auto& row : kV2DefaultExpect) { + auto got = get_v2_default_encoding(row.type); + EXPECT_EQ(row.expected, got) + << "Legacy default mismatch for " << row.name << " (type=" << int(row.type) + << "): expected " << row.expected << ", got " << got; + } +} + +// case 3: index_column_encoding for every type. UNKNOWN_ENCODING means no +// index_column_encoding was registered for that type. +TEST_F(EncodingInfoTest, locked_index_column_encoding_per_type) { + for (const auto& row : kIndexColumnEncodingExpect) { + auto got = EncodingInfo::get_index_column_encoding(row.type); + EXPECT_EQ(row.expected, got) + << "Value-seek default mismatch for " << row.name << " (type=" << int(row.type) + << "): expected " << row.expected << ", got " << got; + } +} + +// case 4: every (type, encoding) combination has the expected presence in the +// global encoding map. +TEST_F(EncodingInfoTest, locked_encoding_map_completeness) { + for (const auto& row : kEncodingMapEntries) { + const EncodingInfo* info = nullptr; + auto status = EncodingInfo::get(row.type, row.encoding, &info); + if (row.should_exist) { + EXPECT_TRUE(status.ok()) + << "Expected entry missing: " << row.name << ": " << status.to_string(); + EXPECT_NE(nullptr, info) << "Entry " << row.name << " is null"; + } else { + EXPECT_FALSE(status.ok()) + << "Unexpected entry present: " << row.name << " (type=" << int(row.type) + << ", encoding=" << row.encoding << ")"; + } + } +} + } // namespace segment_v2 } // namespace doris diff --git a/be/test/storage/segment/external_col_meta_util_test.cpp b/be/test/storage/segment/external_col_meta_util_test.cpp index e2c2f285b45100..b8aed9bfa09d11 100644 --- a/be/test/storage/segment/external_col_meta_util_test.cpp +++ b/be/test/storage/segment/external_col_meta_util_test.cpp @@ -494,7 +494,7 @@ TEST(ExternalColMetaUtilTest, BuildSegmentAndVerifyDataAndFooterMeta) { TabletSchemaSPtr tablet_schema = create_schema(columns, UNIQUE_KEYS); // Enable external ColumnMetaPB so that SegmentWriter produces a V3 footer with // externalized column meta region + column_meta_entries. - tablet_schema->set_external_segment_meta_used_default(true); + tablet_schema->set_storage_format(TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V3); // 2. Use existing build_segment helper (SegmentWriter-based) to write a segment file. SegmentWriterOptions opts; diff --git a/be/test/storage/segment/variant_column_writer_reader_test.cpp b/be/test/storage/segment/variant_column_writer_reader_test.cpp index 6db2dd3bfd013f..7d0ef988c13382 100644 --- a/be/test/storage/segment/variant_column_writer_reader_test.cpp +++ b/be/test/storage/segment/variant_column_writer_reader_test.cpp @@ -226,7 +226,7 @@ class VariantColumnWriterReaderTest : public testing::Test { _tablet_schema->init_from_pb(schema_pb); TabletMetaSharedPtr tablet_meta(new TabletMeta(_tablet_schema)); - _tablet_schema->set_external_segment_meta_used_default(true); + _tablet_schema->set_storage_format(TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V3); tablet_meta->_tablet_id = tablet_id; _tablet = std::make_shared(*_engine_ref, tablet_meta, _data_dir.get()); @@ -499,7 +499,7 @@ class VariantColumnWriterReaderTest : public testing::Test { opts.file_writer = file_writer.get(); opts.footer = footer; opts.rowset_ctx = &rowset_ctx; - _init_column_meta(opts.meta, 0, parent_column, CompressionTypePB::LZ4); + _init_column_meta(opts.meta, 0, parent_column, opts); std::unique_ptr writer; RETURN_IF_ERROR(ColumnWriter::create(opts, &parent_column, file_writer.get(), &writer)); @@ -765,7 +765,7 @@ TEST_F(VariantColumnWriterReaderTest, test_legacy_flat_dot_key_reader_init) { // 2. create tablet TabletMetaSharedPtr tablet_meta(new TabletMeta(_tablet_schema)); - _tablet_schema->set_external_segment_meta_used_default(false); + _tablet_schema->set_storage_format(TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2); tablet_meta->_tablet_id = 20000; _tablet = std::make_shared(*_engine_ref, tablet_meta, _data_dir.get()); EXPECT_TRUE(_tablet->init().ok()); @@ -790,7 +790,9 @@ TEST_F(VariantColumnWriterReaderTest, test_legacy_flat_dot_key_reader_init) { opts.rowset_ctx = &rowset_ctx; opts.rowset_ctx->tablet_schema = _tablet_schema; TabletColumn column = _tablet_schema->column(0); - _init_column_meta(opts.meta, 0, column, CompressionTypePB::LZ4); + opts.compression_type = CompressionTypePB::LZ4; + opts.storage_format = TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2; + _init_column_meta(opts.meta, 0, column, opts); std::unique_ptr writer; EXPECT_TRUE(ColumnWriter::create(opts, &column, file_writer.get(), &writer).ok()); @@ -918,7 +920,9 @@ TEST_F(VariantColumnWriterReaderTest, test_write_data_normal) { bool external_segment_meta_used_default = rand() % 2 == 0; std::cout << "external_segment_meta_used_default: " << external_segment_meta_used_default << std::endl; - _tablet_schema->set_external_segment_meta_used_default(external_segment_meta_used_default); + _tablet_schema->set_storage_format(external_segment_meta_used_default + ? TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V3 + : TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2); tablet_meta->_tablet_id = 10000; _tablet = std::make_shared(*_engine_ref, tablet_meta, _data_dir.get()); @@ -944,7 +948,9 @@ TEST_F(VariantColumnWriterReaderTest, test_write_data_normal) { opts.rowset_ctx = &rowset_ctx; opts.rowset_ctx->tablet_schema = _tablet_schema; TabletColumn column = _tablet_schema->column(0); - _init_column_meta(opts.meta, 0, column, CompressionTypePB::LZ4); + opts.compression_type = CompressionTypePB::LZ4; + opts.storage_format = TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2; + _init_column_meta(opts.meta, 0, column, opts); std::unique_ptr writer; EXPECT_TRUE(ColumnWriter::create(opts, &column, file_writer.get(), &writer).ok()); @@ -1475,7 +1481,9 @@ TEST_F(VariantColumnWriterReaderTest, test_write_doc_and_read_hierarchical_doc) // 2. create tablet TabletMetaSharedPtr tablet_meta(new TabletMeta(_tablet_schema)); bool external_segment_meta_used_default = false; - _tablet_schema->set_external_segment_meta_used_default(external_segment_meta_used_default); + _tablet_schema->set_storage_format(external_segment_meta_used_default + ? TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V3 + : TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2); tablet_meta->_tablet_id = 31000; _tablet = std::make_shared(*_engine_ref, tablet_meta, _data_dir.get()); EXPECT_TRUE(_tablet->init().ok()); @@ -1500,7 +1508,9 @@ TEST_F(VariantColumnWriterReaderTest, test_write_doc_and_read_hierarchical_doc) opts.rowset_ctx = &rowset_ctx; opts.rowset_ctx->tablet_schema = _tablet_schema; TabletColumn parent_column = _tablet_schema->column(0); - _init_column_meta(opts.meta, 0, parent_column, CompressionTypePB::LZ4); + opts.compression_type = CompressionTypePB::LZ4; + opts.storage_format = TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2; + _init_column_meta(opts.meta, 0, parent_column, opts); std::unique_ptr writer; EXPECT_TRUE(ColumnWriter::create(opts, &parent_column, file_writer.get(), &writer).ok()); @@ -1610,7 +1620,7 @@ TEST_F(VariantColumnWriterReaderTest, _tablet_schema->init_from_pb(schema_pb); TabletMetaSharedPtr tablet_meta(new TabletMeta(_tablet_schema)); - _tablet_schema->set_external_segment_meta_used_default(false); + _tablet_schema->set_storage_format(TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2); tablet_meta->_tablet_id = 31002; _tablet = std::make_shared(*_engine_ref, tablet_meta, _data_dir.get()); EXPECT_TRUE(_tablet->init().ok()); @@ -1633,7 +1643,9 @@ TEST_F(VariantColumnWriterReaderTest, opts.rowset_ctx = &rowset_ctx; opts.rowset_ctx->tablet_schema = _tablet_schema; TabletColumn parent_column = _tablet_schema->column(0); - _init_column_meta(opts.meta, 0, parent_column, CompressionTypePB::LZ4); + opts.compression_type = CompressionTypePB::LZ4; + opts.storage_format = TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2; + _init_column_meta(opts.meta, 0, parent_column, opts); std::unique_ptr writer; EXPECT_TRUE(ColumnWriter::create(opts, &parent_column, file_writer.get(), &writer).ok()); @@ -1721,6 +1733,116 @@ TEST_F(VariantColumnWriterReaderTest, EXPECT_TRUE(io::global_local_filesystem()->delete_directory(_tablet->tablet_path()).ok()); } +// Regression: materialized subcolumns in V3 doc-mode tablets must inherit the parent's +// storage_format and resolve V3 default encodings (e.g. integer family = PLAIN, not BIT_SHUFFLE). +// Without propagating base_opts.storage_format into the per-subcolumn ColumnWriterOptions, +// `_init_column_meta` falls back to the V2 default map and writes V2 encodings even for V3 +// tablets, defeating the storage-format-based encoding policy. +TEST_F(VariantColumnWriterReaderTest, test_write_doc_materialized_v3_uses_v3_encoding) { + constexpr int kRows = 200; + constexpr int kDocBuckets = 2; + + TabletSchemaPB schema_pb; + schema_pb.set_keys_type(KeysType::DUP_KEYS); + construct_column(schema_pb.add_column(), 1, "VARIANT", "V1", 3, false, false, + /*variant_sparse_hash_shard_count=*/0, + /*variant_enable_doc_mode=*/true, + /*variant_doc_materialization_min_rows=*/0, + /*variant_doc_hash_shard_count=*/kDocBuckets); + _tablet_schema = std::make_shared(); + _tablet_schema->init_from_pb(schema_pb); + + TabletMetaSharedPtr tablet_meta(new TabletMeta(_tablet_schema)); + _tablet_schema->set_storage_format(TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V3); + tablet_meta->_tablet_id = 31003; + _tablet = std::make_shared(*_engine_ref, tablet_meta, _data_dir.get()); + EXPECT_TRUE(_tablet->init().ok()); + EXPECT_TRUE(io::global_local_filesystem()->delete_directory(_tablet->tablet_path()).ok()); + EXPECT_TRUE(io::global_local_filesystem()->create_directory(_tablet->tablet_path()).ok()); + + io::FileWriterPtr file_writer; + auto file_path = local_segment_path(_tablet->tablet_path(), "0", 0); + auto st = io::global_local_filesystem()->create_file(file_path, &file_writer); + EXPECT_TRUE(st.ok()) << st.msg(); + + SegmentFooterPB footer; + ColumnWriterOptions opts; + opts.meta = footer.add_columns(); + opts.compression_type = CompressionTypePB::LZ4; + opts.file_writer = file_writer.get(); + opts.footer = &footer; + RowsetWriterContext rowset_ctx; + rowset_ctx.write_type = DataWriteType::TYPE_DIRECT; + opts.rowset_ctx = &rowset_ctx; + opts.rowset_ctx->tablet_schema = _tablet_schema; + TabletColumn parent_column = _tablet_schema->column(0); + opts.storage_format = TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V3; + _init_column_meta(opts.meta, 0, parent_column, opts); + + std::unique_ptr writer; + EXPECT_TRUE(ColumnWriter::create(opts, &parent_column, file_writer.get(), &writer).ok()); + EXPECT_TRUE(writer->init().ok()); + + auto olap_data_convertor = std::make_unique(); + auto block = _tablet_schema->create_block(); + auto column_object = (*std::move(block.get_by_position(0).column)).mutate(); + std::unordered_map inserted_jsonstr; + fill_variant_column_with_doc_value_only(column_object, kRows, &inserted_jsonstr); + olap_data_convertor->add_column_data_convertor(parent_column); + olap_data_convertor->set_source_content(&block, 0, kRows); + auto [result, accessor] = olap_data_convertor->convert_column_data(0); + EXPECT_TRUE(result.ok()); + EXPECT_TRUE(accessor != nullptr); + EXPECT_TRUE(writer->append(accessor->get_nullmap(), accessor->get_data(), kRows).ok()); + EXPECT_TRUE(writer->finish().ok()); + EXPECT_TRUE(writer->write_data().ok()); + EXPECT_TRUE(writer->write_ordinal_index().ok()); + EXPECT_TRUE(file_writer->close().ok()); + + // Materialization must have produced extra subcolumns beyond the doc-bucket columns. + EXPECT_GT(footer.columns_size(), 1 + kDocBuckets) << "no subcolumns were materialized"; + + // Locate materialized subcolumns. Doc bucket columns have DOC_VALUE_COLUMN_PATH in their + // path; everything else (other than the root variant at index 0) is a materialized subcolumn. + int integer_subcolumns_checked = 0; + int string_subcolumns_checked = 0; + for (int i = 1; i < footer.columns_size(); ++i) { + const auto& col = footer.columns(i); + if (!col.has_column_path_info()) continue; + PathInData path; + path.from_protobuf(col.column_path_info()); + std::string rel = path.copy_pop_front().get_path(); + if (rel.find(DOC_VALUE_COLUMN_PATH) != std::string::npos) continue; + const auto field_type = static_cast(col.type()); + switch (field_type) { + case FieldType::OLAP_FIELD_TYPE_TINYINT: + case FieldType::OLAP_FIELD_TYPE_SMALLINT: + case FieldType::OLAP_FIELD_TYPE_INT: + case FieldType::OLAP_FIELD_TYPE_BIGINT: + case FieldType::OLAP_FIELD_TYPE_LARGEINT: + EXPECT_EQ(col.encoding(), EncodingTypePB::PLAIN_ENCODING) + << "V3 integer subcolumn '" << rel << "' got " + << EncodingTypePB_Name(col.encoding()) << " instead of PLAIN_ENCODING"; + ++integer_subcolumns_checked; + break; + case FieldType::OLAP_FIELD_TYPE_CHAR: + case FieldType::OLAP_FIELD_TYPE_VARCHAR: + case FieldType::OLAP_FIELD_TYPE_STRING: + EXPECT_EQ(col.encoding(), EncodingTypePB::DICT_ENCODING) + << "V3 string subcolumn '" << rel << "' got " + << EncodingTypePB_Name(col.encoding()) << " instead of DICT_ENCODING"; + ++string_subcolumns_checked; + break; + default: + break; + } + } + EXPECT_GT(integer_subcolumns_checked + string_subcolumns_checked, 0) + << "no scalar materialized subcolumns were found to verify"; + + EXPECT_TRUE(io::global_local_filesystem()->delete_directory(_tablet->tablet_path()).ok()); +} + TEST_F(VariantColumnWriterReaderTest, test_read_doc_compact_from_doc_value_bucket) { constexpr int kRows = 200; constexpr int kDocBuckets = 4; @@ -1738,7 +1860,7 @@ TEST_F(VariantColumnWriterReaderTest, test_read_doc_compact_from_doc_value_bucke // 2. create tablet TabletMetaSharedPtr tablet_meta(new TabletMeta(_tablet_schema)); - _tablet_schema->set_external_segment_meta_used_default(false); + _tablet_schema->set_storage_format(TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2); tablet_meta->_tablet_id = 32000; _tablet = std::make_shared(*_engine_ref, tablet_meta, _data_dir.get()); EXPECT_TRUE(_tablet->init().ok()); @@ -1762,7 +1884,9 @@ TEST_F(VariantColumnWriterReaderTest, test_read_doc_compact_from_doc_value_bucke opts.rowset_ctx = &rowset_ctx; opts.rowset_ctx->tablet_schema = _tablet_schema; TabletColumn parent_column = _tablet_schema->column(0); - _init_column_meta(opts.meta, 0, parent_column, CompressionTypePB::LZ4); + opts.compression_type = CompressionTypePB::LZ4; + opts.storage_format = TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2; + _init_column_meta(opts.meta, 0, parent_column, opts); std::unique_ptr writer; EXPECT_TRUE(ColumnWriter::create(opts, &parent_column, file_writer.get(), &writer).ok()); @@ -1888,7 +2012,7 @@ TEST_F(VariantColumnWriterReaderTest, test_write_doc_compact_writer_and_read_doc // 2. create tablet TabletMetaSharedPtr tablet_meta(new TabletMeta(_tablet_schema)); - _tablet_schema->set_external_segment_meta_used_default(false); + _tablet_schema->set_storage_format(TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2); tablet_meta->_tablet_id = 33000; _tablet = std::make_shared(*_engine_ref, tablet_meta, _data_dir.get()); EXPECT_TRUE(_tablet->init().ok()); @@ -1914,7 +2038,9 @@ TEST_F(VariantColumnWriterReaderTest, test_write_doc_compact_writer_and_read_doc root_opts.file_writer = file_writer.get(); root_opts.footer = &footer; root_opts.rowset_ctx = &rowset_ctx; - _init_column_meta(root_opts.meta, 0, parent_column, CompressionTypePB::LZ4); + root_opts.compression_type = CompressionTypePB::LZ4; + root_opts.storage_format = TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2; + _init_column_meta(root_opts.meta, 0, parent_column, root_opts); std::unique_ptr root_writer; EXPECT_TRUE( @@ -1924,7 +2050,9 @@ TEST_F(VariantColumnWriterReaderTest, test_write_doc_compact_writer_and_read_doc TabletColumn extracted_doc_bucket_col = _tablet_schema->column(1); ColumnWriterOptions doc_compact_opts = root_opts; doc_compact_opts.meta = footer.add_columns(); - _init_column_meta(doc_compact_opts.meta, 0, extracted_doc_bucket_col, CompressionTypePB::LZ4); + doc_compact_opts.compression_type = CompressionTypePB::LZ4; + doc_compact_opts.storage_format = TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2; + _init_column_meta(doc_compact_opts.meta, 0, extracted_doc_bucket_col, doc_compact_opts); std::unique_ptr doc_compact_writer; EXPECT_TRUE(ColumnWriter::create(doc_compact_opts, &extracted_doc_bucket_col, file_writer.get(), &doc_compact_writer) @@ -2086,7 +2214,7 @@ TEST_F(VariantColumnWriterReaderTest, test_doc_compact_sparse_write_array_gap) { _tablet_schema->append_column(extracted_doc_bucket); TabletMetaSharedPtr tablet_meta(new TabletMeta(_tablet_schema)); - _tablet_schema->set_external_segment_meta_used_default(false); + _tablet_schema->set_storage_format(TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2); tablet_meta->_tablet_id = 33001; _tablet = std::make_shared(*_engine_ref, tablet_meta, _data_dir.get()); EXPECT_TRUE(_tablet->init().ok()); @@ -2110,7 +2238,9 @@ TEST_F(VariantColumnWriterReaderTest, test_doc_compact_sparse_write_array_gap) { doc_compact_opts.file_writer = file_writer.get(); doc_compact_opts.footer = &footer; doc_compact_opts.rowset_ctx = &rowset_ctx; - _init_column_meta(doc_compact_opts.meta, 0, extracted_doc_bucket_col, CompressionTypePB::LZ4); + doc_compact_opts.compression_type = CompressionTypePB::LZ4; + doc_compact_opts.storage_format = TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2; + _init_column_meta(doc_compact_opts.meta, 0, extracted_doc_bucket_col, doc_compact_opts); std::unique_ptr doc_compact_writer; EXPECT_TRUE(ColumnWriter::create(doc_compact_opts, &extracted_doc_bucket_col, file_writer.get(), @@ -2188,7 +2318,7 @@ TEST_F(VariantColumnWriterReaderTest, test_write_doc_sparse_write_array_gap_and_ _tablet_schema->init_from_pb(schema_pb); TabletMetaSharedPtr tablet_meta(new TabletMeta(_tablet_schema)); - _tablet_schema->set_external_segment_meta_used_default(false); + _tablet_schema->set_storage_format(TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2); tablet_meta->_tablet_id = 33002; _tablet = std::make_shared(*_engine_ref, tablet_meta, _data_dir.get()); EXPECT_TRUE(_tablet->init().ok()); @@ -2212,7 +2342,9 @@ TEST_F(VariantColumnWriterReaderTest, test_write_doc_sparse_write_array_gap_and_ opts.file_writer = file_writer.get(); opts.footer = &footer; opts.rowset_ctx = &rowset_ctx; - _init_column_meta(opts.meta, 0, parent_column, CompressionTypePB::LZ4); + opts.compression_type = CompressionTypePB::LZ4; + opts.storage_format = TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2; + _init_column_meta(opts.meta, 0, parent_column, opts); std::unique_ptr writer; EXPECT_TRUE(ColumnWriter::create(opts, &parent_column, file_writer.get(), &writer).ok()); @@ -2355,7 +2487,7 @@ TEST_F(VariantColumnWriterReaderTest, test_storage_parse_kv_write_materialized_a opts.file_writer = file_writer.get(); opts.footer = &footer; opts.rowset_ctx = &rowset_ctx; - _init_column_meta(opts.meta, 0, parent_column, CompressionTypePB::LZ4); + _init_column_meta(opts.meta, 0, parent_column, opts); std::unique_ptr writer; ASSERT_TRUE(ColumnWriter::create(opts, &parent_column, file_writer.get(), &writer).ok()); @@ -2882,7 +3014,7 @@ TEST_F(VariantColumnWriterReaderTest, opts.file_writer = file_writer.get(); opts.footer = &footer; opts.rowset_ctx = &rowset_ctx; - _init_column_meta(opts.meta, 0, parent_column, CompressionTypePB::LZ4); + _init_column_meta(opts.meta, 0, parent_column, opts); std::unique_ptr writer; ASSERT_TRUE(ColumnWriter::create(opts, &parent_column, file_writer.get(), &writer).ok()); @@ -2988,7 +3120,9 @@ TEST_F(VariantColumnWriterReaderTest, test_write_data_advanced) { bool external_segment_meta_used_default = rand() % 2 == 0; std::cout << "external_segment_meta_used_default: " << external_segment_meta_used_default << std::endl; - _tablet_schema->set_external_segment_meta_used_default(external_segment_meta_used_default); + _tablet_schema->set_storage_format(external_segment_meta_used_default + ? TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V3 + : TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2); tablet_meta->_tablet_id = 10000; _tablet = std::make_shared(*_engine_ref, tablet_meta, _data_dir.get()); EXPECT_TRUE(_tablet->init().ok()); @@ -3013,7 +3147,9 @@ TEST_F(VariantColumnWriterReaderTest, test_write_data_advanced) { opts.rowset_ctx = &rowset_ctx; opts.rowset_ctx->tablet_schema = _tablet_schema; TabletColumn column = _tablet_schema->column(0); - _init_column_meta(opts.meta, 0, column, CompressionTypePB::LZ4); + opts.compression_type = CompressionTypePB::LZ4; + opts.storage_format = TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2; + _init_column_meta(opts.meta, 0, column, opts); std::unique_ptr writer; EXPECT_TRUE(ColumnWriter::create(opts, &column, file_writer.get(), &writer).ok()); @@ -3189,7 +3325,9 @@ TEST_F(VariantColumnWriterReaderTest, test_write_sub_index) { bool external_segment_meta_used_default = rand() % 2 == 0; std::cout << "external_segment_meta_used_default: " << external_segment_meta_used_default << std::endl; - _tablet_schema->set_external_segment_meta_used_default(external_segment_meta_used_default); + _tablet_schema->set_storage_format(external_segment_meta_used_default + ? TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V3 + : TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2); tablet_meta->_tablet_id = 10000; _tablet = std::make_shared(*_engine_ref, tablet_meta, _data_dir.get()); EXPECT_TRUE(_tablet->init().ok()); @@ -3214,7 +3352,9 @@ TEST_F(VariantColumnWriterReaderTest, test_write_sub_index) { opts.rowset_ctx = &rowset_ctx; opts.rowset_ctx->tablet_schema = _tablet_schema; TabletColumn column = _tablet_schema->column(0); - _init_column_meta(opts.meta, 0, column, CompressionTypePB::LZ4); + opts.compression_type = CompressionTypePB::LZ4; + opts.storage_format = TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2; + _init_column_meta(opts.meta, 0, column, opts); std::unique_ptr writer; EXPECT_TRUE(ColumnWriter::create(opts, &column, file_writer.get(), &writer).ok()); @@ -3260,7 +3400,7 @@ TEST_F(VariantColumnWriterReaderTest, test_find_subcolumn_tablet_indexes_inherit _tablet_schema->init_from_pb(schema_pb); TabletMetaSharedPtr tablet_meta(new TabletMeta(_tablet_schema)); - _tablet_schema->set_external_segment_meta_used_default(false); + _tablet_schema->set_storage_format(TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2); tablet_meta->_tablet_id = 10001; _tablet = std::make_shared(*_engine_ref, tablet_meta, _data_dir.get()); ASSERT_TRUE(_tablet->init().ok()); @@ -3283,7 +3423,9 @@ TEST_F(VariantColumnWriterReaderTest, test_find_subcolumn_tablet_indexes_inherit rowset_ctx.tablet_schema = _tablet_schema; opts.rowset_ctx = &rowset_ctx; TabletColumn column = _tablet_schema->column(0); - _init_column_meta(opts.meta, 0, column, CompressionTypePB::LZ4); + opts.compression_type = CompressionTypePB::LZ4; + opts.storage_format = TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2; + _init_column_meta(opts.meta, 0, column, opts); std::unique_ptr writer; ASSERT_TRUE(ColumnWriter::create(opts, &column, file_writer.get(), &writer).ok()); @@ -3347,7 +3489,7 @@ TEST_F(VariantColumnWriterReaderTest, test_find_subcolumn_tablet_indexes_branch_ _tablet_schema->init_from_pb(schema_pb); TabletMetaSharedPtr tablet_meta(new TabletMeta(_tablet_schema)); - _tablet_schema->set_external_segment_meta_used_default(false); + _tablet_schema->set_storage_format(TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2); tablet_meta->_tablet_id = 10002; _tablet = std::make_shared(*_engine_ref, tablet_meta, _data_dir.get()); ASSERT_TRUE(_tablet->init().ok()); @@ -3370,7 +3512,9 @@ TEST_F(VariantColumnWriterReaderTest, test_find_subcolumn_tablet_indexes_branch_ rowset_ctx.tablet_schema = _tablet_schema; opts.rowset_ctx = &rowset_ctx; TabletColumn root_column = _tablet_schema->column(0); - _init_column_meta(opts.meta, 0, root_column, CompressionTypePB::LZ4); + opts.compression_type = CompressionTypePB::LZ4; + opts.storage_format = TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2; + _init_column_meta(opts.meta, 0, root_column, opts); std::unique_ptr writer; ASSERT_TRUE(ColumnWriter::create(opts, &root_column, file_writer.get(), &writer).ok()); @@ -3533,7 +3677,9 @@ TEST_F(VariantColumnWriterReaderTest, test_write_data_nullable) { bool external_segment_meta_used_default = rand() % 2 == 0; std::cout << "external_segment_meta_used_default: " << external_segment_meta_used_default << std::endl; - _tablet_schema->set_external_segment_meta_used_default(external_segment_meta_used_default); + _tablet_schema->set_storage_format(external_segment_meta_used_default + ? TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V3 + : TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2); tablet_meta->_tablet_id = 10000; _tablet = std::make_shared(*_engine_ref, tablet_meta, _data_dir.get()); EXPECT_TRUE(_tablet->init().ok()); @@ -3559,7 +3705,9 @@ TEST_F(VariantColumnWriterReaderTest, test_write_data_nullable) { opts.rowset_ctx->tablet_schema = _tablet_schema; // nullable variant column TabletColumn column = _tablet_schema->column(0); - _init_column_meta(opts.meta, 0, column, CompressionTypePB::LZ4); + opts.compression_type = CompressionTypePB::LZ4; + opts.storage_format = TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2; + _init_column_meta(opts.meta, 0, column, opts); std::unique_ptr writer; EXPECT_TRUE(ColumnWriter::create(opts, &column, file_writer.get(), &writer).ok()); @@ -3686,7 +3834,9 @@ TEST_F(VariantColumnWriterReaderTest, test_write_data_nullable_without_finalize) bool external_segment_meta_used_default = rand() % 2 == 0; std::cout << "external_segment_meta_used_default: " << external_segment_meta_used_default << std::endl; - _tablet_schema->set_external_segment_meta_used_default(external_segment_meta_used_default); + _tablet_schema->set_storage_format(external_segment_meta_used_default + ? TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V3 + : TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2); tablet_meta->_tablet_id = 10000; _tablet = std::make_shared(*_engine_ref, tablet_meta, _data_dir.get()); EXPECT_TRUE(_tablet->init().ok()); @@ -3712,7 +3862,9 @@ TEST_F(VariantColumnWriterReaderTest, test_write_data_nullable_without_finalize) opts.rowset_ctx->tablet_schema = _tablet_schema; // nullable variant column TabletColumn column = _tablet_schema->column(0); - _init_column_meta(opts.meta, 0, column, CompressionTypePB::LZ4); + opts.compression_type = CompressionTypePB::LZ4; + opts.storage_format = TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2; + _init_column_meta(opts.meta, 0, column, opts); std::unique_ptr writer; EXPECT_TRUE(ColumnWriter::create(opts, &column, file_writer.get(), &writer).ok()); @@ -3781,7 +3933,9 @@ TEST_F(VariantColumnWriterReaderTest, test_write_bm_with_finalize) { bool external_segment_meta_used_default = rand() % 2 == 0; std::cout << "external_segment_meta_used_default: " << external_segment_meta_used_default << std::endl; - _tablet_schema->set_external_segment_meta_used_default(external_segment_meta_used_default); + _tablet_schema->set_storage_format(external_segment_meta_used_default + ? TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V3 + : TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2); tablet_meta->_tablet_id = 10000; _tablet = std::make_shared(*_engine_ref, tablet_meta, _data_dir.get()); EXPECT_TRUE(_tablet->init().ok()); @@ -3807,7 +3961,9 @@ TEST_F(VariantColumnWriterReaderTest, test_write_bm_with_finalize) { opts.rowset_ctx->tablet_schema = _tablet_schema; // nullable variant column TabletColumn column = _tablet_schema->column(0); - _init_column_meta(opts.meta, 0, column, CompressionTypePB::LZ4); + opts.compression_type = CompressionTypePB::LZ4; + opts.storage_format = TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2; + _init_column_meta(opts.meta, 0, column, opts); std::unique_ptr writer; EXPECT_TRUE(ColumnWriter::create(opts, &column, file_writer.get(), &writer).ok()); @@ -3876,7 +4032,9 @@ TEST_F(VariantColumnWriterReaderTest, test_write_bf_with_finalize) { bool external_segment_meta_used_default = rand() % 2 == 0; std::cout << "external_segment_meta_used_default: " << external_segment_meta_used_default << std::endl; - _tablet_schema->set_external_segment_meta_used_default(external_segment_meta_used_default); + _tablet_schema->set_storage_format(external_segment_meta_used_default + ? TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V3 + : TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2); tablet_meta->_tablet_id = 10000; _tablet = std::make_shared(*_engine_ref, tablet_meta, _data_dir.get()); EXPECT_TRUE(_tablet->init().ok()); @@ -3902,7 +4060,9 @@ TEST_F(VariantColumnWriterReaderTest, test_write_bf_with_finalize) { opts.rowset_ctx->tablet_schema = _tablet_schema; // nullable variant column TabletColumn column = _tablet_schema->column(0); - _init_column_meta(opts.meta, 0, column, CompressionTypePB::LZ4); + opts.compression_type = CompressionTypePB::LZ4; + opts.storage_format = TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2; + _init_column_meta(opts.meta, 0, column, opts); std::unique_ptr writer; EXPECT_TRUE(ColumnWriter::create(opts, &column, file_writer.get(), &writer).ok()); @@ -3973,7 +4133,9 @@ TEST_F(VariantColumnWriterReaderTest, test_write_zm_with_finalize) { bool external_segment_meta_used_default = rand() % 2 == 0; std::cout << "external_segment_meta_used_default: " << external_segment_meta_used_default << std::endl; - _tablet_schema->set_external_segment_meta_used_default(external_segment_meta_used_default); + _tablet_schema->set_storage_format(external_segment_meta_used_default + ? TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V3 + : TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2); tablet_meta->_tablet_id = 10000; _tablet = std::make_shared(*_engine_ref, tablet_meta, _data_dir.get()); EXPECT_TRUE(_tablet->init().ok()); @@ -3999,7 +4161,9 @@ TEST_F(VariantColumnWriterReaderTest, test_write_zm_with_finalize) { opts.rowset_ctx->tablet_schema = _tablet_schema; // nullable variant column TabletColumn column = _tablet_schema->column(0); - _init_column_meta(opts.meta, 0, column, CompressionTypePB::LZ4); + opts.compression_type = CompressionTypePB::LZ4; + opts.storage_format = TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2; + _init_column_meta(opts.meta, 0, column, opts); std::unique_ptr writer; EXPECT_TRUE(ColumnWriter::create(opts, &column, file_writer.get(), &writer).ok()); @@ -4070,7 +4234,9 @@ TEST_F(VariantColumnWriterReaderTest, test_write_inverted_with_finalize) { bool external_segment_meta_used_default = rand() % 2 == 0; std::cout << "external_segment_meta_used_default: " << external_segment_meta_used_default << std::endl; - _tablet_schema->set_external_segment_meta_used_default(external_segment_meta_used_default); + _tablet_schema->set_storage_format(external_segment_meta_used_default + ? TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V3 + : TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2); tablet_meta->_tablet_id = 10000; _tablet = std::make_shared(*_engine_ref, tablet_meta, _data_dir.get()); EXPECT_TRUE(_tablet->init().ok()); @@ -4096,7 +4262,9 @@ TEST_F(VariantColumnWriterReaderTest, test_write_inverted_with_finalize) { opts.rowset_ctx->tablet_schema = _tablet_schema; // nullable variant column TabletColumn column = _tablet_schema->column(0); - _init_column_meta(opts.meta, 0, column, CompressionTypePB::LZ4); + opts.compression_type = CompressionTypePB::LZ4; + opts.storage_format = TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2; + _init_column_meta(opts.meta, 0, column, opts); std::unique_ptr writer; EXPECT_TRUE(ColumnWriter::create(opts, &column, file_writer.get(), &writer).ok()); @@ -4166,7 +4334,9 @@ TEST_F(VariantColumnWriterReaderTest, test_no_sub_in_sparse_column) { bool external_segment_meta_used_default = rand() % 2 == 0; std::cout << "external_segment_meta_used_default: " << external_segment_meta_used_default << std::endl; - _tablet_schema->set_external_segment_meta_used_default(external_segment_meta_used_default); + _tablet_schema->set_storage_format(external_segment_meta_used_default + ? TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V3 + : TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2); tablet_meta->_tablet_id = 10001; _tablet = std::make_shared(*_engine_ref, tablet_meta, _data_dir.get()); EXPECT_TRUE(_tablet->init().ok()); @@ -4191,7 +4361,9 @@ TEST_F(VariantColumnWriterReaderTest, test_no_sub_in_sparse_column) { opts.rowset_ctx = &rowset_ctx; opts.rowset_ctx->tablet_schema = _tablet_schema; TabletColumn column = _tablet_schema->column(0); - _init_column_meta(opts.meta, 0, column, CompressionTypePB::LZ4); + opts.compression_type = CompressionTypePB::LZ4; + opts.storage_format = TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2; + _init_column_meta(opts.meta, 0, column, opts); std::unique_ptr writer; EXPECT_TRUE(ColumnWriter::create(opts, &column, file_writer.get(), &writer).ok()); @@ -4299,7 +4471,9 @@ TEST_F(VariantColumnWriterReaderTest, test_prefix_in_sub_and_sparse) { bool external_segment_meta_used_default = rand() % 2 == 0; std::cout << "external_segment_meta_used_default: " << external_segment_meta_used_default << std::endl; - _tablet_schema->set_external_segment_meta_used_default(external_segment_meta_used_default); + _tablet_schema->set_storage_format(external_segment_meta_used_default + ? TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V3 + : TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2); tablet_meta->_tablet_id = 10001; _tablet = std::make_shared(*_engine_ref, tablet_meta, _data_dir.get()); EXPECT_TRUE(_tablet->init().ok()); @@ -4324,7 +4498,9 @@ TEST_F(VariantColumnWriterReaderTest, test_prefix_in_sub_and_sparse) { opts.rowset_ctx = &rowset_ctx; opts.rowset_ctx->tablet_schema = _tablet_schema; TabletColumn column = _tablet_schema->column(0); - _init_column_meta(opts.meta, 0, column, CompressionTypePB::LZ4); + opts.compression_type = CompressionTypePB::LZ4; + opts.storage_format = TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2; + _init_column_meta(opts.meta, 0, column, opts); std::unique_ptr writer; EXPECT_TRUE(ColumnWriter::create(opts, &column, file_writer.get(), &writer).ok()); @@ -4446,7 +4622,9 @@ void test_write_variant_column(StorageEngine* _engine_ref, std::string _absolute bool external_segment_meta_used_default = rand() % 2 == 0; std::cout << "external_segment_meta_used_default: " << external_segment_meta_used_default << std::endl; - _tablet_schema->set_external_segment_meta_used_default(external_segment_meta_used_default); + _tablet_schema->set_storage_format(external_segment_meta_used_default + ? TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V3 + : TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2); tablet_meta->_tablet_id = 10000; EXPECT_TRUE(io::global_local_filesystem()->delete_directory(_absolute_dir).ok()); EXPECT_TRUE(io::global_local_filesystem()->create_directory(_absolute_dir).ok()); @@ -4477,7 +4655,9 @@ void test_write_variant_column(StorageEngine* _engine_ref, std::string _absolute opts.rowset_ctx = &rowset_ctx; opts.rowset_ctx->tablet_schema = _tablet_schema; TabletColumn tablet_column = _tablet_schema->column(0); - _init_column_meta(opts.meta, 0, tablet_column, CompressionTypePB::LZ4); + opts.compression_type = CompressionTypePB::LZ4; + opts.storage_format = TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2; + _init_column_meta(opts.meta, 0, tablet_column, opts); std::unique_ptr writer; EXPECT_TRUE(ColumnWriter::create(opts, &tablet_column, file_writer.get(), &writer).ok()); @@ -4864,7 +5044,9 @@ TEST_F(VariantColumnWriterReaderTest, test_read_with_checksum) { bool external_segment_meta_used_default = rand() % 2 == 0; std::cout << "external_segment_meta_used_default: " << external_segment_meta_used_default << std::endl; - _tablet_schema->set_external_segment_meta_used_default(external_segment_meta_used_default); + _tablet_schema->set_storage_format(external_segment_meta_used_default + ? TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V3 + : TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2); tablet_meta->_tablet_id = 10000; _tablet = std::make_shared(*_engine_ref, tablet_meta, _data_dir.get()); @@ -4890,7 +5072,9 @@ TEST_F(VariantColumnWriterReaderTest, test_read_with_checksum) { opts.rowset_ctx = &rowset_ctx; opts.rowset_ctx->tablet_schema = _tablet_schema; TabletColumn column = _tablet_schema->column(0); - _init_column_meta(opts.meta, 0, column, CompressionTypePB::LZ4); + opts.compression_type = CompressionTypePB::LZ4; + opts.storage_format = TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2; + _init_column_meta(opts.meta, 0, column, opts); std::unique_ptr writer; EXPECT_TRUE(ColumnWriter::create(opts, &column, file_writer.get(), &writer).ok()); @@ -5019,7 +5203,9 @@ TEST_F(VariantColumnWriterReaderTest, test_concurrent_load_external_meta_and_get // VariantColumnReader builds a VariantExternalMetaReader. TabletMetaSharedPtr tablet_meta(new TabletMeta(_tablet_schema)); bool external_segment_meta_used_default = true; - _tablet_schema->set_external_segment_meta_used_default(external_segment_meta_used_default); + _tablet_schema->set_storage_format(external_segment_meta_used_default + ? TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V3 + : TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2); tablet_meta->_tablet_id = 20000; _tablet = std::make_shared(*_engine_ref, tablet_meta, _data_dir.get()); @@ -5045,7 +5231,9 @@ TEST_F(VariantColumnWriterReaderTest, test_concurrent_load_external_meta_and_get opts.rowset_ctx = &rowset_ctx; opts.rowset_ctx->tablet_schema = _tablet_schema; TabletColumn column = _tablet_schema->column(0); - _init_column_meta(opts.meta, 0, column, CompressionTypePB::LZ4); + opts.compression_type = CompressionTypePB::LZ4; + opts.storage_format = TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2; + _init_column_meta(opts.meta, 0, column, opts); std::unique_ptr writer; EXPECT_TRUE(ColumnWriter::create(opts, &column, file_writer.get(), &writer).ok()); @@ -5180,7 +5368,9 @@ TEST_F(VariantColumnWriterReaderTest, opts.rowset_ctx = &rowset_ctx; TabletColumn column = _tablet_schema->column(0); - _init_column_meta(opts.meta, 0, column, CompressionTypePB::LZ4); + opts.compression_type = CompressionTypePB::LZ4; + opts.storage_format = TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2; + _init_column_meta(opts.meta, 0, column, opts); std::unique_ptr writer; ASSERT_TRUE(ColumnWriter::create(opts, &column, file_writer.get(), &writer).ok()); diff --git a/be/test/storage/tablet/tablet_schema_test.cpp b/be/test/storage/tablet/tablet_schema_test.cpp index 9753bad2f67316..e2b8424e8d602d 100644 --- a/be/test/storage/tablet/tablet_schema_test.cpp +++ b/be/test/storage/tablet/tablet_schema_test.cpp @@ -864,4 +864,70 @@ TEST_F(TabletSchemaTest, test_tablet_schema_get_index) { EXPECT_EQ(14002, ann_col_ids[0]); } +// Rolling-upgrade compat: a TabletSchemaPB persisted by an old BE has the three +// legacy V3-flavor flags but no storage_format. init_from_pb must derive V3. +TEST_F(TabletSchemaTest, init_from_pb_legacy_flags_derive_v3) { + TabletSchemaPB pb; + pb.set_keys_type(DUP_KEYS); + pb.set_is_external_segment_column_meta_used(true); + pb.set_integer_type_default_use_plain_encoding(true); + pb.set_binary_plain_encoding_default_impl(BinaryPlainEncodingTypePB::BINARY_PLAIN_ENCODING_V2); + // storage_format is intentionally not set + ASSERT_FALSE(pb.has_storage_format()); + + TabletSchema schema; + schema.init_from_pb(pb); + EXPECT_EQ(TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V3, schema.storage_format()); +} + +// PB with neither storage_format nor any of the legacy flags falls back to V2. +TEST_F(TabletSchemaTest, init_from_pb_no_flags_defaults_v2) { + TabletSchemaPB pb; + pb.set_keys_type(DUP_KEYS); + ASSERT_FALSE(pb.has_storage_format()); + + TabletSchema schema; + schema.init_from_pb(pb); + EXPECT_EQ(TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2, schema.storage_format()); +} + +// Rolling-downgrade compat: a V3 TabletSchema must redundantly emit the three +// legacy flags so an old BE rolled back from a new one can still recognize V3. +TEST_F(TabletSchemaTest, to_schema_pb_v3_emits_legacy_flags) { + TabletSchemaPB in; + in.set_keys_type(DUP_KEYS); + in.set_storage_format(TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V3); + + TabletSchema schema; + schema.init_from_pb(in); + ASSERT_EQ(TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V3, schema.storage_format()); + + TabletSchemaPB out; + schema.to_schema_pb(&out); + EXPECT_EQ(TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V3, out.storage_format()); + EXPECT_TRUE(out.is_external_segment_column_meta_used()); + EXPECT_TRUE(out.integer_type_default_use_plain_encoding()); + EXPECT_EQ(BinaryPlainEncodingTypePB::BINARY_PLAIN_ENCODING_V2, + out.binary_plain_encoding_default_impl()); +} + +// V2 schemas must NOT emit the V3-flavor legacy flags. +TEST_F(TabletSchemaTest, to_schema_pb_v2_skips_legacy_flags) { + TabletSchemaPB in; + in.set_keys_type(DUP_KEYS); + in.set_storage_format(TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2); + + TabletSchema schema; + schema.init_from_pb(in); + ASSERT_EQ(TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2, schema.storage_format()); + + TabletSchemaPB out; + schema.to_schema_pb(&out); + EXPECT_EQ(TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2, out.storage_format()); + EXPECT_FALSE(out.is_external_segment_column_meta_used()); + EXPECT_FALSE(out.integer_type_default_use_plain_encoding()); + EXPECT_NE(BinaryPlainEncodingTypePB::BINARY_PLAIN_ENCODING_V2, + out.binary_plain_encoding_default_impl()); +} + } // namespace doris diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java index 0b2bd821459f96..c3dd64b3071a0f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java @@ -330,15 +330,20 @@ public OlapFile.TabletMetaCloudPB.Builder createTabletMetaBuilder(long tableId, break; } - // Enable external column meta layout when storage_format is V3 (Cloud mode). + // Persist the storage format directly on the schema so the BE doesn't have to + // derive it from the three legacy flags below on the way back from MS. The flags + // are still written for backward-compat with BEs that predate the storage_format + // schema field; both representations agree on every V3 tablet. switch (storageFormat) { case V3: + schemaBuilder.setStorageFormat(OlapFile.TabletStorageFormatPB.TABLET_STORAGE_FORMAT_V3); schemaBuilder.setIsExternalSegmentColumnMetaUsed(true); schemaBuilder.setIntegerTypeDefaultUsePlainEncoding(true); schemaBuilder.setBinaryPlainEncodingDefaultImpl( OlapFile.BinaryPlainEncodingTypePB.BINARY_PLAIN_ENCODING_V2); break; default: + schemaBuilder.setStorageFormat(OlapFile.TabletStorageFormatPB.TABLET_STORAGE_FORMAT_V2); break; } diff --git a/gensrc/proto/olap_file.proto b/gensrc/proto/olap_file.proto index be6715333ccb9d..498f0238efa303 100644 --- a/gensrc/proto/olap_file.proto +++ b/gensrc/proto/olap_file.proto @@ -427,6 +427,15 @@ enum InvertedIndexStorageFormatPB { V3 = 2; } +// Tablet-level storage format. Values match TStorageFormat (Thrift) integer values so +// the C++ side can cast between TStorageFormat::type and this enum 1:1. +enum TabletStorageFormatPB { + TABLET_STORAGE_FORMAT_DEFAULT = 0; + TABLET_STORAGE_FORMAT_V1 = 1; + TABLET_STORAGE_FORMAT_V2 = 2; + TABLET_STORAGE_FORMAT_V3 = 3; +} + message TabletIndexPB { optional int64 index_id = 1; optional string index_name = 2; @@ -491,6 +500,14 @@ message TabletSchemaPB { optional bool integer_type_default_use_plain_encoding = 33; optional BinaryPlainEncodingTypePB binary_plain_encoding_default_impl = 34; + // Persisted TStorageFormat (V2=2, V3=3). This is the new authoritative field. + // The legacy is_external_segment_column_meta_used, integer_type_default_use_plain_encoding + // and binary_plain_encoding_default_impl fields above are still emitted redundantly when + // storage_format == V3 so that an old BE rolled back from a new one can still recover + // the format via the prior "any of those three implies V3" rule. New code should read + // storage_format; the fallback path lives in TabletSchema::init_from_pb. + optional TabletStorageFormatPB storage_format = 36; + optional SplitSchemaPB __split_schema = 1000; // A special field, DO NOT change it. } @@ -533,6 +550,9 @@ message TabletSchemaCloudPB { optional bool integer_type_default_use_plain_encoding = 34; optional BinaryPlainEncodingTypePB binary_plain_encoding_default_impl = 35; + // Persisted TStorageFormat (V2=2, V3=3); see TabletSchemaPB::storage_format. + optional TabletStorageFormatPB storage_format = 37; + optional bool is_dynamic_schema = 100 [default=false]; // FIXME(gavin): deprecate and remove in the future diff --git a/regression-test/suites/table_p0/test_storage_format_controls_encoding.groovy b/regression-test/suites/table_p0/test_storage_format_controls_encoding.groovy index d3aebdb5ec54cc..54d19117262e4d 100644 --- a/regression-test/suites/table_p0/test_storage_format_controls_encoding.groovy +++ b/regression-test/suites/table_p0/test_storage_format_controls_encoding.groovy @@ -38,14 +38,12 @@ suite('test_storage_format_controls_encoding') { logger.info("begin curl ${metaUrl}") def jsonMeta = Http.GET(metaUrl, true, false) + assert jsonMeta.schema.storage_format == "TABLET_STORAGE_FORMAT_V3" + // V3 tablets redundantly emit the three legacy V3-flavor flags so that an old BE + // rolled back from a new deployment can still recognize the tablet as V3. assert jsonMeta.schema.integer_type_default_use_plain_encoding == true assert jsonMeta.schema.binary_plain_encoding_default_impl == "BINARY_PLAIN_ENCODING_V2" - - def res = sql """show variables like "%use_v3_storage_format%";"""; - logger.info("session var use_v3_storage_format: ${res}") - if (res[0][1] == "true") return - tableName = "test_storage_format_controls_encoding2" sql """drop table if exists `${tableName}` force; """ @@ -53,12 +51,12 @@ suite('test_storage_format_controls_encoding') { CREATE TABLE ${tableName} (k int, v1 int, v2 varchar(100)) duplicate KEY(k) - DISTRIBUTED BY HASH (k) + DISTRIBUTED BY HASH (k) BUCKETS 1 PROPERTIES( "replication_num" = "1", "storage_format" = "V2"); """ - + sql "insert into ${tableName} values(1, 1, 'aaa');" sql "select * from ${tableName};" @@ -66,6 +64,9 @@ suite('test_storage_format_controls_encoding') { logger.info("begin curl ${metaUrl}") jsonMeta = Http.GET(metaUrl, true, false) - assert jsonMeta.schema.integer_type_default_use_plain_encoding == false - assert jsonMeta.schema.binary_plain_encoding_default_impl == "BINARY_PLAIN_ENCODING_V1" + assert jsonMeta.schema.storage_format == "TABLET_STORAGE_FORMAT_V2" + // V2 tablets intentionally omit the three legacy V3-flavor flags; absent fields are + // semantically equivalent to false / V1 for an old BE reading this PB. + assert (jsonMeta.schema.integer_type_default_use_plain_encoding ?: false) == false + assert (jsonMeta.schema.binary_plain_encoding_default_impl ?: "BINARY_PLAIN_ENCODING_V1") == "BINARY_PLAIN_ENCODING_V1" } \ No newline at end of file From 9df0fac594524679b61917a3c97e2ff80c5eea57 Mon Sep 17 00:00:00 2001 From: Chenyang Sun Date: Mon, 1 Jun 2026 10:00:52 +0800 Subject: [PATCH 4/5] [refactor](be) remove CHAR padding on read (#63291) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - https://github.com/apache/doris-website/pull/3759/ - Problem: The CHAR padding contract leaked from the storage layer into the compute / predicate layers — every scan stripped padding at the Block level, while predicates re-padded values to match the on-disk shape. Logic was spread out and wasted work on every read. - Fix: On-disk format unchanged. The convertor still pads CHAR to the schema length on write, but the strip is pushed down to the page pre-decoder — the page cache holds unpadded data. All shrink_* / pad_* code above the page cache (SegmentIterator, Block, RowCursor, predicates) is removed. - BloomFilter: BF probing is skipped (return true, fall back to scan) for CHAR predicates — the BF hashes padded bytes but predicate values are unpadded, so the probe would never match. Other indexes (ZoneMap / inverted / bitmap) are unaffected. Issue Number: close #xxx Related PR: #xxx Problem Summary: None - Test - [x] Regression test - [x] Unit Test - [ ] Manual test (add detailed scripts or steps below) - [ ] No need to test or manual test. Explain why: - [ ] This is a refactor/code format and no logic has been changed. - [ ] Previous test can cover this change. - [ ] No code files have been changed. - [ ] Other reason - Behavior changed: - [ ] No. - [ ] Yes. - Does this need documentation? - [ ] No. - [ ] Yes. - [ ] Confirm the release note - [ ] Confirm test cases - [ ] Confirm document - [ ] Add branch pick label Co-authored-by: Claude Opus 4.7 (1M context) (cherry picked from commit e0729979c710736f70c203e0724cb77e98667d81) --- be/src/core/block/block.cpp | 9 - be/src/core/block/block.h | 3 - be/src/core/column/column.h | 4 - be/src/core/column/column_array.cpp | 4 - be/src/core/column/column_array.h | 2 - be/src/core/column/column_dictionary.h | 27 +- be/src/core/column/column_map.cpp | 5 - be/src/core/column/column_map.h | 1 - be/src/core/column/column_nullable.cpp | 4 - be/src/core/column/column_nullable.h | 2 - be/src/core/column/column_string.cpp | 23 -- be/src/core/column/column_string.h | 2 - be/src/core/column/column_struct.cpp | 6 - be/src/core/column/column_struct.h | 2 - be/src/core/column/predicate_column.h | 6 +- .../data_type_string_serde.cpp | 23 +- be/src/exec/rowid_fetcher.cpp | 46 ---- be/src/exprs/bloom_filter_func_adaptor.h | 14 +- be/src/exprs/bloom_filter_func_impl.h | 15 +- be/src/service/point_query_executor.cpp | 5 - be/src/storage/delete/delete_handler.cpp | 20 +- .../storage/predicate/comparison_predicate.h | 7 +- be/src/storage/predicate/in_list_predicate.h | 30 +-- .../storage/predicate/like_column_predicate.h | 2 +- .../predicate_creator_comparison.cpp | 18 +- .../predicate_creator_in_list_in.cpp | 35 +-- .../predicate_creator_in_list_not_in.cpp | 33 +-- be/src/storage/row_cursor.cpp | 11 - be/src/storage/row_cursor.h | 5 - .../segment/binary_dict_page_pre_decoder.h | 30 ++- ...binary_plain_page_char_strip_pre_decoder.h | 98 +++++++ .../binary_plain_page_v2_pre_decoder.h | 248 ++++++++++-------- be/src/storage/segment/binary_prefix_page.h | 2 +- be/src/storage/segment/column_reader.cpp | 5 +- be/src/storage/segment/encoding_info.cpp | 27 +- be/src/storage/segment/page_io.cpp | 14 +- be/src/storage/segment/segment_iterator.cpp | 48 +--- be/src/storage/segment/segment_iterator.h | 8 +- be/test/core/block/block_test.cpp | 2 - be/test/core/column/column_array_test.cpp | 5 - be/test/core/column/column_string_test.cpp | 22 -- be/test/core/column/common_column_test.h | 37 --- be/test/exprs/bloom_filter_func_test.cpp | 6 +- be/test/storage/olap_type_test.cpp | 48 ++++ .../storage/segment/binary_dict_page_test.cpp | 6 +- .../segment/binary_plain_page_v2_test.cpp | 145 +++++++++- .../storage/segment/encoding_info_test.cpp | 43 ++- .../storage/segment/zone_map_index_test.cpp | 48 ++-- 48 files changed, 620 insertions(+), 586 deletions(-) create mode 100644 be/src/storage/segment/binary_plain_page_char_strip_pre_decoder.h diff --git a/be/src/core/block/block.cpp b/be/src/core/block/block.cpp index ad4a031a739c13..deff6ead8c7a0c 100644 --- a/be/src/core/block/block.cpp +++ b/be/src/core/block/block.cpp @@ -1068,15 +1068,6 @@ std::unique_ptr Block::create_same_struct_block(size_t size, bool is_rese return temp_block; } -void Block::shrink_char_type_column_suffix_zero(const std::vector& char_type_idx) { - for (auto idx : char_type_idx) { - if (idx < data.size()) { - auto& col_and_name = this->get_by_position(idx); - col_and_name.column->assume_mutable()->shrink_padding_chars(); - } - } -} - size_t MutableBlock::allocated_bytes() const { size_t res = 0; for (const auto& col : _columns) { diff --git a/be/src/core/block/block.h b/be/src/core/block/block.h index 3b946e9afa4ae2..16cb6148fb853b 100644 --- a/be/src/core/block/block.h +++ b/be/src/core/block/block.h @@ -349,9 +349,6 @@ class Block { return res; } - // for String type or Array type - void shrink_char_type_column_suffix_zero(const std::vector& char_type_idx); - void clear_column_mem_not_keep(const std::vector& column_keep_flags, bool need_keep_first); diff --git a/be/src/core/column/column.h b/be/src/core/column/column.h index eb63c501a17ee9..d9d9e2c2c8d69a 100644 --- a/be/src/core/column/column.h +++ b/be/src/core/column/column.h @@ -113,10 +113,6 @@ class IColumn : public COW { return nullptr; } - // shrink the end zeros for ColumnStr(also for who has it nested). so nest column will call it for all nested. - // for non-str col, will reach here(do nothing). only ColumnStr will really shrink itself. - virtual void shrink_padding_chars() {} - // Only used in ColumnVariant to handle lifecycle of variant. Other columns would do nothing. virtual void finalize() {} diff --git a/be/src/core/column/column_array.cpp b/be/src/core/column/column_array.cpp index 622418cd3ccf6d..a604d9d74bbd24 100644 --- a/be/src/core/column/column_array.cpp +++ b/be/src/core/column/column_array.cpp @@ -98,10 +98,6 @@ ColumnArray::ColumnArray(MutableColumnPtr&& nested_column) : data(std::move(nest offsets = ColumnOffsets::create(); } -void ColumnArray::shrink_padding_chars() { - data->shrink_padding_chars(); -} - std::string ColumnArray::get_name() const { return "Array(" + get_data().get_name() + ")"; } diff --git a/be/src/core/column/column_array.h b/be/src/core/column/column_array.h index 629544415ca2e4..29d25e338eb9ee 100644 --- a/be/src/core/column/column_array.h +++ b/be/src/core/column/column_array.h @@ -117,8 +117,6 @@ class ColumnArray final : public COWHelper { offsets->sanity_check(); } - void shrink_padding_chars() override; - /** On the index i there is an offset to the beginning of the i + 1 -th element. */ using ColumnOffsets = ColumnOffset64; diff --git a/be/src/core/column/column_dictionary.h b/be/src/core/column/column_dictionary.h index f572326b56f754..b70433f338c293 100644 --- a/be/src/core/column/column_dictionary.h +++ b/be/src/core/column/column_dictionary.h @@ -232,7 +232,7 @@ class ColumnDictI32 final : public COWHelper { _dict.initialize_hash_values_for_runtime_filter(); } - uint32_t get_hash_value(uint32_t idx) const { return _dict.get_hash_value(_codes[idx], _type); } + uint32_t get_hash_value(uint32_t idx) const { return _dict.get_hash_value(_codes[idx]); } template void find_codes(const HybridSetType* values, std::vector& selected) const { @@ -279,14 +279,6 @@ class ColumnDictI32 final : public COWHelper { inline const StringRef& get_value(value_type code) const { return _dict.get_value(code); } - inline StringRef get_shrink_value(value_type code) const { - StringRef result = _dict.get_value(code); - if (_type == FieldType::OLAP_FIELD_TYPE_CHAR) { - result.size = strnlen(result.data, result.size); - } - return result; - } - size_t dict_size() const { return _dict.size(); } std::string dict_debug_string() const { return _dict.debug_string(); } @@ -327,26 +319,13 @@ class ColumnDictI32 final : public COWHelper { } } - inline uint32_t get_hash_value(Int32 code, FieldType type) const { + inline uint32_t get_hash_value(Int32 code) const { if (_compute_hash_value_flags[code]) { return _hash_values[code]; } else { auto& sv = (*_dict_data)[code]; - // The char data is stored in the disk with the schema length, - // and zeros are filled if the length is insufficient - - // When reading data, use shrink_char_type_column_suffix_zero(_char_type_idx) - // Remove the suffix 0 - // When writing data, use the CharField::consume function to fill in the trailing 0. - - // For dictionary data of char type, sv.size is the schema length, - // so use strnlen to remove the 0 at the end to get the actual length. - size_t len = sv.size; - if (type == FieldType::OLAP_FIELD_TYPE_CHAR) { - len = strnlen(sv.data, sv.size); - } uint32_t hash_val = - crc32c::Extend(0, (const uint8_t*)sv.data, static_cast(len)); + crc32c::Extend(0, (const uint8_t*)sv.data, static_cast(sv.size)); _hash_values[code] = hash_val; _compute_hash_value_flags[code] = 1; return _hash_values[code]; diff --git a/be/src/core/column/column_map.cpp b/be/src/core/column/column_map.cpp index b251f3d496172c..7d119d4aa8f7bd 100644 --- a/be/src/core/column/column_map.cpp +++ b/be/src/core/column/column_map.cpp @@ -643,11 +643,6 @@ Status ColumnMap::deduplicate_keys(bool recursive) { return Status::OK(); } -void ColumnMap::shrink_padding_chars() { - keys_column->shrink_padding_chars(); - values_column->shrink_padding_chars(); -} - void ColumnMap::reserve(size_t n) { get_offsets().reserve(n); keys_column->reserve(n); diff --git a/be/src/core/column/column_map.h b/be/src/core/column/column_map.h index 1081cd268d50ab..217c39451d3603 100644 --- a/be/src/core/column/column_map.h +++ b/be/src/core/column/column_map.h @@ -115,7 +115,6 @@ class ColumnMap final : public COWHelper { const char* deserialize_and_insert_from_arena(const char* pos) override; void update_hash_with_value(size_t n, SipHash& hash) const override; - void shrink_padding_chars() override; ColumnPtr filter(const Filter& filt, ssize_t result_size_hint) const override; size_t filter(const Filter& filter) override; MutableColumnPtr permute(const Permutation& perm, size_t limit) const override; diff --git a/be/src/core/column/column_nullable.cpp b/be/src/core/column/column_nullable.cpp index 853ea6b0f03edc..3c1c157afdc19f 100644 --- a/be/src/core/column/column_nullable.cpp +++ b/be/src/core/column/column_nullable.cpp @@ -48,10 +48,6 @@ ColumnNullable::ColumnNullable(MutableColumnPtr&& nested_column_, MutableColumnP } } -void ColumnNullable::shrink_padding_chars() { - get_nested_column_ptr()->shrink_padding_chars(); -} - void ColumnNullable::update_xxHash_with_value(size_t start, size_t end, uint64_t& hash, const uint8_t* __restrict null_data) const { if (!has_null(start, end)) { diff --git a/be/src/core/column/column_nullable.h b/be/src/core/column/column_nullable.h index 6701bfc136feb6..020857657b259d 100644 --- a/be/src/core/column/column_nullable.h +++ b/be/src/core/column/column_nullable.h @@ -86,8 +86,6 @@ class ColumnNullable final : public COWHelper { _nested_column->sanity_check(); } - void shrink_padding_chars() override; - bool is_variable_length() const override { return _nested_column->is_variable_length(); } std::string get_name() const override { return "Nullable(" + _nested_column->get_name() + ")"; } diff --git a/be/src/core/column/column_string.cpp b/be/src/core/column/column_string.cpp index f633151effcfd1..9906f84c12b1c2 100644 --- a/be/src/core/column/column_string.cpp +++ b/be/src/core/column/column_string.cpp @@ -81,29 +81,6 @@ MutableColumnPtr ColumnStr::clone_resized(size_t to_size) const { return res; } -template -void ColumnStr::shrink_padding_chars() { - if (size() == 0) { - return; - } - char* data = reinterpret_cast(chars.data()); - auto* offset = offsets.data(); - size_t size = offsets.size(); - - // deal the 0-th element. no need to move. - auto next_start = offset[0]; - offset[0] = static_cast(strnlen(data, size_at(0))); - for (size_t i = 1; i < size; i++) { - // get the i-th length and whole move it to cover the last's trailing void - auto length = strnlen(data + next_start, offset[i] - next_start); - memmove(data + offset[i - 1], data + next_start, length); - // offset i will be changed. so save the old value for (i+1)-th to get its length. - next_start = offset[i]; - offset[i] = offset[i - 1] + static_cast(length); - } - chars.resize_fill(offsets.back()); // just call it to shrink memory here. no possible to expand. -} - // This method is only called by MutableBlock::merge_ignore_overflow // by hash join operator to collect build data to avoid // the total string length of a ColumnStr column exceeds the 4G limit. diff --git a/be/src/core/column/column_string.h b/be/src/core/column/column_string.h index 072b75f5480ce4..608ccb7bc5b5f3 100644 --- a/be/src/core/column/column_string.h +++ b/be/src/core/column/column_string.h @@ -143,8 +143,6 @@ class ColumnStr final : public COWHelper> { MutableColumnPtr clone_resized(size_t to_size) const override; - void shrink_padding_chars() override; - Field operator[](size_t n) const override; void get(size_t n, Field& res) const override; diff --git a/be/src/core/column/column_struct.cpp b/be/src/core/column/column_struct.cpp index ed150bdfd725b8..74e1a34914d7d7 100644 --- a/be/src/core/column/column_struct.cpp +++ b/be/src/core/column/column_struct.cpp @@ -338,12 +338,6 @@ MutableColumnPtr ColumnStruct::permute(const Permutation& perm, size_t limit) co return ColumnStruct::create(new_columns); } -void ColumnStruct::shrink_padding_chars() { - for (auto& column : columns) { - column->shrink_padding_chars(); - } -} - void ColumnStruct::reserve(size_t n) { const size_t tuple_size = columns.size(); for (size_t i = 0; i < tuple_size; ++i) { diff --git a/be/src/core/column/column_struct.h b/be/src/core/column/column_struct.h index 5826372ed6375a..61a2902b0ec8c3 100644 --- a/be/src/core/column/column_struct.h +++ b/be/src/core/column/column_struct.h @@ -150,8 +150,6 @@ class ColumnStruct final : public COWHelper { int compare_at(size_t n, size_t m, const IColumn& rhs_, int nan_direction_hint) const override; - void shrink_padding_chars() override; - void reserve(size_t n) override; void resize(size_t n) override; size_t byte_size() const override; diff --git a/be/src/core/column/predicate_column.h b/be/src/core/column/predicate_column.h index 699aff8a1e0db9..538f1838d3db9d 100644 --- a/be/src/core/column/predicate_column.h +++ b/be/src/core/column/predicate_column.h @@ -105,11 +105,7 @@ class PredicateColumnType final : public COWHelper) { - auto res = reinterpret_cast(data[n]); - if constexpr (Type == TYPE_CHAR) { - res.size = strnlen(res.data, res.size); - } - return res; + return reinterpret_cast(data[n]); } else { throw doris::Exception( ErrorCode::INTERNAL_ERROR, diff --git a/be/src/core/data_type_serde/data_type_string_serde.cpp b/be/src/core/data_type_serde/data_type_string_serde.cpp index e766e4bb563cc0..7d88fcdf809a48 100644 --- a/be/src/core/data_type_serde/data_type_string_serde.cpp +++ b/be/src/core/data_type_serde/data_type_string_serde.cpp @@ -462,25 +462,16 @@ Status DataTypeStringSerDeBase::from_string(StringRef& str, IColumn& // Deserializes a STRING/VARCHAR/CHAR value from its OLAP string representation // (e.g. from ZoneMap protobuf). This is the inverse of to_olap_string(). -// -// For CHAR type: if the string is shorter than the declared column length (_len), -// pads with '\0' bytes to reach _len. This preserves CHAR's fixed-length semantics. -// For STRING/VARCHAR: stores the string as-is. -// -// Examples: -// CHAR(10), str="hello" => field = "hello\0\0\0\0\0" (10 bytes) -// VARCHAR, str="hello" => field = "hello" (5 bytes) template Status DataTypeStringSerDeBase::from_olap_string(const std::string& str, Field& field, const FormatOptions& options) const { - if (cast_set(str.size()) < _len) { - DCHECK_EQ(_type, TYPE_CHAR); - std::string tmp(_len, '\0'); - memcpy(tmp.data(), str.data(), str.size()); - field = Field::create_field(std::move(tmp)); - } else { - field = Field::create_field(str); - } + // CHAR(N) writes through OlapColumnDataConvertorChar are zero-padded to + // the declared schema length, so the serialized OLAP string carries + // trailing '\0' bytes. strnlen() drops that padding to surface the + // logical character content in the Field. VARCHAR / STRING never write + // trailing '\0' through this path, so strnlen is a no-op for them. + size_t len = strnlen(str.data(), str.size()); + field = Field::create_field(std::string(str.data(), len)); return Status::OK(); } diff --git a/be/src/exec/rowid_fetcher.cpp b/be/src/exec/rowid_fetcher.cpp index 985e4bfe340edd..a3f6f6980e1db3 100644 --- a/be/src/exec/rowid_fetcher.cpp +++ b/be/src/exec/rowid_fetcher.cpp @@ -213,30 +213,6 @@ Status RowIDFetcher::_merge_rpc_results(const PMultiGetRequest& request, return Status::OK(); } -bool _has_char_type(const DataTypePtr& type) { - switch (type->get_primitive_type()) { - case TYPE_CHAR: { - return true; - } - case TYPE_ARRAY: { - const auto* arr_type = assert_cast(remove_nullable(type).get()); - return _has_char_type(arr_type->get_nested_type()); - } - case TYPE_MAP: { - const auto* map_type = assert_cast(remove_nullable(type).get()); - return _has_char_type(map_type->get_key_type()) || - _has_char_type(map_type->get_value_type()); - } - case TYPE_STRUCT: { - const auto* struct_type = assert_cast(remove_nullable(type).get()); - return std::any_of(struct_type->get_elements().begin(), struct_type->get_elements().end(), - [&](const DataTypePtr& dt) -> bool { return _has_char_type(dt); }); - } - default: - return false; - } -} - Status RowIDFetcher::fetch(const ColumnPtr& column_row_ids, Block* res_block) { CHECK(!_stubs.empty()); PMultiGetRequest mget_req = _init_fetch_request( @@ -286,16 +262,6 @@ Status RowIDFetcher::fetch(const ColumnPtr& column_row_ids, Block* res_block) { } // Check row consistency RETURN_IF_CATCH_EXCEPTION(res_block->check_number_of_rows()); - // shrink for char type - std::vector char_type_idx; - for (size_t i = 0; i < _fetch_option.desc->slots().size(); i++) { - const auto& column_desc = _fetch_option.desc->slots()[i]; - const auto type = column_desc->type(); - if (_has_char_type(type)) { - char_type_idx.push_back(i); - } - } - res_block->shrink_char_type_column_suffix_zero(char_type_idx); VLOG_DEBUG << "dump block:" << res_block->dump_data(0, 10); return Status::OK(); } @@ -568,15 +534,6 @@ Status RowIdStorageReader::read_by_rowids(const PMultiGetRequestV2& request, for (const auto& pslot : request_block_desc.slots()) { slots.push_back(SlotDescriptor(pslot)); } - // prepare block char vector shrink for char type - std::vector char_type_idx; - for (int j = 0; j < slots.size(); ++j) { - auto slot = slots[j]; - if (_has_char_type(slot.type())) { - char_type_idx.push_back(j); - } - } - try { if (first_file_mapping->type == FileMappingType::INTERNAL) { RETURN_IF_ERROR(read_batch_doris_format_row( @@ -594,9 +551,6 @@ Status RowIdStorageReader::read_by_rowids(const PMultiGetRequestV2& request, return Status::Error(e.code(), "Row id fetch failed because {}", e.what()); } - - // after read the block, shrink char type block - result_blocks[i].shrink_char_type_column_suffix_zero(char_type_idx); } [[maybe_unused]] size_t compressed_size = 0; diff --git a/be/src/exprs/bloom_filter_func_adaptor.h b/be/src/exprs/bloom_filter_func_adaptor.h index d41a12ff64832f..5cc5d33215d359 100644 --- a/be/src/exprs/bloom_filter_func_adaptor.h +++ b/be/src/exprs/bloom_filter_func_adaptor.h @@ -243,18 +243,6 @@ struct StringFindOp : CommonFindOp { } }; -// We do not need to judge whether data is empty, because null will not appear -// when filer used by the storage engine -template -struct FixedStringFindOp : public StringFindOp { - static uint16_t find_batch_olap_engine(const BloomFilterAdaptor& bloom_filter, const char* data, - const uint8_t* nullmap, uint16_t* offsets, int number, - const bool is_parse_column) { - return find_batch_olap( - bloom_filter, data, nullmap, offsets, number, is_parse_column); - } -}; - template struct BloomFilterTypeTraits { using T = typename PrimitiveTypeTraits::CppType; @@ -263,7 +251,7 @@ struct BloomFilterTypeTraits { template struct BloomFilterTypeTraits { - using FindOp = FixedStringFindOp; + using FindOp = StringFindOp; }; template diff --git a/be/src/exprs/bloom_filter_func_impl.h b/be/src/exprs/bloom_filter_func_impl.h index ae00d0d8319737..0ee5da80f881fa 100644 --- a/be/src/exprs/bloom_filter_func_impl.h +++ b/be/src/exprs/bloom_filter_func_impl.h @@ -55,23 +55,12 @@ struct fixed_len_to_uint32_v2 { } }; -template +template uint16_t find_batch_olap(const BloomFilterAdaptor& bloom_filter, const char* data, const uint8_t* nullmap, uint16_t* offsets, int number, const bool is_parse_column) { auto get_element = [](const char* input_data, int idx) { - if constexpr (std::is_same_v && need_trim) { - const auto value = ((const StringRef*)(input_data))[idx]; - int64_t size = value.size; - const char* data = value.data; - // CHAR type may pad the tail with \0, need to trim - while (size > 0 && data[size - 1] == '\0') { - size--; - } - return StringRef(value.data, size); - } else { - return ((const T*)(input_data))[idx]; - } + return ((const T*)(input_data))[idx]; }; uint16_t new_size = 0; diff --git a/be/src/service/point_query_executor.cpp b/be/src/service/point_query_executor.cpp index 572813eab98422..5b753dec7337a2 100644 --- a/be/src/service/point_query_executor.cpp +++ b/be/src/service/point_query_executor.cpp @@ -570,11 +570,6 @@ Status PointQueryExecutor::_lookup_row_data() { RETURN_IF_ERROR(segment->seek_and_read_by_rowid(*_tablet->tablet_schema(), slot, row_id, column, storage_read_options, iter)); - if (_tablet->tablet_schema() - ->column_by_uid(slot->col_unique_id()) - .has_char_type()) { - column->shrink_padding_chars(); - } } } } diff --git a/be/src/storage/delete/delete_handler.cpp b/be/src/storage/delete/delete_handler.cpp index 8aab3c422966fd..1781c7734de7d0 100644 --- a/be/src/storage/delete/delete_handler.cpp +++ b/be/src/storage/delete/delete_handler.cpp @@ -112,9 +112,6 @@ Status convert(const DataTypePtr& data_type, const std::list& str, // Parses a single condition value string into a Field and creates a comparison predicate. // Uses serde->from_fe_string to do the parsing, which handles all type-specific // conversions (including decimal scale, etc.). -// For CHAR type, the value is padded with '\0' to the declared column length, consistent -// with the IN list path in convert() above. -// For VARCHAR/STRING, the Field is created directly from the raw string. Status parse_to_predicate(const uint32_t index, const std::string col_name, const DataTypePtr& type, DeleteHandler::ConditionParseResult& res, Arena& arena, std::shared_ptr& predicate) { @@ -128,22 +125,7 @@ Status parse_to_predicate(const uint32_t index, const std::string col_name, cons } Field v; - if (type->get_primitive_type() == TYPE_CHAR) { - // CHAR type: create Field and pad with '\0' to the declared column length, - // consistent with IN list path (convert() above) and create_comparison_predicate. - const auto& str = res.value_str.front(); - auto char_len = cast_set( - assert_cast(remove_nullable(type).get())->len()); - auto target = std::max(char_len, str.size()); - if (target > str.size()) { - std::string padded(target, '\0'); - memcpy(padded.data(), str.data(), str.size()); - v = Field::create_field(std::move(padded)); - } else { - v = Field::create_field(str); - } - } else if (is_string_type(type->get_primitive_type())) { - // VARCHAR/STRING: create Field directly from the raw string, no padding needed. + if (is_string_type(type->get_primitive_type())) { v = Field::create_field(res.value_str.front()); } else { auto serde = type->get_serde(); diff --git a/be/src/storage/predicate/comparison_predicate.h b/be/src/storage/predicate/comparison_predicate.h index ddb9f46b7752e2..eb33d042e78feb 100644 --- a/be/src/storage/predicate/comparison_predicate.h +++ b/be/src/storage/predicate/comparison_predicate.h @@ -281,7 +281,12 @@ class ComparisonPredicateBase final : public ColumnPredicate { if (bf->is_ngram_bf()) { return true; } - if constexpr (is_string_type(Type)) { + if constexpr (Type == TYPE_CHAR) { + // CHAR BFs hash zero-padded bytes while the predicate value is + // unpadded, so probing the BF would always miss. Skip BF + // pruning for CHAR entirely and let the scan filter the rows. + return true; + } else if constexpr (is_string_type(Type)) { return bf->test_bytes(_value.data(), _value.size()); } else { // DecimalV2 using decimal12_t in bloom filter, should convert value to decimal12_t diff --git a/be/src/storage/predicate/in_list_predicate.h b/be/src/storage/predicate/in_list_predicate.h index b1c229196fc530..eef1796399d990 100644 --- a/be/src/storage/predicate/in_list_predicate.h +++ b/be/src/storage/predicate/in_list_predicate.h @@ -68,18 +68,16 @@ class InListPredicateBase final : public ColumnPredicate { ENABLE_FACTORY_CREATOR(InListPredicateBase); using T = typename PrimitiveTypeTraits::CppType; InListPredicateBase(uint32_t column_id, std::string col_name, - const std::shared_ptr& hybrid_set, bool is_opposite, - size_t char_length = 0) + const std::shared_ptr& hybrid_set, bool is_opposite) : ColumnPredicate(column_id, col_name, Type, is_opposite), _min_value(type_limit::max()), _max_value(type_limit::min()) { CHECK(hybrid_set != nullptr); // String types need a copy because: - // 1. The caller's set is StringSet>, but here we want - // StringSet> for small-set optimization — different - // C++ types, cannot share the pointer. - // 2. CHAR type additionally needs padding to char_length. + // The caller's set is StringSet>, but here we want + // StringSet> for small-set optimization — different + // C++ types, cannot share the pointer. // // Date/DECIMALV2 types do NOT need a copy: their ElementType (CppType) is identical // between the caller's HybridSet and InListPredicateBase's, and InListPredicateBase @@ -93,15 +91,7 @@ class InListPredicateBase final : public ColumnPredicate { HybridSetBase::IteratorBase* iter = hybrid_set->begin(); while (iter->has_next()) { const auto* value = (const StringRef*)(iter->get_value()); - if constexpr (Type == TYPE_CHAR) { - _temp_datas.emplace_back(""); - _temp_datas.back().resize(std::max(char_length, value->size)); - memcpy(_temp_datas.back().data(), value->data, value->size); - const std::string& str = _temp_datas.back(); - _values->insert((void*)str.data(), str.length()); - } else { - _values->insert((void*)value->data, value->size); - } + _values->insert((void*)value->data, value->size); iter->next(); } } else { @@ -129,7 +119,6 @@ class InListPredicateBase final : public ColumnPredicate { _values = other._values; _min_value = other._min_value; _max_value = other._max_value; - _temp_datas = other._temp_datas; DCHECK(_segment_id_to_value_in_dict_flags.empty()); } InListPredicateBase(const InListPredicateBase& other) = delete; @@ -354,6 +343,12 @@ class InListPredicateBase final : public ColumnPredicate { if (bf->is_ngram_bf()) { return true; } + if constexpr (Type == TYPE_CHAR) { + // CHAR BFs hash zero-padded bytes while the predicate value is + // unpadded, so probing the BF would always miss. Skip BF + // pruning for CHAR entirely. + return true; + } HybridSetBase::IteratorBase* iter = _values->begin(); while (iter->has_next()) { if constexpr (is_string_type(Type)) { @@ -655,9 +650,6 @@ class InListPredicateBase final : public ColumnPredicate { _segment_id_to_value_in_dict_flags; T _min_value; T _max_value; - - // temp string for char type column - std::list _temp_datas; }; #include "common/compile_check_end.h" } //namespace doris diff --git a/be/src/storage/predicate/like_column_predicate.h b/be/src/storage/predicate/like_column_predicate.h index 937dc0079bedae..578b98ac6090b7 100644 --- a/be/src/storage/predicate/like_column_predicate.h +++ b/be/src/storage/predicate/like_column_predicate.h @@ -155,7 +155,7 @@ class LikeColumnPredicate final : public ColumnPredicate { std::vector tmp_res(column.dict_size(), false); for (int i = 0; i < column.dict_size(); i++) { - StringRef cell_value = column.get_shrink_value(i); + StringRef cell_value = column.get_value(i); unsigned char flag = 0; THROW_IF_ERROR((_state->scalar_function)( &_like_state, StringRef(cell_value.data, cell_value.size), pattern, &flag)); diff --git a/be/src/storage/predicate/predicate_creator_comparison.cpp b/be/src/storage/predicate/predicate_creator_comparison.cpp index bfec1262cfc1a5..b10a175b016592 100644 --- a/be/src/storage/predicate/predicate_creator_comparison.cpp +++ b/be/src/storage/predicate/predicate_creator_comparison.cpp @@ -77,21 +77,9 @@ std::shared_ptr create_comparison_predicate(const uint32_t cid, opposite); } case TYPE_CHAR: { - auto target = std::max(cast_set(assert_cast( - remove_nullable(data_type).get()) - ->len()), - value.template get().size()); - if (target > value.template get().size()) { - std::string tmp(target, '\0'); - memcpy(tmp.data(), value.template get().data(), - value.template get().size()); - return ComparisonPredicateBase::create_shared( - cid, col_name, Field::create_field(std::move(tmp)), opposite); - } else { - return ComparisonPredicateBase::create_shared( - cid, col_name, Field::create_field(value.template get()), - opposite); - } + return ComparisonPredicateBase::create_shared( + cid, col_name, Field::create_field(value.template get()), + opposite); } case TYPE_VARCHAR: case TYPE_STRING: { diff --git a/be/src/storage/predicate/predicate_creator_in_list_in.cpp b/be/src/storage/predicate/predicate_creator_in_list_in.cpp index e720da632a8ad4..f1ad365b4f523e 100644 --- a/be/src/storage/predicate/predicate_creator_in_list_in.cpp +++ b/be/src/storage/predicate/predicate_creator_in_list_in.cpp @@ -26,42 +26,34 @@ namespace doris { template static std::shared_ptr create_in_list_predicate_impl( const uint32_t cid, const std::string col_name, const std::shared_ptr& set, - bool is_opposite, size_t char_length = 0) { + bool is_opposite) { // Only string types construct their own HybridSetType in the constructor (to convert // from DynamicContainer to FixedContainer), so N dispatch is only needed // for them. All other types directly share the caller's hybrid_set. if constexpr (!is_string_type(TYPE)) { return InListPredicateBase::create_shared( - cid, col_name, set, is_opposite, char_length); + cid, col_name, set, is_opposite); } else { auto set_size = set->size(); if (set_size == 1) { - return InListPredicateBase::create_shared(cid, col_name, set, is_opposite, - char_length); + return InListPredicateBase::create_shared(cid, col_name, set, is_opposite); } else if (set_size == 2) { - return InListPredicateBase::create_shared(cid, col_name, set, is_opposite, - char_length); + return InListPredicateBase::create_shared(cid, col_name, set, is_opposite); } else if (set_size == 3) { - return InListPredicateBase::create_shared(cid, col_name, set, is_opposite, - char_length); + return InListPredicateBase::create_shared(cid, col_name, set, is_opposite); } else if (set_size == 4) { - return InListPredicateBase::create_shared(cid, col_name, set, is_opposite, - char_length); + return InListPredicateBase::create_shared(cid, col_name, set, is_opposite); } else if (set_size == 5) { - return InListPredicateBase::create_shared(cid, col_name, set, is_opposite, - char_length); + return InListPredicateBase::create_shared(cid, col_name, set, is_opposite); } else if (set_size == 6) { - return InListPredicateBase::create_shared(cid, col_name, set, is_opposite, - char_length); + return InListPredicateBase::create_shared(cid, col_name, set, is_opposite); } else if (set_size == 7) { - return InListPredicateBase::create_shared(cid, col_name, set, is_opposite, - char_length); + return InListPredicateBase::create_shared(cid, col_name, set, is_opposite); } else if (set_size == FIXED_CONTAINER_MAX_SIZE) { - return InListPredicateBase::create_shared(cid, col_name, set, is_opposite, - char_length); + return InListPredicateBase::create_shared(cid, col_name, set, is_opposite); } else { return InListPredicateBase::create_shared( - cid, col_name, set, is_opposite, char_length); + cid, col_name, set, is_opposite); } } } @@ -120,9 +112,8 @@ std::shared_ptr create_in_list_predicate( - cid, col_name, set, is_opposite, - assert_cast(remove_nullable(data_type).get())->len()); + return create_in_list_predicate_impl(cid, col_name, set, + is_opposite); } case TYPE_VARCHAR: { return create_in_list_predicate_impl( diff --git a/be/src/storage/predicate/predicate_creator_in_list_not_in.cpp b/be/src/storage/predicate/predicate_creator_in_list_not_in.cpp index 63e8eb37b186a3..e4cb4731a57095 100644 --- a/be/src/storage/predicate/predicate_creator_in_list_not_in.cpp +++ b/be/src/storage/predicate/predicate_creator_in_list_not_in.cpp @@ -26,42 +26,34 @@ namespace doris { template static std::shared_ptr create_in_list_predicate_impl( const uint32_t cid, const std::string col_name, const std::shared_ptr& set, - bool is_opposite, size_t char_length = 0) { + bool is_opposite) { // Only string types construct their own HybridSetType in the constructor (to convert // from DynamicContainer to FixedContainer), so N dispatch is only needed // for them. All other types directly share the caller's hybrid_set. if constexpr (!is_string_type(TYPE)) { return InListPredicateBase::create_shared( - cid, col_name, set, is_opposite, char_length); + cid, col_name, set, is_opposite); } else { auto set_size = set->size(); if (set_size == 1) { - return InListPredicateBase::create_shared(cid, col_name, set, is_opposite, - char_length); + return InListPredicateBase::create_shared(cid, col_name, set, is_opposite); } else if (set_size == 2) { - return InListPredicateBase::create_shared(cid, col_name, set, is_opposite, - char_length); + return InListPredicateBase::create_shared(cid, col_name, set, is_opposite); } else if (set_size == 3) { - return InListPredicateBase::create_shared(cid, col_name, set, is_opposite, - char_length); + return InListPredicateBase::create_shared(cid, col_name, set, is_opposite); } else if (set_size == 4) { - return InListPredicateBase::create_shared(cid, col_name, set, is_opposite, - char_length); + return InListPredicateBase::create_shared(cid, col_name, set, is_opposite); } else if (set_size == 5) { - return InListPredicateBase::create_shared(cid, col_name, set, is_opposite, - char_length); + return InListPredicateBase::create_shared(cid, col_name, set, is_opposite); } else if (set_size == 6) { - return InListPredicateBase::create_shared(cid, col_name, set, is_opposite, - char_length); + return InListPredicateBase::create_shared(cid, col_name, set, is_opposite); } else if (set_size == 7) { - return InListPredicateBase::create_shared(cid, col_name, set, is_opposite, - char_length); + return InListPredicateBase::create_shared(cid, col_name, set, is_opposite); } else if (set_size == FIXED_CONTAINER_MAX_SIZE) { - return InListPredicateBase::create_shared(cid, col_name, set, is_opposite, - char_length); + return InListPredicateBase::create_shared(cid, col_name, set, is_opposite); } else { return InListPredicateBase::create_shared( - cid, col_name, set, is_opposite, char_length); + cid, col_name, set, is_opposite); } } } @@ -121,8 +113,7 @@ std::shared_ptr create_in_list_predicate( - cid, col_name, set, is_opposite, - assert_cast(remove_nullable(data_type).get())->len()); + cid, col_name, set, is_opposite); } case TYPE_VARCHAR: { return create_in_list_predicate_impl( diff --git a/be/src/storage/row_cursor.cpp b/be/src/storage/row_cursor.cpp index f5b99f670c7967..ec7a7ada85aa77 100644 --- a/be/src/storage/row_cursor.cpp +++ b/be/src/storage/row_cursor.cpp @@ -124,17 +124,6 @@ RowCursor RowCursor::clone() const { return result; } -void RowCursor::pad_char_fields() { - for (size_t i = 0; i < _fields.size(); ++i) { - const TabletColumn* col = _schema->column(cast_set(i)); - if (col->type() == FieldType::OLAP_FIELD_TYPE_CHAR && !_fields[i].is_null()) { - String padded = _fields[i].get(); - padded.resize(col->length(), '\0'); - _fields[i] = Field::create_field(std::move(padded)); - } - } -} - std::string RowCursor::to_string() const { std::string result; for (size_t i = 0; i < _fields.size(); ++i) { diff --git a/be/src/storage/row_cursor.h b/be/src/storage/row_cursor.h index 7baa029bcd2929..47ce8437494b21 100644 --- a/be/src/storage/row_cursor.h +++ b/be/src/storage/row_cursor.h @@ -71,11 +71,6 @@ class RowCursor { // Returns a deep copy of this RowCursor with the same schema and field values. RowCursor clone() const; - // Pad all CHAR-type fields in-place to their declared column length using '\0'. - // RowCursor holds CHAR values in compute format (unpadded). Call this before - // comparing against storage-format data (e.g. _seek_block) where CHAR is padded. - void pad_char_fields(); - // Output row cursor content in string format std::string to_string() const; diff --git a/be/src/storage/segment/binary_dict_page_pre_decoder.h b/be/src/storage/segment/binary_dict_page_pre_decoder.h index b488e83f402fcf..c6f6721e70b6b5 100644 --- a/be/src/storage/segment/binary_dict_page_pre_decoder.h +++ b/be/src/storage/segment/binary_dict_page_pre_decoder.h @@ -19,6 +19,7 @@ #include "storage/cache/page_cache.h" #include "storage/segment/binary_dict_page.h" +#include "storage/segment/binary_plain_page_char_strip_pre_decoder.h" #include "storage/segment/binary_plain_page_v2_pre_decoder.h" #include "storage/segment/bitshuffle_page_pre_decoder.h" #include "storage/segment/encoding_info.h" @@ -33,12 +34,18 @@ namespace segment_v2 { * BinaryDictPage data pages can have different encoding types: * 1. DICT_ENCODING: header(4 bytes) + bitshuffle encoded codeword page * 2. PLAIN_ENCODING_V2: header(4 bytes) + BinaryPlainPageV2 encoded data - * 3. PLAIN_ENCODING: header(4 bytes) + BinaryPlainPage encoded data (no pre-decode needed) + * 3. PLAIN_ENCODING: header(4 bytes) + BinaryPlainPage encoded data (no pre-decode needed + * for non-CHAR; CHAR pages get their trailing '\0' padding stripped inline) * * This pre-decoder reads the encoding type from the first 4 bytes, strips the header, * dispatches to the appropriate pre-decoder (BitShufflePagePreDecoder or - * BinaryPlainPageV2PreDecoder), and then restores the header. + * BinaryPlainPageV2PreDecoder), and then restores the header. + * + * When IS_CHAR is true the inline-binary paths (PLAIN_ENCODING / PLAIN_ENCODING_V2) + * use the CHAR-strip variants so the dict-fallback data pages are also unpadded + * once at page load time. */ +template struct BinaryDictPagePreDecoder : public DataPagePreDecoder { /** * @brief Decode BinaryDictPage data page @@ -72,8 +79,9 @@ struct BinaryDictPagePreDecoder : public DataPagePreDecoder { "PLAIN_ENCODING_V2, PLAIN_ENCODING>", encoding_type, file_path); } - // For PLAIN_ENCODING, no pre-decoding needed - if (encoding_type == PLAIN_ENCODING) { + // For PLAIN_ENCODING, non-CHAR pages can be used as-is; CHAR pages + // are routed through the CHAR-strip pre-decoder below. + if (encoding_type == PLAIN_ENCODING && !IS_CHAR) { return Status::OK(); } @@ -102,12 +110,22 @@ struct BinaryDictPagePreDecoder : public DataPagePreDecoder { break; } case PLAIN_ENCODING_V2: { - // Use BinaryPlainPageV2PreDecoder with total_prefix to reserve space - BinaryPlainPageV2PreDecoder v2_decoder; + BinaryPlainPageV2PreDecoder v2_decoder; status = v2_decoder.decode(&decoded_page, &data_without_header, size_of_tail, _use_cache, page_type, file_path, total_prefix); break; } + case PLAIN_ENCODING: { + // Non-CHAR is short-circuited above; CHECK that the invariant + // holds in case the short-circuit gets removed accidentally. + CHECK(IS_CHAR) << "BinaryDictPagePreDecoder reached PLAIN_ENCODING " + "dict-fallback path; expected to short-circuit above. file: " + << file_path; + BinaryPlainPageCharStripPreDecoder v1_decoder; + status = v1_decoder.decode(&decoded_page, &data_without_header, size_of_tail, + _use_cache, page_type, file_path, total_prefix); + break; + } default: // Unknown encoding type, no pre-decoding needed return Status::OK(); diff --git a/be/src/storage/segment/binary_plain_page_char_strip_pre_decoder.h b/be/src/storage/segment/binary_plain_page_char_strip_pre_decoder.h new file mode 100644 index 00000000000000..11947c5abcbb73 --- /dev/null +++ b/be/src/storage/segment/binary_plain_page_char_strip_pre_decoder.h @@ -0,0 +1,98 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include "storage/cache/page_cache.h" +#include "storage/segment/binary_plain_page_v2_pre_decoder.h" // BinaryPlainV1Entry, write_binary_plain_v1_output +#include "storage/segment/encoding_info.h" +#include "util/coding.h" + +namespace doris::segment_v2 { + +// Pre-decoder for BinaryPlainPage (V1) data pages of CHAR columns. +// +// Segments store CHAR(N) zero-padded to N bytes (the on-disk format). The +// pre-decoder strnlens each slice once at page load time, then rewrites the +// page as a tight V1 layout (no trailing '\0' bytes, adjusted offsets) before +// it is placed in the page cache, so the compute layer reads unpadded CHAR. +// +// Both input and output layout are V1: +// Data: |binary1|binary2|...|binaryN| +// Trailer: |offset1|offset2|...|offsetN| num_elems (32-bit) +// +// Reuses the same writer as the V2 pre-decoders (write_binary_plain_v1_output) +// for steps 3-7; only the input scan differs (offsets array instead of varint +// lengths). +struct BinaryPlainPageCharStripPreDecoder : public DataPagePreDecoder { + Status decode(std::unique_ptr* page, Slice* page_slice, size_t size_of_tail, + bool _use_cache, segment_v2::PageTypePB page_type, const std::string& file_path, + size_t size_of_prefix = 0) override { + // Step 1: validate and locate the V1 trailer. + if (page_slice->size < size_of_tail + sizeof(uint32_t)) { + return Status::Corruption( + "Invalid CHAR plain page size: {}, expected at least {} in file: {}", + page_slice->size, size_of_tail + sizeof(uint32_t), file_path); + } + Slice data(page_slice->data, page_slice->size - size_of_tail); + uint32_t num_elems = decode_fixed32_le( + reinterpret_cast(&data[data.size - sizeof(uint32_t)])); + + // Always rewrite the page even for num_elems==0 — callers (e.g. + // BinaryDictPagePreDecoder) may pass a non-zero `size_of_prefix` + // expecting a fresh output buffer with that prefix area reserved. + size_t offsets_pos = data.size - (num_elems + 1) * sizeof(uint32_t); + if (num_elems > 0 && offsets_pos > data.size - sizeof(uint32_t)) { + return Status::Corruption( + "CHAR plain page corruption: offsets pos beyond data, size={}, num_elems={}, " + "offsets_pos={} in file: {}", + data.size, num_elems, offsets_pos, file_path); + } + const auto* offsets_in = reinterpret_cast(&data[offsets_pos]); + + // Step 2: scan entries, strnlen-ing each to drop trailing '\0' padding. + std::vector entries; + entries.reserve(num_elems); + uint32_t total_out_len = 0; + for (uint32_t i = 0; i < num_elems; ++i) { + uint32_t start = decode_fixed32_le(offsets_in + i * sizeof(uint32_t)); + uint32_t end = (i + 1 < num_elems) + ? decode_fixed32_le(offsets_in + (i + 1) * sizeof(uint32_t)) + : static_cast(offsets_pos); + if (end < start || end > offsets_pos) { + return Status::Corruption( + "CHAR plain page corruption: bad offset at {}, start={}, end={}, " + "offsets_pos={} in file: {}", + i, start, end, offsets_pos, file_path); + } + uint32_t raw_size = end - start; + uint32_t out_len = static_cast(strnlen(data.data + start, raw_size)); + entries.push_back({reinterpret_cast(data.data + start), out_len}); + total_out_len += out_len; + } + + // Steps 3-7 (shared with V2 pre-decoders). + return write_binary_plain_v1_output(entries, num_elems, total_out_len, *page_slice, + size_of_tail, size_of_prefix, _use_cache, page_type, + page, page_slice); + } +}; + +} // namespace doris::segment_v2 diff --git a/be/src/storage/segment/binary_plain_page_v2_pre_decoder.h b/be/src/storage/segment/binary_plain_page_v2_pre_decoder.h index bea58094c8a578..b400d583fc9d67 100644 --- a/be/src/storage/segment/binary_plain_page_v2_pre_decoder.h +++ b/be/src/storage/segment/binary_plain_page_v2_pre_decoder.h @@ -17,6 +17,9 @@ #pragma once +#include +#include + #include "storage/cache/page_cache.h" #include "storage/segment/encoding_info.h" #include "util/coding.h" @@ -24,6 +27,63 @@ namespace doris { namespace segment_v2 { +// One source entry feeding the V1 output writer. Variants differ only in how +// `out_len` is derived from the raw input length (raw, strnlen'd, etc.). +struct BinaryPlainV1Entry { + const uint8_t* start; + uint32_t out_len; +}; + +// Allocate a V1 BinaryPlainPage layout output buffer and write +// binary -> offsets -> num_elems -> tail. Shared by V2 pre-decoders (after +// they iterate varint lengths) and the V1 CHAR-strip pre-decoder (after it +// iterates the V1 offsets array). `size_of_prefix` reserves room before the +// V1 data for callers that wrap the page (e.g. BinaryDictPagePreDecoder +// prepending the dict-page header). +inline Status write_binary_plain_v1_output(const std::vector& entries, + uint32_t num_elems, uint32_t total_out_len, + const Slice& source_page_slice, size_t size_of_tail, + size_t size_of_prefix, bool use_cache, + segment_v2::PageTypePB page_type, + std::unique_ptr* out_page, + Slice* out_page_slice) { + size_t offsets_size = num_elems * sizeof(uint32_t); + size_t v1_data_size = total_out_len + offsets_size + sizeof(uint32_t); + size_t total_size = size_of_prefix + v1_data_size + size_of_tail; + + std::unique_ptr decoded_page = + std::make_unique(total_size, use_cache, page_type); + Slice decoded_slice(decoded_page->data(), total_size); + char* output = decoded_slice.data + size_of_prefix; + + // Binary payload. + for (const auto& e : entries) { + memcpy(output, e.start, e.out_len); + output += e.out_len; + } + + // Offsets array (running cursor). + uint32_t running = 0; + for (const auto& e : entries) { + encode_fixed32_le(reinterpret_cast(output), running); + output += sizeof(uint32_t); + running += e.out_len; + } + + // num_elems trailer. + encode_fixed32_le(reinterpret_cast(output), num_elems); + output += sizeof(uint32_t); + + // Tail (footer + null map). + if (size_of_tail > 0) { + memcpy(output, source_page_slice.data + source_page_slice.size - size_of_tail, + size_of_tail); + } + *out_page_slice = decoded_slice; + *out_page = std::move(decoded_page); + return Status::OK(); +} + /** * @brief Pre-decoder for BinaryPlainPageV2 * @@ -37,127 +97,101 @@ namespace segment_v2 { * V1 format (output): * Data: |binary1|binary2|... * Trailer: |offset1(32-bit)|offset2(32-bit)|...| num_elems (32-bit) + * + * The decode pipeline is 7 steps: + * 1. parse header (validate sizes + extract num_elems + iteration bounds) + * 2. scan entries: record (data_start, out_len) per entry, sum total out_len + * 3. allocate the V1 output page (size_of_prefix + binary + offsets + trailer + tail) + * 4. write binary payload + * 5. write offsets array (running cursor over out_len) + * 6. write num_elems trailer + * 7. copy tail (footer + null map) and publish output params + * + * IS_CHAR=true picks the strnlen transform in step 2, so CHAR pages emit + * unpadded slices to the cached page. IS_CHAR=false keeps raw V2 lengths. + * The branch is `if constexpr` — compile-time dispatched, no overhead. */ +template struct BinaryPlainPageV2PreDecoder : public DataPagePreDecoder { - /** - * @brief Decode BinaryPlainPageV2 data to BinaryPlainPage format - * - * @param page unique_ptr to hold page data, will be replaced by decoded data - * @param page_slice data to decode, will be updated to point to decoded data - * @param size_of_tail including size of footer and null map - * @param _use_cache whether to use page cache - * @param page_type the type of page - * @param file_path file path for error reporting - * @param size_of_prefix size of prefix space to reserve (for dict page header) - * @return Status - */ Status decode(std::unique_ptr* page, Slice* page_slice, size_t size_of_tail, bool _use_cache, segment_v2::PageTypePB page_type, const std::string& file_path, size_t size_of_prefix = 0) override { - // Validate input - if (page_slice->size < sizeof(uint32_t) + size_of_tail) { - return Status::Corruption("Invalid page size: {}, expected at least {} in file: {}", - page_slice->size, sizeof(uint32_t) + size_of_tail, file_path); + // Step 1. + Slice data; + uint32_t num_elems = 0; + const uint8_t* ptr = nullptr; + const uint8_t* limit = nullptr; + RETURN_IF_ERROR(parse_header(*page_slice, size_of_tail, file_path, &data, &num_elems, &ptr, + &limit)); + + // Step 2: out_len derived per IS_CHAR. + std::vector entries; + entries.reserve(num_elems); + uint32_t total_out_len = 0; + for (uint32_t i = 0; i < num_elems; ++i) { + uint32_t raw_len = 0; + const uint8_t* data_start = nullptr; + RETURN_IF_ERROR(decode_one(ptr, limit, file_path, i, &data_start, &raw_len)); + uint32_t out_len; + if constexpr (IS_CHAR) { + out_len = static_cast( + strnlen(reinterpret_cast(data_start), raw_len)); + } else { + out_len = raw_len; + } + entries.push_back({data_start, out_len}); + total_out_len += out_len; + ptr = data_start + raw_len; } - // Calculate data portion (excluding tail) - Slice data(page_slice->data, page_slice->size - size_of_tail); + // Steps 3-7. + return write_binary_plain_v1_output(entries, num_elems, total_out_len, *page_slice, + size_of_tail, size_of_prefix, _use_cache, page_type, + page, page_slice); + } - // Read num_elems from the last 4 bytes of data portion - if (data.size < sizeof(uint32_t)) { - return Status::Corruption("Data too small to contain num_elems in file: {}", file_path); +private: + // Step 1: validate the V2 page and extract iteration bounds. + static inline Status parse_header(const Slice& page_slice, size_t size_of_tail, + const std::string& file_path, Slice* out_data, + uint32_t* out_num_elems, const uint8_t** out_ptr, + const uint8_t** out_limit) { + if (page_slice.size < sizeof(uint32_t) + size_of_tail) { + return Status::Corruption("Invalid page size: {}, expected at least {} in file: {}", + page_slice.size, sizeof(uint32_t) + size_of_tail, file_path); } - - uint32_t num_elems = decode_fixed32_le( - reinterpret_cast(&data[data.size - sizeof(uint32_t)])); - - // Calculate required size for V1 format - // V1 format: binary_data + offsets_array + num_elems + tail - // We need to parse V2 to calculate the total binary data size - const auto* ptr = reinterpret_cast(data.data); - const uint8_t* limit = ptr + data.size - sizeof(uint32_t); - - std::vector offsets; - offsets.reserve(num_elems); - - uint32_t current_offset = 0; - for (uint32_t i = 0; i < num_elems; i++) { - if (ptr >= limit) { - return Status::Corruption( - "Unexpected end of data while parsing element {} in file: {}", i, - file_path); - } - - // Decode varuint length - uint32_t length; - const uint8_t* data_start = decode_varint32_ptr(ptr, limit, &length); - if (data_start == nullptr) { - return Status::Corruption("Failed to decode varuint for element {} in file: {}", i, - file_path); - } - - // Store offset for this element - offsets.push_back(current_offset); - current_offset += length; - - // Move to next entry - ptr = data_start + length; - - if (ptr > limit) { - return Status::Corruption("Data extends beyond page for element {} in file: {}", i, - file_path); - } + *out_data = Slice(page_slice.data, page_slice.size - size_of_tail); + if (out_data->size < sizeof(uint32_t)) { + return Status::Corruption("Data too small to contain num_elems in file: {}", file_path); } + *out_num_elems = decode_fixed32_le( + reinterpret_cast(&(*out_data)[out_data->size - sizeof(uint32_t)])); + *out_ptr = reinterpret_cast(out_data->data); + *out_limit = *out_ptr + out_data->size - sizeof(uint32_t); + return Status::OK(); + } - // Calculate size for V1 format - size_t binary_data_size = current_offset; - size_t offsets_size = num_elems * sizeof(uint32_t); - size_t v1_data_size = binary_data_size + offsets_size + sizeof(uint32_t); - size_t total_size = size_of_prefix + v1_data_size + size_of_tail; - - // Allocate new page - Slice decoded_slice; - decoded_slice.size = total_size; - std::unique_ptr decoded_page = - std::make_unique(decoded_slice.size, _use_cache, page_type); - decoded_slice.data = decoded_page->data(); - - // Write V1 format data after the prefix - char* output = decoded_slice.data + size_of_prefix; - - // Step 1: Write binary data (without varint prefixes) - ptr = reinterpret_cast(data.data); - for (uint32_t i = 0; i < num_elems; i++) { - uint32_t length; - const uint8_t* data_start = decode_varint32_ptr(ptr, limit, &length); - - // Copy binary data - memcpy(output, data_start, length); - output += length; - - // Move to next entry - ptr = data_start + length; + // Step 2 helper: decode one varint length and validate the entry bounds. + // The scan loop in decode() / overrides walks the input with this helper + // and decides what `out_len` to record (raw_len here, strnlen'd in the + // CHAR variant). + static inline Status decode_one(const uint8_t* ptr, const uint8_t* limit, + const std::string& file_path, uint32_t i, + const uint8_t** out_data_start, uint32_t* out_raw_len) { + if (ptr >= limit) { + return Status::Corruption("Unexpected end of data while parsing element {} in file: {}", + i, file_path); } - - // Step 2: Write offsets array - for (uint32_t offset : offsets) { - encode_fixed32_le(reinterpret_cast(output), offset); - output += sizeof(uint32_t); + const uint8_t* data_start = decode_varint32_ptr(ptr, limit, out_raw_len); + if (data_start == nullptr) { + return Status::Corruption("Failed to decode varuint for element {} in file: {}", i, + file_path); } - - // Step 3: Write num_elems - encode_fixed32_le(reinterpret_cast(output), num_elems); - output += sizeof(uint32_t); - - // Step 4: Copy tail (footer and null map) - if (size_of_tail > 0) { - memcpy(output, page_slice->data + page_slice->size - size_of_tail, size_of_tail); + if (data_start + *out_raw_len > limit) { + return Status::Corruption("Data extends beyond page for element {} in file: {}", i, + file_path); } - - // Update output parameters - *page_slice = decoded_slice; - *page = std::move(decoded_page); - + *out_data_start = data_start; return Status::OK(); } }; diff --git a/be/src/storage/segment/binary_prefix_page.h b/be/src/storage/segment/binary_prefix_page.h index ce2b363e934635..3e7da14b476d9e 100644 --- a/be/src/storage/segment/binary_prefix_page.h +++ b/be/src/storage/segment/binary_prefix_page.h @@ -94,7 +94,7 @@ class BinaryPrefixPageBuilder : public PageBuilderHelperread_page(_opts, _reader->get_dict_page_pointer(), &_dict_page_handle, &dict_data, &dict_footer, _compress_codec)); const EncodingInfo* encoding_info; - RETURN_IF_ERROR(EncodingInfo::get(FieldType::OLAP_FIELD_TYPE_VARCHAR, + // The dict pool stores strings of the outer column's type. Using the + // outer type (CHAR vs VARCHAR/STRING) lets the EncodingInfo pick a + // CHAR-strip pre-decoder so the cached dict page is already unpadded. + RETURN_IF_ERROR(EncodingInfo::get(_reader->get_meta_type(), dict_footer.dict_page_footer().encoding(), &encoding_info)); RETURN_IF_ERROR(encoding_info->create_page_decoder(dict_data, {}, _dict_decoder)); RETURN_IF_ERROR(_dict_decoder->init()); diff --git a/be/src/storage/segment/encoding_info.cpp b/be/src/storage/segment/encoding_info.cpp index 30700fe0f41fd0..752627c2e285fe 100644 --- a/be/src/storage/segment/encoding_info.cpp +++ b/be/src/storage/segment/encoding_info.cpp @@ -33,6 +33,7 @@ #include "storage/segment/binary_dict_page.h" #include "storage/segment/binary_dict_page_pre_decoder.h" #include "storage/segment/binary_plain_page.h" +#include "storage/segment/binary_plain_page_char_strip_pre_decoder.h" #include "storage/segment/binary_plain_page_v2.h" #include "storage/segment/binary_plain_page_v2_pre_decoder.h" #include "storage/segment/binary_prefix_page.h" @@ -433,15 +434,29 @@ EncodingInfo::EncodingInfo(TraitsClass traits) if (_encoding == BIT_SHUFFLE) { _data_page_pre_decoder = std::make_unique(); } else if (_encoding == DICT_ENCODING) { - _data_page_pre_decoder = std::make_unique(); + if constexpr (TraitsClass::type == FieldType::OLAP_FIELD_TYPE_CHAR) { + _data_page_pre_decoder = std::make_unique>(); + } else { + _data_page_pre_decoder = std::make_unique>(); + } + } else if (_encoding == PLAIN_ENCODING) { + // CHAR plain pages may contain trailing '\0' padding written by older + // BEs; strip it once at page load so the cached page is unpadded. + if constexpr (TraitsClass::type == FieldType::OLAP_FIELD_TYPE_CHAR) { + _data_page_pre_decoder = std::make_unique(); + } } else if (_encoding == PLAIN_ENCODING_V2) { // Only binary types (Slice) need the predecoder for PLAIN_ENCODING_V2 — it converts // varint-encoded lengths to an offset-array format that downstream Slice decoders expect. - // All current (type, PLAIN_ENCODING_V2) registrations are Slice (CHAR/VARCHAR/STRING/ - // JSONB/VARIANT/HLL/BITMAP/QUANTILE_STATE/AGG_STATE per storage/types.h). The else throws - // at construction time to fail loudly if a future non-Slice registration is added. - if constexpr (std::is_same_v) { - _data_page_pre_decoder = std::make_unique(); + // CHAR pages additionally strip trailing '\0' padding written by the convertor; other + // Slice types use the non-CHAR specialization. All current (type, PLAIN_ENCODING_V2) + // registrations are Slice (CHAR/VARCHAR/STRING/JSONB/VARIANT/HLL/BITMAP/QUANTILE_STATE/ + // AGG_STATE per storage/types.h). The else throws at construction time to fail loudly + // if a future non-Slice registration is added. + if constexpr (TraitsClass::type == FieldType::OLAP_FIELD_TYPE_CHAR) { + _data_page_pre_decoder = std::make_unique>(); + } else if constexpr (std::is_same_v) { + _data_page_pre_decoder = std::make_unique>(); } else { throw Exception(Status::FatalError( "PLAIN_ENCODING_V2 is only supported for Slice (binary) types, but got " diff --git a/be/src/storage/segment/page_io.cpp b/be/src/storage/segment/page_io.cpp index 72d33105e52823..f41a425477781e 100644 --- a/be/src/storage/segment/page_io.cpp +++ b/be/src/storage/segment/page_io.cpp @@ -233,11 +233,15 @@ Status PageIO::read_and_decompress_page_(const PageReadOptions& opts, PageHandle if (opts.pre_decode) { const auto* encoding_info = opts.encoding_info; if (footer->type() == DICTIONARY_PAGE) { - // dict page uses its own encoding from footer->dict_page_footer().encoding() - // to look up the pre_decoder - RETURN_IF_ERROR(EncodingInfo::get(FieldType::OLAP_FIELD_TYPE_VARCHAR, - footer->dict_page_footer().encoding(), - &encoding_info)); + // Look up the dict page's encoding_info using the outer column's + // field type (not a hardcoded VARCHAR), so CHAR columns reach the + // CharStrip pre-decoder for the dict pool too. Falls back to + // VARCHAR only when the caller didn't set opts.encoding_info. + FieldType dict_field_type = opts.encoding_info != nullptr + ? opts.encoding_info->type() + : FieldType::OLAP_FIELD_TYPE_VARCHAR; + RETURN_IF_ERROR(EncodingInfo::get( + dict_field_type, footer->dict_page_footer().encoding(), &encoding_info)); } if (encoding_info) { auto* pre_decoder = encoding_info->get_data_page_pre_decoder(); diff --git a/be/src/storage/segment/segment_iterator.cpp b/be/src/storage/segment/segment_iterator.cpp index 87a4ce53e9f4fd..1d1311e33725b1 100644 --- a/be/src/storage/segment/segment_iterator.cpp +++ b/be/src/storage/segment/segment_iterator.cpp @@ -538,7 +538,7 @@ Status SegmentIterator::_lazy_init(Block* block) { } _current_return_columns.resize(_schema->columns().size()); - _vec_init_char_column_id(block); + _vec_init_char_column_id(); for (size_t i = 0; i < _schema->column_ids().size(); i++) { ColumnId cid = _schema->column_ids()[i]; const auto* column_desc = _schema->column(cid); @@ -1695,13 +1695,6 @@ Status SegmentIterator::_lookup_ordinal_from_sk_index(const RowCursor& key, bool const auto& key_col_ids = key.schema()->column_ids(); - // Clone the key once and pad CHAR fields to storage format before the binary search. - // _seek_block holds storage-format data where CHAR is zero-padded to column length, - // while RowCursor holds CHAR in compute format (unpadded). Padding once here avoids - // repeated allocation inside the comparison loop. - RowCursor padded_key = key.clone(); - padded_key.pad_char_fields(); - ssize_t start_block_id = 0; auto start_iter = sk_index_decoder->lower_bound(index_key); if (start_iter.valid()) { @@ -1729,7 +1722,7 @@ Status SegmentIterator::_lookup_ordinal_from_sk_index(const RowCursor& key, bool while (start < end) { rowid_t mid = (start + end) / 2; RETURN_IF_ERROR(_seek_and_peek(mid)); - int cmp = _compare_short_key_with_seek_block(padded_key, key_col_ids); + int cmp = _compare_short_key_with_seek_block(key, key_col_ids); if (cmp > 0) { start = mid + 1; } else if (cmp == 0) { @@ -2085,29 +2078,8 @@ bool SegmentIterator::_can_evaluated_by_vectorized(std::shared_ptrcolumns().size(), false); @@ -2115,14 +2087,6 @@ void SegmentIterator::_vec_init_char_column_id(Block* block) { auto cid = _schema->column_id(i); const TabletColumn* column_desc = _schema->column(cid); - // The additional deleted filter condition will be in the materialized column at the end of the block. - // After _output_column_by_sel_idx, it will be erased, so we do not need to shrink it. - if (i < block->columns()) { - if (_has_char_type(*column_desc)) { - _char_type_idx.emplace_back(i); - } - } - if (column_desc->type() == FieldType::OLAP_FIELD_TYPE_CHAR) { _is_char_type[cid] = true; } @@ -2825,8 +2789,6 @@ Status SegmentIterator::_next_batch_internal(Block* block) { _output_index_result_column(vir_ctxs, sel_rowid_idx, _selected_size, block); } RETURN_IF_ERROR(_materialization_of_virtual_column(block)); - // shrink char_type suffix zero data - block->shrink_char_type_column_suffix_zero(_char_type_idx); return _check_output_block(block); } @@ -2936,7 +2898,6 @@ Status SegmentIterator::_process_common_expr(uint16_t* sel_rowid_idx, uint16_t& common_ctxs.push_back(ctx.get()); } _output_index_result_column(common_ctxs, _sel_rowid_idx.data(), _selected_size, block); - block->shrink_char_type_column_suffix_zero(_char_type_idx); RETURN_IF_ERROR(_execute_common_expr(_sel_rowid_idx.data(), _selected_size, block)); if (need_mock_col) { @@ -3330,7 +3291,6 @@ Status SegmentIterator::_materialization_of_virtual_column(Block* block) { idx_in_block, block->columns(), _vir_cid_to_idx_in_block.size(), column_expr->root()->debug_string()); } - block->shrink_char_type_column_suffix_zero(_char_type_idx); if (check_and_get_column( block->get_by_position(idx_in_block).column.get())) { VLOG_DEBUG << fmt::format("Virtual column is doing materialization, cid {}, col idx {}", diff --git a/be/src/storage/segment/segment_iterator.h b/be/src/storage/segment/segment_iterator.h index e828e6495ddea0..f050cbb7f0e4ea 100644 --- a/be/src/storage/segment/segment_iterator.h +++ b/be/src/storage/segment/segment_iterator.h @@ -203,11 +203,7 @@ class SegmentIterator : public RowwiseIterator { bool _is_literal_node(const TExprNodeType::type& node_type); Status _vec_init_lazy_materialization(); - // TODO: Fix Me - // CHAR type in storage layer padding the 0 in length. But query engine need ignore the padding 0. - // so segment iterator need to shrink char column before output it. only use in vec query engine. - void _vec_init_char_column_id(Block* block); - bool _has_char_type(const TabletColumn& column_desc); + void _vec_init_char_column_id(); uint32_t segment_id() const { return _segment->id(); } uint32_t num_rows() const { return _segment->num_rows(); } @@ -427,8 +423,6 @@ class SegmentIterator : public RowwiseIterator { io::FileReaderSPtr _file_reader; - // char_type or array type columns cid - std::vector _char_type_idx; std::vector _is_char_type; // used for compaction, record selectd rowids of current batch diff --git a/be/test/core/block/block_test.cpp b/be/test/core/block/block_test.cpp index 1bb930bb15a6de..40a6d2ee99ebfa 100644 --- a/be/test/core/block/block_test.cpp +++ b/be/test/core/block/block_test.cpp @@ -1241,8 +1241,6 @@ TEST(BlockTest, others) { auto block = ColumnHelper::create_block({1, 2, 3}); block.insert(ColumnHelper::create_column_with_name({"abc", "efg", "hij"})); - block.shrink_char_type_column_suffix_zero({1, 2}); - SipHash hash; block.update_hash(hash); diff --git a/be/test/core/column/column_array_test.cpp b/be/test/core/column/column_array_test.cpp index e8c0bd4467898c..3b723d8881b281 100644 --- a/be/test/core/column/column_array_test.cpp +++ b/be/test/core/column/column_array_test.cpp @@ -605,11 +605,6 @@ TEST_F(ColumnArrayTest, ColumnStringFuncsTest) { assert_column_string_funcs(array_columns); } -// test shrink_padding_chars_callback -TEST_F(ColumnArrayTest, ShrinkPaddingCharsTest) { - shrink_padding_chars_callback(array_columns, serdes); -} - //////////////////////// special function from column_array.h //////////////////////// TEST_F(ColumnArrayTest, CreateArrayTest) { // Test ColumnArray constructor constraints: nested_column and offsets_column must not be ColumnConst. diff --git a/be/test/core/column/column_string_test.cpp b/be/test/core/column/column_string_test.cpp index 7215f167e20139..511ee87dff0c14 100644 --- a/be/test/core/column/column_string_test.cpp +++ b/be/test/core/column/column_string_test.cpp @@ -1312,28 +1312,6 @@ TEST_F(ColumnStringTest, TestStringInsert) { } } } -TEST_F(ColumnStringTest, shrink_padding_chars) { - ColumnString::MutablePtr col = ColumnString::create(); - col->shrink_padding_chars(); - - col->insert_data("123\0 ", 7); - col->insert_data("456\0xx", 6); - col->insert_data("78", 2); - col->shrink_padding_chars(); - - EXPECT_EQ(col->size(), 3); - EXPECT_EQ(col->get_data_at(0), StringRef("123")); - EXPECT_EQ(col->get_data_at(0).size, 3); - EXPECT_EQ(col->get_data_at(1), StringRef("456")); - EXPECT_EQ(col->get_data_at(1).size, 3); - EXPECT_EQ(col->get_data_at(2), StringRef("78")); - EXPECT_EQ(col->get_data_at(2).size, 2); - - col->insert_data("xyz", 2); // only xy - - EXPECT_EQ(col->size(), 4); - EXPECT_EQ(col->get_data_at(3), StringRef("xy")); -} TEST_F(ColumnStringTest, sort_column) { column_string_common_test(assert_sort_column_callback, false); } diff --git a/be/test/core/column/common_column_test.h b/be/test/core/column/common_column_test.h index 51fc6294e78086..ec69af8818bfc3 100644 --- a/be/test/core/column/common_column_test.h +++ b/be/test/core/column/common_column_test.h @@ -2363,43 +2363,6 @@ class CommonColumnTest : public ::testing::Test { } } - // get_shrinked_column should only happened in char-type column or nested char-type column, - // other column just return the origin column without any data changed, so check file content should be the same as the origin column - // just shrink the end zeros for char-type column which happened in segmentIterator - // eg. column_desc: char(6), insert into char(3), the char(3) will padding the 3 zeros at the end for writing to disk. - // but we select should just print the char(3) without the padding zeros - // limit and topN operation will trigger this function call - void shrink_padding_chars_callback(MutableColumns& load_cols, DataTypeSerDeSPtrs serders) { - auto option = DataTypeSerDe::FormatOptions(); - std::vector> res; - for (size_t i = 0; i < load_cols.size(); i++) { - auto& source_column = load_cols[i]; - LOG(INFO) << "now we are in shrink_padding_chars column : " << load_cols[i]->get_name() - << " for column size : " << source_column->size(); - source_column->shrink_padding_chars(); - // check after get_shrinked_column: 1 in selector present the load cols data is selected and data should be default value - auto ser_col = ColumnString::create(); - ser_col->reserve(source_column->size()); - VectorBufferWriter buffer_writer(*ser_col.get()); - std::vector data; - data.push_back("column: " + source_column->get_name() + - " with shrinked column size: " + std::to_string(source_column->size())); - for (size_t j = 0; j < source_column->size(); ++j) { - if (auto st = serders[i]->serialize_one_cell_to_json(*source_column, j, - buffer_writer, option); - !st) { - LOG(ERROR) << "Failed to serialize column " << i << " at row " << j; - break; - } - buffer_writer.commit(); - std::string actual_str_value = ser_col->get_data_at(j).to_string(); - data.push_back(actual_str_value); - } - res.push_back(data); - } - check_res_file("shrink_padding_chars", res); - } - void assert_size_eq(MutableColumnPtr col, size_t expect_size) { EXPECT_EQ(col->size(), expect_size); } diff --git a/be/test/exprs/bloom_filter_func_test.cpp b/be/test/exprs/bloom_filter_func_test.cpp index 15406cbff7f26b..f06283ddf23cb6 100644 --- a/be/test/exprs/bloom_filter_func_test.cpp +++ b/be/test/exprs/bloom_filter_func_test.cpp @@ -570,8 +570,10 @@ TEST_F(BloomFilterFuncTest, FindFixedLenOlapEngine) { bloom_filter_func2.insert_fixed_len(string_column->clone(), 0); - StringRef strings[] = {StringRef("aa"), StringRef("bb"), StringRef("cc"), - StringRef("dd\0\0", 4), StringRef("ef\0\0", 4)}; + // CHAR padding is stripped at the page decoder now, so the runtime BF + // probe sees natural-length StringRefs; no trailing '\0' bytes here. + StringRef strings[] = {StringRef("aa"), StringRef("bb"), StringRef("cc"), StringRef("dd"), + StringRef("ef")}; PODArray offsets2(5); std::iota(offsets2.begin(), offsets2.end(), 0); diff --git a/be/test/storage/olap_type_test.cpp b/be/test/storage/olap_type_test.cpp index 05775b693e4430..741f75bf47b9ab 100644 --- a/be/test/storage/olap_type_test.cpp +++ b/be/test/storage/olap_type_test.cpp @@ -2083,4 +2083,52 @@ TEST_F(OlapTypeTest, timestamptz_type) { << "serde mismatch for TIMESTAMPTZ expected=" << tc.expected; } } + +// from_olap_string for string types (CHAR / VARCHAR / STRING) strnlens the +// input so any trailing '\0' bytes that came from a fixed-width CHAR write +// are dropped before the value lands in the Field. VARCHAR / STRING ZoneMap +// values do not normally carry trailing '\0' (the writers store the natural +// byte length), so strnlen is a no-op for them. +TEST_F(OlapTypeTest, from_olap_string_strings) { + struct Case { + PrimitiveType type; + std::string input; + std::string expected; + }; + std::vector cases = { + // CHAR(N) ZoneMap min/max from the convertor is padded with '\0' + // — strnlen recovers the logical content. + {TYPE_CHAR, std::string("abc", 3) + std::string(7, '\0'), "abc"}, + {TYPE_CHAR, std::string(10, '\0'), ""}, + {TYPE_CHAR, "alpha", "alpha"}, + // VARCHAR / STRING never carry trailing '\0' in their ZoneMap + // representation, so the helper is a transparent pass-through + // for the typical case. + {TYPE_VARCHAR, "hello", "hello"}, + {TYPE_STRING, "world\nline2", "world\nline2"}, + {TYPE_STRING, "", ""}, + }; + + for (const auto& tc : cases) { + auto data_type = DataTypeFactory::instance().create_data_type( + tc.type, /*is_nullable=*/false, /*precision=*/0, /*scale=*/0, + /*length=*/static_cast(tc.input.size())); + expect_from_storage_string_paths(data_type, tc.input, [&](const Field& field) { + EXPECT_EQ(field.get(), tc.expected) + << "type=" << static_cast(tc.type) << " input.size=" << tc.input.size(); + }); + } +} + +// VARCHAR / STRING values containing an embedded '\0' are truncated at the +// first '\0' — the same strnlen behaviour applies to all string types. This +// is acceptable in practice because Doris string columns do not store +// embedded NULs in their ZoneMap representation; the test pins the contract. +TEST_F(OlapTypeTest, from_olap_string_strings_embedded_null_truncates) { + auto data_type = DataTypeFactory::instance().create_data_type( + TYPE_VARCHAR, /*is_nullable=*/false, 0, 0, /*length=*/32); + expect_from_storage_string_paths(data_type, std::string("ab\0cd", 5), [](const Field& field) { + EXPECT_EQ(field.get(), "ab"); + }); +} } // namespace doris diff --git a/be/test/storage/segment/binary_dict_page_test.cpp b/be/test/storage/segment/binary_dict_page_test.cpp index 36d455c72a5000..79c76ba52d6d60 100644 --- a/be/test/storage/segment/binary_dict_page_test.cpp +++ b/be/test/storage/segment/binary_dict_page_test.cpp @@ -74,7 +74,7 @@ class BinaryDictPageTest : public testing::Test { std::unique_ptr& decoded_page) { // Apply pre-decode for BinaryPlainPageV2 if (encoding_type == PLAIN_ENCODING_V2) { - BinaryPlainPageV2PreDecoder pre_decoder; + BinaryPlainPageV2PreDecoder pre_decoder; Status status = pre_decoder.decode(&decoded_page, &dict_slice, 0, false, PageTypePB::DATA_PAGE, ""); if (!status.ok()) { @@ -107,7 +107,7 @@ class BinaryDictPageTest : public testing::Test { // Apply pre-decode for BinaryDictPage data pages // This method handles all encoding types (bitshuffle, plain V1, plain V2) Status apply_pre_decode(Slice& page_slice, std::unique_ptr& decoded_page) { - BinaryDictPagePreDecoder pre_decoder; + BinaryDictPagePreDecoder pre_decoder; return pre_decoder.decode(&decoded_page, &page_slice, 0, false, PageTypePB::DATA_PAGE, ""); } @@ -674,7 +674,7 @@ TEST_F(BinaryDictPageTest, TestConfigAffectsDictionaryPageEncoding) { // First apply pre-decode for BinaryPlainPageV2 Slice dict_page_slice = dict_slice.slice(); std::unique_ptr decoded_page; - BinaryPlainPageV2PreDecoder pre_decoder; + BinaryPlainPageV2PreDecoder pre_decoder; status = pre_decoder.decode(&decoded_page, &dict_page_slice, 0, false, PageTypePB::DATA_PAGE, ""); EXPECT_TRUE(status.ok()); diff --git a/be/test/storage/segment/binary_plain_page_v2_test.cpp b/be/test/storage/segment/binary_plain_page_v2_test.cpp index 29b74961427aaa..cc27f1ce95e4bd 100644 --- a/be/test/storage/segment/binary_plain_page_v2_test.cpp +++ b/be/test/storage/segment/binary_plain_page_v2_test.cpp @@ -27,6 +27,8 @@ #include "core/column/column_string.h" #include "storage/cache/page_cache.h" #include "storage/olap_common.h" +#include "storage/segment/binary_plain_page.h" +#include "storage/segment/binary_plain_page_char_strip_pre_decoder.h" #include "storage/segment/binary_plain_page_v2_pre_decoder.h" #include "storage/segment/page_builder.h" #include "storage/segment/page_decoder.h" @@ -43,7 +45,7 @@ class BinaryPlainPageV2Test : public testing::Test { // Helper method to apply pre-decode step for BinaryPlainPageV2 // Similar to decode_bitshuffle_page in BinaryDictPageTest Status apply_pre_decode(Slice& page_slice, std::unique_ptr& decoded_page) { - BinaryPlainPageV2PreDecoder pre_decoder; + BinaryPlainPageV2PreDecoder pre_decoder; return pre_decoder.decode(&decoded_page, &page_slice, 0, false, PageTypePB::DATA_PAGE, ""); } @@ -553,5 +555,146 @@ TEST_F(BinaryPlainPageV2Test, TestSeekAndRead) { EXPECT_EQ("e", string_column->get_data_at(2).to_string()); } +// CHAR-specific roundtrip: write padded slices (the on-disk format produced +// by OlapColumnDataConvertorChar) through both CHAR-strip pre-decoders and +// confirm the post-decode column surfaces the unpadded logical content. + +namespace { + +// Build padded slices of fixed width `pad_len`: each input `s` is copied into +// a buffer of size pad_len with trailing '\0' fill. +// Build a vector of zero-padded buffers of fixed width `pad_len`. The caller +// builds Slices into the returned buffers AFTER the vector is settled — keep +// Slice creation outside this helper so the buffers' addresses don't move +// underneath the Slices. +std::vector make_padded_buffers(const std::vector& logical, + size_t pad_len) { + std::vector buffers; + buffers.reserve(logical.size()); + for (const auto& s : logical) { + EXPECT_LE(s.size(), pad_len); + std::string buf = s; + buf.resize(pad_len, '\0'); + buffers.push_back(std::move(buf)); + } + return buffers; +} + +std::vector slices_from(const std::vector& buffers, size_t pad_len) { + std::vector slices; + slices.reserve(buffers.size()); + for (const auto& b : buffers) { + slices.emplace_back(b.data(), pad_len); + } + return slices; +} + +void verify_unpadded_column(MutableColumnPtr& column, const std::vector& expected) { + auto* str_col = assert_cast(column.get()); + ASSERT_EQ(expected.size(), str_col->size()); + for (size_t i = 0; i < expected.size(); ++i) { + auto got = str_col->get_data_at(i).to_string(); + EXPECT_EQ(expected[i], got) << "row " << i; + } +} + +} // namespace + +// V2 plain page with padded CHAR slices → BinaryPlainPageV2PreDecoder +// → BinaryPlainPageDecoder → strings come out unpadded. +TEST_F(BinaryPlainPageV2Test, CharStripPreDecoder_V2_RoundtripPaddedSlices) { + constexpr size_t pad_len = 8; + std::vector logical = {"a", "bc", "", "alpha", "alpaca12"}; + auto buffers = make_padded_buffers(logical, pad_len); + auto slices = slices_from(buffers, pad_len); + + PageBuilderOptions builder_options; + builder_options.data_page_size = 256 * 1024; + PageBuilder* builder_ptr = nullptr; + ASSERT_TRUE(BinaryPlainPageV2Builder::create(&builder_ptr, + builder_options) + .ok()); + std::unique_ptr wrapper(builder_ptr); + auto* page_builder = + static_cast*>(builder_ptr); + size_t count = slices.size(); + ASSERT_TRUE(page_builder->add(reinterpret_cast(slices.data()), &count).ok()); + ASSERT_EQ(slices.size(), count); + + OwnedSlice owned_slice; + ASSERT_TRUE(page_builder->finish(&owned_slice).ok()); + + // Run the CHAR-strip pre-decoder (the one EncodingInfo would pick for a + // CHAR PLAIN_ENCODING_V2 page). + Slice page_slice = owned_slice.slice(); + std::unique_ptr decoded_page; + BinaryPlainPageV2PreDecoder pre_decoder; + ASSERT_TRUE(pre_decoder + .decode(&decoded_page, &page_slice, /*size_of_tail=*/0, + /*use_cache=*/false, PageTypePB::DATA_PAGE, "") + .ok()); + + // Decode the resulting V1 layout and verify rows are unpadded. + PageDecoderOptions decoder_options; + BinaryPlainPageDecoder page_decoder(page_slice, + decoder_options); + ASSERT_TRUE(page_decoder.init().ok()); + ASSERT_EQ(logical.size(), page_decoder.count()); + + MutableColumnPtr column = ColumnString::create(); + size_t num_to_read = logical.size(); + ASSERT_TRUE(page_decoder.next_batch(&num_to_read, column).ok()); + ASSERT_EQ(logical.size(), num_to_read); + verify_unpadded_column(column, logical); +} + +// V1 plain page with padded CHAR slices → BinaryPlainPageCharStripPreDecoder +// → BinaryPlainPageDecoder → strings come out unpadded. This mirrors the +// V2 case above for the PLAIN_ENCODING path. +TEST_F(BinaryPlainPageV2Test, CharStripPreDecoder_V1_RoundtripPaddedSlices) { + constexpr size_t pad_len = 6; + std::vector logical = {"x", "abcd", "", "zzzzzz", "qw"}; + auto buffers = make_padded_buffers(logical, pad_len); + auto slices = slices_from(buffers, pad_len); + + // Build a V1 plain page (no varint length prefixes — data + offsets + + // num_elems trailer). + PageBuilderOptions builder_options; + builder_options.data_page_size = 256 * 1024; + PageBuilder* builder_ptr = nullptr; + ASSERT_TRUE(BinaryPlainPageBuilder::create(&builder_ptr, + builder_options) + .ok()); + std::unique_ptr wrapper(builder_ptr); + auto* page_builder = + static_cast*>(builder_ptr); + size_t count = slices.size(); + ASSERT_TRUE(page_builder->add(reinterpret_cast(slices.data()), &count).ok()); + ASSERT_EQ(slices.size(), count); + + OwnedSlice owned_slice; + ASSERT_TRUE(page_builder->finish(&owned_slice).ok()); + + Slice page_slice = owned_slice.slice(); + std::unique_ptr decoded_page; + BinaryPlainPageCharStripPreDecoder pre_decoder; + ASSERT_TRUE(pre_decoder + .decode(&decoded_page, &page_slice, /*size_of_tail=*/0, + /*use_cache=*/false, PageTypePB::DATA_PAGE, "") + .ok()); + + PageDecoderOptions decoder_options; + BinaryPlainPageDecoder page_decoder(page_slice, + decoder_options); + ASSERT_TRUE(page_decoder.init().ok()); + ASSERT_EQ(logical.size(), page_decoder.count()); + + MutableColumnPtr column = ColumnString::create(); + size_t num_to_read = logical.size(); + ASSERT_TRUE(page_decoder.next_batch(&num_to_read, column).ok()); + ASSERT_EQ(logical.size(), num_to_read); + verify_unpadded_column(column, logical); +} + } // namespace segment_v2 } // namespace doris diff --git a/be/test/storage/segment/encoding_info_test.cpp b/be/test/storage/segment/encoding_info_test.cpp index 641e84e0759fcf..0a60c914e86d85 100644 --- a/be/test/storage/segment/encoding_info_test.cpp +++ b/be/test/storage/segment/encoding_info_test.cpp @@ -28,6 +28,7 @@ #include "runtime/exec_env.h" #include "storage/olap_common.h" #include "storage/segment/binary_dict_page_pre_decoder.h" +#include "storage/segment/binary_plain_page_char_strip_pre_decoder.h" #include "storage/segment/binary_plain_page_v2_pre_decoder.h" #include "storage/segment/bitshuffle_page_pre_decoder.h" #include "storage/types.h" @@ -149,10 +150,15 @@ TEST_F(EncodingInfoTest, test_all_pre_decoders) { auto* pre_decoder = encoding_info->get_data_page_pre_decoder(); ASSERT_NE(nullptr, pre_decoder) << "Type " << static_cast(type) << " with DICT_ENCODING should have pre_decoder"; - auto* dict_decoder = dynamic_cast(pre_decoder); - EXPECT_NE(nullptr, dict_decoder) - << "Type " << static_cast(type) - << " with DICT_ENCODING should have BinaryDictPagePreDecoder"; + // CHAR uses the IS_CHAR=true specialization so it strips trailing '\0' + // padding from inline-binary dict fallbacks; other string types use + // the regular non-CHAR specialization. + bool is_dict_decoder = + (type == FieldType::OLAP_FIELD_TYPE_CHAR) + ? dynamic_cast*>(pre_decoder) != nullptr + : dynamic_cast*>(pre_decoder) != nullptr; + EXPECT_TRUE(is_dict_decoder) << "Type " << static_cast(type) + << " with DICT_ENCODING should have BinaryDictPagePreDecoder"; } // Test PLAIN_ENCODING_V2 with Slice types - should have BinaryPlainPageV2PreDecoder @@ -173,10 +179,16 @@ TEST_F(EncodingInfoTest, test_all_pre_decoders) { auto* pre_decoder = encoding_info->get_data_page_pre_decoder(); ASSERT_NE(nullptr, pre_decoder) << "Type " << static_cast(type) << " with PLAIN_ENCODING_V2 should have pre_decoder"; - auto* v2_decoder = dynamic_cast(pre_decoder); - EXPECT_NE(nullptr, v2_decoder) << "Type " << static_cast(type) - << " with PLAIN_ENCODING_V2 should have " - "BinaryPlainPageV2PreDecoder"; + // CHAR PLAIN_ENCODING_V2 is wired to BinaryPlainPageV2PreDecoder + // so the trailing '\0' padding written by OlapColumnDataConvertorChar + // is stripped at page load time; other binary types use the regular + // instantiation. + bool ok = + (type == FieldType::OLAP_FIELD_TYPE_CHAR) + ? dynamic_cast*>(pre_decoder) != nullptr + : dynamic_cast*>(pre_decoder) != nullptr; + EXPECT_TRUE(ok) << "Type " << static_cast(type) + << " with PLAIN_ENCODING_V2 should have V2 pre-decoder"; } // Test PLAIN_ENCODING - should NOT have pre_decoder @@ -216,8 +228,19 @@ TEST_F(EncodingInfoTest, test_all_pre_decoders) { auto status = EncodingInfo::get(type, PLAIN_ENCODING, &encoding_info); if (status.ok() && encoding_info != nullptr) { auto* pre_decoder = encoding_info->get_data_page_pre_decoder(); - EXPECT_EQ(nullptr, pre_decoder) << "Type " << static_cast(type) - << " with PLAIN_ENCODING should NOT have pre_decoder"; + if (type == FieldType::OLAP_FIELD_TYPE_CHAR) { + // CHAR PLAIN_ENCODING has a CHAR-strip pre-decoder that strips + // the trailing '\0' padding written by the convertor. + EXPECT_NE(nullptr, pre_decoder) + << "CHAR with PLAIN_ENCODING should have a pre_decoder"; + EXPECT_NE(nullptr, dynamic_cast(pre_decoder)) + << "CHAR PLAIN_ENCODING pre-decoder should be " + "BinaryPlainPageCharStripPreDecoder"; + } else { + EXPECT_EQ(nullptr, pre_decoder) + << "Type " << static_cast(type) + << " with PLAIN_ENCODING should NOT have pre_decoder"; + } } } diff --git a/be/test/storage/segment/zone_map_index_test.cpp b/be/test/storage/segment/zone_map_index_test.cpp index 6d0f59da648be8..8c8d0a7cd28f28 100644 --- a/be/test/storage/segment/zone_map_index_test.cpp +++ b/be/test/storage/segment/zone_map_index_test.cpp @@ -249,16 +249,16 @@ class ColumnZoneMapTest : public testing::Test { DataTypeFactory::instance().create_data_type(TYPE_CHAR, true, 0, 0, length); auto tab_col = create_char_key(0, true, length); const TabletColumn* field = tab_col.get(); - std::string s_less_than_schema_length1(length - 1, 'a'); - std::string s_less_than_schema_length1_expect(length, 'a'); - s_less_than_schema_length1_expect[length - 1] = '\0'; - std::string s_less_than_schema_length2(length - 2, 'b'); - std::string s_less_than_schema_length2_expect(length, 'b'); - s_less_than_schema_length2_expect[length - 1] = '\0'; - s_less_than_schema_length2_expect[length - 2] = '\0'; + // ZoneMap writer stores whatever slice bytes it receives. In production + // OlapColumnDataConvertorChar pads CHAR slices to the declared length + // before they reach the writer; from_olap_string strnlens at read time + // so the materialized Field is always unpadded. This test passes raw + // shorter slices directly to the writer to exercise the strnlen path. + std::string s_less_than_char_len1(length - 1, 'a'); + std::string s_less_than_char_len2(length - 2, 'b'); std::unique_ptr writer; ASSERT_TRUE(ZoneMapIndexWriter::create(data_type, field, writer).ok()); - Slice slices[] = {Slice(s_less_than_schema_length1), Slice(s_less_than_schema_length2)}; + Slice slices[] = {Slice(s_less_than_char_len1), Slice(s_less_than_char_len2)}; writer->add_values(&slices, 2); if (pass_all) { writer->reset_page_zone_map(); @@ -274,13 +274,13 @@ class ColumnZoneMapTest : public testing::Test { ASSERT_TRUE(file_writer->close().ok()); const auto& seg_zm = index_meta.zone_map_index().segment_zone_map(); - // Min/Max should be truncated to MAX_ZONE_MAP_INDEX_SIZE and last byte of Max is incremented - EXPECT_EQ(seg_zm.min().size(), s_less_than_schema_length1.size()); - EXPECT_EQ(seg_zm.min(), s_less_than_schema_length1); - EXPECT_EQ(seg_zm.max().size(), s_less_than_schema_length2.size()); - EXPECT_EQ(seg_zm.max(), s_less_than_schema_length2); + // On-disk min/max preserve the raw (unpadded) bytes. + EXPECT_EQ(seg_zm.min().size(), s_less_than_char_len1.size()); + EXPECT_EQ(seg_zm.min(), s_less_than_char_len1); + EXPECT_EQ(seg_zm.max().size(), s_less_than_char_len2.size()); + EXPECT_EQ(seg_zm.max(), s_less_than_char_len2); - // Verify ZoneMap::from_proto can correctly parse the truncated zone map + // Verify ZoneMap::from_proto materializes the unpadded Field. ZoneMap seg_zone_map; ASSERT_TRUE(ZoneMap::from_proto(seg_zm, data_type, seg_zone_map).ok()); EXPECT_EQ(seg_zone_map.has_null, false); @@ -289,10 +289,10 @@ class ColumnZoneMapTest : public testing::Test { EXPECT_EQ(seg_zone_map.has_positive_inf, false); EXPECT_EQ(seg_zone_map.has_negative_inf, false); EXPECT_EQ(seg_zone_map.has_nan, false); - EXPECT_EQ(seg_zone_map.min_value.get().size(), length); - EXPECT_EQ(seg_zone_map.min_value.get(), s_less_than_schema_length1_expect); - EXPECT_EQ(seg_zone_map.max_value.get().size(), length); - EXPECT_EQ(seg_zone_map.max_value.get(), s_less_than_schema_length2_expect); + EXPECT_EQ(seg_zone_map.min_value.get().size(), s_less_than_char_len1.size()); + EXPECT_EQ(seg_zone_map.min_value.get(), s_less_than_char_len1); + EXPECT_EQ(seg_zone_map.max_value.get().size(), s_less_than_char_len2.size()); + EXPECT_EQ(seg_zone_map.max_value.get(), s_less_than_char_len2); io::FileReaderSPtr file_reader; EXPECT_TRUE(_fs->open_file(file_path, &file_reader).ok()); @@ -315,12 +315,12 @@ class ColumnZoneMapTest : public testing::Test { EXPECT_EQ(page_zone_map.has_negative_inf, false); EXPECT_EQ(page_zone_map.has_nan, false); if (!pass_all) { - EXPECT_EQ(page_zone_map.min_value.get().size(), length); - EXPECT_EQ(page_zone_map.min_value.get(), - s_less_than_schema_length1_expect); - EXPECT_EQ(page_zone_map.max_value.get().size(), length); - EXPECT_EQ(page_zone_map.max_value.get(), - s_less_than_schema_length2_expect); + EXPECT_EQ(page_zone_map.min_value.get().size(), + s_less_than_char_len1.size()); + EXPECT_EQ(page_zone_map.min_value.get(), s_less_than_char_len1); + EXPECT_EQ(page_zone_map.max_value.get().size(), + s_less_than_char_len2.size()); + EXPECT_EQ(page_zone_map.max_value.get(), s_less_than_char_len2); } } } From 68b95d1af7cee277bef2792cab98b7889e2b310b Mon Sep 17 00:00:00 2001 From: Chenyang Sun Date: Wed, 10 Jun 2026 18:11:00 +0800 Subject: [PATCH 5/5] [refactor](be) Derive get_storage_field_type from primitive type (#64341) 1. Make IDataType::get_storage_field_type() derive the field type from the primitive type 2. SegmentIterator::_is_char_type (and _vec_init_char_column_id) is deleted Issue Number: close #xxx Related PR: #xxx Problem Summary: ### Release note None ### Check List (For Author) - Test - [ ] Regression test - [ ] Unit Test - [ ] Manual test (add detailed scripts or steps below) - [ ] No need to test or manual test. Explain why: - [x] This is a refactor/code format and no logic has been changed. - [ ] Previous test can cover this change. - [ ] No code files have been changed. - [ ] Other reason - Behavior changed: - [ ] No. - [ ] Yes. - Does this need documentation? - [ ] No. - [ ] Yes. ### Check List (For Reviewer who merge this PR) - [ ] Confirm the release note - [ ] Confirm test cases - [ ] Confirm document - [ ] Add branch pick label Co-authored-by: Claude Opus 4.8 (cherry picked from commit 9d3c35bc9b56385860ae829c8bbb0fc28e2234fe) --- be/src/core/data_type/data_type.cpp | 5 ++ be/src/core/data_type/data_type.h | 4 +- be/src/core/data_type/data_type_agg_state.h | 4 -- be/src/core/data_type/data_type_array.h | 4 -- be/src/core/data_type/data_type_bitmap.h | 4 -- be/src/core/data_type/data_type_date.h | 3 -- .../data_type/data_type_date_or_datetime_v2.h | 6 --- be/src/core/data_type/data_type_date_time.h | 4 -- be/src/core/data_type/data_type_hll.h | 4 -- be/src/core/data_type/data_type_ipv4.h | 4 -- be/src/core/data_type/data_type_ipv6.h | 3 -- be/src/core/data_type/data_type_jsonb.h | 3 -- be/src/core/data_type/data_type_map.h | 3 -- be/src/core/data_type/data_type_number_base.h | 53 ------------------- .../core/data_type/data_type_quantilestate.h | 3 -- be/src/core/data_type/data_type_string.h | 4 -- be/src/core/data_type/data_type_struct.h | 3 -- be/src/core/data_type/data_type_variant.h | 3 -- be/src/storage/segment/segment_iterator.cpp | 22 +------- be/src/storage/segment/segment_iterator.h | 3 -- 20 files changed, 9 insertions(+), 133 deletions(-) diff --git a/be/src/core/data_type/data_type.cpp b/be/src/core/data_type/data_type.cpp index 60e5b6f2ee6204..2781e5df164925 100644 --- a/be/src/core/data_type/data_type.cpp +++ b/be/src/core/data_type/data_type.cpp @@ -34,6 +34,7 @@ #include "core/data_type/define_primitive_type.h" #include "core/data_type_serde/data_type_serde.h" #include "core/field.h" +#include "storage/tablet/tablet_schema.h" namespace doris { class BufferWritable; @@ -46,6 +47,10 @@ IDataType::IDataType() = default; IDataType::~IDataType() = default; +doris::FieldType IDataType::get_storage_field_type() const { + return TabletColumn::get_field_type_by_type(get_primitive_type()); +} + String IDataType::get_name() const { return do_get_name(); } diff --git a/be/src/core/data_type/data_type.h b/be/src/core/data_type/data_type.h index 76b534c1bf5101..bc18f704d81107 100644 --- a/be/src/core/data_type/data_type.h +++ b/be/src/core/data_type/data_type.h @@ -84,7 +84,9 @@ class IDataType : private boost::noncopyable { virtual const std::string get_family_name() const = 0; virtual PrimitiveType get_primitive_type() const = 0; - virtual doris::FieldType get_storage_field_type() const = 0; + // Derived from the primitive type by default (e.g. TYPE_CHAR -> OLAP_FIELD_TYPE_CHAR). + // Types without a direct 1:1 mapping override this. + virtual doris::FieldType get_storage_field_type() const; std::string to_string(const IColumn& column, size_t row_num, const DataTypeSerDe::FormatOptions& options) const; // get specific serializer or deserializer diff --git a/be/src/core/data_type/data_type_agg_state.h b/be/src/core/data_type/data_type_agg_state.h index f1513c97c76a54..378315f9512e84 100644 --- a/be/src/core/data_type/data_type_agg_state.h +++ b/be/src/core/data_type/data_type_agg_state.h @@ -85,10 +85,6 @@ class DataTypeAggState : public DataTypeString { PrimitiveType get_primitive_type() const override { return PrimitiveType::TYPE_AGG_STATE; } - doris::FieldType get_storage_field_type() const override { - return doris::FieldType::OLAP_FIELD_TYPE_AGG_STATE; - } - const DataTypes& get_sub_types() const { return _sub_types; } void to_pb_column_meta(PColumnMeta* col_meta) const override { diff --git a/be/src/core/data_type/data_type_array.h b/be/src/core/data_type/data_type_array.h index 7579ad5cb2604e..f72e2c185376b5 100644 --- a/be/src/core/data_type/data_type_array.h +++ b/be/src/core/data_type/data_type_array.h @@ -57,10 +57,6 @@ class DataTypeArray final : public IDataType { PrimitiveType get_primitive_type() const override { return PrimitiveType::TYPE_ARRAY; } - doris::FieldType get_storage_field_type() const override { - return doris::FieldType::OLAP_FIELD_TYPE_ARRAY; - } - std::string do_get_name() const override { return "Array(" + nested->get_name() + ")"; } const std::string get_family_name() const override { return "Array"; } diff --git a/be/src/core/data_type/data_type_bitmap.h b/be/src/core/data_type/data_type_bitmap.h index c47e9b30a78c15..5e7fb8b67bac98 100644 --- a/be/src/core/data_type/data_type_bitmap.h +++ b/be/src/core/data_type/data_type_bitmap.h @@ -54,10 +54,6 @@ class DataTypeBitMap : public IDataType { const std::string get_family_name() const override { return "BitMap"; } PrimitiveType get_primitive_type() const override { return PrimitiveType::TYPE_BITMAP; } - doris::FieldType get_storage_field_type() const override { - return doris::FieldType::OLAP_FIELD_TYPE_BITMAP; - } - int64_t get_uncompressed_serialized_bytes(const IColumn& column, int be_exec_version) const override; char* serialize(const IColumn& column, char* buf, int be_exec_version) const override; diff --git a/be/src/core/data_type/data_type_date.h b/be/src/core/data_type/data_type_date.h index 8501e0322d3660..10068fcef0706b 100644 --- a/be/src/core/data_type/data_type_date.h +++ b/be/src/core/data_type/data_type_date.h @@ -43,9 +43,6 @@ class DataTypeDate final : public DataTypeNumberBase { static constexpr PrimitiveType PType = TYPE_DATE; PrimitiveType get_primitive_type() const override { return PrimitiveType::TYPE_DATE; } - doris::FieldType get_storage_field_type() const override { - return doris::FieldType::OLAP_FIELD_TYPE_DATE; - } const std::string get_family_name() const override { return "Date"; } std::string do_get_name() const override { return "Date"; } diff --git a/be/src/core/data_type/data_type_date_or_datetime_v2.h b/be/src/core/data_type/data_type_date_or_datetime_v2.h index 4faba14cfab937..c033c0b2948f05 100644 --- a/be/src/core/data_type/data_type_date_or_datetime_v2.h +++ b/be/src/core/data_type/data_type_date_or_datetime_v2.h @@ -55,9 +55,6 @@ class DataTypeDateV2 final : public DataTypeNumberBaseset_scale(_scale); } - doris::FieldType get_storage_field_type() const override { - return doris::FieldType::OLAP_FIELD_TYPE_DATETIMEV2; - } const std::string get_family_name() const override { return "DateTimeV2"; } std::string do_get_name() const override { return "DateTimeV2(" + std::to_string(_scale) + ")"; diff --git a/be/src/core/data_type/data_type_date_time.h b/be/src/core/data_type/data_type_date_time.h index 6a52316907b097..b98b3f9022a2c4 100644 --- a/be/src/core/data_type/data_type_date_time.h +++ b/be/src/core/data_type/data_type_date_time.h @@ -71,10 +71,6 @@ class DataTypeDateTime final : public DataTypeNumberBase { const std::string get_family_name() const override { return "IPv4"; } std::string do_get_name() const override { return "IPv4"; } - doris::FieldType get_storage_field_type() const override { - return doris::FieldType::OLAP_FIELD_TYPE_IPV4; - } - bool equals(const IDataType& rhs) const override; Field get_field(const TExprNode& node) const override; diff --git a/be/src/core/data_type/data_type_ipv6.h b/be/src/core/data_type/data_type_ipv6.h index cc48d3c7284c99..2ab1b15763157e 100644 --- a/be/src/core/data_type/data_type_ipv6.h +++ b/be/src/core/data_type/data_type_ipv6.h @@ -42,9 +42,6 @@ namespace doris { class DataTypeIPv6 final : public DataTypeNumberBase { public: PrimitiveType get_primitive_type() const override { return PrimitiveType::TYPE_IPV6; } - doris::FieldType get_storage_field_type() const override { - return doris::FieldType::OLAP_FIELD_TYPE_IPV6; - } const std::string get_family_name() const override { return "IPv6"; } std::string do_get_name() const override { return "IPv6"; } diff --git a/be/src/core/data_type/data_type_jsonb.h b/be/src/core/data_type/data_type_jsonb.h index c7fe0af35af7ea..f85354a4b30d7a 100644 --- a/be/src/core/data_type/data_type_jsonb.h +++ b/be/src/core/data_type/data_type_jsonb.h @@ -51,9 +51,6 @@ class DataTypeJsonb final : public IDataType { const std::string get_family_name() const override { return "JSONB"; } PrimitiveType get_primitive_type() const override { return PrimitiveType::TYPE_JSONB; } - doris::FieldType get_storage_field_type() const override { - return doris::FieldType::OLAP_FIELD_TYPE_JSONB; - } int64_t get_uncompressed_serialized_bytes(const IColumn& column, int data_version) const override; diff --git a/be/src/core/data_type/data_type_map.h b/be/src/core/data_type/data_type_map.h index ccb155af330574..b442b64da9b52f 100644 --- a/be/src/core/data_type/data_type_map.h +++ b/be/src/core/data_type/data_type_map.h @@ -56,9 +56,6 @@ class DataTypeMap final : public IDataType { DataTypeMap(const DataTypePtr& key_type_, const DataTypePtr& value_type_); PrimitiveType get_primitive_type() const override { return PrimitiveType::TYPE_MAP; } - doris::FieldType get_storage_field_type() const override { - return doris::FieldType::OLAP_FIELD_TYPE_MAP; - } std::string do_get_name() const override { return "Map(" + key_type->get_name() + ", " + value_type->get_name() + ")"; diff --git a/be/src/core/data_type/data_type_number_base.h b/be/src/core/data_type/data_type_number_base.h index 4b3fb37406c6b5..eb8690c1516db9 100644 --- a/be/src/core/data_type/data_type_number_base.h +++ b/be/src/core/data_type/data_type_number_base.h @@ -67,59 +67,6 @@ class DataTypeNumberBase : public IDataType { return T; } - doris::FieldType get_storage_field_type() const override { - // Doris does not support uint8 at present, use uint8 as boolean type - if constexpr (T == TYPE_BOOLEAN) { - return doris::FieldType::OLAP_FIELD_TYPE_BOOL; - } - if constexpr (T == TYPE_TINYINT) { - return doris::FieldType::OLAP_FIELD_TYPE_TINYINT; - } - if constexpr (T == TYPE_SMALLINT) { - return doris::FieldType::OLAP_FIELD_TYPE_SMALLINT; - } - if constexpr (T == TYPE_INT) { - return doris::FieldType::OLAP_FIELD_TYPE_INT; - } - if constexpr (T == TYPE_BIGINT) { - return doris::FieldType::OLAP_FIELD_TYPE_BIGINT; - } - if constexpr (T == TYPE_LARGEINT) { - return doris::FieldType::OLAP_FIELD_TYPE_LARGEINT; - } - if constexpr (T == TYPE_FLOAT) { - return doris::FieldType::OLAP_FIELD_TYPE_FLOAT; - } - if constexpr (T == TYPE_DOUBLE) { - return doris::FieldType::OLAP_FIELD_TYPE_DOUBLE; - } - if constexpr (T == TYPE_DATE) { - return doris::FieldType::OLAP_FIELD_TYPE_DATE; - } - if constexpr (T == TYPE_DATETIME) { - return doris::FieldType::OLAP_FIELD_TYPE_DATETIME; - } - if constexpr (T == TYPE_DATEV2) { - return doris::FieldType::OLAP_FIELD_TYPE_DATEV2; - } - if constexpr (T == TYPE_DATETIMEV2) { - return doris::FieldType::OLAP_FIELD_TYPE_DATETIMEV2; - } - if constexpr (T == TYPE_TIMESTAMPTZ) { - return doris::FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ; - } - if constexpr (T == TYPE_IPV4) { - return doris::FieldType::OLAP_FIELD_TYPE_IPV4; - } - if constexpr (T == TYPE_IPV6) { - return doris::FieldType::OLAP_FIELD_TYPE_IPV6; - } - if constexpr (T == TYPE_TIMEV2) { - return doris::FieldType::OLAP_FIELD_TYPE_TIMEV2; - } - throw Exception(Status::FatalError("__builtin_unreachable")); - } - Field get_field(const TExprNode& node) const override; int64_t get_uncompressed_serialized_bytes(const IColumn& column, diff --git a/be/src/core/data_type/data_type_quantilestate.h b/be/src/core/data_type/data_type_quantilestate.h index 370f9bcb7d487e..d469f6b3588de2 100644 --- a/be/src/core/data_type/data_type_quantilestate.h +++ b/be/src/core/data_type/data_type_quantilestate.h @@ -51,9 +51,6 @@ class DataTypeQuantileState : public IDataType { const std::string get_family_name() const override { return "QuantileState"; } PrimitiveType get_primitive_type() const override { return PrimitiveType::TYPE_QUANTILE_STATE; } - doris::FieldType get_storage_field_type() const override { - return doris::FieldType::OLAP_FIELD_TYPE_QUANTILE_STATE; - } int64_t get_uncompressed_serialized_bytes(const IColumn& column, int be_exec_version) const override; char* serialize(const IColumn& column, char* buf, int be_exec_version) const override; diff --git a/be/src/core/data_type/data_type_string.h b/be/src/core/data_type/data_type_string.h index 32385b7afb1991..bfd86f7d9de8ac 100644 --- a/be/src/core/data_type/data_type_string.h +++ b/be/src/core/data_type/data_type_string.h @@ -55,10 +55,6 @@ class DataTypeString : public IDataType { } PrimitiveType get_primitive_type() const override { return _primitive_type; } - doris::FieldType get_storage_field_type() const override { - return doris::FieldType::OLAP_FIELD_TYPE_STRING; - } - int64_t get_uncompressed_serialized_bytes(const IColumn& column, int be_exec_version) const override; char* serialize(const IColumn& column, char* buf, int be_exec_version) const override; diff --git a/be/src/core/data_type/data_type_struct.h b/be/src/core/data_type/data_type_struct.h index 657364efbca52f..d4e4abd7a88829 100644 --- a/be/src/core/data_type/data_type_struct.h +++ b/be/src/core/data_type/data_type_struct.h @@ -70,9 +70,6 @@ class DataTypeStruct final : public IDataType { DataTypeStruct(const DataTypes& elems, const Strings& names); PrimitiveType get_primitive_type() const override { return PrimitiveType::TYPE_STRUCT; } - doris::FieldType get_storage_field_type() const override { - return doris::FieldType::OLAP_FIELD_TYPE_STRUCT; - } std::string do_get_name() const override; const std::string get_family_name() const override { return "Struct"; } diff --git a/be/src/core/data_type/data_type_variant.h b/be/src/core/data_type/data_type_variant.h index f8ec3484d6b686..3f4e08a0ada620 100644 --- a/be/src/core/data_type/data_type_variant.h +++ b/be/src/core/data_type/data_type_variant.h @@ -60,9 +60,6 @@ class DataTypeVariant : public IDataType { String do_get_name() const override { return name; } const std::string get_family_name() const override { return "Variant"; } - doris::FieldType get_storage_field_type() const override { - return doris::FieldType::OLAP_FIELD_TYPE_VARIANT; - } Status check_column(const IColumn& column) const override { return check_column_non_nested_type(column); } diff --git a/be/src/storage/segment/segment_iterator.cpp b/be/src/storage/segment/segment_iterator.cpp index 1d1311e33725b1..d5e9b68437e217 100644 --- a/be/src/storage/segment/segment_iterator.cpp +++ b/be/src/storage/segment/segment_iterator.cpp @@ -538,21 +538,16 @@ Status SegmentIterator::_lazy_init(Block* block) { } _current_return_columns.resize(_schema->columns().size()); - _vec_init_char_column_id(); for (size_t i = 0; i < _schema->column_ids().size(); i++) { ColumnId cid = _schema->column_ids()[i]; const auto* column_desc = _schema->column(cid); if (_is_pred_column[cid]) { auto storage_column_type = _storage_name_and_type[cid].second; - // Char type is special , since char type's computational datatype is same with string, - // both are DataTypeString, but DataTypeString only return FieldType::OLAP_FIELD_TYPE_STRING - // in get_storage_field_type. RETURN_IF_CATCH_EXCEPTION( // Here, cid will not go out of bounds // because the size of _current_return_columns equals _schema->tablet_columns().size() _current_return_columns[cid] = Schema::get_predicate_column_ptr( - _is_char_type[cid] ? FieldType::OLAP_FIELD_TYPE_CHAR - : storage_column_type->get_storage_field_type(), + storage_column_type->get_storage_field_type(), storage_column_type->is_nullable(), _opts.io_ctx.reader_type)); _current_return_columns[cid]->set_rowset_segment_id( {_segment->rowset_id(), _segment->id()}); @@ -2078,21 +2073,6 @@ bool SegmentIterator::_can_evaluated_by_vectorized(std::shared_ptrcolumns().size(), false); - for (size_t i = 0; i < _schema->num_column_ids(); i++) { - auto cid = _schema->column_id(i); - const TabletColumn* column_desc = _schema->column(cid); - - if (column_desc->type() == FieldType::OLAP_FIELD_TYPE_CHAR) { - _is_char_type[cid] = true; - } - } -} - bool SegmentIterator::_prune_column(ColumnId cid, MutableColumnPtr& column, bool fill_defaults, size_t num_of_defaults) { if (_need_read_data(cid)) { diff --git a/be/src/storage/segment/segment_iterator.h b/be/src/storage/segment/segment_iterator.h index f050cbb7f0e4ea..56c3d026747b69 100644 --- a/be/src/storage/segment/segment_iterator.h +++ b/be/src/storage/segment/segment_iterator.h @@ -203,7 +203,6 @@ class SegmentIterator : public RowwiseIterator { bool _is_literal_node(const TExprNodeType::type& node_type); Status _vec_init_lazy_materialization(); - void _vec_init_char_column_id(); uint32_t segment_id() const { return _segment->id(); } uint32_t num_rows() const { return _segment->num_rows(); } @@ -423,8 +422,6 @@ class SegmentIterator : public RowwiseIterator { io::FileReaderSPtr _file_reader; - std::vector _is_char_type; - // used for compaction, record selectd rowids of current batch uint16_t _selected_size; std::vector _sel_rowid_idx;