diff --git a/be/src/storage/delete/delete_bitmap_calculator.cpp b/be/src/storage/delete/delete_bitmap_calculator.cpp index bf63bf7c437540..a9750b4ce58972 100644 --- a/be/src/storage/delete/delete_bitmap_calculator.cpp +++ b/be/src/storage/delete/delete_bitmap_calculator.cpp @@ -164,14 +164,12 @@ Status MergeIndexDeleteBitmapCalculator::init(RowsetId rowset_id, auto pk_idx = segment->get_primary_key_index(); std::unique_ptr index; RETURN_IF_ERROR(pk_idx->new_iterator(&index, nullptr)); - auto index_type = - DataTypeFactory::instance().create_data_type(pk_idx->type_info()->type(), 1, 0); + auto index_type = DataTypeFactory::instance().create_data_type(pk_idx->type(), 1, 0); _contexts.emplace_back(std::move(index), index_type, segment->id(), pk_idx->num_rows()); _heap->push(&_contexts.back()); } if (_rowid_length > 0) { - _rowid_coder = get_key_coder( - get_scalar_type_info()->type()); + _rowid_coder = get_key_coder(FieldType::OLAP_FIELD_TYPE_UNSIGNED_INT); } }); return Status::OK(); diff --git a/be/src/storage/field.h b/be/src/storage/field.h index ab006b3f72a06a..3fa84c36f7b42d 100644 --- a/be/src/storage/field.h +++ b/be/src/storage/field.h @@ -40,7 +40,7 @@ namespace doris { class StorageField { public: StorageField(const TabletColumn& column) - : _type_info(get_type_info(&column)), + : _type(column.type()), _desc(column), _length(column.length()), _key_coder(get_key_coder(column.type())), @@ -54,7 +54,7 @@ class StorageField { virtual ~StorageField() = default; - size_t size() const { return _type_info->size(); } + size_t size() const { return field_type_size(_type); } size_t length() const { return _length; } size_t field_size() const { return size() + 1; } size_t index_size() const { return _index_size; } @@ -70,8 +70,7 @@ class StorageField { return local; } - FieldType type() const { return _type_info->type(); } - const TypeInfo* type_info() const { return _type_info.get(); } + FieldType type() const { return _type; } bool is_nullable() const { return _is_nullable; } // similar to `full_encode_ascending`, but only encode part (the first `index_size` bytes) of the value. @@ -103,7 +102,7 @@ class StorageField { } protected: - TypeInfoPtr _type_info; + FieldType _type; TabletColumn _desc; // unit : byte // except for strings, other types have fixed lengths @@ -113,7 +112,7 @@ class StorageField { size_t _length; void clone(StorageField* other) const { - other->_type_info = clone_type_info(this->_type_info.get()); + other->_type = this->_type; other->_key_coder = this->_key_coder; other->_name = this->_name; other->_index_size = this->_index_size; diff --git a/be/src/storage/index/bloom_filter/bloom_filter_index_reader.cpp b/be/src/storage/index/bloom_filter/bloom_filter_index_reader.cpp index c8f658b4e34d06..b8c1c9b37440ef 100644 --- a/be/src/storage/index/bloom_filter/bloom_filter_index_reader.cpp +++ b/be/src/storage/index/bloom_filter/bloom_filter_index_reader.cpp @@ -66,8 +66,7 @@ Status BloomFilterIndexReader::new_iterator(std::unique_ptr* bf) { size_t num_to_read = 1; - auto data_type = - DataTypeFactory::instance().create_data_type(_reader->type_info()->type(), 1, 0); + auto data_type = DataTypeFactory::instance().create_data_type(_reader->type(), 1, 0); auto column = data_type->create_column(); RETURN_IF_ERROR(_bloom_filter_iter.seek_to_ordinal(ordinal)); diff --git a/be/src/storage/index/bloom_filter/bloom_filter_index_reader.h b/be/src/storage/index/bloom_filter/bloom_filter_index_reader.h index a04ad8fdff8139..dc0b78f16c3c60 100644 --- a/be/src/storage/index/bloom_filter/bloom_filter_index_reader.h +++ b/be/src/storage/index/bloom_filter/bloom_filter_index_reader.h @@ -42,8 +42,7 @@ class BloomFilterIndexReader : public MetadataAdder { public: explicit BloomFilterIndexReader(io::FileReaderSPtr file_reader, const BloomFilterIndexPB& bloom_filter_index_meta) - : _file_reader(std::move(file_reader)), - _type_info(get_scalar_type_info()) { + : _file_reader(std::move(file_reader)) { _bloom_filter_index_meta.reset(new BloomFilterIndexPB(bloom_filter_index_meta)); } @@ -56,7 +55,7 @@ class BloomFilterIndexReader : public MetadataAdder { Status new_iterator(std::unique_ptr* iterator, OlapReaderStatistics* index_load_stats); - const TypeInfo* type_info() const { return _type_info; } + FieldType type() const { return FieldType::OLAP_FIELD_TYPE_VARCHAR; } private: Status _load(bool use_page_cache, bool kept_in_memory, OlapReaderStatistics* index_load_stats); @@ -68,7 +67,6 @@ class BloomFilterIndexReader : public MetadataAdder { io::FileReaderSPtr _file_reader; DorisCallOnce _load_once; - const TypeInfo* _type_info = nullptr; std::unique_ptr _bloom_filter_index_meta = nullptr; std::unique_ptr _bloom_filter_reader; }; diff --git a/be/src/storage/index/bloom_filter/bloom_filter_index_writer.cpp b/be/src/storage/index/bloom_filter/bloom_filter_index_writer.cpp index 3bef996a50fd91..785e889a831eee 100644 --- a/be/src/storage/index/bloom_filter/bloom_filter_index_writer.cpp +++ b/be/src/storage/index/bloom_filter/bloom_filter_index_writer.cpp @@ -66,9 +66,8 @@ class BloomFilterIndexWriterImpl : public BloomFilterIndexWriter { using CppType = typename CppTypeTraits::CppType; using ValueDict = typename BloomFilterTraits::ValueDict; - explicit BloomFilterIndexWriterImpl(const BloomFilterOptions& bf_options, - const TypeInfo* type_info) - : _bf_options(bf_options), _type_info(type_info) {} + explicit BloomFilterIndexWriterImpl(const BloomFilterOptions& bf_options) + : _bf_options(bf_options) {} ~BloomFilterIndexWriterImpl() override = default; @@ -129,12 +128,11 @@ class BloomFilterIndexWriterImpl : public BloomFilterIndexWriter { meta->set_algorithm(BLOCK_BLOOM_FILTER); // write bloom filters - const auto* bf_type_info = get_scalar_type_info(); IndexedColumnWriterOptions options; options.write_ordinal_index = true; options.write_value_index = false; options.encoding = PLAIN_ENCODING; - IndexedColumnWriter bf_writer(options, bf_type_info, file_writer); + IndexedColumnWriter bf_writer(options, FieldType::OLAP_FIELD_TYPE_VARCHAR, file_writer); RETURN_IF_ERROR(bf_writer.init()); for (auto& bf : _bfs) { Slice data(bf->data(), bf->size()); @@ -162,7 +160,6 @@ class BloomFilterIndexWriterImpl : public BloomFilterIndexWriter { private: BloomFilterOptions _bf_options {}; - const TypeInfo* _type_info = nullptr; Arena _arena; bool _has_null = false; uint64_t _bf_buffer_size = 0; @@ -222,12 +219,11 @@ Status PrimaryKeyBloomFilterIndexWriterImpl::finish(io::FileWriter* file_writer, meta->set_algorithm(BLOCK_BLOOM_FILTER); // write bloom filters - const auto* bf_type_info = get_scalar_type_info(); IndexedColumnWriterOptions options; options.write_ordinal_index = true; options.write_value_index = false; options.encoding = PLAIN_ENCODING; - IndexedColumnWriter bf_writer(options, bf_type_info, file_writer); + IndexedColumnWriter bf_writer(options, FieldType::OLAP_FIELD_TYPE_VARCHAR, file_writer); RETURN_IF_ERROR(bf_writer.init()); for (auto& bf : _bfs) { Slice data(bf->data(), bf->size()); @@ -280,12 +276,11 @@ Status NGramBloomFilterIndexWriterImpl::finish(io::FileWriter* file_writer, meta->set_algorithm(NGRAM_BLOOM_FILTER); // write bloom filters - const TypeInfo* bf_typeinfo = get_scalar_type_info(FieldType::OLAP_FIELD_TYPE_VARCHAR); IndexedColumnWriterOptions options; options.write_ordinal_index = true; options.write_value_index = false; options.encoding = PLAIN_ENCODING; - IndexedColumnWriter bf_writer(options, bf_typeinfo, file_writer); + IndexedColumnWriter bf_writer(options, FieldType::OLAP_FIELD_TYPE_VARCHAR, file_writer); RETURN_IF_ERROR(bf_writer.init()); for (auto& bf : _bfs) { Slice data(bf->data(), bf->size()); @@ -302,8 +297,7 @@ uint64_t NGramBloomFilterIndexWriterImpl::size() { } // TODO currently we don't support bloom filter index for tinyint/hll/float/double -Status BloomFilterIndexWriter::create(const BloomFilterOptions& bf_options, - const TypeInfo* type_info, +Status BloomFilterIndexWriter::create(const BloomFilterOptions& bf_options, FieldType type, std::unique_ptr* res) { DBUG_EXECUTE_IF("BloomFilterIndexWriter::create", { auto fpp = DebugPoints::instance()->get_debug_param_or_default( @@ -316,11 +310,10 @@ Status BloomFilterIndexWriter::create(const BloomFilterOptions& bf_options, } } }) - FieldType type = type_info->type(); switch (type) { -#define M(TYPE) \ - case TYPE: \ - res->reset(new BloomFilterIndexWriterImpl(bf_options, type_info)); \ +#define M(TYPE) \ + case TYPE: \ + res->reset(new BloomFilterIndexWriterImpl(bf_options)); \ break; M(FieldType::OLAP_FIELD_TYPE_BOOL) M(FieldType::OLAP_FIELD_TYPE_TINYINT) @@ -352,11 +345,9 @@ Status BloomFilterIndexWriter::create(const BloomFilterOptions& bf_options, return Status::OK(); } -Status NGramBloomFilterIndexWriterImpl::create(const BloomFilterOptions& bf_options, - const TypeInfo* typeinfo, uint8_t gram_size, - uint16_t gram_bf_size, +Status NGramBloomFilterIndexWriterImpl::create(const BloomFilterOptions& bf_options, FieldType type, + uint8_t gram_size, uint16_t gram_bf_size, std::unique_ptr* res) { - FieldType type = typeinfo->type(); switch (type) { case FieldType::OLAP_FIELD_TYPE_CHAR: case FieldType::OLAP_FIELD_TYPE_VARCHAR: @@ -372,9 +363,8 @@ Status NGramBloomFilterIndexWriterImpl::create(const BloomFilterOptions& bf_opti } Status PrimaryKeyBloomFilterIndexWriterImpl::create(const BloomFilterOptions& bf_options, - const TypeInfo* typeinfo, + FieldType type, std::unique_ptr* res) { - FieldType type = typeinfo->type(); switch (type) { case FieldType::OLAP_FIELD_TYPE_CHAR: case FieldType::OLAP_FIELD_TYPE_VARCHAR: diff --git a/be/src/storage/index/bloom_filter/bloom_filter_index_writer.h b/be/src/storage/index/bloom_filter/bloom_filter_index_writer.h index 0d4aecee850521..d6865b57a6eca5 100644 --- a/be/src/storage/index/bloom_filter/bloom_filter_index_writer.h +++ b/be/src/storage/index/bloom_filter/bloom_filter_index_writer.h @@ -28,12 +28,11 @@ #include "core/arena.h" #include "storage/index/bloom_filter/bloom_filter.h" #include "storage/itoken_extractor.h" +#include "storage/olap_common.h" #include "util/slice.h" namespace doris { -class TypeInfo; - namespace io { class FileWriter; } @@ -44,7 +43,7 @@ class ColumnIndexMetaPB; class BloomFilterIndexWriter { public: - static Status create(const BloomFilterOptions& bf_options, const TypeInfo* typeinfo, + static Status create(const BloomFilterOptions& bf_options, FieldType type, std::unique_ptr* res); BloomFilterIndexWriter() = default; @@ -81,7 +80,7 @@ class PrimaryKeyBloomFilterIndexWriterImpl : public BloomFilterIndexWriter { } }; - static Status create(const BloomFilterOptions& bf_options, const TypeInfo* typeinfo, + static Status create(const BloomFilterOptions& bf_options, FieldType type, std::unique_ptr* res); // This method may allocate large memory for bf, will return error // when memory is exhaused to prevent oom. @@ -107,9 +106,8 @@ class PrimaryKeyBloomFilterIndexWriterImpl : public BloomFilterIndexWriter { class NGramBloomFilterIndexWriterImpl : public BloomFilterIndexWriter { public: - static Status create(const BloomFilterOptions& bf_options, const TypeInfo* typeinfo, - uint8_t gram_size, uint16_t gram_bf_size, - std::unique_ptr* res); + static Status create(const BloomFilterOptions& bf_options, FieldType type, uint8_t gram_size, + uint16_t gram_bf_size, std::unique_ptr* res); NGramBloomFilterIndexWriterImpl(const BloomFilterOptions& bf_options, uint8_t gram_size, uint16_t bf_size); diff --git a/be/src/storage/index/index_writer.cpp b/be/src/storage/index/index_writer.cpp index 98698425b00fd9..d64d6b56f56aed 100644 --- a/be/src/storage/index/index_writer.cpp +++ b/be/src/storage/index/index_writer.cpp @@ -48,8 +48,7 @@ bool IndexColumnWriter::check_support_ann_index(const TabletColumn& column) { Status IndexColumnWriter::create(const StorageField* field, std::unique_ptr* res, IndexFileWriter* index_file_writer, const TabletIndex* index_meta) { - const auto* typeinfo = field->type_info(); - FieldType type = typeinfo->type(); + FieldType type = field->type(); std::string field_name; auto storage_format = index_file_writer->get_storage_format(); if (storage_format == InvertedIndexStorageFormatPB::V1) { @@ -67,12 +66,12 @@ Status IndexColumnWriter::create(const StorageField* field, std::unique_ptris_inverted_index()) { bool single_field = true; if (type == FieldType::OLAP_FIELD_TYPE_ARRAY) { - const auto* array_typeinfo = dynamic_cast(typeinfo); + const auto& column = field->get_desc(); + bool has_item_subcolumn = column.get_subtype_count() > 0; DBUG_EXECUTE_IF("InvertedIndexColumnWriter::create_array_typeinfo_is_nullptr", - { array_typeinfo = nullptr; }) - if (array_typeinfo != nullptr) { - typeinfo = array_typeinfo->item_type_info(); - type = typeinfo->type(); + { has_item_subcolumn = false; }) + if (has_item_subcolumn) { + type = column.get_sub_column(0).type(); single_field = false; } else { return Status::NotSupported("unsupported array type for inverted index: " + diff --git a/be/src/storage/index/indexed_column_reader.cpp b/be/src/storage/index/indexed_column_reader.cpp index 04bcfc1a0f930c..b8fe9a57541a2e 100644 --- a/be/src/storage/index/indexed_column_reader.cpp +++ b/be/src/storage/index/indexed_column_reader.cpp @@ -64,12 +64,12 @@ Status IndexedColumnReader::load(bool use_page_cache, bool kept_in_memory, _use_page_cache = use_page_cache; _kept_in_memory = kept_in_memory; - _type_info = get_scalar_type_info((FieldType)_meta.data_type()); - if (_type_info == nullptr) { + _type = (FieldType)_meta.data_type(); + if (!is_scalar_type(_type)) { return Status::NotSupported("unsupported typeinfo, type={}", _meta.data_type()); } - RETURN_IF_ERROR(EncodingInfo::get(_type_info->type(), _meta.encoding(), {}, &_encoding_info)); - _value_key_coder = get_key_coder(_type_info->type()); + RETURN_IF_ERROR(EncodingInfo::get(_type, _meta.encoding(), {}, &_encoding_info)); + _value_key_coder = get_key_coder(_type); // read and parse ordinal index page when exists if (_meta.has_ordinal_index_meta()) { diff --git a/be/src/storage/index/indexed_column_reader.h b/be/src/storage/index/indexed_column_reader.h index 7d649b4f24179e..1cea4641595dc8 100644 --- a/be/src/storage/index/indexed_column_reader.h +++ b/be/src/storage/index/indexed_column_reader.h @@ -39,7 +39,6 @@ namespace doris { class KeyCoder; -class TypeInfo; class BlockCompressionCodec; namespace segment_v2 { @@ -67,7 +66,7 @@ class IndexedColumnReader : public MetadataAdder { int64_t num_values() const { return _num_values; } const EncodingInfo* encoding_info() const { return _encoding_info; } - const TypeInfo* type_info() const { return _type_info; } + FieldType type() const { return _type; } bool support_ordinal_seek() const { return _meta.has_ordinal_index_meta(); } bool support_value_seek() const { return _meta.has_value_index_meta(); } @@ -99,7 +98,7 @@ class IndexedColumnReader : public MetadataAdder { PageHandle _ordinal_index_page_handle; PageHandle _value_index_page_handle; - const TypeInfo* _type_info = nullptr; + FieldType _type = FieldType::OLAP_FIELD_TYPE_NONE; const EncodingInfo* _encoding_info = nullptr; const KeyCoder* _value_key_coder = nullptr; uint64_t _mem_size = 0; diff --git a/be/src/storage/index/indexed_column_writer.cpp b/be/src/storage/index/indexed_column_writer.cpp index 39c22390728990..87a055792959cc 100644 --- a/be/src/storage/index/indexed_column_writer.cpp +++ b/be/src/storage/index/indexed_column_writer.cpp @@ -39,10 +39,10 @@ namespace doris { namespace segment_v2 { -IndexedColumnWriter::IndexedColumnWriter(const IndexedColumnWriterOptions& options, - const TypeInfo* type_info, io::FileWriter* file_writer) +IndexedColumnWriter::IndexedColumnWriter(const IndexedColumnWriterOptions& options, FieldType type, + io::FileWriter* file_writer) : _options(options), - _type_info(type_info), + _type(type), _file_writer(file_writer), _num_values(0), _num_data_pages(0), @@ -54,7 +54,7 @@ IndexedColumnWriter::~IndexedColumnWriter() = default; Status IndexedColumnWriter::init() { const EncodingInfo* encoding_info; - RETURN_IF_ERROR(EncodingInfo::get(_type_info->type(), _options.encoding, {}, &encoding_info)); + RETURN_IF_ERROR(EncodingInfo::get(_type, _options.encoding, {}, &encoding_info)); _options.encoding = encoding_info->encoding(); // should store more concrete encoding type instead of DEFAULT_ENCODING // because the default encoding of a data type can be changed in the future @@ -72,7 +72,7 @@ Status IndexedColumnWriter::init() { } if (_options.write_value_index) { _value_index_builder.reset(new IndexPageBuilder(_options.index_page_size, true)); - _value_key_coder = get_key_coder(_type_info->type()); + _value_key_coder = get_key_coder(_type); } if (_options.compression != NO_COMPRESSION) { @@ -159,7 +159,7 @@ Status IndexedColumnWriter::finish(IndexedColumnMetaPB* meta) { if (_options.write_value_index) { RETURN_IF_ERROR(_flush_index(_value_index_builder.get(), meta->mutable_value_index_meta())); } - meta->set_data_type(int(_type_info->type())); + meta->set_data_type(int(_type)); meta->set_encoding(_options.encoding); meta->set_num_values(_num_values); meta->set_compression(_options.compression); diff --git a/be/src/storage/index/indexed_column_writer.h b/be/src/storage/index/indexed_column_writer.h index d201cf8fc897a8..98b45ac4c8c771 100644 --- a/be/src/storage/index/indexed_column_writer.h +++ b/be/src/storage/index/indexed_column_writer.h @@ -27,6 +27,7 @@ #include #include "common/status.h" +#include "storage/olap_common.h" #include "storage/segment/common.h" #include "storage/segment/page_pointer.h" @@ -34,7 +35,6 @@ namespace doris { class BlockCompressionCodec; class KeyCoder; -class TypeInfo; namespace io { class FileWriter; @@ -71,8 +71,8 @@ struct IndexedColumnWriterOptions { // TODO test with empty input class IndexedColumnWriter { public: - explicit IndexedColumnWriter(const IndexedColumnWriterOptions& options, - const TypeInfo* type_info, io::FileWriter* file_writer); + explicit IndexedColumnWriter(const IndexedColumnWriterOptions& options, FieldType type, + io::FileWriter* file_writer); ~IndexedColumnWriter(); @@ -93,7 +93,7 @@ class IndexedColumnWriter { Status _flush_index(IndexPageBuilder* index_builder, BTreeMetaPB* meta); IndexedColumnWriterOptions _options; - const TypeInfo* _type_info = nullptr; + FieldType _type; io::FileWriter* _file_writer = nullptr; ordinal_t _num_values; diff --git a/be/src/storage/index/inverted/inverted_index_reader.cpp b/be/src/storage/index/inverted/inverted_index_reader.cpp index 80de3b0aac8744..ec11fb5b28c9f3 100644 --- a/be/src/storage/index/inverted/inverted_index_reader.cpp +++ b/be/src/storage/index/inverted/inverted_index_reader.cpp @@ -637,22 +637,20 @@ Status BkdIndexReader::construct_bkd_query_value(const Field& query_value, std::shared_ptr r, InvertedIndexVisitor* visitor) { if constexpr (QT == InvertedIndexQueryType::EQUAL_QUERY) { - RETURN_IF_ERROR(encode_bkd_field_ascending(_type_info->type(), query_value, - _value_key_coder, &visitor->query_max)); - RETURN_IF_ERROR(encode_bkd_field_ascending(_type_info->type(), query_value, - _value_key_coder, &visitor->query_min)); + RETURN_IF_ERROR(encode_bkd_field_ascending(_type, query_value, _value_key_coder, + &visitor->query_max)); + RETURN_IF_ERROR(encode_bkd_field_ascending(_type, query_value, _value_key_coder, + &visitor->query_min)); } else if constexpr (QT == InvertedIndexQueryType::LESS_THAN_QUERY || QT == InvertedIndexQueryType::LESS_EQUAL_QUERY) { - RETURN_IF_ERROR(encode_bkd_field_ascending(_type_info->type(), query_value, - _value_key_coder, &visitor->query_max)); - RETURN_IF_ERROR(encode_bkd_min_ascending(_type_info->type(), _value_key_coder, - &visitor->query_min)); + RETURN_IF_ERROR(encode_bkd_field_ascending(_type, query_value, _value_key_coder, + &visitor->query_max)); + RETURN_IF_ERROR(encode_bkd_min_ascending(_type, _value_key_coder, &visitor->query_min)); } else if constexpr (QT == InvertedIndexQueryType::GREATER_THAN_QUERY || QT == InvertedIndexQueryType::GREATER_EQUAL_QUERY) { - RETURN_IF_ERROR(encode_bkd_field_ascending(_type_info->type(), query_value, - _value_key_coder, &visitor->query_min)); - RETURN_IF_ERROR(encode_bkd_max_ascending(_type_info->type(), _value_key_coder, - &visitor->query_max)); + RETURN_IF_ERROR(encode_bkd_field_ascending(_type, query_value, _value_key_coder, + &visitor->query_min)); + RETURN_IF_ERROR(encode_bkd_max_ascending(_type, _value_key_coder, &visitor->query_max)); } else { return Status::Error( "invalid query type when query bkd index"); @@ -775,8 +773,8 @@ Status BkdIndexReader::try_query(const IndexQueryContextPtr& context, return st; } std::string query_str; - RETURN_IF_ERROR(encode_bkd_field_ascending(_type_info->type(), query_value, - _value_key_coder, &query_str)); + RETURN_IF_ERROR( + encode_bkd_field_ascending(_type, query_value, _value_key_coder, &query_str)); auto index_file_key = _index_file_reader->get_index_file_cache_key(&_index_meta); InvertedIndexQueryCache::CacheKey cache_key {index_file_key, column_name, query_type, @@ -815,8 +813,8 @@ Status BkdIndexReader::query(const IndexQueryContextPtr& context, const std::str return st; } std::string query_str; - RETURN_IF_ERROR(encode_bkd_field_ascending(_type_info->type(), query_value, - _value_key_coder, &query_str)); + RETURN_IF_ERROR( + encode_bkd_field_ascending(_type, query_value, _value_key_coder, &query_str)); auto index_file_key = _index_file_reader->get_index_file_cache_key(&_index_meta); InvertedIndexQueryCache::CacheKey cache_key {index_file_key, column_name, query_type, @@ -851,15 +849,15 @@ Status BkdIndexReader::get_bkd_reader(const IndexQueryContextPtr& context, auto searcher_variant = inverted_index_cache_handle.get_index_searcher(); bkd_searcher = std::get_if(&searcher_variant); if (bkd_searcher) { - _type_info = get_scalar_type_info((FieldType)(*bkd_searcher)->type); - if (_type_info == nullptr) { + _type = (FieldType)(*bkd_searcher)->type; + if (!is_scalar_type(_type)) { return Status::Error( "unsupported typeinfo, type={}", (*bkd_searcher)->type); } - _value_key_coder = get_key_coder(_type_info->type()); + _value_key_coder = get_key_coder(_type); bkd_reader = *bkd_searcher; if (bkd_reader->bytes_per_dim_ == 0) { - bkd_reader->bytes_per_dim_ = cast_set(_type_info->size()); + bkd_reader->bytes_per_dim_ = cast_set(field_type_size(_type)); } return Status::OK(); } diff --git a/be/src/storage/index/inverted/inverted_index_reader.h b/be/src/storage/index/inverted/inverted_index_reader.h index 38fd2e7cda40d6..0e2f6a120d41e3 100644 --- a/be/src/storage/index/inverted/inverted_index_reader.h +++ b/be/src/storage/index/inverted/inverted_index_reader.h @@ -66,7 +66,6 @@ class Roaring; namespace doris { class KeyCoder; -class TypeInfo; struct OlapReaderStatistics; class RuntimeState; @@ -393,7 +392,7 @@ class BkdIndexReader : public InvertedIndexReader { Status get_bkd_reader(const IndexQueryContextPtr& context, BKDIndexSearcherPtr& reader); private: - const TypeInfo* _type_info {}; + FieldType _type = FieldType::OLAP_FIELD_TYPE_NONE; const KeyCoder* _value_key_coder {}; }; diff --git a/be/src/storage/index/primary_key_index.cpp b/be/src/storage/index/primary_key_index.cpp index 6cda43f01d313d..d91f9e8f586ea6 100644 --- a/be/src/storage/index/primary_key_index.cpp +++ b/be/src/storage/index/primary_key_index.cpp @@ -37,21 +37,21 @@ static bvar::Adder g_primary_key_index_memory_bytes("doris_primary_key_i Status PrimaryKeyIndexBuilder::init() { // TODO(liaoxin) using the column type directly if there's only one column in unique key columns - const auto* type_info = get_scalar_type_info(); + constexpr FieldType type = FieldType::OLAP_FIELD_TYPE_VARCHAR; segment_v2::IndexedColumnWriterOptions options; options.write_ordinal_index = true; options.write_value_index = true; options.data_page_size = config::primary_key_data_page_size; - options.encoding = segment_v2::EncodingInfo::get_default_encoding(type_info->type(), {}, true); + options.encoding = segment_v2::EncodingInfo::get_default_encoding(type, {}, true); options.compression = segment_v2::ZSTD; _primary_key_index_builder.reset( - new segment_v2::IndexedColumnWriter(options, type_info, _file_writer)); + new segment_v2::IndexedColumnWriter(options, type, _file_writer)); RETURN_IF_ERROR(_primary_key_index_builder->init()); auto opt = segment_v2::BloomFilterOptions(); opt.fpp = 0.01; RETURN_IF_ERROR(segment_v2::PrimaryKeyBloomFilterIndexWriterImpl::create( - opt, type_info, &_bloom_filter_index_builder)); + opt, type, &_bloom_filter_index_builder)); return Status::OK(); } diff --git a/be/src/storage/index/primary_key_index.h b/be/src/storage/index/primary_key_index.h index ee7576619887ba..fbb2e39c539c72 100644 --- a/be/src/storage/index/primary_key_index.h +++ b/be/src/storage/index/primary_key_index.h @@ -34,7 +34,6 @@ #include "util/slice.h" namespace doris { -class TypeInfo; namespace io { class FileWriter; @@ -121,9 +120,9 @@ class PrimaryKeyIndexReader { return Status::OK(); } - const TypeInfo* type_info() const { + FieldType type() const { DCHECK(_index_parsed); - return _index_reader->type_info(); + return _index_reader->type(); } // verify whether exist in BloomFilter diff --git a/be/src/storage/index/zone_map/zone_map_index.cpp b/be/src/storage/index/zone_map/zone_map_index.cpp index 61a49715727258..b1735be6543867 100644 --- a/be/src/storage/index/zone_map/zone_map_index.cpp +++ b/be/src/storage/index/zone_map/zone_map_index.cpp @@ -278,14 +278,14 @@ Status TypedZoneMapIndexWriter::finish(io::FileWriter* file_writer, _segment_zone_map.to_proto(meta->mutable_segment_zone_map(), _data_type); // write out zone map for each data pages - const auto* type_info = get_scalar_type_info(); + constexpr FieldType type = FieldType::OLAP_FIELD_TYPE_BITMAP; IndexedColumnWriterOptions options; options.write_ordinal_index = true; options.write_value_index = false; - options.encoding = EncodingInfo::get_default_encoding(type_info->type(), {}, false); + options.encoding = EncodingInfo::get_default_encoding(type, {}, false); options.compression = NO_COMPRESSION; // currently not compressed - IndexedColumnWriter writer(options, type_info, file_writer); + IndexedColumnWriter writer(options, type, file_writer); RETURN_IF_ERROR(writer.init()); for (auto& value : _values) { diff --git a/be/src/storage/merger.cpp b/be/src/storage/merger.cpp index bcb2b72995c660..fa6b4539d496a4 100644 --- a/be/src/storage/merger.cpp +++ b/be/src/storage/merger.cpp @@ -668,7 +668,7 @@ Status Merger::vertical_merge_rowsets(BaseTabletSPtr tablet, ReaderType reader_t // still calls ColumnNullable::insert_many_defaults() for null runs, // which grows the nested PODArray by N * type_size. So the runtime // per-row footprint is at least type_size, no matter how sparse. - int64_t type_size = get_type_info(&col)->size(); + int64_t type_size = field_type_size(col.type()); col_per_row = std::max(raw_per_row, type_size); if (col.is_nullable()) { col_per_row += 1; // null map diff --git a/be/src/storage/segment/column_reader.cpp b/be/src/storage/segment/column_reader.cpp index 20e20879d087b8..189846ad339c27 100644 --- a/be/src/storage/segment/column_reader.cpp +++ b/be/src/storage/segment/column_reader.cpp @@ -310,7 +310,7 @@ void ColumnReader::check_data_by_zone_map_for_test(const MutableColumnPtr& dst) return; } - FieldType type = _type_info->type(); + FieldType type = _type; if (type != FieldType::OLAP_FIELD_TYPE_INT) { return; @@ -346,16 +346,16 @@ void ColumnReader::check_data_by_zone_map_for_test(const MutableColumnPtr& dst) #endif Status ColumnReader::init(const ColumnMetaPB* meta) { - _type_info = get_type_info(meta); + _type = (FieldType)meta->type(); if (meta->has_be_exec_version()) { _be_exec_version = meta->be_exec_version(); } - if (_type_info == nullptr) { + if (_type == FieldType::OLAP_FIELD_TYPE_NONE || _type == FieldType::OLAP_FIELD_TYPE_UNKNOWN) { return Status::NotSupported("unsupported typeinfo, type={}", meta->type()); } - RETURN_IF_ERROR(EncodingInfo::get(_type_info->type(), meta->encoding(), {}, &_encoding_info)); + RETURN_IF_ERROR(EncodingInfo::get(_type, meta->encoding(), {}, &_encoding_info)); for (int i = 0; i < meta->indexes_size(); i++) { const auto& index_meta = meta->indexes(i); @@ -635,7 +635,7 @@ Status ColumnReader::_load_index(const std::shared_ptr& index_f if (_meta_type == FieldType::OLAP_FIELD_TYPE_ARRAY) { type = _meta_children_column_type; } else { - type = _type_info->type(); + type = _type; } if (index_meta->index_type() == IndexType::ANN) { @@ -2512,19 +2512,19 @@ Status DefaultValueColumnIterator::init(const ColumnIteratorOptions& opts) { if (_default_value == "NULL") { _default_value_field = Field::create_field(Null {}); } else { - if (_type_info->type() == FieldType::OLAP_FIELD_TYPE_ARRAY) { + if (_type == FieldType::OLAP_FIELD_TYPE_ARRAY) { if (_default_value != "[]") { return Status::NotSupported("Array default {} is unsupported", _default_value); } else { _default_value_field = Field::create_field(Array {}); return Status::OK(); } - } else if (_type_info->type() == FieldType::OLAP_FIELD_TYPE_STRUCT) { + } else if (_type == FieldType::OLAP_FIELD_TYPE_STRUCT) { return Status::NotSupported("STRUCT default type is unsupported"); - } else if (_type_info->type() == FieldType::OLAP_FIELD_TYPE_MAP) { + } else if (_type == FieldType::OLAP_FIELD_TYPE_MAP) { return Status::NotSupported("MAP default type is unsupported"); } - const auto t = _type_info->type(); + const auto t = _type; const auto serde = DataTypeFactory::instance() .create_data_type(t, _precision, _scale, _len) ->get_serde(); diff --git a/be/src/storage/segment/column_reader.h b/be/src/storage/segment/column_reader.h index 281b6b429944f4..ebf540e7662feb 100644 --- a/be/src/storage/segment/column_reader.h +++ b/be/src/storage/segment/column_reader.h @@ -283,9 +283,8 @@ class ColumnReader : public MetadataAdder, DataTypePtr _data_type; - TypeInfoPtr _type_info = - TypeInfoPtr(nullptr, - nullptr); // initialized in init(), may changed by subclasses. + FieldType _type = + FieldType::OLAP_FIELD_TYPE_NONE; // initialized in init(), may changed by subclasses. const EncodingInfo* _encoding_info = nullptr; // initialized in init(), used for create PageDecoder @@ -816,11 +815,11 @@ class RowIdColumnIteratorV2 : public ColumnIterator { class DefaultValueColumnIterator : public ColumnIterator { public: DefaultValueColumnIterator(bool has_default_value, std::string default_value, bool is_nullable, - TypeInfoPtr type_info, int precision, int scale, int len) + FieldType type, int precision, int scale, int len) : _has_default_value(has_default_value), _default_value(std::move(default_value)), _is_nullable(is_nullable), - _type_info(std::move(type_info)), + _type(type), _precision(precision), _scale(scale), _len(len) {} @@ -854,7 +853,7 @@ class DefaultValueColumnIterator : public ColumnIterator { bool _has_default_value; std::string _default_value; bool _is_nullable; - TypeInfoPtr _type_info; + FieldType _type; int _precision; int _scale; const int _len; diff --git a/be/src/storage/segment/column_writer.cpp b/be/src/storage/segment/column_writer.cpp index a1a22707aa4da3..1ba371233c4671 100644 --- a/be/src/storage/segment/column_writer.cpp +++ b/be/src/storage/segment/column_writer.cpp @@ -131,7 +131,7 @@ inline ScalarColumnWriter* get_null_writer(const ColumnWriterOptions& opts, null_options.meta->set_type(int(null_type)); null_options.meta->set_is_nullable(false); null_options.meta->set_length( - cast_set(get_scalar_type_info()->size())); + cast_set(field_type_size(FieldType::OLAP_FIELD_TYPE_TINYINT))); null_options.meta->set_encoding(DEFAULT_ENCODING); null_options.meta->set_compression(opts.meta->compression()); @@ -210,8 +210,8 @@ Status ColumnWriter::create_array_writer(const ColumnWriterOptions& opts, length_options.meta->set_unique_id(2); length_options.meta->set_type(int(length_type)); length_options.meta->set_is_nullable(false); - length_options.meta->set_length(cast_set( - get_scalar_type_info()->size())); + length_options.meta->set_length( + cast_set(field_type_size(FieldType::OLAP_FIELD_TYPE_UNSIGNED_BIGINT))); length_options.meta->set_encoding(DEFAULT_ENCODING); length_options.meta->set_compression(opts.meta->compression()); @@ -274,8 +274,8 @@ Status ColumnWriter::create_map_writer(const ColumnWriterOptions& opts, const Ta length_options.meta->set_unique_id(column->get_subtype_count() + 1); length_options.meta->set_type(int(length_type)); length_options.meta->set_is_nullable(false); - length_options.meta->set_length(cast_set( - get_scalar_type_info()->size())); + length_options.meta->set_length( + cast_set(field_type_size(FieldType::OLAP_FIELD_TYPE_UNSIGNED_BIGINT))); length_options.meta->set_encoding(DEFAULT_ENCODING); length_options.meta->set_compression(opts.meta->compression()); @@ -576,11 +576,11 @@ Status ScalarColumnWriter::init() { if (_opts.need_bloom_filter) { if (_opts.is_ngram_bf_index) { RETURN_IF_ERROR(NGramBloomFilterIndexWriterImpl::create( - BloomFilterOptions(), get_field()->type_info(), _opts.gram_size, - _opts.gram_bf_size, &_bloom_filter_index_builder)); + BloomFilterOptions(), get_field()->type(), _opts.gram_size, _opts.gram_bf_size, + &_bloom_filter_index_builder)); } else { - RETURN_IF_ERROR(BloomFilterIndexWriter::create( - _opts.bf_options, get_field()->type_info(), &_bloom_filter_index_builder)); + RETURN_IF_ERROR(BloomFilterIndexWriter::create(_opts.bf_options, get_field()->type(), + &_bloom_filter_index_builder)); } } return Status::OK(); diff --git a/be/src/storage/segment/encoding_info.h b/be/src/storage/segment/encoding_info.h index 1f3a5372922560..3ecf817a42ca62 100644 --- a/be/src/storage/segment/encoding_info.h +++ b/be/src/storage/segment/encoding_info.h @@ -30,7 +30,6 @@ namespace doris { -class TypeInfo; enum class FieldType; namespace segment_v2 { diff --git a/be/src/storage/segment/segment.cpp b/be/src/storage/segment/segment.cpp index 103df3f482fd19..aad22566dd6e93 100644 --- a/be/src/storage/segment/segment.cpp +++ b/be/src/storage/segment/segment.cpp @@ -657,10 +657,9 @@ Status Segment::new_default_iterator(const TabletColumn& tablet_column, "column_type={}", tablet_column.unique_id(), tablet_column.name(), tablet_column.type()); } - auto type_info = get_type_info(&tablet_column); std::unique_ptr default_value_iter(new DefaultValueColumnIterator( tablet_column.has_default_value(), tablet_column.default_value(), - tablet_column.is_nullable(), std::move(type_info), tablet_column.precision(), + tablet_column.is_nullable(), tablet_column.type(), tablet_column.precision(), tablet_column.frac(), tablet_column.length())); ColumnIteratorOptions iter_opts; @@ -849,8 +848,7 @@ Status Segment::lookup_row_key(const Slice& key, const TabletSchema* latest_sche row_location->rowset_id = _rowset_id; size_t num_to_read = 1; - auto index_type = DataTypeFactory::instance().create_data_type( - _pk_index_reader->type_info()->type(), 1, 0); + auto index_type = DataTypeFactory::instance().create_data_type(_pk_index_reader->type(), 1, 0); auto index_column = index_type->create_column(); size_t num_read = num_to_read; RETURN_IF_ERROR(index_iterator->next_batch(&num_read, index_column)); @@ -897,8 +895,7 @@ Status Segment::lookup_row_key(const Slice& key, const TabletSchema* latest_sche Slice rowid_slice = Slice(sought_key.get_data() + sought_key_without_seq.get_size() + (segment_has_seq_col ? seq_col_length : 0) + 1, rowid_length - 1); - const auto* type_info = get_scalar_type_info(); - const auto* rowid_coder = get_key_coder(type_info->type()); + const auto* rowid_coder = get_key_coder(FieldType::OLAP_FIELD_TYPE_UNSIGNED_INT); RETURN_IF_ERROR(rowid_coder->decode_ascending(&rowid_slice, rowid_length, (uint8_t*)&row_location->row_id)); } @@ -922,8 +919,7 @@ Status Segment::read_key_by_rowid(uint32_t row_id, std::string* key) { std::unique_ptr iter; RETURN_IF_ERROR(_pk_index_reader->new_iterator(&iter, null_stat)); - auto index_type = DataTypeFactory::instance().create_data_type( - _pk_index_reader->type_info()->type(), 1, 0); + auto index_type = DataTypeFactory::instance().create_data_type(_pk_index_reader->type(), 1, 0); auto index_column = index_type->create_column(); RETURN_IF_ERROR(iter->seek_to_ordinal(row_id)); size_t num_read = 1; diff --git a/be/src/storage/segment/segment_iterator.cpp b/be/src/storage/segment/segment_iterator.cpp index df63306f92baba..8af82b7191eb38 100644 --- a/be/src/storage/segment/segment_iterator.cpp +++ b/be/src/storage/segment/segment_iterator.cpp @@ -1770,7 +1770,7 @@ Status SegmentIterator::_lookup_ordinal_from_pk_index(const RowCursor& key, bool .length() + 1; auto index_type = DataTypeFactory::instance().create_data_type( - _segment->_pk_index_reader->type_info()->type(), 1, 0); + _segment->_pk_index_reader->type(), 1, 0); auto index_column = index_type->create_column(); size_t num_to_read = 1; size_t num_read = num_to_read; diff --git a/be/src/storage/segment/segment_writer.cpp b/be/src/storage/segment/segment_writer.cpp index ac6841a6cf2651..b0e929d8ef8463 100644 --- a/be/src/storage/segment/segment_writer.cpp +++ b/be/src/storage/segment/segment_writer.cpp @@ -124,8 +124,7 @@ SegmentWriter::SegmentWriter(io::FileWriter* file_writer, uint32_t segment_id, } // encode the rowid into the primary key index if (_is_mow_with_cluster_key()) { - const auto* type_info = get_scalar_type_info(); - _rowid_coder = get_key_coder(type_info->type()); + _rowid_coder = get_key_coder(FieldType::OLAP_FIELD_TYPE_UNSIGNED_INT); // primary keys _primary_key_coders.swap(_key_coders); // cluster keys diff --git a/be/src/storage/segment/variant/variant_ext_meta_writer.cpp b/be/src/storage/segment/variant/variant_ext_meta_writer.cpp index 1a1a0619fcc743..1509ac6ef2a710 100644 --- a/be/src/storage/segment/variant/variant_ext_meta_writer.cpp +++ b/be/src/storage/segment/variant/variant_ext_meta_writer.cpp @@ -39,8 +39,8 @@ Status VariantExtMetaWriter::_ensure_inited(Writers* w) { dict_opts.write_ordinal_index = true; dict_opts.encoding = PREFIX_ENCODING; dict_opts.compression = _comp; - const TypeInfo* dict_type = get_scalar_type_info(); - w->key_writer = std::make_unique(dict_opts, dict_type, _fw); + w->key_writer = std::make_unique(dict_opts, + FieldType::OLAP_FIELD_TYPE_VARCHAR, _fw); RETURN_IF_ERROR(w->key_writer->init()); w->inited = true; diff --git a/be/src/storage/segment/vertical_segment_writer.cpp b/be/src/storage/segment/vertical_segment_writer.cpp index 6203bf50b233de..78031d1f7d4429 100644 --- a/be/src/storage/segment/vertical_segment_writer.cpp +++ b/be/src/storage/segment/vertical_segment_writer.cpp @@ -130,8 +130,7 @@ VerticalSegmentWriter::VerticalSegmentWriter(io::FileWriter* file_writer, uint32 } // encode the rowid into the primary key index if (_is_mow_with_cluster_key()) { - const auto* type_info = get_scalar_type_info(); - _rowid_coder = get_key_coder(type_info->type()); + _rowid_coder = get_key_coder(FieldType::OLAP_FIELD_TYPE_UNSIGNED_INT); // primary keys _primary_key_coders.swap(_key_coders); // cluster keys diff --git a/be/src/storage/tablet/base_tablet.cpp b/be/src/storage/tablet/base_tablet.cpp index 6802092c74995e..eb59206165c96a 100644 --- a/be/src/storage/tablet/base_tablet.cpp +++ b/be/src/storage/tablet/base_tablet.cpp @@ -632,8 +632,7 @@ Status BaseTablet::calc_segment_delete_bitmap(RowsetSharedPtr rowset, RETURN_IF_ERROR(pk_idx->new_iterator(&iter, nullptr)); size_t num_to_read = std::min(batch_size, remaining); - auto index_type = - DataTypeFactory::instance().create_data_type(pk_idx->type_info()->type(), 1, 0); + auto index_type = DataTypeFactory::instance().create_data_type(pk_idx->type(), 1, 0); auto index_column = index_type->create_column(); Slice last_key_slice(last_key); RETURN_IF_ERROR(iter->seek_at_or_after(&last_key_slice, &exact_match)); @@ -671,9 +670,7 @@ Status BaseTablet::calc_segment_delete_bitmap(RowsetSharedPtr rowset, Slice rowid_slice = Slice(key.get_data() + key_without_seq.get_size() + seq_col_length + 1, rowid_length - 1); - const auto* type_info = - get_scalar_type_info(); - const auto* rowid_coder = get_key_coder(type_info->type()); + const auto* rowid_coder = get_key_coder(FieldType::OLAP_FIELD_TYPE_UNSIGNED_INT); RETURN_IF_ERROR(rowid_coder->decode_ascending(&rowid_slice, rowid_length, (uint8_t*)&row_id)); } diff --git a/be/src/storage/tablet/tablet_schema.cpp b/be/src/storage/tablet/tablet_schema.cpp index ed4b16bb6e1162..ed729dfec4ce64 100644 --- a/be/src/storage/tablet/tablet_schema.cpp +++ b/be/src/storage/tablet/tablet_schema.cpp @@ -551,7 +551,7 @@ TabletColumn::TabletColumn(FieldAggregationMethod agg, FieldType type) { TabletColumn::TabletColumn(FieldAggregationMethod agg, FieldType filed_type, bool is_nullable) { _aggregation = agg; _type = filed_type; - _length = cast_set(get_scalar_type_info(filed_type)->size()); + _length = cast_set(field_type_size(filed_type)); _is_nullable = is_nullable; } diff --git a/be/src/storage/types.cpp b/be/src/storage/types.cpp index e2137efc3825c3..eb2ccd3c96bfe1 100644 --- a/be/src/storage/types.cpp +++ b/be/src/storage/types.cpp @@ -17,17 +17,8 @@ #include "storage/types.h" -#include - -#include - -#include "common/compiler_util.h" // IWYU pragma: keep -#include "storage/tablet/tablet_schema.h" - namespace doris { -static TypeInfoPtr create_type_info_ptr(const TypeInfo* type_info, bool should_reclaim_memory); - bool is_scalar_type(FieldType field_type) { switch (field_type) { case FieldType::OLAP_FIELD_TYPE_STRUCT: @@ -41,198 +32,4 @@ bool is_scalar_type(FieldType field_type) { } } -bool is_olap_string_type(FieldType field_type) { - switch (field_type) { - case FieldType::OLAP_FIELD_TYPE_CHAR: - case FieldType::OLAP_FIELD_TYPE_VARCHAR: - case FieldType::OLAP_FIELD_TYPE_HLL: - case FieldType::OLAP_FIELD_TYPE_BITMAP: - case FieldType::OLAP_FIELD_TYPE_STRING: - case FieldType::OLAP_FIELD_TYPE_JSONB: - return true; - default: - return false; - } -} - -const TypeInfo* get_scalar_type_info(FieldType field_type) { - // nullptr means that there is no TypeInfo implementation for the corresponding field_type - static const TypeInfo* field_type_array[] = { - nullptr, - get_scalar_type_info(), - nullptr, - get_scalar_type_info(), - nullptr, - get_scalar_type_info(), - get_scalar_type_info(), - get_scalar_type_info(), - get_scalar_type_info(), - get_scalar_type_info(), - get_scalar_type_info(), - get_scalar_type_info(), - nullptr, - get_scalar_type_info(), - get_scalar_type_info(), - get_scalar_type_info(), - get_scalar_type_info(), - get_scalar_type_info(), - nullptr, - nullptr, - nullptr, - nullptr, - nullptr, - get_scalar_type_info(), - get_scalar_type_info(), - get_scalar_type_info(), - get_scalar_type_info(), - get_scalar_type_info(), - get_scalar_type_info(), - get_scalar_type_info(), - get_scalar_type_info(), - get_scalar_type_info(), - get_scalar_type_info(), - get_scalar_type_info(), - get_scalar_type_info(), - get_scalar_type_info(), - get_scalar_type_info(), - get_scalar_type_info(), - get_scalar_type_info(), - get_scalar_type_info(), - get_scalar_type_info(), - nullptr}; - return field_type_array[int(field_type)]; -} - -template -const ArrayTypeInfo* get_init_array_type_info(int32_t iterations) { - static ArrayTypeInfo nested_type_info_0( - create_static_type_info_ptr(get_scalar_type_info())); - static ArrayTypeInfo nested_type_info_1(create_static_type_info_ptr(&nested_type_info_0)); - static ArrayTypeInfo nested_type_info_2(create_static_type_info_ptr(&nested_type_info_1)); - static ArrayTypeInfo nested_type_info_3(create_static_type_info_ptr(&nested_type_info_2)); - static ArrayTypeInfo nested_type_info_4(create_static_type_info_ptr(&nested_type_info_3)); - static ArrayTypeInfo nested_type_info_5(create_static_type_info_ptr(&nested_type_info_4)); - static ArrayTypeInfo nested_type_info_6(create_static_type_info_ptr(&nested_type_info_5)); - static ArrayTypeInfo nested_type_info_7(create_static_type_info_ptr(&nested_type_info_6)); - static ArrayTypeInfo nested_type_info_8(create_static_type_info_ptr(&nested_type_info_7)); - static ArrayTypeInfo* nested_type_info_array[] = { - &nested_type_info_0, &nested_type_info_1, &nested_type_info_2, - &nested_type_info_3, &nested_type_info_4, &nested_type_info_5, - &nested_type_info_6, &nested_type_info_7, &nested_type_info_8}; - return nested_type_info_array[iterations]; -} - -// Produce a struct type info -// TODO(xy): Need refactor to this produce method -const TypeInfo* get_struct_type_info(std::vector field_types) { - std::vector type_infos; - type_infos.reserve(field_types.size()); - for (FieldType& type : field_types) { - if (is_scalar_type(type)) { - type_infos.push_back(create_static_type_info_ptr(get_scalar_type_info(type))); - } else { - // TODO(xy): Not supported nested complex type now - } - } - return new StructTypeInfo(type_infos); -} - -// TODO: Support the type info of the nested array with more than 9 depths. -// TODO(xy): Support the type info of the nested struct -TypeInfoPtr get_type_info(const segment_v2::ColumnMetaPB* column_meta_pb) { - FieldType type = (FieldType)column_meta_pb->type(); - if (UNLIKELY(type == FieldType::OLAP_FIELD_TYPE_STRUCT)) { - std::vector field_types; - for (uint32_t i = 0; i < column_meta_pb->children_columns_size(); i++) { - const auto* child_column = &column_meta_pb->children_columns(i); - field_types.push_back((FieldType)child_column->type()); - } - return create_dynamic_type_info_ptr(get_struct_type_info(field_types)); - } else if (UNLIKELY(type == FieldType::OLAP_FIELD_TYPE_ARRAY)) { - segment_v2::ColumnMetaPB child_column = column_meta_pb->children_columns(0); - TypeInfoPtr child_info = get_type_info(&child_column); - ArrayTypeInfo* array_type_info = new ArrayTypeInfo(std::move(child_info)); - return create_dynamic_type_info_ptr(array_type_info); - } else if (UNLIKELY(type == FieldType::OLAP_FIELD_TYPE_MAP)) { - segment_v2::ColumnMetaPB key_meta = column_meta_pb->children_columns(0); - TypeInfoPtr key_type_info = get_type_info(&key_meta); - segment_v2::ColumnMetaPB value_meta = column_meta_pb->children_columns(1); - TypeInfoPtr value_type_info = get_type_info(&value_meta); - - MapTypeInfo* map_type_info = - new MapTypeInfo(std::move(key_type_info), std::move(value_type_info)); - return create_dynamic_type_info_ptr(map_type_info); - } else { - return create_static_type_info_ptr(get_scalar_type_info(type)); - } -} - -TypeInfoPtr create_static_type_info_ptr(const TypeInfo* type_info) { - return create_type_info_ptr(type_info, false); -} - -TypeInfoPtr create_dynamic_type_info_ptr(const TypeInfo* type_info) { - return create_type_info_ptr(type_info, true); -} - -TypeInfoPtr create_type_info_ptr(const TypeInfo* type_info, bool should_reclaim_memory) { - if (!should_reclaim_memory) { - return TypeInfoPtr(type_info, [](const TypeInfo*) {}); - } else { - return TypeInfoPtr(type_info, [](const TypeInfo* type_info) { delete type_info; }); - } -} - -// TODO: Support the type info of the nested array with more than 9 depths. -TypeInfoPtr get_type_info(const TabletColumn* col) { - auto type = col->type(); - if (UNLIKELY(type == FieldType::OLAP_FIELD_TYPE_STRUCT)) { - std::vector field_types; - for (uint32_t i = 0; i < col->get_subtype_count(); i++) { - const auto* child_column = &col->get_sub_column(i); - field_types.push_back(child_column->type()); - } - return create_dynamic_type_info_ptr(get_struct_type_info(field_types)); - } else if (UNLIKELY(type == FieldType::OLAP_FIELD_TYPE_ARRAY)) { - const auto* child_column = &col->get_sub_column(0); - TypeInfoPtr item_type = get_type_info(child_column); - ArrayTypeInfo* array_type_info = new ArrayTypeInfo(std::move(item_type)); - return create_dynamic_type_info_ptr(array_type_info); - } else if (UNLIKELY(type == FieldType::OLAP_FIELD_TYPE_MAP)) { - const auto* key_column = &col->get_sub_column(0); - TypeInfoPtr key_type = get_type_info(key_column); - const auto* val_column = &col->get_sub_column(1); - TypeInfoPtr value_type = get_type_info(val_column); - MapTypeInfo* map_type_info = new MapTypeInfo(std::move(key_type), std::move(value_type)); - return create_dynamic_type_info_ptr(map_type_info); - } else { - return create_static_type_info_ptr(get_scalar_type_info(type)); - } -} - -TypeInfoPtr clone_type_info(const TypeInfo* type_info) { - auto type = type_info->type(); - if (UNLIKELY(type == FieldType::OLAP_FIELD_TYPE_MAP)) { - const auto map_type_info = dynamic_cast(type_info); - return create_dynamic_type_info_ptr( - new MapTypeInfo(clone_type_info(map_type_info->get_key_type_info()), - clone_type_info(map_type_info->get_value_type_info()))); - } else if (UNLIKELY(type == FieldType::OLAP_FIELD_TYPE_STRUCT)) { - const auto struct_type_info = dynamic_cast(type_info); - std::vector clone_type_infos; - const std::vector* sub_type_infos = struct_type_info->type_infos(); - clone_type_infos.reserve(sub_type_infos->size()); - for (size_t i = 0; i < sub_type_infos->size(); i++) { - clone_type_infos.push_back(clone_type_info((*sub_type_infos)[i].get())); - } - return create_dynamic_type_info_ptr(new StructTypeInfo(clone_type_infos)); - } else if (UNLIKELY(type == FieldType::OLAP_FIELD_TYPE_ARRAY)) { - const auto array_type_info = dynamic_cast(type_info); - return create_dynamic_type_info_ptr( - new ArrayTypeInfo(clone_type_info(array_type_info->item_type_info()))); - } else { - return create_static_type_info_ptr(type_info); - } -} - } // namespace doris diff --git a/be/src/storage/types.h b/be/src/storage/types.h index e33c81751b9086..07b882d6fdb397 100644 --- a/be/src/storage/types.h +++ b/be/src/storage/types.h @@ -51,231 +51,8 @@ namespace doris { -namespace segment_v2 { -class ColumnMetaPB; -} - -class TabletColumn; - -class TypeInfo; - -using TypeInfoPtr = std::unique_ptr; - -TypeInfoPtr create_static_type_info_ptr(const TypeInfo* type_info); -TypeInfoPtr create_dynamic_type_info_ptr(const TypeInfo* type_info); - -class TypeInfo { -public: - virtual ~TypeInfo() = default; - virtual int cmp(const void* left, const void* right) const = 0; - - virtual size_t size() const = 0; - - virtual FieldType type() const = 0; -}; - -class ScalarTypeInfo : public TypeInfo { -public: - int cmp(const void* left, const void* right) const override { return _cmp(left, right); } - - size_t size() const override { return _size; } - - FieldType type() const override { return _field_type; } - - template - ScalarTypeInfo(TypeTraitsClass t) - : _cmp(TypeTraitsClass::cmp), - _size(TypeTraitsClass::size), - _field_type(TypeTraitsClass::type) {} - -private: - int (*_cmp)(const void* left, const void* right); - - const size_t _size; - const FieldType _field_type; - - friend class ScalarTypeInfoResolver; -}; - -class ArrayTypeInfo : public TypeInfo { -public: - explicit ArrayTypeInfo(TypeInfoPtr item_type_info) - : _item_type_info(std::move(item_type_info)), _item_size(_item_type_info->size()) {} - ~ArrayTypeInfo() override = default; - - int cmp(const void* left, const void* right) const override { - auto l_value = reinterpret_cast(left); - auto r_value = reinterpret_cast(right); - size_t l_length = l_value->length(); - size_t r_length = r_value->length(); - size_t cur = 0; - - if (!l_value->has_null() && !r_value->has_null()) { - while (cur < l_length && cur < r_length) { - int result = _item_type_info->cmp((uint8_t*)(l_value->data()) + cur * _item_size, - (uint8_t*)(r_value->data()) + cur * _item_size); - if (result != 0) { - return result; - } - ++cur; - } - } else { - while (cur < l_length && cur < r_length) { - if (l_value->is_null_at(cur)) { - if (!r_value->is_null_at(cur)) { // left is null & right is not null - return -1; - } - } else if (r_value->is_null_at(cur)) { // left is not null & right is null - return 1; - } else { // both are not null - int result = - _item_type_info->cmp((uint8_t*)(l_value->data()) + cur * _item_size, - (uint8_t*)(r_value->data()) + cur * _item_size); - if (result != 0) { - return result; - } - } - ++cur; - } - } - - if (l_length < r_length) { - return -1; - } else if (l_length > r_length) { - return 1; - } else { - return 0; - } - } - - size_t size() const override { return sizeof(CollectionValue); } - - FieldType type() const override { return FieldType::OLAP_FIELD_TYPE_ARRAY; } - - inline const TypeInfo* item_type_info() const { return _item_type_info.get(); } - -private: - TypeInfoPtr _item_type_info; - const size_t _item_size; -}; -///====================== MapType Info ==========================/// -class MapTypeInfo : public TypeInfo { -public: - explicit MapTypeInfo(TypeInfoPtr key_type_info, TypeInfoPtr value_type_info) - : _key_type_info(std::move(key_type_info)), - _value_type_info(std::move(value_type_info)) {} - ~MapTypeInfo() override = default; - - int cmp(const void* left, const void* right) const override { - auto l_value = reinterpret_cast(left); - auto r_value = reinterpret_cast(right); - uint32_t l_size = l_value->size(); - uint32_t r_size = r_value->size(); - if (l_size < r_size) { - return -1; - } else if (l_size > r_size) { - return 1; - } else { - // now we use collection value in array to pack map k-v - auto l_k = reinterpret_cast(l_value->key_data()); - auto l_v = reinterpret_cast(l_value->value_data()); - auto r_k = reinterpret_cast(r_value->key_data()); - auto r_v = reinterpret_cast(r_value->value_data()); - auto key_arr = new ArrayTypeInfo(create_static_type_info_ptr(_key_type_info.get())); - auto val_arr = new ArrayTypeInfo(create_static_type_info_ptr(_value_type_info.get())); - if (int kc = key_arr->cmp(l_k, r_k) != 0) { - return kc; - } else { - return val_arr->cmp(l_v, r_v); - } - } - } - - size_t size() const override { return sizeof(MapValue); } - - FieldType type() const override { return FieldType::OLAP_FIELD_TYPE_MAP; } - - inline const TypeInfo* get_key_type_info() const { return _key_type_info.get(); } - inline const TypeInfo* get_value_type_info() const { return _value_type_info.get(); } - -private: - TypeInfoPtr _key_type_info; - TypeInfoPtr _value_type_info; -}; - -class StructTypeInfo : public TypeInfo { -public: - explicit StructTypeInfo(std::vector& type_infos) { - for (TypeInfoPtr& type_info : type_infos) { - _type_infos.push_back(std::move(type_info)); - } - } - ~StructTypeInfo() override = default; - - int cmp(const void* left, const void* right) const override { - auto l_value = reinterpret_cast(left); - auto r_value = reinterpret_cast(right); - uint32_t l_size = l_value->size(); - uint32_t r_size = r_value->size(); - uint32_t cur = 0; - - if (!l_value->has_null() && !r_value->has_null()) { - while (cur < l_size && cur < r_size) { - int result = - _type_infos[cur]->cmp(l_value->child_value(cur), r_value->child_value(cur)); - if (result != 0) { - return result; - } - ++cur; - } - } else { - while (cur < l_size && cur < r_size) { - if (l_value->is_null_at(cur)) { - if (!r_value->is_null_at(cur)) { // left is null & right is not null - return -1; - } - } else if (r_value->is_null_at(cur)) { // left is not null & right is null - return 1; - } else { // both are not null - int result = _type_infos[cur]->cmp(l_value->child_value(cur), - r_value->child_value(cur)); - if (result != 0) { - return result; - } - } - ++cur; - } - } - - if (l_size < r_size) { - return -1; - } else if (l_size > r_size) { - return 1; - } else { - return 0; - } - } - - size_t size() const override { return sizeof(StructValue); } - - FieldType type() const override { return FieldType::OLAP_FIELD_TYPE_STRUCT; } - - inline const std::vector* type_infos() const { return &_type_infos; } - -private: - std::vector _type_infos; -}; - bool is_scalar_type(FieldType field_type); -const TypeInfo* get_scalar_type_info(FieldType field_type); - -TypeInfoPtr get_type_info(const segment_v2::ColumnMetaPB* column_meta_pb); - -TypeInfoPtr get_type_info(const TabletColumn* col); - -TypeInfoPtr clone_type_info(const TypeInfo* type_info); - // support following formats when convert varchar to date static const std::vector DATE_FORMATS { "%Y-%m-%d", "%y-%m-%d", "%Y%m%d", "%y%m%d", "%Y/%m/%d", "%y/%m/%d", @@ -595,24 +372,51 @@ struct TypeTraits : public FieldTypeTraits { static const int32_t size = sizeof(CppType); }; -template -const TypeInfo* get_scalar_type_info() { - static constexpr TypeTraits traits; - static ScalarTypeInfo scalar_type_info(traits); - return &scalar_type_info; -} - -template -inline const TypeInfo* get_collection_type_info() { - static ArrayTypeInfo collection_type_info( - create_static_type_info_ptr(get_scalar_type_info())); - return &collection_type_info; -} - -// nested array type is unsupported for sub_type of collection -template <> -inline const TypeInfo* get_collection_type_info() { - return nullptr; +inline size_t field_type_size(FieldType field_type) { + switch (field_type) { +#define DORIS_FIELD_TYPE_SIZE_CASE(ft) \ + case FieldType::ft: \ + return sizeof(typename CppTypeTraits::CppType); + DORIS_FIELD_TYPE_SIZE_CASE(OLAP_FIELD_TYPE_BOOL) + DORIS_FIELD_TYPE_SIZE_CASE(OLAP_FIELD_TYPE_TINYINT) + DORIS_FIELD_TYPE_SIZE_CASE(OLAP_FIELD_TYPE_SMALLINT) + DORIS_FIELD_TYPE_SIZE_CASE(OLAP_FIELD_TYPE_INT) + DORIS_FIELD_TYPE_SIZE_CASE(OLAP_FIELD_TYPE_UNSIGNED_INT) + DORIS_FIELD_TYPE_SIZE_CASE(OLAP_FIELD_TYPE_BIGINT) + DORIS_FIELD_TYPE_SIZE_CASE(OLAP_FIELD_TYPE_UNSIGNED_BIGINT) + DORIS_FIELD_TYPE_SIZE_CASE(OLAP_FIELD_TYPE_LARGEINT) + DORIS_FIELD_TYPE_SIZE_CASE(OLAP_FIELD_TYPE_FLOAT) + DORIS_FIELD_TYPE_SIZE_CASE(OLAP_FIELD_TYPE_DOUBLE) + DORIS_FIELD_TYPE_SIZE_CASE(OLAP_FIELD_TYPE_DECIMAL) + DORIS_FIELD_TYPE_SIZE_CASE(OLAP_FIELD_TYPE_DECIMAL32) + DORIS_FIELD_TYPE_SIZE_CASE(OLAP_FIELD_TYPE_DECIMAL64) + DORIS_FIELD_TYPE_SIZE_CASE(OLAP_FIELD_TYPE_DECIMAL128I) + DORIS_FIELD_TYPE_SIZE_CASE(OLAP_FIELD_TYPE_DECIMAL256) + DORIS_FIELD_TYPE_SIZE_CASE(OLAP_FIELD_TYPE_DATE) + DORIS_FIELD_TYPE_SIZE_CASE(OLAP_FIELD_TYPE_DATETIME) + DORIS_FIELD_TYPE_SIZE_CASE(OLAP_FIELD_TYPE_DATEV2) + DORIS_FIELD_TYPE_SIZE_CASE(OLAP_FIELD_TYPE_DATETIMEV2) + DORIS_FIELD_TYPE_SIZE_CASE(OLAP_FIELD_TYPE_TIMEV2) + DORIS_FIELD_TYPE_SIZE_CASE(OLAP_FIELD_TYPE_TIMESTAMPTZ) + DORIS_FIELD_TYPE_SIZE_CASE(OLAP_FIELD_TYPE_IPV4) + DORIS_FIELD_TYPE_SIZE_CASE(OLAP_FIELD_TYPE_IPV6) + DORIS_FIELD_TYPE_SIZE_CASE(OLAP_FIELD_TYPE_CHAR) + DORIS_FIELD_TYPE_SIZE_CASE(OLAP_FIELD_TYPE_VARCHAR) + DORIS_FIELD_TYPE_SIZE_CASE(OLAP_FIELD_TYPE_STRING) + DORIS_FIELD_TYPE_SIZE_CASE(OLAP_FIELD_TYPE_JSONB) + DORIS_FIELD_TYPE_SIZE_CASE(OLAP_FIELD_TYPE_VARIANT) + DORIS_FIELD_TYPE_SIZE_CASE(OLAP_FIELD_TYPE_HLL) + DORIS_FIELD_TYPE_SIZE_CASE(OLAP_FIELD_TYPE_BITMAP) + DORIS_FIELD_TYPE_SIZE_CASE(OLAP_FIELD_TYPE_QUANTILE_STATE) + DORIS_FIELD_TYPE_SIZE_CASE(OLAP_FIELD_TYPE_AGG_STATE) + DORIS_FIELD_TYPE_SIZE_CASE(OLAP_FIELD_TYPE_STRUCT) + DORIS_FIELD_TYPE_SIZE_CASE(OLAP_FIELD_TYPE_ARRAY) + DORIS_FIELD_TYPE_SIZE_CASE(OLAP_FIELD_TYPE_MAP) +#undef DORIS_FIELD_TYPE_SIZE_CASE + default: + LOG(FATAL) << "field_type_size: unsupported FieldType " << int(field_type); + return 0; + } } } // namespace doris diff --git a/be/test/storage/index/ann/ann_index_smoke_test.cpp b/be/test/storage/index/ann/ann_index_smoke_test.cpp index 1672bc3985fe43..32de8d0626bd61 100644 --- a/be/test/storage/index/ann/ann_index_smoke_test.cpp +++ b/be/test/storage/index/ann/ann_index_smoke_test.cpp @@ -57,10 +57,6 @@ class AnnIndexTest : public testing::Test { EXPECT_CALL(*_tablet_column_array, type()) .WillRepeatedly(testing::Return(FieldType::OLAP_FIELD_TYPE_ARRAY)); - EXPECT_CALL(*_tablet_column_array, get_sub_column(0)) - .WillOnce(testing::ReturnRef(*_tablet_column_float)); - EXPECT_CALL(*_tablet_column_float, type()) - .WillOnce(testing::Return(FieldType::OLAP_FIELD_TYPE_FLOAT)); StorageField field(*_tablet_column_array); diff --git a/be/test/storage/index/inverted/compaction/util/index_compaction_utils.cpp b/be/test/storage/index/inverted/compaction/util/index_compaction_utils.cpp index b8eada222697aa..162543c3033dc8 100644 --- a/be/test/storage/index/inverted/compaction/util/index_compaction_utils.cpp +++ b/be/test/storage/index/inverted/compaction/util/index_compaction_utils.cpp @@ -158,9 +158,9 @@ class IndexCompactionUtils { EXPECT_TRUE(searcher_result.has_value()); auto bkd_searcher = std::get_if(&searcher_result.value()); EXPECT_TRUE(bkd_searcher != nullptr); - idx_reader->_type_info = get_scalar_type_info((FieldType)(*bkd_searcher)->type); - EXPECT_TRUE(idx_reader->_type_info != nullptr); - idx_reader->_value_key_coder = get_key_coder(idx_reader->_type_info->type()); + idx_reader->_type = (FieldType)(*bkd_searcher)->type; + EXPECT_TRUE(is_scalar_type(idx_reader->_type)); + idx_reader->_value_key_coder = get_key_coder(idx_reader->_type); for (int i = 0; i < query_data.size(); i++) { Field param_value = Field::create_field(int32_t(query_data[i])); diff --git a/be/test/storage/index/primary_key_index_test.cpp b/be/test/storage/index/primary_key_index_test.cpp index 7f4c47693f5816..83a56beb27420b 100644 --- a/be/test/storage/index/primary_key_index_test.cpp +++ b/be/test/storage/index/primary_key_index_test.cpp @@ -144,8 +144,8 @@ TEST_F(PrimaryKeyIndexTest, builder) { EXPECT_TRUE(index_reader.new_iterator(&iter, nullptr).ok()); size_t num_to_read = std::min(batch_size, remaining); - auto index_type = DataTypeFactory::instance().create_data_type( - index_reader.type_info()->type(), 1, 0); + auto index_type = + DataTypeFactory::instance().create_data_type(index_reader.type(), 1, 0); auto index_column = index_type->create_column(); Slice last_key_slice(last_key); EXPECT_TRUE(iter->seek_at_or_after(&last_key_slice, &exact_match).ok()); diff --git a/be/test/storage/segment/bloom_filter_index_reader_writer_test.cpp b/be/test/storage/segment/bloom_filter_index_reader_writer_test.cpp index 05635aaf407a89..17eeac11a7db15 100644 --- a/be/test/storage/segment/bloom_filter_index_reader_writer_test.cpp +++ b/be/test/storage/segment/bloom_filter_index_reader_writer_test.cpp @@ -66,7 +66,6 @@ Status write_bloom_filter_index_file(const std::string& file_name, const void* v size_t value_count, size_t null_count, ColumnIndexMetaPB* index_meta, bool use_primary_key_bloom_filter = false, double fpp = 0.05) { - const auto* type_info = get_scalar_type_info(); using CppType = typename CppTypeTraits::CppType; std::string fname = dname + "/" + file_name; auto fs = io::global_local_filesystem(); @@ -80,10 +79,10 @@ Status write_bloom_filter_index_file(const std::string& file_name, const void* v bf_options.fpp = fpp; // Set the expected FPP if (use_primary_key_bloom_filter) { RETURN_IF_ERROR(PrimaryKeyBloomFilterIndexWriterImpl::create( - bf_options, type_info, &bloom_filter_index_writer)); + bf_options, type, &bloom_filter_index_writer)); } else { - RETURN_IF_ERROR(BloomFilterIndexWriter::create(bf_options, type_info, - &bloom_filter_index_writer)); + RETURN_IF_ERROR( + BloomFilterIndexWriter::create(bf_options, type, &bloom_filter_index_writer)); } const CppType* vals = (const CppType*)values; @@ -616,7 +615,7 @@ TEST_F(BloomFilterIndexReaderWriterTest, test_ipv6) { template Status write_ngram_bloom_filter_index_file(const std::string& file_name, Slice* values, - size_t num_values, const TypeInfo* type_info, + size_t num_values, BloomFilterIndexWriter* bf_index_writer, ColumnIndexMetaPB* meta) { auto fs = io::global_local_filesystem(); @@ -685,16 +684,15 @@ template Status test_ngram_bloom_filter_index_reader_writer(const std::string& file_name, Slice* values, size_t num_values, uint8_t gram_size, uint16_t bf_size) { - const auto* type_info = get_scalar_type_info(); ColumnIndexMetaPB meta; BloomFilterOptions bf_options; std::unique_ptr bf_index_writer; - RETURN_IF_ERROR(NGramBloomFilterIndexWriterImpl::create(bf_options, type_info, gram_size, - bf_size, &bf_index_writer)); + RETURN_IF_ERROR(NGramBloomFilterIndexWriterImpl::create(bf_options, type, gram_size, bf_size, + &bf_index_writer)); - RETURN_IF_ERROR(write_ngram_bloom_filter_index_file( - file_name, values, num_values, type_info, bf_index_writer.get(), &meta)); + RETURN_IF_ERROR(write_ngram_bloom_filter_index_file(file_name, values, num_values, + bf_index_writer.get(), &meta)); std::vector test_patterns = {"ngram15", "ngram1000", "ngram1499", "non-existent-string"}; @@ -734,7 +732,7 @@ TEST_F(BloomFilterIndexReaderWriterTest, test_ngram_bloom_filter) { EXPECT_EQ(st.code(), TStatusCode::NOT_IMPLEMENTED_ERROR); } void test_ngram_bloom_filter_with_size(uint16_t bf_size) { - const auto* type_info = get_scalar_type_info(); + constexpr FieldType type = FieldType::OLAP_FIELD_TYPE_VARCHAR; ColumnIndexMetaPB meta; BloomFilterOptions bf_options; @@ -751,13 +749,13 @@ void test_ngram_bloom_filter_with_size(uint16_t bf_size) { uint8_t gram_size = 5; std::unique_ptr bf_index_writer; - auto st = NGramBloomFilterIndexWriterImpl::create(bf_options, type_info, gram_size, bf_size, + auto st = NGramBloomFilterIndexWriterImpl::create(bf_options, type, gram_size, bf_size, &bf_index_writer); EXPECT_TRUE(st.ok()); std::string file_name = "bloom_filter_ngram_varchar_size_" + std::to_string(bf_size); - st = write_ngram_bloom_filter_index_file( - file_name, slices.data(), num, type_info, bf_index_writer.get(), &meta); + st = write_ngram_bloom_filter_index_file(file_name, slices.data(), num, + bf_index_writer.get(), &meta); EXPECT_TRUE(st.ok()); EXPECT_EQ(bf_index_writer->size(), static_cast(bf_size) * total_pages); } @@ -770,10 +768,10 @@ TEST_F(BloomFilterIndexReaderWriterTest, test_ngram_bloom_filter_size) { } TEST_F(BloomFilterIndexReaderWriterTest, test_unsupported_type) { - auto type_info = get_scalar_type_info(); BloomFilterOptions bf_options; std::unique_ptr bloom_filter_index_writer; - auto st = BloomFilterIndexWriter::create(bf_options, type_info, &bloom_filter_index_writer); + auto st = BloomFilterIndexWriter::create(bf_options, FieldType::OLAP_FIELD_TYPE_FLOAT, + &bloom_filter_index_writer); EXPECT_FALSE(st.ok()); EXPECT_EQ(st.code(), TStatusCode::NOT_IMPLEMENTED_ERROR); } diff --git a/be/test/storage/segment/encoding_info_test.cpp b/be/test/storage/segment/encoding_info_test.cpp index 5583a18df8a290..666363c9566c83 100644 --- a/be/test/storage/segment/encoding_info_test.cpp +++ b/be/test/storage/segment/encoding_info_test.cpp @@ -42,36 +42,31 @@ class EncodingInfoTest : public testing::Test { }; TEST_F(EncodingInfoTest, normal) { - const auto* type_info = get_scalar_type_info(); + constexpr FieldType type = FieldType::OLAP_FIELD_TYPE_BIGINT; const EncodingInfo* encoding_info = nullptr; EncodingPreference encoding_preference; - auto status = EncodingInfo::get(type_info->type(), PLAIN_ENCODING, encoding_preference, - &encoding_info); + auto status = EncodingInfo::get(type, PLAIN_ENCODING, encoding_preference, &encoding_info); EXPECT_TRUE(status.ok()); EXPECT_NE(nullptr, encoding_info); } TEST_F(EncodingInfoTest, no_encoding) { - const auto* type_info = get_scalar_type_info(); + constexpr FieldType type = FieldType::OLAP_FIELD_TYPE_BIGINT; const EncodingInfo* encoding_info = nullptr; EncodingPreference encoding_preference; - auto status = EncodingInfo::get(type_info->type(), DICT_ENCODING, encoding_preference, - &encoding_info); + auto status = EncodingInfo::get(type, DICT_ENCODING, encoding_preference, &encoding_info); EXPECT_FALSE(status.ok()); } TEST_F(EncodingInfoTest, test_use_plain_binary_v2_config) { // Helper lambda to test string/JSON types with DICT_ENCODING as default auto test_dict_type_encoding = [](FieldType type, const std::string& type_name) { - const auto* type_info = get_scalar_type_info(type); - // Test with BINARY_PLAIN_ENCODING_V1 (default) // String and JSON types default to DICT_ENCODING EncodingPreference pref_v1; pref_v1.binary_plain_encoding_default_impl = BinaryPlainEncodingTypePB::BINARY_PLAIN_ENCODING_V1; - EncodingTypePB encoding_type = - EncodingInfo::get_default_encoding(type_info->type(), pref_v1, false); + EncodingTypePB encoding_type = EncodingInfo::get_default_encoding(type, pref_v1, false); EXPECT_EQ(DICT_ENCODING, encoding_type) << "Type " << type_name << " should use DICT_ENCODING with V1 preference"; @@ -80,21 +75,18 @@ TEST_F(EncodingInfoTest, test_use_plain_binary_v2_config) { EncodingPreference pref_v2; pref_v2.binary_plain_encoding_default_impl = BinaryPlainEncodingTypePB::BINARY_PLAIN_ENCODING_V2; - encoding_type = EncodingInfo::get_default_encoding(type_info->type(), pref_v2, false); + encoding_type = EncodingInfo::get_default_encoding(type, pref_v2, false); EXPECT_EQ(DICT_ENCODING, encoding_type) << "Type " << type_name << " should still use DICT_ENCODING with V2 preference"; }; // Helper lambda to test aggregate state types with PLAIN_ENCODING as default auto test_plain_type_encoding = [](FieldType type, const std::string& type_name) { - const auto* type_info = get_scalar_type_info(type); - // Test with BINARY_PLAIN_ENCODING_V1 (default) EncodingPreference pref_v1; pref_v1.binary_plain_encoding_default_impl = BinaryPlainEncodingTypePB::BINARY_PLAIN_ENCODING_V1; - EncodingTypePB encoding_type = - EncodingInfo::get_default_encoding(type_info->type(), pref_v1, false); + EncodingTypePB encoding_type = EncodingInfo::get_default_encoding(type, pref_v1, false); EXPECT_EQ(PLAIN_ENCODING, encoding_type) << "Type " << type_name << " should use PLAIN_ENCODING with V1 preference"; @@ -102,7 +94,7 @@ TEST_F(EncodingInfoTest, test_use_plain_binary_v2_config) { EncodingPreference pref_v2; pref_v2.binary_plain_encoding_default_impl = BinaryPlainEncodingTypePB::BINARY_PLAIN_ENCODING_V2; - encoding_type = EncodingInfo::get_default_encoding(type_info->type(), pref_v2, false); + encoding_type = EncodingInfo::get_default_encoding(type, pref_v2, false); EXPECT_EQ(PLAIN_ENCODING_V2, encoding_type) << "Type " << type_name << " should use PLAIN_ENCODING_V2 with V2 preference"; }; @@ -123,15 +115,15 @@ TEST_F(EncodingInfoTest, test_use_plain_binary_v2_config) { test_plain_type_encoding(FieldType::OLAP_FIELD_TYPE_AGG_STATE, "AGG_STATE"); // Test non-binary type (BIGINT) - should not be affected by binary preference - const auto* bigint_type_info = get_scalar_type_info(); + constexpr FieldType bigint_type = FieldType::OLAP_FIELD_TYPE_BIGINT; // Test with plain encoding disabled for integers (default) EncodingPreference pref_plain_disabled; pref_plain_disabled.integer_type_default_use_plain_encoding = false; pref_plain_disabled.binary_plain_encoding_default_impl = BinaryPlainEncodingTypePB::BINARY_PLAIN_ENCODING_V1; - EncodingTypePB encoding_type = EncodingInfo::get_default_encoding(bigint_type_info->type(), - pref_plain_disabled, false); + EncodingTypePB encoding_type = + EncodingInfo::get_default_encoding(bigint_type, pref_plain_disabled, false); EXPECT_EQ(BIT_SHUFFLE, encoding_type); // Test with plain encoding enabled for integers @@ -139,15 +131,13 @@ TEST_F(EncodingInfoTest, test_use_plain_binary_v2_config) { pref_plain_enabled.integer_type_default_use_plain_encoding = true; pref_plain_enabled.binary_plain_encoding_default_impl = BinaryPlainEncodingTypePB::BINARY_PLAIN_ENCODING_V1; - encoding_type = - EncodingInfo::get_default_encoding(bigint_type_info->type(), pref_plain_enabled, false); + encoding_type = EncodingInfo::get_default_encoding(bigint_type, pref_plain_enabled, false); EXPECT_EQ(PLAIN_ENCODING, encoding_type); // Verify binary preference doesn't affect integer types pref_plain_enabled.binary_plain_encoding_default_impl = BinaryPlainEncodingTypePB::BINARY_PLAIN_ENCODING_V2; - encoding_type = - EncodingInfo::get_default_encoding(bigint_type_info->type(), pref_plain_enabled, false); + encoding_type = EncodingInfo::get_default_encoding(bigint_type, pref_plain_enabled, false); EXPECT_EQ(PLAIN_ENCODING, encoding_type); // Should still be PLAIN_ENCODING } diff --git a/be/test/storage/storage_types_test.cpp b/be/test/storage/storage_types_test.cpp index 5d79e4f56ec320..f0d285b5861dc2 100644 --- a/be/test/storage/storage_types_test.cpp +++ b/be/test/storage/storage_types_test.cpp @@ -41,20 +41,16 @@ class TypesTest : public testing::Test { template void common_test(typename TypeTraits::CppType src_val) { - const auto* type = get_scalar_type_info(); - - EXPECT_EQ(field_type, type->type()); - EXPECT_EQ(sizeof(src_val), type->size()); + EXPECT_EQ(sizeof(src_val), field_type_size(field_type)); } template void test_char(Slice src_val) { StorageField* field = StorageFieldFactory::create_by_type(fieldType); field->_length = src_val.size; - const auto* type = field->type_info(); EXPECT_EQ(field->type(), fieldType); - EXPECT_EQ(sizeof(src_val), type->size()); + EXPECT_EQ(sizeof(src_val), field->size()); delete field; } @@ -105,9 +101,7 @@ void common_test_array(CollectionValue src_val) { 0, item_length); list_column.add_sub_column(item_column); - auto array_type = get_type_info(&list_column); - ASSERT_EQ(item_type, - dynamic_cast(array_type.get())->item_type_info()->type()); + ASSERT_EQ(item_type, list_column.get_sub_column(0).type()); } TEST(ArrayTypeTest, copy_and_equal) {