Skip to content

[refactor](storage) replace TypeInfo polymorphic hierarchy with FieldType#63153

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
airborne12 merged 1 commit into from
May 13, 2026
Merged

[refactor](storage) replace TypeInfo polymorphic hierarchy with FieldType #63153

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 2 additions & 4 deletions be/src/storage/delete/delete_bitmap_calculator.cpp
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -164,14 +164,12 @@ Status MergeIndexDeleteBitmapCalculator::init(RowsetId rowset_id,
auto pk_idx = segment->get_primary_key_index();
std::unique_ptr<segment_v2::IndexedColumnIterator> index;
RETURN_IF_ERROR(pk_idx->new_iterator(&index, nullptr));
auto index_type =
DataTypeFactory::instance().create_data_type(pk_idx->type_info()->type(), 1, 0);
auto index_type = DataTypeFactory::instance().create_data_type(pk_idx->type(), 1, 0);
_contexts.emplace_back(std::move(index), index_type, segment->id(), pk_idx->num_rows());
_heap->push(&_contexts.back());
}
if (_rowid_length > 0) {
_rowid_coder = get_key_coder(
get_scalar_type_info<FieldType::OLAP_FIELD_TYPE_UNSIGNED_INT>()->type());
_rowid_coder = get_key_coder(FieldType::OLAP_FIELD_TYPE_UNSIGNED_INT);
}
});
return Status::OK();
Expand Down
11 changes: 5 additions & 6 deletions be/src/storage/field.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -40,7 +40,7 @@ namespace doris {
class StorageField {
public:
StorageField(const TabletColumn& column)
: _type_info(get_type_info(&column)),
: _type(column.type()),
_desc(column),
_length(column.length()),
_key_coder(get_key_coder(column.type())),
Expand All@@ -54,7 +54,7 @@ class StorageField {

virtual ~StorageField() = default;

size_t size() const { return _type_info->size(); }
size_t size() const { return field_type_size(_type); }
Comment thread
csun5285 marked this conversation as resolved.
size_t length() const { return _length; }
size_t field_size() const { return size() + 1; }
size_t index_size() const { return _index_size; }
Expand All@@ -70,8 +70,7 @@ class StorageField {
return local;
}

FieldType type() const { return _type_info->type(); }
const TypeInfo* type_info() const { return _type_info.get(); }
FieldType type() const { return _type; }
bool is_nullable() const { return _is_nullable; }

// similar to `full_encode_ascending`, but only encode part (the first `index_size` bytes) of the value.
Expand DownExpand Up@@ -103,7 +102,7 @@ class StorageField {
}

protected:
TypeInfoPtr _type_info;
FieldType _type;
TabletColumn _desc;
// unit : byte
// except for strings, other types have fixed lengths
Expand All@@ -113,7 +112,7 @@ class StorageField {
size_t _length;

void clone(StorageField* other) const {
other->_type_info = clone_type_info(this->_type_info.get());
other->_type = this->_type;
other->_key_coder = this->_key_coder;
other->_name = this->_name;
other->_index_size = this->_index_size;
Expand Down
3 changes: 1 addition & 2 deletions be/src/storage/index/bloom_filter/bloom_filter_index_reader.cpp
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -66,8 +66,7 @@ Status BloomFilterIndexReader::new_iterator(std::unique_ptr<BloomFilterIndexIter
Status BloomFilterIndexIterator::read_bloom_filter(rowid_t ordinal,
std::unique_ptr<BloomFilter>* bf) {
size_t num_to_read = 1;
auto data_type =
DataTypeFactory::instance().create_data_type(_reader->type_info()->type(), 1, 0);
auto data_type = DataTypeFactory::instance().create_data_type(_reader->type(), 1, 0);
auto column = data_type->create_column();

RETURN_IF_ERROR(_bloom_filter_iter.seek_to_ordinal(ordinal));
Expand Down
6 changes: 2 additions & 4 deletions be/src/storage/index/bloom_filter/bloom_filter_index_reader.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -42,8 +42,7 @@ class BloomFilterIndexReader : public MetadataAdder<BloomFilterIndexReader> {
public:
explicit BloomFilterIndexReader(io::FileReaderSPtr file_reader,
const BloomFilterIndexPB& bloom_filter_index_meta)
: _file_reader(std::move(file_reader)),
_type_info(get_scalar_type_info<FieldType::OLAP_FIELD_TYPE_VARCHAR>()) {
: _file_reader(std::move(file_reader)) {
_bloom_filter_index_meta.reset(new BloomFilterIndexPB(bloom_filter_index_meta));
}

Expand All@@ -56,7 +55,7 @@ class BloomFilterIndexReader : public MetadataAdder<BloomFilterIndexReader> {
Status new_iterator(std::unique_ptr<BloomFilterIndexIterator>* iterator,
OlapReaderStatistics* index_load_stats);

const TypeInfo* type_info() const { return _type_info; }
FieldType type() const { return FieldType::OLAP_FIELD_TYPE_VARCHAR; }

private:
Status _load(bool use_page_cache, bool kept_in_memory, OlapReaderStatistics* index_load_stats);
Expand All@@ -68,7 +67,6 @@ class BloomFilterIndexReader : public MetadataAdder<BloomFilterIndexReader> {

io::FileReaderSPtr _file_reader;
DorisCallOnce<Status> _load_once;
const TypeInfo* _type_info = nullptr;
std::unique_ptr<BloomFilterIndexPB> _bloom_filter_index_meta = nullptr;
std::unique_ptr<IndexedColumnReader> _bloom_filter_reader;
};
Expand Down
34 changes: 12 additions & 22 deletions be/src/storage/index/bloom_filter/bloom_filter_index_writer.cpp
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -66,9 +66,8 @@ class BloomFilterIndexWriterImpl : public BloomFilterIndexWriter {
using CppType = typename CppTypeTraits<field_type>::CppType;
using ValueDict = typename BloomFilterTraits<CppType>::ValueDict;

explicit BloomFilterIndexWriterImpl(const BloomFilterOptions& bf_options,
const TypeInfo* type_info)
: _bf_options(bf_options), _type_info(type_info) {}
explicit BloomFilterIndexWriterImpl(const BloomFilterOptions& bf_options)
: _bf_options(bf_options) {}

~BloomFilterIndexWriterImpl() override = default;

Expand DownExpand Up@@ -129,12 +128,11 @@ class BloomFilterIndexWriterImpl : public BloomFilterIndexWriter {
meta->set_algorithm(BLOCK_BLOOM_FILTER);

// write bloom filters
const auto* bf_type_info = get_scalar_type_info<FieldType::OLAP_FIELD_TYPE_VARCHAR>();
IndexedColumnWriterOptions options;
options.write_ordinal_index = true;
options.write_value_index = false;
options.encoding = PLAIN_ENCODING;
IndexedColumnWriter bf_writer(options, bf_type_info, file_writer);
IndexedColumnWriter bf_writer(options, FieldType::OLAP_FIELD_TYPE_VARCHAR, file_writer);
RETURN_IF_ERROR(bf_writer.init());
for (auto& bf : _bfs) {
Slice data(bf->data(), bf->size());
Expand DownExpand Up@@ -162,7 +160,6 @@ class BloomFilterIndexWriterImpl : public BloomFilterIndexWriter {

private:
BloomFilterOptions _bf_options {};
const TypeInfo* _type_info = nullptr;
Arena _arena;
bool _has_null = false;
uint64_t _bf_buffer_size = 0;
Expand DownExpand Up@@ -222,12 +219,11 @@ Status PrimaryKeyBloomFilterIndexWriterImpl::finish(io::FileWriter* file_writer,
meta->set_algorithm(BLOCK_BLOOM_FILTER);

// write bloom filters
const auto* bf_type_info = get_scalar_type_info<FieldType::OLAP_FIELD_TYPE_VARCHAR>();
IndexedColumnWriterOptions options;
options.write_ordinal_index = true;
options.write_value_index = false;
options.encoding = PLAIN_ENCODING;
IndexedColumnWriter bf_writer(options, bf_type_info, file_writer);
IndexedColumnWriter bf_writer(options, FieldType::OLAP_FIELD_TYPE_VARCHAR, file_writer);
RETURN_IF_ERROR(bf_writer.init());
for (auto& bf : _bfs) {
Slice data(bf->data(), bf->size());
Expand DownExpand Up@@ -280,12 +276,11 @@ Status NGramBloomFilterIndexWriterImpl::finish(io::FileWriter* file_writer,
meta->set_algorithm(NGRAM_BLOOM_FILTER);

// write bloom filters
const TypeInfo* bf_typeinfo = get_scalar_type_info(FieldType::OLAP_FIELD_TYPE_VARCHAR);
IndexedColumnWriterOptions options;
options.write_ordinal_index = true;
options.write_value_index = false;
options.encoding = PLAIN_ENCODING;
IndexedColumnWriter bf_writer(options, bf_typeinfo, file_writer);
IndexedColumnWriter bf_writer(options, FieldType::OLAP_FIELD_TYPE_VARCHAR, file_writer);
RETURN_IF_ERROR(bf_writer.init());
for (auto& bf : _bfs) {
Slice data(bf->data(), bf->size());
Expand All@@ -302,8 +297,7 @@ uint64_t NGramBloomFilterIndexWriterImpl::size() {
}

// TODO currently we don't support bloom filter index for tinyint/hll/float/double
Status BloomFilterIndexWriter::create(const BloomFilterOptions& bf_options,
const TypeInfo* type_info,
Status BloomFilterIndexWriter::create(const BloomFilterOptions& bf_options, FieldType type,
std::unique_ptr<BloomFilterIndexWriter>* res) {
DBUG_EXECUTE_IF("BloomFilterIndexWriter::create", {
auto fpp = DebugPoints::instance()->get_debug_param_or_default<std::string>(
Expand All@@ -316,11 +310,10 @@ Status BloomFilterIndexWriter::create(const BloomFilterOptions& bf_options,
}
}
})
FieldType type = type_info->type();
switch (type) {
#define M(TYPE) \
case TYPE: \
res->reset(new BloomFilterIndexWriterImpl<TYPE>(bf_options, type_info)); \
#define M(TYPE) \
case TYPE: \
res->reset(new BloomFilterIndexWriterImpl<TYPE>(bf_options)); \
break;
M(FieldType::OLAP_FIELD_TYPE_BOOL)
M(FieldType::OLAP_FIELD_TYPE_TINYINT)
Expand DownExpand Up@@ -352,11 +345,9 @@ Status BloomFilterIndexWriter::create(const BloomFilterOptions& bf_options,
return Status::OK();
}

Status NGramBloomFilterIndexWriterImpl::create(const BloomFilterOptions& bf_options,
const TypeInfo* typeinfo, uint8_t gram_size,
uint16_t gram_bf_size,
Status NGramBloomFilterIndexWriterImpl::create(const BloomFilterOptions& bf_options, FieldType type,
uint8_t gram_size, uint16_t gram_bf_size,
std::unique_ptr<BloomFilterIndexWriter>* res) {
FieldType type = typeinfo->type();
switch (type) {
case FieldType::OLAP_FIELD_TYPE_CHAR:
case FieldType::OLAP_FIELD_TYPE_VARCHAR:
Expand All@@ -372,9 +363,8 @@ Status NGramBloomFilterIndexWriterImpl::create(const BloomFilterOptions& bf_opti
}

Status PrimaryKeyBloomFilterIndexWriterImpl::create(const BloomFilterOptions& bf_options,
const TypeInfo* typeinfo,
FieldType type,
std::unique_ptr<BloomFilterIndexWriter>* res) {
FieldType type = typeinfo->type();
switch (type) {
case FieldType::OLAP_FIELD_TYPE_CHAR:
case FieldType::OLAP_FIELD_TYPE_VARCHAR:
Expand Down
12 changes: 5 additions & 7 deletions be/src/storage/index/bloom_filter/bloom_filter_index_writer.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -28,12 +28,11 @@
#include "core/arena.h"
#include "storage/index/bloom_filter/bloom_filter.h"
#include "storage/itoken_extractor.h"
#include "storage/olap_common.h"
#include "util/slice.h"

namespace doris {

class TypeInfo;

namespace io {
class FileWriter;
}
Expand All@@ -44,7 +43,7 @@ class ColumnIndexMetaPB;

class BloomFilterIndexWriter {
public:
static Status create(const BloomFilterOptions& bf_options, const TypeInfo* typeinfo,
static Status create(const BloomFilterOptions& bf_options, FieldType type,
std::unique_ptr<BloomFilterIndexWriter>* res);

BloomFilterIndexWriter() = default;
Expand DownExpand Up@@ -81,7 +80,7 @@ class PrimaryKeyBloomFilterIndexWriterImpl : public BloomFilterIndexWriter {
}
};

static Status create(const BloomFilterOptions& bf_options, const TypeInfo* typeinfo,
static Status create(const BloomFilterOptions& bf_options, FieldType type,
std::unique_ptr<BloomFilterIndexWriter>* res);
// This method may allocate large memory for bf, will return error
// when memory is exhaused to prevent oom.
Expand All@@ -107,9 +106,8 @@ class PrimaryKeyBloomFilterIndexWriterImpl : public BloomFilterIndexWriter {

class NGramBloomFilterIndexWriterImpl : public BloomFilterIndexWriter {
public:
static Status create(const BloomFilterOptions& bf_options, const TypeInfo* typeinfo,
uint8_t gram_size, uint16_t gram_bf_size,
std::unique_ptr<BloomFilterIndexWriter>* res);
static Status create(const BloomFilterOptions& bf_options, FieldType type, uint8_t gram_size,
uint16_t gram_bf_size, std::unique_ptr<BloomFilterIndexWriter>* res);

NGramBloomFilterIndexWriterImpl(const BloomFilterOptions& bf_options, uint8_t gram_size,
uint16_t bf_size);
Expand Down
13 changes: 6 additions & 7 deletions be/src/storage/index/index_writer.cpp
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -48,8 +48,7 @@ bool IndexColumnWriter::check_support_ann_index(const TabletColumn& column) {
Status IndexColumnWriter::create(const StorageField* field, std::unique_ptr<IndexColumnWriter>* res,
IndexFileWriter* index_file_writer,
const TabletIndex* index_meta) {
const auto* typeinfo = field->type_info();
FieldType type = typeinfo->type();
FieldType type = field->type();
std::string field_name;
auto storage_format = index_file_writer->get_storage_format();
if (storage_format == InvertedIndexStorageFormatPB::V1) {
Expand All@@ -67,12 +66,12 @@ Status IndexColumnWriter::create(const StorageField* field, std::unique_ptr<Inde
if (index_meta->is_inverted_index()) {
bool single_field = true;
if (type == FieldType::OLAP_FIELD_TYPE_ARRAY) {
const auto* array_typeinfo = dynamic_cast<const ArrayTypeInfo*>(typeinfo);
const auto& column = field->get_desc();
Comment thread
csun5285 marked this conversation as resolved.
bool has_item_subcolumn = column.get_subtype_count() > 0;
DBUG_EXECUTE_IF("InvertedIndexColumnWriter::create_array_typeinfo_is_nullptr",
{ array_typeinfo = nullptr; })
if (array_typeinfo != nullptr) {
typeinfo = array_typeinfo->item_type_info();
type = typeinfo->type();
{ has_item_subcolumn = false; })
if (has_item_subcolumn) {
type = column.get_sub_column(0).type();
single_field = false;
} else {
return Status::NotSupported("unsupported array type for inverted index: " +
Expand Down
8 changes: 4 additions & 4 deletions be/src/storage/index/indexed_column_reader.cpp
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -64,12 +64,12 @@ Status IndexedColumnReader::load(bool use_page_cache, bool kept_in_memory,
_use_page_cache = use_page_cache;
_kept_in_memory = kept_in_memory;

_type_info = get_scalar_type_info((FieldType)_meta.data_type());
if (_type_info == nullptr) {
_type = (FieldType)_meta.data_type();
if (!is_scalar_type(_type)) {
Comment thread
csun5285 marked this conversation as resolved.
return Status::NotSupported("unsupported typeinfo, type={}", _meta.data_type());
}
RETURN_IF_ERROR(EncodingInfo::get(_type_info->type(), _meta.encoding(), {}, &_encoding_info));
_value_key_coder = get_key_coder(_type_info->type());
RETURN_IF_ERROR(EncodingInfo::get(_type, _meta.encoding(), {}, &_encoding_info));
_value_key_coder = get_key_coder(_type);

// read and parse ordinal index page when exists
if (_meta.has_ordinal_index_meta()) {
Expand Down
5 changes: 2 additions & 3 deletions be/src/storage/index/indexed_column_reader.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -39,7 +39,6 @@
namespace doris {

class KeyCoder;
class TypeInfo;
class BlockCompressionCodec;

namespace segment_v2 {
Expand DownExpand Up@@ -67,7 +66,7 @@ class IndexedColumnReader : public MetadataAdder<IndexedColumnReader> {

int64_t num_values() const { return _num_values; }
const EncodingInfo* encoding_info() const { return _encoding_info; }
const TypeInfo* type_info() const { return _type_info; }
FieldType type() const { return _type; }
bool support_ordinal_seek() const { return _meta.has_ordinal_index_meta(); }
bool support_value_seek() const { return _meta.has_value_index_meta(); }

Expand DownExpand Up@@ -99,7 +98,7 @@ class IndexedColumnReader : public MetadataAdder<IndexedColumnReader> {
PageHandle _ordinal_index_page_handle;
PageHandle _value_index_page_handle;

const TypeInfo* _type_info = nullptr;
FieldType _type = FieldType::OLAP_FIELD_TYPE_NONE;
const EncodingInfo* _encoding_info = nullptr;
const KeyCoder* _value_key_coder = nullptr;
uint64_t _mem_size = 0;
Expand Down
12 changes: 6 additions & 6 deletions be/src/storage/index/indexed_column_writer.cpp
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -39,10 +39,10 @@
namespace doris {
namespace segment_v2 {

IndexedColumnWriter::IndexedColumnWriter(const IndexedColumnWriterOptions& options,
const TypeInfo* type_info, io::FileWriter* file_writer)
IndexedColumnWriter::IndexedColumnWriter(const IndexedColumnWriterOptions& options, FieldType type,
io::FileWriter* file_writer)
: _options(options),
_type_info(type_info),
_type(type),
_file_writer(file_writer),
_num_values(0),
_num_data_pages(0),
Expand All@@ -54,7 +54,7 @@ IndexedColumnWriter::~IndexedColumnWriter() = default;

Status IndexedColumnWriter::init() {
const EncodingInfo* encoding_info;
RETURN_IF_ERROR(EncodingInfo::get(_type_info->type(), _options.encoding, {}, &encoding_info));
RETURN_IF_ERROR(EncodingInfo::get(_type, _options.encoding, {}, &encoding_info));
_options.encoding = encoding_info->encoding();
// should store more concrete encoding type instead of DEFAULT_ENCODING
// because the default encoding of a data type can be changed in the future
Expand All@@ -72,7 +72,7 @@ Status IndexedColumnWriter::init() {
}
if (_options.write_value_index) {
_value_index_builder.reset(new IndexPageBuilder(_options.index_page_size, true));
_value_key_coder = get_key_coder(_type_info->type());
_value_key_coder = get_key_coder(_type);
}

if (_options.compression != NO_COMPRESSION) {
Expand DownExpand Up@@ -159,7 +159,7 @@ Status IndexedColumnWriter::finish(IndexedColumnMetaPB* meta) {
if (_options.write_value_index) {
RETURN_IF_ERROR(_flush_index(_value_index_builder.get(), meta->mutable_value_index_meta()));
}
meta->set_data_type(int(_type_info->type()));
meta->set_data_type(int(_type));
meta->set_encoding(_options.encoding);
meta->set_num_values(_num_values);
meta->set_compression(_options.compression);
Expand Down
Loading
Loading