From a9c39c0d31c927b7b424897f45f28fadef9f8493 Mon Sep 17 00:00:00 2001 From: csun5285 Date: Mon, 8 Jun 2026 17:17:27 +0800 Subject: [PATCH] [refactor](be) Derive get_storage_field_type from primitive type Make IDataType::get_storage_field_type() a non-pure virtual with a base implementation that derives the storage field type from the primitive type via TabletColumn::get_field_type_by_type(get_primitive_type()), and drop the ~17 trivial 1:1 overrides that just duplicated that mapping. Only the data types whose storage field type genuinely diverges from the primitive->field mapping keep an override: - DataTypeNullable: delegates to the nested type (e.g. nullable(decimalv2)) - DataTypeDecimal: DECIMALV2 -> OLAP_FIELD_TYPE_DECIMAL - DataTypeNothing / DataTypeFixedLengthObject: -> OLAP_FIELD_TYPE_NONE - DataTypeVarbinary: not implemented (throws) DataTypeString no longer collapses CHAR/VARCHAR into STRING; a char column now honestly reports OLAP_FIELD_TYPE_CHAR (varchar -> VARCHAR). Every call site tolerates this: inverted index uses is_string_type() (covers all three), get_predicate_column_ptr maps VARCHAR and STRING to the same predicate column, _can_evaluated_by_vectorized treats CHAR/VARCHAR/STRING together, and zone map dispatches via the data type serde. Because get_storage_field_type() now returns OLAP_FIELD_TYPE_CHAR for char columns directly, the SegmentIterator::_is_char_type machinery (and _vec_init_char_column_id) is dead and removed: its only consumer forced CHAR for char columns, which the derived field type already does. Variant subcolumns can never be CHAR (Type.variantSubTypes only allows STRING), so no case relies on the old guard. Co-Authored-By: Claude Opus 4.8 --- be/src/core/data_type/data_type.cpp | 5 ++ be/src/core/data_type/data_type.h | 4 +- be/src/core/data_type/data_type_agg_state.h | 4 -- be/src/core/data_type/data_type_array.h | 4 -- be/src/core/data_type/data_type_bitmap.h | 4 -- be/src/core/data_type/data_type_date.h | 3 -- .../data_type/data_type_date_or_datetime_v2.h | 6 --- be/src/core/data_type/data_type_date_time.h | 4 -- be/src/core/data_type/data_type_hll.h | 4 -- be/src/core/data_type/data_type_ipv4.h | 4 -- be/src/core/data_type/data_type_ipv6.h | 3 -- be/src/core/data_type/data_type_jsonb.h | 3 -- be/src/core/data_type/data_type_map.h | 3 -- be/src/core/data_type/data_type_number_base.h | 53 ------------------- .../core/data_type/data_type_quantilestate.h | 3 -- be/src/core/data_type/data_type_string.h | 4 -- be/src/core/data_type/data_type_struct.h | 3 -- be/src/core/data_type/data_type_variant.h | 3 -- be/src/storage/segment/segment_iterator.cpp | 22 +------- be/src/storage/segment/segment_iterator.h | 3 -- 20 files changed, 9 insertions(+), 133 deletions(-) diff --git a/be/src/core/data_type/data_type.cpp b/be/src/core/data_type/data_type.cpp index a11c17a6cd24f7..94f7b6c38c9940 100644 --- a/be/src/core/data_type/data_type.cpp +++ b/be/src/core/data_type/data_type.cpp @@ -34,6 +34,7 @@ #include "core/data_type/define_primitive_type.h" #include "core/data_type_serde/data_type_serde.h" #include "core/field.h" +#include "storage/tablet/tablet_schema.h" namespace doris { class BufferWritable; @@ -45,6 +46,10 @@ IDataType::IDataType() = default; IDataType::~IDataType() = default; +doris::FieldType IDataType::get_storage_field_type() const { + return TabletColumn::get_field_type_by_type(get_primitive_type()); +} + String IDataType::get_name() const { return do_get_name(); } diff --git a/be/src/core/data_type/data_type.h b/be/src/core/data_type/data_type.h index 9fb341302c0333..253bd8e49f99db 100644 --- a/be/src/core/data_type/data_type.h +++ b/be/src/core/data_type/data_type.h @@ -83,7 +83,9 @@ class IDataType : private boost::noncopyable { virtual const std::string get_family_name() const = 0; virtual PrimitiveType get_primitive_type() const = 0; - virtual doris::FieldType get_storage_field_type() const = 0; + // Derived from the primitive type by default (e.g. TYPE_CHAR -> OLAP_FIELD_TYPE_CHAR). + // Types without a direct 1:1 mapping override this. + virtual doris::FieldType get_storage_field_type() const; std::string to_string(const IColumn& column, size_t row_num, const DataTypeSerDe::FormatOptions& options) const; // get specific serializer or deserializer diff --git a/be/src/core/data_type/data_type_agg_state.h b/be/src/core/data_type/data_type_agg_state.h index f1513c97c76a54..378315f9512e84 100644 --- a/be/src/core/data_type/data_type_agg_state.h +++ b/be/src/core/data_type/data_type_agg_state.h @@ -85,10 +85,6 @@ class DataTypeAggState : public DataTypeString { PrimitiveType get_primitive_type() const override { return PrimitiveType::TYPE_AGG_STATE; } - doris::FieldType get_storage_field_type() const override { - return doris::FieldType::OLAP_FIELD_TYPE_AGG_STATE; - } - const DataTypes& get_sub_types() const { return _sub_types; } void to_pb_column_meta(PColumnMeta* col_meta) const override { diff --git a/be/src/core/data_type/data_type_array.h b/be/src/core/data_type/data_type_array.h index 5ddc5236c41e2c..264798591216dc 100644 --- a/be/src/core/data_type/data_type_array.h +++ b/be/src/core/data_type/data_type_array.h @@ -59,10 +59,6 @@ class DataTypeArray final : public IDataType { PrimitiveType get_primitive_type() const override { return PrimitiveType::TYPE_ARRAY; } - doris::FieldType get_storage_field_type() const override { - return doris::FieldType::OLAP_FIELD_TYPE_ARRAY; - } - std::string do_get_name() const override { return "Array(" + nested->get_name() + ")"; } const std::string get_family_name() const override { return "Array"; } diff --git a/be/src/core/data_type/data_type_bitmap.h b/be/src/core/data_type/data_type_bitmap.h index c47e9b30a78c15..5e7fb8b67bac98 100644 --- a/be/src/core/data_type/data_type_bitmap.h +++ b/be/src/core/data_type/data_type_bitmap.h @@ -54,10 +54,6 @@ class DataTypeBitMap : public IDataType { const std::string get_family_name() const override { return "BitMap"; } PrimitiveType get_primitive_type() const override { return PrimitiveType::TYPE_BITMAP; } - doris::FieldType get_storage_field_type() const override { - return doris::FieldType::OLAP_FIELD_TYPE_BITMAP; - } - int64_t get_uncompressed_serialized_bytes(const IColumn& column, int be_exec_version) const override; char* serialize(const IColumn& column, char* buf, int be_exec_version) const override; diff --git a/be/src/core/data_type/data_type_date.h b/be/src/core/data_type/data_type_date.h index d048eff6aaab28..8acdb2f20e14e3 100644 --- a/be/src/core/data_type/data_type_date.h +++ b/be/src/core/data_type/data_type_date.h @@ -42,9 +42,6 @@ class DataTypeDate final : public DataTypeNumberBase { static constexpr PrimitiveType PType = TYPE_DATE; PrimitiveType get_primitive_type() const override { return PrimitiveType::TYPE_DATE; } - doris::FieldType get_storage_field_type() const override { - return doris::FieldType::OLAP_FIELD_TYPE_DATE; - } const std::string get_family_name() const override { return "Date"; } std::string do_get_name() const override { return "Date"; } diff --git a/be/src/core/data_type/data_type_date_or_datetime_v2.h b/be/src/core/data_type/data_type_date_or_datetime_v2.h index 80b04000c6636f..0e16644863ec9c 100644 --- a/be/src/core/data_type/data_type_date_or_datetime_v2.h +++ b/be/src/core/data_type/data_type_date_or_datetime_v2.h @@ -54,9 +54,6 @@ class DataTypeDateV2 final : public DataTypeNumberBaseset_scale(_scale); } - doris::FieldType get_storage_field_type() const override { - return doris::FieldType::OLAP_FIELD_TYPE_DATETIMEV2; - } const std::string get_family_name() const override { return "DateTimeV2"; } std::string do_get_name() const override { return "DateTimeV2(" + std::to_string(_scale) + ")"; diff --git a/be/src/core/data_type/data_type_date_time.h b/be/src/core/data_type/data_type_date_time.h index 0cdfda2405d11f..8e4d49913af148 100644 --- a/be/src/core/data_type/data_type_date_time.h +++ b/be/src/core/data_type/data_type_date_time.h @@ -69,10 +69,6 @@ class DataTypeDateTime final : public DataTypeNumberBase { const std::string get_family_name() const override { return "IPv4"; } std::string do_get_name() const override { return "IPv4"; } - doris::FieldType get_storage_field_type() const override { - return doris::FieldType::OLAP_FIELD_TYPE_IPV4; - } - bool equals(const IDataType& rhs) const override; Field get_field(const TExprNode& node) const override; diff --git a/be/src/core/data_type/data_type_ipv6.h b/be/src/core/data_type/data_type_ipv6.h index cc48d3c7284c99..2ab1b15763157e 100644 --- a/be/src/core/data_type/data_type_ipv6.h +++ b/be/src/core/data_type/data_type_ipv6.h @@ -42,9 +42,6 @@ namespace doris { class DataTypeIPv6 final : public DataTypeNumberBase { public: PrimitiveType get_primitive_type() const override { return PrimitiveType::TYPE_IPV6; } - doris::FieldType get_storage_field_type() const override { - return doris::FieldType::OLAP_FIELD_TYPE_IPV6; - } const std::string get_family_name() const override { return "IPv6"; } std::string do_get_name() const override { return "IPv6"; } diff --git a/be/src/core/data_type/data_type_jsonb.h b/be/src/core/data_type/data_type_jsonb.h index 7cd866d2c621cd..483ef874fe9899 100644 --- a/be/src/core/data_type/data_type_jsonb.h +++ b/be/src/core/data_type/data_type_jsonb.h @@ -50,9 +50,6 @@ class DataTypeJsonb final : public IDataType { const std::string get_family_name() const override { return "JSONB"; } PrimitiveType get_primitive_type() const override { return PrimitiveType::TYPE_JSONB; } - doris::FieldType get_storage_field_type() const override { - return doris::FieldType::OLAP_FIELD_TYPE_JSONB; - } int64_t get_uncompressed_serialized_bytes(const IColumn& column, int be_exec_version) const override; diff --git a/be/src/core/data_type/data_type_map.h b/be/src/core/data_type/data_type_map.h index ccb155af330574..b442b64da9b52f 100644 --- a/be/src/core/data_type/data_type_map.h +++ b/be/src/core/data_type/data_type_map.h @@ -56,9 +56,6 @@ class DataTypeMap final : public IDataType { DataTypeMap(const DataTypePtr& key_type_, const DataTypePtr& value_type_); PrimitiveType get_primitive_type() const override { return PrimitiveType::TYPE_MAP; } - doris::FieldType get_storage_field_type() const override { - return doris::FieldType::OLAP_FIELD_TYPE_MAP; - } std::string do_get_name() const override { return "Map(" + key_type->get_name() + ", " + value_type->get_name() + ")"; diff --git a/be/src/core/data_type/data_type_number_base.h b/be/src/core/data_type/data_type_number_base.h index cc87f8d6b041af..f3fe2544196fce 100644 --- a/be/src/core/data_type/data_type_number_base.h +++ b/be/src/core/data_type/data_type_number_base.h @@ -66,59 +66,6 @@ class DataTypeNumberBase : public IDataType { return T; } - doris::FieldType get_storage_field_type() const override { - // Doris does not support uint8 at present, use uint8 as boolean type - if constexpr (T == TYPE_BOOLEAN) { - return doris::FieldType::OLAP_FIELD_TYPE_BOOL; - } - if constexpr (T == TYPE_TINYINT) { - return doris::FieldType::OLAP_FIELD_TYPE_TINYINT; - } - if constexpr (T == TYPE_SMALLINT) { - return doris::FieldType::OLAP_FIELD_TYPE_SMALLINT; - } - if constexpr (T == TYPE_INT) { - return doris::FieldType::OLAP_FIELD_TYPE_INT; - } - if constexpr (T == TYPE_BIGINT) { - return doris::FieldType::OLAP_FIELD_TYPE_BIGINT; - } - if constexpr (T == TYPE_LARGEINT) { - return doris::FieldType::OLAP_FIELD_TYPE_LARGEINT; - } - if constexpr (T == TYPE_FLOAT) { - return doris::FieldType::OLAP_FIELD_TYPE_FLOAT; - } - if constexpr (T == TYPE_DOUBLE) { - return doris::FieldType::OLAP_FIELD_TYPE_DOUBLE; - } - if constexpr (T == TYPE_DATE) { - return doris::FieldType::OLAP_FIELD_TYPE_DATE; - } - if constexpr (T == TYPE_DATETIME) { - return doris::FieldType::OLAP_FIELD_TYPE_DATETIME; - } - if constexpr (T == TYPE_DATEV2) { - return doris::FieldType::OLAP_FIELD_TYPE_DATEV2; - } - if constexpr (T == TYPE_DATETIMEV2) { - return doris::FieldType::OLAP_FIELD_TYPE_DATETIMEV2; - } - if constexpr (T == TYPE_TIMESTAMPTZ) { - return doris::FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ; - } - if constexpr (T == TYPE_IPV4) { - return doris::FieldType::OLAP_FIELD_TYPE_IPV4; - } - if constexpr (T == TYPE_IPV6) { - return doris::FieldType::OLAP_FIELD_TYPE_IPV6; - } - if constexpr (T == TYPE_TIMEV2) { - return doris::FieldType::OLAP_FIELD_TYPE_TIMEV2; - } - throw Exception(Status::FatalError("__builtin_unreachable")); - } - Field get_field(const TExprNode& node) const override; int64_t get_uncompressed_serialized_bytes(const IColumn& column, diff --git a/be/src/core/data_type/data_type_quantilestate.h b/be/src/core/data_type/data_type_quantilestate.h index 370f9bcb7d487e..d469f6b3588de2 100644 --- a/be/src/core/data_type/data_type_quantilestate.h +++ b/be/src/core/data_type/data_type_quantilestate.h @@ -51,9 +51,6 @@ class DataTypeQuantileState : public IDataType { const std::string get_family_name() const override { return "QuantileState"; } PrimitiveType get_primitive_type() const override { return PrimitiveType::TYPE_QUANTILE_STATE; } - doris::FieldType get_storage_field_type() const override { - return doris::FieldType::OLAP_FIELD_TYPE_QUANTILE_STATE; - } int64_t get_uncompressed_serialized_bytes(const IColumn& column, int be_exec_version) const override; char* serialize(const IColumn& column, char* buf, int be_exec_version) const override; diff --git a/be/src/core/data_type/data_type_string.h b/be/src/core/data_type/data_type_string.h index 32385b7afb1991..bfd86f7d9de8ac 100644 --- a/be/src/core/data_type/data_type_string.h +++ b/be/src/core/data_type/data_type_string.h @@ -55,10 +55,6 @@ class DataTypeString : public IDataType { } PrimitiveType get_primitive_type() const override { return _primitive_type; } - doris::FieldType get_storage_field_type() const override { - return doris::FieldType::OLAP_FIELD_TYPE_STRING; - } - int64_t get_uncompressed_serialized_bytes(const IColumn& column, int be_exec_version) const override; char* serialize(const IColumn& column, char* buf, int be_exec_version) const override; diff --git a/be/src/core/data_type/data_type_struct.h b/be/src/core/data_type/data_type_struct.h index 657364efbca52f..d4e4abd7a88829 100644 --- a/be/src/core/data_type/data_type_struct.h +++ b/be/src/core/data_type/data_type_struct.h @@ -70,9 +70,6 @@ class DataTypeStruct final : public IDataType { DataTypeStruct(const DataTypes& elems, const Strings& names); PrimitiveType get_primitive_type() const override { return PrimitiveType::TYPE_STRUCT; } - doris::FieldType get_storage_field_type() const override { - return doris::FieldType::OLAP_FIELD_TYPE_STRUCT; - } std::string do_get_name() const override; const std::string get_family_name() const override { return "Struct"; } diff --git a/be/src/core/data_type/data_type_variant.h b/be/src/core/data_type/data_type_variant.h index f8ec3484d6b686..3f4e08a0ada620 100644 --- a/be/src/core/data_type/data_type_variant.h +++ b/be/src/core/data_type/data_type_variant.h @@ -60,9 +60,6 @@ class DataTypeVariant : public IDataType { String do_get_name() const override { return name; } const std::string get_family_name() const override { return "Variant"; } - doris::FieldType get_storage_field_type() const override { - return doris::FieldType::OLAP_FIELD_TYPE_VARIANT; - } Status check_column(const IColumn& column) const override { return check_column_non_nested_type(column); } diff --git a/be/src/storage/segment/segment_iterator.cpp b/be/src/storage/segment/segment_iterator.cpp index dc6930777f5508..c1ba1b1777828e 100644 --- a/be/src/storage/segment/segment_iterator.cpp +++ b/be/src/storage/segment/segment_iterator.cpp @@ -654,21 +654,16 @@ Status SegmentIterator::_lazy_init(Block* block) { } _current_return_columns.resize(_schema->columns().size()); - _vec_init_char_column_id(); for (size_t i = 0; i < _schema->column_ids().size(); i++) { ColumnId cid = _schema->column_ids()[i]; const auto* column_desc = _schema->column(cid); if (_is_pred_column[cid]) { auto storage_column_type = _storage_name_and_type[cid].second; - // Char type is special , since char type's computational datatype is same with string, - // both are DataTypeString, but DataTypeString only return FieldType::OLAP_FIELD_TYPE_STRING - // in get_storage_field_type. RETURN_IF_CATCH_EXCEPTION( // Here, cid will not go out of bounds // because the size of _current_return_columns equals _schema->tablet_columns().size() _current_return_columns[cid] = Schema::get_predicate_column_ptr( - _is_char_type[cid] ? FieldType::OLAP_FIELD_TYPE_CHAR - : storage_column_type->get_storage_field_type(), + storage_column_type->get_storage_field_type(), storage_column_type->is_nullable(), _opts.io_ctx.reader_type)); _current_return_columns[cid]->set_rowset_segment_id( {_segment->rowset_id(), _segment->id()}); @@ -2201,21 +2196,6 @@ bool SegmentIterator::_can_evaluated_by_vectorized(std::shared_ptrcolumns().size(), false); - for (size_t i = 0; i < _schema->num_column_ids(); i++) { - auto cid = _schema->column_id(i); - const TabletColumn* column_desc = _schema->column(cid); - - if (column_desc->type() == FieldType::OLAP_FIELD_TYPE_CHAR) { - _is_char_type[cid] = true; - } - } -} - bool SegmentIterator::_prune_column(ColumnId cid, MutableColumnPtr& column, bool fill_defaults, size_t num_of_defaults) { if (_need_read_data(cid)) { diff --git a/be/src/storage/segment/segment_iterator.h b/be/src/storage/segment/segment_iterator.h index c7faf3fdb51d17..8fd143867ed97b 100644 --- a/be/src/storage/segment/segment_iterator.h +++ b/be/src/storage/segment/segment_iterator.h @@ -204,7 +204,6 @@ class SegmentIterator : public RowwiseIterator { bool _is_literal_node(const TExprNodeType::type& node_type); Status _vec_init_lazy_materialization(); - void _vec_init_char_column_id(); uint32_t segment_id() const { return _segment->id(); } uint32_t num_rows() const { return _segment->num_rows(); } @@ -427,8 +426,6 @@ class SegmentIterator : public RowwiseIterator { io::FileReaderSPtr _file_reader; - std::vector _is_char_type; - // used for compaction, record selectd rowids of current batch uint16_t _selected_size; std::vector _sel_rowid_idx;