diff --git a/be/src/core/data_type/data_type.cpp b/be/src/core/data_type/data_type.cpp index a11c17a6cd24f7..94f7b6c38c9940 100644 --- a/be/src/core/data_type/data_type.cpp +++ b/be/src/core/data_type/data_type.cpp @@ -34,6 +34,7 @@ #include "core/data_type/define_primitive_type.h" #include "core/data_type_serde/data_type_serde.h" #include "core/field.h" +#include "storage/tablet/tablet_schema.h" namespace doris { class BufferWritable; @@ -45,6 +46,10 @@ IDataType::IDataType() = default; IDataType::~IDataType() = default; +doris::FieldType IDataType::get_storage_field_type() const { + return TabletColumn::get_field_type_by_type(get_primitive_type()); +} + String IDataType::get_name() const { return do_get_name(); } diff --git a/be/src/core/data_type/data_type.h b/be/src/core/data_type/data_type.h index 9fb341302c0333..253bd8e49f99db 100644 --- a/be/src/core/data_type/data_type.h +++ b/be/src/core/data_type/data_type.h @@ -83,7 +83,9 @@ class IDataType : private boost::noncopyable { virtual const std::string get_family_name() const = 0; virtual PrimitiveType get_primitive_type() const = 0; - virtual doris::FieldType get_storage_field_type() const = 0; + // Derived from the primitive type by default (e.g. TYPE_CHAR -> OLAP_FIELD_TYPE_CHAR). + // Types without a direct 1:1 mapping override this. + virtual doris::FieldType get_storage_field_type() const; std::string to_string(const IColumn& column, size_t row_num, const DataTypeSerDe::FormatOptions& options) const; // get specific serializer or deserializer diff --git a/be/src/core/data_type/data_type_agg_state.h b/be/src/core/data_type/data_type_agg_state.h index f1513c97c76a54..378315f9512e84 100644 --- a/be/src/core/data_type/data_type_agg_state.h +++ b/be/src/core/data_type/data_type_agg_state.h @@ -85,10 +85,6 @@ class DataTypeAggState : public DataTypeString { PrimitiveType get_primitive_type() const override { return PrimitiveType::TYPE_AGG_STATE; } - doris::FieldType get_storage_field_type() const override { - return doris::FieldType::OLAP_FIELD_TYPE_AGG_STATE; - } - const DataTypes& get_sub_types() const { return _sub_types; } void to_pb_column_meta(PColumnMeta* col_meta) const override { diff --git a/be/src/core/data_type/data_type_array.h b/be/src/core/data_type/data_type_array.h index 5ddc5236c41e2c..264798591216dc 100644 --- a/be/src/core/data_type/data_type_array.h +++ b/be/src/core/data_type/data_type_array.h @@ -59,10 +59,6 @@ class DataTypeArray final : public IDataType { PrimitiveType get_primitive_type() const override { return PrimitiveType::TYPE_ARRAY; } - doris::FieldType get_storage_field_type() const override { - return doris::FieldType::OLAP_FIELD_TYPE_ARRAY; - } - std::string do_get_name() const override { return "Array(" + nested->get_name() + ")"; } const std::string get_family_name() const override { return "Array"; } diff --git a/be/src/core/data_type/data_type_bitmap.h b/be/src/core/data_type/data_type_bitmap.h index c47e9b30a78c15..5e7fb8b67bac98 100644 --- a/be/src/core/data_type/data_type_bitmap.h +++ b/be/src/core/data_type/data_type_bitmap.h @@ -54,10 +54,6 @@ class DataTypeBitMap : public IDataType { const std::string get_family_name() const override { return "BitMap"; } PrimitiveType get_primitive_type() const override { return PrimitiveType::TYPE_BITMAP; } - doris::FieldType get_storage_field_type() const override { - return doris::FieldType::OLAP_FIELD_TYPE_BITMAP; - } - int64_t get_uncompressed_serialized_bytes(const IColumn& column, int be_exec_version) const override; char* serialize(const IColumn& column, char* buf, int be_exec_version) const override; diff --git a/be/src/core/data_type/data_type_date.h b/be/src/core/data_type/data_type_date.h index d048eff6aaab28..8acdb2f20e14e3 100644 --- a/be/src/core/data_type/data_type_date.h +++ b/be/src/core/data_type/data_type_date.h @@ -42,9 +42,6 @@ class DataTypeDate final : public DataTypeNumberBase { static constexpr PrimitiveType PType = TYPE_DATE; PrimitiveType get_primitive_type() const override { return PrimitiveType::TYPE_DATE; } - doris::FieldType get_storage_field_type() const override { - return doris::FieldType::OLAP_FIELD_TYPE_DATE; - } const std::string get_family_name() const override { return "Date"; } std::string do_get_name() const override { return "Date"; } diff --git a/be/src/core/data_type/data_type_date_or_datetime_v2.h b/be/src/core/data_type/data_type_date_or_datetime_v2.h index 80b04000c6636f..0e16644863ec9c 100644 --- a/be/src/core/data_type/data_type_date_or_datetime_v2.h +++ b/be/src/core/data_type/data_type_date_or_datetime_v2.h @@ -54,9 +54,6 @@ class DataTypeDateV2 final : public DataTypeNumberBaseset_scale(_scale); } - doris::FieldType get_storage_field_type() const override { - return doris::FieldType::OLAP_FIELD_TYPE_DATETIMEV2; - } const std::string get_family_name() const override { return "DateTimeV2"; } std::string do_get_name() const override { return "DateTimeV2(" + std::to_string(_scale) + ")"; diff --git a/be/src/core/data_type/data_type_date_time.h b/be/src/core/data_type/data_type_date_time.h index 0cdfda2405d11f..8e4d49913af148 100644 --- a/be/src/core/data_type/data_type_date_time.h +++ b/be/src/core/data_type/data_type_date_time.h @@ -69,10 +69,6 @@ class DataTypeDateTime final : public DataTypeNumberBase { const std::string get_family_name() const override { return "IPv4"; } std::string do_get_name() const override { return "IPv4"; } - doris::FieldType get_storage_field_type() const override { - return doris::FieldType::OLAP_FIELD_TYPE_IPV4; - } - bool equals(const IDataType& rhs) const override; Field get_field(const TExprNode& node) const override; diff --git a/be/src/core/data_type/data_type_ipv6.h b/be/src/core/data_type/data_type_ipv6.h index cc48d3c7284c99..2ab1b15763157e 100644 --- a/be/src/core/data_type/data_type_ipv6.h +++ b/be/src/core/data_type/data_type_ipv6.h @@ -42,9 +42,6 @@ namespace doris { class DataTypeIPv6 final : public DataTypeNumberBase { public: PrimitiveType get_primitive_type() const override { return PrimitiveType::TYPE_IPV6; } - doris::FieldType get_storage_field_type() const override { - return doris::FieldType::OLAP_FIELD_TYPE_IPV6; - } const std::string get_family_name() const override { return "IPv6"; } std::string do_get_name() const override { return "IPv6"; } diff --git a/be/src/core/data_type/data_type_jsonb.h b/be/src/core/data_type/data_type_jsonb.h index 7cd866d2c621cd..483ef874fe9899 100644 --- a/be/src/core/data_type/data_type_jsonb.h +++ b/be/src/core/data_type/data_type_jsonb.h @@ -50,9 +50,6 @@ class DataTypeJsonb final : public IDataType { const std::string get_family_name() const override { return "JSONB"; } PrimitiveType get_primitive_type() const override { return PrimitiveType::TYPE_JSONB; } - doris::FieldType get_storage_field_type() const override { - return doris::FieldType::OLAP_FIELD_TYPE_JSONB; - } int64_t get_uncompressed_serialized_bytes(const IColumn& column, int be_exec_version) const override; diff --git a/be/src/core/data_type/data_type_map.h b/be/src/core/data_type/data_type_map.h index ccb155af330574..b442b64da9b52f 100644 --- a/be/src/core/data_type/data_type_map.h +++ b/be/src/core/data_type/data_type_map.h @@ -56,9 +56,6 @@ class DataTypeMap final : public IDataType { DataTypeMap(const DataTypePtr& key_type_, const DataTypePtr& value_type_); PrimitiveType get_primitive_type() const override { return PrimitiveType::TYPE_MAP; } - doris::FieldType get_storage_field_type() const override { - return doris::FieldType::OLAP_FIELD_TYPE_MAP; - } std::string do_get_name() const override { return "Map(" + key_type->get_name() + ", " + value_type->get_name() + ")"; diff --git a/be/src/core/data_type/data_type_number_base.h b/be/src/core/data_type/data_type_number_base.h index cc87f8d6b041af..f3fe2544196fce 100644 --- a/be/src/core/data_type/data_type_number_base.h +++ b/be/src/core/data_type/data_type_number_base.h @@ -66,59 +66,6 @@ class DataTypeNumberBase : public IDataType { return T; } - doris::FieldType get_storage_field_type() const override { - // Doris does not support uint8 at present, use uint8 as boolean type - if constexpr (T == TYPE_BOOLEAN) { - return doris::FieldType::OLAP_FIELD_TYPE_BOOL; - } - if constexpr (T == TYPE_TINYINT) { - return doris::FieldType::OLAP_FIELD_TYPE_TINYINT; - } - if constexpr (T == TYPE_SMALLINT) { - return doris::FieldType::OLAP_FIELD_TYPE_SMALLINT; - } - if constexpr (T == TYPE_INT) { - return doris::FieldType::OLAP_FIELD_TYPE_INT; - } - if constexpr (T == TYPE_BIGINT) { - return doris::FieldType::OLAP_FIELD_TYPE_BIGINT; - } - if constexpr (T == TYPE_LARGEINT) { - return doris::FieldType::OLAP_FIELD_TYPE_LARGEINT; - } - if constexpr (T == TYPE_FLOAT) { - return doris::FieldType::OLAP_FIELD_TYPE_FLOAT; - } - if constexpr (T == TYPE_DOUBLE) { - return doris::FieldType::OLAP_FIELD_TYPE_DOUBLE; - } - if constexpr (T == TYPE_DATE) { - return doris::FieldType::OLAP_FIELD_TYPE_DATE; - } - if constexpr (T == TYPE_DATETIME) { - return doris::FieldType::OLAP_FIELD_TYPE_DATETIME; - } - if constexpr (T == TYPE_DATEV2) { - return doris::FieldType::OLAP_FIELD_TYPE_DATEV2; - } - if constexpr (T == TYPE_DATETIMEV2) { - return doris::FieldType::OLAP_FIELD_TYPE_DATETIMEV2; - } - if constexpr (T == TYPE_TIMESTAMPTZ) { - return doris::FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ; - } - if constexpr (T == TYPE_IPV4) { - return doris::FieldType::OLAP_FIELD_TYPE_IPV4; - } - if constexpr (T == TYPE_IPV6) { - return doris::FieldType::OLAP_FIELD_TYPE_IPV6; - } - if constexpr (T == TYPE_TIMEV2) { - return doris::FieldType::OLAP_FIELD_TYPE_TIMEV2; - } - throw Exception(Status::FatalError("__builtin_unreachable")); - } - Field get_field(const TExprNode& node) const override; int64_t get_uncompressed_serialized_bytes(const IColumn& column, diff --git a/be/src/core/data_type/data_type_quantilestate.h b/be/src/core/data_type/data_type_quantilestate.h index 370f9bcb7d487e..d469f6b3588de2 100644 --- a/be/src/core/data_type/data_type_quantilestate.h +++ b/be/src/core/data_type/data_type_quantilestate.h @@ -51,9 +51,6 @@ class DataTypeQuantileState : public IDataType { const std::string get_family_name() const override { return "QuantileState"; } PrimitiveType get_primitive_type() const override { return PrimitiveType::TYPE_QUANTILE_STATE; } - doris::FieldType get_storage_field_type() const override { - return doris::FieldType::OLAP_FIELD_TYPE_QUANTILE_STATE; - } int64_t get_uncompressed_serialized_bytes(const IColumn& column, int be_exec_version) const override; char* serialize(const IColumn& column, char* buf, int be_exec_version) const override; diff --git a/be/src/core/data_type/data_type_string.h b/be/src/core/data_type/data_type_string.h index 32385b7afb1991..bfd86f7d9de8ac 100644 --- a/be/src/core/data_type/data_type_string.h +++ b/be/src/core/data_type/data_type_string.h @@ -55,10 +55,6 @@ class DataTypeString : public IDataType { } PrimitiveType get_primitive_type() const override { return _primitive_type; } - doris::FieldType get_storage_field_type() const override { - return doris::FieldType::OLAP_FIELD_TYPE_STRING; - } - int64_t get_uncompressed_serialized_bytes(const IColumn& column, int be_exec_version) const override; char* serialize(const IColumn& column, char* buf, int be_exec_version) const override; diff --git a/be/src/core/data_type/data_type_struct.h b/be/src/core/data_type/data_type_struct.h index 657364efbca52f..d4e4abd7a88829 100644 --- a/be/src/core/data_type/data_type_struct.h +++ b/be/src/core/data_type/data_type_struct.h @@ -70,9 +70,6 @@ class DataTypeStruct final : public IDataType { DataTypeStruct(const DataTypes& elems, const Strings& names); PrimitiveType get_primitive_type() const override { return PrimitiveType::TYPE_STRUCT; } - doris::FieldType get_storage_field_type() const override { - return doris::FieldType::OLAP_FIELD_TYPE_STRUCT; - } std::string do_get_name() const override; const std::string get_family_name() const override { return "Struct"; } diff --git a/be/src/core/data_type/data_type_variant.h b/be/src/core/data_type/data_type_variant.h index f8ec3484d6b686..3f4e08a0ada620 100644 --- a/be/src/core/data_type/data_type_variant.h +++ b/be/src/core/data_type/data_type_variant.h @@ -60,9 +60,6 @@ class DataTypeVariant : public IDataType { String do_get_name() const override { return name; } const std::string get_family_name() const override { return "Variant"; } - doris::FieldType get_storage_field_type() const override { - return doris::FieldType::OLAP_FIELD_TYPE_VARIANT; - } Status check_column(const IColumn& column) const override { return check_column_non_nested_type(column); } diff --git a/be/src/storage/segment/segment_iterator.cpp b/be/src/storage/segment/segment_iterator.cpp index dc6930777f5508..c1ba1b1777828e 100644 --- a/be/src/storage/segment/segment_iterator.cpp +++ b/be/src/storage/segment/segment_iterator.cpp @@ -654,21 +654,16 @@ Status SegmentIterator::_lazy_init(Block* block) { } _current_return_columns.resize(_schema->columns().size()); - _vec_init_char_column_id(); for (size_t i = 0; i < _schema->column_ids().size(); i++) { ColumnId cid = _schema->column_ids()[i]; const auto* column_desc = _schema->column(cid); if (_is_pred_column[cid]) { auto storage_column_type = _storage_name_and_type[cid].second; - // Char type is special , since char type's computational datatype is same with string, - // both are DataTypeString, but DataTypeString only return FieldType::OLAP_FIELD_TYPE_STRING - // in get_storage_field_type. RETURN_IF_CATCH_EXCEPTION( // Here, cid will not go out of bounds // because the size of _current_return_columns equals _schema->tablet_columns().size() _current_return_columns[cid] = Schema::get_predicate_column_ptr( - _is_char_type[cid] ? FieldType::OLAP_FIELD_TYPE_CHAR - : storage_column_type->get_storage_field_type(), + storage_column_type->get_storage_field_type(), storage_column_type->is_nullable(), _opts.io_ctx.reader_type)); _current_return_columns[cid]->set_rowset_segment_id( {_segment->rowset_id(), _segment->id()}); @@ -2201,21 +2196,6 @@ bool SegmentIterator::_can_evaluated_by_vectorized(std::shared_ptrcolumns().size(), false); - for (size_t i = 0; i < _schema->num_column_ids(); i++) { - auto cid = _schema->column_id(i); - const TabletColumn* column_desc = _schema->column(cid); - - if (column_desc->type() == FieldType::OLAP_FIELD_TYPE_CHAR) { - _is_char_type[cid] = true; - } - } -} - bool SegmentIterator::_prune_column(ColumnId cid, MutableColumnPtr& column, bool fill_defaults, size_t num_of_defaults) { if (_need_read_data(cid)) { diff --git a/be/src/storage/segment/segment_iterator.h b/be/src/storage/segment/segment_iterator.h index c7faf3fdb51d17..8fd143867ed97b 100644 --- a/be/src/storage/segment/segment_iterator.h +++ b/be/src/storage/segment/segment_iterator.h @@ -204,7 +204,6 @@ class SegmentIterator : public RowwiseIterator { bool _is_literal_node(const TExprNodeType::type& node_type); Status _vec_init_lazy_materialization(); - void _vec_init_char_column_id(); uint32_t segment_id() const { return _segment->id(); } uint32_t num_rows() const { return _segment->num_rows(); } @@ -427,8 +426,6 @@ class SegmentIterator : public RowwiseIterator { io::FileReaderSPtr _file_reader; - std::vector _is_char_type; - // used for compaction, record selectd rowids of current batch uint16_t _selected_size; std::vector _sel_rowid_idx;