Skip to content

[refactor](be) Derive get_storage_field_type from primitive type#64341

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
yiguolei merged 1 commit into from
Jun 10, 2026
Merged

[refactor](be) Derive get_storage_field_type from primitive type #64341

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions be/src/core/data_type/data_type.cpp
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -34,6 +34,7 @@
#include "core/data_type/define_primitive_type.h"
#include "core/data_type_serde/data_type_serde.h"
#include "core/field.h"
#include "storage/tablet/tablet_schema.h"

namespace doris {
class BufferWritable;
Expand All@@ -45,6 +46,10 @@ IDataType::IDataType() = default;

IDataType::~IDataType() = default;

doris::FieldType IDataType::get_storage_field_type() const {
return TabletColumn::get_field_type_by_type(get_primitive_type());
}

String IDataType::get_name() const {
return do_get_name();
}
Expand Down
4 changes: 3 additions & 1 deletion be/src/core/data_type/data_type.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -83,7 +83,9 @@ class IDataType : private boost::noncopyable {
virtual const std::string get_family_name() const = 0;
virtual PrimitiveType get_primitive_type() const = 0;

virtual doris::FieldType get_storage_field_type() const = 0;
// Derived from the primitive type by default (e.g. TYPE_CHAR -> OLAP_FIELD_TYPE_CHAR).
// Types without a direct 1:1 mapping override this.
virtual doris::FieldType get_storage_field_type() const;
std::string to_string(const IColumn& column, size_t row_num,
const DataTypeSerDe::FormatOptions& options) const;
// get specific serializer or deserializer
Expand Down
4 changes: 0 additions & 4 deletions be/src/core/data_type/data_type_agg_state.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -85,10 +85,6 @@ class DataTypeAggState : public DataTypeString {

PrimitiveType get_primitive_type() const override { return PrimitiveType::TYPE_AGG_STATE; }

doris::FieldType get_storage_field_type() const override {
return doris::FieldType::OLAP_FIELD_TYPE_AGG_STATE;
}

const DataTypes& get_sub_types() const { return _sub_types; }

void to_pb_column_meta(PColumnMeta* col_meta) const override {
Expand Down
4 changes: 0 additions & 4 deletions be/src/core/data_type/data_type_array.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -59,10 +59,6 @@ class DataTypeArray final : public IDataType {

PrimitiveType get_primitive_type() const override { return PrimitiveType::TYPE_ARRAY; }

doris::FieldType get_storage_field_type() const override {
return doris::FieldType::OLAP_FIELD_TYPE_ARRAY;
}

std::string do_get_name() const override { return "Array(" + nested->get_name() + ")"; }

const std::string get_family_name() const override { return "Array"; }
Expand Down
4 changes: 0 additions & 4 deletions be/src/core/data_type/data_type_bitmap.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -54,10 +54,6 @@ class DataTypeBitMap : public IDataType {
const std::string get_family_name() const override { return "BitMap"; }
PrimitiveType get_primitive_type() const override { return PrimitiveType::TYPE_BITMAP; }

doris::FieldType get_storage_field_type() const override {
return doris::FieldType::OLAP_FIELD_TYPE_BITMAP;
}

int64_t get_uncompressed_serialized_bytes(const IColumn& column,
int be_exec_version) const override;
char* serialize(const IColumn& column, char* buf, int be_exec_version) const override;
Expand Down
3 changes: 0 additions & 3 deletions be/src/core/data_type/data_type_date.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -42,9 +42,6 @@ class DataTypeDate final : public DataTypeNumberBase<PrimitiveType::TYPE_DATE> {
static constexpr PrimitiveType PType = TYPE_DATE;
PrimitiveType get_primitive_type() const override { return PrimitiveType::TYPE_DATE; }

doris::FieldType get_storage_field_type() const override {
return doris::FieldType::OLAP_FIELD_TYPE_DATE;
}
const std::string get_family_name() const override { return "Date"; }
std::string do_get_name() const override { return "Date"; }

Expand Down
6 changes: 0 additions & 6 deletions be/src/core/data_type/data_type_date_or_datetime_v2.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -54,9 +54,6 @@ class DataTypeDateV2 final : public DataTypeNumberBase<PrimitiveType::TYPE_DATEV
static constexpr PrimitiveType PType = TYPE_DATEV2;
PrimitiveType get_primitive_type() const override { return PrimitiveType::TYPE_DATEV2; }

doris::FieldType get_storage_field_type() const override {
return doris::FieldType::OLAP_FIELD_TYPE_DATEV2;
}
const std::string get_family_name() const override { return "DateV2"; }
std::string do_get_name() const override { return "DateV2"; }

Expand DownExpand Up@@ -111,9 +108,6 @@ class DataTypeDateTimeV2 final : public DataTypeNumberBase<PrimitiveType::TYPE_D
scalar_type->set_scale(_scale);
}

doris::FieldType get_storage_field_type() const override {
return doris::FieldType::OLAP_FIELD_TYPE_DATETIMEV2;
}
const std::string get_family_name() const override { return "DateTimeV2"; }
std::string do_get_name() const override {
return "DateTimeV2(" + std::to_string(_scale) + ")";
Expand Down
4 changes: 0 additions & 4 deletions be/src/core/data_type/data_type_date_time.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -69,10 +69,6 @@ class DataTypeDateTime final : public DataTypeNumberBase<PrimitiveType::TYPE_DAT
std::string do_get_name() const override { return "DateTime"; }
PrimitiveType get_primitive_type() const override { return PrimitiveType::TYPE_DATETIME; }

doris::FieldType get_storage_field_type() const override {
return doris::FieldType::OLAP_FIELD_TYPE_DATETIME;
}

bool equals(const IDataType& rhs) const override;
#ifdef BE_TEST
/// TODO: remove this in the future
Expand Down
4 changes: 0 additions & 4 deletions be/src/core/data_type/data_type_hll.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -54,10 +54,6 @@ class DataTypeHLL : public IDataType {

PrimitiveType get_primitive_type() const override { return PrimitiveType::TYPE_HLL; }

doris::FieldType get_storage_field_type() const override {
return doris::FieldType::OLAP_FIELD_TYPE_HLL;
}

int64_t get_uncompressed_serialized_bytes(const IColumn& column,
int be_exec_version) const override;
char* serialize(const IColumn& column, char* buf, int be_exec_version) const override;
Expand Down
4 changes: 0 additions & 4 deletions be/src/core/data_type/data_type_ipv4.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -45,10 +45,6 @@ class DataTypeIPv4 final : public DataTypeNumberBase<PrimitiveType::TYPE_IPV4> {
const std::string get_family_name() const override { return "IPv4"; }
std::string do_get_name() const override { return "IPv4"; }

doris::FieldType get_storage_field_type() const override {
return doris::FieldType::OLAP_FIELD_TYPE_IPV4;
}

bool equals(const IDataType& rhs) const override;

Field get_field(const TExprNode& node) const override;
Expand Down
3 changes: 0 additions & 3 deletions be/src/core/data_type/data_type_ipv6.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -42,9 +42,6 @@ namespace doris {
class DataTypeIPv6 final : public DataTypeNumberBase<PrimitiveType::TYPE_IPV6> {
public:
PrimitiveType get_primitive_type() const override { return PrimitiveType::TYPE_IPV6; }
doris::FieldType get_storage_field_type() const override {
return doris::FieldType::OLAP_FIELD_TYPE_IPV6;
}
const std::string get_family_name() const override { return "IPv6"; }
std::string do_get_name() const override { return "IPv6"; }

Expand Down
3 changes: 0 additions & 3 deletions be/src/core/data_type/data_type_jsonb.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -50,9 +50,6 @@ class DataTypeJsonb final : public IDataType {

const std::string get_family_name() const override { return "JSONB"; }
PrimitiveType get_primitive_type() const override { return PrimitiveType::TYPE_JSONB; }
doris::FieldType get_storage_field_type() const override {
return doris::FieldType::OLAP_FIELD_TYPE_JSONB;
}

int64_t get_uncompressed_serialized_bytes(const IColumn& column,
int be_exec_version) const override;
Expand Down
3 changes: 0 additions & 3 deletions be/src/core/data_type/data_type_map.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -56,9 +56,6 @@ class DataTypeMap final : public IDataType {

DataTypeMap(const DataTypePtr& key_type_, const DataTypePtr& value_type_);
PrimitiveType get_primitive_type() const override { return PrimitiveType::TYPE_MAP; }
doris::FieldType get_storage_field_type() const override {
return doris::FieldType::OLAP_FIELD_TYPE_MAP;
}

std::string do_get_name() const override {
return "Map(" + key_type->get_name() + ", " + value_type->get_name() + ")";
Expand Down
53 changes: 0 additions & 53 deletions be/src/core/data_type/data_type_number_base.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -66,59 +66,6 @@ class DataTypeNumberBase : public IDataType {
return T;
}

doris::FieldType get_storage_field_type() const override {
// Doris does not support uint8 at present, use uint8 as boolean type
if constexpr (T == TYPE_BOOLEAN) {
return doris::FieldType::OLAP_FIELD_TYPE_BOOL;
}
if constexpr (T == TYPE_TINYINT) {
return doris::FieldType::OLAP_FIELD_TYPE_TINYINT;
}
if constexpr (T == TYPE_SMALLINT) {
return doris::FieldType::OLAP_FIELD_TYPE_SMALLINT;
}
if constexpr (T == TYPE_INT) {
return doris::FieldType::OLAP_FIELD_TYPE_INT;
}
if constexpr (T == TYPE_BIGINT) {
return doris::FieldType::OLAP_FIELD_TYPE_BIGINT;
}
if constexpr (T == TYPE_LARGEINT) {
return doris::FieldType::OLAP_FIELD_TYPE_LARGEINT;
}
if constexpr (T == TYPE_FLOAT) {
return doris::FieldType::OLAP_FIELD_TYPE_FLOAT;
}
if constexpr (T == TYPE_DOUBLE) {
return doris::FieldType::OLAP_FIELD_TYPE_DOUBLE;
}
if constexpr (T == TYPE_DATE) {
return doris::FieldType::OLAP_FIELD_TYPE_DATE;
}
if constexpr (T == TYPE_DATETIME) {
return doris::FieldType::OLAP_FIELD_TYPE_DATETIME;
}
if constexpr (T == TYPE_DATEV2) {
return doris::FieldType::OLAP_FIELD_TYPE_DATEV2;
}
if constexpr (T == TYPE_DATETIMEV2) {
return doris::FieldType::OLAP_FIELD_TYPE_DATETIMEV2;
}
if constexpr (T == TYPE_TIMESTAMPTZ) {
return doris::FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ;
}
if constexpr (T == TYPE_IPV4) {
return doris::FieldType::OLAP_FIELD_TYPE_IPV4;
}
if constexpr (T == TYPE_IPV6) {
return doris::FieldType::OLAP_FIELD_TYPE_IPV6;
}
if constexpr (T == TYPE_TIMEV2) {
return doris::FieldType::OLAP_FIELD_TYPE_TIMEV2;
}
throw Exception(Status::FatalError("__builtin_unreachable"));
}

Field get_field(const TExprNode& node) const override;

int64_t get_uncompressed_serialized_bytes(const IColumn& column,
Expand Down
3 changes: 0 additions & 3 deletions be/src/core/data_type/data_type_quantilestate.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -51,9 +51,6 @@ class DataTypeQuantileState : public IDataType {
const std::string get_family_name() const override { return "QuantileState"; }
PrimitiveType get_primitive_type() const override { return PrimitiveType::TYPE_QUANTILE_STATE; }

doris::FieldType get_storage_field_type() const override {
return doris::FieldType::OLAP_FIELD_TYPE_QUANTILE_STATE;
}
int64_t get_uncompressed_serialized_bytes(const IColumn& column,
int be_exec_version) const override;
char* serialize(const IColumn& column, char* buf, int be_exec_version) const override;
Expand Down
4 changes: 0 additions & 4 deletions be/src/core/data_type/data_type_string.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -55,10 +55,6 @@ class DataTypeString : public IDataType {
}
PrimitiveType get_primitive_type() const override { return _primitive_type; }

doris::FieldType get_storage_field_type() const override {
return doris::FieldType::OLAP_FIELD_TYPE_STRING;
}

int64_t get_uncompressed_serialized_bytes(const IColumn& column,
int be_exec_version) const override;
char* serialize(const IColumn& column, char* buf, int be_exec_version) const override;
Expand Down
3 changes: 0 additions & 3 deletions be/src/core/data_type/data_type_struct.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -70,9 +70,6 @@ class DataTypeStruct final : public IDataType {
DataTypeStruct(const DataTypes& elems, const Strings& names);
PrimitiveType get_primitive_type() const override { return PrimitiveType::TYPE_STRUCT; }

doris::FieldType get_storage_field_type() const override {
return doris::FieldType::OLAP_FIELD_TYPE_STRUCT;
}
std::string do_get_name() const override;
const std::string get_family_name() const override { return "Struct"; }

Expand Down
3 changes: 0 additions & 3 deletions be/src/core/data_type/data_type_variant.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -60,9 +60,6 @@ class DataTypeVariant : public IDataType {
String do_get_name() const override { return name; }
const std::string get_family_name() const override { return "Variant"; }

doris::FieldType get_storage_field_type() const override {
return doris::FieldType::OLAP_FIELD_TYPE_VARIANT;
}
Status check_column(const IColumn& column) const override {
return check_column_non_nested_type<ColumnVariant>(column);
}
Expand Down
22 changes: 1 addition & 21 deletions be/src/storage/segment/segment_iterator.cpp
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -654,21 +654,16 @@ Status SegmentIterator::_lazy_init(Block* block) {
}
_current_return_columns.resize(_schema->columns().size());

_vec_init_char_column_id();
for (size_t i = 0; i < _schema->column_ids().size(); i++) {
ColumnId cid = _schema->column_ids()[i];
const auto* column_desc = _schema->column(cid);
if (_is_pred_column[cid]) {
auto storage_column_type = _storage_name_and_type[cid].second;
// Char type is special , since char type's computational datatype is same with string,
// both are DataTypeString, but DataTypeString only return FieldType::OLAP_FIELD_TYPE_STRING
// in get_storage_field_type.
RETURN_IF_CATCH_EXCEPTION(
// Here, cid will not go out of bounds
// because the size of _current_return_columns equals _schema->tablet_columns().size()
_current_return_columns[cid] = Schema::get_predicate_column_ptr(
_is_char_type[cid] ? FieldType::OLAP_FIELD_TYPE_CHAR
: storage_column_type->get_storage_field_type(),
storage_column_type->get_storage_field_type(),
storage_column_type->is_nullable(), _opts.io_ctx.reader_type));
_current_return_columns[cid]->set_rowset_segment_id(
{_segment->rowset_id(), _segment->id()});
Expand DownExpand Up@@ -2201,21 +2196,6 @@ bool SegmentIterator::_can_evaluated_by_vectorized(std::shared_ptr<ColumnPredica
}
}

void SegmentIterator::_vec_init_char_column_id() {
if (!_is_char_type.empty()) {
return;
}
_is_char_type.resize(_schema->columns().size(), false);
for (size_t i = 0; i < _schema->num_column_ids(); i++) {
auto cid = _schema->column_id(i);
const TabletColumn* column_desc = _schema->column(cid);

if (column_desc->type() == FieldType::OLAP_FIELD_TYPE_CHAR) {
_is_char_type[cid] = true;
}
}
}

bool SegmentIterator::_prune_column(ColumnId cid, MutableColumnPtr& column, bool fill_defaults,
size_t num_of_defaults) {
if (_need_read_data(cid)) {
Expand Down
3 changes: 0 additions & 3 deletions be/src/storage/segment/segment_iterator.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -204,7 +204,6 @@ class SegmentIterator : public RowwiseIterator {
bool _is_literal_node(const TExprNodeType::type& node_type);

Status _vec_init_lazy_materialization();
void _vec_init_char_column_id();

uint32_t segment_id() const { return _segment->id(); }
uint32_t num_rows() const { return _segment->num_rows(); }
Expand DownExpand Up@@ -427,8 +426,6 @@ class SegmentIterator : public RowwiseIterator {

io::FileReaderSPtr _file_reader;

std::vector<bool> _is_char_type;

// used for compaction, record selectd rowids of current batch
uint16_t _selected_size;
std::vector<uint16_t> _sel_rowid_idx;
Expand Down
Loading