Skip to content

[branch-4.1](be) backport storage type/encoding refactor series to branch-4.1 (#63153 #63233 #63622 #63291 #64341)#64550

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
yiguolei merged 5 commits into from
Jun 17, 2026
Merged

[branch-4.1](be) backport storage type/encoding refactor series to branch-4.1 (#63153 #63233 #63622 #63291 #64341) #64550

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
c4be0ec
[refactor](storage) replace TypeInfo polymorphic hierarchy with Field…
csun5285 May 13, 2026
ff4644b
[refactor](storage) drop StorageField wrapper and clean up related de…
csun5285 May 21, 2026
b3f7ac4
[refactor](BE) split EncodingInfo defaults into 4 explicit maps (#63622)
csun5285 May 29, 2026
9df0fac
[refactor](be) remove CHAR padding on read (#63291)
csun5285 Jun 1, 2026
68b95d1
[refactor](be) Derive get_storage_field_type from primitive type (#64…
csun5285 Jun 10, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 1 addition & 3 deletions be/benchmark/benchmark_zone_map_index.hpp
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -34,7 +34,6 @@

#include "core/data_type/data_type_factory.hpp"
#include "core/string_ref.h"
#include "storage/field.h"
#include "storage/index/zone_map/zone_map_index.h"
#include "storage/tablet/tablet_schema.h"
#include "util/slice.h"
Expand DownExpand Up@@ -116,9 +115,8 @@ std::unique_ptr<ZoneMapIndexWriter> make_writer() {
col = make_column(FieldType::OLAP_FIELD_TYPE_VARCHAR, 64, 1);
dtype = DataTypeFactory::instance().create_data_type(TYPE_VARCHAR, false, 0, 0, 64);
}
std::unique_ptr<StorageField> field(StorageFieldFactory::create(*col));
std::unique_ptr<ZoneMapIndexWriter> w;
(void)ZoneMapIndexWriter::create(dtype, field.get(), w);
(void)ZoneMapIndexWriter::create(dtype, col.get(), w);
return w;
}

Expand Down
12 changes: 12 additions & 0 deletions be/src/cloud/pb_convert.cpp
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -432,6 +432,9 @@ void doris_tablet_schema_to_cloud(TabletSchemaCloudPB* out, const TabletSchemaPB
if (in.has_binary_plain_encoding_default_impl()) {
out->set_binary_plain_encoding_default_impl(in.binary_plain_encoding_default_impl());
}
if (in.has_storage_format()) {
out->set_storage_format(in.storage_format());
}

if (in.has___split_schema()) {
out->mutable___split_schema()->CopyFrom(in.__split_schema());
Expand DownExpand Up@@ -475,6 +478,9 @@ void doris_tablet_schema_to_cloud(TabletSchemaCloudPB* out, TabletSchemaPB&& in)
if (in.has_binary_plain_encoding_default_impl()) {
out->set_binary_plain_encoding_default_impl(in.binary_plain_encoding_default_impl());
}
if (in.has_storage_format()) {
out->set_storage_format(in.storage_format());
}

if (in.has___split_schema()) {
out->mutable___split_schema()->CopyFrom(in.__split_schema());
Expand DownExpand Up@@ -531,6 +537,9 @@ void cloud_tablet_schema_to_doris(TabletSchemaPB* out, const TabletSchemaCloudPB
if (in.has_binary_plain_encoding_default_impl()) {
out->set_binary_plain_encoding_default_impl(in.binary_plain_encoding_default_impl());
}
if (in.has_storage_format()) {
out->set_storage_format(in.storage_format());
}

if (in.has___split_schema()) {
out->mutable___split_schema()->CopyFrom(in.__split_schema());
Expand DownExpand Up@@ -575,6 +584,9 @@ void cloud_tablet_schema_to_doris(TabletSchemaPB* out, TabletSchemaCloudPB&& in)
if (in.has_binary_plain_encoding_default_impl()) {
out->set_binary_plain_encoding_default_impl(in.binary_plain_encoding_default_impl());
}
if (in.has_storage_format()) {
out->set_storage_format(in.storage_format());
}

if (in.has___split_schema()) {
out->mutable___split_schema()->CopyFrom(in.__split_schema());
Expand Down
9 changes: 0 additions & 9 deletions be/src/core/block/block.cpp
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -1068,15 +1068,6 @@ std::unique_ptr<Block> Block::create_same_struct_block(size_t size, bool is_rese
return temp_block;
}

void Block::shrink_char_type_column_suffix_zero(const std::vector<size_t>& char_type_idx) {
for (auto idx : char_type_idx) {
if (idx < data.size()) {
auto& col_and_name = this->get_by_position(idx);
col_and_name.column->assume_mutable()->shrink_padding_chars();
}
}
}

size_t MutableBlock::allocated_bytes() const {
size_t res = 0;
for (const auto& col : _columns) {
Expand Down
3 changes: 0 additions & 3 deletions be/src/core/block/block.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -349,9 +349,6 @@ class Block {
return res;
}

// for String type or Array<String> type
void shrink_char_type_column_suffix_zero(const std::vector<size_t>& char_type_idx);

void clear_column_mem_not_keep(const std::vector<bool>& column_keep_flags,
bool need_keep_first);

Expand Down
4 changes: 0 additions & 4 deletions be/src/core/column/column.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -113,10 +113,6 @@ class IColumn : public COW<IColumn> {
return nullptr;
}

// shrink the end zeros for ColumnStr(also for who has it nested). so nest column will call it for all nested.
// for non-str col, will reach here(do nothing). only ColumnStr will really shrink itself.
virtual void shrink_padding_chars() {}

// Only used in ColumnVariant to handle lifecycle of variant. Other columns would do nothing.
virtual void finalize() {}

Expand Down
4 changes: 0 additions & 4 deletions be/src/core/column/column_array.cpp
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -98,10 +98,6 @@ ColumnArray::ColumnArray(MutableColumnPtr&& nested_column) : data(std::move(nest
offsets = ColumnOffsets::create();
}

void ColumnArray::shrink_padding_chars() {
data->shrink_padding_chars();
}

std::string ColumnArray::get_name() const {
return "Array(" + get_data().get_name() + ")";
}
Expand Down
2 changes: 0 additions & 2 deletions be/src/core/column/column_array.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -117,8 +117,6 @@ class ColumnArray final : public COWHelper<IColumn, ColumnArray> {
offsets->sanity_check();
}

void shrink_padding_chars() override;

/** On the index i there is an offset to the beginning of the i + 1 -th element. */
using ColumnOffsets = ColumnOffset64;

Expand Down
27 changes: 3 additions & 24 deletions be/src/core/column/column_dictionary.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -232,7 +232,7 @@ class ColumnDictI32 final : public COWHelper<IColumn, ColumnDictI32> {
_dict.initialize_hash_values_for_runtime_filter();
}

uint32_t get_hash_value(uint32_t idx) const { return _dict.get_hash_value(_codes[idx], _type); }
uint32_t get_hash_value(uint32_t idx) const { return _dict.get_hash_value(_codes[idx]); }

template <typename HybridSetType>
void find_codes(const HybridSetType* values, std::vector<UInt8>& selected) const {
Expand DownExpand Up@@ -279,14 +279,6 @@ class ColumnDictI32 final : public COWHelper<IColumn, ColumnDictI32> {

inline const StringRef& get_value(value_type code) const { return _dict.get_value(code); }

inline StringRef get_shrink_value(value_type code) const {
StringRef result = _dict.get_value(code);
if (_type == FieldType::OLAP_FIELD_TYPE_CHAR) {
result.size = strnlen(result.data, result.size);
}
return result;
}

size_t dict_size() const { return _dict.size(); }

std::string dict_debug_string() const { return _dict.debug_string(); }
Expand DownExpand Up@@ -327,26 +319,13 @@ class ColumnDictI32 final : public COWHelper<IColumn, ColumnDictI32> {
}
}

inline uint32_t get_hash_value(Int32 code, FieldType type) const {
inline uint32_t get_hash_value(Int32 code) const {
if (_compute_hash_value_flags[code]) {
return _hash_values[code];
} else {
auto& sv = (*_dict_data)[code];
// The char data is stored in the disk with the schema length,
// and zeros are filled if the length is insufficient

// When reading data, use shrink_char_type_column_suffix_zero(_char_type_idx)
// Remove the suffix 0
// When writing data, use the CharField::consume function to fill in the trailing 0.

// For dictionary data of char type, sv.size is the schema length,
// so use strnlen to remove the 0 at the end to get the actual length.
size_t len = sv.size;
if (type == FieldType::OLAP_FIELD_TYPE_CHAR) {
len = strnlen(sv.data, sv.size);
}
uint32_t hash_val =
crc32c::Extend(0, (const uint8_t*)sv.data, static_cast<uint32_t>(len));
crc32c::Extend(0, (const uint8_t*)sv.data, static_cast<uint32_t>(sv.size));
_hash_values[code] = hash_val;
_compute_hash_value_flags[code] = 1;
return _hash_values[code];
Expand Down
5 changes: 0 additions & 5 deletions be/src/core/column/column_map.cpp
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -643,11 +643,6 @@ Status ColumnMap::deduplicate_keys(bool recursive) {
return Status::OK();
}

void ColumnMap::shrink_padding_chars() {
keys_column->shrink_padding_chars();
values_column->shrink_padding_chars();
}

void ColumnMap::reserve(size_t n) {
get_offsets().reserve(n);
keys_column->reserve(n);
Expand Down
1 change: 0 additions & 1 deletion be/src/core/column/column_map.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -115,7 +115,6 @@ class ColumnMap final : public COWHelper<IColumn, ColumnMap> {
const char* deserialize_and_insert_from_arena(const char* pos) override;

void update_hash_with_value(size_t n, SipHash& hash) const override;
void shrink_padding_chars() override;
ColumnPtr filter(const Filter& filt, ssize_t result_size_hint) const override;
size_t filter(const Filter& filter) override;
MutableColumnPtr permute(const Permutation& perm, size_t limit) const override;
Expand Down
4 changes: 0 additions & 4 deletions be/src/core/column/column_nullable.cpp
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -48,10 +48,6 @@ ColumnNullable::ColumnNullable(MutableColumnPtr&& nested_column_, MutableColumnP
}
}

void ColumnNullable::shrink_padding_chars() {
get_nested_column_ptr()->shrink_padding_chars();
}

void ColumnNullable::update_xxHash_with_value(size_t start, size_t end, uint64_t& hash,
const uint8_t* __restrict null_data) const {
if (!has_null(start, end)) {
Expand Down
2 changes: 0 additions & 2 deletions be/src/core/column/column_nullable.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -86,8 +86,6 @@ class ColumnNullable final : public COWHelper<IColumn, ColumnNullable> {
_nested_column->sanity_check();
}

void shrink_padding_chars() override;

bool is_variable_length() const override { return _nested_column->is_variable_length(); }

std::string get_name() const override { return "Nullable(" + _nested_column->get_name() + ")"; }
Expand Down
23 changes: 0 additions & 23 deletions be/src/core/column/column_string.cpp
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -81,29 +81,6 @@ MutableColumnPtr ColumnStr<T>::clone_resized(size_t to_size) const {
return res;
}

template <typename T>
void ColumnStr<T>::shrink_padding_chars() {
if (size() == 0) {
return;
}
char* data = reinterpret_cast<char*>(chars.data());
auto* offset = offsets.data();
size_t size = offsets.size();

// deal the 0-th element. no need to move.
auto next_start = offset[0];
offset[0] = static_cast<T>(strnlen(data, size_at(0)));
for (size_t i = 1; i < size; i++) {
// get the i-th length and whole move it to cover the last's trailing void
auto length = strnlen(data + next_start, offset[i] - next_start);
memmove(data + offset[i - 1], data + next_start, length);
// offset i will be changed. so save the old value for (i+1)-th to get its length.
next_start = offset[i];
offset[i] = offset[i - 1] + static_cast<T>(length);
}
chars.resize_fill(offsets.back()); // just call it to shrink memory here. no possible to expand.
}

// This method is only called by MutableBlock::merge_ignore_overflow
// by hash join operator to collect build data to avoid
// the total string length of a ColumnStr<uint32_t> column exceeds the 4G limit.
Expand Down
2 changes: 0 additions & 2 deletions be/src/core/column/column_string.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -143,8 +143,6 @@ class ColumnStr final : public COWHelper<IColumn, ColumnStr<T>> {

MutableColumnPtr clone_resized(size_t to_size) const override;

void shrink_padding_chars() override;

Field operator[](size_t n) const override;

void get(size_t n, Field& res) const override;
Expand Down
6 changes: 0 additions & 6 deletions be/src/core/column/column_struct.cpp
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -338,12 +338,6 @@ MutableColumnPtr ColumnStruct::permute(const Permutation& perm, size_t limit) co
return ColumnStruct::create(new_columns);
}

void ColumnStruct::shrink_padding_chars() {
for (auto& column : columns) {
column->shrink_padding_chars();
}
}

void ColumnStruct::reserve(size_t n) {
const size_t tuple_size = columns.size();
for (size_t i = 0; i < tuple_size; ++i) {
Expand Down
2 changes: 0 additions & 2 deletions be/src/core/column/column_struct.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -150,8 +150,6 @@ class ColumnStruct final : public COWHelper<IColumn, ColumnStruct> {

int compare_at(size_t n, size_t m, const IColumn& rhs_, int nan_direction_hint) const override;

void shrink_padding_chars() override;

void reserve(size_t n) override;
void resize(size_t n) override;
size_t byte_size() const override;
Expand Down
6 changes: 1 addition & 5 deletions be/src/core/column/predicate_column.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -105,11 +105,7 @@ class PredicateColumnType final : public COWHelper<IColumn, PredicateColumnType<

StringRef get_data_at(size_t n) const override {
if constexpr (std::is_same_v<T, StringRef>) {
auto res = reinterpret_cast<const StringRef&>(data[n]);
if constexpr (Type == TYPE_CHAR) {
res.size = strnlen(res.data, res.size);
}
return res;
return reinterpret_cast<const StringRef&>(data[n]);
} else {
throw doris::Exception(
ErrorCode::INTERNAL_ERROR,
Expand Down
5 changes: 5 additions & 0 deletions be/src/core/data_type/data_type.cpp
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -34,6 +34,7 @@
#include "core/data_type/define_primitive_type.h"
#include "core/data_type_serde/data_type_serde.h"
#include "core/field.h"
#include "storage/tablet/tablet_schema.h"

namespace doris {
class BufferWritable;
Expand All@@ -46,6 +47,10 @@ IDataType::IDataType() = default;

IDataType::~IDataType() = default;

doris::FieldType IDataType::get_storage_field_type() const {
return TabletColumn::get_field_type_by_type(get_primitive_type());
}

String IDataType::get_name() const {
return do_get_name();
}
Expand Down
4 changes: 3 additions & 1 deletion be/src/core/data_type/data_type.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -84,7 +84,9 @@ class IDataType : private boost::noncopyable {
virtual const std::string get_family_name() const = 0;
virtual PrimitiveType get_primitive_type() const = 0;

virtual doris::FieldType get_storage_field_type() const = 0;
// Derived from the primitive type by default (e.g. TYPE_CHAR -> OLAP_FIELD_TYPE_CHAR).
// Types without a direct 1:1 mapping override this.
virtual doris::FieldType get_storage_field_type() const;
std::string to_string(const IColumn& column, size_t row_num,
const DataTypeSerDe::FormatOptions& options) const;
// get specific serializer or deserializer
Expand Down
4 changes: 0 additions & 4 deletions be/src/core/data_type/data_type_agg_state.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -85,10 +85,6 @@ class DataTypeAggState : public DataTypeString {

PrimitiveType get_primitive_type() const override { return PrimitiveType::TYPE_AGG_STATE; }

doris::FieldType get_storage_field_type() const override {
return doris::FieldType::OLAP_FIELD_TYPE_AGG_STATE;
}

const DataTypes& get_sub_types() const { return _sub_types; }

void to_pb_column_meta(PColumnMeta* col_meta) const override {
Expand Down
4 changes: 0 additions & 4 deletions be/src/core/data_type/data_type_array.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -57,10 +57,6 @@ class DataTypeArray final : public IDataType {

PrimitiveType get_primitive_type() const override { return PrimitiveType::TYPE_ARRAY; }

doris::FieldType get_storage_field_type() const override {
return doris::FieldType::OLAP_FIELD_TYPE_ARRAY;
}

std::string do_get_name() const override { return "Array(" + nested->get_name() + ")"; }

const std::string get_family_name() const override { return "Array"; }
Expand Down
4 changes: 0 additions & 4 deletions be/src/core/data_type/data_type_bitmap.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -54,10 +54,6 @@ class DataTypeBitMap : public IDataType {
const std::string get_family_name() const override { return "BitMap"; }
PrimitiveType get_primitive_type() const override { return PrimitiveType::TYPE_BITMAP; }

doris::FieldType get_storage_field_type() const override {
return doris::FieldType::OLAP_FIELD_TYPE_BITMAP;
}

int64_t get_uncompressed_serialized_bytes(const IColumn& column,
int be_exec_version) const override;
char* serialize(const IColumn& column, char* buf, int be_exec_version) const override;
Expand Down
3 changes: 0 additions & 3 deletions be/src/core/data_type/data_type_date.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -43,9 +43,6 @@ class DataTypeDate final : public DataTypeNumberBase<PrimitiveType::TYPE_DATE> {
static constexpr PrimitiveType PType = TYPE_DATE;
PrimitiveType get_primitive_type() const override { return PrimitiveType::TYPE_DATE; }

doris::FieldType get_storage_field_type() const override {
return doris::FieldType::OLAP_FIELD_TYPE_DATE;
}
const std::string get_family_name() const override { return "Date"; }
std::string do_get_name() const override { return "Date"; }

Expand Down
6 changes: 0 additions & 6 deletions be/src/core/data_type/data_type_date_or_datetime_v2.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -55,9 +55,6 @@ class DataTypeDateV2 final : public DataTypeNumberBase<PrimitiveType::TYPE_DATEV
static constexpr PrimitiveType PType = TYPE_DATEV2;
PrimitiveType get_primitive_type() const override { return PrimitiveType::TYPE_DATEV2; }

doris::FieldType get_storage_field_type() const override {
return doris::FieldType::OLAP_FIELD_TYPE_DATEV2;
}
const std::string get_family_name() const override { return "DateV2"; }
std::string do_get_name() const override { return "DateV2"; }

Expand DownExpand Up@@ -112,9 +109,6 @@ class DataTypeDateTimeV2 final : public DataTypeNumberBase<PrimitiveType::TYPE_D
scalar_type->set_scale(_scale);
}

doris::FieldType get_storage_field_type() const override {
return doris::FieldType::OLAP_FIELD_TYPE_DATETIMEV2;
}
const std::string get_family_name() const override { return "DateTimeV2"; }
std::string do_get_name() const override {
return "DateTimeV2(" + std::to_string(_scale) + ")";
Expand Down
4 changes: 0 additions & 4 deletions be/src/core/data_type/data_type_date_time.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -71,10 +71,6 @@ class DataTypeDateTime final : public DataTypeNumberBase<PrimitiveType::TYPE_DAT
std::string do_get_name() const override { return "DateTime"; }
PrimitiveType get_primitive_type() const override { return PrimitiveType::TYPE_DATETIME; }

doris::FieldType get_storage_field_type() const override {
return doris::FieldType::OLAP_FIELD_TYPE_DATETIME;
}

bool equals(const IDataType& rhs) const override;
#ifdef BE_TEST
/// TODO: remove this in the future
Expand Down
Loading
Loading