Skip to content

[Feature](encoding) Add float_type_default_use_plain_encoding table property#62085

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
bobhan1 wants to merge 2 commits into from
Closed

[Feature](encoding) Add float_type_default_use_plain_encoding table property #62085

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
fe5ebb6
Add float_type_default_use_plain_encoding table property
bobhan1 Apr 3, 2026
dfcec16
Only show float_type_default_use_plain_encoding in SHOW CREATE TABLE …
bobhan1 Apr 3, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions be/src/cloud/pb_convert.cpp
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -410,6 +410,9 @@ void doris_tablet_schema_to_cloud(TabletSchemaCloudPB* out, const TabletSchemaPB
out->set_integer_type_default_use_plain_encoding(
in.integer_type_default_use_plain_encoding());
}
if (in.has_float_type_default_use_plain_encoding()) {
out->set_float_type_default_use_plain_encoding(in.float_type_default_use_plain_encoding());
}
if (in.has_binary_plain_encoding_default_impl()) {
out->set_binary_plain_encoding_default_impl(in.binary_plain_encoding_default_impl());
}
Expand DownExpand Up@@ -457,6 +460,9 @@ void doris_tablet_schema_to_cloud(TabletSchemaCloudPB* out, TabletSchemaPB&& in)
out->set_integer_type_default_use_plain_encoding(
in.integer_type_default_use_plain_encoding());
}
if (in.has_float_type_default_use_plain_encoding()) {
out->set_float_type_default_use_plain_encoding(in.float_type_default_use_plain_encoding());
}
if (in.has_binary_plain_encoding_default_impl()) {
out->set_binary_plain_encoding_default_impl(in.binary_plain_encoding_default_impl());
}
Expand DownExpand Up@@ -517,6 +523,9 @@ void cloud_tablet_schema_to_doris(TabletSchemaPB* out, const TabletSchemaCloudPB
out->set_integer_type_default_use_plain_encoding(
in.integer_type_default_use_plain_encoding());
}
if (in.has_float_type_default_use_plain_encoding()) {
out->set_float_type_default_use_plain_encoding(in.float_type_default_use_plain_encoding());
}
if (in.has_binary_plain_encoding_default_impl()) {
out->set_binary_plain_encoding_default_impl(in.binary_plain_encoding_default_impl());
}
Expand DownExpand Up@@ -565,6 +574,9 @@ void cloud_tablet_schema_to_doris(TabletSchemaPB* out, TabletSchemaCloudPB&& in)
out->set_integer_type_default_use_plain_encoding(
in.integer_type_default_use_plain_encoding());
}
if (in.has_float_type_default_use_plain_encoding()) {
out->set_float_type_default_use_plain_encoding(in.float_type_default_use_plain_encoding());
}
if (in.has_binary_plain_encoding_default_impl()) {
out->set_binary_plain_encoding_default_impl(in.binary_plain_encoding_default_impl());
}
Expand Down
15 changes: 15 additions & 0 deletions be/src/storage/segment/encoding_info.cpp
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -348,6 +348,10 @@ bool is_integer_type(FieldType type) {
type == FieldType::OLAP_FIELD_TYPE_BIGINT || type == FieldType::OLAP_FIELD_TYPE_LARGEINT;
}

bool is_float_type(FieldType type) {
return type == FieldType::OLAP_FIELD_TYPE_FLOAT || type == FieldType::OLAP_FIELD_TYPE_DOUBLE;
}

bool is_binary_type(FieldType type) {
return type == FieldType::OLAP_FIELD_TYPE_CHAR || type == FieldType::OLAP_FIELD_TYPE_VARCHAR ||
type == FieldType::OLAP_FIELD_TYPE_STRING || type == FieldType::OLAP_FIELD_TYPE_JSONB ||
Expand DownExpand Up@@ -396,6 +400,17 @@ EncodingTypePB EncodingInfoResolver::get_default_encoding(FieldType type,
return is_integer_type(type) &&
encoding_preference.integer_type_default_use_plain_encoding;
},
.target_encoding = PLAIN_ENCODING},

// Hook 3: Float types - any encoding -> PLAIN_ENCODING
// Applies when: type is float/double and config enables plain encoding for floats
EncodingTransform {
.predicate =
[](FieldType type, EncodingTypePB encoding,
EncodingPreference encoding_preference, bool optimize_value_seek) {
return is_float_type(type) &&
encoding_preference.float_type_default_use_plain_encoding;
},
.target_encoding = PLAIN_ENCODING}};

auto& encoding_map =
Expand Down
1 change: 1 addition & 0 deletions be/src/storage/segment/options.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -31,6 +31,7 @@ constexpr long ROW_STORE_PAGE_SIZE_DEFAULT_VALUE = 16384; // default row store p

struct EncodingPreference {
bool integer_type_default_use_plain_encoding {false};
bool float_type_default_use_plain_encoding {false};
BinaryPlainEncodingTypePB binary_plain_encoding_default_impl {
BinaryPlainEncodingTypePB::BINARY_PLAIN_ENCODING_V1};
};
Expand Down
2 changes: 2 additions & 0 deletions be/src/storage/segment/segment_writer.cpp
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -307,6 +307,8 @@ Status SegmentWriter::_create_column_writer(uint32_t cid, const TabletColumn& co
}
opts.encoding_preference = {.integer_type_default_use_plain_encoding =
_tablet_schema->integer_type_default_use_plain_encoding(),
.float_type_default_use_plain_encoding =
_tablet_schema->float_type_default_use_plain_encoding(),
.binary_plain_encoding_default_impl =
_tablet_schema->binary_plain_encoding_default_impl()};

Expand Down
2 changes: 2 additions & 0 deletions be/src/storage/segment/vertical_segment_writer.cpp
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -306,6 +306,8 @@ Status VerticalSegmentWriter::_create_column_writer(uint32_t cid, const TabletCo

opts.encoding_preference = {.integer_type_default_use_plain_encoding =
_tablet_schema->integer_type_default_use_plain_encoding(),
.float_type_default_use_plain_encoding =
_tablet_schema->float_type_default_use_plain_encoding(),
.binary_plain_encoding_default_impl =
_tablet_schema->binary_plain_encoding_default_impl()};
std::unique_ptr<ColumnWriter> writer;
Expand Down
4 changes: 4 additions & 0 deletions be/src/storage/tablet/tablet_meta.cpp
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -389,6 +389,10 @@ TabletMeta::TabletMeta(int64_t table_id, int64_t partition_id, int64_t tablet_id
if (tablet_schema.__isset.storage_dict_page_size) {
schema->set_storage_dict_page_size(tablet_schema.storage_dict_page_size);
}
if (tablet_schema.__isset.float_type_default_use_plain_encoding) {
schema->set_float_type_default_use_plain_encoding(
tablet_schema.float_type_default_use_plain_encoding);
}
if (tablet_schema.__isset.skip_write_index_on_load) {
schema->set_skip_write_index_on_load(tablet_schema.skip_write_index_on_load);
}
Expand Down
7 changes: 7 additions & 0 deletions be/src/storage/tablet/tablet_schema.cpp
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -1300,6 +1300,9 @@ void TabletSchema::init_from_pb(const TabletSchemaPB& schema, bool ignore_extrac
if (schema.has_integer_type_default_use_plain_encoding()) {
_integer_type_default_use_plain_encoding = schema.integer_type_default_use_plain_encoding();
}
if (schema.has_float_type_default_use_plain_encoding()) {
_float_type_default_use_plain_encoding = schema.float_type_default_use_plain_encoding();
}
if (schema.has_binary_plain_encoding_default_impl()) {
_binary_plain_encoding_default_impl = schema.binary_plain_encoding_default_impl();
}
Expand DownExpand Up@@ -1578,6 +1581,8 @@ void TabletSchema::to_schema_pb(TabletSchemaPB* tablet_schema_pb) const {
_is_external_segment_column_meta_used);
tablet_schema_pb->set_integer_type_default_use_plain_encoding(
_integer_type_default_use_plain_encoding);
tablet_schema_pb->set_float_type_default_use_plain_encoding(
_float_type_default_use_plain_encoding);
tablet_schema_pb->set_binary_plain_encoding_default_impl(_binary_plain_encoding_default_impl);
auto column_groups_pb = tablet_schema_pb->mutable_seq_map();
for (const auto& it : _seq_col_uid_to_value_cols_uid) {
Expand DownExpand Up@@ -1975,6 +1980,8 @@ bool operator==(const TabletSchema& a, const TabletSchema& b) {
return false;
if (a._integer_type_default_use_plain_encoding != b._integer_type_default_use_plain_encoding)
return false;
if (a._float_type_default_use_plain_encoding != b._float_type_default_use_plain_encoding)
return false;
if (a._binary_plain_encoding_default_impl != b._binary_plain_encoding_default_impl)
return false;
return true;
Expand Down
9 changes: 9 additions & 0 deletions be/src/storage/tablet/tablet_schema.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -752,6 +752,14 @@ class TabletSchema : public MetadataAdder<TabletSchema> {
_integer_type_default_use_plain_encoding = v;
}

bool float_type_default_use_plain_encoding() const {
return _float_type_default_use_plain_encoding;
}

void set_float_type_default_use_plain_encoding(bool v) {
_float_type_default_use_plain_encoding = v;
}

BinaryPlainEncodingTypePB binary_plain_encoding_default_impl() const {
return _binary_plain_encoding_default_impl;
}
Expand DownExpand Up@@ -843,6 +851,7 @@ class TabletSchema : public MetadataAdder<TabletSchema> {
bool _is_external_segment_column_meta_used = false;

bool _integer_type_default_use_plain_encoding {false};
bool _float_type_default_use_plain_encoding {false};
BinaryPlainEncodingTypePB _binary_plain_encoding_default_impl {
BinaryPlainEncodingTypePB::BINARY_PLAIN_ENCODING_V1};
// Sequence column unique id mapping to value columns unique id
Expand Down
37 changes: 37 additions & 0 deletions be/test/storage/segment/encoding_info_test.cpp
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -151,6 +151,43 @@ TEST_F(EncodingInfoTest, test_use_plain_binary_v2_config) {
EXPECT_EQ(PLAIN_ENCODING, encoding_type); // Should still be PLAIN_ENCODING
}

// Test encoding preference for float types
TEST_F(EncodingInfoTest, test_float_type_plain_encoding_preference) {
// Test FLOAT type
const auto* float_type_info = get_scalar_type_info<FieldType::OLAP_FIELD_TYPE_FLOAT>();

// Default: float_type_default_use_plain_encoding = false -> BIT_SHUFFLE
EncodingPreference pref_default;
pref_default.float_type_default_use_plain_encoding = false;
EncodingTypePB encoding_type =
EncodingInfo::get_default_encoding(float_type_info->type(), pref_default, false);
EXPECT_EQ(BIT_SHUFFLE, encoding_type);

// Enabled: float_type_default_use_plain_encoding = true -> PLAIN_ENCODING
EncodingPreference pref_enabled;
pref_enabled.float_type_default_use_plain_encoding = true;
encoding_type =
EncodingInfo::get_default_encoding(float_type_info->type(), pref_enabled, false);
EXPECT_EQ(PLAIN_ENCODING, encoding_type);

// Test DOUBLE type
const auto* double_type_info = get_scalar_type_info<FieldType::OLAP_FIELD_TYPE_DOUBLE>();

encoding_type =
EncodingInfo::get_default_encoding(double_type_info->type(), pref_default, false);
EXPECT_EQ(BIT_SHUFFLE, encoding_type);

encoding_type =
EncodingInfo::get_default_encoding(double_type_info->type(), pref_enabled, false);
EXPECT_EQ(PLAIN_ENCODING, encoding_type);

// Verify float preference doesn't affect integer types
const auto* bigint_type_info = get_scalar_type_info<FieldType::OLAP_FIELD_TYPE_BIGINT>();
encoding_type =
EncodingInfo::get_default_encoding(bigint_type_info->type(), pref_enabled, false);
EXPECT_EQ(BIT_SHUFFLE, encoding_type); // Should still be BIT_SHUFFLE
}

// Comprehensive test for _data_page_pre_decoder for all encoding types
TEST_F(EncodingInfoTest, test_all_pre_decoders) {
EncodingPreference encoding_preference;
Expand Down
1 change: 1 addition & 0 deletions fe/fe-core/src/main/java/org/apache/doris/alter/CloudRollupJobV2.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -240,6 +240,7 @@ private void createRollupReplicaForPartition(OlapTable tbl) throws Exception {
tbl.getTimeSeriesCompactionEmptyRowsetsThreshold(),
tbl.getTimeSeriesCompactionLevelThreshold(),
tbl.disableAutoCompaction(),
tbl.floatTypeDefaultUsePlainEncoding(),
tbl.getRowStoreColumnsUniqueIds(rowStoreColumns),
null,
tbl.rowStorePageSize(),
Expand Down
1 change: 1 addition & 0 deletions fe/fe-core/src/main/java/org/apache/doris/alter/CloudSchemaChangeJobV2.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -262,6 +262,7 @@ private void createShadowIndexReplicaForPartition(OlapTable tbl) throws Exceptio
tbl.getTimeSeriesCompactionEmptyRowsetsThreshold(),
tbl.getTimeSeriesCompactionLevelThreshold(),
tbl.disableAutoCompaction(),
tbl.floatTypeDefaultUsePlainEncoding(),
tbl.getRowStoreColumnsUniqueIds(rowStoreColumns),
tbl.getInvertedIndexFileStorageFormat(),
tbl.rowStorePageSize(),
Expand Down
1 change: 1 addition & 0 deletions fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -257,6 +257,7 @@ protected void createRollupReplica() throws AlterCancelException {
tbl.getCompressionType(),
tbl.getEnableUniqueKeyMergeOnWrite(), tbl.getStoragePolicy(),
tbl.disableAutoCompaction(),
tbl.floatTypeDefaultUsePlainEncoding(),
tbl.enableSingleReplicaCompaction(),
tbl.skipWriteIndexOnLoad(),
tbl.getCompactionPolicy(),
Expand Down
1 change: 1 addition & 0 deletions fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -319,6 +319,7 @@ protected void createShadowIndexReplica() throws AlterCancelException {
tbl.getCompressionType(),
tbl.getEnableUniqueKeyMergeOnWrite(), tbl.getStoragePolicy(),
tbl.disableAutoCompaction(),
tbl.floatTypeDefaultUsePlainEncoding(),
tbl.enableSingleReplicaCompaction(),
tbl.skipWriteIndexOnLoad(),
tbl.getCompactionPolicy(),
Expand Down
1 change: 1 addition & 0 deletions fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -1440,6 +1440,7 @@ protected void createReplicas(Database db, OlapTable localTbl, Partition restore
localTbl.getCompressionType(),
localTbl.getEnableUniqueKeyMergeOnWrite(), localTbl.getStoragePolicy(),
localTbl.disableAutoCompaction(),
localTbl.floatTypeDefaultUsePlainEncoding(),
localTbl.enableSingleReplicaCompaction(),
localTbl.skipWriteIndexOnLoad(),
localTbl.getCompactionPolicy(),
Expand Down
7 changes: 7 additions & 0 deletions fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -4023,6 +4023,13 @@ private static void addOlapTablePropertyInfo(OlapTable olapTable, StringBuilder
sb.append(",\n\"").append(PropertyAnalyzer.PROPERTIES_DISABLE_AUTO_COMPACTION).append("\" = \"");
sb.append(olapTable.disableAutoCompaction()).append("\"");

// float type default use plain encoding
if (olapTable.floatTypeDefaultUsePlainEncoding()) {
sb.append(",\n\"").append(PropertyAnalyzer.PROPERTIES_FLOAT_TYPE_DEFAULT_USE_PLAIN_ENCODING)
.append("\" = \"");
sb.append(olapTable.floatTypeDefaultUsePlainEncoding()).append("\"");
}

if (olapTable.variantEnableFlattenNested()) {
// enable flatten nested type in variant
sb.append(",\n\"").append(PropertyAnalyzer.PROPERTIES_VARIANT_ENABLE_FLATTEN_NESTED).append("\" = \"");
Expand Down
15 changes: 15 additions & 0 deletions fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -2618,6 +2618,21 @@ public Boolean disableAutoCompaction() {
return false;
}

public void setFloatTypeDefaultUsePlainEncoding(boolean floatTypeDefaultUsePlainEncoding) {
TableProperty tableProperty = getOrCreatTableProperty();
tableProperty.modifyTableProperties(
PropertyAnalyzer.PROPERTIES_FLOAT_TYPE_DEFAULT_USE_PLAIN_ENCODING,
Boolean.valueOf(floatTypeDefaultUsePlainEncoding).toString());
tableProperty.buildFloatTypeDefaultUsePlainEncoding();
}

public Boolean floatTypeDefaultUsePlainEncoding() {
if (tableProperty != null) {
return tableProperty.floatTypeDefaultUsePlainEncoding();
}
return false;
}

@Deprecated
public void setVariantEnableFlattenNested(boolean flattenNested) throws AnalysisException {
TableProperty tableProperty = getOrCreatTableProperty();
Expand Down
15 changes: 15 additions & 0 deletions fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -94,6 +94,8 @@ public class TableProperty implements GsonPostProcessable {

private boolean disableAutoCompaction = false;

private boolean floatTypeDefaultUsePlainEncoding = false;

private boolean variantEnableFlattenNested = false;

private boolean enableSingleReplicaCompaction = false;
Expand DownExpand Up@@ -171,6 +173,7 @@ public TableProperty buildProperty(short opCode) {
buildEnableSingleReplicaCompaction();
buildVerticalCompactionNumColumnsPerGroup();
buildDisableAutoCompaction();
buildFloatTypeDefaultUsePlainEncoding();
buildTimeSeriesCompactionEmptyRowsetsThreshold();
buildTimeSeriesCompactionLevelThreshold();
buildTTLSeconds();
Expand DownExpand Up@@ -295,6 +298,17 @@ public TableProperty buildDisableAutoCompaction() {
return this;
}

public TableProperty buildFloatTypeDefaultUsePlainEncoding() {
floatTypeDefaultUsePlainEncoding = Boolean.parseBoolean(
properties.getOrDefault(
PropertyAnalyzer.PROPERTIES_FLOAT_TYPE_DEFAULT_USE_PLAIN_ENCODING, "false"));
return this;
}

public boolean floatTypeDefaultUsePlainEncoding() {
return floatTypeDefaultUsePlainEncoding;
}

public TableProperty buildAutoAnalyzeProperty() {
autoAnalyzePolicy = properties.getOrDefault(PropertyAnalyzer.PROPERTIES_AUTO_ANALYZE_POLICY,
PropertyAnalyzer.ENABLE_AUTO_ANALYZE_POLICY);
Expand DownExpand Up@@ -917,6 +931,7 @@ public void gsonPostProcess() throws IOException {
buildTimeSeriesCompactionFileCountThreshold();
buildTimeSeriesCompactionTimeThresholdSeconds();
buildDisableAutoCompaction();
buildFloatTypeDefaultUsePlainEncoding();
buildEnableSingleReplicaCompaction();
buildVerticalCompactionNumColumnsPerGroup();
buildTimeSeriesCompactionEmptyRowsetsThreshold();
Expand Down
1 change: 1 addition & 0 deletions fe/fe-core/src/main/java/org/apache/doris/cloud/backup/CloudRestoreJob.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -398,6 +398,7 @@ public void createReplicas(Database db, OlapTable localTbl, Partition restorePar
localTbl.getTimeSeriesCompactionTimeThresholdSeconds(),
localTbl.getTimeSeriesCompactionEmptyRowsetsThreshold(),
localTbl.getTimeSeriesCompactionLevelThreshold(), localTbl.disableAutoCompaction(),
localTbl.floatTypeDefaultUsePlainEncoding(),
localTbl.getRowStoreColumnsUniqueIds(rowStoreColumns),
localTbl.getInvertedIndexFileStorageFormat(),
localTbl.rowStorePageSize(),
Expand Down
3 changes: 3 additions & 0 deletions fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -185,6 +185,7 @@ protected Partition createPartitionWithIndices(long dbId, OlapTable tbl, long pa
tbl.getTimeSeriesCompactionEmptyRowsetsThreshold(),
tbl.getTimeSeriesCompactionLevelThreshold(),
tbl.disableAutoCompaction(),
tbl.floatTypeDefaultUsePlainEncoding(),
tbl.getRowStoreColumnsUniqueIds(rowStoreColumns),
tbl.getInvertedIndexFileStorageFormat(),
tbl.rowStorePageSize(),
Expand DownExpand Up@@ -222,6 +223,7 @@ public OlapFile.TabletMetaCloudPB.Builder createTabletMetaBuilder(long tableId,
Long timeSeriesCompactionGoalSizeMbytes, Long timeSeriesCompactionFileCountThreshold,
Long timeSeriesCompactionTimeThresholdSeconds, Long timeSeriesCompactionEmptyRowsetsThreshold,
Long timeSeriesCompactionLevelThreshold, boolean disableAutoCompaction,
boolean floatTypeDefaultUsePlainEncoding,
List<Integer> rowStoreColumnUniqueIds,
TInvertedIndexFileStorageFormat invertedIndexFileStorageFormat, long pageSize,
boolean variantEnableFlattenNested, List<Integer> clusterKeyUids,
Expand DownExpand Up@@ -361,6 +363,7 @@ public OlapFile.TabletMetaCloudPB.Builder createTabletMetaBuilder(long tableId,
schemaBuilder.addAllRowStoreColumnUniqueIds(rowStoreColumnUniqueIds);
}
schemaBuilder.setDisableAutoCompaction(disableAutoCompaction);
schemaBuilder.setFloatTypeDefaultUsePlainEncoding(floatTypeDefaultUsePlainEncoding);

if (invertedIndexFileStorageFormat != null) {
if (invertedIndexFileStorageFormat == TInvertedIndexFileStorageFormat.V1) {
Expand Down
Loading
Loading