diff --git a/be/src/cloud/pb_convert.cpp b/be/src/cloud/pb_convert.cpp index 157a303c2c6a45..40f52adaef831e 100644 --- a/be/src/cloud/pb_convert.cpp +++ b/be/src/cloud/pb_convert.cpp @@ -410,6 +410,9 @@ void doris_tablet_schema_to_cloud(TabletSchemaCloudPB* out, const TabletSchemaPB out->set_integer_type_default_use_plain_encoding( in.integer_type_default_use_plain_encoding()); } + if (in.has_float_type_default_use_plain_encoding()) { + out->set_float_type_default_use_plain_encoding(in.float_type_default_use_plain_encoding()); + } if (in.has_binary_plain_encoding_default_impl()) { out->set_binary_plain_encoding_default_impl(in.binary_plain_encoding_default_impl()); } @@ -457,6 +460,9 @@ void doris_tablet_schema_to_cloud(TabletSchemaCloudPB* out, TabletSchemaPB&& in) out->set_integer_type_default_use_plain_encoding( in.integer_type_default_use_plain_encoding()); } + if (in.has_float_type_default_use_plain_encoding()) { + out->set_float_type_default_use_plain_encoding(in.float_type_default_use_plain_encoding()); + } if (in.has_binary_plain_encoding_default_impl()) { out->set_binary_plain_encoding_default_impl(in.binary_plain_encoding_default_impl()); } @@ -517,6 +523,9 @@ void cloud_tablet_schema_to_doris(TabletSchemaPB* out, const TabletSchemaCloudPB out->set_integer_type_default_use_plain_encoding( in.integer_type_default_use_plain_encoding()); } + if (in.has_float_type_default_use_plain_encoding()) { + out->set_float_type_default_use_plain_encoding(in.float_type_default_use_plain_encoding()); + } if (in.has_binary_plain_encoding_default_impl()) { out->set_binary_plain_encoding_default_impl(in.binary_plain_encoding_default_impl()); } @@ -565,6 +574,9 @@ void cloud_tablet_schema_to_doris(TabletSchemaPB* out, TabletSchemaCloudPB&& in) out->set_integer_type_default_use_plain_encoding( in.integer_type_default_use_plain_encoding()); } + if (in.has_float_type_default_use_plain_encoding()) { + out->set_float_type_default_use_plain_encoding(in.float_type_default_use_plain_encoding()); + } if (in.has_binary_plain_encoding_default_impl()) { out->set_binary_plain_encoding_default_impl(in.binary_plain_encoding_default_impl()); } diff --git a/be/src/storage/segment/encoding_info.cpp b/be/src/storage/segment/encoding_info.cpp index 9fdad8504063aa..67a015a836c596 100644 --- a/be/src/storage/segment/encoding_info.cpp +++ b/be/src/storage/segment/encoding_info.cpp @@ -348,6 +348,10 @@ bool is_integer_type(FieldType type) { type == FieldType::OLAP_FIELD_TYPE_BIGINT || type == FieldType::OLAP_FIELD_TYPE_LARGEINT; } +bool is_float_type(FieldType type) { + return type == FieldType::OLAP_FIELD_TYPE_FLOAT || type == FieldType::OLAP_FIELD_TYPE_DOUBLE; +} + bool is_binary_type(FieldType type) { return type == FieldType::OLAP_FIELD_TYPE_CHAR || type == FieldType::OLAP_FIELD_TYPE_VARCHAR || type == FieldType::OLAP_FIELD_TYPE_STRING || type == FieldType::OLAP_FIELD_TYPE_JSONB || @@ -396,6 +400,17 @@ EncodingTypePB EncodingInfoResolver::get_default_encoding(FieldType type, return is_integer_type(type) && encoding_preference.integer_type_default_use_plain_encoding; }, + .target_encoding = PLAIN_ENCODING}, + + // Hook 3: Float types - any encoding -> PLAIN_ENCODING + // Applies when: type is float/double and config enables plain encoding for floats + EncodingTransform { + .predicate = + [](FieldType type, EncodingTypePB encoding, + EncodingPreference encoding_preference, bool optimize_value_seek) { + return is_float_type(type) && + encoding_preference.float_type_default_use_plain_encoding; + }, .target_encoding = PLAIN_ENCODING}}; auto& encoding_map = diff --git a/be/src/storage/segment/options.h b/be/src/storage/segment/options.h index 72509cb4602eaa..900786b7f0cb1c 100644 --- a/be/src/storage/segment/options.h +++ b/be/src/storage/segment/options.h @@ -31,6 +31,7 @@ constexpr long ROW_STORE_PAGE_SIZE_DEFAULT_VALUE = 16384; // default row store p struct EncodingPreference { bool integer_type_default_use_plain_encoding {false}; + bool float_type_default_use_plain_encoding {false}; BinaryPlainEncodingTypePB binary_plain_encoding_default_impl { BinaryPlainEncodingTypePB::BINARY_PLAIN_ENCODING_V1}; }; diff --git a/be/src/storage/segment/segment_writer.cpp b/be/src/storage/segment/segment_writer.cpp index deef80e8e207da..ca7e31fe5cce90 100644 --- a/be/src/storage/segment/segment_writer.cpp +++ b/be/src/storage/segment/segment_writer.cpp @@ -307,6 +307,8 @@ Status SegmentWriter::_create_column_writer(uint32_t cid, const TabletColumn& co } opts.encoding_preference = {.integer_type_default_use_plain_encoding = _tablet_schema->integer_type_default_use_plain_encoding(), + .float_type_default_use_plain_encoding = + _tablet_schema->float_type_default_use_plain_encoding(), .binary_plain_encoding_default_impl = _tablet_schema->binary_plain_encoding_default_impl()}; diff --git a/be/src/storage/segment/vertical_segment_writer.cpp b/be/src/storage/segment/vertical_segment_writer.cpp index a3728111fa2bce..df36060bc4f8da 100644 --- a/be/src/storage/segment/vertical_segment_writer.cpp +++ b/be/src/storage/segment/vertical_segment_writer.cpp @@ -306,6 +306,8 @@ Status VerticalSegmentWriter::_create_column_writer(uint32_t cid, const TabletCo opts.encoding_preference = {.integer_type_default_use_plain_encoding = _tablet_schema->integer_type_default_use_plain_encoding(), + .float_type_default_use_plain_encoding = + _tablet_schema->float_type_default_use_plain_encoding(), .binary_plain_encoding_default_impl = _tablet_schema->binary_plain_encoding_default_impl()}; std::unique_ptr writer; diff --git a/be/src/storage/tablet/tablet_meta.cpp b/be/src/storage/tablet/tablet_meta.cpp index 05e01fec6d8d03..d858e01521d5f3 100644 --- a/be/src/storage/tablet/tablet_meta.cpp +++ b/be/src/storage/tablet/tablet_meta.cpp @@ -389,6 +389,10 @@ TabletMeta::TabletMeta(int64_t table_id, int64_t partition_id, int64_t tablet_id if (tablet_schema.__isset.storage_dict_page_size) { schema->set_storage_dict_page_size(tablet_schema.storage_dict_page_size); } + if (tablet_schema.__isset.float_type_default_use_plain_encoding) { + schema->set_float_type_default_use_plain_encoding( + tablet_schema.float_type_default_use_plain_encoding); + } if (tablet_schema.__isset.skip_write_index_on_load) { schema->set_skip_write_index_on_load(tablet_schema.skip_write_index_on_load); } diff --git a/be/src/storage/tablet/tablet_schema.cpp b/be/src/storage/tablet/tablet_schema.cpp index 9caa121bcc3e13..15e5b325d22a6f 100644 --- a/be/src/storage/tablet/tablet_schema.cpp +++ b/be/src/storage/tablet/tablet_schema.cpp @@ -1300,6 +1300,9 @@ void TabletSchema::init_from_pb(const TabletSchemaPB& schema, bool ignore_extrac if (schema.has_integer_type_default_use_plain_encoding()) { _integer_type_default_use_plain_encoding = schema.integer_type_default_use_plain_encoding(); } + if (schema.has_float_type_default_use_plain_encoding()) { + _float_type_default_use_plain_encoding = schema.float_type_default_use_plain_encoding(); + } if (schema.has_binary_plain_encoding_default_impl()) { _binary_plain_encoding_default_impl = schema.binary_plain_encoding_default_impl(); } @@ -1578,6 +1581,8 @@ void TabletSchema::to_schema_pb(TabletSchemaPB* tablet_schema_pb) const { _is_external_segment_column_meta_used); tablet_schema_pb->set_integer_type_default_use_plain_encoding( _integer_type_default_use_plain_encoding); + tablet_schema_pb->set_float_type_default_use_plain_encoding( + _float_type_default_use_plain_encoding); tablet_schema_pb->set_binary_plain_encoding_default_impl(_binary_plain_encoding_default_impl); auto column_groups_pb = tablet_schema_pb->mutable_seq_map(); for (const auto& it : _seq_col_uid_to_value_cols_uid) { @@ -1975,6 +1980,8 @@ bool operator==(const TabletSchema& a, const TabletSchema& b) { return false; if (a._integer_type_default_use_plain_encoding != b._integer_type_default_use_plain_encoding) return false; + if (a._float_type_default_use_plain_encoding != b._float_type_default_use_plain_encoding) + return false; if (a._binary_plain_encoding_default_impl != b._binary_plain_encoding_default_impl) return false; return true; diff --git a/be/src/storage/tablet/tablet_schema.h b/be/src/storage/tablet/tablet_schema.h index 7564872fdd236e..0567389feadcc1 100644 --- a/be/src/storage/tablet/tablet_schema.h +++ b/be/src/storage/tablet/tablet_schema.h @@ -752,6 +752,14 @@ class TabletSchema : public MetadataAdder { _integer_type_default_use_plain_encoding = v; } + bool float_type_default_use_plain_encoding() const { + return _float_type_default_use_plain_encoding; + } + + void set_float_type_default_use_plain_encoding(bool v) { + _float_type_default_use_plain_encoding = v; + } + BinaryPlainEncodingTypePB binary_plain_encoding_default_impl() const { return _binary_plain_encoding_default_impl; } @@ -843,6 +851,7 @@ class TabletSchema : public MetadataAdder { bool _is_external_segment_column_meta_used = false; bool _integer_type_default_use_plain_encoding {false}; + bool _float_type_default_use_plain_encoding {false}; BinaryPlainEncodingTypePB _binary_plain_encoding_default_impl { BinaryPlainEncodingTypePB::BINARY_PLAIN_ENCODING_V1}; // Sequence column unique id mapping to value columns unique id diff --git a/be/test/storage/segment/encoding_info_test.cpp b/be/test/storage/segment/encoding_info_test.cpp index 5583a18df8a290..a56d1cc7efe005 100644 --- a/be/test/storage/segment/encoding_info_test.cpp +++ b/be/test/storage/segment/encoding_info_test.cpp @@ -151,6 +151,43 @@ TEST_F(EncodingInfoTest, test_use_plain_binary_v2_config) { EXPECT_EQ(PLAIN_ENCODING, encoding_type); // Should still be PLAIN_ENCODING } +// Test encoding preference for float types +TEST_F(EncodingInfoTest, test_float_type_plain_encoding_preference) { + // Test FLOAT type + const auto* float_type_info = get_scalar_type_info(); + + // Default: float_type_default_use_plain_encoding = false -> BIT_SHUFFLE + EncodingPreference pref_default; + pref_default.float_type_default_use_plain_encoding = false; + EncodingTypePB encoding_type = + EncodingInfo::get_default_encoding(float_type_info->type(), pref_default, false); + EXPECT_EQ(BIT_SHUFFLE, encoding_type); + + // Enabled: float_type_default_use_plain_encoding = true -> PLAIN_ENCODING + EncodingPreference pref_enabled; + pref_enabled.float_type_default_use_plain_encoding = true; + encoding_type = + EncodingInfo::get_default_encoding(float_type_info->type(), pref_enabled, false); + EXPECT_EQ(PLAIN_ENCODING, encoding_type); + + // Test DOUBLE type + const auto* double_type_info = get_scalar_type_info(); + + encoding_type = + EncodingInfo::get_default_encoding(double_type_info->type(), pref_default, false); + EXPECT_EQ(BIT_SHUFFLE, encoding_type); + + encoding_type = + EncodingInfo::get_default_encoding(double_type_info->type(), pref_enabled, false); + EXPECT_EQ(PLAIN_ENCODING, encoding_type); + + // Verify float preference doesn't affect integer types + const auto* bigint_type_info = get_scalar_type_info(); + encoding_type = + EncodingInfo::get_default_encoding(bigint_type_info->type(), pref_enabled, false); + EXPECT_EQ(BIT_SHUFFLE, encoding_type); // Should still be BIT_SHUFFLE +} + // Comprehensive test for _data_page_pre_decoder for all encoding types TEST_F(EncodingInfoTest, test_all_pre_decoders) { EncodingPreference encoding_preference; diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/CloudRollupJobV2.java b/fe/fe-core/src/main/java/org/apache/doris/alter/CloudRollupJobV2.java index e3192f5e4911b3..6ca74426e8bf73 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/CloudRollupJobV2.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/CloudRollupJobV2.java @@ -240,6 +240,7 @@ private void createRollupReplicaForPartition(OlapTable tbl) throws Exception { tbl.getTimeSeriesCompactionEmptyRowsetsThreshold(), tbl.getTimeSeriesCompactionLevelThreshold(), tbl.disableAutoCompaction(), + tbl.floatTypeDefaultUsePlainEncoding(), tbl.getRowStoreColumnsUniqueIds(rowStoreColumns), null, tbl.rowStorePageSize(), diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/CloudSchemaChangeJobV2.java b/fe/fe-core/src/main/java/org/apache/doris/alter/CloudSchemaChangeJobV2.java index cd1b90fb923316..e96074cfd57b9c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/CloudSchemaChangeJobV2.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/CloudSchemaChangeJobV2.java @@ -262,6 +262,7 @@ private void createShadowIndexReplicaForPartition(OlapTable tbl) throws Exceptio tbl.getTimeSeriesCompactionEmptyRowsetsThreshold(), tbl.getTimeSeriesCompactionLevelThreshold(), tbl.disableAutoCompaction(), + tbl.floatTypeDefaultUsePlainEncoding(), tbl.getRowStoreColumnsUniqueIds(rowStoreColumns), tbl.getInvertedIndexFileStorageFormat(), tbl.rowStorePageSize(), diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java b/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java index 27a1fdcc8df136..af0acbbf621c61 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java @@ -257,6 +257,7 @@ protected void createRollupReplica() throws AlterCancelException { tbl.getCompressionType(), tbl.getEnableUniqueKeyMergeOnWrite(), tbl.getStoragePolicy(), tbl.disableAutoCompaction(), + tbl.floatTypeDefaultUsePlainEncoding(), tbl.enableSingleReplicaCompaction(), tbl.skipWriteIndexOnLoad(), tbl.getCompactionPolicy(), diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java index 955af7162fd62b..aac0328086895f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java @@ -319,6 +319,7 @@ protected void createShadowIndexReplica() throws AlterCancelException { tbl.getCompressionType(), tbl.getEnableUniqueKeyMergeOnWrite(), tbl.getStoragePolicy(), tbl.disableAutoCompaction(), + tbl.floatTypeDefaultUsePlainEncoding(), tbl.enableSingleReplicaCompaction(), tbl.skipWriteIndexOnLoad(), tbl.getCompactionPolicy(), diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java index 0395378de476e4..3b34ae6e22978e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java @@ -1440,6 +1440,7 @@ protected void createReplicas(Database db, OlapTable localTbl, Partition restore localTbl.getCompressionType(), localTbl.getEnableUniqueKeyMergeOnWrite(), localTbl.getStoragePolicy(), localTbl.disableAutoCompaction(), + localTbl.floatTypeDefaultUsePlainEncoding(), localTbl.enableSingleReplicaCompaction(), localTbl.skipWriteIndexOnLoad(), localTbl.getCompactionPolicy(), diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java index cf5849e247adcb..2373448ba18875 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java @@ -4023,6 +4023,13 @@ private static void addOlapTablePropertyInfo(OlapTable olapTable, StringBuilder sb.append(",\n\"").append(PropertyAnalyzer.PROPERTIES_DISABLE_AUTO_COMPACTION).append("\" = \""); sb.append(olapTable.disableAutoCompaction()).append("\""); + // float type default use plain encoding + if (olapTable.floatTypeDefaultUsePlainEncoding()) { + sb.append(",\n\"").append(PropertyAnalyzer.PROPERTIES_FLOAT_TYPE_DEFAULT_USE_PLAIN_ENCODING) + .append("\" = \""); + sb.append(olapTable.floatTypeDefaultUsePlainEncoding()).append("\""); + } + if (olapTable.variantEnableFlattenNested()) { // enable flatten nested type in variant sb.append(",\n\"").append(PropertyAnalyzer.PROPERTIES_VARIANT_ENABLE_FLATTEN_NESTED).append("\" = \""); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java index ba1b8d3c715739..15f9e359fae705 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java @@ -2618,6 +2618,21 @@ public Boolean disableAutoCompaction() { return false; } + public void setFloatTypeDefaultUsePlainEncoding(boolean floatTypeDefaultUsePlainEncoding) { + TableProperty tableProperty = getOrCreatTableProperty(); + tableProperty.modifyTableProperties( + PropertyAnalyzer.PROPERTIES_FLOAT_TYPE_DEFAULT_USE_PLAIN_ENCODING, + Boolean.valueOf(floatTypeDefaultUsePlainEncoding).toString()); + tableProperty.buildFloatTypeDefaultUsePlainEncoding(); + } + + public Boolean floatTypeDefaultUsePlainEncoding() { + if (tableProperty != null) { + return tableProperty.floatTypeDefaultUsePlainEncoding(); + } + return false; + } + @Deprecated public void setVariantEnableFlattenNested(boolean flattenNested) throws AnalysisException { TableProperty tableProperty = getOrCreatTableProperty(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java index 7ada8286d9878d..5ee42d52edbbcc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java @@ -94,6 +94,8 @@ public class TableProperty implements GsonPostProcessable { private boolean disableAutoCompaction = false; + private boolean floatTypeDefaultUsePlainEncoding = false; + private boolean variantEnableFlattenNested = false; private boolean enableSingleReplicaCompaction = false; @@ -171,6 +173,7 @@ public TableProperty buildProperty(short opCode) { buildEnableSingleReplicaCompaction(); buildVerticalCompactionNumColumnsPerGroup(); buildDisableAutoCompaction(); + buildFloatTypeDefaultUsePlainEncoding(); buildTimeSeriesCompactionEmptyRowsetsThreshold(); buildTimeSeriesCompactionLevelThreshold(); buildTTLSeconds(); @@ -295,6 +298,17 @@ public TableProperty buildDisableAutoCompaction() { return this; } + public TableProperty buildFloatTypeDefaultUsePlainEncoding() { + floatTypeDefaultUsePlainEncoding = Boolean.parseBoolean( + properties.getOrDefault( + PropertyAnalyzer.PROPERTIES_FLOAT_TYPE_DEFAULT_USE_PLAIN_ENCODING, "false")); + return this; + } + + public boolean floatTypeDefaultUsePlainEncoding() { + return floatTypeDefaultUsePlainEncoding; + } + public TableProperty buildAutoAnalyzeProperty() { autoAnalyzePolicy = properties.getOrDefault(PropertyAnalyzer.PROPERTIES_AUTO_ANALYZE_POLICY, PropertyAnalyzer.ENABLE_AUTO_ANALYZE_POLICY); @@ -917,6 +931,7 @@ public void gsonPostProcess() throws IOException { buildTimeSeriesCompactionFileCountThreshold(); buildTimeSeriesCompactionTimeThresholdSeconds(); buildDisableAutoCompaction(); + buildFloatTypeDefaultUsePlainEncoding(); buildEnableSingleReplicaCompaction(); buildVerticalCompactionNumColumnsPerGroup(); buildTimeSeriesCompactionEmptyRowsetsThreshold(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/backup/CloudRestoreJob.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/backup/CloudRestoreJob.java index 980e75ee2b58e8..e356eaad357411 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/backup/CloudRestoreJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/backup/CloudRestoreJob.java @@ -398,6 +398,7 @@ public void createReplicas(Database db, OlapTable localTbl, Partition restorePar localTbl.getTimeSeriesCompactionTimeThresholdSeconds(), localTbl.getTimeSeriesCompactionEmptyRowsetsThreshold(), localTbl.getTimeSeriesCompactionLevelThreshold(), localTbl.disableAutoCompaction(), + localTbl.floatTypeDefaultUsePlainEncoding(), localTbl.getRowStoreColumnsUniqueIds(rowStoreColumns), localTbl.getInvertedIndexFileStorageFormat(), localTbl.rowStorePageSize(), diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java index 1c1c0385c5b736..77e25615427e8b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java @@ -185,6 +185,7 @@ protected Partition createPartitionWithIndices(long dbId, OlapTable tbl, long pa tbl.getTimeSeriesCompactionEmptyRowsetsThreshold(), tbl.getTimeSeriesCompactionLevelThreshold(), tbl.disableAutoCompaction(), + tbl.floatTypeDefaultUsePlainEncoding(), tbl.getRowStoreColumnsUniqueIds(rowStoreColumns), tbl.getInvertedIndexFileStorageFormat(), tbl.rowStorePageSize(), @@ -222,6 +223,7 @@ public OlapFile.TabletMetaCloudPB.Builder createTabletMetaBuilder(long tableId, Long timeSeriesCompactionGoalSizeMbytes, Long timeSeriesCompactionFileCountThreshold, Long timeSeriesCompactionTimeThresholdSeconds, Long timeSeriesCompactionEmptyRowsetsThreshold, Long timeSeriesCompactionLevelThreshold, boolean disableAutoCompaction, + boolean floatTypeDefaultUsePlainEncoding, List rowStoreColumnUniqueIds, TInvertedIndexFileStorageFormat invertedIndexFileStorageFormat, long pageSize, boolean variantEnableFlattenNested, List clusterKeyUids, @@ -361,6 +363,7 @@ public OlapFile.TabletMetaCloudPB.Builder createTabletMetaBuilder(long tableId, schemaBuilder.addAllRowStoreColumnUniqueIds(rowStoreColumnUniqueIds); } schemaBuilder.setDisableAutoCompaction(disableAutoCompaction); + schemaBuilder.setFloatTypeDefaultUsePlainEncoding(floatTypeDefaultUsePlainEncoding); if (invertedIndexFileStorageFormat != null) { if (invertedIndexFileStorageFormat == TInvertedIndexFileStorageFormat.V1) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java index 5bbd2d83bb6efd..934ab0ed0ee8a1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java @@ -158,6 +158,9 @@ public class PropertyAnalyzer { public static final String PROPERTIES_DISABLE_AUTO_COMPACTION = "disable_auto_compaction"; + public static final String PROPERTIES_FLOAT_TYPE_DEFAULT_USE_PLAIN_ENCODING = + "float_type_default_use_plain_encoding"; + // Legacy persisted switch for flatten-nested variant behavior before it was deprecated. @Deprecated public static final String LEGACY_PROPERTIES_VARIANT_ENABLE_FLATTEN_NESTED = "variant_enable_flatten_nested"; @@ -829,6 +832,25 @@ public static Boolean analyzeDisableAutoCompaction(Map propertie + " must be `true` or `false`"); } + public static Boolean analyzeFloatTypeDefaultUsePlainEncoding(Map properties) + throws AnalysisException { + if (properties == null || properties.isEmpty()) { + return false; + } + String value = properties.get(PROPERTIES_FLOAT_TYPE_DEFAULT_USE_PLAIN_ENCODING); + if (null == value) { + return false; + } + properties.remove(PROPERTIES_FLOAT_TYPE_DEFAULT_USE_PLAIN_ENCODING); + if (value.equalsIgnoreCase("true")) { + return true; + } else if (value.equalsIgnoreCase("false")) { + return false; + } + throw new AnalysisException(PROPERTIES_FLOAT_TYPE_DEFAULT_USE_PLAIN_ENCODING + + " must be `true` or `false`"); + } + @Deprecated public static Boolean analyzeVariantFlattenNested(Map properties) throws AnalysisException { if (properties == null || properties.isEmpty()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java index b97304e618c1cd..37e5ad421f84c3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java @@ -1543,6 +1543,11 @@ public void addPartition(Database db, String tableName, AddPartitionOp addPartit properties.put(PropertyAnalyzer.PROPERTIES_DISABLE_AUTO_COMPACTION, olapTable.disableAutoCompaction().toString()); } + if (!properties.containsKey( + PropertyAnalyzer.PROPERTIES_FLOAT_TYPE_DEFAULT_USE_PLAIN_ENCODING)) { + properties.put(PropertyAnalyzer.PROPERTIES_FLOAT_TYPE_DEFAULT_USE_PLAIN_ENCODING, + olapTable.floatTypeDefaultUsePlainEncoding().toString()); + } if (!properties.containsKey(PropertyAnalyzer.PROPERTIES_VARIANT_ENABLE_FLATTEN_NESTED)) { properties.put(PropertyAnalyzer.PROPERTIES_VARIANT_ENABLE_FLATTEN_NESTED, olapTable.variantEnableFlattenNested().toString()); @@ -2149,6 +2154,7 @@ protected Partition createPartitionWithIndices(long dbId, OlapTable tbl, long pa indexes, tbl.isInMemory(), tabletType, tbl.getDataSortInfo(), tbl.getCompressionType(), tbl.getEnableUniqueKeyMergeOnWrite(), storagePolicy, tbl.disableAutoCompaction(), + tbl.floatTypeDefaultUsePlainEncoding(), tbl.enableSingleReplicaCompaction(), tbl.skipWriteIndexOnLoad(), tbl.getCompactionPolicy(), tbl.getTimeSeriesCompactionGoalSizeMbytes(), tbl.getTimeSeriesCompactionFileCountThreshold(), @@ -2436,6 +2442,15 @@ private boolean createOlapTable(Database db, CreateTableInfo createTableInfo) th // use light schema change optimization olapTable.setDisableAutoCompaction(disableAutoCompaction); + boolean floatTypeDefaultUsePlainEncoding = false; + try { + floatTypeDefaultUsePlainEncoding = + PropertyAnalyzer.analyzeFloatTypeDefaultUsePlainEncoding(properties); + } catch (AnalysisException e) { + throw new DdlException(e.getMessage()); + } + olapTable.setFloatTypeDefaultUsePlainEncoding(floatTypeDefaultUsePlainEncoding); + // set compaction policy String compactionPolicy = PropertyAnalyzer.SIZE_BASED_COMPACTION_POLICY; try { diff --git a/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java b/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java index cf7df808eda14d..ebaccdf973d6ef 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java @@ -1074,6 +1074,7 @@ private static void deleteFromMeta(ListMultimap tabletDeleteFromMeta olapTable.getCompressionType(), olapTable.getEnableUniqueKeyMergeOnWrite(), olapTable.getStoragePolicy(), olapTable.disableAutoCompaction(), + olapTable.floatTypeDefaultUsePlainEncoding(), olapTable.enableSingleReplicaCompaction(), olapTable.skipWriteIndexOnLoad(), olapTable.getCompactionPolicy(), olapTable.getTimeSeriesCompactionGoalSizeMbytes(), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ModifyTablePropertiesOp.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ModifyTablePropertiesOp.java index 0e23752c5dd264..df61739cbfb51d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ModifyTablePropertiesOp.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ModifyTablePropertiesOp.java @@ -360,6 +360,10 @@ public void validate(ConnectContext ctx) throws UserException { } else if (properties.containsKey(PropertyAnalyzer.PROPERTIES_STORAGE_PAGE_SIZE) || properties.containsKey(PropertyAnalyzer.PROPERTIES_STORAGE_DICT_PAGE_SIZE)) { throw new AnalysisException("You can not modify storage_page_size|storage_dict_page_size property."); + } else if (properties.containsKey( + PropertyAnalyzer.PROPERTIES_FLOAT_TYPE_DEFAULT_USE_PLAIN_ENCODING)) { + throw new AnalysisException("You can not modify " + + PropertyAnalyzer.PROPERTIES_FLOAT_TYPE_DEFAULT_USE_PLAIN_ENCODING + " property."); } else if (properties.containsKey(PropertyAnalyzer.PROPERTIES_STORAGE_MEDIUM)) { this.opType = AlterOpType.MODIFY_TABLE_PROPERTY_SYNC; } else { diff --git a/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java b/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java index 1d03b7bfe9b2a6..53b3ba3e62fb98 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java @@ -109,6 +109,8 @@ public class CreateReplicaTask extends AgentTask { private boolean disableAutoCompaction; + private boolean floatTypeDefaultUsePlainEncoding; + private boolean enableSingleReplicaCompaction; private boolean skipWriteIndexOnLoad; @@ -152,6 +154,7 @@ public CreateReplicaTask(long backendId, long dbId, long tableId, long partition TCompressionType compressionType, boolean enableUniqueKeyMergeOnWrite, String storagePolicy, boolean disableAutoCompaction, + boolean floatTypeDefaultUsePlainEncoding, boolean enableSingleReplicaCompaction, boolean skipWriteIndexOnLoad, String compactionPolicy, @@ -203,6 +206,7 @@ public CreateReplicaTask(long backendId, long dbId, long tableId, long partition } } this.disableAutoCompaction = disableAutoCompaction; + this.floatTypeDefaultUsePlainEncoding = floatTypeDefaultUsePlainEncoding; this.enableSingleReplicaCompaction = enableSingleReplicaCompaction; this.skipWriteIndexOnLoad = skipWriteIndexOnLoad; this.compactionPolicy = compactionPolicy; @@ -376,6 +380,7 @@ public TCreateTabletReq toThrift() { } tSchema.setIsInMemory(isInMemory); tSchema.setDisableAutoCompaction(disableAutoCompaction); + tSchema.setFloatTypeDefaultUsePlainEncoding(floatTypeDefaultUsePlainEncoding); tSchema.setVariantEnableFlattenNested(variantEnableFlattenNested); tSchema.setEnableSingleReplicaCompaction(enableSingleReplicaCompaction); tSchema.setSkipWriteIndexOnLoad(skipWriteIndexOnLoad); diff --git a/fe/fe-core/src/test/java/org/apache/doris/task/AgentTaskTest.java b/fe/fe-core/src/test/java/org/apache/doris/task/AgentTaskTest.java index a1cef6161bdec7..e7f90a03ca85df 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/task/AgentTaskTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/task/AgentTaskTest.java @@ -115,7 +115,7 @@ public void setUp() throws AnalysisException { createReplicaTask = new CreateReplicaTask(backendId1, dbId, tableId, partitionId, indexId1, tabletId1, replicaId1, shortKeyNum, schemaHash1, version, KeysType.AGG_KEYS, storageType, TStorageMedium.SSD, columns, null, 0, latch, null, false, TTabletType.TABLET_TYPE_DISK, null, - TCompressionType.LZ4F, false, "", false, false, false, "", 0, 0, 0, 0, 0, false, null, null, objectPool, rowStorePageSize, false, + TCompressionType.LZ4F, false, "", false, false, false, false, "", 0, 0, 0, 0, 0, false, null, null, objectPool, rowStorePageSize, false, storagePageSize, TEncryptionAlgorithm.PLAINTEXT, storageDictPageSize, new HashMap<>(), 5); // drop diff --git a/gensrc/proto/olap_file.proto b/gensrc/proto/olap_file.proto index 0d4359fe16215f..91632bfd89b21b 100644 --- a/gensrc/proto/olap_file.proto +++ b/gensrc/proto/olap_file.proto @@ -491,6 +491,7 @@ message TabletSchemaPB { optional BinaryPlainEncodingTypePB binary_plain_encoding_default_impl = 34; // mapping from seq column unique id to value column unique id optional ColumnGroupsPB seq_map = 35; + optional bool float_type_default_use_plain_encoding = 36; optional SplitSchemaPB __split_schema = 1000; // A special field, DO NOT change it. } @@ -535,6 +536,7 @@ message TabletSchemaCloudPB { optional BinaryPlainEncodingTypePB binary_plain_encoding_default_impl = 35; // mapping from seq column unique id to value column unique id optional ColumnGroupsPB seq_map = 36; + optional bool float_type_default_use_plain_encoding = 37; optional bool is_dynamic_schema = 100 [default=false]; diff --git a/gensrc/thrift/AgentService.thrift b/gensrc/thrift/AgentService.thrift index c8c65513dc26e3..8c7cfa64b439b7 100644 --- a/gensrc/thrift/AgentService.thrift +++ b/gensrc/thrift/AgentService.thrift @@ -52,6 +52,7 @@ struct TTabletSchema { 23: optional i64 storage_page_size = 65536 24: optional i64 storage_dict_page_size = 262144 25: optional list seq_map + 26: optional bool float_type_default_use_plain_encoding = false } // this enum stands for different storage format in src_backends diff --git a/regression-test/data/query_p0/system/test_table_properties.out b/regression-test/data/query_p0/system/test_table_properties.out index 497f300612039f..345e859a78a347 100644 --- a/regression-test/data/query_p0/system/test_table_properties.out +++ b/regression-test/data/query_p0/system/test_table_properties.out @@ -1,6 +1,6 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !select_check_1 -- -114 +117 -- !select_check_2 -- internal test_table_properties_db duplicate_table _auto_bucket false @@ -18,6 +18,7 @@ internal test_table_properties_db duplicate_table enable_mow_light_delete false internal test_table_properties_db duplicate_table enable_single_replica_compaction false internal test_table_properties_db duplicate_table enable_unique_key_merge_on_write false internal test_table_properties_db duplicate_table file_cache_ttl_seconds 0 +internal test_table_properties_db duplicate_table float_type_default_use_plain_encoding false internal test_table_properties_db duplicate_table group_commit_data_bytes 134217728 internal test_table_properties_db duplicate_table group_commit_interval_ms 10000 internal test_table_properties_db duplicate_table group_commit_mode off_mode @@ -55,6 +56,7 @@ internal test_table_properties_db listtable enable_mow_light_delete false internal test_table_properties_db listtable enable_single_replica_compaction false internal test_table_properties_db listtable enable_unique_key_merge_on_write false internal test_table_properties_db listtable file_cache_ttl_seconds 0 +internal test_table_properties_db listtable float_type_default_use_plain_encoding false internal test_table_properties_db listtable group_commit_data_bytes 134217728 internal test_table_properties_db listtable group_commit_interval_ms 10000 internal test_table_properties_db listtable group_commit_mode off_mode @@ -92,6 +94,7 @@ internal test_table_properties_db unique_table enable_mow_light_delete false internal test_table_properties_db unique_table enable_single_replica_compaction false internal test_table_properties_db unique_table enable_unique_key_merge_on_write true internal test_table_properties_db unique_table file_cache_ttl_seconds 0 +internal test_table_properties_db unique_table float_type_default_use_plain_encoding false internal test_table_properties_db unique_table group_commit_data_bytes 134217728 internal test_table_properties_db unique_table group_commit_interval_ms 10000 internal test_table_properties_db unique_table group_commit_mode off_mode @@ -131,6 +134,7 @@ internal test_table_properties_db duplicate_table enable_mow_light_delete false internal test_table_properties_db duplicate_table enable_single_replica_compaction false internal test_table_properties_db duplicate_table enable_unique_key_merge_on_write false internal test_table_properties_db duplicate_table file_cache_ttl_seconds 0 +internal test_table_properties_db duplicate_table float_type_default_use_plain_encoding false internal test_table_properties_db duplicate_table group_commit_data_bytes 134217728 internal test_table_properties_db duplicate_table group_commit_interval_ms 10000 internal test_table_properties_db duplicate_table group_commit_mode off_mode @@ -168,6 +172,7 @@ internal test_table_properties_db unique_table enable_mow_light_delete false internal test_table_properties_db unique_table enable_single_replica_compaction false internal test_table_properties_db unique_table enable_unique_key_merge_on_write true internal test_table_properties_db unique_table file_cache_ttl_seconds 0 +internal test_table_properties_db unique_table float_type_default_use_plain_encoding false internal test_table_properties_db unique_table group_commit_data_bytes 134217728 internal test_table_properties_db unique_table group_commit_interval_ms 10000 internal test_table_properties_db unique_table group_commit_mode off_mode @@ -209,6 +214,7 @@ internal test_table_properties_db duplicate_table enable_mow_light_delete false internal test_table_properties_db duplicate_table enable_single_replica_compaction false internal test_table_properties_db duplicate_table enable_unique_key_merge_on_write false internal test_table_properties_db duplicate_table file_cache_ttl_seconds 0 +internal test_table_properties_db duplicate_table float_type_default_use_plain_encoding false internal test_table_properties_db duplicate_table group_commit_data_bytes 134217728 internal test_table_properties_db duplicate_table group_commit_interval_ms 10000 internal test_table_properties_db duplicate_table group_commit_mode off_mode diff --git a/regression-test/suites/table_p0/test_float_type_default_use_plain_encoding.groovy b/regression-test/suites/table_p0/test_float_type_default_use_plain_encoding.groovy new file mode 100644 index 00000000000000..60435828025ca4 --- /dev/null +++ b/regression-test/suites/table_p0/test_float_type_default_use_plain_encoding.groovy @@ -0,0 +1,86 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.util.Http + +suite('test_float_type_default_use_plain_encoding') { + + // Test 1: Property explicitly set to true + def tableName1 = "test_float_plain_encoding_enabled" + sql """drop table if exists `${tableName1}` force; """ + + sql """ + CREATE TABLE ${tableName1} + (k int, v1 float, v2 double) + duplicate KEY(k) + DISTRIBUTED BY HASH (k) + BUCKETS 1 PROPERTIES( + "replication_num" = "1", + "float_type_default_use_plain_encoding" = "true"); + """ + + sql "insert into ${tableName1} values(1, 1.5, 2.5);" + sql "select * from ${tableName1};" + + def metaUrl = sql_return_maparray("show tablets from ${tableName1};").get(0).MetaUrl + logger.info("begin curl ${metaUrl}") + def jsonMeta = Http.GET(metaUrl, true, false) + + assert jsonMeta.schema.float_type_default_use_plain_encoding == true + + // Verify property is shown in SHOW CREATE TABLE + def createTableStmt = sql "show create table ${tableName1};" + logger.info("show create table: ${createTableStmt}") + assert createTableStmt[0][1].contains('"float_type_default_use_plain_encoding" = "true"') + + // Test 2: Property explicitly set to false + def tableName2 = "test_float_plain_encoding_disabled" + sql """drop table if exists `${tableName2}` force; """ + + sql """ + CREATE TABLE ${tableName2} + (k int, v1 float, v2 double) + duplicate KEY(k) + DISTRIBUTED BY HASH (k) + BUCKETS 1 PROPERTIES( + "replication_num" = "1", + "float_type_default_use_plain_encoding" = "false"); + """ + + sql "insert into ${tableName2} values(1, 1.5, 2.5);" + sql "select * from ${tableName2};" + + metaUrl = sql_return_maparray("show tablets from ${tableName2};").get(0).MetaUrl + logger.info("begin curl ${metaUrl}") + jsonMeta = Http.GET(metaUrl, true, false) + + assert jsonMeta.schema.float_type_default_use_plain_encoding == false + + // Verify SHOW CREATE TABLE does NOT show false (only shown when true) + createTableStmt = sql "show create table ${tableName2};" + assert !createTableStmt[0][1].contains('float_type_default_use_plain_encoding') + + // Test 3: Property is immutable - ALTER TABLE should fail + test { + sql """ALTER TABLE ${tableName2} SET ("float_type_default_use_plain_encoding" = "true");""" + exception "You can not modify float_type_default_use_plain_encoding property" + } + + // Cleanup + sql """drop table if exists `${tableName1}` force; """ + sql """drop table if exists `${tableName2}` force; """ +}