From 0bc9c745456541d4f612edd67d4dd958980e34e9 Mon Sep 17 00:00:00 2001 From: Yukang-Lian Date: Wed, 26 Aug 2026 22:15:56 +0800 Subject: [PATCH 1/2] [improvement](compaction) Remove MOW version-count promotion ### What problem does this PR solve? Issue Number: None Related PR: #49383 Problem Summary: Size-based cumulative compaction promotes a merge-on-write output rowset to the base layer once its version span exceeds compaction_promotion_version_count, even when the rowset is only a few hundred KB. This causes high-frequency tiny loads to enter base compaction prematurely and trigger unnecessary base compactions. PR #49383 makes cumulative compaction aggregate and remove historical delete bitmap versions, so raw version span is no longer needed as a proxy for delete bitmap pressure. Remove the version-count promotion from local and cloud policies and remove its obsolete configuration. Size-based and delete-version promotion remain unchanged. ### Release note MOW cumulative compaction no longer promotes small output rowsets based only on version count. ### Check List (For Author) - Test: Unit Test - ./run-be-ut.sh --run --filter=TestSizeBasedCumulativeCompactionPolicy.*:TestCloudSizeBasedCumulativeCompactionPolicy.* -j80 - build-support/check-format.sh - build-support/check-build-hygiene.sh - Behavior changed: Yes. Small MOW cumulative output rowsets remain in the cumulative layer until normal size or delete-version promotion applies. - Does this need documentation: No --- .../cloud_cumulative_compaction_policy.cpp | 11 +-------- be/src/common/config.cpp | 6 ----- be/src/common/config.h | 6 ----- .../cumulative_compaction_policy.cpp | 12 +--------- .../compaction/cumulative_compaction_policy.h | 3 --- ...loud_cumulative_compaction_policy_test.cpp | 13 +++++++++++ .../cumulative_compaction_policy_test.cpp | 23 +++++++++++++++++++ ...compaction_update_big_delete_bitmap.groovy | 3 --- ...paction_and_read_stale_cloud_docker.groovy | 1 - ...egments_and_read_stale_cloud_docker.groovy | 1 - ...on_agg_and_remove_pre_delete_bitmap.groovy | 3 --- .../test_mow_compaction_and_read_stale.groovy | 3 --- ...paction_and_read_stale_cloud_docker.groovy | 1 - ...mow_compaction_and_rowset_not_exist.groovy | 3 --- ..._mow_delete_unused_rowset_dm_docker.groovy | 1 - 15 files changed, 38 insertions(+), 52 deletions(-) diff --git a/be/src/cloud/cloud_cumulative_compaction_policy.cpp b/be/src/cloud/cloud_cumulative_compaction_policy.cpp index 890527b7e173c5..52d8fbdd1f50d3 100644 --- a/be/src/cloud/cloud_cumulative_compaction_policy.cpp +++ b/be/src/cloud/cloud_cumulative_compaction_policy.cpp @@ -324,19 +324,10 @@ int64_t CloudSizeBasedCumulativeCompactionPolicy::new_cumulative_point( return cumu_point; } }); - // for MoW table, if there's too many versions, the delete bitmap will grow to - // a very big size, which may cause the tablet meta too big and the `save_meta` - // operation too slow. - // if the rowset should not promotion according to it's disk size, we should also - // consider it's version count here. - bool satisfy_promotion_version = tablet->enable_unique_key_merge_on_write() && - output_rowset->end_version() - output_rowset->start_version() > - config::compaction_promotion_version_count; // if rowsets have delete version, move to the last directly. // if rowsets have no delete version, check output_rowset total disk size satisfies promotion size. return (last_delete_version.first != -1 || - output_rowset->total_disk_size() >= cloud_promotion_size(tablet) || - satisfy_promotion_version) + output_rowset->total_disk_size() >= cloud_promotion_size(tablet)) ? output_rowset->end_version() + 1 : last_cumulative_point; } diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index 75495dcb1a2106..5f91f56bb1d986 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -523,12 +523,6 @@ DEFINE_mDouble(compaction_promotion_ratio, "0.05"); // rowset will be not given to base compaction. The unit is m byte. DEFINE_mInt64(compaction_promotion_min_size_mbytes, "128"); -// When output rowset of cumulative compaction total version count (end_version - start_version) -// exceed this config count, the rowset will be moved to base compaction -// NOTE: this config will work for unique key merge-on-write table only, to reduce version count -// related cost on delete bitmap more effectively. -DEFINE_mInt64(compaction_promotion_version_count, "1000"); - // The lower bound size to do cumulative compaction. When total disk size of candidate rowsets is less than // this size, size_based policy may not do to cumulative compaction. The unit is m byte. DEFINE_mInt64(compaction_min_size_mbytes, "64"); diff --git a/be/src/common/config.h b/be/src/common/config.h index b9e33288396033..a04ace12c14a11 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -613,12 +613,6 @@ DECLARE_mDouble(compaction_promotion_ratio); // rowset will be not given to base compaction. The unit is m byte. DECLARE_mInt64(compaction_promotion_min_size_mbytes); -// When output rowset of cumulative compaction total version count (end_version - start_version) -// exceed this config count, the rowset will be moved to base compaction -// NOTE: this config will work for unique key merge-on-write table only, to reduce version count -// related cost on delete bitmap more effectively. -DECLARE_mInt64(compaction_promotion_version_count); - // The lower bound size to do cumulative compaction. When total disk size of candidate rowsets is less than // this size, size_based policy may not do to cumulative compaction. The unit is m byte. DECLARE_mInt64(compaction_min_size_mbytes); diff --git a/be/src/storage/compaction/cumulative_compaction_policy.cpp b/be/src/storage/compaction/cumulative_compaction_policy.cpp index 07e7488d8e5010..d10b60a167a279 100644 --- a/be/src/storage/compaction/cumulative_compaction_policy.cpp +++ b/be/src/storage/compaction/cumulative_compaction_policy.cpp @@ -39,11 +39,10 @@ namespace doris { SizeBasedCumulativeCompactionPolicy::SizeBasedCumulativeCompactionPolicy( int64_t promotion_size, double promotion_ratio, int64_t promotion_min_size, - int64_t promotion_version_count, int64_t compaction_min_size) + int64_t compaction_min_size) : _promotion_size(promotion_size), _promotion_ratio(promotion_ratio), _promotion_min_size(promotion_min_size), - _promotion_version_count(promotion_version_count), _compaction_min_size(compaction_min_size) {} void SizeBasedCumulativeCompactionPolicy::calculate_cumulative_point( @@ -159,15 +158,6 @@ void SizeBasedCumulativeCompactionPolicy::update_cumulative_point( size_t total_size = output_rowset->rowset_meta()->total_disk_size(); if (total_size >= tablet->cumulative_promotion_size()) { tablet->set_cumulative_layer_point(output_rowset->end_version() + 1); - } else if (tablet->enable_unique_key_merge_on_write() && - output_rowset->end_version() - output_rowset->start_version() > - _promotion_version_count) { - // for MoW table, if there's too many versions, the delete bitmap will grow to - // a very big size, which may cause the tablet meta too big and the `save_meta` - // operation too slow. - // if the rowset should not promotion according to it's disk size, we should also - // consider it's version count here. - tablet->set_cumulative_layer_point(output_rowset->end_version() + 1); } } } diff --git a/be/src/storage/compaction/cumulative_compaction_policy.h b/be/src/storage/compaction/cumulative_compaction_policy.h index 45ea6265c1087a..377e6e92f866ba 100644 --- a/be/src/storage/compaction/cumulative_compaction_policy.h +++ b/be/src/storage/compaction/cumulative_compaction_policy.h @@ -119,7 +119,6 @@ class SizeBasedCumulativeCompactionPolicy final : public CumulativeCompactionPol int64_t promotion_size = config::compaction_promotion_size_mbytes * 1024 * 1024, double promotion_ratio = config::compaction_promotion_ratio, int64_t promotion_min_size = config::compaction_promotion_min_size_mbytes * 1024 * 1024, - int64_t promotion_version_count = config::compaction_promotion_version_count, int64_t compaction_min_size = config::compaction_min_size_mbytes * 1024 * 1024); /// Destructor function of SizeBasedCumulativeCompactionPolicy. @@ -179,8 +178,6 @@ class SizeBasedCumulativeCompactionPolicy final : public CumulativeCompactionPol double _promotion_ratio; /// cumulative compaction promotion min size, unit is byte. int64_t _promotion_min_size; - // cululative compaction promotion version count, only works for unique key MoW table - int64_t _promotion_version_count; /// lower bound size to do compaction compaction. int64_t _compaction_min_size; }; diff --git a/be/test/cloud/cloud_cumulative_compaction_policy_test.cpp b/be/test/cloud/cloud_cumulative_compaction_policy_test.cpp index 35a2598617fca9..30a8695e9586ef 100644 --- a/be/test/cloud/cloud_cumulative_compaction_policy_test.cpp +++ b/be/test/cloud/cloud_cumulative_compaction_policy_test.cpp @@ -212,6 +212,19 @@ TEST_F(TestCloudSizeBasedCumulativeCompactionPolicy, new_cumulative_point) { EXPECT_EQ(policy.new_cumulative_point(&_tablet, output_rowset, version, 2), 6); } +TEST_F(TestCloudSizeBasedCumulativeCompactionPolicy, + new_cumulative_point_does_not_promote_small_mow_rowset_by_version_count) { + CloudTablet tablet(_engine, _tablet_meta); + tablet._tablet_meta->_enable_unique_key_merge_on_write = true; + tablet._base_size = kGiB; + + CloudSizeBasedCumulativeCompactionPolicy policy; + RowsetSharedPtr output_rowset = create_rowset(Version(2, 1003), 1, false, kMiB); + Version last_delete_version {-1, -1}; + + EXPECT_EQ(2, policy.new_cumulative_point(&tablet, output_rowset, last_delete_version, 2)); +} + TEST_F(TestCloudSizeBasedCumulativeCompactionPolicy, pick_input_rowsets_notready_keeps_latest_versions) { auto base_rowset = create_rowset(Version(0, 1), 1, false, kGiB); diff --git a/be/test/storage/compaction/cumulative_compaction_policy_test.cpp b/be/test/storage/compaction/cumulative_compaction_policy_test.cpp index ad1c536451da7f..11800833bdc9cc 100644 --- a/be/test/storage/compaction/cumulative_compaction_policy_test.cpp +++ b/be/test/storage/compaction/cumulative_compaction_policy_test.cpp @@ -1238,6 +1238,29 @@ TEST_F(TestSizeBasedCumulativeCompactionPolicy, EXPECT_EQ(0, compaction_score); } +TEST_F(TestSizeBasedCumulativeCompactionPolicy, + update_cumulative_point_does_not_promote_small_mow_rowset_by_version_count) { + _tablet_meta->set_enable_unique_key_merge_on_write(true); + TabletSharedPtr tablet( + new Tablet(_engine, _tablet_meta, nullptr, CUMULATIVE_SIZE_BASED_POLICY)); + ASSERT_TRUE(tablet->init().ok()); + tablet->set_cumulative_layer_point(1); + tablet->set_cumulative_promotion_size(64L * kMiB); + + RowsetMetaSharedPtr output_meta(new RowsetMeta()); + init_rs_meta(output_meta, 1, 1002); + output_meta->set_total_disk_size(kMiB); + RowsetSharedPtr output_rowset; + ASSERT_TRUE(RowsetFactory::create_rowset(nullptr, "", output_meta, &output_rowset).ok()); + + std::vector input_rowsets; + Version last_delete_version {-1, -1}; + tablet->_cumulative_compaction_policy->update_cumulative_point( + tablet.get(), input_rowsets, output_rowset, last_delete_version); + + EXPECT_EQ(1, tablet->cumulative_layer_point()); +} + TEST_F(TestSizeBasedCumulativeCompactionPolicy, pick_input_rowsets_large_head_single_overlapping_tail_selected) { std::vector rs_metas; diff --git a/regression-test/suites/compaction/test_compaction_update_big_delete_bitmap.groovy b/regression-test/suites/compaction/test_compaction_update_big_delete_bitmap.groovy index 0a24e637cce1f2..d2a40b0eedf385 100644 --- a/regression-test/suites/compaction/test_compaction_update_big_delete_bitmap.groovy +++ b/regression-test/suites/compaction/test_compaction_update_big_delete_bitmap.groovy @@ -175,10 +175,8 @@ suite("test_compaction_update_big_delete_bitmap", "nonConcurrent") { try { // store the original value - get_be_param("compaction_promotion_version_count") get_be_param("cumulative_compaction_max_deltas") get_be_param("cumulative_compaction_min_deltas") - set_be_param("compaction_promotion_version_count", "5") set_be_param("cumulative_compaction_max_deltas", "3") set_be_param("cumulative_compaction_min_deltas", "2") @@ -235,7 +233,6 @@ suite("test_compaction_update_big_delete_bitmap", "nonConcurrent") { } } finally { - reset_be_param("compaction_promotion_version_count") reset_be_param("cumulative_compaction_max_deltas") reset_be_param("cumulative_compaction_min_deltas") GetDebugPoint().disableDebugPointForAllBEs("CloudMetaMgr::test_update_big_delete_bitmap") diff --git a/regression-test/suites/compaction/test_filecache_compaction_and_read_stale_cloud_docker.groovy b/regression-test/suites/compaction/test_filecache_compaction_and_read_stale_cloud_docker.groovy index 97d8d7dfae9f3e..21e90b1348c46e 100644 --- a/regression-test/suites/compaction/test_filecache_compaction_and_read_stale_cloud_docker.groovy +++ b/regression-test/suites/compaction/test_filecache_compaction_and_read_stale_cloud_docker.groovy @@ -27,7 +27,6 @@ suite("test_filecache_compaction_and_read_stale_cloud_docker", "docker") { options.setBeNum(1) options.enableDebugPoints() options.feConfigs.add("enable_workload_group=false") - options.beConfigs.add('compaction_promotion_version_count=5') options.beConfigs.add('tablet_rowset_stale_sweep_time_sec=0') options.beConfigs.add('vacuum_stale_rowsets_interval_s=10') options.beConfigs.add('enable_java_support=false') diff --git a/regression-test/suites/compaction/test_filecache_compaction_multisegments_and_read_stale_cloud_docker.groovy b/regression-test/suites/compaction/test_filecache_compaction_multisegments_and_read_stale_cloud_docker.groovy index 956a70c382e06c..924040656abec4 100644 --- a/regression-test/suites/compaction/test_filecache_compaction_multisegments_and_read_stale_cloud_docker.groovy +++ b/regression-test/suites/compaction/test_filecache_compaction_multisegments_and_read_stale_cloud_docker.groovy @@ -27,7 +27,6 @@ suite("test_filecache_compaction_multisegments_and_read_stale_cloud_docker", "do options.setBeNum(1) options.enableDebugPoints() options.feConfigs.add("enable_workload_group=false") - options.beConfigs.add('compaction_promotion_version_count=5') options.beConfigs.add('tablet_rowset_stale_sweep_time_sec=0') options.beConfigs.add('vacuum_stale_rowsets_interval_s=10') options.beConfigs.add('enable_java_support=false') diff --git a/regression-test/suites/compaction/test_mow_compaction_agg_and_remove_pre_delete_bitmap.groovy b/regression-test/suites/compaction/test_mow_compaction_agg_and_remove_pre_delete_bitmap.groovy index bcff773e764c21..bee95f77e999a7 100644 --- a/regression-test/suites/compaction/test_mow_compaction_agg_and_remove_pre_delete_bitmap.groovy +++ b/regression-test/suites/compaction/test_mow_compaction_agg_and_remove_pre_delete_bitmap.groovy @@ -168,12 +168,10 @@ suite("test_mow_compaction_agg_and_remove_pre_delete_bitmap", "nonConcurrent") { GetDebugPoint().clearDebugPointsForAllBEs() get_be_param("tablet_rowset_stale_sweep_time_sec") - get_be_param("compaction_promotion_version_count") get_be_param("enable_agg_and_remove_pre_rowsets_delete_bitmap") try { set_be_param("tablet_rowset_stale_sweep_time_sec", "0") - set_be_param("compaction_promotion_version_count", "5") set_be_param("enable_agg_and_remove_pre_rowsets_delete_bitmap", "true") // solution 2 def testTable = "test_mow_compaction" @@ -332,7 +330,6 @@ suite("test_mow_compaction_agg_and_remove_pre_delete_bitmap", "nonConcurrent") { logger.info("local_dm 2.2: " + local_dm) } finally { reset_be_param("tablet_rowset_stale_sweep_time_sec") - reset_be_param("compaction_promotion_version_count") reset_be_param("enable_agg_and_remove_pre_rowsets_delete_bitmap") GetDebugPoint().clearDebugPointsForAllBEs() } diff --git a/regression-test/suites/compaction/test_mow_compaction_and_read_stale.groovy b/regression-test/suites/compaction/test_mow_compaction_and_read_stale.groovy index 75393c5286137f..bc526e92630afa 100644 --- a/regression-test/suites/compaction/test_mow_compaction_and_read_stale.groovy +++ b/regression-test/suites/compaction/test_mow_compaction_and_read_stale.groovy @@ -208,9 +208,7 @@ suite("test_mow_compaction_and_read_stale", "nonConcurrent") { String tablet_id = tablet.TabletId GetDebugPoint().clearDebugPointsForAllBEs() - get_be_param("compaction_promotion_version_count") get_be_param("tablet_rowset_stale_sweep_time_sec") - set_be_param("compaction_promotion_version_count", "5") set_be_param("tablet_rowset_stale_sweep_time_sec", "0") try { @@ -308,7 +306,6 @@ suite("test_mow_compaction_and_read_stale", "nonConcurrent") { assertTrue(is_local_dm_deleted, "delete bitmap of compaction2 stale rowsets are not deleted") order_qt_sql5 "select * from ${testTable}" } finally { - reset_be_param("compaction_promotion_version_count") reset_be_param("tablet_rowset_stale_sweep_time_sec") GetDebugPoint().clearDebugPointsForAllBEs() } diff --git a/regression-test/suites/compaction/test_mow_compaction_and_read_stale_cloud_docker.groovy b/regression-test/suites/compaction/test_mow_compaction_and_read_stale_cloud_docker.groovy index 1cad3818306a89..1819ef5957ef63 100644 --- a/regression-test/suites/compaction/test_mow_compaction_and_read_stale_cloud_docker.groovy +++ b/regression-test/suites/compaction/test_mow_compaction_and_read_stale_cloud_docker.groovy @@ -26,7 +26,6 @@ suite("test_mow_compaction_and_read_stale_cloud_docker", "docker") { options.setBeNum(1) options.enableDebugPoints() options.feConfigs.add("enable_workload_group=false") - options.beConfigs.add('compaction_promotion_version_count=5') options.beConfigs.add('tablet_rowset_stale_sweep_time_sec=0') options.beConfigs.add('vacuum_stale_rowsets_interval_s=10') options.beConfigs.add('enable_java_support=false') diff --git a/regression-test/suites/compaction/test_mow_compaction_and_rowset_not_exist.groovy b/regression-test/suites/compaction/test_mow_compaction_and_rowset_not_exist.groovy index f0ac88db49f018..198da5f5a42d5c 100644 --- a/regression-test/suites/compaction/test_mow_compaction_and_rowset_not_exist.groovy +++ b/regression-test/suites/compaction/test_mow_compaction_and_rowset_not_exist.groovy @@ -202,9 +202,7 @@ suite("test_mow_compaction_and_rowset_not_exist", "nonConcurrent") { } GetDebugPoint().clearDebugPointsForAllBEs() - get_be_param("compaction_promotion_version_count") get_be_param("tablet_rowset_stale_sweep_time_sec") - set_be_param("compaction_promotion_version_count", "5") set_be_param("tablet_rowset_stale_sweep_time_sec", "0") try { @@ -321,7 +319,6 @@ suite("test_mow_compaction_and_rowset_not_exist", "nonConcurrent") { // check ms update_delete_bitmap log contains skip // check recycler checker abnormal_rowsets_num=0 } finally { - reset_be_param("compaction_promotion_version_count") reset_be_param("tablet_rowset_stale_sweep_time_sec") GetDebugPoint().clearDebugPointsForAllBEs() } diff --git a/regression-test/suites/compaction/test_mow_delete_unused_rowset_dm_docker.groovy b/regression-test/suites/compaction/test_mow_delete_unused_rowset_dm_docker.groovy index 9002a6366f42fe..9fcaf032536781 100644 --- a/regression-test/suites/compaction/test_mow_delete_unused_rowset_dm_docker.groovy +++ b/regression-test/suites/compaction/test_mow_delete_unused_rowset_dm_docker.groovy @@ -28,7 +28,6 @@ suite("test_mow_delete_unused_rowset_dm_docker", "docker") { options.enableDebugPoints() options.feConfigs.add("enable_workload_group=false") // beConfigs - options.beConfigs.add('compaction_promotion_version_count=5') options.beConfigs.add('tablet_rowset_stale_sweep_time_sec=0') options.beConfigs.add('enable_mow_verbose_log=true') options.beConfigs.add('enable_java_support=false') From 3cb81f90bd57e2b4d41f21f29a0be5378f03c014 Mon Sep 17 00:00:00 2001 From: Yukang-Lian Date: Tue, 1 Sep 2026 18:00:13 +0800 Subject: [PATCH 2/2] [improvement](compaction) Keep MOW version promotion configurable ### What problem does this PR solve? Issue Number: None Related PR: #49383 Problem Summary: The MOW version-count promotion protects against excessive delete bitmap metadata, but its default threshold of 1000 can promote tiny cumulative output rowsets and trigger frequent base compactions. Retain the local and cloud promotion logic as an operational fallback, while changing the default threshold to the maximum int64 value so version count alone does not promote rowsets by default. Operators can lower compaction_promotion_version_count if the fallback is needed. ### Release note MOW cumulative compaction no longer promotes output rowsets by version count under the default configuration. The existing configuration remains available as a fallback. ### Check List (For Author) - Test: Unit Test - ./run-be-ut.sh --run --filter=TestSizeBasedCumulativeCompactionPolicy.*:TestCloudSizeBasedCumulativeCompactionPolicy.* -j80 - build-support/check-format.sh - build-support/check-build-hygiene.sh - Behavior changed: Yes. Version-count promotion is disabled by default but remains configurable. - Does this need documentation: No --- be/src/cloud/cloud_cumulative_compaction_policy.cpp | 11 ++++++++++- be/src/common/config.cpp | 6 ++++++ be/src/common/config.h | 6 ++++++ .../compaction/cumulative_compaction_policy.cpp | 12 +++++++++++- .../compaction/cumulative_compaction_policy.h | 3 +++ .../cloud_cumulative_compaction_policy_test.cpp | 2 +- .../compaction/cumulative_compaction_policy_test.cpp | 2 +- .../test_compaction_update_big_delete_bitmap.groovy | 3 +++ ...che_compaction_and_read_stale_cloud_docker.groovy | 1 + ..._multisegments_and_read_stale_cloud_docker.groovy | 1 + ...ompaction_agg_and_remove_pre_delete_bitmap.groovy | 3 +++ .../test_mow_compaction_and_read_stale.groovy | 3 +++ ...mow_compaction_and_read_stale_cloud_docker.groovy | 1 + .../test_mow_compaction_and_rowset_not_exist.groovy | 3 +++ .../test_mow_delete_unused_rowset_dm_docker.groovy | 1 + 15 files changed, 54 insertions(+), 4 deletions(-) diff --git a/be/src/cloud/cloud_cumulative_compaction_policy.cpp b/be/src/cloud/cloud_cumulative_compaction_policy.cpp index 52d8fbdd1f50d3..890527b7e173c5 100644 --- a/be/src/cloud/cloud_cumulative_compaction_policy.cpp +++ b/be/src/cloud/cloud_cumulative_compaction_policy.cpp @@ -324,10 +324,19 @@ int64_t CloudSizeBasedCumulativeCompactionPolicy::new_cumulative_point( return cumu_point; } }); + // for MoW table, if there's too many versions, the delete bitmap will grow to + // a very big size, which may cause the tablet meta too big and the `save_meta` + // operation too slow. + // if the rowset should not promotion according to it's disk size, we should also + // consider it's version count here. + bool satisfy_promotion_version = tablet->enable_unique_key_merge_on_write() && + output_rowset->end_version() - output_rowset->start_version() > + config::compaction_promotion_version_count; // if rowsets have delete version, move to the last directly. // if rowsets have no delete version, check output_rowset total disk size satisfies promotion size. return (last_delete_version.first != -1 || - output_rowset->total_disk_size() >= cloud_promotion_size(tablet)) + output_rowset->total_disk_size() >= cloud_promotion_size(tablet) || + satisfy_promotion_version) ? output_rowset->end_version() + 1 : last_cumulative_point; } diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index 5f91f56bb1d986..474f74433b501e 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -523,6 +523,12 @@ DEFINE_mDouble(compaction_promotion_ratio, "0.05"); // rowset will be not given to base compaction. The unit is m byte. DEFINE_mInt64(compaction_promotion_min_size_mbytes, "128"); +// When output rowset of cumulative compaction total version count (end_version - start_version) +// exceed this config count, the rowset will be moved to base compaction. +// NOTE: this config only works for unique key merge-on-write tables. The default maximum value +// disables version-count-based promotion while retaining the option to enable it when needed. +DEFINE_mInt64(compaction_promotion_version_count, "9223372036854775807"); + // The lower bound size to do cumulative compaction. When total disk size of candidate rowsets is less than // this size, size_based policy may not do to cumulative compaction. The unit is m byte. DEFINE_mInt64(compaction_min_size_mbytes, "64"); diff --git a/be/src/common/config.h b/be/src/common/config.h index a04ace12c14a11..1c93d4a4f39ee3 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -613,6 +613,12 @@ DECLARE_mDouble(compaction_promotion_ratio); // rowset will be not given to base compaction. The unit is m byte. DECLARE_mInt64(compaction_promotion_min_size_mbytes); +// When output rowset of cumulative compaction total version count (end_version - start_version) +// exceed this config count, the rowset will be moved to base compaction. +// NOTE: this config only works for unique key merge-on-write tables. The default maximum value +// disables version-count-based promotion while retaining the option to enable it when needed. +DECLARE_mInt64(compaction_promotion_version_count); + // The lower bound size to do cumulative compaction. When total disk size of candidate rowsets is less than // this size, size_based policy may not do to cumulative compaction. The unit is m byte. DECLARE_mInt64(compaction_min_size_mbytes); diff --git a/be/src/storage/compaction/cumulative_compaction_policy.cpp b/be/src/storage/compaction/cumulative_compaction_policy.cpp index d10b60a167a279..07e7488d8e5010 100644 --- a/be/src/storage/compaction/cumulative_compaction_policy.cpp +++ b/be/src/storage/compaction/cumulative_compaction_policy.cpp @@ -39,10 +39,11 @@ namespace doris { SizeBasedCumulativeCompactionPolicy::SizeBasedCumulativeCompactionPolicy( int64_t promotion_size, double promotion_ratio, int64_t promotion_min_size, - int64_t compaction_min_size) + int64_t promotion_version_count, int64_t compaction_min_size) : _promotion_size(promotion_size), _promotion_ratio(promotion_ratio), _promotion_min_size(promotion_min_size), + _promotion_version_count(promotion_version_count), _compaction_min_size(compaction_min_size) {} void SizeBasedCumulativeCompactionPolicy::calculate_cumulative_point( @@ -158,6 +159,15 @@ void SizeBasedCumulativeCompactionPolicy::update_cumulative_point( size_t total_size = output_rowset->rowset_meta()->total_disk_size(); if (total_size >= tablet->cumulative_promotion_size()) { tablet->set_cumulative_layer_point(output_rowset->end_version() + 1); + } else if (tablet->enable_unique_key_merge_on_write() && + output_rowset->end_version() - output_rowset->start_version() > + _promotion_version_count) { + // for MoW table, if there's too many versions, the delete bitmap will grow to + // a very big size, which may cause the tablet meta too big and the `save_meta` + // operation too slow. + // if the rowset should not promotion according to it's disk size, we should also + // consider it's version count here. + tablet->set_cumulative_layer_point(output_rowset->end_version() + 1); } } } diff --git a/be/src/storage/compaction/cumulative_compaction_policy.h b/be/src/storage/compaction/cumulative_compaction_policy.h index 377e6e92f866ba..45ea6265c1087a 100644 --- a/be/src/storage/compaction/cumulative_compaction_policy.h +++ b/be/src/storage/compaction/cumulative_compaction_policy.h @@ -119,6 +119,7 @@ class SizeBasedCumulativeCompactionPolicy final : public CumulativeCompactionPol int64_t promotion_size = config::compaction_promotion_size_mbytes * 1024 * 1024, double promotion_ratio = config::compaction_promotion_ratio, int64_t promotion_min_size = config::compaction_promotion_min_size_mbytes * 1024 * 1024, + int64_t promotion_version_count = config::compaction_promotion_version_count, int64_t compaction_min_size = config::compaction_min_size_mbytes * 1024 * 1024); /// Destructor function of SizeBasedCumulativeCompactionPolicy. @@ -178,6 +179,8 @@ class SizeBasedCumulativeCompactionPolicy final : public CumulativeCompactionPol double _promotion_ratio; /// cumulative compaction promotion min size, unit is byte. int64_t _promotion_min_size; + // cululative compaction promotion version count, only works for unique key MoW table + int64_t _promotion_version_count; /// lower bound size to do compaction compaction. int64_t _compaction_min_size; }; diff --git a/be/test/cloud/cloud_cumulative_compaction_policy_test.cpp b/be/test/cloud/cloud_cumulative_compaction_policy_test.cpp index 30a8695e9586ef..0bb4dbfc3bf63d 100644 --- a/be/test/cloud/cloud_cumulative_compaction_policy_test.cpp +++ b/be/test/cloud/cloud_cumulative_compaction_policy_test.cpp @@ -213,7 +213,7 @@ TEST_F(TestCloudSizeBasedCumulativeCompactionPolicy, new_cumulative_point) { } TEST_F(TestCloudSizeBasedCumulativeCompactionPolicy, - new_cumulative_point_does_not_promote_small_mow_rowset_by_version_count) { + new_cumulative_point_does_not_promote_small_mow_rowset_with_default_version_limit) { CloudTablet tablet(_engine, _tablet_meta); tablet._tablet_meta->_enable_unique_key_merge_on_write = true; tablet._base_size = kGiB; diff --git a/be/test/storage/compaction/cumulative_compaction_policy_test.cpp b/be/test/storage/compaction/cumulative_compaction_policy_test.cpp index 11800833bdc9cc..a82c52719b1b86 100644 --- a/be/test/storage/compaction/cumulative_compaction_policy_test.cpp +++ b/be/test/storage/compaction/cumulative_compaction_policy_test.cpp @@ -1239,7 +1239,7 @@ TEST_F(TestSizeBasedCumulativeCompactionPolicy, } TEST_F(TestSizeBasedCumulativeCompactionPolicy, - update_cumulative_point_does_not_promote_small_mow_rowset_by_version_count) { + update_cumulative_point_does_not_promote_small_mow_rowset_with_default_version_limit) { _tablet_meta->set_enable_unique_key_merge_on_write(true); TabletSharedPtr tablet( new Tablet(_engine, _tablet_meta, nullptr, CUMULATIVE_SIZE_BASED_POLICY)); diff --git a/regression-test/suites/compaction/test_compaction_update_big_delete_bitmap.groovy b/regression-test/suites/compaction/test_compaction_update_big_delete_bitmap.groovy index d2a40b0eedf385..0a24e637cce1f2 100644 --- a/regression-test/suites/compaction/test_compaction_update_big_delete_bitmap.groovy +++ b/regression-test/suites/compaction/test_compaction_update_big_delete_bitmap.groovy @@ -175,8 +175,10 @@ suite("test_compaction_update_big_delete_bitmap", "nonConcurrent") { try { // store the original value + get_be_param("compaction_promotion_version_count") get_be_param("cumulative_compaction_max_deltas") get_be_param("cumulative_compaction_min_deltas") + set_be_param("compaction_promotion_version_count", "5") set_be_param("cumulative_compaction_max_deltas", "3") set_be_param("cumulative_compaction_min_deltas", "2") @@ -233,6 +235,7 @@ suite("test_compaction_update_big_delete_bitmap", "nonConcurrent") { } } finally { + reset_be_param("compaction_promotion_version_count") reset_be_param("cumulative_compaction_max_deltas") reset_be_param("cumulative_compaction_min_deltas") GetDebugPoint().disableDebugPointForAllBEs("CloudMetaMgr::test_update_big_delete_bitmap") diff --git a/regression-test/suites/compaction/test_filecache_compaction_and_read_stale_cloud_docker.groovy b/regression-test/suites/compaction/test_filecache_compaction_and_read_stale_cloud_docker.groovy index 21e90b1348c46e..97d8d7dfae9f3e 100644 --- a/regression-test/suites/compaction/test_filecache_compaction_and_read_stale_cloud_docker.groovy +++ b/regression-test/suites/compaction/test_filecache_compaction_and_read_stale_cloud_docker.groovy @@ -27,6 +27,7 @@ suite("test_filecache_compaction_and_read_stale_cloud_docker", "docker") { options.setBeNum(1) options.enableDebugPoints() options.feConfigs.add("enable_workload_group=false") + options.beConfigs.add('compaction_promotion_version_count=5') options.beConfigs.add('tablet_rowset_stale_sweep_time_sec=0') options.beConfigs.add('vacuum_stale_rowsets_interval_s=10') options.beConfigs.add('enable_java_support=false') diff --git a/regression-test/suites/compaction/test_filecache_compaction_multisegments_and_read_stale_cloud_docker.groovy b/regression-test/suites/compaction/test_filecache_compaction_multisegments_and_read_stale_cloud_docker.groovy index 924040656abec4..956a70c382e06c 100644 --- a/regression-test/suites/compaction/test_filecache_compaction_multisegments_and_read_stale_cloud_docker.groovy +++ b/regression-test/suites/compaction/test_filecache_compaction_multisegments_and_read_stale_cloud_docker.groovy @@ -27,6 +27,7 @@ suite("test_filecache_compaction_multisegments_and_read_stale_cloud_docker", "do options.setBeNum(1) options.enableDebugPoints() options.feConfigs.add("enable_workload_group=false") + options.beConfigs.add('compaction_promotion_version_count=5') options.beConfigs.add('tablet_rowset_stale_sweep_time_sec=0') options.beConfigs.add('vacuum_stale_rowsets_interval_s=10') options.beConfigs.add('enable_java_support=false') diff --git a/regression-test/suites/compaction/test_mow_compaction_agg_and_remove_pre_delete_bitmap.groovy b/regression-test/suites/compaction/test_mow_compaction_agg_and_remove_pre_delete_bitmap.groovy index bee95f77e999a7..bcff773e764c21 100644 --- a/regression-test/suites/compaction/test_mow_compaction_agg_and_remove_pre_delete_bitmap.groovy +++ b/regression-test/suites/compaction/test_mow_compaction_agg_and_remove_pre_delete_bitmap.groovy @@ -168,10 +168,12 @@ suite("test_mow_compaction_agg_and_remove_pre_delete_bitmap", "nonConcurrent") { GetDebugPoint().clearDebugPointsForAllBEs() get_be_param("tablet_rowset_stale_sweep_time_sec") + get_be_param("compaction_promotion_version_count") get_be_param("enable_agg_and_remove_pre_rowsets_delete_bitmap") try { set_be_param("tablet_rowset_stale_sweep_time_sec", "0") + set_be_param("compaction_promotion_version_count", "5") set_be_param("enable_agg_and_remove_pre_rowsets_delete_bitmap", "true") // solution 2 def testTable = "test_mow_compaction" @@ -330,6 +332,7 @@ suite("test_mow_compaction_agg_and_remove_pre_delete_bitmap", "nonConcurrent") { logger.info("local_dm 2.2: " + local_dm) } finally { reset_be_param("tablet_rowset_stale_sweep_time_sec") + reset_be_param("compaction_promotion_version_count") reset_be_param("enable_agg_and_remove_pre_rowsets_delete_bitmap") GetDebugPoint().clearDebugPointsForAllBEs() } diff --git a/regression-test/suites/compaction/test_mow_compaction_and_read_stale.groovy b/regression-test/suites/compaction/test_mow_compaction_and_read_stale.groovy index bc526e92630afa..75393c5286137f 100644 --- a/regression-test/suites/compaction/test_mow_compaction_and_read_stale.groovy +++ b/regression-test/suites/compaction/test_mow_compaction_and_read_stale.groovy @@ -208,7 +208,9 @@ suite("test_mow_compaction_and_read_stale", "nonConcurrent") { String tablet_id = tablet.TabletId GetDebugPoint().clearDebugPointsForAllBEs() + get_be_param("compaction_promotion_version_count") get_be_param("tablet_rowset_stale_sweep_time_sec") + set_be_param("compaction_promotion_version_count", "5") set_be_param("tablet_rowset_stale_sweep_time_sec", "0") try { @@ -306,6 +308,7 @@ suite("test_mow_compaction_and_read_stale", "nonConcurrent") { assertTrue(is_local_dm_deleted, "delete bitmap of compaction2 stale rowsets are not deleted") order_qt_sql5 "select * from ${testTable}" } finally { + reset_be_param("compaction_promotion_version_count") reset_be_param("tablet_rowset_stale_sweep_time_sec") GetDebugPoint().clearDebugPointsForAllBEs() } diff --git a/regression-test/suites/compaction/test_mow_compaction_and_read_stale_cloud_docker.groovy b/regression-test/suites/compaction/test_mow_compaction_and_read_stale_cloud_docker.groovy index 1819ef5957ef63..1cad3818306a89 100644 --- a/regression-test/suites/compaction/test_mow_compaction_and_read_stale_cloud_docker.groovy +++ b/regression-test/suites/compaction/test_mow_compaction_and_read_stale_cloud_docker.groovy @@ -26,6 +26,7 @@ suite("test_mow_compaction_and_read_stale_cloud_docker", "docker") { options.setBeNum(1) options.enableDebugPoints() options.feConfigs.add("enable_workload_group=false") + options.beConfigs.add('compaction_promotion_version_count=5') options.beConfigs.add('tablet_rowset_stale_sweep_time_sec=0') options.beConfigs.add('vacuum_stale_rowsets_interval_s=10') options.beConfigs.add('enable_java_support=false') diff --git a/regression-test/suites/compaction/test_mow_compaction_and_rowset_not_exist.groovy b/regression-test/suites/compaction/test_mow_compaction_and_rowset_not_exist.groovy index 198da5f5a42d5c..f0ac88db49f018 100644 --- a/regression-test/suites/compaction/test_mow_compaction_and_rowset_not_exist.groovy +++ b/regression-test/suites/compaction/test_mow_compaction_and_rowset_not_exist.groovy @@ -202,7 +202,9 @@ suite("test_mow_compaction_and_rowset_not_exist", "nonConcurrent") { } GetDebugPoint().clearDebugPointsForAllBEs() + get_be_param("compaction_promotion_version_count") get_be_param("tablet_rowset_stale_sweep_time_sec") + set_be_param("compaction_promotion_version_count", "5") set_be_param("tablet_rowset_stale_sweep_time_sec", "0") try { @@ -319,6 +321,7 @@ suite("test_mow_compaction_and_rowset_not_exist", "nonConcurrent") { // check ms update_delete_bitmap log contains skip // check recycler checker abnormal_rowsets_num=0 } finally { + reset_be_param("compaction_promotion_version_count") reset_be_param("tablet_rowset_stale_sweep_time_sec") GetDebugPoint().clearDebugPointsForAllBEs() } diff --git a/regression-test/suites/compaction/test_mow_delete_unused_rowset_dm_docker.groovy b/regression-test/suites/compaction/test_mow_delete_unused_rowset_dm_docker.groovy index 9fcaf032536781..9002a6366f42fe 100644 --- a/regression-test/suites/compaction/test_mow_delete_unused_rowset_dm_docker.groovy +++ b/regression-test/suites/compaction/test_mow_delete_unused_rowset_dm_docker.groovy @@ -28,6 +28,7 @@ suite("test_mow_delete_unused_rowset_dm_docker", "docker") { options.enableDebugPoints() options.feConfigs.add("enable_workload_group=false") // beConfigs + options.beConfigs.add('compaction_promotion_version_count=5') options.beConfigs.add('tablet_rowset_stale_sweep_time_sec=0') options.beConfigs.add('enable_mow_verbose_log=true') options.beConfigs.add('enable_java_support=false')