From 22f494a3f51154474f34dcbe162f84af49ec2437 Mon Sep 17 00:00:00 2001 From: liaoxin Date: Mon, 20 Jul 2026 00:44:24 +0800 Subject: [PATCH 01/22] [improvement](s3) CPU-aware object storage rate limiter with unified decorator Redesign the S3 GET/PUT rate limiting path: - New configs s3_{get,put}_qps_per_core / _qps_max and s3_{get,put}_bytes_per_second_per_core / _max derive per-BE limits from CPU cores (cgroup-quota aware), so heterogeneous BEs get proportional budgets. -1 keeps the legacy absolute token configs bit-for-bit. - All rate limiting now lives in one decorator, RateLimitedObjStorageClient, wrapped by S3ClientFactory at construction time. Provider clients (S3, Azure, future GCP) contain no rate limiting code anymore. - In cloud mode only internal storage-vault buckets are limited; external buckets (S3 load, TVF, external catalogs) get the bare client. Non-cloud mode wraps every client, preserving legacy behavior. - New bytes-per-second buckets charge payload size with reserve+settle accounting: short reads are refunded, single reservations are clamped to 1s of bandwidth, and the guard pins the exact bucket generation it charged so a concurrent limiter reset cannot make the refund pollute a fresh bucket. The derived per-BE bytes/s should not be set below the single IO upper bound per second (s3_write_buffer_size); documented on the configs. - S3RateLimiterManager::refresh() is idempotent and driven by a daemon thread, picking up both dynamic config changes (now effective in non-cloud mode too) and in-place cgroup CPU resizes for serverless BEs; a positive s3_rate_limiter_cpu_cores overrides detection for control-plane pushes. - Fix S3 client cache and ObjClientHolder::reset treating get_hash() equality as configuration identity (XOR of crc32s collides easily). - Fix TokenBucketRateLimiterHolder sleeping while holding the read lock, which blocked reset() for the whole throttle duration. --- be/src/cloud/cloud_storage_engine.cpp | 12 +- be/src/common/config.cpp | 38 +++ be/src/common/config.h | 16 + be/src/common/daemon.cpp | 17 + be/src/common/daemon.h | 1 + be/src/io/fs/azure_obj_storage_client.cpp | 67 ++-- .../io/fs/rate_limited_obj_storage_client.cpp | 139 ++++++++ .../io/fs/rate_limited_obj_storage_client.h | 70 ++++ be/src/io/fs/s3_file_system.cpp | 13 +- be/src/io/fs/s3_obj_storage_client.cpp | 70 ++-- be/src/util/s3_rate_limiter_manager.cpp | 204 +++++++++++ be/src/util/s3_rate_limiter_manager.h | 113 +++++++ be/src/util/s3_util.cpp | 102 +----- be/src/util/s3_util.h | 32 +- be/test/io/client/s3_file_system_test.cpp | 30 +- .../rate_limited_obj_storage_client_test.cpp | 193 +++++++++++ be/test/util/s3_rate_limiter_manager_test.cpp | 317 ++++++++++++++++++ be/test/util/s3_util_test.cpp | 28 +- common/cpp/token_bucket_rate_limiter.cpp | 39 ++- common/cpp/token_bucket_rate_limiter.h | 20 +- 20 files changed, 1288 insertions(+), 233 deletions(-) create mode 100644 be/src/io/fs/rate_limited_obj_storage_client.cpp create mode 100644 be/src/io/fs/rate_limited_obj_storage_client.h create mode 100644 be/src/util/s3_rate_limiter_manager.cpp create mode 100644 be/src/util/s3_rate_limiter_manager.h create mode 100644 be/test/io/fs/rate_limited_obj_storage_client_test.cpp create mode 100644 be/test/util/s3_rate_limiter_manager_test.cpp diff --git a/be/src/cloud/cloud_storage_engine.cpp b/be/src/cloud/cloud_storage_engine.cpp index 0dc8321c2c4e3a..bb1ea005665b36 100644 --- a/be/src/cloud/cloud_storage_engine.cpp +++ b/be/src/cloud/cloud_storage_engine.cpp @@ -453,15 +453,9 @@ void CloudStorageEngine::_refresh_storage_vault_info_thread_callback() { while (!_stop_background_threads_latch.wait_for( std::chrono::seconds(config::refresh_s3_info_interval_s))) { sync_storage_vault(); - // The other place that rebuilds the S3 rate limiter is S3ClientFactory::create(), which - // is not called when an existing vault's conf is unchanged. Trigger the check here as well - // so that dynamically modified s3_{get,put}_* rate limiter configs take effect within - // refresh_s3_info_interval_s even when no vault is created or its conf does not change. - // Gate it behind enable_s3_rate_limiter so that clusters with rate limiting disabled - // (e.g. HDFS-only vaults) do not force-initialize S3ClientFactory / the AWS SDK here. - if (config::enable_s3_rate_limiter) { - check_s3_rate_limiter_config_changed(); - } + // Dynamically modified s3_{get,put}_* rate limiter configs and cgroup CPU quota + // changes are picked up by the daemon's s3_rate_limiter_refresh_thread, which + // runs in both cloud and non-cloud mode. } } diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index 9f697a2dcbed81..05871682376f41 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -1543,6 +1543,44 @@ DEFINE_mInt64(s3_put_token_limit, "0"); DEFINE_mInt64(s3_rate_limiter_log_interval, "1000"); DEFINE_Validator(s3_rate_limiter_log_interval, [](int64_t config) -> bool { return config >= 0; }); +// CPU-aware S3 rate limiter. Effective GET/PUT QPS = qps_per_core * BE cpu cores, capped by +// the corresponding qps_max. -1 means unset: fall back to the legacy absolute +// s3_{get,put}_token_* configs above. 0 disables QPS limiting for that operation. +DEFINE_mInt64(s3_get_qps_per_core, "-1"); +DEFINE_Validator(s3_get_qps_per_core, [](int64_t config) -> bool { return config >= -1; }); +DEFINE_mInt64(s3_put_qps_per_core, "-1"); +DEFINE_Validator(s3_put_qps_per_core, [](int64_t config) -> bool { return config >= -1; }); +// Hard caps for the CPU-derived GET/PUT QPS. 0 means no cap. +DEFINE_mInt64(s3_get_qps_max, "0"); +DEFINE_Validator(s3_get_qps_max, [](int64_t config) -> bool { return config >= 0; }); +DEFINE_mInt64(s3_put_qps_max, "0"); +DEFINE_Validator(s3_put_qps_max, [](int64_t config) -> bool { return config >= 0; }); + +// CPU-aware S3 bandwidth limiter. Effective GET/PUT bytes/s = bytes_per_second_per_core * +// BE cpu cores, capped by the corresponding bytes_per_second_max. -1 and 0 both disable +// byte-rate limiting for that operation (there is no legacy fallback for bandwidth). +// Note: the derived per-BE bytes/s should not be set below the single IO upper bound +// per second (s3_write_buffer_size, 5MB by default). A single IO larger than 1 second +// of quota only reserves 1 second worth of tokens; the excess bytes are not accounted +// (reservation clamp in S3RateLimitGuard). +DEFINE_mInt64(s3_get_bytes_per_second_per_core, "-1"); +DEFINE_Validator(s3_get_bytes_per_second_per_core, + [](int64_t config) -> bool { return config >= -1; }); +DEFINE_mInt64(s3_put_bytes_per_second_per_core, "-1"); +DEFINE_Validator(s3_put_bytes_per_second_per_core, + [](int64_t config) -> bool { return config >= -1; }); +// Hard caps for the CPU-derived GET/PUT bytes/s. 0 means no cap. +DEFINE_mInt64(s3_get_bytes_per_second_max, "0"); +DEFINE_Validator(s3_get_bytes_per_second_max, [](int64_t config) -> bool { return config >= 0; }); +DEFINE_mInt64(s3_put_bytes_per_second_max, "0"); +DEFINE_Validator(s3_put_bytes_per_second_max, [](int64_t config) -> bool { return config >= 0; }); + +// CPU cores used to derive the effective S3 rate limits. 0 means auto-detect from the +// cgroup cpu quota (fall back to physical cores). A positive value overrides detection; +// the control plane can push it via /api/update_config when resizing a serverless BE. +DEFINE_mInt64(s3_rate_limiter_cpu_cores, "0"); +DEFINE_Validator(s3_rate_limiter_cpu_cores, [](int64_t config) -> bool { return config >= 0; }); + DEFINE_String(trino_connector_plugin_dir, "${DORIS_HOME}/plugins/connectors"); // ca_cert_file is in this path by default, Normally no modification is required diff --git a/be/src/common/config.h b/be/src/common/config.h index f5c72eb776d5d3..b7fd4b93073753 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -1618,6 +1618,22 @@ DECLARE_mInt64(s3_put_bucket_tokens); DECLARE_mInt64(s3_put_token_per_second); DECLARE_mInt64(s3_put_token_limit); DECLARE_mInt64(s3_rate_limiter_log_interval); + +// CPU-aware S3 rate limiter: GET/PUT QPS per CPU core. -1 = unset, fall back to the +// legacy absolute token configs above; 0 disables QPS limiting for that operation. +DECLARE_mInt64(s3_get_qps_per_core); +DECLARE_mInt64(s3_put_qps_per_core); +// Hard caps for the CPU-derived GET/PUT QPS. 0 means no cap. +DECLARE_mInt64(s3_get_qps_max); +DECLARE_mInt64(s3_put_qps_max); +// GET/PUT bytes per second per CPU core. -1 and 0 both disable byte-rate limiting. +DECLARE_mInt64(s3_get_bytes_per_second_per_core); +DECLARE_mInt64(s3_put_bytes_per_second_per_core); +// Hard caps for the CPU-derived GET/PUT bytes/s. 0 means no cap. +DECLARE_mInt64(s3_get_bytes_per_second_max); +DECLARE_mInt64(s3_put_bytes_per_second_max); +// Cores used to derive effective limits: 0 = auto-detect from cgroup quota; >0 overrides. +DECLARE_mInt64(s3_rate_limiter_cpu_cores); // max s3 client retry times DECLARE_mInt32(max_s3_client_retry); // When meet s3 429 error, the "get" request will diff --git a/be/src/common/daemon.cpp b/be/src/common/daemon.cpp index 54986277356782..9b0c76bdddd2d9 100644 --- a/be/src/common/daemon.cpp +++ b/be/src/common/daemon.cpp @@ -60,6 +60,7 @@ #include "util/algorithm_util.h" #include "util/mem_info.h" #include "util/perf_counters.h" +#include "util/s3_rate_limiter_manager.h" #include "util/time.h" namespace doris { @@ -582,6 +583,18 @@ void Daemon::calculate_workload_group_metrics_thread() { } } +void Daemon::s3_rate_limiter_refresh_thread() { + // Single trigger for dynamic rate limiter changes: picks up both mutable + // s3_{get,put}_* config updates and cgroup CPU quota changes (serverless BEs can + // be resized in place). refresh() is idempotent and compares against the buckets' + // own parameters, so quiet iterations are cheap no-ops. + while (!_stop_background_threads_latch.wait_for(std::chrono::seconds(10))) { + if (config::enable_s3_rate_limiter) { + S3RateLimiterManager::instance().refresh(); + } + } +} + void Daemon::start() { Status st; st = Thread::create( @@ -603,6 +616,10 @@ void Daemon::start() { [this]() { this->calculate_metrics_thread(); }, &_threads.emplace_back()); CHECK(st.ok()) << st; } + st = Thread::create( + "Daemon", "s3_rate_limiter_refresh_thread", + [this]() { this->s3_rate_limiter_refresh_thread(); }, &_threads.emplace_back()); + CHECK(st.ok()) << st; st = Thread::create( "Daemon", "je_reset_dirty_decay_thread", [this]() { this->je_reset_dirty_decay_thread(); }, &_threads.emplace_back()); diff --git a/be/src/common/daemon.h b/be/src/common/daemon.h index e12c25c9cb7eeb..fdc815119f63f7 100644 --- a/be/src/common/daemon.h +++ b/be/src/common/daemon.h @@ -46,6 +46,7 @@ class Daemon { void report_runtime_query_statistics_thread(); void be_proc_monitor_thread(); void calculate_workload_group_metrics_thread(); + void s3_rate_limiter_refresh_thread(); CountDownLatch _stop_background_threads_latch; std::vector> _threads; diff --git a/be/src/io/fs/azure_obj_storage_client.cpp b/be/src/io/fs/azure_obj_storage_client.cpp index 4ac9117fad9ae3..92c99d06409fd6 100644 --- a/be/src/io/fs/azure_obj_storage_client.cpp +++ b/be/src/io/fs/azure_obj_storage_client.cpp @@ -70,27 +70,8 @@ auto base64_encode_part_num(int part_num) { return Aws::Utils::HashingUtils::Base64Encode({buf, sizeof(buf)}); } -template -auto s3_rate_limit(doris::S3RateLimitType op, Func callback) -> decltype(callback()) { - if (!doris::config::enable_s3_rate_limiter) { - return callback(); - } - auto sleep_duration = doris::apply_s3_rate_limit(op); - if (sleep_duration < 0) { - throw std::runtime_error("Azure exceeds request limit"); - } - return callback(); -} - -template -auto s3_get_rate_limit(Func callback) -> decltype(callback()) { - return s3_rate_limit(doris::S3RateLimitType::GET, std::move(callback)); -} - -template -auto s3_put_rate_limit(Func callback) -> decltype(callback()) { - return s3_rate_limit(doris::S3RateLimitType::PUT, std::move(callback)); -} +// Rate limiting is applied by RateLimitedObjStorageClient, the decorator that +// S3ClientFactory wraps around this client when the bucket is subject to limiting. constexpr char SAS_TOKEN_URL_TEMPLATE[] = "{}/{}/{}{}"; constexpr char BlobNotFound[] = "BlobNotFound"; @@ -163,10 +144,10 @@ struct AzureBatchDeleter { } auto resp = do_azure_client_call( [&]() { - s3_put_rate_limit([&]() { + { SCOPED_BVAR_LATENCY(s3_bvar::s3_delete_objects_latency); _client->SubmitBatch(_batch); - }); + } }, _opts, _tls_debug_context); if (resp.status.code != ErrorCode::OK) { @@ -228,11 +209,11 @@ ObjectStorageResponse AzureObjStorageClient::put_object(const ObjectStoragePathO auto client = _client->GetBlockBlobClient(opts.key); return do_azure_client_call( [&]() { - s3_put_rate_limit([&]() { + { SCOPED_BVAR_LATENCY(s3_bvar::s3_put_latency); client.UploadFrom(reinterpret_cast(stream.data()), stream.size()); - }); + } }, opts, _tls_debug_context); } @@ -246,10 +227,10 @@ ObjectStorageUploadResponse AzureObjStorageClient::upload_part(const ObjectStora Azure::Core::IO::MemoryBodyStream memory_body( reinterpret_cast(stream.data()), stream.size()); // The blockId must be base64 encoded - s3_put_rate_limit([&]() { + { SCOPED_BVAR_LATENCY(s3_bvar::s3_multi_part_upload_latency); client.StageBlock(base64_encode_part_num(part_num), memory_body); - }); + } }, opts, _tls_debug_context); return ObjectStorageUploadResponse { @@ -267,10 +248,10 @@ ObjectStorageResponse AzureObjStorageClient::complete_multipart_upload( [](const ObjectCompleteMultiPart& i) { return base64_encode_part_num(i.part_num); }); return do_azure_client_call( [&]() { - s3_put_rate_limit([&]() { + { SCOPED_BVAR_LATENCY(s3_bvar::s3_multi_part_upload_latency); client.CommitBlockList(string_block_ids); - }); + } }, opts, _tls_debug_context); } @@ -279,10 +260,10 @@ ObjectStorageHeadResponse AzureObjStorageClient::head_object(const ObjectStorage Models::BlobProperties properties {}; auto resp = do_azure_client_call( [&]() { - properties = s3_get_rate_limit([&]() { + properties = [&]() { SCOPED_BVAR_LATENCY(s3_bvar::s3_head_latency); return _client->GetBlockBlobClient(opts.key).GetProperties().Value; - }); + }(); }, opts, _tls_debug_context); if (resp.http_code == static_cast(Azure::Core::Http::HttpStatusCode::NotFound)) { @@ -308,11 +289,11 @@ ObjectStorageResponse AzureObjStorageClient::get_object(const ObjectStoragePathO DownloadBlobToOptions download_opts; Azure::Core::Http::HttpRange range {static_cast(offset), bytes_read}; download_opts.Range = range; - auto resp = s3_get_rate_limit([&]() { + auto resp = [&]() { SCOPED_BVAR_LATENCY(s3_bvar::s3_get_latency); return client.DownloadTo(reinterpret_cast(buffer), bytes_read, download_opts); - }); + }(); *size_return = resp.Value.ContentRange.Length.Value(); }, opts, _tls_debug_context); @@ -330,17 +311,17 @@ ObjectStorageResponse AzureObjStorageClient::list_objects(const ObjectStoragePat [&]() { ListBlobsOptions list_opts; list_opts.Prefix = opts.prefix; - auto resp = s3_get_rate_limit([&]() { + auto resp = [&]() { SCOPED_BVAR_LATENCY(s3_bvar::s3_list_latency); return _client->ListBlobs(list_opts); - }); + }(); get_file_file(resp); while (resp.NextPageToken.HasValue()) { list_opts.ContinuationToken = resp.NextPageToken; - resp = s3_get_rate_limit([&]() { + resp = [&]() { SCOPED_BVAR_LATENCY(s3_bvar::s3_list_latency); return _client->ListBlobs(list_opts); - }); + }(); get_file_file(resp); } }, @@ -376,10 +357,10 @@ ObjectStorageResponse AzureObjStorageClient::delete_objects(const ObjectStorageP ObjectStorageResponse AzureObjStorageClient::delete_object(const ObjectStoragePathOptions& opts) { return do_azure_client_call( [&]() { - auto resp = s3_put_rate_limit([&]() { + auto resp = [&]() { SCOPED_BVAR_LATENCY(s3_bvar::s3_delete_object_latency); return _client->DeleteBlob(opts.key); - }); + }(); if (!resp.Value.Deleted) { throw Exception(Status::IOError("Delete azure blob failed")); } @@ -407,10 +388,10 @@ ObjectStorageResponse AzureObjStorageClient::delete_objects_recursively( ListBlobsPagedResponse resp; auto list_resp = do_azure_client_call( [&]() { - resp = s3_get_rate_limit([&]() { + resp = [&]() { SCOPED_BVAR_LATENCY(s3_bvar::s3_list_latency); return _client->ListBlobs(list_opts); - }); + }(); }, opts, _tls_debug_context); if (list_resp.status.code != ErrorCode::OK) { @@ -425,10 +406,10 @@ ObjectStorageResponse AzureObjStorageClient::delete_objects_recursively( list_opts.ContinuationToken = resp.NextPageToken; list_resp = do_azure_client_call( [&]() { - resp = s3_get_rate_limit([&]() { + resp = [&]() { SCOPED_BVAR_LATENCY(s3_bvar::s3_list_latency); return _client->ListBlobs(list_opts); - }); + }(); }, opts, _tls_debug_context); if (list_resp.status.code != ErrorCode::OK) { diff --git a/be/src/io/fs/rate_limited_obj_storage_client.cpp b/be/src/io/fs/rate_limited_obj_storage_client.cpp new file mode 100644 index 00000000000000..b24123b1ab488d --- /dev/null +++ b/be/src/io/fs/rate_limited_obj_storage_client.cpp @@ -0,0 +1,139 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "io/fs/rate_limited_obj_storage_client.h" + +#include "common/status.h" +#include "util/s3_rate_limiter_manager.h" + +namespace doris::io { +namespace { + +ObjectStorageResponse rate_limited_response(S3RateLimitType type) { + return {.status = convert_to_obj_response(Status::Error( + "s3 {} request exceeds request limit, rejected by BE rate limiter", + to_string(type))), + .http_code = 429}; +} + +} // namespace + +ObjectStorageUploadResponse RateLimitedObjStorageClient::create_multipart_upload( + const ObjectStoragePathOptions& opts) { + S3RateLimitGuard guard(S3RateLimitType::PUT, 0); + if (!guard.ok()) { + return {.resp = rate_limited_response(S3RateLimitType::PUT)}; + } + return _inner->create_multipart_upload(opts); +} + +ObjectStorageResponse RateLimitedObjStorageClient::put_object(const ObjectStoragePathOptions& opts, + std::string_view stream) { + S3RateLimitGuard guard(S3RateLimitType::PUT, stream.size()); + if (!guard.ok()) { + return rate_limited_response(S3RateLimitType::PUT); + } + return _inner->put_object(opts, stream); +} + +ObjectStorageUploadResponse RateLimitedObjStorageClient::upload_part( + const ObjectStoragePathOptions& opts, std::string_view stream, int part_num) { + S3RateLimitGuard guard(S3RateLimitType::PUT, stream.size()); + if (!guard.ok()) { + return {.resp = rate_limited_response(S3RateLimitType::PUT)}; + } + return _inner->upload_part(opts, stream, part_num); +} + +ObjectStorageResponse RateLimitedObjStorageClient::complete_multipart_upload( + const ObjectStoragePathOptions& opts, + const std::vector& completed_parts) { + S3RateLimitGuard guard(S3RateLimitType::PUT, 0); + if (!guard.ok()) { + return rate_limited_response(S3RateLimitType::PUT); + } + return _inner->complete_multipart_upload(opts, completed_parts); +} + +ObjectStorageHeadResponse RateLimitedObjStorageClient::head_object( + const ObjectStoragePathOptions& opts) { + S3RateLimitGuard guard(S3RateLimitType::GET, 0); + if (!guard.ok()) { + return {.resp = rate_limited_response(S3RateLimitType::GET)}; + } + return _inner->head_object(opts); +} + +ObjectStorageResponse RateLimitedObjStorageClient::get_object(const ObjectStoragePathOptions& opts, + void* buffer, size_t offset, + size_t bytes_read, + size_t* size_return) { + S3RateLimitGuard guard(S3RateLimitType::GET, bytes_read); + if (!guard.ok()) { + return rate_limited_response(S3RateLimitType::GET); + } + auto resp = _inner->get_object(opts, buffer, offset, bytes_read, size_return); + if (resp.status.code == 0) { + // Refund the difference for short reads (e.g. requested range crosses EOF). + guard.settle(*size_return); + } + return resp; +} + +ObjectStorageResponse RateLimitedObjStorageClient::list_objects( + const ObjectStoragePathOptions& opts, std::vector* files) { + S3RateLimitGuard guard(S3RateLimitType::GET, 0); + if (!guard.ok()) { + return rate_limited_response(S3RateLimitType::GET); + } + return _inner->list_objects(opts, files); +} + +ObjectStorageResponse RateLimitedObjStorageClient::delete_objects( + const ObjectStoragePathOptions& opts, std::vector objs) { + S3RateLimitGuard guard(S3RateLimitType::PUT, 0); + if (!guard.ok()) { + return rate_limited_response(S3RateLimitType::PUT); + } + return _inner->delete_objects(opts, std::move(objs)); +} + +ObjectStorageResponse RateLimitedObjStorageClient::delete_object( + const ObjectStoragePathOptions& opts) { + S3RateLimitGuard guard(S3RateLimitType::PUT, 0); + if (!guard.ok()) { + return rate_limited_response(S3RateLimitType::PUT); + } + return _inner->delete_object(opts); +} + +ObjectStorageResponse RateLimitedObjStorageClient::delete_objects_recursively( + const ObjectStoragePathOptions& opts) { + S3RateLimitGuard guard(S3RateLimitType::PUT, 0); + if (!guard.ok()) { + return rate_limited_response(S3RateLimitType::PUT); + } + return _inner->delete_objects_recursively(opts); +} + +std::string RateLimitedObjStorageClient::generate_presigned_url( + const ObjectStoragePathOptions& opts, int64_t expiration_secs, const S3ClientConf& conf) { + // Generating a presigned URL is a local computation, no request goes out. + return _inner->generate_presigned_url(opts, expiration_secs, conf); +} + +} // namespace doris::io diff --git a/be/src/io/fs/rate_limited_obj_storage_client.h b/be/src/io/fs/rate_limited_obj_storage_client.h new file mode 100644 index 00000000000000..00725d7edcb299 --- /dev/null +++ b/be/src/io/fs/rate_limited_obj_storage_client.h @@ -0,0 +1,70 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include "io/fs/obj_storage_client.h" + +namespace doris::io { + +// Decorator that applies the process-wide S3 GET/PUT QPS and bandwidth rate limiters +// in front of any ObjStorageClient. This is the single place where rate limiting is +// wired into the object storage path: provider clients (S3, Azure, future GCP, ...) +// contain no rate limiting code, and S3ClientFactory decides at construction time +// whether to wrap a client (internal storage-vault buckets) or return it bare +// (external buckets: S3 load, TVF, external catalogs in cloud mode). +// +// Each public API call is charged once against the QPS bucket, and data-carrying +// calls additionally reserve their payload size from the bytes bucket (reconciled +// with the actually transferred size for reads). Note that APIs which internally +// paginate (list_objects, delete_objects_recursively) are charged once per logical +// call, not once per underlying HTTP request. +class RateLimitedObjStorageClient final : public ObjStorageClient { +public: + explicit RateLimitedObjStorageClient(std::shared_ptr inner) + : _inner(std::move(inner)) {} + ~RateLimitedObjStorageClient() override = default; + + ObjectStorageUploadResponse create_multipart_upload( + const ObjectStoragePathOptions& opts) override; + ObjectStorageResponse put_object(const ObjectStoragePathOptions& opts, + std::string_view stream) override; + ObjectStorageUploadResponse upload_part(const ObjectStoragePathOptions& opts, + std::string_view stream, int part_num) override; + ObjectStorageResponse complete_multipart_upload( + const ObjectStoragePathOptions& opts, + const std::vector& completed_parts) override; + ObjectStorageHeadResponse head_object(const ObjectStoragePathOptions& opts) override; + ObjectStorageResponse get_object(const ObjectStoragePathOptions& opts, void* buffer, + size_t offset, size_t bytes_read, + size_t* size_return) override; + ObjectStorageResponse list_objects(const ObjectStoragePathOptions& opts, + std::vector* files) override; + ObjectStorageResponse delete_objects(const ObjectStoragePathOptions& opts, + std::vector objs) override; + ObjectStorageResponse delete_object(const ObjectStoragePathOptions& opts) override; + ObjectStorageResponse delete_objects_recursively(const ObjectStoragePathOptions& opts) override; + std::string generate_presigned_url(const ObjectStoragePathOptions& opts, + int64_t expiration_secs, const S3ClientConf& conf) override; + +private: + std::shared_ptr _inner; +}; + +} // namespace doris::io diff --git a/be/src/io/fs/s3_file_system.cpp b/be/src/io/fs/s3_file_system.cpp index 63be8f1955a59c..45251eab6e9d2a 100644 --- a/be/src/io/fs/s3_file_system.cpp +++ b/be/src/io/fs/s3_file_system.cpp @@ -88,10 +88,6 @@ Status ObjClientHolder::reset(const S3ClientConf& conf) { S3ClientConf reset_conf; { std::shared_lock lock(_mtx); - if (conf.get_hash() == _conf.get_hash()) { - return Status::OK(); // Same conf - } - reset_conf = _conf; reset_conf.ak = conf.ak; reset_conf.sk = conf.sk; @@ -101,11 +97,18 @@ Status ObjClientHolder::reset(const S3ClientConf& conf) { reset_conf.max_connections = conf.max_connections; reset_conf.request_timeout_ms = conf.request_timeout_ms; reset_conf.use_virtual_addressing = conf.use_virtual_addressing; + reset_conf.is_internal_bucket = conf.is_internal_bucket; reset_conf.role_arn = conf.role_arn; reset_conf.external_id = conf.external_id; reset_conf.cred_provider_type = conf.cred_provider_type; - // Should check endpoint here? + + // Compare full-field equality of the merged conf, not get_hash(): the hash is + // an XOR of crc32s and distinct configurations can collide, which would skip a + // required client rebuild (e.g. a credential update). + if (reset_conf == _conf) { + return Status::OK(); // Same conf + } } auto client = S3ClientFactory::instance().create(reset_conf); diff --git a/be/src/io/fs/s3_obj_storage_client.cpp b/be/src/io/fs/s3_obj_storage_client.cpp index f9ed8e155ff59c..0c0b0370f8097f 100644 --- a/be/src/io/fs/s3_obj_storage_client.cpp +++ b/be/src/io/fs/s3_obj_storage_client.cpp @@ -72,34 +72,9 @@ #include "io/fs/s3_common.h" #include "util/bvar_helper.h" +// Rate limiting is applied by RateLimitedObjStorageClient, the decorator that +// S3ClientFactory wraps around this client when the bucket is subject to limiting. namespace { -inline ::Aws::Client::AWSError<::Aws::S3::S3Errors> s3_error_factory() { - return {::Aws::S3::S3Errors::INTERNAL_FAILURE, "exceeds limit", "exceeds limit", false}; -} - -template -auto s3_rate_limit(doris::S3RateLimitType op, Func callback) -> decltype(callback()) { - using T = decltype(callback()); - if (!doris::config::enable_s3_rate_limiter) { - return callback(); - } - auto sleep_duration = doris::apply_s3_rate_limit(op); - if (sleep_duration < 0) { - return T(s3_error_factory()); - } - return callback(); -} - -template -auto s3_get_rate_limit(Func callback) -> decltype(callback()) { - return s3_rate_limit(doris::S3RateLimitType::GET, std::move(callback)); -} - -template -auto s3_put_rate_limit(Func callback) -> decltype(callback()) { - return s3_rate_limit(doris::S3RateLimitType::PUT, std::move(callback)); -} - void record_s3_request_failed(const Aws::S3::S3Error& error) { doris::record_object_request_failed(static_cast(error.GetResponseCode())); } @@ -130,9 +105,9 @@ ObjectStorageUploadResponse S3ObjStorageClient::create_multipart_upload( MonotonicStopWatch watch; watch.start(); - auto outcome = SYNC_POINT_HOOK_RETURN_VALUE( - s3_put_rate_limit([&]() { return _client->CreateMultipartUpload(request); }), - "s3_file_writer::create_multi_part_upload", std::cref(request).get()); + auto outcome = SYNC_POINT_HOOK_RETURN_VALUE(_client->CreateMultipartUpload(request), + "s3_file_writer::create_multi_part_upload", + std::cref(request).get()); SYNC_POINT_CALLBACK("s3_file_writer::_open", &outcome); watch.stop(); @@ -172,9 +147,9 @@ ObjectStorageResponse S3ObjStorageClient::put_object(const ObjectStoragePathOpti MonotonicStopWatch watch; watch.start(); - auto outcome = SYNC_POINT_HOOK_RETURN_VALUE( - s3_put_rate_limit([&]() { return _client->PutObject(request); }), - "s3_file_writer::put_object", std::cref(request).get(), &stream); + auto outcome = + SYNC_POINT_HOOK_RETURN_VALUE(_client->PutObject(request), "s3_file_writer::put_object", + std::cref(request).get(), &stream); watch.stop(); @@ -217,9 +192,9 @@ ObjectStorageUploadResponse S3ObjStorageClient::upload_part(const ObjectStorageP MonotonicStopWatch watch; watch.start(); - auto outcome = SYNC_POINT_HOOK_RETURN_VALUE( - s3_put_rate_limit([&]() { return _client->UploadPart(request); }), - "s3_file_writer::upload_part", std::cref(request).get(), &stream); + auto outcome = SYNC_POINT_HOOK_RETURN_VALUE(_client->UploadPart(request), + "s3_file_writer::upload_part", + std::cref(request).get(), &stream); watch.stop(); @@ -273,9 +248,9 @@ ObjectStorageResponse S3ObjStorageClient::complete_multipart_upload( MonotonicStopWatch watch; watch.start(); - auto outcome = SYNC_POINT_HOOK_RETURN_VALUE( - s3_put_rate_limit([&]() { return _client->CompleteMultipartUpload(request); }), - "s3_file_writer::complete_multi_part", std::cref(request).get()); + auto outcome = SYNC_POINT_HOOK_RETURN_VALUE(_client->CompleteMultipartUpload(request), + "s3_file_writer::complete_multi_part", + std::cref(request).get()); watch.stop(); s3_bvar::s3_multi_part_upload_latency << watch.elapsed_time_microseconds(); @@ -306,8 +281,7 @@ ObjectStorageHeadResponse S3ObjStorageClient::head_object(const ObjectStoragePat SCOPED_BVAR_LATENCY(s3_bvar::s3_head_latency); auto outcome = SYNC_POINT_HOOK_RETURN_VALUE( - s3_get_rate_limit([&]() { return _client->HeadObject(request); }), - "s3_file_system::head_object", std::ref(request).get()); + _client->HeadObject(request), "s3_file_system::head_object", std::ref(request).get()); if (outcome.IsSuccess()) { return {.resp = {convert_to_obj_response(Status::OK())}, .file_size = outcome.GetResult().GetContentLength()}; @@ -332,7 +306,7 @@ ObjectStorageResponse S3ObjStorageClient::get_object(const ObjectStoragePathOpti request.SetResponseStreamFactory(AwsWriteableStreamFactory(buffer, bytes_read)); SCOPED_BVAR_LATENCY(s3_bvar::s3_get_latency); - auto outcome = s3_get_rate_limit([&]() { return _client->GetObject(request); }); + auto outcome = _client->GetObject(request); if (!outcome.IsSuccess()) { record_s3_request_failed(outcome.GetError()); return {convert_to_obj_response(s3fs_error( @@ -360,7 +334,7 @@ ObjectStorageResponse S3ObjStorageClient::list_objects(const ObjectStoragePathOp Aws::S3::Model::ListObjectsV2Outcome outcome; { SCOPED_BVAR_LATENCY(s3_bvar::s3_list_latency); - outcome = s3_get_rate_limit([&]() { return _client->ListObjectsV2(request); }); + outcome = _client->ListObjectsV2(request); } if (!outcome.IsSuccess()) { files->clear(); @@ -415,8 +389,7 @@ ObjectStorageResponse S3ObjStorageClient::delete_objects(const ObjectStoragePath del.WithObjects(std::move(objects)).SetQuiet(true); delete_request.SetDelete(std::move(del)); SCOPED_BVAR_LATENCY(s3_bvar::s3_delete_objects_latency); - auto delete_outcome = - s3_put_rate_limit([&]() { return _client->DeleteObjects(delete_request); }); + auto delete_outcome = _client->DeleteObjects(delete_request); if (!delete_outcome.IsSuccess()) { record_s3_request_failed(delete_outcome.GetError()); return {convert_to_obj_response( @@ -441,7 +414,7 @@ ObjectStorageResponse S3ObjStorageClient::delete_object(const ObjectStoragePathO request.WithBucket(opts.bucket).WithKey(opts.key); SCOPED_BVAR_LATENCY(s3_bvar::s3_delete_object_latency); - auto outcome = s3_put_rate_limit([&]() { return _client->DeleteObject(request); }); + auto outcome = _client->DeleteObject(request); if (outcome.IsSuccess() || outcome.GetError().GetResponseCode() == Aws::Http::HttpResponseCode::NOT_FOUND) { return ObjectStorageResponse::OK(); @@ -464,7 +437,7 @@ ObjectStorageResponse S3ObjStorageClient::delete_objects_recursively( Aws::S3::Model::ListObjectsV2Outcome outcome; { SCOPED_BVAR_LATENCY(s3_bvar::s3_list_latency); - outcome = s3_get_rate_limit([&]() { return _client->ListObjectsV2(request); }); + outcome = _client->ListObjectsV2(request); } if (!outcome.IsSuccess()) { record_s3_request_failed(outcome.GetError()); @@ -485,8 +458,7 @@ ObjectStorageResponse S3ObjStorageClient::delete_objects_recursively( del.WithObjects(std::move(objects)).SetQuiet(true); delete_request.SetDelete(std::move(del)); SCOPED_BVAR_LATENCY(s3_bvar::s3_delete_objects_latency); - auto delete_outcome = - s3_put_rate_limit([&]() { return _client->DeleteObjects(delete_request); }); + auto delete_outcome = _client->DeleteObjects(delete_request); if (!delete_outcome.IsSuccess()) { record_s3_request_failed(delete_outcome.GetError()); return {convert_to_obj_response( diff --git a/be/src/util/s3_rate_limiter_manager.cpp b/be/src/util/s3_rate_limiter_manager.cpp new file mode 100644 index 00000000000000..cf351a4324b9e8 --- /dev/null +++ b/be/src/util/s3_rate_limiter_manager.cpp @@ -0,0 +1,204 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "util/s3_rate_limiter_manager.h" + +#include +#include +#include + +#include "common/config.h" +#include "common/logging.h" +#include "util/cgroup_util.h" + +namespace doris { + +bvar::Adder s3_get_bytes_rate_limit_sleep_ns("s3_get_bytes_rate_limit_sleep_ns"); +bvar::Adder s3_get_bytes_rate_limit_sleep_count("s3_get_bytes_rate_limit_sleep_count"); +bvar::Adder s3_put_bytes_rate_limit_sleep_ns("s3_put_bytes_rate_limit_sleep_ns"); +bvar::Adder s3_put_bytes_rate_limit_sleep_count("s3_put_bytes_rate_limit_sleep_count"); + +namespace { + +std::function bytes_rate_limiter_metric_func(S3RateLimitType type) { + switch (type) { + case S3RateLimitType::GET: + return metric_func_factory(s3_get_bytes_rate_limit_sleep_ns, + s3_get_bytes_rate_limit_sleep_count); + case S3RateLimitType::PUT: + return metric_func_factory(s3_put_bytes_rate_limit_sleep_ns, + s3_put_bytes_rate_limit_sleep_count); + default: + return [](int64_t) {}; + } +} + +// min(per_core * cores, cap) with overflow protection; cap == 0 means no cap. +int64_t cap_multiply(int64_t per_core, int64_t cores, int64_t cap) { + cap = cap > 0 ? cap : std::numeric_limits::max(); + if (per_core > cap / cores) { + return cap; + } + return per_core * cores; +} + +size_t index_of(S3RateLimitType type) { + CHECK(type == S3RateLimitType::GET || type == S3RateLimitType::PUT) << to_string(type); + return static_cast(type); +} + +} // namespace + +S3EffectiveRateLimit resolve_s3_rate_limit(S3RateLimitType type, int64_t cores) { + const bool is_get = type == S3RateLimitType::GET; + const int64_t qps_per_core = is_get ? config::s3_get_qps_per_core : config::s3_put_qps_per_core; + const int64_t qps_max = is_get ? config::s3_get_qps_max : config::s3_put_qps_max; + const int64_t bytes_per_core = is_get ? config::s3_get_bytes_per_second_per_core + : config::s3_put_bytes_per_second_per_core; + const int64_t bytes_max = + is_get ? config::s3_get_bytes_per_second_max : config::s3_put_bytes_per_second_max; + cores = std::max(1, cores); + + S3EffectiveRateLimit limit; + if (qps_per_core < 0) { + // Unset: the legacy absolute configs stay in charge, bit-for-bit compatible. + limit.qps = is_get ? config::s3_get_token_per_second : config::s3_put_token_per_second; + limit.burst = is_get ? config::s3_get_bucket_tokens : config::s3_put_bucket_tokens; + limit.count_limit = is_get ? config::s3_get_token_limit : config::s3_put_token_limit; + } else if (qps_per_core > 0) { + limit.qps = cap_multiply(qps_per_core, cores, qps_max); + limit.burst = limit.qps; // burst = 1 second worth of quota + } // qps_per_core == 0: QPS limiting disabled, all fields stay 0. + + if (bytes_per_core > 0) { + limit.bytes_per_second = cap_multiply(bytes_per_core, cores, bytes_max); + } + return limit; +} + +int reset_s3_rate_limiter(S3RateLimitType type, size_t max_speed, size_t max_burst, size_t limit) { + if (type == S3RateLimitType::UNKNOWN) { + return -1; + } + return S3RateLimiterManager::instance().qps_limiter(type)->reset(max_speed, max_burst, limit); +} + +int64_t s3_rate_limiter_cpu_cores() { + if (int64_t overridden = config::s3_rate_limiter_cpu_cores; overridden > 0) { + return overridden; + } + int physical = static_cast(std::thread::hardware_concurrency()); + // Re-read the cgroup quota on every call: serverless BEs can be resized in place, + // and the daemon refresh thread picks the change up through here. + int limited = CGroupUtil::get_cgroup_limited_cpu_number(physical); + return std::max(1, limited); +} + +S3RateLimiterManager::S3RateLimiterManager() { + const int64_t cores = s3_rate_limiter_cpu_cores(); + for (auto type : {S3RateLimitType::GET, S3RateLimitType::PUT}) { + auto limit = resolve_s3_rate_limit(type, cores); + _qps_limiters[index_of(type)] = std::make_unique( + limit.qps, limit.burst, limit.count_limit, s3_rate_limiter_metric_func(type)); + _bytes_limiters[index_of(type)] = std::make_unique( + limit.bytes_per_second, limit.bytes_per_second, 0, + bytes_rate_limiter_metric_func(type)); + } +} + +S3RateLimiterManager& S3RateLimiterManager::instance() { + static S3RateLimiterManager ret; + return ret; +} + +S3RateLimiterHolder* S3RateLimiterManager::qps_limiter(S3RateLimitType type) { + return _qps_limiters[index_of(type)].get(); +} + +S3RateLimiterHolder* S3RateLimiterManager::bytes_limiter(S3RateLimitType type) { + return _bytes_limiters[index_of(type)].get(); +} + +void S3RateLimiterManager::refresh() { + std::lock_guard guard(_refresh_lock); + const int64_t cores = s3_rate_limiter_cpu_cores(); + for (auto type : {S3RateLimitType::GET, S3RateLimitType::PUT}) { + const auto limit = resolve_s3_rate_limit(type, cores); + + auto* qps = qps_limiter(type); + if (qps->get_max_speed() != static_cast(limit.qps) || + qps->get_max_burst() != static_cast(limit.burst) || + qps->get_limit() != static_cast(limit.count_limit)) { + qps->reset(limit.qps, limit.burst, limit.count_limit); + LOG(INFO) << "reset S3 " << to_string(type) << " QPS rate limiter, qps=" << limit.qps + << ", burst=" << limit.burst << ", count_limit=" << limit.count_limit + << ", cores=" << cores; + } + + auto* bytes = bytes_limiter(type); + if (bytes->get_max_speed() != static_cast(limit.bytes_per_second)) { + bytes->reset(limit.bytes_per_second, limit.bytes_per_second, 0); + LOG(INFO) << "reset S3 " << to_string(type) + << " bytes rate limiter, bytes_per_second=" << limit.bytes_per_second + << ", cores=" << cores; + } + } +} + +S3RateLimitGuard::S3RateLimitGuard(S3RateLimitType type, size_t estimated_bytes) { + if (!config::enable_s3_rate_limiter) { + return; + } + auto& mgr = S3RateLimiterManager::instance(); + + auto* qps = mgr.qps_limiter(type); + if (qps->is_enabled() && + apply_s3_rate_limit(type, qps, config::s3_rate_limiter_log_interval) < 0) { + _ok = false; + return; + } + + if (estimated_bytes == 0) { + return; + } + auto* bytes = mgr.bytes_limiter(type); + if (!bytes->is_enabled()) { + return; + } + // Clamp the reservation to 1 second worth of bandwidth so a single oversized IO + // (e.g. a whole-file read_at) cannot create unbounded upfront debt. The clamped + // remainder is intentionally not accounted; effective quotas below the single-IO + // upper bound are excluded by the config contract (see config.cpp). + _reserved = std::min(estimated_bytes, bytes->get_max_speed()); + if (_reserved > 0) { + // Debt model: may sleep, never rejects (count limit is 0). Pin the charged + // bucket generation for settle(). + _charged_bucket = bytes->charge(_reserved); + } +} + +void S3RateLimitGuard::settle(size_t actual_bytes) { + if (_settled) { + return; + } + _settled = true; + if (_charged_bucket != nullptr && _reserved > actual_bytes) { + _charged_bucket->refund(_reserved - actual_bytes); + } +} + +} // namespace doris diff --git a/be/src/util/s3_rate_limiter_manager.h b/be/src/util/s3_rate_limiter_manager.h new file mode 100644 index 00000000000000..8d6c61c5413a7d --- /dev/null +++ b/be/src/util/s3_rate_limiter_manager.h @@ -0,0 +1,113 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include + +#include "cpp/token_bucket_rate_limiter.h" + +namespace doris { + +// The final rate limit values applied to the token buckets, resolved from all the +// s3_{get,put}_* configs plus the current CPU core count. All precedence rules +// (per-core vs legacy, caps, overflow guard) live in resolve_s3_rate_limit() only. +struct S3EffectiveRateLimit { + int64_t qps = 0; // QPS bucket rate; 0 = unlimited + int64_t burst = 0; // QPS bucket capacity + int64_t count_limit = 0; // cumulative request cap (legacy path only); 0 = unlimited + int64_t bytes_per_second = 0; // bytes bucket rate; 0 = unlimited + + bool operator==(const S3EffectiveRateLimit&) const = default; +}; + +// Pure function of configs and `cores`; no side effects. +// s3_{get,put}_qps_per_core == -1 falls back to the legacy absolute token configs and +// `cores` does not participate; otherwise qps = min(per_core * cores, qps_max). +S3EffectiveRateLimit resolve_s3_rate_limit(S3RateLimitType type, int64_t cores); + +// Cores used to derive effective limits: config::s3_rate_limiter_cpu_cores override +// (> 0) wins; otherwise re-read the cgroup cpu quota (serverless BEs can be resized +// in place), falling back to physical cores. Always >= 1. +int64_t s3_rate_limiter_cpu_cores(); + +// Directly reset the GET/PUT QPS bucket. Note that the daemon refresh thread will +// override a manual reset as soon as the config-resolved parameters differ. +int reset_s3_rate_limiter(S3RateLimitType type, size_t max_speed, size_t max_burst, size_t limit); + +// Owns the 4 process-wide token buckets (GET/PUT x QPS/bytes). Independent of +// S3ClientFactory so that instantiating it never initializes the AWS SDK. +class S3RateLimiterManager { +public: + static S3RateLimiterManager& instance(); + + // Idempotent: re-resolve effective limits from the current configs and core count, + // and reset only the buckets whose parameters actually changed. The comparison + // baseline is each bucket's own parameters -- there is no shadow state to drift. + // Called from the daemon refresh thread; safe to call from anywhere. + void refresh(); + + S3RateLimiterHolder* qps_limiter(S3RateLimitType type); + S3RateLimiterHolder* bytes_limiter(S3RateLimitType type); + + S3RateLimiterManager(const S3RateLimiterManager&) = delete; + S3RateLimiterManager& operator=(const S3RateLimiterManager&) = delete; + +private: + S3RateLimiterManager(); + + std::mutex _refresh_lock; + std::array, 2> _qps_limiters; + std::array, 2> _bytes_limiters; +}; + +// RAII admission for one logical object storage request. +// +// The constructor charges the QPS bucket (may sleep when throttled; rejected only by +// the legacy token_limit cumulative cap) and then reserves `estimated_bytes` from the +// bytes bucket, clamped to at most 1 second worth of bandwidth so a single huge IO +// cannot create unbounded upfront debt (may sleep; never rejects). +// +// settle(actual) refunds the difference when the actual transferred bytes are smaller +// than the reservation (e.g. a short read at EOF). An unsettled guard keeps the full +// reservation charged, which is the conservative choice for failed requests. +// +// The guard pins the bucket generation it charged: if refresh() resets the bytes +// bucket while the request is in flight, settle() refunds on the old generation +// (kept alive by the guard's shared_ptr) instead of polluting the fresh bucket. +class S3RateLimitGuard { +public: + S3RateLimitGuard(S3RateLimitType type, size_t estimated_bytes); + ~S3RateLimitGuard() = default; + + S3RateLimitGuard(const S3RateLimitGuard&) = delete; + S3RateLimitGuard& operator=(const S3RateLimitGuard&) = delete; + + bool ok() const { return _ok; } + void settle(size_t actual_bytes); + +private: + size_t _reserved = 0; + std::shared_ptr _charged_bucket; + bool _ok = true; + bool _settled = false; +}; + +} // namespace doris diff --git a/be/src/util/s3_util.cpp b/be/src/util/s3_util.cpp index c6e86a7f4b290f..1e03308fae4bec 100644 --- a/be/src/util/s3_util.cpp +++ b/be/src/util/s3_util.cpp @@ -60,8 +60,10 @@ #ifdef USE_AZURE #include "io/fs/azure_obj_storage_client.h" #endif +#include "cloud/config.h" #include "exec/scan/scanner_scheduler.h" #include "io/fs/obj_storage_client.h" +#include "io/fs/rate_limited_obj_storage_client.h" #include "io/fs/s3_obj_storage_client.h" #include "runtime/exec_env.h" #include "util/s3_uri.h" @@ -158,78 +160,6 @@ constexpr char S3_EXTERNAL_ID[] = "AWS_EXTERNAL_ID"; constexpr char S3_CREDENTIALS_PROVIDER_TYPE[] = "AWS_CREDENTIALS_PROVIDER_TYPE"; } // namespace -static std::atomic last_s3_get_token_bucket_tokens {0}; -static std::atomic last_s3_get_token_limit {0}; -static std::atomic last_s3_get_token_per_second {0}; -static std::atomic last_s3_put_token_per_second {0}; -static std::atomic last_s3_put_token_bucket_tokens {0}; -static std::atomic last_s3_put_token_limit {0}; - -static std::atomic updating_get_limiter {false}; -static std::atomic updating_put_limiter {false}; - -S3RateLimiterHolder* S3ClientFactory::rate_limiter(S3RateLimitType type) { - CHECK(type == S3RateLimitType::GET || type == S3RateLimitType::PUT) << to_string(type); - return _rate_limiters[static_cast(type)].get(); -} - -template -void update_rate_limiter_if_changed(int64_t current_tps, int64_t current_bucket, - int64_t current_limit, std::atomic& last_tps, - std::atomic& last_bucket, - std::atomic& last_limit, - std::atomic& updating_flag, const char* limiter_name) { - if (last_tps.load(std::memory_order_relaxed) != current_tps || - last_bucket.load(std::memory_order_relaxed) != current_bucket || - last_limit.load(std::memory_order_relaxed) != current_limit) { - bool expected = false; - if (!updating_flag.compare_exchange_strong(expected, true, std::memory_order_acq_rel)) { - return; - } - if (last_tps.load(std::memory_order_acquire) != current_tps || - last_bucket.load(std::memory_order_acquire) != current_bucket || - last_limit.load(std::memory_order_acquire) != current_limit) { - int ret = - reset_s3_rate_limiter(LimiterType, current_tps, current_bucket, current_limit); - - if (ret == 0) { - last_tps.store(current_tps, std::memory_order_release); - last_bucket.store(current_bucket, std::memory_order_release); - last_limit.store(current_limit, std::memory_order_release); - } else { - LOG(WARNING) << "Failed to reset S3 " << limiter_name - << " rate limiter, error code: " << ret; - } - } - - updating_flag.store(false, std::memory_order_release); - } -} - -void check_s3_rate_limiter_config_changed() { - update_rate_limiter_if_changed( - config::s3_get_token_per_second, config::s3_get_bucket_tokens, - config::s3_get_token_limit, last_s3_get_token_per_second, - last_s3_get_token_bucket_tokens, last_s3_get_token_limit, updating_get_limiter, "GET"); - - update_rate_limiter_if_changed( - config::s3_put_token_per_second, config::s3_put_bucket_tokens, - config::s3_put_token_limit, last_s3_put_token_per_second, - last_s3_put_token_bucket_tokens, last_s3_put_token_limit, updating_put_limiter, "PUT"); -} - -int reset_s3_rate_limiter(S3RateLimitType type, size_t max_speed, size_t max_burst, size_t limit) { - if (type == S3RateLimitType::UNKNOWN) { - return -1; - } - return S3ClientFactory::instance().rate_limiter(type)->reset(max_speed, max_burst, limit); -} - -int64_t apply_s3_rate_limit(S3RateLimitType type) { - return doris::apply_s3_rate_limit(type, S3ClientFactory::instance().rate_limiter(type), - config::s3_rate_limiter_log_interval); -} - S3ClientFactory::S3ClientFactory() { _aws_options = Aws::SDKOptions {}; auto logLevel = static_cast(config::aws_log_level); @@ -239,13 +169,6 @@ S3ClientFactory::S3ClientFactory() { }; Aws::InitAPI(_aws_options); _ca_cert_file_path = get_valid_ca_cert_path(doris::split(config::ca_cert_file_paths, ";")); - _rate_limiters = { - std::make_unique( - config::s3_get_token_per_second, config::s3_get_bucket_tokens, - config::s3_get_token_limit, s3_rate_limiter_metric_func(S3RateLimitType::GET)), - std::make_unique( - config::s3_put_token_per_second, config::s3_put_bucket_tokens, - config::s3_put_token_limit, s3_rate_limiter_metric_func(S3RateLimitType::PUT))}; #ifdef USE_AZURE auto azureLogLevel = @@ -289,8 +212,6 @@ std::shared_ptr S3ClientFactory::create(const S3ClientConf return nullptr; } - check_s3_rate_limiter_config_changed(); - #ifdef BE_TEST { std::lock_guard l(_lock); @@ -301,9 +222,8 @@ std::shared_ptr S3ClientFactory::create(const S3ClientConf #endif { - uint64_t hash = s3_conf.get_hash(); std::lock_guard l(_lock); - auto it = _cache.find(hash); + auto it = _cache.find(s3_conf); if (it != _cache.end()) { return it->second; } @@ -313,12 +233,19 @@ std::shared_ptr S3ClientFactory::create(const S3ClientConf ? _create_azure_client(s3_conf) : _create_s3_client(s3_conf); + // Rate limiting lives in one decorator, decided here at construction time: + // in cloud mode only internal storage-vault buckets are limited; external buckets + // (S3 load, TVF, external catalogs) get the bare client. In non-cloud mode every + // client is wrapped, preserving the legacy behavior. + if (obj_client != nullptr && (!config::is_cloud_mode() || s3_conf.is_internal_bucket)) { + obj_client = std::make_shared(std::move(obj_client)); + } + { - uint64_t hash = s3_conf.get_hash(); std::lock_guard l(_lock); - _cache[hash] = obj_client; + auto [it, _] = _cache.emplace(s3_conf, std::move(obj_client)); + return it->second; } - return obj_client; } #ifdef BE_TEST @@ -659,6 +586,9 @@ S3Conf S3Conf::get_s3_conf(const cloud::ObjectStoreInfoPB& info) { .role_arn = info.role_arn(), .external_id = info.external_id(), + // ObjectStoreInfoPB always describes a storage vault, i.e. a Doris + // internal bucket in cloud mode. + .is_internal_bucket = true, }, .sse_enabled = info.sse_enabled(), }; diff --git a/be/src/util/s3_util.h b/be/src/util/s3_util.h index 064f88accd8539..ef938d00c15122 100644 --- a/be/src/util/s3_util.h +++ b/be/src/util/s3_util.h @@ -63,11 +63,6 @@ extern bvar::LatencyRecorder s3_copy_object_latency; }; // namespace s3_bvar std::string hide_access_key(const std::string& ak); -int reset_s3_rate_limiter(S3RateLimitType type, size_t max_speed, size_t max_burst, size_t limit); -int64_t apply_s3_rate_limit(S3RateLimitType type); -// Rebuild the S3 GET/PUT rate limiters if the related configs have changed. -// Safe to call periodically; it is a no-op when nothing changed. -void check_s3_rate_limiter_config_changed(); class S3URI; struct S3ClientConf { @@ -89,6 +84,16 @@ struct S3ClientConf { CredProviderType cred_provider_type = CredProviderType::Default; std::string role_arn; std::string external_id; + // True when this client is bound to a Doris internal object storage bucket + // (a storage vault in cloud mode). S3ClientFactory wraps such clients with the + // shared rate limiter; external buckets (S3 load, TVF, external catalogs) are + // returned bare in cloud mode. + bool is_internal_bucket = false; + + // Full-field identity. get_hash() is only good for picking an unordered_map + // bucket; distinct configurations can collide, so never treat hash equality as + // configuration equality. + bool operator==(const S3ClientConf&) const = default; uint64_t get_hash() const { uint64_t hash_code = 0; @@ -107,6 +112,7 @@ struct S3ClientConf { hash_code ^= static_cast(cred_provider_type); hash_code ^= crc32_hash(role_arn); hash_code ^= crc32_hash(external_id); + hash_code ^= is_internal_bucket; return hash_code; } @@ -114,10 +120,16 @@ struct S3ClientConf { return fmt::format( "(ak={}, token={}, endpoint={}, region={}, bucket={}, max_connections={}, " "request_timeout_ms={}, connect_timeout_ms={}, use_virtual_addressing={}, " - "cred_provider_type={},role_arn={}, external_id={}", + "cred_provider_type={},role_arn={}, external_id={}, is_internal_bucket={}", hide_access_key(ak), token, endpoint, region, bucket, max_connections, request_timeout_ms, connect_timeout_ms, use_virtual_addressing, cred_provider_type, - role_arn, external_id); + role_arn, external_id, is_internal_bucket); + } +}; + +struct S3ClientConfHash { + size_t operator()(const S3ClientConf& conf) const { + return static_cast(conf.get_hash()); } }; @@ -158,8 +170,6 @@ class S3ClientFactory { return instance; } - S3RateLimiterHolder* rate_limiter(S3RateLimitType type); - std::shared_ptr get_aws_credentials_provider( const S3ClientConf& s3_conf); @@ -184,9 +194,9 @@ class S3ClientFactory { Aws::SDKOptions _aws_options; std::mutex _lock; - std::unordered_map> _cache; + std::unordered_map, S3ClientConfHash> + _cache; std::string _ca_cert_file_path; - std::array, 2> _rate_limiters; #ifdef BE_TEST std::function(const S3ClientConf&)> _test_client_creator; #endif diff --git a/be/test/io/client/s3_file_system_test.cpp b/be/test/io/client/s3_file_system_test.cpp index 71034020120d12..f811faf1a2601a 100644 --- a/be/test/io/client/s3_file_system_test.cpp +++ b/be/test/io/client/s3_file_system_test.cpp @@ -34,6 +34,7 @@ #include "io/fs/file_writer.h" #include "io/fs/obj_storage_client.h" #include "runtime/exec_env.h" +#include "util/s3_rate_limiter_manager.h" #include "util/s3_util.h" namespace doris { @@ -2456,15 +2457,13 @@ TEST_F(S3FileSystemTest, DynamicUpdateRateLimiterConfig) { // Save original config values int64_t original_get_bucket_tokens = config::s3_get_bucket_tokens; int64_t original_get_token_per_second = config::s3_get_token_per_second; - int64_t original_get_token_limit = config::s3_get_token_limit; - - std::cout << "Original GET config: bucket_tokens=" << original_get_bucket_tokens - << ", token_per_second=" << original_get_token_per_second - << ", limit=" << original_get_token_limit << std::endl; + int64_t original_get_qps_per_core = config::s3_get_qps_per_core; int64_t new_s3_get_bucket_tokens_val = 50; int64_t new_s3_get_token_per_second_val = 1; + // Legacy configs are only effective while the per-core config is unset. + ASSERT_TRUE(config::set_config("s3_get_qps_per_core", "-1").ok()); auto [success1, msg7] = config::set_config( "s3_get_bucket_tokens", std::to_string(new_s3_get_bucket_tokens_val), false, false); ASSERT_EQ(success1, 0) << "Failed to set s3_get_bucket_tokens: " << msg7; @@ -2473,14 +2472,25 @@ TEST_F(S3FileSystemTest, DynamicUpdateRateLimiterConfig) { std::to_string(new_s3_get_token_per_second_val), false, false); ASSERT_EQ(success2, 0) << "Failed to set s3_get_token_per_second: " << msg8; - auto st = create_client(); - ASSERT_TRUE(st.ok()); + // Dynamic config changes take effect through the periodic idempotent refresh. + auto& manager = S3RateLimiterManager::instance(); + manager.refresh(); - // Verify restoration - EXPECT_EQ(S3ClientFactory::instance().rate_limiter(S3RateLimitType::GET)->get_max_burst(), + EXPECT_EQ(manager.qps_limiter(S3RateLimitType::GET)->get_max_burst(), new_s3_get_bucket_tokens_val); - EXPECT_EQ(S3ClientFactory::instance().rate_limiter(S3RateLimitType::GET)->get_max_speed(), + EXPECT_EQ(manager.qps_limiter(S3RateLimitType::GET)->get_max_speed(), new_s3_get_token_per_second_val); + + // Restore configs and re-apply so other tests are unaffected. + ASSERT_TRUE( + config::set_config("s3_get_bucket_tokens", std::to_string(original_get_bucket_tokens)) + .ok()); + ASSERT_TRUE(config::set_config("s3_get_token_per_second", + std::to_string(original_get_token_per_second)) + .ok()); + ASSERT_TRUE(config::set_config("s3_get_qps_per_core", std::to_string(original_get_qps_per_core)) + .ok()); + manager.refresh(); } } // namespace doris diff --git a/be/test/io/fs/rate_limited_obj_storage_client_test.cpp b/be/test/io/fs/rate_limited_obj_storage_client_test.cpp new file mode 100644 index 00000000000000..0d567a15fde47e --- /dev/null +++ b/be/test/io/fs/rate_limited_obj_storage_client_test.cpp @@ -0,0 +1,193 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "io/fs/rate_limited_obj_storage_client.h" + +#include + +#include "common/config.h" +#include "util/s3_rate_limiter_manager.h" +#include "util/s3_util.h" + +namespace doris::io { +namespace { + +// Provider-free fake: counts calls and reports a configurable read size. +class FakeObjStorageClient : public ObjStorageClient { +public: + ObjectStorageUploadResponse create_multipart_upload( + const ObjectStoragePathOptions& opts) override { + ++calls; + return {}; + } + ObjectStorageResponse put_object(const ObjectStoragePathOptions& opts, + std::string_view stream) override { + ++calls; + return ObjectStorageResponse::OK(); + } + ObjectStorageUploadResponse upload_part(const ObjectStoragePathOptions& opts, + std::string_view stream, int part_num) override { + ++calls; + return {}; + } + ObjectStorageResponse complete_multipart_upload( + const ObjectStoragePathOptions& opts, + const std::vector& completed_parts) override { + ++calls; + return ObjectStorageResponse::OK(); + } + ObjectStorageHeadResponse head_object(const ObjectStoragePathOptions& opts) override { + ++calls; + return {}; + } + ObjectStorageResponse get_object(const ObjectStoragePathOptions& opts, void* buffer, + size_t offset, size_t bytes_read, + size_t* size_return) override { + ++calls; + *size_return = actual_read_size; + return ObjectStorageResponse::OK(); + } + ObjectStorageResponse list_objects(const ObjectStoragePathOptions& opts, + std::vector* files) override { + ++calls; + return ObjectStorageResponse::OK(); + } + ObjectStorageResponse delete_objects(const ObjectStoragePathOptions& opts, + std::vector objs) override { + ++calls; + return ObjectStorageResponse::OK(); + } + ObjectStorageResponse delete_object(const ObjectStoragePathOptions& opts) override { + ++calls; + return ObjectStorageResponse::OK(); + } + ObjectStorageResponse delete_objects_recursively( + const ObjectStoragePathOptions& opts) override { + ++calls; + return ObjectStorageResponse::OK(); + } + std::string generate_presigned_url(const ObjectStoragePathOptions& opts, + int64_t expiration_secs, const S3ClientConf& conf) override { + ++calls; + return "presigned"; + } + + int calls = 0; + size_t actual_read_size = 0; +}; + +struct RateLimiterConfigGuard { + bool enable = config::enable_s3_rate_limiter; + + ~RateLimiterConfigGuard() { + config::enable_s3_rate_limiter = enable; + S3RateLimiterManager::instance().refresh(); + } +}; + +} // namespace + +TEST(RateLimitedObjStorageClientTest, forwards_all_calls_when_disabled) { + RateLimiterConfigGuard guard; + config::enable_s3_rate_limiter = false; + + auto fake = std::make_shared(); + RateLimitedObjStorageClient client(fake); + ObjectStoragePathOptions opts {.bucket = "b", .key = "k"}; + + size_t size_return = 0; + EXPECT_EQ(0, client.create_multipart_upload(opts).resp.status.code); + EXPECT_EQ(0, client.put_object(opts, "data").status.code); + EXPECT_EQ(0, client.upload_part(opts, "data", 1).resp.status.code); + EXPECT_EQ(0, client.complete_multipart_upload(opts, {}).status.code); + EXPECT_EQ(0, client.head_object(opts).resp.status.code); + EXPECT_EQ(0, client.get_object(opts, nullptr, 0, 4, &size_return).status.code); + std::vector files; + EXPECT_EQ(0, client.list_objects(opts, &files).status.code); + EXPECT_EQ(0, client.delete_objects(opts, {}).status.code); + EXPECT_EQ(0, client.delete_object(opts).status.code); + EXPECT_EQ(0, client.delete_objects_recursively(opts).status.code); + EXPECT_EQ("presigned", client.generate_presigned_url(opts, 60, S3ClientConf {})); + EXPECT_EQ(11, fake->calls); +} + +TEST(RateLimitedObjStorageClientTest, get_rejected_by_count_limit_does_not_reach_inner) { + RateLimiterConfigGuard guard; + config::enable_s3_rate_limiter = true; + auto& manager = S3RateLimiterManager::instance(); + manager.qps_limiter(S3RateLimitType::GET)->reset(0, 0, 1); + manager.qps_limiter(S3RateLimitType::PUT)->reset(0, 0, 0); + + auto fake = std::make_shared(); + RateLimitedObjStorageClient client(fake); + ObjectStoragePathOptions opts {.bucket = "b", .key = "k"}; + + EXPECT_EQ(0, client.head_object(opts).resp.status.code); + EXPECT_EQ(1, fake->calls); + + auto resp = client.head_object(opts); + EXPECT_NE(0, resp.resp.status.code); + EXPECT_EQ(429, resp.resp.http_code); + EXPECT_NE(std::string::npos, resp.resp.status.msg.find("exceeds request limit")); + EXPECT_EQ(1, fake->calls); // rejected before reaching the provider + + // PUT uses an independent bucket and is unaffected. + EXPECT_EQ(0, client.put_object(opts, "data").status.code); + EXPECT_EQ(2, fake->calls); +} + +TEST(RateLimitedObjStorageClientTest, get_object_settles_short_read) { + RateLimiterConfigGuard guard; + config::enable_s3_rate_limiter = true; + auto& manager = S3RateLimiterManager::instance(); + manager.qps_limiter(S3RateLimitType::GET)->reset(0, 0, 0); + auto* bytes = manager.bytes_limiter(S3RateLimitType::GET); + bytes->reset(1000, 1000, 0); + + auto fake = std::make_shared(); + fake->actual_read_size = 100; // short read: 600 requested, 100 returned + RateLimitedObjStorageClient client(fake); + ObjectStoragePathOptions opts {.bucket = "b", .key = "k"}; + + size_t size_return = 0; + EXPECT_EQ(0, client.get_object(opts, nullptr, 0, 600, &size_return).status.code); + EXPECT_EQ(100, size_return); + + // Only 100 tokens were effectively consumed, so another 900 pass without sleeping. + EXPECT_EQ(0, bytes->add(900)); + EXPECT_GT(bytes->add(100), 0); +} + +TEST(RateLimitedObjStorageClientTest, put_object_charges_payload_bytes) { + RateLimiterConfigGuard guard; + config::enable_s3_rate_limiter = true; + auto& manager = S3RateLimiterManager::instance(); + manager.qps_limiter(S3RateLimitType::PUT)->reset(0, 0, 0); + auto* bytes = manager.bytes_limiter(S3RateLimitType::PUT); + bytes->reset(1000, 1000, 0); + + auto fake = std::make_shared(); + RateLimitedObjStorageClient client(fake); + ObjectStoragePathOptions opts {.bucket = "b", .key = "k"}; + + std::string payload(600, 'x'); + EXPECT_EQ(0, client.put_object(opts, payload).status.code); + EXPECT_EQ(0, bytes->add(400)); // exactly the remainder of the bucket + EXPECT_GT(bytes->add(100), 0); +} + +} // namespace doris::io diff --git a/be/test/util/s3_rate_limiter_manager_test.cpp b/be/test/util/s3_rate_limiter_manager_test.cpp new file mode 100644 index 00000000000000..cf79e07d4d2005 --- /dev/null +++ b/be/test/util/s3_rate_limiter_manager_test.cpp @@ -0,0 +1,317 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "util/s3_rate_limiter_manager.h" + +#include + +#include + +#include "common/config.h" + +namespace doris { + +namespace { + +// Saves every rate limiter related config on construction, restores it and re-applies +// the limiters on destruction so tests do not leak state into each other. +struct RateLimiterConfigGuard { + bool enable = config::enable_s3_rate_limiter; + int64_t get_tps = config::s3_get_token_per_second; + int64_t get_bucket = config::s3_get_bucket_tokens; + int64_t get_limit = config::s3_get_token_limit; + int64_t put_tps = config::s3_put_token_per_second; + int64_t put_bucket = config::s3_put_bucket_tokens; + int64_t put_limit = config::s3_put_token_limit; + int64_t get_per_core = config::s3_get_qps_per_core; + int64_t put_per_core = config::s3_put_qps_per_core; + int64_t get_qps_max = config::s3_get_qps_max; + int64_t put_qps_max = config::s3_put_qps_max; + int64_t get_bytes_per_core = config::s3_get_bytes_per_second_per_core; + int64_t put_bytes_per_core = config::s3_put_bytes_per_second_per_core; + int64_t get_bytes_max = config::s3_get_bytes_per_second_max; + int64_t put_bytes_max = config::s3_put_bytes_per_second_max; + int64_t cpu_cores = config::s3_rate_limiter_cpu_cores; + + ~RateLimiterConfigGuard() { + config::enable_s3_rate_limiter = enable; + config::s3_get_token_per_second = get_tps; + config::s3_get_bucket_tokens = get_bucket; + config::s3_get_token_limit = get_limit; + config::s3_put_token_per_second = put_tps; + config::s3_put_bucket_tokens = put_bucket; + config::s3_put_token_limit = put_limit; + config::s3_get_qps_per_core = get_per_core; + config::s3_put_qps_per_core = put_per_core; + config::s3_get_qps_max = get_qps_max; + config::s3_put_qps_max = put_qps_max; + config::s3_get_bytes_per_second_per_core = get_bytes_per_core; + config::s3_put_bytes_per_second_per_core = put_bytes_per_core; + config::s3_get_bytes_per_second_max = get_bytes_max; + config::s3_put_bytes_per_second_max = put_bytes_max; + config::s3_rate_limiter_cpu_cores = cpu_cores; + S3RateLimiterManager::instance().refresh(); + } +}; + +constexpr int64_t kCores = 4; + +} // namespace + +TEST(S3RateLimiterResolveTest, legacy_config_wins_when_per_core_unset) { + RateLimiterConfigGuard guard; + config::s3_get_qps_per_core = -1; + config::s3_get_token_per_second = 123; + config::s3_get_bucket_tokens = 456; + config::s3_get_token_limit = 789; + config::s3_get_qps_max = 9; // must be ignored on the legacy path + config::s3_get_bytes_per_second_per_core = -1; + + auto limit = resolve_s3_rate_limit(S3RateLimitType::GET, kCores); + EXPECT_EQ(123, limit.qps); + EXPECT_EQ(456, limit.burst); + EXPECT_EQ(789, limit.count_limit); + EXPECT_EQ(0, limit.bytes_per_second); +} + +TEST(S3RateLimiterResolveTest, per_core_config_overrides_legacy) { + RateLimiterConfigGuard guard; + config::s3_put_qps_per_core = 7; + config::s3_put_qps_max = 0; + config::s3_put_token_per_second = 123; + config::s3_put_bucket_tokens = 456; + config::s3_put_token_limit = 789; + + auto limit = resolve_s3_rate_limit(S3RateLimitType::PUT, kCores); + EXPECT_EQ(7 * kCores, limit.qps); + EXPECT_EQ(7 * kCores, limit.burst); + EXPECT_EQ(0, limit.count_limit); // legacy count cap is dropped on the per-core path +} + +TEST(S3RateLimiterResolveTest, per_core_zero_disables_qps_limiting) { + RateLimiterConfigGuard guard; + config::s3_get_qps_per_core = 0; + config::s3_get_token_per_second = 123; + config::s3_get_token_limit = 789; + + auto limit = resolve_s3_rate_limit(S3RateLimitType::GET, kCores); + EXPECT_EQ(0, limit.qps); + EXPECT_EQ(0, limit.burst); + EXPECT_EQ(0, limit.count_limit); +} + +TEST(S3RateLimiterResolveTest, qps_max_caps_per_core_result) { + RateLimiterConfigGuard guard; + config::s3_get_qps_per_core = 1000000; + config::s3_get_qps_max = 256; + + auto limit = resolve_s3_rate_limit(S3RateLimitType::GET, kCores); + EXPECT_EQ(256, limit.qps); + EXPECT_EQ(256, limit.burst); +} + +TEST(S3RateLimiterResolveTest, overflowing_multiplication_is_capped) { + RateLimiterConfigGuard guard; + config::s3_get_qps_per_core = std::numeric_limits::max(); + config::s3_get_qps_max = 1024; + + auto limit = resolve_s3_rate_limit(S3RateLimitType::GET, kCores); + EXPECT_EQ(1024, limit.qps); + + // Without a cap the overflowing product saturates instead of wrapping around. + config::s3_get_qps_max = 0; + limit = resolve_s3_rate_limit(S3RateLimitType::GET, kCores); + EXPECT_EQ(std::numeric_limits::max(), limit.qps); +} + +TEST(S3RateLimiterResolveTest, bytes_per_core_derives_bandwidth) { + RateLimiterConfigGuard guard; + config::s3_get_bytes_per_second_per_core = 1024; + config::s3_get_bytes_per_second_max = 0; + + auto limit = resolve_s3_rate_limit(S3RateLimitType::GET, kCores); + EXPECT_EQ(1024 * kCores, limit.bytes_per_second); + + config::s3_get_bytes_per_second_max = 2048; + limit = resolve_s3_rate_limit(S3RateLimitType::GET, kCores); + EXPECT_EQ(2048, limit.bytes_per_second); + + // -1 and 0 both disable bandwidth limiting. + config::s3_get_bytes_per_second_per_core = 0; + limit = resolve_s3_rate_limit(S3RateLimitType::GET, kCores); + EXPECT_EQ(0, limit.bytes_per_second); +} + +TEST(S3RateLimiterResolveTest, cpu_cores_override_config_wins) { + RateLimiterConfigGuard guard; + config::s3_rate_limiter_cpu_cores = 16; + EXPECT_EQ(16, s3_rate_limiter_cpu_cores()); + + config::s3_rate_limiter_cpu_cores = 0; + EXPECT_GE(s3_rate_limiter_cpu_cores(), 1); // auto-detect always yields >= 1 +} + +TEST(S3RateLimiterManagerTest, refresh_is_idempotent_and_applies_core_changes) { + RateLimiterConfigGuard guard; + auto& manager = S3RateLimiterManager::instance(); + auto* get_qps = manager.qps_limiter(S3RateLimitType::GET); + + config::s3_rate_limiter_cpu_cores = kCores; + config::s3_get_qps_per_core = 100; + config::s3_get_qps_max = 0; + manager.refresh(); + EXPECT_EQ(100 * kCores, get_qps->get_max_speed()); + EXPECT_EQ(100 * kCores, get_qps->get_max_burst()); + + // No config change -> no reset (params stay identical). + manager.refresh(); + EXPECT_EQ(100 * kCores, get_qps->get_max_speed()); + + // Simulate a serverless resize: the core count is an input of refresh(). + config::s3_rate_limiter_cpu_cores = 2 * kCores; + manager.refresh(); + EXPECT_EQ(100 * 2 * kCores, get_qps->get_max_speed()); +} + +TEST(S3RateLimiterManagerTest, refresh_applies_bytes_limit_and_enables_bucket) { + RateLimiterConfigGuard guard; + auto& manager = S3RateLimiterManager::instance(); + auto* put_bytes = manager.bytes_limiter(S3RateLimitType::PUT); + + config::s3_rate_limiter_cpu_cores = kCores; + config::s3_put_bytes_per_second_per_core = -1; + manager.refresh(); + EXPECT_FALSE(put_bytes->is_enabled()); + + config::s3_put_bytes_per_second_per_core = 1000; + manager.refresh(); + EXPECT_TRUE(put_bytes->is_enabled()); + EXPECT_EQ(1000 * kCores, put_bytes->get_max_speed()); + + config::s3_put_bytes_per_second_per_core = -1; + manager.refresh(); + EXPECT_FALSE(put_bytes->is_enabled()); +} + +TEST(S3RateLimitGuardTest, disabled_limiter_admits_everything) { + RateLimiterConfigGuard guard; + config::enable_s3_rate_limiter = false; + S3RateLimiterManager::instance().qps_limiter(S3RateLimitType::GET)->reset(0, 0, 1); + + for (int i = 0; i < 3; ++i) { + S3RateLimitGuard g(S3RateLimitType::GET, 100); + EXPECT_TRUE(g.ok()); + } +} + +TEST(S3RateLimitGuardTest, legacy_count_limit_rejects) { + RateLimiterConfigGuard guard; + config::enable_s3_rate_limiter = true; + auto& manager = S3RateLimiterManager::instance(); + // No throttling, hard count limit of 2. + manager.qps_limiter(S3RateLimitType::GET)->reset(0, 0, 2); + + S3RateLimitGuard g1(S3RateLimitType::GET, 0); + EXPECT_TRUE(g1.ok()); + S3RateLimitGuard g2(S3RateLimitType::GET, 0); + EXPECT_TRUE(g2.ok()); + S3RateLimitGuard g3(S3RateLimitType::GET, 0); + EXPECT_FALSE(g3.ok()); +} + +TEST(S3RateLimitGuardTest, settle_refunds_short_read) { + RateLimiterConfigGuard guard; + config::enable_s3_rate_limiter = true; + auto& manager = S3RateLimiterManager::instance(); + auto* bytes = manager.bytes_limiter(S3RateLimitType::GET); + manager.qps_limiter(S3RateLimitType::GET)->reset(0, 0, 0); + // Big enough bucket that nothing throttles; we only observe token accounting. + bytes->reset(1000, 1000, 0); + + { + S3RateLimitGuard g(S3RateLimitType::GET, 600); + ASSERT_TRUE(g.ok()); + g.settle(100); // short read: 500 tokens must come back + } + // 1000 - 600 + 500 = 900 tokens remain; a 900-byte reservation passes without + // sleeping, which we observe as add() returning 0. + EXPECT_EQ(0, bytes->add(900)); + // Now the bucket is empty; the next add must throttle (sleep > 0). + EXPECT_GT(bytes->add(100), 0); +} + +TEST(S3RateLimitGuardTest, unsettled_guard_keeps_reservation) { + RateLimiterConfigGuard guard; + config::enable_s3_rate_limiter = true; + auto& manager = S3RateLimiterManager::instance(); + auto* bytes = manager.bytes_limiter(S3RateLimitType::PUT); + manager.qps_limiter(S3RateLimitType::PUT)->reset(0, 0, 0); + bytes->reset(1000, 1000, 0); + + { + S3RateLimitGuard g(S3RateLimitType::PUT, 600); + ASSERT_TRUE(g.ok()); + // No settle: e.g. the request failed. The reservation stays charged. + } + EXPECT_EQ(0, bytes->add(400)); // exactly the remainder + EXPECT_GT(bytes->add(100), 0); // anything more throttles +} + +TEST(S3RateLimitGuardTest, settle_across_reset_does_not_pollute_new_bucket) { + RateLimiterConfigGuard guard; + config::enable_s3_rate_limiter = true; + auto& manager = S3RateLimiterManager::instance(); + auto* bytes = manager.bytes_limiter(S3RateLimitType::GET); + manager.qps_limiter(S3RateLimitType::GET)->reset(0, 0, 0); + bytes->reset(1000, 1000, 0); + + { + S3RateLimitGuard g(S3RateLimitType::GET, 600); + ASSERT_TRUE(g.ok()); + // The bucket is swapped while the request is in flight. + bytes->reset(1000, 1000, 0); + g.settle(100); // refund lands on the OLD generation, not the fresh bucket + } + // The fresh bucket must still be exactly full (1000): a 1000-byte reservation + // passes without sleeping, anything more throttles. If the 500-byte refund had + // leaked into it, the bucket would be over-filled... which the burst cap masks, + // so verify the other direction: no tokens were taken from it either. + EXPECT_EQ(0, bytes->add(1000)); + EXPECT_GT(bytes->add(100), 0); +} + +TEST(S3RateLimitGuardTest, reservation_is_clamped_to_one_second_of_bandwidth) { + RateLimiterConfigGuard guard; + config::enable_s3_rate_limiter = true; + auto& manager = S3RateLimiterManager::instance(); + auto* bytes = manager.bytes_limiter(S3RateLimitType::GET); + manager.qps_limiter(S3RateLimitType::GET)->reset(0, 0, 0); + bytes->reset(1000, 1000, 0); + + { + // A huge IO only reserves max_speed (=1000) instead of going into deep debt. + S3RateLimitGuard g(S3RateLimitType::GET, 1000000); + ASSERT_TRUE(g.ok()); + } + // The bucket was drained exactly to zero, not to -999000: a following small add + // throttles briefly instead of sleeping for ~1000 seconds. + auto sleep_ns = bytes->add(100); + EXPECT_GT(sleep_ns, 0); + EXPECT_LT(sleep_ns, 1000000000L); // well under 1 second of debt +} + +} // namespace doris diff --git a/be/test/util/s3_util_test.cpp b/be/test/util/s3_util_test.cpp index a87d0e9a2d2980..eeeb1cec9e5340 100644 --- a/be/test/util/s3_util_test.cpp +++ b/be/test/util/s3_util_test.cpp @@ -23,6 +23,7 @@ #include "common/config.h" #include "gtest/gtest_pred_impl.h" +#include "util/s3_rate_limiter_manager.h" #include "util/s3_uri.h" namespace doris { @@ -77,29 +78,31 @@ TEST_F(S3UTILTest, hide_access_key_typical_aws_key) { EXPECT_EQ("xxxxxxxFODNN7xxxxxxx", result); } -// Verifies that check_s3_rate_limiter_config_changed() rebuilds the global GET rate -// limiter when the related configs change. This is the behavior the cloud vault refresh -// thread relies on to apply dynamically modified s3_get_* rate limiter configs without -// having to (re)create an S3 client. -TEST_F(S3UTILTest, check_s3_rate_limiter_config_changed_rebuilds_limiter) { - auto* get_limiter = S3ClientFactory::instance().rate_limiter(S3RateLimitType::GET); +// Verifies that S3RateLimiterManager::refresh() rebuilds the global GET rate limiter +// when the related configs change. This is the behavior the daemon refresh thread +// relies on to apply dynamically modified s3_get_* rate limiter configs. +TEST_F(S3UTILTest, refresh_rebuilds_limiter_on_config_change) { + auto& manager = S3RateLimiterManager::instance(); + auto* get_limiter = manager.qps_limiter(S3RateLimitType::GET); ASSERT_NE(get_limiter, nullptr); // Save originals so other tests are not affected. const int64_t orig_tps = config::s3_get_token_per_second; const int64_t orig_bucket = config::s3_get_bucket_tokens; const int64_t orig_limit = config::s3_get_token_limit; + const int64_t orig_per_core = config::s3_get_qps_per_core; - // Establish a known baseline (no count limit, no throttling). + // Establish a known baseline (legacy path, no count limit, no throttling). + config::s3_get_qps_per_core = -1; config::s3_get_token_per_second = 1000000000; config::s3_get_bucket_tokens = 1000000000; config::s3_get_token_limit = 0; - check_s3_rate_limiter_config_changed(); + manager.refresh(); // Impose a hard request-count limit of 3. Since the limit value changes (0 -> 3), // the limiter is rebuilt with a fresh counter. config::s3_get_token_limit = 3; - check_s3_rate_limiter_config_changed(); + manager.refresh(); // The bucket/speed are huge so add() never throttles (returns 0); only the count // limit takes effect: the first 3 requests pass, the 4th is rejected (-1). @@ -108,17 +111,18 @@ TEST_F(S3UTILTest, check_s3_rate_limiter_config_changed_rebuilds_limiter) { EXPECT_GE(get_limiter->add(1), 0); EXPECT_LT(get_limiter->add(1), 0); - // Raise the limit. The checker must rebuild the limiter so the exhausted counter is + // Raise the limit. The refresh must rebuild the limiter so the exhausted counter is // reset; otherwise the next request would still be rejected. config::s3_get_token_limit = 100; - check_s3_rate_limiter_config_changed(); + manager.refresh(); EXPECT_GE(get_limiter->add(1), 0); // Restore original configs and apply them back to the limiter. config::s3_get_token_per_second = orig_tps; config::s3_get_bucket_tokens = orig_bucket; config::s3_get_token_limit = orig_limit; - check_s3_rate_limiter_config_changed(); + config::s3_get_qps_per_core = orig_per_core; + manager.refresh(); } } // end namespace doris diff --git a/common/cpp/token_bucket_rate_limiter.cpp b/common/cpp/token_bucket_rate_limiter.cpp index 128501ae7cf469..b557adee4c0346 100644 --- a/common/cpp/token_bucket_rate_limiter.cpp +++ b/common/cpp/token_bucket_rate_limiter.cpp @@ -124,10 +124,19 @@ int64_t TokenBucketRateLimiter::add(size_t amount) { return sleep_time_ns; } +void TokenBucketRateLimiter::refund(size_t amount) { + std::lock_guard lock(*_mutex); + if (_max_speed) { + _remain_tokens = std::min(_remain_tokens + amount, _max_burst); + } + _count = (_count >= amount) ? _count - amount : 0; +} + TokenBucketRateLimiterHolder::TokenBucketRateLimiterHolder(size_t max_speed, size_t max_burst, size_t limit, std::function metric_func) - : rate_limiter(std::make_unique(max_speed, max_burst, limit)), + : rate_limiter(std::make_shared(max_speed, max_burst, limit)), + _enabled(max_speed > 0 || limit > 0), metric_func(std::move(metric_func)) {} int64_t TokenBucketRateLimiterHolder::add(size_t amount) { @@ -135,22 +144,38 @@ int64_t TokenBucketRateLimiterHolder::add(size_t amount) { } TokenBucketRateLimiterResult TokenBucketRateLimiterHolder::add_with_config(size_t amount) { - TokenBucketRateLimiterResult result; + // Snapshot the current limiter and call add() outside the read lock: add() may + // sleep for a long time when throttled, and holding the read lock across the + // sleep would block reset() (dynamic config update) for the whole duration. + std::shared_ptr limiter; { std::shared_lock read {rate_limiter_rw_lock}; - result = {.sleep_duration = rate_limiter->add(amount), - .max_speed = rate_limiter->get_max_speed(), - .max_burst = rate_limiter->get_max_burst(), - .limit = rate_limiter->get_limit()}; + limiter = rate_limiter; } + TokenBucketRateLimiterResult result = {.sleep_duration = limiter->add(amount), + .max_speed = limiter->get_max_speed(), + .max_burst = limiter->get_max_burst(), + .limit = limiter->get_limit()}; metric_func(result.sleep_duration); return result; } +std::shared_ptr TokenBucketRateLimiterHolder::charge(size_t amount) { + std::shared_ptr limiter; + { + std::shared_lock read {rate_limiter_rw_lock}; + limiter = rate_limiter; + } + metric_func(limiter->add(amount)); + return limiter; +} + int TokenBucketRateLimiterHolder::reset(size_t max_speed, size_t max_burst, size_t limit) { + auto new_rate_limiter = std::make_shared(max_speed, max_burst, limit); { std::unique_lock write {rate_limiter_rw_lock}; - rate_limiter = std::make_unique(max_speed, max_burst, limit); + rate_limiter = std::move(new_rate_limiter); + _enabled.store(max_speed > 0 || limit > 0, std::memory_order_release); } return 0; } diff --git a/common/cpp/token_bucket_rate_limiter.h b/common/cpp/token_bucket_rate_limiter.h index 57f9205527acc6..ae11afa0569e4d 100644 --- a/common/cpp/token_bucket_rate_limiter.h +++ b/common/cpp/token_bucket_rate_limiter.h @@ -18,6 +18,7 @@ #pragma once #include +#include #include #include #include @@ -58,6 +59,11 @@ class TokenBucketRateLimiter { // Returns duration of sleep in nanoseconds (to distinguish sleeping on different kinds of S3RateLimiters for metrics) int64_t add(size_t amount); + // Return `amount` tokens to the bucket (capped at max_burst) and roll back the + // cumulative counter. Used to reconcile a reservation with the actually consumed + // amount, e.g. a short read at EOF. + void refund(size_t amount); + size_t get_max_speed() const { return _max_speed; } size_t get_max_burst() const { return _max_burst; } @@ -93,15 +99,27 @@ class TokenBucketRateLimiterHolder { int64_t add(size_t amount); TokenBucketRateLimiterResult add_with_config(size_t amount); + // Charge `amount` like add(), but return the limiter generation the tokens were + // taken from. Callers that later refund a reservation must refund on the returned + // object, so that a concurrent reset() cannot make the refund pollute a fresh + // bucket that never saw the original charge. + std::shared_ptr charge(size_t amount); + int reset(size_t max_speed, size_t max_burst, size_t limit); + // Whether the currently published limiter can throttle or reject at all + // (max_speed > 0 or limit > 0). Lock-free fast path for callers that want to + // skip disabled limiters. + bool is_enabled() const { return _enabled.load(std::memory_order_acquire); } + size_t get_max_speed() const; size_t get_max_burst() const; size_t get_limit() const; private: mutable std::shared_mutex rate_limiter_rw_lock; - std::unique_ptr rate_limiter; + std::shared_ptr rate_limiter; + std::atomic _enabled; // Record the correspoding sleeping time(unit is ms) std::function metric_func; }; From a12fe2e0b286cdc907c1be0712345ce4a4087a47 Mon Sep 17 00:00:00 2001 From: Refrain Date: Mon, 20 Jul 2026 09:49:54 +0800 Subject: [PATCH 02/22] [test](be) Cover unified object storage rate limiter ### What problem does this PR solve? Issue Number: None Related PR: #65420 Problem Summary: After adopting the unified object-storage rate limiter implementation, deterministic coverage was needed for refresh state, CPU-based resizing, reservation settlement, logical pagination and operation mapping, factory wrapping policy, and full-key client cache behavior. This change adds that coverage, removes wall-clock-sensitive assertions, and makes dynamic config restoration safe on early test exits. ### Release note None ### Check List (For Author) - Test: Unit Test / Build - Targeted BE unit tests: 28 passed, 1 skipped because the S3 client is disabled locally - BUILD_TYPE=Debug ./build.sh --fe --be -j64 - build-support/check-format.sh - Debug BE UT objects compiled; final link is blocked by the pre-existing unresolved __lsan_disable/__lsan_enable references in cached_remote_file_reader_peer_test.cpp - Behavior changed: No. Test coverage only. - Does this need documentation: No --- be/test/io/client/s3_file_system_test.cpp | 21 ++-- .../rate_limited_obj_storage_client_test.cpp | 95 ++++++++++++++- .../io/fs/s3_obj_stroage_client_mock_test.cpp | 43 +++++-- be/test/io/s3_client_factory_test.cpp | 111 ++++++++++++++++++ be/test/util/s3_rate_limiter_manager_test.cpp | 77 +++++++----- 5 files changed, 291 insertions(+), 56 deletions(-) diff --git a/be/test/io/client/s3_file_system_test.cpp b/be/test/io/client/s3_file_system_test.cpp index f811faf1a2601a..87fb45165200df 100644 --- a/be/test/io/client/s3_file_system_test.cpp +++ b/be/test/io/client/s3_file_system_test.cpp @@ -34,6 +34,7 @@ #include "io/fs/file_writer.h" #include "io/fs/obj_storage_client.h" #include "runtime/exec_env.h" +#include "util/defer_op.h" #include "util/s3_rate_limiter_manager.h" #include "util/s3_util.h" @@ -2459,6 +2460,14 @@ TEST_F(S3FileSystemTest, DynamicUpdateRateLimiterConfig) { int64_t original_get_token_per_second = config::s3_get_token_per_second; int64_t original_get_qps_per_core = config::s3_get_qps_per_core; + auto& manager = S3RateLimiterManager::instance(); + Defer restore_configs {[&] { + config::s3_get_bucket_tokens = original_get_bucket_tokens; + config::s3_get_token_per_second = original_get_token_per_second; + config::s3_get_qps_per_core = original_get_qps_per_core; + manager.refresh(); + }}; + int64_t new_s3_get_bucket_tokens_val = 50; int64_t new_s3_get_token_per_second_val = 1; @@ -2473,24 +2482,12 @@ TEST_F(S3FileSystemTest, DynamicUpdateRateLimiterConfig) { ASSERT_EQ(success2, 0) << "Failed to set s3_get_token_per_second: " << msg8; // Dynamic config changes take effect through the periodic idempotent refresh. - auto& manager = S3RateLimiterManager::instance(); manager.refresh(); EXPECT_EQ(manager.qps_limiter(S3RateLimitType::GET)->get_max_burst(), new_s3_get_bucket_tokens_val); EXPECT_EQ(manager.qps_limiter(S3RateLimitType::GET)->get_max_speed(), new_s3_get_token_per_second_val); - - // Restore configs and re-apply so other tests are unaffected. - ASSERT_TRUE( - config::set_config("s3_get_bucket_tokens", std::to_string(original_get_bucket_tokens)) - .ok()); - ASSERT_TRUE(config::set_config("s3_get_token_per_second", - std::to_string(original_get_token_per_second)) - .ok()); - ASSERT_TRUE(config::set_config("s3_get_qps_per_core", std::to_string(original_get_qps_per_core)) - .ok()); - manager.refresh(); } } // namespace doris diff --git a/be/test/io/fs/rate_limited_obj_storage_client_test.cpp b/be/test/io/fs/rate_limited_obj_storage_client_test.cpp index 0d567a15fde47e..ccfd0e7400c328 100644 --- a/be/test/io/fs/rate_limited_obj_storage_client_test.cpp +++ b/be/test/io/fs/rate_limited_obj_storage_client_test.cpp @@ -26,12 +26,17 @@ namespace doris::io { namespace { +constexpr size_t kNoThrottleBytesPerSecond = 1ULL << 40; + // Provider-free fake: counts calls and reports a configurable read size. class FakeObjStorageClient : public ObjStorageClient { public: ObjectStorageUploadResponse create_multipart_upload( const ObjectStoragePathOptions& opts) override { ++calls; + ++create_multipart_upload_calls; + create_multipart_upload_provider_calls += + create_multipart_upload_provider_calls_per_logical_call; return {}; } ObjectStorageResponse put_object(const ObjectStoragePathOptions& opts, @@ -78,6 +83,9 @@ class FakeObjStorageClient : public ObjStorageClient { ObjectStorageResponse delete_objects_recursively( const ObjectStoragePathOptions& opts) override { ++calls; + ++delete_objects_recursively_calls; + delete_objects_recursively_provider_calls += + delete_objects_recursively_provider_calls_per_logical_call; return ObjectStorageResponse::OK(); } std::string generate_presigned_url(const ObjectStoragePathOptions& opts, @@ -88,6 +96,12 @@ class FakeObjStorageClient : public ObjStorageClient { int calls = 0; size_t actual_read_size = 0; + int create_multipart_upload_calls = 0; + int create_multipart_upload_provider_calls = 0; + int create_multipart_upload_provider_calls_per_logical_call = 1; + int delete_objects_recursively_calls = 0; + int delete_objects_recursively_provider_calls = 0; + int delete_objects_recursively_provider_calls_per_logical_call = 1; }; struct RateLimiterConfigGuard { @@ -156,7 +170,7 @@ TEST(RateLimitedObjStorageClientTest, get_object_settles_short_read) { auto& manager = S3RateLimiterManager::instance(); manager.qps_limiter(S3RateLimitType::GET)->reset(0, 0, 0); auto* bytes = manager.bytes_limiter(S3RateLimitType::GET); - bytes->reset(1000, 1000, 0); + bytes->reset(kNoThrottleBytesPerSecond, kNoThrottleBytesPerSecond, 1000); auto fake = std::make_shared(); fake->actual_read_size = 100; // short read: 600 requested, 100 returned @@ -167,9 +181,9 @@ TEST(RateLimitedObjStorageClientTest, get_object_settles_short_read) { EXPECT_EQ(0, client.get_object(opts, nullptr, 0, 600, &size_return).status.code); EXPECT_EQ(100, size_return); - // Only 100 tokens were effectively consumed, so another 900 pass without sleeping. + // Only 100 bytes remain cumulatively charged, so exactly 900 more are admitted. EXPECT_EQ(0, bytes->add(900)); - EXPECT_GT(bytes->add(100), 0); + EXPECT_EQ(-1, bytes->add(1)); } TEST(RateLimitedObjStorageClientTest, put_object_charges_payload_bytes) { @@ -178,7 +192,7 @@ TEST(RateLimitedObjStorageClientTest, put_object_charges_payload_bytes) { auto& manager = S3RateLimiterManager::instance(); manager.qps_limiter(S3RateLimitType::PUT)->reset(0, 0, 0); auto* bytes = manager.bytes_limiter(S3RateLimitType::PUT); - bytes->reset(1000, 1000, 0); + bytes->reset(kNoThrottleBytesPerSecond, kNoThrottleBytesPerSecond, 1000); auto fake = std::make_shared(); RateLimitedObjStorageClient client(fake); @@ -186,8 +200,77 @@ TEST(RateLimitedObjStorageClientTest, put_object_charges_payload_bytes) { std::string payload(600, 'x'); EXPECT_EQ(0, client.put_object(opts, payload).status.code); - EXPECT_EQ(0, bytes->add(400)); // exactly the remainder of the bucket - EXPECT_GT(bytes->add(100), 0); + EXPECT_EQ(0, bytes->add(400)); // exactly the cumulative count remainder + EXPECT_EQ(-1, bytes->add(1)); +} + +TEST(RateLimitedObjStorageClientTest, recursive_delete_charges_one_put_qps) { + RateLimiterConfigGuard guard; + config::enable_s3_rate_limiter = true; + auto& manager = S3RateLimiterManager::instance(); + manager.qps_limiter(S3RateLimitType::GET)->reset(0, 0, 1); + manager.qps_limiter(S3RateLimitType::PUT)->reset(0, 0, 1); + manager.bytes_limiter(S3RateLimitType::PUT)->reset(0, 0, 0); + + auto fake = std::make_shared(); + fake->delete_objects_recursively_provider_calls_per_logical_call = 4; + RateLimitedObjStorageClient client(fake); + ObjectStoragePathOptions opts {.bucket = "b", .prefix = "p"}; + + // Exhaust GET first. Recursive delete still succeeds because the logical API is PUT. + EXPECT_EQ(0, client.head_object(opts).resp.status.code); + EXPECT_EQ(0, client.delete_objects_recursively(opts).status.code); + EXPECT_EQ(1, fake->delete_objects_recursively_calls); + EXPECT_EQ(4, fake->delete_objects_recursively_provider_calls); + + auto resp = client.delete_objects_recursively(opts); + EXPECT_NE(0, resp.status.code); + EXPECT_EQ(429, resp.http_code); + EXPECT_EQ(1, fake->delete_objects_recursively_calls); + EXPECT_EQ(4, fake->delete_objects_recursively_provider_calls); +} + +TEST(RateLimitedObjStorageClientTest, azure_noop_multipart_create_charges_one_put_qps) { + RateLimiterConfigGuard guard; + config::enable_s3_rate_limiter = true; + auto& manager = S3RateLimiterManager::instance(); + manager.qps_limiter(S3RateLimitType::PUT)->reset(0, 0, 1); + manager.bytes_limiter(S3RateLimitType::PUT)->reset(0, 0, 0); + + auto fake = std::make_shared(); + // Azure implements create_multipart_upload as a provider-side no-op. + fake->create_multipart_upload_provider_calls_per_logical_call = 0; + RateLimitedObjStorageClient client(fake); + ObjectStoragePathOptions opts {.bucket = "b", .key = "k"}; + + EXPECT_EQ(0, client.create_multipart_upload(opts).resp.status.code); + EXPECT_EQ(1, fake->create_multipart_upload_calls); + EXPECT_EQ(0, fake->create_multipart_upload_provider_calls); + + auto resp = client.create_multipart_upload(opts); + EXPECT_NE(0, resp.resp.status.code); + EXPECT_EQ(429, resp.resp.http_code); + EXPECT_EQ(1, fake->create_multipart_upload_calls); + EXPECT_EQ(0, fake->create_multipart_upload_provider_calls); +} + +TEST(RateLimitedObjStorageClientTest, presigned_url_bypasses_rate_limiters) { + RateLimiterConfigGuard guard; + config::enable_s3_rate_limiter = true; + auto& manager = S3RateLimiterManager::instance(); + auto* get_qps = manager.qps_limiter(S3RateLimitType::GET); + auto* put_qps = manager.qps_limiter(S3RateLimitType::PUT); + get_qps->reset(0, 0, 1); + put_qps->reset(0, 0, 1); + EXPECT_EQ(0, get_qps->add(1)); + EXPECT_EQ(0, put_qps->add(1)); + + auto fake = std::make_shared(); + RateLimitedObjStorageClient client(fake); + ObjectStoragePathOptions opts {.bucket = "b", .key = "k"}; + + EXPECT_EQ("presigned", client.generate_presigned_url(opts, 60, S3ClientConf {})); + EXPECT_EQ(1, fake->calls); } } // namespace doris::io diff --git a/be/test/io/fs/s3_obj_stroage_client_mock_test.cpp b/be/test/io/fs/s3_obj_stroage_client_mock_test.cpp index b7e635c1f1d29f..c010c120480c67 100644 --- a/be/test/io/fs/s3_obj_stroage_client_mock_test.cpp +++ b/be/test/io/fs/s3_obj_stroage_client_mock_test.cpp @@ -22,13 +22,28 @@ #include #include "gmock/gmock.h" +#include "io/fs/rate_limited_obj_storage_client.h" #include "io/fs/s3_obj_storage_client.h" +#include "util/s3_rate_limiter_manager.h" #include "util/s3_util.h" #include "util/string_util.h" using namespace Aws::S3::Model; namespace doris::io { +namespace { + +struct RateLimiterConfigGuard { + bool enable = config::enable_s3_rate_limiter; + + ~RateLimiterConfigGuard() { + config::enable_s3_rate_limiter = enable; + S3RateLimiterManager::instance().refresh(); + } +}; + +} // namespace + class MockS3Client : public Aws::S3::S3Client { public: MockS3Client() {}; @@ -81,9 +96,16 @@ ListObjectsV2Result CreatePageResult(const std::string& nextToken, return result; } -TEST_F(S3ObjStorageClientMockTest, list_objects_with_pagination) { +TEST_F(S3ObjStorageClientMockTest, list_objects_pagination_charges_one_get_qps) { + RateLimiterConfigGuard guard; + config::enable_s3_rate_limiter = true; + auto& manager = S3RateLimiterManager::instance(); + manager.qps_limiter(S3RateLimitType::GET)->reset(0, 0, 1); + manager.bytes_limiter(S3RateLimitType::GET)->reset(0, 0, 0); + auto mock_s3_client = std::make_shared(); - S3ObjStorageClient s3_obj_storage_client(mock_s3_client); + auto s3_obj_storage_client = std::make_shared(mock_s3_client); + RateLimitedObjStorageClient rate_limited_client(s3_obj_storage_client); std::vector> pages = { {"key1", "key2"}, // page1 @@ -110,14 +132,19 @@ TEST_F(S3ObjStorageClientMockTest, list_objects_with_pagination) { }); std::vector files; - auto response = s3_obj_storage_client.list_objects( - {.bucket = "dummy-bucket", - .prefix = "S3ObjStorageClientMockTest/list_objects_with_pagination"}, - &files); + const ObjectStoragePathOptions opts { + .bucket = "dummy-bucket", + .prefix = "S3ObjStorageClientMockTest/list_objects_with_pagination"}; + auto response = rate_limited_client.list_objects(opts, &files); EXPECT_EQ(response.status.code, ErrorCode::OK); EXPECT_EQ(files.size(), 5); - files.clear(); + + // The first logical list used one GET token despite issuing three provider requests. + // A second logical list is rejected before it reaches the provider. + response = rate_limited_client.list_objects(opts, &files); + EXPECT_NE(response.status.code, ErrorCode::OK); + EXPECT_EQ(response.http_code, 429); } TEST_F(S3ObjStorageClientMockTest, test_ca_cert) { @@ -125,4 +152,4 @@ TEST_F(S3ObjStorageClientMockTest, test_ca_cert) { LOG(INFO) << "config:" << config::ca_cert_file_paths << " path:" << path; ASSERT_FALSE(path.empty()); } -} // namespace doris::io \ No newline at end of file +} // namespace doris::io diff --git a/be/test/io/s3_client_factory_test.cpp b/be/test/io/s3_client_factory_test.cpp index 792d815cbca5f8..de8149380cbcbc 100644 --- a/be/test/io/s3_client_factory_test.cpp +++ b/be/test/io/s3_client_factory_test.cpp @@ -21,9 +21,14 @@ #include #include +#include +#include #include +#include "cloud/config.h" #include "cpp/custom_aws_credentials_provider_chain.h" +#include "io/fs/rate_limited_obj_storage_client.h" +#include "io/fs/s3_obj_storage_client.h" #include "util/s3_uri.h" #include "util/s3_util.h" @@ -31,8 +36,114 @@ namespace doris { class S3ClientFactoryTest : public testing::Test { FRIEND_TEST(S3ClientFactoryTest, S3ClientFactory); + +protected: + void TearDown() override { S3ClientFactory::instance().clear_client_creator_for_test(); } }; +namespace { + +class CloudModeConfigGuard { +public: + explicit CloudModeConfigGuard(bool cloud_mode) + : _deploy_mode(config::deploy_mode), _cloud_unique_id(config::cloud_unique_id) { + config::deploy_mode = cloud_mode ? "cloud" : ""; + config::cloud_unique_id.clear(); + } + + ~CloudModeConfigGuard() { + config::deploy_mode = _deploy_mode; + config::cloud_unique_id = _cloud_unique_id; + } + +private: + std::string _deploy_mode; + std::string _cloud_unique_id; +}; + +S3ClientConf make_factory_conf(std::string endpoint, bool is_internal_bucket) { + S3ClientConf conf; + conf.endpoint = std::move(endpoint); + conf.region = "us-east-1"; + conf.cred_provider_type = CredProviderType::Anonymous; + conf.is_internal_bucket = is_internal_bucket; + return conf; +} + +S3ClientConf make_hash_collision_conf(std::string endpoint, bool is_internal_bucket) { + auto conf = make_factory_conf(std::move(endpoint), is_internal_bucket); + conf.use_virtual_addressing = !is_internal_bucket; + return conf; +} + +} // namespace + +TEST_F(S3ClientFactoryTest, WrapsAllClientsInNonCloudMode) { + CloudModeConfigGuard guard(false); + auto& factory = S3ClientFactory::instance(); + + auto external_client = + factory.create(make_factory_conf("non-cloud-external-rate-limit.example.com", false)); + auto internal_client = + factory.create(make_factory_conf("non-cloud-internal-rate-limit.example.com", true)); + + ASSERT_NE(external_client, nullptr); + ASSERT_NE(internal_client, nullptr); + EXPECT_NE(std::dynamic_pointer_cast(external_client), nullptr); + EXPECT_NE(std::dynamic_pointer_cast(internal_client), nullptr); +} + +TEST_F(S3ClientFactoryTest, WrapsOnlyInternalClientsInCloudModeAndDistinguishesHashCollisions) { + CloudModeConfigGuard guard(true); + auto external_conf = + make_hash_collision_conf("cloud-rate-limit-hash-collision.example.com", false); + auto internal_conf = + make_hash_collision_conf("cloud-rate-limit-hash-collision.example.com", true); + ASSERT_EQ(external_conf.get_hash(), internal_conf.get_hash()); + ASSERT_NE(external_conf, internal_conf); + + auto& factory = S3ClientFactory::instance(); + auto external_client = factory.create(external_conf); + auto internal_client = factory.create(internal_conf); + + ASSERT_NE(external_client, nullptr); + ASSERT_NE(internal_client, nullptr); + EXPECT_EQ(std::dynamic_pointer_cast(external_client), nullptr); + EXPECT_NE(std::dynamic_pointer_cast(internal_client), nullptr); + EXPECT_NE(external_client, internal_client); + EXPECT_EQ(factory.create(external_conf), external_client); + EXPECT_EQ(factory.create(internal_conf), internal_client); +} + +TEST_F(S3ClientFactoryTest, ObjClientHolderResetDistinguishesHashCollisions) { + auto external_conf = + make_hash_collision_conf("s3-client-holder-hash-collision.example.com", false); + auto internal_conf = + make_hash_collision_conf("s3-client-holder-hash-collision.example.com", true); + ASSERT_EQ(external_conf.get_hash(), internal_conf.get_hash()); + + auto external_client = + std::make_shared(std::shared_ptr {}); + auto internal_client = + std::make_shared(std::shared_ptr {}); + int create_count = 0; + S3ClientFactory::instance().set_client_creator_for_test( + [&](const S3ClientConf& conf) -> std::shared_ptr { + ++create_count; + return conf.is_internal_bucket ? internal_client : external_client; + }); + + io::ObjClientHolder holder(external_conf); + ASSERT_TRUE(holder.init().ok()); + EXPECT_EQ(create_count, 1); + EXPECT_EQ(holder.get(), external_client); + + ASSERT_TRUE(holder.reset(internal_conf).ok()); + EXPECT_EQ(create_count, 2); + EXPECT_EQ(holder.get(), internal_client); + EXPECT_EQ(holder.s3_client_conf(), internal_conf); +} + TEST_F(S3ClientFactoryTest, AwsCredentialsProvider) { S3ClientFactory& factory = S3ClientFactory::instance(); S3ClientConf anonymous_conf; diff --git a/be/test/util/s3_rate_limiter_manager_test.cpp b/be/test/util/s3_rate_limiter_manager_test.cpp index cf79e07d4d2005..a830150ccc5046 100644 --- a/be/test/util/s3_rate_limiter_manager_test.cpp +++ b/be/test/util/s3_rate_limiter_manager_test.cpp @@ -69,6 +69,7 @@ struct RateLimiterConfigGuard { }; constexpr int64_t kCores = 4; +constexpr size_t kNoThrottle = 1ULL << 40; } // namespace @@ -165,7 +166,27 @@ TEST(S3RateLimiterResolveTest, cpu_cores_override_config_wins) { EXPECT_GE(s3_rate_limiter_cpu_cores(), 1); // auto-detect always yields >= 1 } -TEST(S3RateLimiterManagerTest, refresh_is_idempotent_and_applies_core_changes) { +TEST(S3RateLimiterManagerTest, refresh_with_same_parameters_keeps_consumed_state) { + RateLimiterConfigGuard guard; + auto& manager = S3RateLimiterManager::instance(); + auto* get_qps = manager.qps_limiter(S3RateLimitType::GET); + + config::s3_get_qps_per_core = -1; + config::s3_get_token_per_second = kNoThrottle; + config::s3_get_bucket_tokens = kNoThrottle; + config::s3_get_token_limit = 2; + manager.refresh(); + + EXPECT_EQ(0, get_qps->add(1)); + EXPECT_EQ(0, get_qps->add(1)); + + // Identical parameters must preserve the published bucket and its cumulative count. + // If refresh() reset the bucket, this third request would be admitted. + manager.refresh(); + EXPECT_EQ(-1, get_qps->add(1)); +} + +TEST(S3RateLimiterManagerTest, refresh_applies_core_changes) { RateLimiterConfigGuard guard; auto& manager = S3RateLimiterManager::instance(); auto* get_qps = manager.qps_limiter(S3RateLimitType::GET); @@ -177,10 +198,6 @@ TEST(S3RateLimiterManagerTest, refresh_is_idempotent_and_applies_core_changes) { EXPECT_EQ(100 * kCores, get_qps->get_max_speed()); EXPECT_EQ(100 * kCores, get_qps->get_max_burst()); - // No config change -> no reset (params stay identical). - manager.refresh(); - EXPECT_EQ(100 * kCores, get_qps->get_max_speed()); - // Simulate a serverless resize: the core count is an input of refresh(). config::s3_rate_limiter_cpu_cores = 2 * kCores; manager.refresh(); @@ -239,36 +256,35 @@ TEST(S3RateLimitGuardTest, settle_refunds_short_read) { auto& manager = S3RateLimiterManager::instance(); auto* bytes = manager.bytes_limiter(S3RateLimitType::GET); manager.qps_limiter(S3RateLimitType::GET)->reset(0, 0, 0); - // Big enough bucket that nothing throttles; we only observe token accounting. - bytes->reset(1000, 1000, 0); + // A large speed/burst removes wall-clock refill from the assertion. The count limit + // makes the refund observable exactly. + bytes->reset(kNoThrottle, kNoThrottle, 1000); { S3RateLimitGuard g(S3RateLimitType::GET, 600); ASSERT_TRUE(g.ok()); g.settle(100); // short read: 500 tokens must come back } - // 1000 - 600 + 500 = 900 tokens remain; a 900-byte reservation passes without - // sleeping, which we observe as add() returning 0. + // The guard leaves 100 cumulatively charged bytes, so exactly 900 more are admitted. EXPECT_EQ(0, bytes->add(900)); - // Now the bucket is empty; the next add must throttle (sleep > 0). - EXPECT_GT(bytes->add(100), 0); + EXPECT_EQ(-1, bytes->add(1)); } -TEST(S3RateLimitGuardTest, unsettled_guard_keeps_reservation) { +TEST(S3RateLimitGuardTest, put_payload_reservation_remains_charged_without_settle) { RateLimiterConfigGuard guard; config::enable_s3_rate_limiter = true; auto& manager = S3RateLimiterManager::instance(); auto* bytes = manager.bytes_limiter(S3RateLimitType::PUT); manager.qps_limiter(S3RateLimitType::PUT)->reset(0, 0, 0); - bytes->reset(1000, 1000, 0); + bytes->reset(kNoThrottle, kNoThrottle, 1000); { S3RateLimitGuard g(S3RateLimitType::PUT, 600); ASSERT_TRUE(g.ok()); // No settle: e.g. the request failed. The reservation stays charged. } - EXPECT_EQ(0, bytes->add(400)); // exactly the remainder - EXPECT_GT(bytes->add(100), 0); // anything more throttles + EXPECT_EQ(0, bytes->add(400)); // exactly the cumulative count remainder + EXPECT_EQ(-1, bytes->add(1)); } TEST(S3RateLimitGuardTest, settle_across_reset_does_not_pollute_new_bucket) { @@ -277,21 +293,20 @@ TEST(S3RateLimitGuardTest, settle_across_reset_does_not_pollute_new_bucket) { auto& manager = S3RateLimiterManager::instance(); auto* bytes = manager.bytes_limiter(S3RateLimitType::GET); manager.qps_limiter(S3RateLimitType::GET)->reset(0, 0, 0); - bytes->reset(1000, 1000, 0); + bytes->reset(kNoThrottle, kNoThrottle, 1000); { S3RateLimitGuard g(S3RateLimitType::GET, 600); ASSERT_TRUE(g.ok()); // The bucket is swapped while the request is in flight. - bytes->reset(1000, 1000, 0); + bytes->reset(kNoThrottle, kNoThrottle, 1000); + EXPECT_EQ(0, bytes->add(600)); g.settle(100); // refund lands on the OLD generation, not the fresh bucket } - // The fresh bucket must still be exactly full (1000): a 1000-byte reservation - // passes without sleeping, anything more throttles. If the 500-byte refund had - // leaked into it, the bucket would be over-filled... which the burst cap masks, - // so verify the other direction: no tokens were taken from it either. - EXPECT_EQ(0, bytes->add(1000)); - EXPECT_GT(bytes->add(100), 0); + // The fresh generation remains charged for 600 bytes. A refund to the wrong + // generation would reduce its cumulative count to 100 and admit the last request. + EXPECT_EQ(0, bytes->add(400)); + EXPECT_EQ(-1, bytes->add(1)); } TEST(S3RateLimitGuardTest, reservation_is_clamped_to_one_second_of_bandwidth) { @@ -300,18 +315,20 @@ TEST(S3RateLimitGuardTest, reservation_is_clamped_to_one_second_of_bandwidth) { auto& manager = S3RateLimiterManager::instance(); auto* bytes = manager.bytes_limiter(S3RateLimitType::GET); manager.qps_limiter(S3RateLimitType::GET)->reset(0, 0, 0); - bytes->reset(1000, 1000, 0); + // Keep max_speed at 1000 to define the one-second reservation, but use a huge + // burst to make the test independent from elapsed time while observing the exact + // charged amount through the cumulative limit. + bytes->reset(1000, kNoThrottle, 1500); { - // A huge IO only reserves max_speed (=1000) instead of going into deep debt. + // A huge IO only reserves max_speed (=1000). Since actual_bytes exceeds that + // reservation, settle() does not refund it. S3RateLimitGuard g(S3RateLimitType::GET, 1000000); ASSERT_TRUE(g.ok()); + g.settle(2000); } - // The bucket was drained exactly to zero, not to -999000: a following small add - // throttles briefly instead of sleeping for ~1000 seconds. - auto sleep_ns = bytes->add(100); - EXPECT_GT(sleep_ns, 0); - EXPECT_LT(sleep_ns, 1000000000L); // well under 1 second of debt + EXPECT_EQ(0, bytes->add(500)); + EXPECT_EQ(-1, bytes->add(1)); } } // namespace doris From 3e1c0d3e05df5336f7f8854e3ba608e9ce8d068e Mon Sep 17 00:00:00 2001 From: Refrain Date: Mon, 20 Jul 2026 16:03:03 +0800 Subject: [PATCH 03/22] [fix](be) Bound pending S3 byte reservations ### What problem does this PR solve? Issue Number: None Related PR: #65420 Problem Summary: The S3 byte limiter could admit an unbounded amount of reserved work. Each admitted request added token debt before sleeping, so bursts could produce progressively longer waits while occupying object-storage worker threads. This change reuses the limiter count as the outstanding byte reservation, caps it at one second of effective bandwidth, atomically rolls back rejected charges, and separates short-read token settlement from request-completion count release. ### Release note Bound outstanding S3 byte-rate reservations to prevent unbounded limiter waits. ### Check List (For Author) - Test: Unit Test - ./run-be-ut.sh --run --filter=TokenBucketRateLimiterTest.*:S3RateLimiterManagerTest.*:S3RateLimitGuardTest.*:RateLimitedObjStorageClientTest.* -j48 - Behavior changed: Yes. S3 byte reservations exceeding one second of effective bandwidth are rejected without leaving byte-token or count debt. - Does this need documentation: No --- be/src/common/config.cpp | 3 + be/src/util/s3_rate_limiter_manager.cpp | 22 ++- be/src/util/s3_rate_limiter_manager.h | 15 ++- .../rate_limited_obj_storage_client_test.cpp | 25 ++-- be/test/util/s3_rate_limiter_manager_test.cpp | 126 ++++++++++++------ common/cpp/token_bucket_rate_limiter.cpp | 22 ++- common/cpp/token_bucket_rate_limiter.h | 20 +-- 7 files changed, 160 insertions(+), 73 deletions(-) diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index 05871682376f41..dc933b18be03ef 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -1559,6 +1559,9 @@ DEFINE_Validator(s3_put_qps_max, [](int64_t config) -> bool { return config >= 0 // CPU-aware S3 bandwidth limiter. Effective GET/PUT bytes/s = bytes_per_second_per_core * // BE cpu cores, capped by the corresponding bytes_per_second_max. -1 and 0 both disable // byte-rate limiting for that operation (there is no legacy fallback for bandwidth). +// Waiting and executing reservations are capped at one second worth of effective bandwidth. +// A byte reservation that would exceed that cap is rejected and its byte-rate token charge is +// rolled back. // Note: the derived per-BE bytes/s should not be set below the single IO upper bound // per second (s3_write_buffer_size, 5MB by default). A single IO larger than 1 second // of quota only reserves 1 second worth of tokens; the excess bytes are not accounted diff --git a/be/src/util/s3_rate_limiter_manager.cpp b/be/src/util/s3_rate_limiter_manager.cpp index cf351a4324b9e8..3f9c1a149bb220 100644 --- a/be/src/util/s3_rate_limiter_manager.cpp +++ b/be/src/util/s3_rate_limiter_manager.cpp @@ -115,7 +115,7 @@ S3RateLimiterManager::S3RateLimiterManager() { _qps_limiters[index_of(type)] = std::make_unique( limit.qps, limit.burst, limit.count_limit, s3_rate_limiter_metric_func(type)); _bytes_limiters[index_of(type)] = std::make_unique( - limit.bytes_per_second, limit.bytes_per_second, 0, + limit.bytes_per_second, limit.bytes_per_second, limit.bytes_per_second, bytes_rate_limiter_metric_func(type)); } } @@ -150,8 +150,10 @@ void S3RateLimiterManager::refresh() { } auto* bytes = bytes_limiter(type); - if (bytes->get_max_speed() != static_cast(limit.bytes_per_second)) { - bytes->reset(limit.bytes_per_second, limit.bytes_per_second, 0); + const auto bytes_per_second = static_cast(limit.bytes_per_second); + if (bytes->get_max_speed() != bytes_per_second || + bytes->get_max_burst() != bytes_per_second || bytes->get_limit() != bytes_per_second) { + bytes->reset(bytes_per_second, bytes_per_second, bytes_per_second); LOG(INFO) << "reset S3 " << to_string(type) << " bytes rate limiter, bytes_per_second=" << limit.bytes_per_second << ", cores=" << cores; @@ -185,9 +187,17 @@ S3RateLimitGuard::S3RateLimitGuard(S3RateLimitType type, size_t estimated_bytes) // upper bound are excluded by the config contract (see config.cpp). _reserved = std::min(estimated_bytes, bytes->get_max_speed()); if (_reserved > 0) { - // Debt model: may sleep, never rejects (count limit is 0). Pin the charged - // bucket generation for settle(). + // Pin the admitted bucket generation for settlement and count release. _charged_bucket = bytes->charge(_reserved); + if (_charged_bucket == nullptr) { + _ok = false; + } + } +} + +S3RateLimitGuard::~S3RateLimitGuard() { + if (_charged_bucket != nullptr) { + _charged_bucket->refund_count(_reserved); } } @@ -197,7 +207,7 @@ void S3RateLimitGuard::settle(size_t actual_bytes) { } _settled = true; if (_charged_bucket != nullptr && _reserved > actual_bytes) { - _charged_bucket->refund(_reserved - actual_bytes); + _charged_bucket->refund_tokens(_reserved - actual_bytes); } } diff --git a/be/src/util/s3_rate_limiter_manager.h b/be/src/util/s3_rate_limiter_manager.h index 8d6c61c5413a7d..9707f08b0b4dea 100644 --- a/be/src/util/s3_rate_limiter_manager.h +++ b/be/src/util/s3_rate_limiter_manager.h @@ -83,19 +83,20 @@ class S3RateLimiterManager { // The constructor charges the QPS bucket (may sleep when throttled; rejected only by // the legacy token_limit cumulative cap) and then reserves `estimated_bytes` from the // bytes bucket, clamped to at most 1 second worth of bandwidth so a single huge IO -// cannot create unbounded upfront debt (may sleep; never rejects). +// cannot create unbounded upfront debt. The bytes bucket rejects a reservation when +// admitting it would make the total waiting and executing bytes exceed that amount. // -// settle(actual) refunds the difference when the actual transferred bytes are smaller -// than the reservation (e.g. a short read at EOF). An unsettled guard keeps the full -// reservation charged, which is the conservative choice for failed requests. +// settle(actual) returns only unused rate tokens when the actual transferred bytes are +// smaller than the reservation (e.g. a short read at EOF). Destruction always releases +// the full reservation from the bytes bucket count without returning rate tokens. // // The guard pins the bucket generation it charged: if refresh() resets the bytes -// bucket while the request is in flight, settle() refunds on the old generation -// (kept alive by the guard's shared_ptr) instead of polluting the fresh bucket. +// bucket while the request is in flight, token settlement and count release apply to +// the old generation instead of polluting the fresh bucket. class S3RateLimitGuard { public: S3RateLimitGuard(S3RateLimitType type, size_t estimated_bytes); - ~S3RateLimitGuard() = default; + ~S3RateLimitGuard(); S3RateLimitGuard(const S3RateLimitGuard&) = delete; S3RateLimitGuard& operator=(const S3RateLimitGuard&) = delete; diff --git a/be/test/io/fs/rate_limited_obj_storage_client_test.cpp b/be/test/io/fs/rate_limited_obj_storage_client_test.cpp index ccfd0e7400c328..77aab7289815d6 100644 --- a/be/test/io/fs/rate_limited_obj_storage_client_test.cpp +++ b/be/test/io/fs/rate_limited_obj_storage_client_test.cpp @@ -170,7 +170,7 @@ TEST(RateLimitedObjStorageClientTest, get_object_settles_short_read) { auto& manager = S3RateLimiterManager::instance(); manager.qps_limiter(S3RateLimitType::GET)->reset(0, 0, 0); auto* bytes = manager.bytes_limiter(S3RateLimitType::GET); - bytes->reset(kNoThrottleBytesPerSecond, kNoThrottleBytesPerSecond, 1000); + bytes->reset(1000, 1000, 1000); auto fake = std::make_shared(); fake->actual_read_size = 100; // short read: 600 requested, 100 returned @@ -181,12 +181,13 @@ TEST(RateLimitedObjStorageClientTest, get_object_settles_short_read) { EXPECT_EQ(0, client.get_object(opts, nullptr, 0, 600, &size_return).status.code); EXPECT_EQ(100, size_return); - // Only 100 bytes remain cumulatively charged, so exactly 900 more are admitted. + // The count reservation is released, while only the 500 unused rate tokens are + // returned. The resulting 900-token balance admits exactly this request immediately. EXPECT_EQ(0, bytes->add(900)); - EXPECT_EQ(-1, bytes->add(1)); + bytes->reset(0, 0, 0); } -TEST(RateLimitedObjStorageClientTest, put_object_charges_payload_bytes) { +TEST(RateLimitedObjStorageClientTest, bytes_limit_rejects_before_calling_inner) { RateLimiterConfigGuard guard; config::enable_s3_rate_limiter = true; auto& manager = S3RateLimiterManager::instance(); @@ -198,10 +199,18 @@ TEST(RateLimitedObjStorageClientTest, put_object_charges_payload_bytes) { RateLimitedObjStorageClient client(fake); ObjectStoragePathOptions opts {.bucket = "b", .key = "k"}; - std::string payload(600, 'x'); - EXPECT_EQ(0, client.put_object(opts, payload).status.code); - EXPECT_EQ(0, bytes->add(400)); // exactly the cumulative count remainder - EXPECT_EQ(-1, bytes->add(1)); + { + S3RateLimitGuard in_flight(S3RateLimitType::PUT, 1000); + ASSERT_TRUE(in_flight.ok()); + + auto resp = client.put_object(opts, "x"); + EXPECT_NE(0, resp.status.code); + EXPECT_EQ(429, resp.http_code); + EXPECT_EQ(0, fake->calls); + } + + EXPECT_EQ(0, client.put_object(opts, std::string(1000, 'x')).status.code); + EXPECT_EQ(1, fake->calls); } TEST(RateLimitedObjStorageClientTest, recursive_delete_charges_one_put_qps) { diff --git a/be/test/util/s3_rate_limiter_manager_test.cpp b/be/test/util/s3_rate_limiter_manager_test.cpp index a830150ccc5046..681a6156bb1423 100644 --- a/be/test/util/s3_rate_limiter_manager_test.cpp +++ b/be/test/util/s3_rate_limiter_manager_test.cpp @@ -218,12 +218,49 @@ TEST(S3RateLimiterManagerTest, refresh_applies_bytes_limit_and_enables_bucket) { manager.refresh(); EXPECT_TRUE(put_bytes->is_enabled()); EXPECT_EQ(1000 * kCores, put_bytes->get_max_speed()); + EXPECT_EQ(1000 * kCores, put_bytes->get_max_burst()); + EXPECT_EQ(1000 * kCores, put_bytes->get_limit()); config::s3_put_bytes_per_second_per_core = -1; manager.refresh(); EXPECT_FALSE(put_bytes->is_enabled()); } +TEST(TokenBucketRateLimiterTest, rejected_add_rolls_back_count_and_tokens) { + TokenBucketRateLimiter limiter(1000, 1000, 600); + + EXPECT_EQ(0, limiter.add(600)); + EXPECT_EQ(-1, limiter.add(500)); + + // Releasing the admitted request makes room for another request. The rejected + // request must not leave either count or token debt behind. + limiter.refund_count(600); + EXPECT_EQ(0, limiter.add(400)); + limiter.refund_count(400); +} + +TEST(TokenBucketRateLimiterTest, token_refund_does_not_release_count) { + TokenBucketRateLimiter limiter(kNoThrottle, kNoThrottle, 600); + + EXPECT_EQ(0, limiter.add(600)); + limiter.refund_tokens(500); + EXPECT_EQ(-1, limiter.add(1)); + + limiter.refund_count(600); + EXPECT_EQ(0, limiter.add(600)); + limiter.refund_count(600); +} + +TEST(TokenBucketRateLimiterTest, count_refund_does_not_return_tokens) { + TokenBucketRateLimiter limiter(10000, 1000, 1000); + + EXPECT_EQ(0, limiter.add(1000)); + limiter.refund_count(1000); + + EXPECT_GT(limiter.add(1000), 0); + limiter.refund_count(1000); +} + TEST(S3RateLimitGuardTest, disabled_limiter_admits_everything) { RateLimiterConfigGuard guard; config::enable_s3_rate_limiter = false; @@ -242,49 +279,59 @@ TEST(S3RateLimitGuardTest, legacy_count_limit_rejects) { // No throttling, hard count limit of 2. manager.qps_limiter(S3RateLimitType::GET)->reset(0, 0, 2); - S3RateLimitGuard g1(S3RateLimitType::GET, 0); - EXPECT_TRUE(g1.ok()); - S3RateLimitGuard g2(S3RateLimitType::GET, 0); - EXPECT_TRUE(g2.ok()); + { + S3RateLimitGuard g1(S3RateLimitType::GET, 0); + EXPECT_TRUE(g1.ok()); + } + { + S3RateLimitGuard g2(S3RateLimitType::GET, 0); + EXPECT_TRUE(g2.ok()); + } + // QPS guards do not release the legacy cumulative count on destruction. S3RateLimitGuard g3(S3RateLimitType::GET, 0); EXPECT_FALSE(g3.ok()); } -TEST(S3RateLimitGuardTest, settle_refunds_short_read) { +TEST(S3RateLimitGuardTest, settle_refunds_only_short_read_tokens) { RateLimiterConfigGuard guard; config::enable_s3_rate_limiter = true; auto& manager = S3RateLimiterManager::instance(); auto* bytes = manager.bytes_limiter(S3RateLimitType::GET); manager.qps_limiter(S3RateLimitType::GET)->reset(0, 0, 0); - // A large speed/burst removes wall-clock refill from the assertion. The count limit - // makes the refund observable exactly. - bytes->reset(kNoThrottle, kNoThrottle, 1000); + bytes->reset(kNoThrottle, kNoThrottle, 600); { S3RateLimitGuard g(S3RateLimitType::GET, 600); ASSERT_TRUE(g.ok()); g.settle(100); // short read: 500 tokens must come back + + // Token settlement does not release the in-flight reservation count. + S3RateLimitGuard rejected(S3RateLimitType::GET, 1); + EXPECT_FALSE(rejected.ok()); } - // The guard leaves 100 cumulatively charged bytes, so exactly 900 more are admitted. - EXPECT_EQ(0, bytes->add(900)); - EXPECT_EQ(-1, bytes->add(1)); + + S3RateLimitGuard admitted(S3RateLimitType::GET, 600); + EXPECT_TRUE(admitted.ok()); } -TEST(S3RateLimitGuardTest, put_payload_reservation_remains_charged_without_settle) { +TEST(S3RateLimitGuardTest, put_payload_count_is_released_on_destruction) { RateLimiterConfigGuard guard; config::enable_s3_rate_limiter = true; auto& manager = S3RateLimiterManager::instance(); auto* bytes = manager.bytes_limiter(S3RateLimitType::PUT); manager.qps_limiter(S3RateLimitType::PUT)->reset(0, 0, 0); - bytes->reset(kNoThrottle, kNoThrottle, 1000); + bytes->reset(kNoThrottle, kNoThrottle, 600); { S3RateLimitGuard g(S3RateLimitType::PUT, 600); ASSERT_TRUE(g.ok()); - // No settle: e.g. the request failed. The reservation stays charged. + + S3RateLimitGuard rejected(S3RateLimitType::PUT, 1); + EXPECT_FALSE(rejected.ok()); } - EXPECT_EQ(0, bytes->add(400)); // exactly the cumulative count remainder - EXPECT_EQ(-1, bytes->add(1)); + + S3RateLimitGuard admitted(S3RateLimitType::PUT, 600); + EXPECT_TRUE(admitted.ok()); } TEST(S3RateLimitGuardTest, settle_across_reset_does_not_pollute_new_bucket) { @@ -293,20 +340,23 @@ TEST(S3RateLimitGuardTest, settle_across_reset_does_not_pollute_new_bucket) { auto& manager = S3RateLimiterManager::instance(); auto* bytes = manager.bytes_limiter(S3RateLimitType::GET); manager.qps_limiter(S3RateLimitType::GET)->reset(0, 0, 0); + bytes->reset(kNoThrottle, kNoThrottle, 600); + + auto old_guard = std::make_unique(S3RateLimitType::GET, 600); + ASSERT_TRUE(old_guard->ok()); + + // The bucket is swapped while the old request is in flight. bytes->reset(kNoThrottle, kNoThrottle, 1000); + S3RateLimitGuard fresh_guard(S3RateLimitType::GET, 600); + ASSERT_TRUE(fresh_guard.ok()); - { - S3RateLimitGuard g(S3RateLimitType::GET, 600); - ASSERT_TRUE(g.ok()); - // The bucket is swapped while the request is in flight. - bytes->reset(kNoThrottle, kNoThrottle, 1000); - EXPECT_EQ(0, bytes->add(600)); - g.settle(100); // refund lands on the OLD generation, not the fresh bucket - } - // The fresh generation remains charged for 600 bytes. A refund to the wrong - // generation would reduce its cumulative count to 100 and admit the last request. - EXPECT_EQ(0, bytes->add(400)); - EXPECT_EQ(-1, bytes->add(1)); + old_guard->settle(100); + old_guard.reset(); // token and count refunds must land on the old generation + + S3RateLimitGuard fill_fresh_bucket(S3RateLimitType::GET, 400); + EXPECT_TRUE(fill_fresh_bucket.ok()); + S3RateLimitGuard rejected(S3RateLimitType::GET, 1); + EXPECT_FALSE(rejected.ok()); } TEST(S3RateLimitGuardTest, reservation_is_clamped_to_one_second_of_bandwidth) { @@ -316,19 +366,17 @@ TEST(S3RateLimitGuardTest, reservation_is_clamped_to_one_second_of_bandwidth) { auto* bytes = manager.bytes_limiter(S3RateLimitType::GET); manager.qps_limiter(S3RateLimitType::GET)->reset(0, 0, 0); // Keep max_speed at 1000 to define the one-second reservation, but use a huge - // burst to make the test independent from elapsed time while observing the exact - // charged amount through the cumulative limit. + // burst to make the test independent from elapsed time. bytes->reset(1000, kNoThrottle, 1500); - { - // A huge IO only reserves max_speed (=1000). Since actual_bytes exceeds that - // reservation, settle() does not refund it. - S3RateLimitGuard g(S3RateLimitType::GET, 1000000); - ASSERT_TRUE(g.ok()); - g.settle(2000); - } - EXPECT_EQ(0, bytes->add(500)); - EXPECT_EQ(-1, bytes->add(1)); + // A huge IO only reserves max_speed (=1000), leaving 500 bytes of count capacity. + S3RateLimitGuard g(S3RateLimitType::GET, 1000000); + ASSERT_TRUE(g.ok()); + g.settle(2000); + S3RateLimitGuard fill_limit(S3RateLimitType::GET, 500); + EXPECT_TRUE(fill_limit.ok()); + S3RateLimitGuard rejected(S3RateLimitType::GET, 1); + EXPECT_FALSE(rejected.ok()); } } // namespace doris diff --git a/common/cpp/token_bucket_rate_limiter.cpp b/common/cpp/token_bucket_rate_limiter.cpp index b557adee4c0346..073cf772233414 100644 --- a/common/cpp/token_bucket_rate_limiter.cpp +++ b/common/cpp/token_bucket_rate_limiter.cpp @@ -97,6 +97,14 @@ std::pair TokenBucketRateLimiter::_update_remain_token(long now, } _count += amount; count_value = _count; + if (_limit && count_value > _limit) { + // Keep rejection side-effect free. Roll back before releasing the lock so + // concurrent callers cannot observe debt from a request that will not run. + _count -= amount; + if (_max_speed) { + _remain_tokens = std::min(_remain_tokens + amount, _max_burst); + } + } tokens_value = _remain_tokens; _prev_ns_count = now; } @@ -124,12 +132,17 @@ int64_t TokenBucketRateLimiter::add(size_t amount) { return sleep_time_ns; } -void TokenBucketRateLimiter::refund(size_t amount) { +void TokenBucketRateLimiter::refund_tokens(size_t amount) { std::lock_guard lock(*_mutex); if (_max_speed) { _remain_tokens = std::min(_remain_tokens + amount, _max_burst); } - _count = (_count >= amount) ? _count - amount : 0; +} + +void TokenBucketRateLimiter::refund_count(size_t amount) { + std::lock_guard lock(*_mutex); + CHECK_GE(_count, amount); + _count -= amount; } TokenBucketRateLimiterHolder::TokenBucketRateLimiterHolder(size_t max_speed, size_t max_burst, @@ -166,8 +179,9 @@ std::shared_ptr TokenBucketRateLimiterHolder::charge(siz std::shared_lock read {rate_limiter_rw_lock}; limiter = rate_limiter; } - metric_func(limiter->add(amount)); - return limiter; + int64_t sleep_duration = limiter->add(amount); + metric_func(sleep_duration); + return sleep_duration < 0 ? nullptr : limiter; } int TokenBucketRateLimiterHolder::reset(size_t max_speed, size_t max_burst, size_t limit) { diff --git a/common/cpp/token_bucket_rate_limiter.h b/common/cpp/token_bucket_rate_limiter.h index ae11afa0569e4d..34b732196020d3 100644 --- a/common/cpp/token_bucket_rate_limiter.h +++ b/common/cpp/token_bucket_rate_limiter.h @@ -55,14 +55,16 @@ class TokenBucketRateLimiter { TokenBucketRateLimiter(size_t max_speed, size_t max_burst, size_t limit); ~TokenBucketRateLimiter(); - // Use `amount` remain_tokens, sleeps if required or throws exception on limit overflow. - // Returns duration of sleep in nanoseconds (to distinguish sleeping on different kinds of S3RateLimiters for metrics) + // Use `amount` remain_tokens and count, sleeping when rate tokens are insufficient. + // Returns the sleep duration in nanoseconds, or -1 when the count limit rejects the add. int64_t add(size_t amount); - // Return `amount` tokens to the bucket (capped at max_burst) and roll back the - // cumulative counter. Used to reconcile a reservation with the actually consumed - // amount, e.g. a short read at EOF. - void refund(size_t amount); + // Return `amount` rate tokens to the bucket, capped at max_burst. This does not + // release the count charged by add(). + void refund_tokens(size_t amount); + + // Release `amount` from the count charged by add(). This does not return rate tokens. + void refund_count(size_t amount); size_t get_max_speed() const { return _max_speed; } @@ -100,9 +102,9 @@ class TokenBucketRateLimiterHolder { TokenBucketRateLimiterResult add_with_config(size_t amount); // Charge `amount` like add(), but return the limiter generation the tokens were - // taken from. Callers that later refund a reservation must refund on the returned - // object, so that a concurrent reset() cannot make the refund pollute a fresh - // bucket that never saw the original charge. + // taken from, or nullptr when the count limit rejects the charge. Callers that later + // refund a reservation must refund on the returned object, so that a concurrent + // reset() cannot make the refund pollute a fresh bucket that never saw the charge. std::shared_ptr charge(size_t amount); int reset(size_t max_speed, size_t max_burst, size_t limit); From d6b0e4f0d7a9d16f885c82332fc193279c144bc7 Mon Sep 17 00:00:00 2001 From: Refrain Date: Tue, 21 Jul 2026 00:23:34 +0800 Subject: [PATCH 04/22] [fix](be) Refine S3 limiter rejection handling ### What problem does this PR solve? Issue Number: None Related PR: #65420 Problem Summary: S3 rate-limit rejections lacked byte-specific counters and returned the same local error text as QPS rejections. The previous change also split token and count refunds and used the byte limiter count as an outstanding reservation cap. This change adds separate GET/PUT byte rejection bvars, identifies QPS versus byte rejection in the 429 error text, restores unified refund accounting and a zero byte count limit, and retains the atomic rollback that prevents rejected charges from contributing token or count debt. ### Release note Add separate S3 byte limiter rejection metrics and distinguish QPS and byte limiter rejection messages. ### Check List (For Author) - Test: Unit Test - ./run-be-ut.sh --run --filter=TokenBucketRateLimiterTest.*:S3RateLimiterManagerTest.*:S3RateLimitGuardTest.*:RateLimitedObjStorageClientTest.* -j48 - Behavior changed: Yes. S3 limiter 429 messages identify QPS or byte rejection, and byte limiter rejection counters are exposed. - Does this need documentation: No --- be/src/common/config.cpp | 3 - .../io/fs/rate_limited_obj_storage_client.cpp | 29 ++--- be/src/util/s3_rate_limiter_manager.cpp | 30 ++--- be/src/util/s3_rate_limiter_manager.h | 24 ++-- .../rate_limited_obj_storage_client_test.cpp | 66 ++++++++--- be/test/util/s3_rate_limiter_manager_test.cpp | 109 +++++++----------- common/cpp/token_bucket_rate_limiter.cpp | 9 +- common/cpp/token_bucket_rate_limiter.h | 10 +- 8 files changed, 143 insertions(+), 137 deletions(-) diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index dc933b18be03ef..05871682376f41 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -1559,9 +1559,6 @@ DEFINE_Validator(s3_put_qps_max, [](int64_t config) -> bool { return config >= 0 // CPU-aware S3 bandwidth limiter. Effective GET/PUT bytes/s = bytes_per_second_per_core * // BE cpu cores, capped by the corresponding bytes_per_second_max. -1 and 0 both disable // byte-rate limiting for that operation (there is no legacy fallback for bandwidth). -// Waiting and executing reservations are capped at one second worth of effective bandwidth. -// A byte reservation that would exceed that cap is rejected and its byte-rate token charge is -// rolled back. // Note: the derived per-BE bytes/s should not be set below the single IO upper bound // per second (s3_write_buffer_size, 5MB by default). A single IO larger than 1 second // of quota only reserves 1 second worth of tokens; the excess bytes are not accounted diff --git a/be/src/io/fs/rate_limited_obj_storage_client.cpp b/be/src/io/fs/rate_limited_obj_storage_client.cpp index b24123b1ab488d..8ed27a4454667a 100644 --- a/be/src/io/fs/rate_limited_obj_storage_client.cpp +++ b/be/src/io/fs/rate_limited_obj_storage_client.cpp @@ -17,16 +17,19 @@ #include "io/fs/rate_limited_obj_storage_client.h" +#include "common/logging.h" #include "common/status.h" #include "util/s3_rate_limiter_manager.h" namespace doris::io { namespace { -ObjectStorageResponse rate_limited_response(S3RateLimitType type) { +ObjectStorageResponse rate_limited_response(S3RateLimitType type, S3RateLimitRejectReason reason) { + CHECK(reason != S3RateLimitRejectReason::NONE); + const auto* limit_type = reason == S3RateLimitRejectReason::QPS ? "QPS" : "bytes"; return {.status = convert_to_obj_response(Status::Error( - "s3 {} request exceeds request limit, rejected by BE rate limiter", - to_string(type))), + "s3 {} request exceeds {} limit, rejected by BE rate limiter", to_string(type), + limit_type)), .http_code = 429}; } @@ -36,7 +39,7 @@ ObjectStorageUploadResponse RateLimitedObjStorageClient::create_multipart_upload const ObjectStoragePathOptions& opts) { S3RateLimitGuard guard(S3RateLimitType::PUT, 0); if (!guard.ok()) { - return {.resp = rate_limited_response(S3RateLimitType::PUT)}; + return {.resp = rate_limited_response(S3RateLimitType::PUT, guard.reject_reason())}; } return _inner->create_multipart_upload(opts); } @@ -45,7 +48,7 @@ ObjectStorageResponse RateLimitedObjStorageClient::put_object(const ObjectStorag std::string_view stream) { S3RateLimitGuard guard(S3RateLimitType::PUT, stream.size()); if (!guard.ok()) { - return rate_limited_response(S3RateLimitType::PUT); + return rate_limited_response(S3RateLimitType::PUT, guard.reject_reason()); } return _inner->put_object(opts, stream); } @@ -54,7 +57,7 @@ ObjectStorageUploadResponse RateLimitedObjStorageClient::upload_part( const ObjectStoragePathOptions& opts, std::string_view stream, int part_num) { S3RateLimitGuard guard(S3RateLimitType::PUT, stream.size()); if (!guard.ok()) { - return {.resp = rate_limited_response(S3RateLimitType::PUT)}; + return {.resp = rate_limited_response(S3RateLimitType::PUT, guard.reject_reason())}; } return _inner->upload_part(opts, stream, part_num); } @@ -64,7 +67,7 @@ ObjectStorageResponse RateLimitedObjStorageClient::complete_multipart_upload( const std::vector& completed_parts) { S3RateLimitGuard guard(S3RateLimitType::PUT, 0); if (!guard.ok()) { - return rate_limited_response(S3RateLimitType::PUT); + return rate_limited_response(S3RateLimitType::PUT, guard.reject_reason()); } return _inner->complete_multipart_upload(opts, completed_parts); } @@ -73,7 +76,7 @@ ObjectStorageHeadResponse RateLimitedObjStorageClient::head_object( const ObjectStoragePathOptions& opts) { S3RateLimitGuard guard(S3RateLimitType::GET, 0); if (!guard.ok()) { - return {.resp = rate_limited_response(S3RateLimitType::GET)}; + return {.resp = rate_limited_response(S3RateLimitType::GET, guard.reject_reason())}; } return _inner->head_object(opts); } @@ -84,7 +87,7 @@ ObjectStorageResponse RateLimitedObjStorageClient::get_object(const ObjectStorag size_t* size_return) { S3RateLimitGuard guard(S3RateLimitType::GET, bytes_read); if (!guard.ok()) { - return rate_limited_response(S3RateLimitType::GET); + return rate_limited_response(S3RateLimitType::GET, guard.reject_reason()); } auto resp = _inner->get_object(opts, buffer, offset, bytes_read, size_return); if (resp.status.code == 0) { @@ -98,7 +101,7 @@ ObjectStorageResponse RateLimitedObjStorageClient::list_objects( const ObjectStoragePathOptions& opts, std::vector* files) { S3RateLimitGuard guard(S3RateLimitType::GET, 0); if (!guard.ok()) { - return rate_limited_response(S3RateLimitType::GET); + return rate_limited_response(S3RateLimitType::GET, guard.reject_reason()); } return _inner->list_objects(opts, files); } @@ -107,7 +110,7 @@ ObjectStorageResponse RateLimitedObjStorageClient::delete_objects( const ObjectStoragePathOptions& opts, std::vector objs) { S3RateLimitGuard guard(S3RateLimitType::PUT, 0); if (!guard.ok()) { - return rate_limited_response(S3RateLimitType::PUT); + return rate_limited_response(S3RateLimitType::PUT, guard.reject_reason()); } return _inner->delete_objects(opts, std::move(objs)); } @@ -116,7 +119,7 @@ ObjectStorageResponse RateLimitedObjStorageClient::delete_object( const ObjectStoragePathOptions& opts) { S3RateLimitGuard guard(S3RateLimitType::PUT, 0); if (!guard.ok()) { - return rate_limited_response(S3RateLimitType::PUT); + return rate_limited_response(S3RateLimitType::PUT, guard.reject_reason()); } return _inner->delete_object(opts); } @@ -125,7 +128,7 @@ ObjectStorageResponse RateLimitedObjStorageClient::delete_objects_recursively( const ObjectStoragePathOptions& opts) { S3RateLimitGuard guard(S3RateLimitType::PUT, 0); if (!guard.ok()) { - return rate_limited_response(S3RateLimitType::PUT); + return rate_limited_response(S3RateLimitType::PUT, guard.reject_reason()); } return _inner->delete_objects_recursively(opts); } diff --git a/be/src/util/s3_rate_limiter_manager.cpp b/be/src/util/s3_rate_limiter_manager.cpp index 3f9c1a149bb220..b3210d473a42e3 100644 --- a/be/src/util/s3_rate_limiter_manager.cpp +++ b/be/src/util/s3_rate_limiter_manager.cpp @@ -29,8 +29,12 @@ namespace doris { bvar::Adder s3_get_bytes_rate_limit_sleep_ns("s3_get_bytes_rate_limit_sleep_ns"); bvar::Adder s3_get_bytes_rate_limit_sleep_count("s3_get_bytes_rate_limit_sleep_count"); +bvar::Adder s3_get_bytes_rate_limit_rejected_count( + "s3_get_bytes_rate_limit_rejected_count"); bvar::Adder s3_put_bytes_rate_limit_sleep_ns("s3_put_bytes_rate_limit_sleep_ns"); bvar::Adder s3_put_bytes_rate_limit_sleep_count("s3_put_bytes_rate_limit_sleep_count"); +bvar::Adder s3_put_bytes_rate_limit_rejected_count( + "s3_put_bytes_rate_limit_rejected_count"); namespace { @@ -38,10 +42,12 @@ std::function bytes_rate_limiter_metric_func(S3RateLimitType type switch (type) { case S3RateLimitType::GET: return metric_func_factory(s3_get_bytes_rate_limit_sleep_ns, - s3_get_bytes_rate_limit_sleep_count); + s3_get_bytes_rate_limit_sleep_count, + &s3_get_bytes_rate_limit_rejected_count); case S3RateLimitType::PUT: return metric_func_factory(s3_put_bytes_rate_limit_sleep_ns, - s3_put_bytes_rate_limit_sleep_count); + s3_put_bytes_rate_limit_sleep_count, + &s3_put_bytes_rate_limit_rejected_count); default: return [](int64_t) {}; } @@ -115,7 +121,7 @@ S3RateLimiterManager::S3RateLimiterManager() { _qps_limiters[index_of(type)] = std::make_unique( limit.qps, limit.burst, limit.count_limit, s3_rate_limiter_metric_func(type)); _bytes_limiters[index_of(type)] = std::make_unique( - limit.bytes_per_second, limit.bytes_per_second, limit.bytes_per_second, + limit.bytes_per_second, limit.bytes_per_second, 0, bytes_rate_limiter_metric_func(type)); } } @@ -150,10 +156,8 @@ void S3RateLimiterManager::refresh() { } auto* bytes = bytes_limiter(type); - const auto bytes_per_second = static_cast(limit.bytes_per_second); - if (bytes->get_max_speed() != bytes_per_second || - bytes->get_max_burst() != bytes_per_second || bytes->get_limit() != bytes_per_second) { - bytes->reset(bytes_per_second, bytes_per_second, bytes_per_second); + if (bytes->get_max_speed() != static_cast(limit.bytes_per_second)) { + bytes->reset(limit.bytes_per_second, limit.bytes_per_second, 0); LOG(INFO) << "reset S3 " << to_string(type) << " bytes rate limiter, bytes_per_second=" << limit.bytes_per_second << ", cores=" << cores; @@ -171,6 +175,7 @@ S3RateLimitGuard::S3RateLimitGuard(S3RateLimitType type, size_t estimated_bytes) if (qps->is_enabled() && apply_s3_rate_limit(type, qps, config::s3_rate_limiter_log_interval) < 0) { _ok = false; + _reject_reason = S3RateLimitRejectReason::QPS; return; } @@ -187,27 +192,22 @@ S3RateLimitGuard::S3RateLimitGuard(S3RateLimitType type, size_t estimated_bytes) // upper bound are excluded by the config contract (see config.cpp). _reserved = std::min(estimated_bytes, bytes->get_max_speed()); if (_reserved > 0) { - // Pin the admitted bucket generation for settlement and count release. + // Pin the admitted bucket generation for settle(). _charged_bucket = bytes->charge(_reserved); if (_charged_bucket == nullptr) { _ok = false; + _reject_reason = S3RateLimitRejectReason::BYTES; } } } -S3RateLimitGuard::~S3RateLimitGuard() { - if (_charged_bucket != nullptr) { - _charged_bucket->refund_count(_reserved); - } -} - void S3RateLimitGuard::settle(size_t actual_bytes) { if (_settled) { return; } _settled = true; if (_charged_bucket != nullptr && _reserved > actual_bytes) { - _charged_bucket->refund_tokens(_reserved - actual_bytes); + _charged_bucket->refund(_reserved - actual_bytes); } } diff --git a/be/src/util/s3_rate_limiter_manager.h b/be/src/util/s3_rate_limiter_manager.h index 9707f08b0b4dea..0c7a63b979fe19 100644 --- a/be/src/util/s3_rate_limiter_manager.h +++ b/be/src/util/s3_rate_limiter_manager.h @@ -52,6 +52,12 @@ int64_t s3_rate_limiter_cpu_cores(); // override a manual reset as soon as the config-resolved parameters differ. int reset_s3_rate_limiter(S3RateLimitType type, size_t max_speed, size_t max_burst, size_t limit); +enum class S3RateLimitRejectReason { + NONE, + QPS, + BYTES, +}; + // Owns the 4 process-wide token buckets (GET/PUT x QPS/bytes). Independent of // S3ClientFactory so that instantiating it never initializes the AWS SDK. class S3RateLimiterManager { @@ -83,25 +89,26 @@ class S3RateLimiterManager { // The constructor charges the QPS bucket (may sleep when throttled; rejected only by // the legacy token_limit cumulative cap) and then reserves `estimated_bytes` from the // bytes bucket, clamped to at most 1 second worth of bandwidth so a single huge IO -// cannot create unbounded upfront debt. The bytes bucket rejects a reservation when -// admitting it would make the total waiting and executing bytes exceed that amount. +// cannot create unbounded upfront debt (may sleep; normally does not reject because its +// count limit is 0). // -// settle(actual) returns only unused rate tokens when the actual transferred bytes are -// smaller than the reservation (e.g. a short read at EOF). Destruction always releases -// the full reservation from the bytes bucket count without returning rate tokens. +// settle(actual) refunds the difference when the actual transferred bytes are smaller +// than the reservation (e.g. a short read at EOF). An unsettled guard keeps the full +// reservation charged, which is the conservative choice for failed requests. // // The guard pins the bucket generation it charged: if refresh() resets the bytes -// bucket while the request is in flight, token settlement and count release apply to -// the old generation instead of polluting the fresh bucket. +// bucket while the request is in flight, settle() refunds on the old generation +// (kept alive by the guard's shared_ptr) instead of polluting the fresh bucket. class S3RateLimitGuard { public: S3RateLimitGuard(S3RateLimitType type, size_t estimated_bytes); - ~S3RateLimitGuard(); + ~S3RateLimitGuard() = default; S3RateLimitGuard(const S3RateLimitGuard&) = delete; S3RateLimitGuard& operator=(const S3RateLimitGuard&) = delete; bool ok() const { return _ok; } + S3RateLimitRejectReason reject_reason() const { return _reject_reason; } void settle(size_t actual_bytes); private: @@ -109,6 +116,7 @@ class S3RateLimitGuard { std::shared_ptr _charged_bucket; bool _ok = true; bool _settled = false; + S3RateLimitRejectReason _reject_reason = S3RateLimitRejectReason::NONE; }; } // namespace doris diff --git a/be/test/io/fs/rate_limited_obj_storage_client_test.cpp b/be/test/io/fs/rate_limited_obj_storage_client_test.cpp index 77aab7289815d6..5643f6e0d690b0 100644 --- a/be/test/io/fs/rate_limited_obj_storage_client_test.cpp +++ b/be/test/io/fs/rate_limited_obj_storage_client_test.cpp @@ -23,6 +23,13 @@ #include "util/s3_rate_limiter_manager.h" #include "util/s3_util.h" +namespace doris { + +extern bvar::Adder s3_get_bytes_rate_limit_rejected_count; +extern bvar::Adder s3_put_bytes_rate_limit_rejected_count; + +} // namespace doris + namespace doris::io { namespace { @@ -145,6 +152,8 @@ TEST(RateLimitedObjStorageClientTest, get_rejected_by_count_limit_does_not_reach auto& manager = S3RateLimiterManager::instance(); manager.qps_limiter(S3RateLimitType::GET)->reset(0, 0, 1); manager.qps_limiter(S3RateLimitType::PUT)->reset(0, 0, 0); + manager.bytes_limiter(S3RateLimitType::GET)->reset(0, 0, 0); + manager.bytes_limiter(S3RateLimitType::PUT)->reset(0, 0, 0); auto fake = std::make_shared(); RateLimitedObjStorageClient client(fake); @@ -156,7 +165,7 @@ TEST(RateLimitedObjStorageClientTest, get_rejected_by_count_limit_does_not_reach auto resp = client.head_object(opts); EXPECT_NE(0, resp.resp.status.code); EXPECT_EQ(429, resp.resp.http_code); - EXPECT_NE(std::string::npos, resp.resp.status.msg.find("exceeds request limit")); + EXPECT_NE(std::string::npos, resp.resp.status.msg.find("exceeds QPS limit")); EXPECT_EQ(1, fake->calls); // rejected before reaching the provider // PUT uses an independent bucket and is unaffected. @@ -170,7 +179,7 @@ TEST(RateLimitedObjStorageClientTest, get_object_settles_short_read) { auto& manager = S3RateLimiterManager::instance(); manager.qps_limiter(S3RateLimitType::GET)->reset(0, 0, 0); auto* bytes = manager.bytes_limiter(S3RateLimitType::GET); - bytes->reset(1000, 1000, 1000); + bytes->reset(kNoThrottleBytesPerSecond, kNoThrottleBytesPerSecond, 1000); auto fake = std::make_shared(); fake->actual_read_size = 100; // short read: 600 requested, 100 returned @@ -181,13 +190,12 @@ TEST(RateLimitedObjStorageClientTest, get_object_settles_short_read) { EXPECT_EQ(0, client.get_object(opts, nullptr, 0, 600, &size_return).status.code); EXPECT_EQ(100, size_return); - // The count reservation is released, while only the 500 unused rate tokens are - // returned. The resulting 900-token balance admits exactly this request immediately. + // Only 100 bytes remain cumulatively charged, so exactly 900 more are admitted. EXPECT_EQ(0, bytes->add(900)); - bytes->reset(0, 0, 0); + EXPECT_EQ(-1, bytes->add(1)); } -TEST(RateLimitedObjStorageClientTest, bytes_limit_rejects_before_calling_inner) { +TEST(RateLimitedObjStorageClientTest, put_object_charges_payload_bytes) { RateLimiterConfigGuard guard; config::enable_s3_rate_limiter = true; auto& manager = S3RateLimiterManager::instance(); @@ -199,18 +207,44 @@ TEST(RateLimitedObjStorageClientTest, bytes_limit_rejects_before_calling_inner) RateLimitedObjStorageClient client(fake); ObjectStoragePathOptions opts {.bucket = "b", .key = "k"}; - { - S3RateLimitGuard in_flight(S3RateLimitType::PUT, 1000); - ASSERT_TRUE(in_flight.ok()); + std::string payload(600, 'x'); + EXPECT_EQ(0, client.put_object(opts, payload).status.code); + EXPECT_EQ(0, bytes->add(400)); // exactly the cumulative count remainder + EXPECT_EQ(-1, bytes->add(1)); +} - auto resp = client.put_object(opts, "x"); - EXPECT_NE(0, resp.status.code); - EXPECT_EQ(429, resp.http_code); - EXPECT_EQ(0, fake->calls); - } +TEST(RateLimitedObjStorageClientTest, bytes_rejections_have_distinct_text_and_metrics) { + RateLimiterConfigGuard guard; + config::enable_s3_rate_limiter = true; + auto& manager = S3RateLimiterManager::instance(); + manager.qps_limiter(S3RateLimitType::GET)->reset(0, 0, 0); + manager.qps_limiter(S3RateLimitType::PUT)->reset(0, 0, 0); + manager.bytes_limiter(S3RateLimitType::GET) + ->reset(kNoThrottleBytesPerSecond, kNoThrottleBytesPerSecond, 1); + manager.bytes_limiter(S3RateLimitType::PUT) + ->reset(kNoThrottleBytesPerSecond, kNoThrottleBytesPerSecond, 1); - EXPECT_EQ(0, client.put_object(opts, std::string(1000, 'x')).status.code); - EXPECT_EQ(1, fake->calls); + const int64_t get_rejected_before = s3_get_bytes_rate_limit_rejected_count.get_value(); + const int64_t put_rejected_before = s3_put_bytes_rate_limit_rejected_count.get_value(); + + auto fake = std::make_shared(); + RateLimitedObjStorageClient client(fake); + ObjectStoragePathOptions opts {.bucket = "b", .key = "k"}; + + size_t size_return = 0; + auto get_resp = client.get_object(opts, nullptr, 0, 2, &size_return); + EXPECT_NE(0, get_resp.status.code); + EXPECT_EQ(429, get_resp.http_code); + EXPECT_NE(std::string::npos, get_resp.status.msg.find("exceeds bytes limit")); + EXPECT_EQ(get_rejected_before + 1, s3_get_bytes_rate_limit_rejected_count.get_value()); + + auto put_resp = client.put_object(opts, "xx"); + EXPECT_NE(0, put_resp.status.code); + EXPECT_EQ(429, put_resp.http_code); + EXPECT_NE(std::string::npos, put_resp.status.msg.find("exceeds bytes limit")); + EXPECT_EQ(put_rejected_before + 1, s3_put_bytes_rate_limit_rejected_count.get_value()); + + EXPECT_EQ(0, fake->calls); } TEST(RateLimitedObjStorageClientTest, recursive_delete_charges_one_put_qps) { diff --git a/be/test/util/s3_rate_limiter_manager_test.cpp b/be/test/util/s3_rate_limiter_manager_test.cpp index 681a6156bb1423..56350b32a20887 100644 --- a/be/test/util/s3_rate_limiter_manager_test.cpp +++ b/be/test/util/s3_rate_limiter_manager_test.cpp @@ -219,7 +219,7 @@ TEST(S3RateLimiterManagerTest, refresh_applies_bytes_limit_and_enables_bucket) { EXPECT_TRUE(put_bytes->is_enabled()); EXPECT_EQ(1000 * kCores, put_bytes->get_max_speed()); EXPECT_EQ(1000 * kCores, put_bytes->get_max_burst()); - EXPECT_EQ(1000 * kCores, put_bytes->get_limit()); + EXPECT_EQ(0, put_bytes->get_limit()); config::s3_put_bytes_per_second_per_core = -1; manager.refresh(); @@ -227,38 +227,15 @@ TEST(S3RateLimiterManagerTest, refresh_applies_bytes_limit_and_enables_bucket) { } TEST(TokenBucketRateLimiterTest, rejected_add_rolls_back_count_and_tokens) { - TokenBucketRateLimiter limiter(1000, 1000, 600); + TokenBucketRateLimiter limiter(1000, 1000, 1000); - EXPECT_EQ(0, limiter.add(600)); - EXPECT_EQ(-1, limiter.add(500)); + EXPECT_EQ(0, limiter.add(1000)); + EXPECT_EQ(-1, limiter.add(1000)); // Releasing the admitted request makes room for another request. The rejected // request must not leave either count or token debt behind. - limiter.refund_count(600); - EXPECT_EQ(0, limiter.add(400)); - limiter.refund_count(400); -} - -TEST(TokenBucketRateLimiterTest, token_refund_does_not_release_count) { - TokenBucketRateLimiter limiter(kNoThrottle, kNoThrottle, 600); - - EXPECT_EQ(0, limiter.add(600)); - limiter.refund_tokens(500); - EXPECT_EQ(-1, limiter.add(1)); - - limiter.refund_count(600); - EXPECT_EQ(0, limiter.add(600)); - limiter.refund_count(600); -} - -TEST(TokenBucketRateLimiterTest, count_refund_does_not_return_tokens) { - TokenBucketRateLimiter limiter(10000, 1000, 1000); - + limiter.refund(1000); EXPECT_EQ(0, limiter.add(1000)); - limiter.refund_count(1000); - - EXPECT_GT(limiter.add(1000), 0); - limiter.refund_count(1000); } TEST(S3RateLimitGuardTest, disabled_limiter_admits_everything) { @@ -292,46 +269,41 @@ TEST(S3RateLimitGuardTest, legacy_count_limit_rejects) { EXPECT_FALSE(g3.ok()); } -TEST(S3RateLimitGuardTest, settle_refunds_only_short_read_tokens) { +TEST(S3RateLimitGuardTest, settle_refunds_short_read) { RateLimiterConfigGuard guard; config::enable_s3_rate_limiter = true; auto& manager = S3RateLimiterManager::instance(); auto* bytes = manager.bytes_limiter(S3RateLimitType::GET); manager.qps_limiter(S3RateLimitType::GET)->reset(0, 0, 0); - bytes->reset(kNoThrottle, kNoThrottle, 600); + // A large speed/burst removes wall-clock refill from the assertion. The count limit + // makes the refund observable exactly. + bytes->reset(kNoThrottle, kNoThrottle, 1000); { S3RateLimitGuard g(S3RateLimitType::GET, 600); ASSERT_TRUE(g.ok()); g.settle(100); // short read: 500 tokens must come back - - // Token settlement does not release the in-flight reservation count. - S3RateLimitGuard rejected(S3RateLimitType::GET, 1); - EXPECT_FALSE(rejected.ok()); } - - S3RateLimitGuard admitted(S3RateLimitType::GET, 600); - EXPECT_TRUE(admitted.ok()); + // The guard leaves 100 cumulatively charged bytes, so exactly 900 more are admitted. + EXPECT_EQ(0, bytes->add(900)); + EXPECT_EQ(-1, bytes->add(1)); } -TEST(S3RateLimitGuardTest, put_payload_count_is_released_on_destruction) { +TEST(S3RateLimitGuardTest, put_payload_reservation_remains_charged_without_settle) { RateLimiterConfigGuard guard; config::enable_s3_rate_limiter = true; auto& manager = S3RateLimiterManager::instance(); auto* bytes = manager.bytes_limiter(S3RateLimitType::PUT); manager.qps_limiter(S3RateLimitType::PUT)->reset(0, 0, 0); - bytes->reset(kNoThrottle, kNoThrottle, 600); + bytes->reset(kNoThrottle, kNoThrottle, 1000); { S3RateLimitGuard g(S3RateLimitType::PUT, 600); ASSERT_TRUE(g.ok()); - - S3RateLimitGuard rejected(S3RateLimitType::PUT, 1); - EXPECT_FALSE(rejected.ok()); + // No settle: e.g. the request failed. The reservation stays charged. } - - S3RateLimitGuard admitted(S3RateLimitType::PUT, 600); - EXPECT_TRUE(admitted.ok()); + EXPECT_EQ(0, bytes->add(400)); // exactly the cumulative count remainder + EXPECT_EQ(-1, bytes->add(1)); } TEST(S3RateLimitGuardTest, settle_across_reset_does_not_pollute_new_bucket) { @@ -340,23 +312,20 @@ TEST(S3RateLimitGuardTest, settle_across_reset_does_not_pollute_new_bucket) { auto& manager = S3RateLimiterManager::instance(); auto* bytes = manager.bytes_limiter(S3RateLimitType::GET); manager.qps_limiter(S3RateLimitType::GET)->reset(0, 0, 0); - bytes->reset(kNoThrottle, kNoThrottle, 600); - - auto old_guard = std::make_unique(S3RateLimitType::GET, 600); - ASSERT_TRUE(old_guard->ok()); - - // The bucket is swapped while the old request is in flight. bytes->reset(kNoThrottle, kNoThrottle, 1000); - S3RateLimitGuard fresh_guard(S3RateLimitType::GET, 600); - ASSERT_TRUE(fresh_guard.ok()); - - old_guard->settle(100); - old_guard.reset(); // token and count refunds must land on the old generation - S3RateLimitGuard fill_fresh_bucket(S3RateLimitType::GET, 400); - EXPECT_TRUE(fill_fresh_bucket.ok()); - S3RateLimitGuard rejected(S3RateLimitType::GET, 1); - EXPECT_FALSE(rejected.ok()); + { + S3RateLimitGuard g(S3RateLimitType::GET, 600); + ASSERT_TRUE(g.ok()); + // The bucket is swapped while the request is in flight. + bytes->reset(kNoThrottle, kNoThrottle, 1000); + EXPECT_EQ(0, bytes->add(600)); + g.settle(100); // refund lands on the OLD generation, not the fresh bucket + } + // The fresh generation remains charged for 600 bytes. A refund to the wrong + // generation would reduce its cumulative count to 100 and admit the last request. + EXPECT_EQ(0, bytes->add(400)); + EXPECT_EQ(-1, bytes->add(1)); } TEST(S3RateLimitGuardTest, reservation_is_clamped_to_one_second_of_bandwidth) { @@ -366,17 +335,19 @@ TEST(S3RateLimitGuardTest, reservation_is_clamped_to_one_second_of_bandwidth) { auto* bytes = manager.bytes_limiter(S3RateLimitType::GET); manager.qps_limiter(S3RateLimitType::GET)->reset(0, 0, 0); // Keep max_speed at 1000 to define the one-second reservation, but use a huge - // burst to make the test independent from elapsed time. + // burst to make the test independent from elapsed time while observing the exact + // charged amount through the cumulative limit. bytes->reset(1000, kNoThrottle, 1500); - // A huge IO only reserves max_speed (=1000), leaving 500 bytes of count capacity. - S3RateLimitGuard g(S3RateLimitType::GET, 1000000); - ASSERT_TRUE(g.ok()); - g.settle(2000); - S3RateLimitGuard fill_limit(S3RateLimitType::GET, 500); - EXPECT_TRUE(fill_limit.ok()); - S3RateLimitGuard rejected(S3RateLimitType::GET, 1); - EXPECT_FALSE(rejected.ok()); + { + // A huge IO only reserves max_speed (=1000). Since actual_bytes exceeds that + // reservation, settle() does not refund it. + S3RateLimitGuard g(S3RateLimitType::GET, 1000000); + ASSERT_TRUE(g.ok()); + g.settle(2000); + } + EXPECT_EQ(0, bytes->add(500)); + EXPECT_EQ(-1, bytes->add(1)); } } // namespace doris diff --git a/common/cpp/token_bucket_rate_limiter.cpp b/common/cpp/token_bucket_rate_limiter.cpp index 073cf772233414..666fba7dbf1fee 100644 --- a/common/cpp/token_bucket_rate_limiter.cpp +++ b/common/cpp/token_bucket_rate_limiter.cpp @@ -132,17 +132,12 @@ int64_t TokenBucketRateLimiter::add(size_t amount) { return sleep_time_ns; } -void TokenBucketRateLimiter::refund_tokens(size_t amount) { +void TokenBucketRateLimiter::refund(size_t amount) { std::lock_guard lock(*_mutex); if (_max_speed) { _remain_tokens = std::min(_remain_tokens + amount, _max_burst); } -} - -void TokenBucketRateLimiter::refund_count(size_t amount) { - std::lock_guard lock(*_mutex); - CHECK_GE(_count, amount); - _count -= amount; + _count = (_count >= amount) ? _count - amount : 0; } TokenBucketRateLimiterHolder::TokenBucketRateLimiterHolder(size_t max_speed, size_t max_burst, diff --git a/common/cpp/token_bucket_rate_limiter.h b/common/cpp/token_bucket_rate_limiter.h index 34b732196020d3..77eed4929e406c 100644 --- a/common/cpp/token_bucket_rate_limiter.h +++ b/common/cpp/token_bucket_rate_limiter.h @@ -59,12 +59,10 @@ class TokenBucketRateLimiter { // Returns the sleep duration in nanoseconds, or -1 when the count limit rejects the add. int64_t add(size_t amount); - // Return `amount` rate tokens to the bucket, capped at max_burst. This does not - // release the count charged by add(). - void refund_tokens(size_t amount); - - // Release `amount` from the count charged by add(). This does not return rate tokens. - void refund_count(size_t amount); + // Return `amount` tokens to the bucket (capped at max_burst) and roll back the + // cumulative counter. Used to reconcile a reservation with the actually consumed + // amount, e.g. a short read at EOF. + void refund(size_t amount); size_t get_max_speed() const { return _max_speed; } From 032e2366ba6b593af4cda5af88d279d305fac2ab Mon Sep 17 00:00:00 2001 From: Refrain Date: Tue, 21 Jul 2026 12:28:29 +0800 Subject: [PATCH 05/22] [test](be) Cover S3 rate limiter behavior ### What problem does this PR solve? Issue Number: None Related PR: #65420 Problem Summary: Existing coverage did not fully verify CPU-derived GET/PUT limits, independent QPS and bytes buckets, dynamic refresh, object-storage API mappings, or the real S3 SQL paths. Add focused BE unit and mapping tests, plus a non-cloud real-S3 regression that checks dynamic config readback, directional bvar growth, and concurrent GET/PUT behavior. ### Release note None ### Check List (For Author) - Test: Unit Test and Regression Test - `./run-be-ut.sh --run --filter=S3RateLimiterResolveTest.*:S3RateLimiterManagerTest.*:TokenBucketRateLimiterTest.rejected_add_rolls_back_count_and_tokens:S3RateLimitGuardTest.*:S3RateLimiterMetricsTest.*:RateLimitedObjStorageClientTest.* -j48` - `./run-regression-test.sh --run -d external_table_p2/s3_rate_limiter -s test_s3_rate_limiter -genOut` - `./run-regression-test.sh --run -d external_table_p2/s3_rate_limiter -s test_s3_rate_limiter` - Behavior changed: No - Does this need documentation: No --- .../rate_limited_obj_storage_client_test.cpp | 182 ++++++++++++ be/test/util/s3_rate_limiter_manager_test.cpp | 199 +++++++++++-- .../s3_rate_limiter/test_s3_rate_limiter.out | 16 ++ .../test_s3_rate_limiter.groovy | 266 ++++++++++++++++++ 4 files changed, 634 insertions(+), 29 deletions(-) create mode 100644 regression-test/data/external_table_p2/s3_rate_limiter/test_s3_rate_limiter.out create mode 100644 regression-test/suites/external_table_p2/s3_rate_limiter/test_s3_rate_limiter.groovy diff --git a/be/test/io/fs/rate_limited_obj_storage_client_test.cpp b/be/test/io/fs/rate_limited_obj_storage_client_test.cpp index 5643f6e0d690b0..4bf4ea904518aa 100644 --- a/be/test/io/fs/rate_limited_obj_storage_client_test.cpp +++ b/be/test/io/fs/rate_limited_obj_storage_client_test.cpp @@ -173,6 +173,89 @@ TEST(RateLimitedObjStorageClientTest, get_rejected_by_count_limit_does_not_reach EXPECT_EQ(2, fake->calls); } +TEST(RateLimitedObjStorageClientTest, put_rejected_by_count_limit_does_not_reach_inner) { + RateLimiterConfigGuard guard; + config::enable_s3_rate_limiter = true; + auto& manager = S3RateLimiterManager::instance(); + manager.qps_limiter(S3RateLimitType::GET)->reset(0, 0, 0); + manager.qps_limiter(S3RateLimitType::PUT)->reset(0, 0, 1); + manager.bytes_limiter(S3RateLimitType::GET)->reset(0, 0, 0); + manager.bytes_limiter(S3RateLimitType::PUT)->reset(0, 0, 0); + + auto fake = std::make_shared(); + RateLimitedObjStorageClient client(fake); + ObjectStoragePathOptions opts {.bucket = "b", .key = "k"}; + + EXPECT_EQ(0, client.put_object(opts, "data").status.code); + EXPECT_EQ(1, fake->calls); + + auto resp = client.put_object(opts, "data"); + EXPECT_NE(0, resp.status.code); + EXPECT_EQ(429, resp.http_code); + EXPECT_NE(std::string::npos, resp.status.msg.find("exceeds QPS limit")); + EXPECT_EQ(1, fake->calls); // rejected before reaching the provider + + // GET uses an independent bucket and is unaffected. + EXPECT_EQ(0, client.head_object(opts).resp.status.code); + EXPECT_EQ(2, fake->calls); +} + +TEST(RateLimitedObjStorageClientTest, head_and_list_map_to_get_qps_without_bytes) { + RateLimiterConfigGuard guard; + config::enable_s3_rate_limiter = true; + auto& manager = S3RateLimiterManager::instance(); + auto* get_bytes = manager.bytes_limiter(S3RateLimitType::GET); + manager.qps_limiter(S3RateLimitType::GET) + ->reset(kNoThrottleBytesPerSecond, kNoThrottleBytesPerSecond, 2); + get_bytes->reset(kNoThrottleBytesPerSecond, kNoThrottleBytesPerSecond, 1); + + auto fake = std::make_shared(); + RateLimitedObjStorageClient client(fake); + ObjectStoragePathOptions opts {.bucket = "b", .key = "k", .prefix = "p"}; + + EXPECT_EQ(0, client.head_object(opts).resp.status.code); + std::vector files; + EXPECT_EQ(0, client.list_objects(opts, &files).status.code); + + auto rejected = client.head_object(opts); + EXPECT_NE(0, rejected.resp.status.code); + EXPECT_EQ(429, rejected.resp.http_code); + EXPECT_NE(std::string::npos, rejected.resp.status.msg.find("exceeds QPS limit")); + EXPECT_EQ(2, fake->calls); + + // Neither HEAD nor LIST carries payload bytes. + EXPECT_EQ(0, get_bytes->add(1)); + EXPECT_EQ(-1, get_bytes->add(1)); +} + +TEST(RateLimitedObjStorageClientTest, get_object_maps_to_get_qps_and_get_bytes) { + RateLimiterConfigGuard guard; + config::enable_s3_rate_limiter = true; + auto& manager = S3RateLimiterManager::instance(); + auto* get_bytes = manager.bytes_limiter(S3RateLimitType::GET); + manager.qps_limiter(S3RateLimitType::GET) + ->reset(kNoThrottleBytesPerSecond, kNoThrottleBytesPerSecond, 1); + get_bytes->reset(kNoThrottleBytesPerSecond, kNoThrottleBytesPerSecond, 4); + + auto fake = std::make_shared(); + fake->actual_read_size = 4; + RateLimitedObjStorageClient client(fake); + ObjectStoragePathOptions opts {.bucket = "b", .key = "k"}; + + size_t size_return = 0; + EXPECT_EQ(0, client.get_object(opts, nullptr, 0, 4, &size_return).status.code); + EXPECT_EQ(4, size_return); + + auto rejected = client.get_object(opts, nullptr, 0, 4, &size_return); + EXPECT_NE(0, rejected.status.code); + EXPECT_EQ(429, rejected.http_code); + EXPECT_NE(std::string::npos, rejected.status.msg.find("exceeds QPS limit")); + EXPECT_EQ(1, fake->calls); + + // The admitted GET charged exactly its returned payload bytes. + EXPECT_EQ(-1, get_bytes->add(1)); +} + TEST(RateLimitedObjStorageClientTest, get_object_settles_short_read) { RateLimiterConfigGuard guard; config::enable_s3_rate_limiter = true; @@ -213,6 +296,105 @@ TEST(RateLimitedObjStorageClientTest, put_object_charges_payload_bytes) { EXPECT_EQ(-1, bytes->add(1)); } +TEST(RateLimitedObjStorageClientTest, put_object_maps_to_put_qps_and_put_bytes) { + RateLimiterConfigGuard guard; + config::enable_s3_rate_limiter = true; + auto& manager = S3RateLimiterManager::instance(); + auto* put_bytes = manager.bytes_limiter(S3RateLimitType::PUT); + manager.qps_limiter(S3RateLimitType::PUT) + ->reset(kNoThrottleBytesPerSecond, kNoThrottleBytesPerSecond, 1); + put_bytes->reset(kNoThrottleBytesPerSecond, kNoThrottleBytesPerSecond, 4); + + auto fake = std::make_shared(); + RateLimitedObjStorageClient client(fake); + ObjectStoragePathOptions opts {.bucket = "b", .key = "k"}; + + EXPECT_EQ(0, client.put_object(opts, "data").status.code); + auto rejected = client.put_object(opts, "data"); + EXPECT_NE(0, rejected.status.code); + EXPECT_EQ(429, rejected.http_code); + EXPECT_NE(std::string::npos, rejected.status.msg.find("exceeds QPS limit")); + EXPECT_EQ(1, fake->calls); + + EXPECT_EQ(-1, put_bytes->add(1)); +} + +TEST(RateLimitedObjStorageClientTest, upload_part_maps_to_put_qps_and_put_bytes) { + RateLimiterConfigGuard guard; + config::enable_s3_rate_limiter = true; + auto& manager = S3RateLimiterManager::instance(); + auto* put_bytes = manager.bytes_limiter(S3RateLimitType::PUT); + manager.qps_limiter(S3RateLimitType::PUT) + ->reset(kNoThrottleBytesPerSecond, kNoThrottleBytesPerSecond, 1); + put_bytes->reset(kNoThrottleBytesPerSecond, kNoThrottleBytesPerSecond, 4); + + auto fake = std::make_shared(); + RateLimitedObjStorageClient client(fake); + ObjectStoragePathOptions opts {.bucket = "b", .key = "k", .upload_id = "upload"}; + + EXPECT_EQ(0, client.upload_part(opts, "data", 1).resp.status.code); + auto rejected = client.upload_part(opts, "data", 2); + EXPECT_NE(0, rejected.resp.status.code); + EXPECT_EQ(429, rejected.resp.http_code); + EXPECT_NE(std::string::npos, rejected.resp.status.msg.find("exceeds QPS limit")); + EXPECT_EQ(1, fake->calls); + + EXPECT_EQ(-1, put_bytes->add(1)); +} + +TEST(RateLimitedObjStorageClientTest, multipart_control_apis_map_to_put_qps_without_bytes) { + RateLimiterConfigGuard guard; + config::enable_s3_rate_limiter = true; + auto& manager = S3RateLimiterManager::instance(); + auto* put_bytes = manager.bytes_limiter(S3RateLimitType::PUT); + manager.qps_limiter(S3RateLimitType::PUT) + ->reset(kNoThrottleBytesPerSecond, kNoThrottleBytesPerSecond, 2); + put_bytes->reset(kNoThrottleBytesPerSecond, kNoThrottleBytesPerSecond, 1); + + auto fake = std::make_shared(); + RateLimitedObjStorageClient client(fake); + ObjectStoragePathOptions opts {.bucket = "b", .key = "k", .upload_id = "upload"}; + + EXPECT_EQ(0, client.create_multipart_upload(opts).resp.status.code); + EXPECT_EQ(0, client.complete_multipart_upload(opts, {}).status.code); + + auto rejected = client.create_multipart_upload(opts); + EXPECT_NE(0, rejected.resp.status.code); + EXPECT_EQ(429, rejected.resp.http_code); + EXPECT_NE(std::string::npos, rejected.resp.status.msg.find("exceeds QPS limit")); + EXPECT_EQ(2, fake->calls); + + EXPECT_EQ(0, put_bytes->add(1)); + EXPECT_EQ(-1, put_bytes->add(1)); +} + +TEST(RateLimitedObjStorageClientTest, delete_apis_map_to_put_qps_without_bytes) { + RateLimiterConfigGuard guard; + config::enable_s3_rate_limiter = true; + auto& manager = S3RateLimiterManager::instance(); + auto* put_bytes = manager.bytes_limiter(S3RateLimitType::PUT); + manager.qps_limiter(S3RateLimitType::PUT) + ->reset(kNoThrottleBytesPerSecond, kNoThrottleBytesPerSecond, 3); + put_bytes->reset(kNoThrottleBytesPerSecond, kNoThrottleBytesPerSecond, 1); + + auto fake = std::make_shared(); + RateLimitedObjStorageClient client(fake); + ObjectStoragePathOptions opts {.bucket = "b", .key = "k", .prefix = "p"}; + + EXPECT_EQ(0, client.delete_object(opts).status.code); + EXPECT_EQ(0, client.delete_objects(opts, {"a", "b"}).status.code); + EXPECT_EQ(0, client.delete_objects_recursively(opts).status.code); + + auto rejected = client.delete_object(opts); + EXPECT_NE(0, rejected.status.code); + EXPECT_EQ(429, rejected.http_code); + EXPECT_NE(std::string::npos, rejected.status.msg.find("exceeds QPS limit")); + EXPECT_EQ(3, fake->calls); + + EXPECT_EQ(0, put_bytes->add(1)); + EXPECT_EQ(-1, put_bytes->add(1)); +} + TEST(RateLimitedObjStorageClientTest, bytes_rejections_have_distinct_text_and_metrics) { RateLimiterConfigGuard guard; config::enable_s3_rate_limiter = true; diff --git a/be/test/util/s3_rate_limiter_manager_test.cpp b/be/test/util/s3_rate_limiter_manager_test.cpp index 56350b32a20887..a4c4a021684617 100644 --- a/be/test/util/s3_rate_limiter_manager_test.cpp +++ b/be/test/util/s3_rate_limiter_manager_test.cpp @@ -25,6 +25,9 @@ namespace doris { +extern bvar::Adder s3_get_bytes_rate_limit_sleep_count; +extern bvar::Adder s3_get_bytes_rate_limit_rejected_count; + namespace { // Saves every rate limiter related config on construction, restores it and re-applies @@ -73,6 +76,15 @@ constexpr size_t kNoThrottle = 1ULL << 40; } // namespace +TEST(S3RateLimiterResolveTest, registered_defaults_preserve_legacy_compatibility) { + const auto& fields = *config::Register::_s_field_map; + EXPECT_STREQ("-1", fields.at("s3_get_qps_per_core").defval); + EXPECT_STREQ("-1", fields.at("s3_put_qps_per_core").defval); + EXPECT_STREQ("-1", fields.at("s3_get_bytes_per_second_per_core").defval); + EXPECT_STREQ("-1", fields.at("s3_put_bytes_per_second_per_core").defval); + EXPECT_STREQ("0", fields.at("s3_rate_limiter_cpu_cores").defval); +} + TEST(S3RateLimiterResolveTest, legacy_config_wins_when_per_core_unset) { RateLimiterConfigGuard guard; config::s3_get_qps_per_core = -1; @@ -80,27 +92,49 @@ TEST(S3RateLimiterResolveTest, legacy_config_wins_when_per_core_unset) { config::s3_get_bucket_tokens = 456; config::s3_get_token_limit = 789; config::s3_get_qps_max = 9; // must be ignored on the legacy path + config::s3_put_qps_per_core = -1; + config::s3_put_token_per_second = 321; + config::s3_put_bucket_tokens = 654; + config::s3_put_token_limit = 987; + config::s3_put_qps_max = 8; // must be ignored on the legacy path config::s3_get_bytes_per_second_per_core = -1; + config::s3_put_bytes_per_second_per_core = -1; - auto limit = resolve_s3_rate_limit(S3RateLimitType::GET, kCores); - EXPECT_EQ(123, limit.qps); - EXPECT_EQ(456, limit.burst); - EXPECT_EQ(789, limit.count_limit); - EXPECT_EQ(0, limit.bytes_per_second); + auto get_limit = resolve_s3_rate_limit(S3RateLimitType::GET, kCores); + EXPECT_EQ(123, get_limit.qps); + EXPECT_EQ(456, get_limit.burst); + EXPECT_EQ(789, get_limit.count_limit); + EXPECT_EQ(0, get_limit.bytes_per_second); + + auto put_limit = resolve_s3_rate_limit(S3RateLimitType::PUT, kCores); + EXPECT_EQ(321, put_limit.qps); + EXPECT_EQ(654, put_limit.burst); + EXPECT_EQ(987, put_limit.count_limit); + EXPECT_EQ(0, put_limit.bytes_per_second); } -TEST(S3RateLimiterResolveTest, per_core_config_overrides_legacy) { +TEST(S3RateLimiterResolveTest, get_and_put_per_core_configs_override_legacy_independently) { RateLimiterConfigGuard guard; + config::s3_get_qps_per_core = 3; + config::s3_get_qps_max = 0; + config::s3_get_token_per_second = 321; + config::s3_get_bucket_tokens = 654; + config::s3_get_token_limit = 987; config::s3_put_qps_per_core = 7; config::s3_put_qps_max = 0; config::s3_put_token_per_second = 123; config::s3_put_bucket_tokens = 456; config::s3_put_token_limit = 789; - auto limit = resolve_s3_rate_limit(S3RateLimitType::PUT, kCores); - EXPECT_EQ(7 * kCores, limit.qps); - EXPECT_EQ(7 * kCores, limit.burst); - EXPECT_EQ(0, limit.count_limit); // legacy count cap is dropped on the per-core path + auto get_limit = resolve_s3_rate_limit(S3RateLimitType::GET, kCores); + EXPECT_EQ(3 * kCores, get_limit.qps); + EXPECT_EQ(3 * kCores, get_limit.burst); + EXPECT_EQ(0, get_limit.count_limit); + + auto put_limit = resolve_s3_rate_limit(S3RateLimitType::PUT, kCores); + EXPECT_EQ(7 * kCores, put_limit.qps); + EXPECT_EQ(7 * kCores, put_limit.burst); + EXPECT_EQ(0, put_limit.count_limit); } TEST(S3RateLimiterResolveTest, per_core_zero_disables_qps_limiting) { @@ -108,21 +142,35 @@ TEST(S3RateLimiterResolveTest, per_core_zero_disables_qps_limiting) { config::s3_get_qps_per_core = 0; config::s3_get_token_per_second = 123; config::s3_get_token_limit = 789; - - auto limit = resolve_s3_rate_limit(S3RateLimitType::GET, kCores); - EXPECT_EQ(0, limit.qps); - EXPECT_EQ(0, limit.burst); - EXPECT_EQ(0, limit.count_limit); + config::s3_put_qps_per_core = 0; + config::s3_put_token_per_second = 321; + config::s3_put_token_limit = 987; + + auto get_limit = resolve_s3_rate_limit(S3RateLimitType::GET, kCores); + EXPECT_EQ(0, get_limit.qps); + EXPECT_EQ(0, get_limit.burst); + EXPECT_EQ(0, get_limit.count_limit); + + auto put_limit = resolve_s3_rate_limit(S3RateLimitType::PUT, kCores); + EXPECT_EQ(0, put_limit.qps); + EXPECT_EQ(0, put_limit.burst); + EXPECT_EQ(0, put_limit.count_limit); } -TEST(S3RateLimiterResolveTest, qps_max_caps_per_core_result) { +TEST(S3RateLimiterResolveTest, get_and_put_qps_max_cap_per_core_results_independently) { RateLimiterConfigGuard guard; config::s3_get_qps_per_core = 1000000; config::s3_get_qps_max = 256; + config::s3_put_qps_per_core = 2000000; + config::s3_put_qps_max = 512; - auto limit = resolve_s3_rate_limit(S3RateLimitType::GET, kCores); - EXPECT_EQ(256, limit.qps); - EXPECT_EQ(256, limit.burst); + auto get_limit = resolve_s3_rate_limit(S3RateLimitType::GET, kCores); + EXPECT_EQ(256, get_limit.qps); + EXPECT_EQ(256, get_limit.burst); + + auto put_limit = resolve_s3_rate_limit(S3RateLimitType::PUT, kCores); + EXPECT_EQ(512, put_limit.qps); + EXPECT_EQ(512, put_limit.burst); } TEST(S3RateLimiterResolveTest, overflowing_multiplication_is_capped) { @@ -139,22 +187,30 @@ TEST(S3RateLimiterResolveTest, overflowing_multiplication_is_capped) { EXPECT_EQ(std::numeric_limits::max(), limit.qps); } -TEST(S3RateLimiterResolveTest, bytes_per_core_derives_bandwidth) { +TEST(S3RateLimiterResolveTest, get_and_put_bytes_caps_are_resolved_independently) { RateLimiterConfigGuard guard; config::s3_get_bytes_per_second_per_core = 1024; config::s3_get_bytes_per_second_max = 0; + config::s3_put_bytes_per_second_per_core = 2048; + config::s3_put_bytes_per_second_max = 6144; - auto limit = resolve_s3_rate_limit(S3RateLimitType::GET, kCores); - EXPECT_EQ(1024 * kCores, limit.bytes_per_second); + auto get_limit = resolve_s3_rate_limit(S3RateLimitType::GET, kCores); + EXPECT_EQ(1024 * kCores, get_limit.bytes_per_second); + + auto put_limit = resolve_s3_rate_limit(S3RateLimitType::PUT, kCores); + EXPECT_EQ(6144, put_limit.bytes_per_second); config::s3_get_bytes_per_second_max = 2048; - limit = resolve_s3_rate_limit(S3RateLimitType::GET, kCores); - EXPECT_EQ(2048, limit.bytes_per_second); + get_limit = resolve_s3_rate_limit(S3RateLimitType::GET, kCores); + EXPECT_EQ(2048, get_limit.bytes_per_second); // -1 and 0 both disable bandwidth limiting. config::s3_get_bytes_per_second_per_core = 0; - limit = resolve_s3_rate_limit(S3RateLimitType::GET, kCores); - EXPECT_EQ(0, limit.bytes_per_second); + config::s3_put_bytes_per_second_per_core = -1; + get_limit = resolve_s3_rate_limit(S3RateLimitType::GET, kCores); + put_limit = resolve_s3_rate_limit(S3RateLimitType::PUT, kCores); + EXPECT_EQ(0, get_limit.bytes_per_second); + EXPECT_EQ(0, put_limit.bytes_per_second); } TEST(S3RateLimiterResolveTest, cpu_cores_override_config_wins) { @@ -186,22 +242,34 @@ TEST(S3RateLimiterManagerTest, refresh_with_same_parameters_keeps_consumed_state EXPECT_EQ(-1, get_qps->add(1)); } -TEST(S3RateLimiterManagerTest, refresh_applies_core_changes) { +TEST(S3RateLimiterManagerTest, refresh_applies_qps_speed_burst_and_limit_changes) { RateLimiterConfigGuard guard; auto& manager = S3RateLimiterManager::instance(); auto* get_qps = manager.qps_limiter(S3RateLimitType::GET); + config::s3_get_qps_per_core = -1; + config::s3_get_token_per_second = 100; + config::s3_get_bucket_tokens = 200; + config::s3_get_token_limit = 300; + manager.refresh(); + EXPECT_EQ(100, get_qps->get_max_speed()); + EXPECT_EQ(200, get_qps->get_max_burst()); + EXPECT_EQ(300, get_qps->get_limit()); + config::s3_rate_limiter_cpu_cores = kCores; config::s3_get_qps_per_core = 100; config::s3_get_qps_max = 0; manager.refresh(); EXPECT_EQ(100 * kCores, get_qps->get_max_speed()); EXPECT_EQ(100 * kCores, get_qps->get_max_burst()); + EXPECT_EQ(0, get_qps->get_limit()); // Simulate a serverless resize: the core count is an input of refresh(). config::s3_rate_limiter_cpu_cores = 2 * kCores; manager.refresh(); EXPECT_EQ(100 * 2 * kCores, get_qps->get_max_speed()); + EXPECT_EQ(100 * 2 * kCores, get_qps->get_max_burst()); + EXPECT_EQ(0, get_qps->get_limit()); } TEST(S3RateLimiterManagerTest, refresh_applies_bytes_limit_and_enables_bucket) { @@ -238,15 +306,88 @@ TEST(TokenBucketRateLimiterTest, rejected_add_rolls_back_count_and_tokens) { EXPECT_EQ(0, limiter.add(1000)); } -TEST(S3RateLimitGuardTest, disabled_limiter_admits_everything) { +TEST(S3RateLimiterManagerTest, qps_and_bytes_buckets_are_independent) { + RateLimiterConfigGuard guard; + auto& manager = S3RateLimiterManager::instance(); + auto* qps = manager.qps_limiter(S3RateLimitType::GET); + auto* bytes = manager.bytes_limiter(S3RateLimitType::GET); + qps->reset(kNoThrottle, kNoThrottle, 1); + bytes->reset(kNoThrottle, kNoThrottle, 100); + + EXPECT_EQ(0, qps->add(1)); + EXPECT_EQ(-1, qps->add(1)); + EXPECT_EQ(0, bytes->add(100)); + EXPECT_EQ(-1, bytes->add(1)); +} + +TEST(S3RateLimiterManagerTest, get_and_put_qps_buckets_are_independent) { + RateLimiterConfigGuard guard; + auto& manager = S3RateLimiterManager::instance(); + auto* get_qps = manager.qps_limiter(S3RateLimitType::GET); + auto* put_qps = manager.qps_limiter(S3RateLimitType::PUT); + get_qps->reset(kNoThrottle, kNoThrottle, 1); + put_qps->reset(kNoThrottle, kNoThrottle, 2); + + EXPECT_EQ(0, get_qps->add(1)); + EXPECT_EQ(-1, get_qps->add(1)); + EXPECT_EQ(0, put_qps->add(2)); + EXPECT_EQ(-1, put_qps->add(1)); +} + +TEST(S3RateLimiterManagerTest, get_and_put_bytes_buckets_are_independent) { + RateLimiterConfigGuard guard; + auto& manager = S3RateLimiterManager::instance(); + auto* get_bytes = manager.bytes_limiter(S3RateLimitType::GET); + auto* put_bytes = manager.bytes_limiter(S3RateLimitType::PUT); + get_bytes->reset(kNoThrottle, kNoThrottle, 1); + put_bytes->reset(kNoThrottle, kNoThrottle, 2); + + EXPECT_EQ(0, get_bytes->add(1)); + EXPECT_EQ(-1, get_bytes->add(1)); + EXPECT_EQ(0, put_bytes->add(2)); + EXPECT_EQ(-1, put_bytes->add(1)); +} + +TEST(S3RateLimitGuardTest, disabled_limiter_does_not_consume_qps_or_bytes) { RateLimiterConfigGuard guard; config::enable_s3_rate_limiter = false; - S3RateLimiterManager::instance().qps_limiter(S3RateLimitType::GET)->reset(0, 0, 1); + auto& manager = S3RateLimiterManager::instance(); + auto* qps = manager.qps_limiter(S3RateLimitType::GET); + auto* bytes = manager.bytes_limiter(S3RateLimitType::GET); + qps->reset(0, 0, 1); + bytes->reset(0, 0, 100); for (int i = 0; i < 3; ++i) { S3RateLimitGuard g(S3RateLimitType::GET, 100); EXPECT_TRUE(g.ok()); } + + // The disabled guards must not consume either cumulative limit. Each bucket still + // admits exactly its configured limit after the guards have completed. + EXPECT_EQ(0, qps->add(1)); + EXPECT_EQ(-1, qps->add(1)); + EXPECT_EQ(0, bytes->add(100)); + EXPECT_EQ(-1, bytes->add(1)); +} + +TEST(S3RateLimiterMetricsTest, bytes_wait_and_rejection_have_distinct_counters) { + RateLimiterConfigGuard guard; + auto* bytes = S3RateLimiterManager::instance().bytes_limiter(S3RateLimitType::GET); + const int64_t sleep_count_before = s3_get_bytes_rate_limit_sleep_count.get_value(); + const int64_t rejected_count_before = s3_get_bytes_rate_limit_rejected_count.get_value(); + + // The first add exceeds the one-token burst. It waits for the debt to refill and + // succeeds instead of being rejected. + bytes->reset(1000, 1, 0); + EXPECT_GT(bytes->add(2), 0); + EXPECT_EQ(sleep_count_before + 1, s3_get_bytes_rate_limit_sleep_count.get_value()); + EXPECT_EQ(rejected_count_before, s3_get_bytes_rate_limit_rejected_count.get_value()); + + // A cumulative-limit rejection is immediate and must not be reported as a wait. + bytes->reset(kNoThrottle, kNoThrottle, 1); + EXPECT_EQ(-1, bytes->add(2)); + EXPECT_EQ(sleep_count_before + 1, s3_get_bytes_rate_limit_sleep_count.get_value()); + EXPECT_EQ(rejected_count_before + 1, s3_get_bytes_rate_limit_rejected_count.get_value()); } TEST(S3RateLimitGuardTest, legacy_count_limit_rejects) { diff --git a/regression-test/data/external_table_p2/s3_rate_limiter/test_s3_rate_limiter.out b/regression-test/data/external_table_p2/s3_rate_limiter/test_s3_rate_limiter.out new file mode 100644 index 00000000000000..be551c64e42c70 --- /dev/null +++ b/regression-test/data/external_table_p2/s3_rate_limiter/test_s3_rate_limiter.out @@ -0,0 +1,16 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !get_qps_result -- +20 + +-- !get_bytes_result -- +100000 + +-- !put_qps_result -- +20 + +-- !put_bytes_result -- +100000 + +-- !mixed_result -- +100000 + diff --git a/regression-test/suites/external_table_p2/s3_rate_limiter/test_s3_rate_limiter.groovy b/regression-test/suites/external_table_p2/s3_rate_limiter/test_s3_rate_limiter.groovy new file mode 100644 index 00000000000000..118165fdf5d9eb --- /dev/null +++ b/regression-test/suites/external_table_p2/s3_rate_limiter/test_s3_rate_limiter.groovy @@ -0,0 +1,266 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import java.util.regex.Pattern + +suite("test_s3_rate_limiter", "p2,external,nonConcurrent") { + // In cloud mode, external S3 TVF/Outfile clients intentionally bypass this limiter; + // only internal storage-vault clients are wrapped. The SQL paths below exercise the + // non-cloud compatibility contract where every object-storage client is wrapped. + if (isCloudMode()) { + return + } + + String ak = getS3AK() + String sk = getS3SK() + String endpoint = getS3Endpoint() + String region = getS3Region() + String bucket = getS3BucketName() + String provider = getS3Provider() + String runId = UUID.randomUUID().toString() + String rootPath = "${bucket}/regression/s3_rate_limiter/${runId}" + long bytesPerSecond = 4L * 1024 * 1024 + + def backendIdToIp = [:] + def backendIdToHttpPort = [:] + def backendIdToBrpcPort = [:] + getBackendIpHttpAndBrpcPort(backendIdToIp, backendIdToHttpPort, backendIdToBrpcPort) + + List metricNames = [ + "s3_get_rate_limit_sleep_count", + "s3_put_rate_limit_sleep_count", + "s3_get_bytes_rate_limit_sleep_count", + "s3_put_bytes_rate_limit_sleep_count" + ] + + def getMetricTotal = { String metricName -> + long total = 0 + backendIdToIp.each { id, ip -> + String brpcPort = backendIdToBrpcPort[id] + def (code, out, err) = + curl("GET", "http://${ip}:${brpcPort}/vars/${metricName}") + assertEquals(0, code, "failed to read ${metricName} from BE ${id}: ${err}") + def matcher = out =~ Pattern.compile( + '(?m)^' + Pattern.quote(metricName) + '\\s*:\\s*(\\d+)\\s*$') + assertTrue(matcher.find(), "missing ${metricName} from BE ${id}: ${out}") + total += matcher.group(1).toLong() + } + return total + } + + def snapshotMetrics = { + metricNames.collectEntries { name -> [(name): getMetricTotal(name)] } + } + + def assertMetricChanges = { Map before, Map after, + Collection expectedGrowth -> + metricNames.each { name -> + if (expectedGrowth.contains(name)) { + assertTrue(after[name] > before[name], + "${name} should grow: before=${before[name]}, after=${after[name]}") + } else { + assertEquals(before[name], after[name], + "${name} should not grow while its limiter is disabled") + } + } + } + + def assertBeConfig = { String key, Object expected -> + Map actualByBackend = get_be_param(key) + actualByBackend.each { id, actual -> + assertEquals(expected.toString(), actual, + "unexpected ${key} on BE ${id}") + } + } + + def applyLimiterPhase = { long getQps, long putQps, long getBytes, long putBytes -> + Map phase = [ + "s3_get_qps_per_core": getQps, + "s3_put_qps_per_core": putQps, + "s3_get_bytes_per_second_per_core": getBytes, + "s3_put_bytes_per_second_per_core": putBytes + ] + phase.each { key, value -> set_be_param(key, value) } + set_be_param("enable_s3_rate_limiter", true) + phase.each { key, value -> assertBeConfig(key, value) } + assertBeConfig("enable_s3_rate_limiter", true) + + // S3RateLimiterManager is refreshed by the daemon every 10 seconds. Waiting + // for one complete interval makes the following bvar delta prove that the + // dynamically updated limiter, rather than only the config value, took effect. + sleep(12000) + } + + def outfileSql = { String path, String querySuffix -> + return """ + SELECT id, payload FROM ${context.dbName}.test_s3_rate_limiter + ${querySuffix} + INTO OUTFILE "s3://${path}" + FORMAT AS csv + PROPERTIES ( + "column_separator" = "|", + "s3.endpoint" = "${endpoint}", + "s3.region" = "${region}", + "s3.secret_key" = "${sk}", + "s3.access_key" = "${ak}", + "provider" = "${provider}" + ) + """ + } + + def tvfSql = { String path -> + return """ + SELECT count(*) FROM S3( + "uri" = "s3://${path}*", + "s3.endpoint" = "${endpoint}", + "s3.region" = "${region}", + "s3.secret_key" = "${sk}", + "s3.access_key" = "${ak}", + "provider" = "${provider}", + "format" = "csv", + "column_separator" = "|", + "csv_schema" = "id:bigint;payload:string" + ) + """ + } + + Map temporaryConfig = [ + "enable_s3_rate_limiter": false, + "s3_rate_limiter_cpu_cores": 1, + "s3_get_qps_per_core": 0, + "s3_put_qps_per_core": 0, + "s3_get_qps_max": 0, + "s3_put_qps_max": 0, + "s3_get_bytes_per_second_per_core": 0, + "s3_put_bytes_per_second_per_core": 0, + "s3_get_bytes_per_second_max": 0, + "s3_put_bytes_per_second_max": 0, + "s3_rate_limiter_log_interval": 1 + ] + + try { + setBeConfigTemporary(temporaryConfig) { + sql "SET enable_parallel_outfile = true" + sql "SET parallel_pipeline_task_num = 2" + sql "DROP TABLE IF EXISTS test_s3_rate_limiter" + sql """ + CREATE TABLE test_s3_rate_limiter ( + id BIGINT, + payload VARCHAR(256) + ) + DUPLICATE KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 2 + PROPERTIES("replication_num" = "1") + """ + sql """ + INSERT INTO test_s3_rate_limiter + SELECT number, repeat('x', 128) + FROM numbers("number" = "100000") + """ + + // Prepare two explicit small objects for GET QPS and one large object for + // GET bytes. Do this before enabling any limiter so setup is not observed. + String getQpsSourcePath = "${rootPath}/get_qps_source_" + sql outfileSql("${getQpsSourcePath}first_", "WHERE id < 10") + sql outfileSql("${getQpsSourcePath}second_", "WHERE id < 10") + String getQpsReadSql = tvfSql(getQpsSourcePath) + String largeSourcePath = "${rootPath}/large_source_" + sql outfileSql(largeSourcePath, "") + String largeSourceReadSql = tvfSql(largeSourcePath) + + // GET QPS: bytes and all PUT limiters are disabled. + applyLimiterPhase(1, 0, 0, 0) + def before = snapshotMetrics() + qt_get_qps_result getQpsReadSql + def after = snapshotMetrics() + assertMetricChanges(before, after, ["s3_get_rate_limit_sleep_count"]) + + // GET bytes: QPS and all PUT limiters are disabled. + applyLimiterPhase(0, 0, bytesPerSecond, 0) + before = snapshotMetrics() + qt_get_bytes_result largeSourceReadSql + after = snapshotMetrics() + assertMetricChanges(before, after, ["s3_get_bytes_rate_limit_sleep_count"]) + + // PUT QPS: bytes and all GET limiters are disabled. + applyLimiterPhase(0, 1, 0, 0) + String putQpsPath = "${rootPath}/put_qps_" + before = snapshotMetrics() + sql outfileSql("${putQpsPath}first_", "WHERE id < 10") + sql outfileSql("${putQpsPath}second_", "WHERE id < 10") + after = snapshotMetrics() + assertMetricChanges(before, after, ["s3_put_rate_limit_sleep_count"]) + qt_put_qps_result tvfSql(putQpsPath) + + // PUT bytes: QPS and all GET limiters are disabled. + applyLimiterPhase(0, 0, 0, bytesPerSecond) + String putBytesPath = "${rootPath}/put_bytes_" + before = snapshotMetrics() + sql outfileSql(putBytesPath, "") + after = snapshotMetrics() + assertMetricChanges(before, after, ["s3_put_bytes_rate_limit_sleep_count"]) + qt_put_bytes_result tvfSql(putBytesPath) + + // Mixed: run a real S3 read and write concurrently. The GET and PUT QPS + // counters must both grow while the disabled bytes counters stay unchanged, + // proving that the directional request limiters are independent. + set_be_param("enable_s3_rate_limiter", false) + String mixedSourcePath = "${rootPath}/mixed_source_" + sql outfileSql(mixedSourcePath, "") + String mixedSourceReadSql = tvfSql(mixedSourcePath) + applyLimiterPhase(1, 1, 0, 0) + String mixedPath = "${rootPath}/mixed_output_" + List errors = Collections.synchronizedList(new ArrayList()) + before = snapshotMetrics() + def reader = Thread.start { + try { + connect(context.config.jdbcUser, context.config.jdbcPassword, + context.config.jdbcUrl) { + sql mixedSourceReadSql + } + } catch (Throwable t) { + errors.add(t) + } + } + def writer = Thread.start { + try { + connect(context.config.jdbcUser, context.config.jdbcPassword, + context.config.jdbcUrl) { + sql "SET enable_parallel_outfile = true" + sql "SET parallel_pipeline_task_num = 2" + sql outfileSql(mixedPath, "") + } + } catch (Throwable t) { + errors.add(t) + } + } + reader.join() + writer.join() + assertTrue(errors.isEmpty(), "mixed S3 operations failed: ${errors}") + after = snapshotMetrics() + assertMetricChanges(before, after, [ + "s3_get_rate_limit_sleep_count", + "s3_put_rate_limit_sleep_count" + ]) + qt_mixed_result tvfSql(mixedPath) + } + } finally { + // setBeConfigTemporary restores the values first. Allow the daemon to publish + // the restored bucket parameters before another suite can use object storage. + sleep(12000) + } +} From 0ed49ce55569e783a196e44319762f775f69c458 Mon Sep 17 00:00:00 2001 From: Refrain Date: Tue, 21 Jul 2026 16:21:16 +0800 Subject: [PATCH 06/22] [fix](be) Validate dynamic limiter configs against new values ### What problem does this PR solve? Issue Number: None Related PR: #65420 Problem Summary: Dynamic config validators were invoked before the converted value was published to the registered field, so validators that inspect the field validated the old value. This could allow an invalid S3 limiter value such as per_core=-2 through the HTTP update path. Assign the candidate before validation and restore the old value on failure. Expand S3 limiter unit coverage for invalid values, legacy fallback throttling, disabled GET/PUT accounting, oversized byte reservations, and extend the real-S3 regression suite with rejection metrics and direction-specific QPS rejection messages. ### Release note Invalid dynamic configuration updates are now rejected using the candidate value without changing the active configuration. ### Check List (For Author) - Test: Unit Test / Manual test - 40 focused BE unit tests passed. - ASAN BE build passed. - S3 regression was attempted but stopped during fixture creation because the private OSS credential returned InvalidAccessKeyId. - Behavior changed: Yes. Dynamic validators now validate the candidate value and restore the old value on failure. - Does this need documentation: No --- be/src/common/config.cpp | 2 +- be/test/util/s3_rate_limiter_manager_test.cpp | 101 ++++++++++++++++-- .../test_s3_rate_limiter.groovy | 55 +++++++++- 3 files changed, 146 insertions(+), 12 deletions(-) diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index 05871682376f41..a6f8db0fde85d2 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -2241,6 +2241,7 @@ bool init(const char* conf_file, bool fill_conf_map, bool must_exist, bool set_t } \ TYPE& ref_conf_value = *reinterpret_cast((FIELD).storage); \ TYPE old_value = ref_conf_value; \ + ref_conf_value = new_value; \ if (RegisterConfValidator::_s_field_validator != nullptr) { \ auto validator = RegisterConfValidator::_s_field_validator->find((FIELD).name); \ if (validator != RegisterConfValidator::_s_field_validator->end() && \ @@ -2250,7 +2251,6 @@ bool init(const char* conf_file, bool fill_conf_map, bool must_exist, bool set_t (FIELD).name, new_value); \ } \ } \ - ref_conf_value = new_value; \ if (full_conf_map != nullptr) { \ std::ostringstream oss; \ oss << new_value; \ diff --git a/be/test/util/s3_rate_limiter_manager_test.cpp b/be/test/util/s3_rate_limiter_manager_test.cpp index a4c4a021684617..273433d7c65b21 100644 --- a/be/test/util/s3_rate_limiter_manager_test.cpp +++ b/be/test/util/s3_rate_limiter_manager_test.cpp @@ -22,6 +22,7 @@ #include #include "common/config.h" +#include "common/status.h" namespace doris { @@ -85,6 +86,35 @@ TEST(S3RateLimiterResolveTest, registered_defaults_preserve_legacy_compatibility EXPECT_STREQ("0", fields.at("s3_rate_limiter_cpu_cores").defval); } +TEST(S3RateLimiterConfigTest, invalid_dynamic_values_are_rejected_without_changing_config) { + RateLimiterConfigGuard guard; + config::enable_s3_rate_limiter = false; + config::s3_get_qps_per_core = -1; + config::s3_get_bytes_per_second_per_core = -1; + config::s3_put_bytes_per_second_max = 0; + + auto status = config::set_config("s3_get_qps_per_core", "-2"); + EXPECT_FALSE(status.ok()); + EXPECT_NE(std::string::npos, status.to_string().find("validate s3_get_qps_per_core=-2 failed")); + EXPECT_EQ(-1, config::s3_get_qps_per_core); + + status = config::set_config("s3_get_bytes_per_second_per_core", "Ab"); + EXPECT_FALSE(status.ok()); + EXPECT_NE(std::string::npos, status.to_string().find("convert 'Ab' as int64_t failed")); + EXPECT_EQ(-1, config::s3_get_bytes_per_second_per_core); + + status = config::set_config("enable_s3_rate_limiter", "A"); + EXPECT_FALSE(status.ok()); + EXPECT_NE(std::string::npos, status.to_string().find("convert 'A' as bool failed")); + EXPECT_FALSE(config::enable_s3_rate_limiter); + + status = config::set_config("s3_put_bytes_per_second_max", "9223372036854775808"); + EXPECT_FALSE(status.ok()); + EXPECT_NE(std::string::npos, + status.to_string().find("convert '9223372036854775808' as int64_t failed")); + EXPECT_EQ(0, config::s3_put_bytes_per_second_max); +} + TEST(S3RateLimiterResolveTest, legacy_config_wins_when_per_core_unset) { RateLimiterConfigGuard guard; config::s3_get_qps_per_core = -1; @@ -242,6 +272,25 @@ TEST(S3RateLimiterManagerTest, refresh_with_same_parameters_keeps_consumed_state EXPECT_EQ(-1, get_qps->add(1)); } +TEST(S3RateLimiterManagerTest, legacy_get_config_throttles_when_per_core_is_unset) { + RateLimiterConfigGuard guard; + auto& manager = S3RateLimiterManager::instance(); + auto* get_qps = manager.qps_limiter(S3RateLimitType::GET); + + config::s3_get_qps_per_core = -1; + config::s3_get_token_per_second = 100; + config::s3_get_bucket_tokens = 1; + config::s3_get_token_limit = 0; + manager.refresh(); + + EXPECT_EQ(100, get_qps->get_max_speed()); + EXPECT_EQ(1, get_qps->get_max_burst()); + EXPECT_EQ(0, get_qps->get_limit()); + // Charging two requests against a one-token burst must wait for the legacy + // token_per_second refill, proving that fallback affects runtime behavior. + EXPECT_GT(get_qps->add(2), 0); +} + TEST(S3RateLimiterManagerTest, refresh_applies_qps_speed_burst_and_limit_changes) { RateLimiterConfigGuard guard; auto& manager = S3RateLimiterManager::instance(); @@ -352,22 +401,32 @@ TEST(S3RateLimitGuardTest, disabled_limiter_does_not_consume_qps_or_bytes) { RateLimiterConfigGuard guard; config::enable_s3_rate_limiter = false; auto& manager = S3RateLimiterManager::instance(); - auto* qps = manager.qps_limiter(S3RateLimitType::GET); - auto* bytes = manager.bytes_limiter(S3RateLimitType::GET); - qps->reset(0, 0, 1); - bytes->reset(0, 0, 100); + auto* get_qps = manager.qps_limiter(S3RateLimitType::GET); + auto* put_qps = manager.qps_limiter(S3RateLimitType::PUT); + auto* get_bytes = manager.bytes_limiter(S3RateLimitType::GET); + auto* put_bytes = manager.bytes_limiter(S3RateLimitType::PUT); + get_qps->reset(0, 0, 1); + put_qps->reset(0, 0, 1); + get_bytes->reset(0, 0, 100); + put_bytes->reset(0, 0, 100); for (int i = 0; i < 3; ++i) { - S3RateLimitGuard g(S3RateLimitType::GET, 100); - EXPECT_TRUE(g.ok()); + S3RateLimitGuard get_guard(S3RateLimitType::GET, 100); + S3RateLimitGuard put_guard(S3RateLimitType::PUT, 100); + EXPECT_TRUE(get_guard.ok()); + EXPECT_TRUE(put_guard.ok()); } // The disabled guards must not consume either cumulative limit. Each bucket still // admits exactly its configured limit after the guards have completed. - EXPECT_EQ(0, qps->add(1)); - EXPECT_EQ(-1, qps->add(1)); - EXPECT_EQ(0, bytes->add(100)); - EXPECT_EQ(-1, bytes->add(1)); + for (auto type : {S3RateLimitType::GET, S3RateLimitType::PUT}) { + auto* qps = manager.qps_limiter(type); + auto* bytes = manager.bytes_limiter(type); + EXPECT_EQ(0, qps->add(1)); + EXPECT_EQ(-1, qps->add(1)); + EXPECT_EQ(0, bytes->add(100)); + EXPECT_EQ(-1, bytes->add(1)); + } } TEST(S3RateLimiterMetricsTest, bytes_wait_and_rejection_have_distinct_counters) { @@ -491,4 +550,26 @@ TEST(S3RateLimitGuardTest, reservation_is_clamped_to_one_second_of_bandwidth) { EXPECT_EQ(-1, bytes->add(1)); } +TEST(S3RateLimitGuardTest, first_oversized_reservation_is_admitted_without_waiting) { + RateLimiterConfigGuard guard; + config::enable_s3_rate_limiter = true; + auto& manager = S3RateLimiterManager::instance(); + auto* bytes = manager.bytes_limiter(S3RateLimitType::GET); + manager.qps_limiter(S3RateLimitType::GET)->reset(0, 0, 0); + bytes->reset(1000, 1000, 0); + const int64_t sleep_count_before = s3_get_bytes_rate_limit_sleep_count.get_value(); + + // The guard clamps this request to max_speed (= one full bucket). Charging exactly + // the initial bucket does not create debt, so the request is admitted without wait. + S3RateLimitGuard g(S3RateLimitType::GET, 1000000); + EXPECT_TRUE(g.ok()); + EXPECT_EQ(sleep_count_before, s3_get_bytes_rate_limit_sleep_count.get_value()); + + // A later oversized request is clamped in the same way. With the initial burst + // already consumed, it waits for refill and is still admitted rather than rejected. + S3RateLimitGuard next(S3RateLimitType::GET, 1000000); + EXPECT_TRUE(next.ok()); + EXPECT_GT(s3_get_bytes_rate_limit_sleep_count.get_value(), sleep_count_before); +} + } // namespace doris diff --git a/regression-test/suites/external_table_p2/s3_rate_limiter/test_s3_rate_limiter.groovy b/regression-test/suites/external_table_p2/s3_rate_limiter/test_s3_rate_limiter.groovy index 118165fdf5d9eb..6889bedd791ce1 100644 --- a/regression-test/suites/external_table_p2/s3_rate_limiter/test_s3_rate_limiter.groovy +++ b/regression-test/suites/external_table_p2/s3_rate_limiter/test_s3_rate_limiter.groovy @@ -44,7 +44,11 @@ suite("test_s3_rate_limiter", "p2,external,nonConcurrent") { "s3_get_rate_limit_sleep_count", "s3_put_rate_limit_sleep_count", "s3_get_bytes_rate_limit_sleep_count", - "s3_put_bytes_rate_limit_sleep_count" + "s3_put_bytes_rate_limit_sleep_count", + "s3_get_rate_limit_rejected_count", + "s3_put_rate_limit_rejected_count", + "s3_get_bytes_rate_limit_rejected_count", + "s3_put_bytes_rate_limit_rejected_count" ] def getMetricTotal = { String metricName -> @@ -105,6 +109,26 @@ suite("test_s3_rate_limiter", "p2,external,nonConcurrent") { sleep(12000) } + def applyLegacyCountLimitPhase = { long getLimit, long putLimit -> + Map phase = [ + "s3_get_qps_per_core": getLimit > 0 ? -1 : 0, + "s3_put_qps_per_core": putLimit > 0 ? -1 : 0, + "s3_get_token_per_second": 1000000000000000000L, + "s3_put_token_per_second": 1000000000000000000L, + "s3_get_bucket_tokens": 1000000000000000000L, + "s3_put_bucket_tokens": 1000000000000000000L, + "s3_get_token_limit": getLimit, + "s3_put_token_limit": putLimit, + "s3_get_bytes_per_second_per_core": 0, + "s3_put_bytes_per_second_per_core": 0 + ] + phase.each { key, value -> set_be_param(key, value) } + set_be_param("enable_s3_rate_limiter", true) + phase.each { key, value -> assertBeConfig(key, value) } + assertBeConfig("enable_s3_rate_limiter", true) + sleep(12000) + } + def outfileSql = { String path, String querySuffix -> return """ SELECT id, payload FROM ${context.dbName}.test_s3_rate_limiter @@ -145,6 +169,12 @@ suite("test_s3_rate_limiter", "p2,external,nonConcurrent") { "s3_put_qps_per_core": 0, "s3_get_qps_max": 0, "s3_put_qps_max": 0, + "s3_get_token_per_second": 1000000000000000000L, + "s3_put_token_per_second": 1000000000000000000L, + "s3_get_bucket_tokens": 1000000000000000000L, + "s3_put_bucket_tokens": 1000000000000000000L, + "s3_get_token_limit": 0, + "s3_put_token_limit": 0, "s3_get_bytes_per_second_per_core": 0, "s3_put_bytes_per_second_per_core": 0, "s3_get_bytes_per_second_max": 0, @@ -257,6 +287,29 @@ suite("test_s3_rate_limiter", "p2,external,nonConcurrent") { "s3_put_rate_limit_sleep_count" ]) qt_mixed_result tvfSql(mixedPath) + + // Legacy GET token_limit: force an admission rejection and verify both the + // user-visible direction/reason and the dedicated rejection bvar. + applyLegacyCountLimitPhase(1, 0) + before = snapshotMetrics() + test { + sql getQpsReadSql + exception "s3 get request exceeds QPS limit" + exception "rejected by BE rate limiter" + } + after = snapshotMetrics() + assertMetricChanges(before, after, ["s3_get_rate_limit_rejected_count"]) + + // Legacy PUT token_limit has an independent counter and error message. + applyLegacyCountLimitPhase(0, 1) + before = snapshotMetrics() + test { + sql outfileSql("${rootPath}/put_rejected_", "WHERE id < 10") + exception "s3 put request exceeds QPS limit" + exception "rejected by BE rate limiter" + } + after = snapshotMetrics() + assertMetricChanges(before, after, ["s3_put_rate_limit_rejected_count"]) } } finally { // setBeConfigTemporary restores the values first. Allow the daemon to publish From f65493846fb10e53eda9bd9b697aaadc9b414887 Mon Sep 17 00:00:00 2001 From: Refrain Date: Tue, 21 Jul 2026 18:24:08 +0800 Subject: [PATCH 07/22] [test](cloud) Validate S3 rate limiting on internal vault IO ### What problem does this PR solve? Issue Number: None Related PR: #65420 Problem Summary: The previous regression used S3 TVF and Outfile in non-cloud mode, so it did not validate the Cloud internal-vault boundary where the limiter is enabled. Replace it with real internal OLAP scans and writes, isolate GET and PUT QPS/bytes phases, validate dynamic configuration and bvar deltas, and cover legacy token-limit rejection behavior. ### Release note None ### Check List (For Author) - Test: Regression test - `cloud_p0/s3/test_s3_rate_limiter` on a local three-BE Cloud cluster - Behavior changed: No - Does this need documentation: No --- .../data/cloud_p0/s3/test_s3_rate_limiter.out | 7 + .../cloud_p0/s3/test_s3_rate_limiter.groovy | 395 ++++++++++++++++++ .../test_s3_rate_limiter.groovy | 319 -------------- 3 files changed, 402 insertions(+), 319 deletions(-) create mode 100644 regression-test/data/cloud_p0/s3/test_s3_rate_limiter.out create mode 100644 regression-test/suites/cloud_p0/s3/test_s3_rate_limiter.groovy delete mode 100644 regression-test/suites/external_table_p2/s3_rate_limiter/test_s3_rate_limiter.groovy diff --git a/regression-test/data/cloud_p0/s3/test_s3_rate_limiter.out b/regression-test/data/cloud_p0/s3/test_s3_rate_limiter.out new file mode 100644 index 00000000000000..10555600c25245 --- /dev/null +++ b/regression-test/data/cloud_p0/s3/test_s3_rate_limiter.out @@ -0,0 +1,7 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !get_qps_result -- +6000 768000 + +-- !get_bytes_result -- +200000 25600000 + diff --git a/regression-test/suites/cloud_p0/s3/test_s3_rate_limiter.groovy b/regression-test/suites/cloud_p0/s3/test_s3_rate_limiter.groovy new file mode 100644 index 00000000000000..1390e2f6a21207 --- /dev/null +++ b/regression-test/suites/cloud_p0/s3/test_s3_rate_limiter.groovy @@ -0,0 +1,395 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import groovy.json.JsonSlurper + +import java.util.concurrent.CountDownLatch +import java.util.regex.Pattern + +suite("test_s3_rate_limiter", "p0,nonConcurrent") { + if (!isCloudMode()) { + return + } + + long bytesPerSecond = 5L * 1024 * 1024 + int smallRowsetCount = 6 + int smallRowsPerRowset = 1000 + int largeRows = 200000 + + def backendIdToIp = [:] + def backendIdToHttpPort = [:] + def backendIdToBrpcPort = [:] + getBackendIpHttpAndBrpcPort(backendIdToIp, backendIdToHttpPort, backendIdToBrpcPort) + + List metricNames = [ + "s3_get_rate_limit_sleep_ns", + "s3_get_rate_limit_sleep_count", + "s3_put_rate_limit_sleep_ns", + "s3_put_rate_limit_sleep_count", + "s3_get_bytes_rate_limit_sleep_ns", + "s3_get_bytes_rate_limit_sleep_count", + "s3_put_bytes_rate_limit_sleep_ns", + "s3_put_bytes_rate_limit_sleep_count", + "s3_get_rate_limit_rejected_count", + "s3_put_rate_limit_rejected_count", + "s3_get_bytes_rate_limit_rejected_count", + "s3_put_bytes_rate_limit_rejected_count" + ] + + def getMetricTotal = { String metricName -> + long total = 0 + backendIdToIp.each { id, ip -> + String brpcPort = backendIdToBrpcPort[id] + def (code, out, err) = curl("GET", "http://${ip}:${brpcPort}/vars/${metricName}") + assertEquals(0, code, "failed to read ${metricName} from BE ${id}: ${err}") + def matcher = out =~ Pattern.compile( + '(?m)^' + Pattern.quote(metricName) + '\\s*:\\s*(\\d+)\\s*$') + assertTrue(matcher.find(), "missing ${metricName} from BE ${id}: ${out}") + total += matcher.group(1).toLong() + } + return total + } + + def snapshotMetrics = { + metricNames.collectEntries { name -> [(name): getMetricTotal(name)] } + } + + def assertMetricChanges = { Map before, Map after, + Collection expectedGrowth -> + metricNames.each { name -> + if (expectedGrowth.contains(name)) { + assertTrue(after[name] > before[name], + "${name} should grow: before=${before[name]}, after=${after[name]}") + } else { + assertEquals(before[name], after[name], + "${name} should not grow while its limiter is disabled") + } + } + } + + def assertBeConfig = { String key, Object expected -> + Map actualByBackend = get_be_param(key) + actualByBackend.each { id, actual -> + assertEquals(expected.toString(), actual, "unexpected ${key} on BE ${id}") + } + } + + def applyLimiterPhase = { long getQps, long putQps, long getBytes, long putBytes -> + Map phase = [ + "s3_get_qps_per_core": getQps, + "s3_put_qps_per_core": putQps, + "s3_get_bytes_per_second_per_core": getBytes, + "s3_put_bytes_per_second_per_core": putBytes + ] + phase.each { key, value -> set_be_param(key, value) } + set_be_param("enable_s3_rate_limiter", true) + phase.each { key, value -> assertBeConfig(key, value) } + assertBeConfig("enable_s3_rate_limiter", true) + + // The daemon refreshes S3RateLimiterManager every 10 seconds. The config + // assertions plus the following phase-specific bvar delta verify that the + // dynamically rebuilt bucket, rather than only the config value, took effect. + sleep(12000) + } + + def applyLegacyCountLimitPhase = { long getLimit, long putLimit -> + Map phase = [ + "s3_get_qps_per_core": getLimit > 0 ? -1 : 0, + "s3_put_qps_per_core": putLimit > 0 ? -1 : 0, + "s3_get_token_per_second": 1000000000000000000L, + "s3_put_token_per_second": 1000000000000000000L, + "s3_get_bucket_tokens": 1000000000000000000L, + "s3_put_bucket_tokens": 1000000000000000000L, + "s3_get_token_limit": getLimit, + "s3_put_token_limit": putLimit, + "s3_get_bytes_per_second_per_core": 0, + "s3_put_bytes_per_second_per_core": 0 + ] + phase.each { key, value -> set_be_param(key, value) } + set_be_param("enable_s3_rate_limiter", true) + phase.each { key, value -> assertBeConfig(key, value) } + assertBeConfig("enable_s3_rate_limiter", true) + sleep(12000) + } + + def clearFileCacheOnAllBackends = { + backendIdToIp.each { id, ip -> + String httpPort = backendIdToHttpPort[id] + String response = new URL( + "http://${ip}:${httpPort}/api/file_cache?op=clear&sync=true").text + def json = new JsonSlurper().parseText(response) + assertEquals("OK", json.status, "failed to clear file cache on BE ${id}: ${response}") + } + sleep(5000) + } + + String largePayload = "concat(" + + "md5(cast(number as string))," + + "md5(cast(number + 200000 as string))," + + "md5(cast(number + 400000 as string))," + + "md5(cast(number + 600000 as string)))" + + def runConcurrentSmallInserts = { int insertCount -> + CountDownLatch ready = new CountDownLatch(insertCount) + CountDownLatch start = new CountDownLatch(1) + List errors = Collections.synchronizedList(new ArrayList()) + List writers = (0.. + Thread.start { + ready.countDown() + start.await() + try { + connect(context.config.jdbcUser, context.config.jdbcPassword, + context.config.jdbcUrl) { + sql """ + INSERT INTO ${context.dbName}.test_s3_rate_limiter_put + SELECT ${index * 1000} + number, repeat(md5(cast(number as string)), 4) + FROM numbers("number" = "100") + """ + } + } catch (Throwable t) { + errors.add(t) + } + } + } + ready.await() + start.countDown() + writers.each { it.join() } + assertTrue(errors.isEmpty(), "concurrent internal PUT operations failed: ${errors}") + } + + Map temporaryConfig = [ + "enable_s3_rate_limiter": false, + "enable_packed_file": false, + // File cache clearing is not enough for repeated scans: decoded column + // pages may still be served from the storage page cache without S3 IO. + "disable_storage_page_cache": true, + // Keep PUT phases directionally pure: the optional post-upload HeadObject + // would otherwise add GET traffic to an internal write. + "enable_s3_object_check_after_upload": false, + "s3_rate_limiter_cpu_cores": 1, + "s3_get_qps_per_core": 0, + "s3_put_qps_per_core": 0, + "s3_get_qps_max": 0, + "s3_put_qps_max": 0, + "s3_get_token_per_second": 1000000000000000000L, + "s3_put_token_per_second": 1000000000000000000L, + "s3_get_bucket_tokens": 1000000000000000000L, + "s3_put_bucket_tokens": 1000000000000000000L, + "s3_get_token_limit": 0, + "s3_put_token_limit": 0, + "s3_get_bytes_per_second_per_core": 0, + "s3_put_bytes_per_second_per_core": 0, + "s3_get_bytes_per_second_max": 0, + "s3_put_bytes_per_second_max": 0, + "s3_rate_limiter_log_interval": 1 + ] + + try { + setBeConfigTemporary(temporaryConfig) { + sql "SET disable_file_cache = true" + sql "DROP TABLE IF EXISTS test_s3_rate_limiter_get" + sql "DROP TABLE IF EXISTS test_s3_rate_limiter_put" + sql """ + CREATE TABLE test_s3_rate_limiter_get ( + id BIGINT, + payload STRING + ) + DUPLICATE KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES( + "replication_num" = "1", + "disable_auto_compaction" = "true" + ) + """ + sql """ + CREATE TABLE test_s3_rate_limiter_put ( + id BIGINT, + payload STRING + ) + DUPLICATE KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES( + "replication_num" = "1", + "disable_auto_compaction" = "true" + ) + """ + + // Build multiple small rowsets and one incompressible large rowset while + // limiting is disabled. One bucket keeps all segment IO on one tablet/BE. + for (int batch = 0; batch < smallRowsetCount; batch++) { + sql """ + INSERT INTO test_s3_rate_limiter_get + SELECT ${batch * smallRowsPerRowset} + number, + repeat(md5(cast(number + ${batch * smallRowsPerRowset} as string)), 4) + FROM numbers("number" = "${smallRowsPerRowset}") + """ + } + sql """ + INSERT INTO test_s3_rate_limiter_get + SELECT 1000000 + number, ${largePayload} + FROM numbers("number" = "${largeRows}") + """ + + // GET QPS: a cache-free scan reads all rowset segments from the internal vault. + applyLimiterPhase(1, 0, 0, 0) + clearFileCacheOnAllBackends() + def before = snapshotMetrics() + qt_get_qps_result """ + SELECT count(*), sum(length(payload)) + FROM test_s3_rate_limiter_get + WHERE id < 1000000 + """ + def after = snapshotMetrics() + assertMetricChanges(before, after, [ + "s3_get_rate_limit_sleep_ns", + "s3_get_rate_limit_sleep_count" + ]) + + // GET bytes: 5 MiB/s is no lower than the default single S3 IO upper bound. + applyLimiterPhase(0, 0, bytesPerSecond, 0) + clearFileCacheOnAllBackends() + before = snapshotMetrics() + qt_get_bytes_result """ + SELECT count(*), sum(length(payload)) + FROM test_s3_rate_limiter_get + WHERE id >= 1000000 + """ + after = snapshotMetrics() + assertMetricChanges(before, after, [ + "s3_get_bytes_rate_limit_sleep_ns", + "s3_get_bytes_rate_limit_sleep_count" + ]) + + // PUT QPS: concurrent small inserts create independent small segment files + // for the same tablet, guaranteeing multiple requests against one BE bucket. + applyLimiterPhase(0, 1, 0, 0) + before = snapshotMetrics() + runConcurrentSmallInserts(4) + after = snapshotMetrics() + assertMetricChanges(before, after, [ + "s3_put_rate_limit_sleep_ns", + "s3_put_rate_limit_sleep_count" + ]) + + // PUT bytes: the large, poorly compressible segment produces multiple + // s3_write_buffer_size upload parts to the internal vault. + applyLimiterPhase(0, 0, 0, bytesPerSecond) + before = snapshotMetrics() + sql """ + INSERT INTO test_s3_rate_limiter_put + SELECT 1000000 + number, ${largePayload} + FROM numbers("number" = "${largeRows}") + """ + after = snapshotMetrics() + assertMetricChanges(before, after, [ + "s3_put_bytes_rate_limit_sleep_ns", + "s3_put_bytes_rate_limit_sleep_count" + ]) + + // Mixed internal IO: scan uncached vault data while another large segment + // is uploaded. GET and PUT QPS metrics must grow independently. + applyLimiterPhase(1, 1, 0, 0) + clearFileCacheOnAllBackends() + List mixedErrors = Collections.synchronizedList(new ArrayList()) + CountDownLatch mixedStart = new CountDownLatch(1) + before = snapshotMetrics() + def reader = Thread.start { + mixedStart.await() + try { + connect(context.config.jdbcUser, context.config.jdbcPassword, + context.config.jdbcUrl) { + sql "SET disable_file_cache = true" + sql """ + SELECT count(*), sum(length(payload)) + FROM ${context.dbName}.test_s3_rate_limiter_get + """ + } + } catch (Throwable t) { + mixedErrors.add(t) + } + } + def writer = Thread.start { + mixedStart.await() + try { + connect(context.config.jdbcUser, context.config.jdbcPassword, + context.config.jdbcUrl) { + sql """ + INSERT INTO ${context.dbName}.test_s3_rate_limiter_put + SELECT 2000000 + number, ${largePayload} + FROM numbers("number" = "${largeRows}") + """ + } + } catch (Throwable t) { + mixedErrors.add(t) + } + } + mixedStart.countDown() + reader.join() + writer.join() + assertTrue(mixedErrors.isEmpty(), "mixed internal S3 operations failed: ${mixedErrors}") + after = snapshotMetrics() + assertMetricChanges(before, after, [ + "s3_get_rate_limit_sleep_ns", + "s3_get_rate_limit_sleep_count", + "s3_put_rate_limit_sleep_ns", + "s3_put_rate_limit_sleep_count" + ]) + + // Legacy GET token_limit: a cache-free scan must be rejected by the + // internal GET bucket. Append a fresh rowset with limiting disabled so + // no process-local cache can make this phase vacuous. + applyLegacyCountLimitPhase(1, 0) + set_be_param("enable_s3_rate_limiter", false) + sql """ + INSERT INTO test_s3_rate_limiter_get + SELECT 3000000 + number, repeat(md5(cast(number + 3000000 as string)), 4) + FROM numbers("number" = "1000") + """ + set_be_param("enable_s3_rate_limiter", true) + clearFileCacheOnAllBackends() + before = snapshotMetrics() + test { + sql """ + SELECT count(*), sum(length(payload)) + FROM test_s3_rate_limiter_get + """ + check { result, exception, startTime, endTime -> + assertNotNull(exception, "internal GET should be rejected by token_limit") + logger.info("internal GET token_limit error: ${exception}") + } + } + after = snapshotMetrics() + assertMetricChanges(before, after, ["s3_get_rate_limit_rejected_count"]) + + // Legacy PUT token_limit: the first one-object segment consumes the token; + // the second is rejected with an independently attributable error/bvar. + applyLegacyCountLimitPhase(0, 1) + sql "INSERT INTO test_s3_rate_limiter_put VALUES (9000000, 'first token')" + before = snapshotMetrics() + test { + sql "INSERT INTO test_s3_rate_limiter_put VALUES (9000001, 'rejected token')" + exception "s3 put request exceeds QPS limit, rejected by BE rate limiter" + } + after = snapshotMetrics() + assertMetricChanges(before, after, ["s3_put_rate_limit_rejected_count"]) + } + } finally { + // Let the daemon publish the restored bucket parameters before another suite + // enables object-storage limiting. + sleep(12000) + } +} diff --git a/regression-test/suites/external_table_p2/s3_rate_limiter/test_s3_rate_limiter.groovy b/regression-test/suites/external_table_p2/s3_rate_limiter/test_s3_rate_limiter.groovy deleted file mode 100644 index 6889bedd791ce1..00000000000000 --- a/regression-test/suites/external_table_p2/s3_rate_limiter/test_s3_rate_limiter.groovy +++ /dev/null @@ -1,319 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -import java.util.regex.Pattern - -suite("test_s3_rate_limiter", "p2,external,nonConcurrent") { - // In cloud mode, external S3 TVF/Outfile clients intentionally bypass this limiter; - // only internal storage-vault clients are wrapped. The SQL paths below exercise the - // non-cloud compatibility contract where every object-storage client is wrapped. - if (isCloudMode()) { - return - } - - String ak = getS3AK() - String sk = getS3SK() - String endpoint = getS3Endpoint() - String region = getS3Region() - String bucket = getS3BucketName() - String provider = getS3Provider() - String runId = UUID.randomUUID().toString() - String rootPath = "${bucket}/regression/s3_rate_limiter/${runId}" - long bytesPerSecond = 4L * 1024 * 1024 - - def backendIdToIp = [:] - def backendIdToHttpPort = [:] - def backendIdToBrpcPort = [:] - getBackendIpHttpAndBrpcPort(backendIdToIp, backendIdToHttpPort, backendIdToBrpcPort) - - List metricNames = [ - "s3_get_rate_limit_sleep_count", - "s3_put_rate_limit_sleep_count", - "s3_get_bytes_rate_limit_sleep_count", - "s3_put_bytes_rate_limit_sleep_count", - "s3_get_rate_limit_rejected_count", - "s3_put_rate_limit_rejected_count", - "s3_get_bytes_rate_limit_rejected_count", - "s3_put_bytes_rate_limit_rejected_count" - ] - - def getMetricTotal = { String metricName -> - long total = 0 - backendIdToIp.each { id, ip -> - String brpcPort = backendIdToBrpcPort[id] - def (code, out, err) = - curl("GET", "http://${ip}:${brpcPort}/vars/${metricName}") - assertEquals(0, code, "failed to read ${metricName} from BE ${id}: ${err}") - def matcher = out =~ Pattern.compile( - '(?m)^' + Pattern.quote(metricName) + '\\s*:\\s*(\\d+)\\s*$') - assertTrue(matcher.find(), "missing ${metricName} from BE ${id}: ${out}") - total += matcher.group(1).toLong() - } - return total - } - - def snapshotMetrics = { - metricNames.collectEntries { name -> [(name): getMetricTotal(name)] } - } - - def assertMetricChanges = { Map before, Map after, - Collection expectedGrowth -> - metricNames.each { name -> - if (expectedGrowth.contains(name)) { - assertTrue(after[name] > before[name], - "${name} should grow: before=${before[name]}, after=${after[name]}") - } else { - assertEquals(before[name], after[name], - "${name} should not grow while its limiter is disabled") - } - } - } - - def assertBeConfig = { String key, Object expected -> - Map actualByBackend = get_be_param(key) - actualByBackend.each { id, actual -> - assertEquals(expected.toString(), actual, - "unexpected ${key} on BE ${id}") - } - } - - def applyLimiterPhase = { long getQps, long putQps, long getBytes, long putBytes -> - Map phase = [ - "s3_get_qps_per_core": getQps, - "s3_put_qps_per_core": putQps, - "s3_get_bytes_per_second_per_core": getBytes, - "s3_put_bytes_per_second_per_core": putBytes - ] - phase.each { key, value -> set_be_param(key, value) } - set_be_param("enable_s3_rate_limiter", true) - phase.each { key, value -> assertBeConfig(key, value) } - assertBeConfig("enable_s3_rate_limiter", true) - - // S3RateLimiterManager is refreshed by the daemon every 10 seconds. Waiting - // for one complete interval makes the following bvar delta prove that the - // dynamically updated limiter, rather than only the config value, took effect. - sleep(12000) - } - - def applyLegacyCountLimitPhase = { long getLimit, long putLimit -> - Map phase = [ - "s3_get_qps_per_core": getLimit > 0 ? -1 : 0, - "s3_put_qps_per_core": putLimit > 0 ? -1 : 0, - "s3_get_token_per_second": 1000000000000000000L, - "s3_put_token_per_second": 1000000000000000000L, - "s3_get_bucket_tokens": 1000000000000000000L, - "s3_put_bucket_tokens": 1000000000000000000L, - "s3_get_token_limit": getLimit, - "s3_put_token_limit": putLimit, - "s3_get_bytes_per_second_per_core": 0, - "s3_put_bytes_per_second_per_core": 0 - ] - phase.each { key, value -> set_be_param(key, value) } - set_be_param("enable_s3_rate_limiter", true) - phase.each { key, value -> assertBeConfig(key, value) } - assertBeConfig("enable_s3_rate_limiter", true) - sleep(12000) - } - - def outfileSql = { String path, String querySuffix -> - return """ - SELECT id, payload FROM ${context.dbName}.test_s3_rate_limiter - ${querySuffix} - INTO OUTFILE "s3://${path}" - FORMAT AS csv - PROPERTIES ( - "column_separator" = "|", - "s3.endpoint" = "${endpoint}", - "s3.region" = "${region}", - "s3.secret_key" = "${sk}", - "s3.access_key" = "${ak}", - "provider" = "${provider}" - ) - """ - } - - def tvfSql = { String path -> - return """ - SELECT count(*) FROM S3( - "uri" = "s3://${path}*", - "s3.endpoint" = "${endpoint}", - "s3.region" = "${region}", - "s3.secret_key" = "${sk}", - "s3.access_key" = "${ak}", - "provider" = "${provider}", - "format" = "csv", - "column_separator" = "|", - "csv_schema" = "id:bigint;payload:string" - ) - """ - } - - Map temporaryConfig = [ - "enable_s3_rate_limiter": false, - "s3_rate_limiter_cpu_cores": 1, - "s3_get_qps_per_core": 0, - "s3_put_qps_per_core": 0, - "s3_get_qps_max": 0, - "s3_put_qps_max": 0, - "s3_get_token_per_second": 1000000000000000000L, - "s3_put_token_per_second": 1000000000000000000L, - "s3_get_bucket_tokens": 1000000000000000000L, - "s3_put_bucket_tokens": 1000000000000000000L, - "s3_get_token_limit": 0, - "s3_put_token_limit": 0, - "s3_get_bytes_per_second_per_core": 0, - "s3_put_bytes_per_second_per_core": 0, - "s3_get_bytes_per_second_max": 0, - "s3_put_bytes_per_second_max": 0, - "s3_rate_limiter_log_interval": 1 - ] - - try { - setBeConfigTemporary(temporaryConfig) { - sql "SET enable_parallel_outfile = true" - sql "SET parallel_pipeline_task_num = 2" - sql "DROP TABLE IF EXISTS test_s3_rate_limiter" - sql """ - CREATE TABLE test_s3_rate_limiter ( - id BIGINT, - payload VARCHAR(256) - ) - DUPLICATE KEY(id) - DISTRIBUTED BY HASH(id) BUCKETS 2 - PROPERTIES("replication_num" = "1") - """ - sql """ - INSERT INTO test_s3_rate_limiter - SELECT number, repeat('x', 128) - FROM numbers("number" = "100000") - """ - - // Prepare two explicit small objects for GET QPS and one large object for - // GET bytes. Do this before enabling any limiter so setup is not observed. - String getQpsSourcePath = "${rootPath}/get_qps_source_" - sql outfileSql("${getQpsSourcePath}first_", "WHERE id < 10") - sql outfileSql("${getQpsSourcePath}second_", "WHERE id < 10") - String getQpsReadSql = tvfSql(getQpsSourcePath) - String largeSourcePath = "${rootPath}/large_source_" - sql outfileSql(largeSourcePath, "") - String largeSourceReadSql = tvfSql(largeSourcePath) - - // GET QPS: bytes and all PUT limiters are disabled. - applyLimiterPhase(1, 0, 0, 0) - def before = snapshotMetrics() - qt_get_qps_result getQpsReadSql - def after = snapshotMetrics() - assertMetricChanges(before, after, ["s3_get_rate_limit_sleep_count"]) - - // GET bytes: QPS and all PUT limiters are disabled. - applyLimiterPhase(0, 0, bytesPerSecond, 0) - before = snapshotMetrics() - qt_get_bytes_result largeSourceReadSql - after = snapshotMetrics() - assertMetricChanges(before, after, ["s3_get_bytes_rate_limit_sleep_count"]) - - // PUT QPS: bytes and all GET limiters are disabled. - applyLimiterPhase(0, 1, 0, 0) - String putQpsPath = "${rootPath}/put_qps_" - before = snapshotMetrics() - sql outfileSql("${putQpsPath}first_", "WHERE id < 10") - sql outfileSql("${putQpsPath}second_", "WHERE id < 10") - after = snapshotMetrics() - assertMetricChanges(before, after, ["s3_put_rate_limit_sleep_count"]) - qt_put_qps_result tvfSql(putQpsPath) - - // PUT bytes: QPS and all GET limiters are disabled. - applyLimiterPhase(0, 0, 0, bytesPerSecond) - String putBytesPath = "${rootPath}/put_bytes_" - before = snapshotMetrics() - sql outfileSql(putBytesPath, "") - after = snapshotMetrics() - assertMetricChanges(before, after, ["s3_put_bytes_rate_limit_sleep_count"]) - qt_put_bytes_result tvfSql(putBytesPath) - - // Mixed: run a real S3 read and write concurrently. The GET and PUT QPS - // counters must both grow while the disabled bytes counters stay unchanged, - // proving that the directional request limiters are independent. - set_be_param("enable_s3_rate_limiter", false) - String mixedSourcePath = "${rootPath}/mixed_source_" - sql outfileSql(mixedSourcePath, "") - String mixedSourceReadSql = tvfSql(mixedSourcePath) - applyLimiterPhase(1, 1, 0, 0) - String mixedPath = "${rootPath}/mixed_output_" - List errors = Collections.synchronizedList(new ArrayList()) - before = snapshotMetrics() - def reader = Thread.start { - try { - connect(context.config.jdbcUser, context.config.jdbcPassword, - context.config.jdbcUrl) { - sql mixedSourceReadSql - } - } catch (Throwable t) { - errors.add(t) - } - } - def writer = Thread.start { - try { - connect(context.config.jdbcUser, context.config.jdbcPassword, - context.config.jdbcUrl) { - sql "SET enable_parallel_outfile = true" - sql "SET parallel_pipeline_task_num = 2" - sql outfileSql(mixedPath, "") - } - } catch (Throwable t) { - errors.add(t) - } - } - reader.join() - writer.join() - assertTrue(errors.isEmpty(), "mixed S3 operations failed: ${errors}") - after = snapshotMetrics() - assertMetricChanges(before, after, [ - "s3_get_rate_limit_sleep_count", - "s3_put_rate_limit_sleep_count" - ]) - qt_mixed_result tvfSql(mixedPath) - - // Legacy GET token_limit: force an admission rejection and verify both the - // user-visible direction/reason and the dedicated rejection bvar. - applyLegacyCountLimitPhase(1, 0) - before = snapshotMetrics() - test { - sql getQpsReadSql - exception "s3 get request exceeds QPS limit" - exception "rejected by BE rate limiter" - } - after = snapshotMetrics() - assertMetricChanges(before, after, ["s3_get_rate_limit_rejected_count"]) - - // Legacy PUT token_limit has an independent counter and error message. - applyLegacyCountLimitPhase(0, 1) - before = snapshotMetrics() - test { - sql outfileSql("${rootPath}/put_rejected_", "WHERE id < 10") - exception "s3 put request exceeds QPS limit" - exception "rejected by BE rate limiter" - } - after = snapshotMetrics() - assertMetricChanges(before, after, ["s3_put_rate_limit_rejected_count"]) - } - } finally { - // setBeConfigTemporary restores the values first. Allow the daemon to publish - // the restored bucket parameters before another suite can use object storage. - sleep(12000) - } -} From 9ffbf4dc5d4f7c5cf4c4b9e70418021439a6d915 Mon Sep 17 00:00:00 2001 From: Refrain Date: Tue, 21 Jul 2026 19:58:51 +0800 Subject: [PATCH 08/22] [fix](be) Preserve S3 GET limiter error after retries ### What problem does this PR solve? Issue Number: None Related PR: #65420 Problem Summary: S3FileReader retried BE-local rate limiter 429 responses through the normal object-storage retry path, but replaced the limiter-specific reason with a generic read failure after retries. Preserve the existing retry, sleep, and counter behavior while including the last limiter response in the final error. ### Release note S3 GET rate-limit rejection errors now retain the QPS or bytes limit reason after retries. ### Check List (For Author) - Test: Regression test - cloud_p0/s3/test_s3_rate_limiter - Behavior changed: Yes. Final GET limiter errors include the limiter-specific reason while retry and metrics behavior stay unchanged. - Does this need documentation: No --- be/src/io/fs/s3_file_reader.cpp | 6 +++++- .../suites/cloud_p0/s3/test_s3_rate_limiter.groovy | 5 +---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/be/src/io/fs/s3_file_reader.cpp b/be/src/io/fs/s3_file_reader.cpp index 4eaa10f3311e06..191168d71e8c7d 100644 --- a/be/src/io/fs/s3_file_reader.cpp +++ b/be/src/io/fs/s3_file_reader.cpp @@ -159,11 +159,12 @@ Status S3FileReader::read_at_impl(size_t offset, Slice result, size_t* bytes_rea SCOPED_RAW_TIMER(&_s3_stats.total_get_request_time_ns); int total_sleep_time = 0; + ObjectStorageResponse resp; while (retry_count <= max_retries) { *bytes_read = 0; s3_file_reader_read_counter << 1; // clang-format off - auto resp = client->get_object( { .bucket = _bucket, .key = _key, }, + resp = client->get_object( { .bucket = _bucket, .key = _key, }, to, offset, bytes_req, bytes_read); // clang-format on _s3_stats.total_get_request_counter++; @@ -207,6 +208,9 @@ Status S3FileReader::read_at_impl(size_t offset, Slice result, size_t* bytes_rea "failed to get object, path={} offset={} bytes_req={} bytes_read={} file_size={} " "tries={}", _path.native(), offset, bytes_req, *bytes_read, _file_size, (max_retries + 1)); + if (resp.status.code == ErrorCode::EXCEEDED_LIMIT) { + msg.append(fmt::format(", err={}", resp.status.msg)); + } LOG(WARNING) << msg; return Status::InternalError(msg); } diff --git a/regression-test/suites/cloud_p0/s3/test_s3_rate_limiter.groovy b/regression-test/suites/cloud_p0/s3/test_s3_rate_limiter.groovy index 1390e2f6a21207..14902d31242897 100644 --- a/regression-test/suites/cloud_p0/s3/test_s3_rate_limiter.groovy +++ b/regression-test/suites/cloud_p0/s3/test_s3_rate_limiter.groovy @@ -367,10 +367,7 @@ suite("test_s3_rate_limiter", "p0,nonConcurrent") { SELECT count(*), sum(length(payload)) FROM test_s3_rate_limiter_get """ - check { result, exception, startTime, endTime -> - assertNotNull(exception, "internal GET should be rejected by token_limit") - logger.info("internal GET token_limit error: ${exception}") - } + exception "s3 get request exceeds QPS limit, rejected by BE rate limiter" } after = snapshotMetrics() assertMetricChanges(before, after, ["s3_get_rate_limit_rejected_count"]) From 4c07c974e3325e501a81f293d2abcf38c8af3b99 Mon Sep 17 00:00:00 2001 From: Refrain Date: Tue, 21 Jul 2026 23:41:59 +0800 Subject: [PATCH 09/22] r --- .../s3_rate_limiter/test_s3_rate_limiter.out | 16 ---------------- 1 file changed, 16 deletions(-) delete mode 100644 regression-test/data/external_table_p2/s3_rate_limiter/test_s3_rate_limiter.out diff --git a/regression-test/data/external_table_p2/s3_rate_limiter/test_s3_rate_limiter.out b/regression-test/data/external_table_p2/s3_rate_limiter/test_s3_rate_limiter.out deleted file mode 100644 index be551c64e42c70..00000000000000 --- a/regression-test/data/external_table_p2/s3_rate_limiter/test_s3_rate_limiter.out +++ /dev/null @@ -1,16 +0,0 @@ --- This file is automatically generated. You should know what you did if you want to edit this --- !get_qps_result -- -20 - --- !get_bytes_result -- -100000 - --- !put_qps_result -- -20 - --- !put_bytes_result -- -100000 - --- !mixed_result -- -100000 - From 72cdf97357a77746525e6f7bd6e96a824dc68241 Mon Sep 17 00:00:00 2001 From: Refrain Date: Wed, 22 Jul 2026 06:43:07 +0800 Subject: [PATCH 10/22] [test](regression) Add cgroup CPU resize coverage for S3 limiter ### What problem does this PR solve? Issue Number: None Related PR: #65420 Problem Summary: Add a Docker regression case that changes a running BE cgroup v2 CPU quota from one to two cores and verifies both cpu.max and the rebuilt GET/PUT QPS and bytes limiter parameters through BE logs. ### Release note None ### Check List (For Author) - Test: Regression test script loading; Docker execution was skipped because the current user cannot access the Docker socket - Behavior changed: No - Does this need documentation: No --- ...t_s3_rate_limiter_cgroup_cpu_resize.groovy | 116 ++++++++++++++++++ 1 file changed, 116 insertions(+) create mode 100644 regression-test/suites/cloud_p0/s3/test_s3_rate_limiter_cgroup_cpu_resize.groovy diff --git a/regression-test/suites/cloud_p0/s3/test_s3_rate_limiter_cgroup_cpu_resize.groovy b/regression-test/suites/cloud_p0/s3/test_s3_rate_limiter_cgroup_cpu_resize.groovy new file mode 100644 index 00000000000000..7ce530a5d91f3e --- /dev/null +++ b/regression-test/suites/cloud_p0/s3/test_s3_rate_limiter_cgroup_cpu_resize.groovy @@ -0,0 +1,116 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.suite.ClusterOptions + +suite("test_s3_rate_limiter_cgroup_cpu_resize", "docker") { + long getQpsPerCore = 11 + long putQpsPerCore = 13 + long getBytesPerCore = 1024 * 1024 + long putBytesPerCore = 2 * 1024 * 1024 + + def options = new ClusterOptions() + options.cloudMode = false + options.feNum = 1 + options.beNum = 1 + options.msNum = 0 + options.recyclerNum = 0 + options.beConfigs += [ + "enable_s3_rate_limiter=true", + "s3_rate_limiter_cpu_cores=0", + "s3_get_qps_per_core=${getQpsPerCore}", + "s3_put_qps_per_core=${putQpsPerCore}", + "s3_get_qps_max=0", + "s3_put_qps_max=0", + "s3_get_bytes_per_second_per_core=${getBytesPerCore}", + "s3_put_bytes_per_second_per_core=${putBytesPerCore}", + "s3_get_bytes_per_second_max=0", + "s3_put_bytes_per_second_max=0" + ] + + docker(options) { + // This case isolates the process-wide cgroup-to-limiter rebuild. Internal-vault + // request mapping and throttling are covered by test_s3_rate_limiter. + def be = cluster.getBeByIndex(1) + assertNotNull(be) + + String beContainer = "doris-${cluster.name}-be-1" + File beLog = new File(be.getLogFilePath()) + assertTrue(beLog.exists(), "BE log does not exist: ${beLog}") + + String cgroupV2Enabled = cmd("docker exec ${beContainer} sh -c " + + "'if [ -f /sys/fs/cgroup/cgroup.controllers ]; then echo true; else echo false; fi'") + .trim() + assertEquals("true", cgroupV2Enabled, "this case requires a cgroup v2 Docker host") + + int onlineCpus = cmd("docker exec ${beContainer} getconf _NPROCESSORS_ONLN") + .trim().toInteger() + assertTrue(onlineCpus >= 2, + "the Docker host needs at least 2 online CPUs, actual=${onlineCpus}") + + String cgroupMembership = cmd("docker exec ${beContainer} cat /proc/self/cgroup").trim() + logger.info("BE container cgroup membership: ${cgroupMembership}") + + def readCpuQuotaCores = { + String cpuMax = cmd("docker exec ${beContainer} cat /sys/fs/cgroup/cpu.max").trim() + logger.info("BE container cpu.max: ${cpuMax}") + def fields = cpuMax.split(/\s+/) + assertEquals(2, fields.size(), "invalid cpu.max content: ${cpuMax}") + assertFalse(fields[0] == "max", "cpu.max is unlimited: ${cpuMax}") + + long quota = fields[0].toLong() + long period = fields[1].toLong() + assertTrue(quota > 0 && period > 0, "invalid cpu.max content: ${cpuMax}") + return (quota + period - 1).intdiv(period) + } + + def expectedResetLogs = { int cores -> + long getQps = getQpsPerCore * cores + long putQps = putQpsPerCore * cores + long getBytes = getBytesPerCore * cores + long putBytes = putBytesPerCore * cores + return [ + "reset S3 get QPS rate limiter, qps=${getQps}, burst=${getQps}, " + + "count_limit=0, cores=${cores}", + "reset S3 put QPS rate limiter, qps=${putQps}, burst=${putQps}, " + + "count_limit=0, cores=${cores}", + "reset S3 get bytes rate limiter, bytes_per_second=${getBytes}, cores=${cores}", + "reset S3 put bytes rate limiter, bytes_per_second=${putBytes}, cores=${cores}" + ] + } + + def updateCpuQuotaAndWaitForLimiter = { int cores -> + int logOffset = beLog.getText("UTF-8").length() + cmd("docker update --cpus=${cores} ${beContainer}") + assertEquals(cores as long, readCpuQuotaCores(), + "docker update did not apply a ${cores}-CPU quota") + + List expectedLogs = expectedResetLogs(cores) + awaitUntil(35, 1) { + String currentLog = beLog.getText("UTF-8") + String newLog = currentLog.substring(Math.min(logOffset, currentLog.length())) + return expectedLogs.every { newLog.contains(it) } + } + } + + // Ensure the daemon's first 10-second refresh has initialized the manager before + // changing the cgroup quota, so both updates exercise an in-place limiter rebuild. + sleep(12000) + updateCpuQuotaAndWaitForLimiter(1) + updateCpuQuotaAndWaitForLimiter(2) + } +} From 2bed35fe39fb57c4de7a768351c7dcf6e9f0e78e Mon Sep 17 00:00:00 2001 From: Refrain Date: Wed, 22 Jul 2026 10:53:18 +0800 Subject: [PATCH 11/22] [fix](be) Fix S3 rate limiter CI failures ### What problem does this PR solve? Issue Number: None Related PR: #65420 Problem Summary: The full BE UT suite cleared the global config registry in ConfigTest and did not restore it, causing later S3 limiter config tests to fail with missing fields. The performance build also rejected a designated ObjectStorageHeadResponse initializer under GCC with -Wmissing-field-initializers. Preserve and restore the config registry around ConfigTest, and construct the rate-limited head response explicitly. ### Release note None ### Check List (For Author) - Test: Unit Test - ConfigTest and the two previously failing S3 limiter tests in one process - 39 focused S3 limiter tests - BE clang-format, check-format, and clang-tidy - Behavior changed: No - Does this need documentation: No --- be/src/io/fs/rate_limited_obj_storage_client.cpp | 4 +++- be/test/common/config_test.cpp | 6 ++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/be/src/io/fs/rate_limited_obj_storage_client.cpp b/be/src/io/fs/rate_limited_obj_storage_client.cpp index 8ed27a4454667a..ae65d80b3d2e10 100644 --- a/be/src/io/fs/rate_limited_obj_storage_client.cpp +++ b/be/src/io/fs/rate_limited_obj_storage_client.cpp @@ -76,7 +76,9 @@ ObjectStorageHeadResponse RateLimitedObjStorageClient::head_object( const ObjectStoragePathOptions& opts) { S3RateLimitGuard guard(S3RateLimitType::GET, 0); if (!guard.ok()) { - return {.resp = rate_limited_response(S3RateLimitType::GET, guard.reject_reason())}; + ObjectStorageHeadResponse response; + response.resp = rate_limited_response(S3RateLimitType::GET, guard.reject_reason()); + return response; } return _inner->head_object(opts); } diff --git a/be/test/common/config_test.cpp b/be/test/common/config_test.cpp index afeff7010c6e43..a1a3de49fd5bd5 100644 --- a/be/test/common/config_test.cpp +++ b/be/test/common/config_test.cpp @@ -32,8 +32,10 @@ namespace doris { using namespace config; class ConfigTest : public testing::Test { - void SetUp() override { config::Register::_s_field_map->clear(); } - void TearDown() override { config::Register::_s_field_map->clear(); } + void SetUp() override { _saved_fields.swap(*config::Register::_s_field_map); } + void TearDown() override { _saved_fields.swap(*config::Register::_s_field_map); } + + std::map _saved_fields; }; TEST_F(ConfigTest, DumpAllConfigs) { From 44db46a6ee7fdbd9fe48c811112a70bfc757644f Mon Sep 17 00:00:00 2001 From: Refrain Date: Wed, 22 Jul 2026 18:04:04 +0800 Subject: [PATCH 12/22] [fix](be) Fix broken disk test config reference ### What problem does this PR solve? Issue Number: None Related PR: #65420 Problem Summary: StorageEngineTest.TestBrokenDisk declared a local broken_storage_path config with the same name as the production config. With the global config registry correctly preserved, persisting broken paths updates the registered production variable while the test asserted against the empty local variable. Remove the duplicate local definition so the test checks the registered configuration value. ### Release note None ### Check List (For Author) - Test: Unit Test - ./run-be-ut.sh --run --filter=StorageEngineTest.TestBrokenDisk -j48 - Behavior changed: No - Does this need documentation: No --- be/test/storage/storage_engine_test.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/be/test/storage/storage_engine_test.cpp b/be/test/storage/storage_engine_test.cpp index d9e25b726cdf42..9099601d434620 100644 --- a/be/test/storage/storage_engine_test.cpp +++ b/be/test/storage/storage_engine_test.cpp @@ -69,7 +69,6 @@ class StorageEngineTest : public testing::Test { }; TEST_F(StorageEngineTest, TestBrokenDisk) { - DEFINE_mString(broken_storage_path, ""); std::string path = config::custom_config_dir + "/be_custom.conf"; std::error_code ec; From e154e2935f168fd28e778905143efcb2e6028ea3 Mon Sep 17 00:00:00 2001 From: Refrain Date: Fri, 24 Jul 2026 14:31:06 +0800 Subject: [PATCH 13/22] [test](be) Verify S3 limiter metric mode switching ### What problem does this PR solve? Issue Number: None Related PR: #65420 Problem Summary: Add deterministic BE and cloud regression coverage proving that CPU-aware byte limiting updates only the new byte-limit metrics while the original QPS metrics remain unchanged, and that switching qps_per_core back to -1 disables CPU-aware byte limiting and restores the legacy QPS metric path for both GET and PUT. ### Release note None ### Check List (For Author) - Test: Unit Test / Regression test - ./run-be-ut.sh --run --filter=S3RateLimiterMetricsTest.*:S3RateLimiterManagerTest.*:S3RateLimitGuardTest.* -j48 - ./run-regression-test.sh --run -d cloud_p0/s3 -s test_s3_rate_limiter against a local 3-BE cloud cluster - Behavior changed: No - Does this need documentation: No --- be/test/util/s3_rate_limiter_manager_test.cpp | 142 ++++++++++++++++++ .../cloud_p0/s3/test_s3_rate_limiter.groovy | 58 ++++++- 2 files changed, 196 insertions(+), 4 deletions(-) diff --git a/be/test/util/s3_rate_limiter_manager_test.cpp b/be/test/util/s3_rate_limiter_manager_test.cpp index 273433d7c65b21..5a7a413f511977 100644 --- a/be/test/util/s3_rate_limiter_manager_test.cpp +++ b/be/test/util/s3_rate_limiter_manager_test.cpp @@ -26,11 +26,139 @@ namespace doris { +extern bvar::Adder s3_get_rate_limit_sleep_ns; +extern bvar::Adder s3_get_rate_limit_sleep_count; +extern bvar::Adder s3_get_rate_limit_rejected_count; extern bvar::Adder s3_get_bytes_rate_limit_sleep_count; extern bvar::Adder s3_get_bytes_rate_limit_rejected_count; +extern bvar::Adder s3_get_bytes_rate_limit_sleep_ns; +extern bvar::Adder s3_put_rate_limit_sleep_ns; +extern bvar::Adder s3_put_rate_limit_sleep_count; +extern bvar::Adder s3_put_rate_limit_rejected_count; +extern bvar::Adder s3_put_bytes_rate_limit_sleep_ns; +extern bvar::Adder s3_put_bytes_rate_limit_sleep_count; +extern bvar::Adder s3_put_bytes_rate_limit_rejected_count; namespace { +struct RateLimiterMetricSnapshot { + int64_t qps_sleep_ns; + int64_t qps_sleep_count; + int64_t qps_rejected_count; + int64_t bytes_sleep_ns; + int64_t bytes_sleep_count; + int64_t bytes_rejected_count; +}; + +RateLimiterMetricSnapshot metric_snapshot(S3RateLimitType type) { + CHECK(type == S3RateLimitType::GET || type == S3RateLimitType::PUT); + if (type == S3RateLimitType::GET) { + return {.qps_sleep_ns = s3_get_rate_limit_sleep_ns.get_value(), + .qps_sleep_count = s3_get_rate_limit_sleep_count.get_value(), + .qps_rejected_count = s3_get_rate_limit_rejected_count.get_value(), + .bytes_sleep_ns = s3_get_bytes_rate_limit_sleep_ns.get_value(), + .bytes_sleep_count = s3_get_bytes_rate_limit_sleep_count.get_value(), + .bytes_rejected_count = s3_get_bytes_rate_limit_rejected_count.get_value()}; + } + return {.qps_sleep_ns = s3_put_rate_limit_sleep_ns.get_value(), + .qps_sleep_count = s3_put_rate_limit_sleep_count.get_value(), + .qps_rejected_count = s3_put_rate_limit_rejected_count.get_value(), + .bytes_sleep_ns = s3_put_bytes_rate_limit_sleep_ns.get_value(), + .bytes_sleep_count = s3_put_bytes_rate_limit_sleep_count.get_value(), + .bytes_rejected_count = s3_put_bytes_rate_limit_rejected_count.get_value()}; +} + +void expect_metrics_unchanged(const RateLimiterMetricSnapshot& before, + const RateLimiterMetricSnapshot& after) { + EXPECT_EQ(before.qps_sleep_ns, after.qps_sleep_ns); + EXPECT_EQ(before.qps_sleep_count, after.qps_sleep_count); + EXPECT_EQ(before.qps_rejected_count, after.qps_rejected_count); + EXPECT_EQ(before.bytes_sleep_ns, after.bytes_sleep_ns); + EXPECT_EQ(before.bytes_sleep_count, after.bytes_sleep_count); + EXPECT_EQ(before.bytes_rejected_count, after.bytes_rejected_count); +} + +void expect_only_bytes_sleep_grows(const RateLimiterMetricSnapshot& before, + const RateLimiterMetricSnapshot& after) { + EXPECT_EQ(before.qps_sleep_ns, after.qps_sleep_ns); + EXPECT_EQ(before.qps_sleep_count, after.qps_sleep_count); + EXPECT_EQ(before.qps_rejected_count, after.qps_rejected_count); + EXPECT_LT(before.bytes_sleep_ns, after.bytes_sleep_ns); + EXPECT_EQ(before.bytes_sleep_count + 1, after.bytes_sleep_count); + EXPECT_EQ(before.bytes_rejected_count, after.bytes_rejected_count); +} + +void expect_only_qps_sleep_grows(const RateLimiterMetricSnapshot& before, + const RateLimiterMetricSnapshot& after) { + EXPECT_LT(before.qps_sleep_ns, after.qps_sleep_ns); + EXPECT_EQ(before.qps_sleep_count + 1, after.qps_sleep_count); + EXPECT_EQ(before.qps_rejected_count, after.qps_rejected_count); + EXPECT_EQ(before.bytes_sleep_ns, after.bytes_sleep_ns); + EXPECT_EQ(before.bytes_sleep_count, after.bytes_sleep_count); + EXPECT_EQ(before.bytes_rejected_count, after.bytes_rejected_count); +} + +void verify_cpu_aware_bytes_and_legacy_qps_metrics(S3RateLimitType type, S3RateLimitType other_type, + int64_t& qps_per_core, int64_t& bytes_per_core) { + config::enable_s3_rate_limiter = true; + config::s3_rate_limiter_cpu_cores = 1; + config::s3_get_qps_max = 0; + config::s3_put_qps_max = 0; + config::s3_get_bytes_per_second_max = 0; + config::s3_put_bytes_per_second_max = 0; + config::s3_get_token_per_second = 1000; + config::s3_put_token_per_second = 1000; + config::s3_get_bucket_tokens = 1; + config::s3_put_bucket_tokens = 1; + config::s3_get_token_limit = 0; + config::s3_put_token_limit = 0; + + auto& manager = S3RateLimiterManager::instance(); + + // Enable only the target direction's CPU-aware bytes limiter. Keep a low + // legacy QPS configuration in place to prove qps_per_core=0 bypasses it. + config::s3_get_qps_per_core = 0; + config::s3_put_qps_per_core = 0; + config::s3_get_bytes_per_second_per_core = 0; + config::s3_put_bytes_per_second_per_core = 0; + bytes_per_core = 1000; + manager.refresh(); + + auto* qps = manager.qps_limiter(type); + auto* bytes = manager.bytes_limiter(type); + EXPECT_FALSE(qps->is_enabled()); + ASSERT_TRUE(bytes->is_enabled()); + EXPECT_EQ(1000, bytes->get_max_speed()); + EXPECT_EQ(1000, bytes->get_max_burst()); + + auto target_before = metric_snapshot(type); + auto other_before = metric_snapshot(other_type); + EXPECT_GT(bytes->add(bytes->get_max_burst() + 1), 0); + auto target_after = metric_snapshot(type); + auto other_after = metric_snapshot(other_type); + expect_only_bytes_sleep_grows(target_before, target_after); + expect_metrics_unchanged(other_before, other_after); + + // Disable CPU-aware bytes and select qps_per_core=-1. The same low legacy + // configuration now becomes effective, so only the original QPS metrics grow. + qps_per_core = -1; + bytes_per_core = 0; + manager.refresh(); + + EXPECT_TRUE(qps->is_enabled()); + EXPECT_EQ(1000, qps->get_max_speed()); + EXPECT_EQ(1, qps->get_max_burst()); + EXPECT_FALSE(bytes->is_enabled()); + + target_before = metric_snapshot(type); + other_before = metric_snapshot(other_type); + EXPECT_GT(qps->add(qps->get_max_burst() + 1), 0); + target_after = metric_snapshot(type); + other_after = metric_snapshot(other_type); + expect_only_qps_sleep_grows(target_before, target_after); + expect_metrics_unchanged(other_before, other_after); +} + // Saves every rate limiter related config on construction, restores it and re-applies // the limiters on destruction so tests do not leak state into each other. struct RateLimiterConfigGuard { @@ -449,6 +577,20 @@ TEST(S3RateLimiterMetricsTest, bytes_wait_and_rejection_have_distinct_counters) EXPECT_EQ(rejected_count_before + 1, s3_get_bytes_rate_limit_rejected_count.get_value()); } +TEST(S3RateLimiterMetricsTest, get_cpu_aware_bytes_and_legacy_qps_update_separate_metrics) { + RateLimiterConfigGuard guard; + verify_cpu_aware_bytes_and_legacy_qps_metrics(S3RateLimitType::GET, S3RateLimitType::PUT, + config::s3_get_qps_per_core, + config::s3_get_bytes_per_second_per_core); +} + +TEST(S3RateLimiterMetricsTest, put_cpu_aware_bytes_and_legacy_qps_update_separate_metrics) { + RateLimiterConfigGuard guard; + verify_cpu_aware_bytes_and_legacy_qps_metrics(S3RateLimitType::PUT, S3RateLimitType::GET, + config::s3_put_qps_per_core, + config::s3_put_bytes_per_second_per_core); +} + TEST(S3RateLimitGuardTest, legacy_count_limit_rejects) { RateLimiterConfigGuard guard; config::enable_s3_rate_limiter = true; diff --git a/regression-test/suites/cloud_p0/s3/test_s3_rate_limiter.groovy b/regression-test/suites/cloud_p0/s3/test_s3_rate_limiter.groovy index 14902d31242897..3c7e5323c1188d 100644 --- a/regression-test/suites/cloud_p0/s3/test_s3_rate_limiter.groovy +++ b/regression-test/suites/cloud_p0/s3/test_s3_rate_limiter.groovy @@ -106,6 +106,26 @@ suite("test_s3_rate_limiter", "p0,nonConcurrent") { sleep(12000) } + def applyLegacyQpsPhase = { long getQps, long putQps -> + Map phase = [ + "s3_get_qps_per_core": getQps > 0 ? -1 : 0, + "s3_put_qps_per_core": putQps > 0 ? -1 : 0, + "s3_get_token_per_second": getQps > 0 ? getQps : 1, + "s3_put_token_per_second": putQps > 0 ? putQps : 1, + "s3_get_bucket_tokens": 1, + "s3_put_bucket_tokens": 1, + "s3_get_token_limit": 0, + "s3_put_token_limit": 0, + "s3_get_bytes_per_second_per_core": 0, + "s3_put_bytes_per_second_per_core": 0 + ] + phase.each { key, value -> set_be_param(key, value) } + set_be_param("enable_s3_rate_limiter", true) + phase.each { key, value -> assertBeConfig(key, value) } + assertBeConfig("enable_s3_rate_limiter", true) + sleep(12000) + } + def applyLegacyCountLimitPhase = { long getLimit, long putLimit -> Map phase = [ "s3_get_qps_per_core": getLimit > 0 ? -1 : 0, @@ -185,10 +205,12 @@ suite("test_s3_rate_limiter", "p0,nonConcurrent") { "s3_put_qps_per_core": 0, "s3_get_qps_max": 0, "s3_put_qps_max": 0, - "s3_get_token_per_second": 1000000000000000000L, - "s3_put_token_per_second": 1000000000000000000L, - "s3_get_bucket_tokens": 1000000000000000000L, - "s3_put_bucket_tokens": 1000000000000000000L, + // Keep the legacy QPS settings low. CPU-aware phases must ignore them + // unless qps_per_core is explicitly switched back to -1. + "s3_get_token_per_second": 1, + "s3_put_token_per_second": 1, + "s3_get_bucket_tokens": 1, + "s3_put_bucket_tokens": 1, "s3_get_token_limit": 0, "s3_put_token_limit": 0, "s3_get_bytes_per_second_per_core": 0, @@ -274,6 +296,23 @@ suite("test_s3_rate_limiter", "p0,nonConcurrent") { "s3_get_bytes_rate_limit_sleep_count" ]) + // Disable CPU-aware GET bytes and switch qps_per_core to -1. The low + // legacy QPS config that was ignored above must now throttle the same + // internal-vault read. Only the original QPS metrics may grow. + applyLegacyQpsPhase(1, 0) + clearFileCacheOnAllBackends() + before = snapshotMetrics() + sql """ + SELECT count(*), sum(length(payload)) + FROM test_s3_rate_limiter_get + WHERE id < 1000000 + """ + after = snapshotMetrics() + assertMetricChanges(before, after, [ + "s3_get_rate_limit_sleep_ns", + "s3_get_rate_limit_sleep_count" + ]) + // PUT QPS: concurrent small inserts create independent small segment files // for the same tablet, guaranteeing multiple requests against one BE bucket. applyLimiterPhase(0, 1, 0, 0) @@ -300,6 +339,17 @@ suite("test_s3_rate_limiter", "p0,nonConcurrent") { "s3_put_bytes_rate_limit_sleep_count" ]) + // Disable CPU-aware PUT bytes and switch qps_per_core to -1. Concurrent + // internal-vault writes must now update only the original PUT QPS metrics. + applyLegacyQpsPhase(0, 1) + before = snapshotMetrics() + runConcurrentSmallInserts(4) + after = snapshotMetrics() + assertMetricChanges(before, after, [ + "s3_put_rate_limit_sleep_ns", + "s3_put_rate_limit_sleep_count" + ]) + // Mixed internal IO: scan uncached vault data while another large segment // is uploaded. GET and PUT QPS metrics must grow independently. applyLimiterPhase(1, 1, 0, 0) From a2c4675a24b64d638effaa1365953b4ee5e7db9a Mon Sep 17 00:00:00 2001 From: 0AyanamiRei <3244156674@qq.com> Date: Thu, 30 Jul 2026 12:10:40 +0800 Subject: [PATCH 14/22] useless change --- be/src/common/config.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index a6f8db0fde85d2..551176e2f2f314 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -2241,7 +2241,6 @@ bool init(const char* conf_file, bool fill_conf_map, bool must_exist, bool set_t } \ TYPE& ref_conf_value = *reinterpret_cast((FIELD).storage); \ TYPE old_value = ref_conf_value; \ - ref_conf_value = new_value; \ if (RegisterConfValidator::_s_field_validator != nullptr) { \ auto validator = RegisterConfValidator::_s_field_validator->find((FIELD).name); \ if (validator != RegisterConfValidator::_s_field_validator->end() && \ From 016677539d59afc54a06037e04d964e2b737a050 Mon Sep 17 00:00:00 2001 From: 0AyanamiRei <3244156674@qq.com> Date: Thu, 30 Jul 2026 12:44:16 +0800 Subject: [PATCH 15/22] small change --- be/src/io/fs/azure_obj_storage_client.cpp | 12 ++++-------- be/src/util/s3_rate_limiter_manager.cpp | 2 +- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/be/src/io/fs/azure_obj_storage_client.cpp b/be/src/io/fs/azure_obj_storage_client.cpp index 92c99d06409fd6..3beeeb2b05218d 100644 --- a/be/src/io/fs/azure_obj_storage_client.cpp +++ b/be/src/io/fs/azure_obj_storage_client.cpp @@ -248,10 +248,8 @@ ObjectStorageResponse AzureObjStorageClient::complete_multipart_upload( [](const ObjectCompleteMultiPart& i) { return base64_encode_part_num(i.part_num); }); return do_azure_client_call( [&]() { - { - SCOPED_BVAR_LATENCY(s3_bvar::s3_multi_part_upload_latency); - client.CommitBlockList(string_block_ids); - } + SCOPED_BVAR_LATENCY(s3_bvar::s3_multi_part_upload_latency); + client.CommitBlockList(string_block_ids); }, opts, _tls_debug_context); } @@ -260,10 +258,8 @@ ObjectStorageHeadResponse AzureObjStorageClient::head_object(const ObjectStorage Models::BlobProperties properties {}; auto resp = do_azure_client_call( [&]() { - properties = [&]() { - SCOPED_BVAR_LATENCY(s3_bvar::s3_head_latency); - return _client->GetBlockBlobClient(opts.key).GetProperties().Value; - }(); + SCOPED_BVAR_LATENCY(s3_bvar::s3_head_latency); + properties = _client->GetBlockBlobClient(opts.key).GetProperties().Value; }, opts, _tls_debug_context); if (resp.http_code == static_cast(Azure::Core::Http::HttpStatusCode::NotFound)) { diff --git a/be/src/util/s3_rate_limiter_manager.cpp b/be/src/util/s3_rate_limiter_manager.cpp index b3210d473a42e3..c241e071959ecd 100644 --- a/be/src/util/s3_rate_limiter_manager.cpp +++ b/be/src/util/s3_rate_limiter_manager.cpp @@ -63,7 +63,7 @@ int64_t cap_multiply(int64_t per_core, int64_t cores, int64_t cap) { } size_t index_of(S3RateLimitType type) { - CHECK(type == S3RateLimitType::GET || type == S3RateLimitType::PUT) << to_string(type); + DCHECK(type == S3RateLimitType::GET || type == S3RateLimitType::PUT) << to_string(type); return static_cast(type); } From 7c295dfcba319634230bb1bc1d7a1cc72271e65e Mon Sep 17 00:00:00 2001 From: 0AyanamiRei <3244156674@qq.com> Date: Thu, 30 Jul 2026 13:09:44 +0800 Subject: [PATCH 16/22] change http429 --- .../io/fs/rate_limited_obj_storage_client.cpp | 3 +- be/src/io/fs/s3_file_reader.cpp | 6 +-- .../rate_limited_obj_storage_client_test.cpp | 48 +++++++++---------- .../io/fs/s3_obj_stroage_client_mock_test.cpp | 4 +- 4 files changed, 29 insertions(+), 32 deletions(-) diff --git a/be/src/io/fs/rate_limited_obj_storage_client.cpp b/be/src/io/fs/rate_limited_obj_storage_client.cpp index ae65d80b3d2e10..1b8730847162df 100644 --- a/be/src/io/fs/rate_limited_obj_storage_client.cpp +++ b/be/src/io/fs/rate_limited_obj_storage_client.cpp @@ -30,7 +30,8 @@ ObjectStorageResponse rate_limited_response(S3RateLimitType type, S3RateLimitRej return {.status = convert_to_obj_response(Status::Error( "s3 {} request exceeds {} limit, rejected by BE rate limiter", to_string(type), limit_type)), - .http_code = 429}; + // The BE rate limiter rejected the request before it reached the provider. + .http_code = 0}; } } // namespace diff --git a/be/src/io/fs/s3_file_reader.cpp b/be/src/io/fs/s3_file_reader.cpp index 191168d71e8c7d..4eaa10f3311e06 100644 --- a/be/src/io/fs/s3_file_reader.cpp +++ b/be/src/io/fs/s3_file_reader.cpp @@ -159,12 +159,11 @@ Status S3FileReader::read_at_impl(size_t offset, Slice result, size_t* bytes_rea SCOPED_RAW_TIMER(&_s3_stats.total_get_request_time_ns); int total_sleep_time = 0; - ObjectStorageResponse resp; while (retry_count <= max_retries) { *bytes_read = 0; s3_file_reader_read_counter << 1; // clang-format off - resp = client->get_object( { .bucket = _bucket, .key = _key, }, + auto resp = client->get_object( { .bucket = _bucket, .key = _key, }, to, offset, bytes_req, bytes_read); // clang-format on _s3_stats.total_get_request_counter++; @@ -208,9 +207,6 @@ Status S3FileReader::read_at_impl(size_t offset, Slice result, size_t* bytes_rea "failed to get object, path={} offset={} bytes_req={} bytes_read={} file_size={} " "tries={}", _path.native(), offset, bytes_req, *bytes_read, _file_size, (max_retries + 1)); - if (resp.status.code == ErrorCode::EXCEEDED_LIMIT) { - msg.append(fmt::format(", err={}", resp.status.msg)); - } LOG(WARNING) << msg; return Status::InternalError(msg); } diff --git a/be/test/io/fs/rate_limited_obj_storage_client_test.cpp b/be/test/io/fs/rate_limited_obj_storage_client_test.cpp index 4bf4ea904518aa..657b139c0a4fcd 100644 --- a/be/test/io/fs/rate_limited_obj_storage_client_test.cpp +++ b/be/test/io/fs/rate_limited_obj_storage_client_test.cpp @@ -163,8 +163,8 @@ TEST(RateLimitedObjStorageClientTest, get_rejected_by_count_limit_does_not_reach EXPECT_EQ(1, fake->calls); auto resp = client.head_object(opts); - EXPECT_NE(0, resp.resp.status.code); - EXPECT_EQ(429, resp.resp.http_code); + EXPECT_EQ(ErrorCode::EXCEEDED_LIMIT, resp.resp.status.code); + EXPECT_EQ(0, resp.resp.http_code); EXPECT_NE(std::string::npos, resp.resp.status.msg.find("exceeds QPS limit")); EXPECT_EQ(1, fake->calls); // rejected before reaching the provider @@ -190,8 +190,8 @@ TEST(RateLimitedObjStorageClientTest, put_rejected_by_count_limit_does_not_reach EXPECT_EQ(1, fake->calls); auto resp = client.put_object(opts, "data"); - EXPECT_NE(0, resp.status.code); - EXPECT_EQ(429, resp.http_code); + EXPECT_EQ(ErrorCode::EXCEEDED_LIMIT, resp.status.code); + EXPECT_EQ(0, resp.http_code); EXPECT_NE(std::string::npos, resp.status.msg.find("exceeds QPS limit")); EXPECT_EQ(1, fake->calls); // rejected before reaching the provider @@ -218,8 +218,8 @@ TEST(RateLimitedObjStorageClientTest, head_and_list_map_to_get_qps_without_bytes EXPECT_EQ(0, client.list_objects(opts, &files).status.code); auto rejected = client.head_object(opts); - EXPECT_NE(0, rejected.resp.status.code); - EXPECT_EQ(429, rejected.resp.http_code); + EXPECT_EQ(ErrorCode::EXCEEDED_LIMIT, rejected.resp.status.code); + EXPECT_EQ(0, rejected.resp.http_code); EXPECT_NE(std::string::npos, rejected.resp.status.msg.find("exceeds QPS limit")); EXPECT_EQ(2, fake->calls); @@ -247,8 +247,8 @@ TEST(RateLimitedObjStorageClientTest, get_object_maps_to_get_qps_and_get_bytes) EXPECT_EQ(4, size_return); auto rejected = client.get_object(opts, nullptr, 0, 4, &size_return); - EXPECT_NE(0, rejected.status.code); - EXPECT_EQ(429, rejected.http_code); + EXPECT_EQ(ErrorCode::EXCEEDED_LIMIT, rejected.status.code); + EXPECT_EQ(0, rejected.http_code); EXPECT_NE(std::string::npos, rejected.status.msg.find("exceeds QPS limit")); EXPECT_EQ(1, fake->calls); @@ -311,8 +311,8 @@ TEST(RateLimitedObjStorageClientTest, put_object_maps_to_put_qps_and_put_bytes) EXPECT_EQ(0, client.put_object(opts, "data").status.code); auto rejected = client.put_object(opts, "data"); - EXPECT_NE(0, rejected.status.code); - EXPECT_EQ(429, rejected.http_code); + EXPECT_EQ(ErrorCode::EXCEEDED_LIMIT, rejected.status.code); + EXPECT_EQ(0, rejected.http_code); EXPECT_NE(std::string::npos, rejected.status.msg.find("exceeds QPS limit")); EXPECT_EQ(1, fake->calls); @@ -334,8 +334,8 @@ TEST(RateLimitedObjStorageClientTest, upload_part_maps_to_put_qps_and_put_bytes) EXPECT_EQ(0, client.upload_part(opts, "data", 1).resp.status.code); auto rejected = client.upload_part(opts, "data", 2); - EXPECT_NE(0, rejected.resp.status.code); - EXPECT_EQ(429, rejected.resp.http_code); + EXPECT_EQ(ErrorCode::EXCEEDED_LIMIT, rejected.resp.status.code); + EXPECT_EQ(0, rejected.resp.http_code); EXPECT_NE(std::string::npos, rejected.resp.status.msg.find("exceeds QPS limit")); EXPECT_EQ(1, fake->calls); @@ -359,8 +359,8 @@ TEST(RateLimitedObjStorageClientTest, multipart_control_apis_map_to_put_qps_with EXPECT_EQ(0, client.complete_multipart_upload(opts, {}).status.code); auto rejected = client.create_multipart_upload(opts); - EXPECT_NE(0, rejected.resp.status.code); - EXPECT_EQ(429, rejected.resp.http_code); + EXPECT_EQ(ErrorCode::EXCEEDED_LIMIT, rejected.resp.status.code); + EXPECT_EQ(0, rejected.resp.http_code); EXPECT_NE(std::string::npos, rejected.resp.status.msg.find("exceeds QPS limit")); EXPECT_EQ(2, fake->calls); @@ -386,8 +386,8 @@ TEST(RateLimitedObjStorageClientTest, delete_apis_map_to_put_qps_without_bytes) EXPECT_EQ(0, client.delete_objects_recursively(opts).status.code); auto rejected = client.delete_object(opts); - EXPECT_NE(0, rejected.status.code); - EXPECT_EQ(429, rejected.http_code); + EXPECT_EQ(ErrorCode::EXCEEDED_LIMIT, rejected.status.code); + EXPECT_EQ(0, rejected.http_code); EXPECT_NE(std::string::npos, rejected.status.msg.find("exceeds QPS limit")); EXPECT_EQ(3, fake->calls); @@ -415,14 +415,14 @@ TEST(RateLimitedObjStorageClientTest, bytes_rejections_have_distinct_text_and_me size_t size_return = 0; auto get_resp = client.get_object(opts, nullptr, 0, 2, &size_return); - EXPECT_NE(0, get_resp.status.code); - EXPECT_EQ(429, get_resp.http_code); + EXPECT_EQ(ErrorCode::EXCEEDED_LIMIT, get_resp.status.code); + EXPECT_EQ(0, get_resp.http_code); EXPECT_NE(std::string::npos, get_resp.status.msg.find("exceeds bytes limit")); EXPECT_EQ(get_rejected_before + 1, s3_get_bytes_rate_limit_rejected_count.get_value()); auto put_resp = client.put_object(opts, "xx"); - EXPECT_NE(0, put_resp.status.code); - EXPECT_EQ(429, put_resp.http_code); + EXPECT_EQ(ErrorCode::EXCEEDED_LIMIT, put_resp.status.code); + EXPECT_EQ(0, put_resp.http_code); EXPECT_NE(std::string::npos, put_resp.status.msg.find("exceeds bytes limit")); EXPECT_EQ(put_rejected_before + 1, s3_put_bytes_rate_limit_rejected_count.get_value()); @@ -449,8 +449,8 @@ TEST(RateLimitedObjStorageClientTest, recursive_delete_charges_one_put_qps) { EXPECT_EQ(4, fake->delete_objects_recursively_provider_calls); auto resp = client.delete_objects_recursively(opts); - EXPECT_NE(0, resp.status.code); - EXPECT_EQ(429, resp.http_code); + EXPECT_EQ(ErrorCode::EXCEEDED_LIMIT, resp.status.code); + EXPECT_EQ(0, resp.http_code); EXPECT_EQ(1, fake->delete_objects_recursively_calls); EXPECT_EQ(4, fake->delete_objects_recursively_provider_calls); } @@ -473,8 +473,8 @@ TEST(RateLimitedObjStorageClientTest, azure_noop_multipart_create_charges_one_pu EXPECT_EQ(0, fake->create_multipart_upload_provider_calls); auto resp = client.create_multipart_upload(opts); - EXPECT_NE(0, resp.resp.status.code); - EXPECT_EQ(429, resp.resp.http_code); + EXPECT_EQ(ErrorCode::EXCEEDED_LIMIT, resp.resp.status.code); + EXPECT_EQ(0, resp.resp.http_code); EXPECT_EQ(1, fake->create_multipart_upload_calls); EXPECT_EQ(0, fake->create_multipart_upload_provider_calls); } diff --git a/be/test/io/fs/s3_obj_stroage_client_mock_test.cpp b/be/test/io/fs/s3_obj_stroage_client_mock_test.cpp index c010c120480c67..1a00bcb8642f0c 100644 --- a/be/test/io/fs/s3_obj_stroage_client_mock_test.cpp +++ b/be/test/io/fs/s3_obj_stroage_client_mock_test.cpp @@ -143,8 +143,8 @@ TEST_F(S3ObjStorageClientMockTest, list_objects_pagination_charges_one_get_qps) // The first logical list used one GET token despite issuing three provider requests. // A second logical list is rejected before it reaches the provider. response = rate_limited_client.list_objects(opts, &files); - EXPECT_NE(response.status.code, ErrorCode::OK); - EXPECT_EQ(response.http_code, 429); + EXPECT_EQ(ErrorCode::EXCEEDED_LIMIT, response.status.code); + EXPECT_EQ(0, response.http_code); } TEST_F(S3ObjStorageClientMockTest, test_ca_cert) { From 9bb7831f12373f86fc8e92e47562369396152352 Mon Sep 17 00:00:00 2001 From: 0AyanamiRei <3244156674@qq.com> Date: Thu, 30 Jul 2026 13:12:38 +0800 Subject: [PATCH 17/22] bk --- be/src/common/config.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index 551176e2f2f314..05871682376f41 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -2250,6 +2250,7 @@ bool init(const char* conf_file, bool fill_conf_map, bool must_exist, bool set_t (FIELD).name, new_value); \ } \ } \ + ref_conf_value = new_value; \ if (full_conf_map != nullptr) { \ std::ostringstream oss; \ oss << new_value; \ From 6e1df4b42df58bbc87119c2c651c2fa8523b2cee Mon Sep 17 00:00:00 2001 From: Refrain Date: Thu, 30 Jul 2026 15:40:16 +0800 Subject: [PATCH 18/22] rewr --- be/src/common/config.cpp | 11 ++-- be/src/common/config.h | 4 +- be/src/util/s3_rate_limiter_manager.cpp | 31 +++++++--- be/src/util/s3_rate_limiter_manager.h | 2 +- be/test/util/s3_rate_limiter_manager_test.cpp | 24 +++++--- .../cloud_p0/s3/test_s3_rate_limiter.groovy | 56 +------------------ ...t_s3_rate_limiter_cgroup_cpu_resize.groovy | 2 +- 7 files changed, 50 insertions(+), 80 deletions(-) diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index 05871682376f41..2b53d6f358f230 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -1575,11 +1575,12 @@ DEFINE_Validator(s3_get_bytes_per_second_max, [](int64_t config) -> bool { retur DEFINE_mInt64(s3_put_bytes_per_second_max, "0"); DEFINE_Validator(s3_put_bytes_per_second_max, [](int64_t config) -> bool { return config >= 0; }); -// CPU cores used to derive the effective S3 rate limits. 0 means auto-detect from the -// cgroup cpu quota (fall back to physical cores). A positive value overrides detection; -// the control plane can push it via /api/update_config when resizing a serverless BE. -DEFINE_mInt64(s3_rate_limiter_cpu_cores, "0"); -DEFINE_Validator(s3_rate_limiter_cpu_cores, [](int64_t config) -> bool { return config >= 0; }); +// Override the CPU cores used to derive the effective S3 rate limits. 0 means auto-detect +// from the cgroup cpu quota (fall back to physical cores); the control plane can push a +// positive value via /api/update_config when resizing a serverless BE. +DEFINE_mInt32(s3_rate_limiter_cpu_cores_override, "0"); +DEFINE_Validator(s3_rate_limiter_cpu_cores_override, + [](int32_t config) -> bool { return config >= 0; }); DEFINE_String(trino_connector_plugin_dir, "${DORIS_HOME}/plugins/connectors"); diff --git a/be/src/common/config.h b/be/src/common/config.h index b7fd4b93073753..1c7ec164f0a044 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -1632,8 +1632,8 @@ DECLARE_mInt64(s3_put_bytes_per_second_per_core); // Hard caps for the CPU-derived GET/PUT bytes/s. 0 means no cap. DECLARE_mInt64(s3_get_bytes_per_second_max); DECLARE_mInt64(s3_put_bytes_per_second_max); -// Cores used to derive effective limits: 0 = auto-detect from cgroup quota; >0 overrides. -DECLARE_mInt64(s3_rate_limiter_cpu_cores); +// Override for cores used to derive effective limits: 0 = auto-detect from cgroup quota. +DECLARE_mInt32(s3_rate_limiter_cpu_cores_override); // max s3 client retry times DECLARE_mInt32(max_s3_client_retry); // When meet s3 429 error, the "get" request will diff --git a/be/src/util/s3_rate_limiter_manager.cpp b/be/src/util/s3_rate_limiter_manager.cpp index c241e071959ecd..f4360091ba53f8 100644 --- a/be/src/util/s3_rate_limiter_manager.cpp +++ b/be/src/util/s3_rate_limiter_manager.cpp @@ -20,6 +20,7 @@ #include #include #include +#include #include "common/config.h" #include "common/logging.h" @@ -38,16 +39,32 @@ bvar::Adder s3_put_bytes_rate_limit_rejected_count( namespace { +constexpr int64_t kBytesRateLimitLongSleepNs = 500 * 1000 * 1000; +constexpr int kBytesRateLimitLongSleepLogInterval = 50; + +std::function bytes_rate_limiter_metric_func_with_log( + S3RateLimitType type, bvar::Adder& sleep_ns_bvar, + bvar::Adder& sleep_count_bvar, bvar::Adder& rejected_count_bvar) { + auto metric_func = metric_func_factory(sleep_ns_bvar, sleep_count_bvar, &rejected_count_bvar); + return [type, metric_func = std::move(metric_func)](int64_t sleep_ns) { + metric_func(sleep_ns); + LOG_IF_EVERY_N(WARNING, sleep_ns > kBytesRateLimitLongSleepNs, + kBytesRateLimitLongSleepLogInterval) + << "S3 " << to_string(type) << " request is throttled by bytes rate limiter" + << ", sleep_ms=" << sleep_ns / 1000000; + }; +} + std::function bytes_rate_limiter_metric_func(S3RateLimitType type) { switch (type) { case S3RateLimitType::GET: - return metric_func_factory(s3_get_bytes_rate_limit_sleep_ns, - s3_get_bytes_rate_limit_sleep_count, - &s3_get_bytes_rate_limit_rejected_count); + return bytes_rate_limiter_metric_func_with_log(type, s3_get_bytes_rate_limit_sleep_ns, + s3_get_bytes_rate_limit_sleep_count, + s3_get_bytes_rate_limit_rejected_count); case S3RateLimitType::PUT: - return metric_func_factory(s3_put_bytes_rate_limit_sleep_ns, - s3_put_bytes_rate_limit_sleep_count, - &s3_put_bytes_rate_limit_rejected_count); + return bytes_rate_limiter_metric_func_with_log(type, s3_put_bytes_rate_limit_sleep_ns, + s3_put_bytes_rate_limit_sleep_count, + s3_put_bytes_rate_limit_rejected_count); default: return [](int64_t) {}; } @@ -104,7 +121,7 @@ int reset_s3_rate_limiter(S3RateLimitType type, size_t max_speed, size_t max_bur } int64_t s3_rate_limiter_cpu_cores() { - if (int64_t overridden = config::s3_rate_limiter_cpu_cores; overridden > 0) { + if (int32_t overridden = config::s3_rate_limiter_cpu_cores_override; overridden > 0) { return overridden; } int physical = static_cast(std::thread::hardware_concurrency()); diff --git a/be/src/util/s3_rate_limiter_manager.h b/be/src/util/s3_rate_limiter_manager.h index 0c7a63b979fe19..ea6ed1eecb1049 100644 --- a/be/src/util/s3_rate_limiter_manager.h +++ b/be/src/util/s3_rate_limiter_manager.h @@ -43,7 +43,7 @@ struct S3EffectiveRateLimit { // `cores` does not participate; otherwise qps = min(per_core * cores, qps_max). S3EffectiveRateLimit resolve_s3_rate_limit(S3RateLimitType type, int64_t cores); -// Cores used to derive effective limits: config::s3_rate_limiter_cpu_cores override +// Cores used to derive effective limits: config::s3_rate_limiter_cpu_cores_override // (> 0) wins; otherwise re-read the cgroup cpu quota (serverless BEs can be resized // in place), falling back to physical cores. Always >= 1. int64_t s3_rate_limiter_cpu_cores(); diff --git a/be/test/util/s3_rate_limiter_manager_test.cpp b/be/test/util/s3_rate_limiter_manager_test.cpp index 5a7a413f511977..984a0c9ec9fbe0 100644 --- a/be/test/util/s3_rate_limiter_manager_test.cpp +++ b/be/test/util/s3_rate_limiter_manager_test.cpp @@ -101,7 +101,7 @@ void expect_only_qps_sleep_grows(const RateLimiterMetricSnapshot& before, void verify_cpu_aware_bytes_and_legacy_qps_metrics(S3RateLimitType type, S3RateLimitType other_type, int64_t& qps_per_core, int64_t& bytes_per_core) { config::enable_s3_rate_limiter = true; - config::s3_rate_limiter_cpu_cores = 1; + config::s3_rate_limiter_cpu_cores_override = 1; config::s3_get_qps_max = 0; config::s3_put_qps_max = 0; config::s3_get_bytes_per_second_max = 0; @@ -177,7 +177,7 @@ struct RateLimiterConfigGuard { int64_t put_bytes_per_core = config::s3_put_bytes_per_second_per_core; int64_t get_bytes_max = config::s3_get_bytes_per_second_max; int64_t put_bytes_max = config::s3_put_bytes_per_second_max; - int64_t cpu_cores = config::s3_rate_limiter_cpu_cores; + int32_t cpu_cores_override = config::s3_rate_limiter_cpu_cores_override; ~RateLimiterConfigGuard() { config::enable_s3_rate_limiter = enable; @@ -195,7 +195,7 @@ struct RateLimiterConfigGuard { config::s3_put_bytes_per_second_per_core = put_bytes_per_core; config::s3_get_bytes_per_second_max = get_bytes_max; config::s3_put_bytes_per_second_max = put_bytes_max; - config::s3_rate_limiter_cpu_cores = cpu_cores; + config::s3_rate_limiter_cpu_cores_override = cpu_cores_override; S3RateLimiterManager::instance().refresh(); } }; @@ -211,7 +211,7 @@ TEST(S3RateLimiterResolveTest, registered_defaults_preserve_legacy_compatibility EXPECT_STREQ("-1", fields.at("s3_put_qps_per_core").defval); EXPECT_STREQ("-1", fields.at("s3_get_bytes_per_second_per_core").defval); EXPECT_STREQ("-1", fields.at("s3_put_bytes_per_second_per_core").defval); - EXPECT_STREQ("0", fields.at("s3_rate_limiter_cpu_cores").defval); + EXPECT_STREQ("0", fields.at("s3_rate_limiter_cpu_cores_override").defval); } TEST(S3RateLimiterConfigTest, invalid_dynamic_values_are_rejected_without_changing_config) { @@ -220,6 +220,7 @@ TEST(S3RateLimiterConfigTest, invalid_dynamic_values_are_rejected_without_changi config::s3_get_qps_per_core = -1; config::s3_get_bytes_per_second_per_core = -1; config::s3_put_bytes_per_second_max = 0; + config::s3_rate_limiter_cpu_cores_override = 0; auto status = config::set_config("s3_get_qps_per_core", "-2"); EXPECT_FALSE(status.ok()); @@ -241,6 +242,11 @@ TEST(S3RateLimiterConfigTest, invalid_dynamic_values_are_rejected_without_changi EXPECT_NE(std::string::npos, status.to_string().find("convert '9223372036854775808' as int64_t failed")); EXPECT_EQ(0, config::s3_put_bytes_per_second_max); + + status = config::set_config("s3_rate_limiter_cpu_cores_override", "2147483648"); + EXPECT_FALSE(status.ok()); + EXPECT_NE(std::string::npos, status.to_string().find("convert '2147483648' as int32_t failed")); + EXPECT_EQ(0, config::s3_rate_limiter_cpu_cores_override); } TEST(S3RateLimiterResolveTest, legacy_config_wins_when_per_core_unset) { @@ -373,10 +379,10 @@ TEST(S3RateLimiterResolveTest, get_and_put_bytes_caps_are_resolved_independently TEST(S3RateLimiterResolveTest, cpu_cores_override_config_wins) { RateLimiterConfigGuard guard; - config::s3_rate_limiter_cpu_cores = 16; + config::s3_rate_limiter_cpu_cores_override = 16; EXPECT_EQ(16, s3_rate_limiter_cpu_cores()); - config::s3_rate_limiter_cpu_cores = 0; + config::s3_rate_limiter_cpu_cores_override = 0; EXPECT_GE(s3_rate_limiter_cpu_cores(), 1); // auto-detect always yields >= 1 } @@ -433,7 +439,7 @@ TEST(S3RateLimiterManagerTest, refresh_applies_qps_speed_burst_and_limit_changes EXPECT_EQ(200, get_qps->get_max_burst()); EXPECT_EQ(300, get_qps->get_limit()); - config::s3_rate_limiter_cpu_cores = kCores; + config::s3_rate_limiter_cpu_cores_override = kCores; config::s3_get_qps_per_core = 100; config::s3_get_qps_max = 0; manager.refresh(); @@ -442,7 +448,7 @@ TEST(S3RateLimiterManagerTest, refresh_applies_qps_speed_burst_and_limit_changes EXPECT_EQ(0, get_qps->get_limit()); // Simulate a serverless resize: the core count is an input of refresh(). - config::s3_rate_limiter_cpu_cores = 2 * kCores; + config::s3_rate_limiter_cpu_cores_override = 2 * kCores; manager.refresh(); EXPECT_EQ(100 * 2 * kCores, get_qps->get_max_speed()); EXPECT_EQ(100 * 2 * kCores, get_qps->get_max_burst()); @@ -454,7 +460,7 @@ TEST(S3RateLimiterManagerTest, refresh_applies_bytes_limit_and_enables_bucket) { auto& manager = S3RateLimiterManager::instance(); auto* put_bytes = manager.bytes_limiter(S3RateLimitType::PUT); - config::s3_rate_limiter_cpu_cores = kCores; + config::s3_rate_limiter_cpu_cores_override = kCores; config::s3_put_bytes_per_second_per_core = -1; manager.refresh(); EXPECT_FALSE(put_bytes->is_enabled()); diff --git a/regression-test/suites/cloud_p0/s3/test_s3_rate_limiter.groovy b/regression-test/suites/cloud_p0/s3/test_s3_rate_limiter.groovy index 3c7e5323c1188d..ecd743e5a0323d 100644 --- a/regression-test/suites/cloud_p0/s3/test_s3_rate_limiter.groovy +++ b/regression-test/suites/cloud_p0/s3/test_s3_rate_limiter.groovy @@ -126,26 +126,6 @@ suite("test_s3_rate_limiter", "p0,nonConcurrent") { sleep(12000) } - def applyLegacyCountLimitPhase = { long getLimit, long putLimit -> - Map phase = [ - "s3_get_qps_per_core": getLimit > 0 ? -1 : 0, - "s3_put_qps_per_core": putLimit > 0 ? -1 : 0, - "s3_get_token_per_second": 1000000000000000000L, - "s3_put_token_per_second": 1000000000000000000L, - "s3_get_bucket_tokens": 1000000000000000000L, - "s3_put_bucket_tokens": 1000000000000000000L, - "s3_get_token_limit": getLimit, - "s3_put_token_limit": putLimit, - "s3_get_bytes_per_second_per_core": 0, - "s3_put_bytes_per_second_per_core": 0 - ] - phase.each { key, value -> set_be_param(key, value) } - set_be_param("enable_s3_rate_limiter", true) - phase.each { key, value -> assertBeConfig(key, value) } - assertBeConfig("enable_s3_rate_limiter", true) - sleep(12000) - } - def clearFileCacheOnAllBackends = { backendIdToIp.each { id, ip -> String httpPort = backendIdToHttpPort[id] @@ -200,7 +180,7 @@ suite("test_s3_rate_limiter", "p0,nonConcurrent") { // Keep PUT phases directionally pure: the optional post-upload HeadObject // would otherwise add GET traffic to an internal write. "enable_s3_object_check_after_upload": false, - "s3_rate_limiter_cpu_cores": 1, + "s3_rate_limiter_cpu_cores_override": 1, "s3_get_qps_per_core": 0, "s3_put_qps_per_core": 0, "s3_get_qps_max": 0, @@ -399,40 +379,6 @@ suite("test_s3_rate_limiter", "p0,nonConcurrent") { "s3_put_rate_limit_sleep_count" ]) - // Legacy GET token_limit: a cache-free scan must be rejected by the - // internal GET bucket. Append a fresh rowset with limiting disabled so - // no process-local cache can make this phase vacuous. - applyLegacyCountLimitPhase(1, 0) - set_be_param("enable_s3_rate_limiter", false) - sql """ - INSERT INTO test_s3_rate_limiter_get - SELECT 3000000 + number, repeat(md5(cast(number + 3000000 as string)), 4) - FROM numbers("number" = "1000") - """ - set_be_param("enable_s3_rate_limiter", true) - clearFileCacheOnAllBackends() - before = snapshotMetrics() - test { - sql """ - SELECT count(*), sum(length(payload)) - FROM test_s3_rate_limiter_get - """ - exception "s3 get request exceeds QPS limit, rejected by BE rate limiter" - } - after = snapshotMetrics() - assertMetricChanges(before, after, ["s3_get_rate_limit_rejected_count"]) - - // Legacy PUT token_limit: the first one-object segment consumes the token; - // the second is rejected with an independently attributable error/bvar. - applyLegacyCountLimitPhase(0, 1) - sql "INSERT INTO test_s3_rate_limiter_put VALUES (9000000, 'first token')" - before = snapshotMetrics() - test { - sql "INSERT INTO test_s3_rate_limiter_put VALUES (9000001, 'rejected token')" - exception "s3 put request exceeds QPS limit, rejected by BE rate limiter" - } - after = snapshotMetrics() - assertMetricChanges(before, after, ["s3_put_rate_limit_rejected_count"]) } } finally { // Let the daemon publish the restored bucket parameters before another suite diff --git a/regression-test/suites/cloud_p0/s3/test_s3_rate_limiter_cgroup_cpu_resize.groovy b/regression-test/suites/cloud_p0/s3/test_s3_rate_limiter_cgroup_cpu_resize.groovy index 7ce530a5d91f3e..3c6e7a82906573 100644 --- a/regression-test/suites/cloud_p0/s3/test_s3_rate_limiter_cgroup_cpu_resize.groovy +++ b/regression-test/suites/cloud_p0/s3/test_s3_rate_limiter_cgroup_cpu_resize.groovy @@ -31,7 +31,7 @@ suite("test_s3_rate_limiter_cgroup_cpu_resize", "docker") { options.recyclerNum = 0 options.beConfigs += [ "enable_s3_rate_limiter=true", - "s3_rate_limiter_cpu_cores=0", + "s3_rate_limiter_cpu_cores_override=0", "s3_get_qps_per_core=${getQpsPerCore}", "s3_put_qps_per_core=${putQpsPerCore}", "s3_get_qps_max=0", From 7c75a83056f6aab4c995d38665d4a87899a8f48d Mon Sep 17 00:00:00 2001 From: 0AyanamiRei <3244156674@qq.com> Date: Thu, 30 Jul 2026 19:38:23 +0800 Subject: [PATCH 19/22] [fix](be) Remove duplicate Trino plugin config definition ### What problem does this PR solve? Issue Number: None Related PR: #65420 Problem Summary: Merging the latest master into the S3 rate limiter branch kept both the old and new definitions of trino_connector_plugin_dir in BE config.cpp. Clang rejected the duplicate config variable and registration object definitions. Remove the obsolete plugins/connectors definition and keep the master definition that matches the FE plugins/trino_plugins default. ### Release note None ### Check List (For Author) - Test: No need to test (single merge-conflict cleanup; local testing was explicitly not requested) - Behavior changed: No - Does this need documentation: No --- be/src/common/config.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index ea58b2c119008f..f4abf9063728cc 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -1589,7 +1589,6 @@ DEFINE_mInt32(s3_rate_limiter_cpu_cores_override, "0"); DEFINE_Validator(s3_rate_limiter_cpu_cores_override, [](int32_t config) -> bool { return config >= 0; }); -DEFINE_String(trino_connector_plugin_dir, "${DORIS_HOME}/plugins/connectors"); // The dir TrinoConnectorPluginLoader loads Trino's own plugins from, used verbatim. Keep the default // in sync with FE Config.trino_connector_plugin_dir: FE and BE load the same plugins and an operator // who leaves both untouched expects both to find them. From fffe93b9ad07330c3ca143c6365e3da84dbcc7d2 Mon Sep 17 00:00:00 2001 From: Refrain Date: Thu, 30 Jul 2026 22:44:55 +0800 Subject: [PATCH 20/22] [fix](be) Validate dynamic configs against candidate values ### What problem does this PR solve? Issue Number: None Related PR: #65420 Problem Summary: The BE UT exposed that dynamic config validators checked the active value before the converted candidate was assigned. An invalid S3 limiter update such as s3_get_qps_per_core=-2 therefore passed validation and changed the live config. Publish the candidate before invoking the registered validator; the existing failure path restores the previous value. ### Release note Invalid dynamic BE configuration values are rejected without changing the active value. ### Check List (For Author) - Test: Unit Test - ./run-be-ut.sh --run --filter=S3RateLimiterConfigTest.*:ConfigValidatorTest.* -j48 - Behavior changed: Yes. Dynamic validators now evaluate the candidate value and roll back invalid updates. - Does this need documentation: No --- be/src/common/config.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index f4abf9063728cc..7b5bf350d91b1f 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -2252,6 +2252,7 @@ bool init(const char* conf_file, bool fill_conf_map, bool must_exist, bool set_t } \ TYPE& ref_conf_value = *reinterpret_cast((FIELD).storage); \ TYPE old_value = ref_conf_value; \ + ref_conf_value = new_value; \ if (RegisterConfValidator::_s_field_validator != nullptr) { \ auto validator = RegisterConfValidator::_s_field_validator->find((FIELD).name); \ if (validator != RegisterConfValidator::_s_field_validator->end() && \ @@ -2261,7 +2262,6 @@ bool init(const char* conf_file, bool fill_conf_map, bool must_exist, bool set_t (FIELD).name, new_value); \ } \ } \ - ref_conf_value = new_value; \ if (full_conf_map != nullptr) { \ std::ostringstream oss; \ oss << new_value; \ From d76d454b5de96bd334de835966421ed5ca4038fd Mon Sep 17 00:00:00 2001 From: Refrain Date: Thu, 30 Jul 2026 23:17:51 +0800 Subject: [PATCH 21/22] [fix](be) Align S3 limiter config semantics ### What problem does this PR solve? Issue Number: None Related PR: #65420 Problem Summary: The S3 limiter follow-up added redundant Azure timing scopes and immediately invoked lambdas. It also restricted new limiter configs to sentinel values even though the resolver already treats any negative or non-positive value as fallback, disabled, uncapped, or auto-detected. Remove the redundant Azure wrappers, align the config declarations and comments with the resolver semantics, restore the existing UPDATE_FIELD behavior, and remove the invalid dynamic-value unit test. ### Release note None ### Check List (For Author) - Test: No test run per request; clang-format and check-format passed - Behavior changed: Yes; negative limiter tuning values now follow the existing resolver semantics - Does this need documentation: No --- be/src/common/config.cpp | 28 +++------ be/src/common/config.h | 12 ++-- be/src/io/fs/azure_obj_storage_client.cpp | 58 +++++++------------ be/src/util/s3_rate_limiter_manager.cpp | 2 +- be/src/util/s3_rate_limiter_manager.h | 2 +- be/test/util/s3_rate_limiter_manager_test.cpp | 38 +----------- 6 files changed, 39 insertions(+), 101 deletions(-) diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index 7b5bf350d91b1f..31a0a20f3c25ab 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -1551,43 +1551,31 @@ DEFINE_mInt64(s3_rate_limiter_log_interval, "1000"); DEFINE_Validator(s3_rate_limiter_log_interval, [](int64_t config) -> bool { return config >= 0; }); // CPU-aware S3 rate limiter. Effective GET/PUT QPS = qps_per_core * BE cpu cores, capped by -// the corresponding qps_max. -1 means unset: fall back to the legacy absolute +// the corresponding qps_max. A negative value means unset: fall back to the legacy absolute // s3_{get,put}_token_* configs above. 0 disables QPS limiting for that operation. DEFINE_mInt64(s3_get_qps_per_core, "-1"); -DEFINE_Validator(s3_get_qps_per_core, [](int64_t config) -> bool { return config >= -1; }); DEFINE_mInt64(s3_put_qps_per_core, "-1"); -DEFINE_Validator(s3_put_qps_per_core, [](int64_t config) -> bool { return config >= -1; }); -// Hard caps for the CPU-derived GET/PUT QPS. 0 means no cap. +// Hard caps for the CPU-derived GET/PUT QPS. A non-positive value means no cap. DEFINE_mInt64(s3_get_qps_max, "0"); -DEFINE_Validator(s3_get_qps_max, [](int64_t config) -> bool { return config >= 0; }); DEFINE_mInt64(s3_put_qps_max, "0"); -DEFINE_Validator(s3_put_qps_max, [](int64_t config) -> bool { return config >= 0; }); // CPU-aware S3 bandwidth limiter. Effective GET/PUT bytes/s = bytes_per_second_per_core * -// BE cpu cores, capped by the corresponding bytes_per_second_max. -1 and 0 both disable +// BE cpu cores, capped by the corresponding bytes_per_second_max. A non-positive value disables // byte-rate limiting for that operation (there is no legacy fallback for bandwidth). // Note: the derived per-BE bytes/s should not be set below the single IO upper bound // per second (s3_write_buffer_size, 5MB by default). A single IO larger than 1 second // of quota only reserves 1 second worth of tokens; the excess bytes are not accounted // (reservation clamp in S3RateLimitGuard). DEFINE_mInt64(s3_get_bytes_per_second_per_core, "-1"); -DEFINE_Validator(s3_get_bytes_per_second_per_core, - [](int64_t config) -> bool { return config >= -1; }); DEFINE_mInt64(s3_put_bytes_per_second_per_core, "-1"); -DEFINE_Validator(s3_put_bytes_per_second_per_core, - [](int64_t config) -> bool { return config >= -1; }); -// Hard caps for the CPU-derived GET/PUT bytes/s. 0 means no cap. +// Hard caps for the CPU-derived GET/PUT bytes/s. A non-positive value means no cap. DEFINE_mInt64(s3_get_bytes_per_second_max, "0"); -DEFINE_Validator(s3_get_bytes_per_second_max, [](int64_t config) -> bool { return config >= 0; }); DEFINE_mInt64(s3_put_bytes_per_second_max, "0"); -DEFINE_Validator(s3_put_bytes_per_second_max, [](int64_t config) -> bool { return config >= 0; }); -// Override the CPU cores used to derive the effective S3 rate limits. 0 means auto-detect -// from the cgroup cpu quota (fall back to physical cores); the control plane can push a -// positive value via /api/update_config when resizing a serverless BE. +// Override the CPU cores used to derive the effective S3 rate limits. A non-positive value +// means auto-detect from the cgroup cpu quota (fall back to physical cores); the control plane +// can push a positive value via /api/update_config when resizing a serverless BE. DEFINE_mInt32(s3_rate_limiter_cpu_cores_override, "0"); -DEFINE_Validator(s3_rate_limiter_cpu_cores_override, - [](int32_t config) -> bool { return config >= 0; }); // The dir TrinoConnectorPluginLoader loads Trino's own plugins from, used verbatim. Keep the default // in sync with FE Config.trino_connector_plugin_dir: FE and BE load the same plugins and an operator @@ -2252,7 +2240,6 @@ bool init(const char* conf_file, bool fill_conf_map, bool must_exist, bool set_t } \ TYPE& ref_conf_value = *reinterpret_cast((FIELD).storage); \ TYPE old_value = ref_conf_value; \ - ref_conf_value = new_value; \ if (RegisterConfValidator::_s_field_validator != nullptr) { \ auto validator = RegisterConfValidator::_s_field_validator->find((FIELD).name); \ if (validator != RegisterConfValidator::_s_field_validator->end() && \ @@ -2262,6 +2249,7 @@ bool init(const char* conf_file, bool fill_conf_map, bool must_exist, bool set_t (FIELD).name, new_value); \ } \ } \ + ref_conf_value = new_value; \ if (full_conf_map != nullptr) { \ std::ostringstream oss; \ oss << new_value; \ diff --git a/be/src/common/config.h b/be/src/common/config.h index c39b54a37a7568..3b504941827d95 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -1622,20 +1622,20 @@ DECLARE_mInt64(s3_put_token_per_second); DECLARE_mInt64(s3_put_token_limit); DECLARE_mInt64(s3_rate_limiter_log_interval); -// CPU-aware S3 rate limiter: GET/PUT QPS per CPU core. -1 = unset, fall back to the -// legacy absolute token configs above; 0 disables QPS limiting for that operation. +// CPU-aware S3 rate limiter: GET/PUT QPS per CPU core. A negative value means unset and +// falls back to the legacy absolute token configs above; 0 disables QPS limiting. DECLARE_mInt64(s3_get_qps_per_core); DECLARE_mInt64(s3_put_qps_per_core); -// Hard caps for the CPU-derived GET/PUT QPS. 0 means no cap. +// Hard caps for the CPU-derived GET/PUT QPS. A non-positive value means no cap. DECLARE_mInt64(s3_get_qps_max); DECLARE_mInt64(s3_put_qps_max); -// GET/PUT bytes per second per CPU core. -1 and 0 both disable byte-rate limiting. +// GET/PUT bytes per second per CPU core. A non-positive value disables byte-rate limiting. DECLARE_mInt64(s3_get_bytes_per_second_per_core); DECLARE_mInt64(s3_put_bytes_per_second_per_core); -// Hard caps for the CPU-derived GET/PUT bytes/s. 0 means no cap. +// Hard caps for the CPU-derived GET/PUT bytes/s. A non-positive value means no cap. DECLARE_mInt64(s3_get_bytes_per_second_max); DECLARE_mInt64(s3_put_bytes_per_second_max); -// Override for cores used to derive effective limits: 0 = auto-detect from cgroup quota. +// Override for cores used to derive effective limits: a non-positive value means auto-detect. DECLARE_mInt32(s3_rate_limiter_cpu_cores_override); // max s3 client retry times DECLARE_mInt32(max_s3_client_retry); diff --git a/be/src/io/fs/azure_obj_storage_client.cpp b/be/src/io/fs/azure_obj_storage_client.cpp index 3beeeb2b05218d..9702c87b3b304b 100644 --- a/be/src/io/fs/azure_obj_storage_client.cpp +++ b/be/src/io/fs/azure_obj_storage_client.cpp @@ -144,10 +144,8 @@ struct AzureBatchDeleter { } auto resp = do_azure_client_call( [&]() { - { - SCOPED_BVAR_LATENCY(s3_bvar::s3_delete_objects_latency); - _client->SubmitBatch(_batch); - } + SCOPED_BVAR_LATENCY(s3_bvar::s3_delete_objects_latency); + _client->SubmitBatch(_batch); }, _opts, _tls_debug_context); if (resp.status.code != ErrorCode::OK) { @@ -209,11 +207,8 @@ ObjectStorageResponse AzureObjStorageClient::put_object(const ObjectStoragePathO auto client = _client->GetBlockBlobClient(opts.key); return do_azure_client_call( [&]() { - { - SCOPED_BVAR_LATENCY(s3_bvar::s3_put_latency); - client.UploadFrom(reinterpret_cast(stream.data()), - stream.size()); - } + SCOPED_BVAR_LATENCY(s3_bvar::s3_put_latency); + client.UploadFrom(reinterpret_cast(stream.data()), stream.size()); }, opts, _tls_debug_context); } @@ -227,10 +222,8 @@ ObjectStorageUploadResponse AzureObjStorageClient::upload_part(const ObjectStora Azure::Core::IO::MemoryBodyStream memory_body( reinterpret_cast(stream.data()), stream.size()); // The blockId must be base64 encoded - { - SCOPED_BVAR_LATENCY(s3_bvar::s3_multi_part_upload_latency); - client.StageBlock(base64_encode_part_num(part_num), memory_body); - } + SCOPED_BVAR_LATENCY(s3_bvar::s3_multi_part_upload_latency); + client.StageBlock(base64_encode_part_num(part_num), memory_body); }, opts, _tls_debug_context); return ObjectStorageUploadResponse { @@ -285,11 +278,9 @@ ObjectStorageResponse AzureObjStorageClient::get_object(const ObjectStoragePathO DownloadBlobToOptions download_opts; Azure::Core::Http::HttpRange range {static_cast(offset), bytes_read}; download_opts.Range = range; - auto resp = [&]() { - SCOPED_BVAR_LATENCY(s3_bvar::s3_get_latency); - return client.DownloadTo(reinterpret_cast(buffer), bytes_read, - download_opts); - }(); + SCOPED_BVAR_LATENCY(s3_bvar::s3_get_latency); + auto resp = client.DownloadTo(reinterpret_cast(buffer), bytes_read, + download_opts); *size_return = resp.Value.ContentRange.Length.Value(); }, opts, _tls_debug_context); @@ -307,17 +298,18 @@ ObjectStorageResponse AzureObjStorageClient::list_objects(const ObjectStoragePat [&]() { ListBlobsOptions list_opts; list_opts.Prefix = opts.prefix; - auto resp = [&]() { + ListBlobsPagedResponse resp; + { SCOPED_BVAR_LATENCY(s3_bvar::s3_list_latency); - return _client->ListBlobs(list_opts); - }(); + resp = _client->ListBlobs(list_opts); + } get_file_file(resp); while (resp.NextPageToken.HasValue()) { list_opts.ContinuationToken = resp.NextPageToken; - resp = [&]() { + { SCOPED_BVAR_LATENCY(s3_bvar::s3_list_latency); - return _client->ListBlobs(list_opts); - }(); + resp = _client->ListBlobs(list_opts); + } get_file_file(resp); } }, @@ -353,10 +345,8 @@ ObjectStorageResponse AzureObjStorageClient::delete_objects(const ObjectStorageP ObjectStorageResponse AzureObjStorageClient::delete_object(const ObjectStoragePathOptions& opts) { return do_azure_client_call( [&]() { - auto resp = [&]() { - SCOPED_BVAR_LATENCY(s3_bvar::s3_delete_object_latency); - return _client->DeleteBlob(opts.key); - }(); + SCOPED_BVAR_LATENCY(s3_bvar::s3_delete_object_latency); + auto resp = _client->DeleteBlob(opts.key); if (!resp.Value.Deleted) { throw Exception(Status::IOError("Delete azure blob failed")); } @@ -384,10 +374,8 @@ ObjectStorageResponse AzureObjStorageClient::delete_objects_recursively( ListBlobsPagedResponse resp; auto list_resp = do_azure_client_call( [&]() { - resp = [&]() { - SCOPED_BVAR_LATENCY(s3_bvar::s3_list_latency); - return _client->ListBlobs(list_opts); - }(); + SCOPED_BVAR_LATENCY(s3_bvar::s3_list_latency); + resp = _client->ListBlobs(list_opts); }, opts, _tls_debug_context); if (list_resp.status.code != ErrorCode::OK) { @@ -402,10 +390,8 @@ ObjectStorageResponse AzureObjStorageClient::delete_objects_recursively( list_opts.ContinuationToken = resp.NextPageToken; list_resp = do_azure_client_call( [&]() { - resp = [&]() { - SCOPED_BVAR_LATENCY(s3_bvar::s3_list_latency); - return _client->ListBlobs(list_opts); - }(); + SCOPED_BVAR_LATENCY(s3_bvar::s3_list_latency); + resp = _client->ListBlobs(list_opts); }, opts, _tls_debug_context); if (list_resp.status.code != ErrorCode::OK) { diff --git a/be/src/util/s3_rate_limiter_manager.cpp b/be/src/util/s3_rate_limiter_manager.cpp index f4360091ba53f8..1b9d93a42a8612 100644 --- a/be/src/util/s3_rate_limiter_manager.cpp +++ b/be/src/util/s3_rate_limiter_manager.cpp @@ -70,7 +70,7 @@ std::function bytes_rate_limiter_metric_func(S3RateLimitType type } } -// min(per_core * cores, cap) with overflow protection; cap == 0 means no cap. +// min(per_core * cores, cap) with overflow protection; cap <= 0 means no cap. int64_t cap_multiply(int64_t per_core, int64_t cores, int64_t cap) { cap = cap > 0 ? cap : std::numeric_limits::max(); if (per_core > cap / cores) { diff --git a/be/src/util/s3_rate_limiter_manager.h b/be/src/util/s3_rate_limiter_manager.h index ea6ed1eecb1049..7c68d358c7b1ea 100644 --- a/be/src/util/s3_rate_limiter_manager.h +++ b/be/src/util/s3_rate_limiter_manager.h @@ -39,7 +39,7 @@ struct S3EffectiveRateLimit { }; // Pure function of configs and `cores`; no side effects. -// s3_{get,put}_qps_per_core == -1 falls back to the legacy absolute token configs and +// s3_{get,put}_qps_per_core < 0 falls back to the legacy absolute token configs and // `cores` does not participate; otherwise qps = min(per_core * cores, qps_max). S3EffectiveRateLimit resolve_s3_rate_limit(S3RateLimitType type, int64_t cores); diff --git a/be/test/util/s3_rate_limiter_manager_test.cpp b/be/test/util/s3_rate_limiter_manager_test.cpp index 984a0c9ec9fbe0..373c9a5f932cd1 100644 --- a/be/test/util/s3_rate_limiter_manager_test.cpp +++ b/be/test/util/s3_rate_limiter_manager_test.cpp @@ -22,7 +22,6 @@ #include #include "common/config.h" -#include "common/status.h" namespace doris { @@ -214,41 +213,6 @@ TEST(S3RateLimiterResolveTest, registered_defaults_preserve_legacy_compatibility EXPECT_STREQ("0", fields.at("s3_rate_limiter_cpu_cores_override").defval); } -TEST(S3RateLimiterConfigTest, invalid_dynamic_values_are_rejected_without_changing_config) { - RateLimiterConfigGuard guard; - config::enable_s3_rate_limiter = false; - config::s3_get_qps_per_core = -1; - config::s3_get_bytes_per_second_per_core = -1; - config::s3_put_bytes_per_second_max = 0; - config::s3_rate_limiter_cpu_cores_override = 0; - - auto status = config::set_config("s3_get_qps_per_core", "-2"); - EXPECT_FALSE(status.ok()); - EXPECT_NE(std::string::npos, status.to_string().find("validate s3_get_qps_per_core=-2 failed")); - EXPECT_EQ(-1, config::s3_get_qps_per_core); - - status = config::set_config("s3_get_bytes_per_second_per_core", "Ab"); - EXPECT_FALSE(status.ok()); - EXPECT_NE(std::string::npos, status.to_string().find("convert 'Ab' as int64_t failed")); - EXPECT_EQ(-1, config::s3_get_bytes_per_second_per_core); - - status = config::set_config("enable_s3_rate_limiter", "A"); - EXPECT_FALSE(status.ok()); - EXPECT_NE(std::string::npos, status.to_string().find("convert 'A' as bool failed")); - EXPECT_FALSE(config::enable_s3_rate_limiter); - - status = config::set_config("s3_put_bytes_per_second_max", "9223372036854775808"); - EXPECT_FALSE(status.ok()); - EXPECT_NE(std::string::npos, - status.to_string().find("convert '9223372036854775808' as int64_t failed")); - EXPECT_EQ(0, config::s3_put_bytes_per_second_max); - - status = config::set_config("s3_rate_limiter_cpu_cores_override", "2147483648"); - EXPECT_FALSE(status.ok()); - EXPECT_NE(std::string::npos, status.to_string().find("convert '2147483648' as int32_t failed")); - EXPECT_EQ(0, config::s3_rate_limiter_cpu_cores_override); -} - TEST(S3RateLimiterResolveTest, legacy_config_wins_when_per_core_unset) { RateLimiterConfigGuard guard; config::s3_get_qps_per_core = -1; @@ -368,7 +332,7 @@ TEST(S3RateLimiterResolveTest, get_and_put_bytes_caps_are_resolved_independently get_limit = resolve_s3_rate_limit(S3RateLimitType::GET, kCores); EXPECT_EQ(2048, get_limit.bytes_per_second); - // -1 and 0 both disable bandwidth limiting. + // Non-positive per-core values disable bandwidth limiting. config::s3_get_bytes_per_second_per_core = 0; config::s3_put_bytes_per_second_per_core = -1; get_limit = resolve_s3_rate_limit(S3RateLimitType::GET, kCores); From beaa8ea2bf6e5f300be4ced32505ceb9eb33278c Mon Sep 17 00:00:00 2001 From: Refrain Date: Thu, 30 Jul 2026 23:31:02 +0800 Subject: [PATCH 22/22] [improvement](be) Rename S3 request rate configs ### What problem does this PR solve? Issue Number: None Related PR: #65420 Problem Summary: The CPU-aware request-rate configs used the QPS abbreviation while the bandwidth configs spell out their units. Rename the four GET/PUT per-core and maximum request-rate configs to use requests_per_second consistently, and update all source, unit-test, and regression-test references. ### Release note Rename the new S3 CPU-aware request-rate configs to s3_get_requests_per_second_per_core, s3_put_requests_per_second_per_core, s3_get_requests_per_second_max, and s3_put_requests_per_second_max. ### Check List (For Author) - Test: No test run per request; clang-format and check-format passed - Behavior changed: Yes; the four new request-rate configuration names changed - Does this need documentation: No --- be/src/common/config.cpp | 15 ++-- be/src/common/config.h | 8 +- be/src/util/s3_rate_limiter_manager.cpp | 6 +- be/src/util/s3_rate_limiter_manager.h | 4 +- be/test/io/client/s3_file_system_test.cpp | 6 +- be/test/util/s3_rate_limiter_manager_test.cpp | 80 +++++++++---------- be/test/util/s3_util_test.cpp | 6 +- .../cloud_p0/s3/test_s3_rate_limiter.groovy | 22 ++--- ...t_s3_rate_limiter_cgroup_cpu_resize.groovy | 8 +- 9 files changed, 79 insertions(+), 76 deletions(-) diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index 31a0a20f3c25ab..3065d5b0fba09f 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -1550,14 +1550,15 @@ DEFINE_mInt64(s3_put_token_limit, "0"); DEFINE_mInt64(s3_rate_limiter_log_interval, "1000"); DEFINE_Validator(s3_rate_limiter_log_interval, [](int64_t config) -> bool { return config >= 0; }); -// CPU-aware S3 rate limiter. Effective GET/PUT QPS = qps_per_core * BE cpu cores, capped by -// the corresponding qps_max. A negative value means unset: fall back to the legacy absolute -// s3_{get,put}_token_* configs above. 0 disables QPS limiting for that operation. -DEFINE_mInt64(s3_get_qps_per_core, "-1"); -DEFINE_mInt64(s3_put_qps_per_core, "-1"); +// CPU-aware S3 rate limiter. Effective GET/PUT requests per second = +// requests_per_second_per_core * BE cpu cores, capped by the corresponding +// requests_per_second_max. A negative value means unset: fall back to the legacy absolute +// s3_{get,put}_token_* configs above. 0 disables request-rate limiting for that operation. +DEFINE_mInt64(s3_get_requests_per_second_per_core, "-1"); +DEFINE_mInt64(s3_put_requests_per_second_per_core, "-1"); // Hard caps for the CPU-derived GET/PUT QPS. A non-positive value means no cap. -DEFINE_mInt64(s3_get_qps_max, "0"); -DEFINE_mInt64(s3_put_qps_max, "0"); +DEFINE_mInt64(s3_get_requests_per_second_max, "0"); +DEFINE_mInt64(s3_put_requests_per_second_max, "0"); // CPU-aware S3 bandwidth limiter. Effective GET/PUT bytes/s = bytes_per_second_per_core * // BE cpu cores, capped by the corresponding bytes_per_second_max. A non-positive value disables diff --git a/be/src/common/config.h b/be/src/common/config.h index 3b504941827d95..ec48e288fec883 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -1624,11 +1624,11 @@ DECLARE_mInt64(s3_rate_limiter_log_interval); // CPU-aware S3 rate limiter: GET/PUT QPS per CPU core. A negative value means unset and // falls back to the legacy absolute token configs above; 0 disables QPS limiting. -DECLARE_mInt64(s3_get_qps_per_core); -DECLARE_mInt64(s3_put_qps_per_core); +DECLARE_mInt64(s3_get_requests_per_second_per_core); +DECLARE_mInt64(s3_put_requests_per_second_per_core); // Hard caps for the CPU-derived GET/PUT QPS. A non-positive value means no cap. -DECLARE_mInt64(s3_get_qps_max); -DECLARE_mInt64(s3_put_qps_max); +DECLARE_mInt64(s3_get_requests_per_second_max); +DECLARE_mInt64(s3_put_requests_per_second_max); // GET/PUT bytes per second per CPU core. A non-positive value disables byte-rate limiting. DECLARE_mInt64(s3_get_bytes_per_second_per_core); DECLARE_mInt64(s3_put_bytes_per_second_per_core); diff --git a/be/src/util/s3_rate_limiter_manager.cpp b/be/src/util/s3_rate_limiter_manager.cpp index 1b9d93a42a8612..d553a5ca63b57e 100644 --- a/be/src/util/s3_rate_limiter_manager.cpp +++ b/be/src/util/s3_rate_limiter_manager.cpp @@ -88,8 +88,10 @@ size_t index_of(S3RateLimitType type) { S3EffectiveRateLimit resolve_s3_rate_limit(S3RateLimitType type, int64_t cores) { const bool is_get = type == S3RateLimitType::GET; - const int64_t qps_per_core = is_get ? config::s3_get_qps_per_core : config::s3_put_qps_per_core; - const int64_t qps_max = is_get ? config::s3_get_qps_max : config::s3_put_qps_max; + const int64_t qps_per_core = is_get ? config::s3_get_requests_per_second_per_core + : config::s3_put_requests_per_second_per_core; + const int64_t qps_max = is_get ? config::s3_get_requests_per_second_max + : config::s3_put_requests_per_second_max; const int64_t bytes_per_core = is_get ? config::s3_get_bytes_per_second_per_core : config::s3_put_bytes_per_second_per_core; const int64_t bytes_max = diff --git a/be/src/util/s3_rate_limiter_manager.h b/be/src/util/s3_rate_limiter_manager.h index 7c68d358c7b1ea..ed668b02f0db0f 100644 --- a/be/src/util/s3_rate_limiter_manager.h +++ b/be/src/util/s3_rate_limiter_manager.h @@ -39,8 +39,8 @@ struct S3EffectiveRateLimit { }; // Pure function of configs and `cores`; no side effects. -// s3_{get,put}_qps_per_core < 0 falls back to the legacy absolute token configs and -// `cores` does not participate; otherwise qps = min(per_core * cores, qps_max). +// s3_{get,put}_requests_per_second_per_core < 0 falls back to the legacy absolute token +// configs and `cores` does not participate; otherwise qps = min(per_core * cores, qps_max). S3EffectiveRateLimit resolve_s3_rate_limit(S3RateLimitType type, int64_t cores); // Cores used to derive effective limits: config::s3_rate_limiter_cpu_cores_override diff --git a/be/test/io/client/s3_file_system_test.cpp b/be/test/io/client/s3_file_system_test.cpp index 87fb45165200df..1cc11876bde4c8 100644 --- a/be/test/io/client/s3_file_system_test.cpp +++ b/be/test/io/client/s3_file_system_test.cpp @@ -2458,13 +2458,13 @@ TEST_F(S3FileSystemTest, DynamicUpdateRateLimiterConfig) { // Save original config values int64_t original_get_bucket_tokens = config::s3_get_bucket_tokens; int64_t original_get_token_per_second = config::s3_get_token_per_second; - int64_t original_get_qps_per_core = config::s3_get_qps_per_core; + int64_t original_get_requests_per_second_per_core = config::s3_get_requests_per_second_per_core; auto& manager = S3RateLimiterManager::instance(); Defer restore_configs {[&] { config::s3_get_bucket_tokens = original_get_bucket_tokens; config::s3_get_token_per_second = original_get_token_per_second; - config::s3_get_qps_per_core = original_get_qps_per_core; + config::s3_get_requests_per_second_per_core = original_get_requests_per_second_per_core; manager.refresh(); }}; @@ -2472,7 +2472,7 @@ TEST_F(S3FileSystemTest, DynamicUpdateRateLimiterConfig) { int64_t new_s3_get_token_per_second_val = 1; // Legacy configs are only effective while the per-core config is unset. - ASSERT_TRUE(config::set_config("s3_get_qps_per_core", "-1").ok()); + ASSERT_TRUE(config::set_config("s3_get_requests_per_second_per_core", "-1").ok()); auto [success1, msg7] = config::set_config( "s3_get_bucket_tokens", std::to_string(new_s3_get_bucket_tokens_val), false, false); ASSERT_EQ(success1, 0) << "Failed to set s3_get_bucket_tokens: " << msg7; diff --git a/be/test/util/s3_rate_limiter_manager_test.cpp b/be/test/util/s3_rate_limiter_manager_test.cpp index 373c9a5f932cd1..78bdbdfd9991a0 100644 --- a/be/test/util/s3_rate_limiter_manager_test.cpp +++ b/be/test/util/s3_rate_limiter_manager_test.cpp @@ -101,8 +101,8 @@ void verify_cpu_aware_bytes_and_legacy_qps_metrics(S3RateLimitType type, S3RateL int64_t& qps_per_core, int64_t& bytes_per_core) { config::enable_s3_rate_limiter = true; config::s3_rate_limiter_cpu_cores_override = 1; - config::s3_get_qps_max = 0; - config::s3_put_qps_max = 0; + config::s3_get_requests_per_second_max = 0; + config::s3_put_requests_per_second_max = 0; config::s3_get_bytes_per_second_max = 0; config::s3_put_bytes_per_second_max = 0; config::s3_get_token_per_second = 1000; @@ -115,9 +115,9 @@ void verify_cpu_aware_bytes_and_legacy_qps_metrics(S3RateLimitType type, S3RateL auto& manager = S3RateLimiterManager::instance(); // Enable only the target direction's CPU-aware bytes limiter. Keep a low - // legacy QPS configuration in place to prove qps_per_core=0 bypasses it. - config::s3_get_qps_per_core = 0; - config::s3_put_qps_per_core = 0; + // legacy QPS configuration in place to prove requests_per_second_per_core=0 bypasses it. + config::s3_get_requests_per_second_per_core = 0; + config::s3_put_requests_per_second_per_core = 0; config::s3_get_bytes_per_second_per_core = 0; config::s3_put_bytes_per_second_per_core = 0; bytes_per_core = 1000; @@ -138,7 +138,7 @@ void verify_cpu_aware_bytes_and_legacy_qps_metrics(S3RateLimitType type, S3RateL expect_only_bytes_sleep_grows(target_before, target_after); expect_metrics_unchanged(other_before, other_after); - // Disable CPU-aware bytes and select qps_per_core=-1. The same low legacy + // Disable CPU-aware bytes and select requests_per_second_per_core=-1. The same low legacy // configuration now becomes effective, so only the original QPS metrics grow. qps_per_core = -1; bytes_per_core = 0; @@ -168,10 +168,10 @@ struct RateLimiterConfigGuard { int64_t put_tps = config::s3_put_token_per_second; int64_t put_bucket = config::s3_put_bucket_tokens; int64_t put_limit = config::s3_put_token_limit; - int64_t get_per_core = config::s3_get_qps_per_core; - int64_t put_per_core = config::s3_put_qps_per_core; - int64_t get_qps_max = config::s3_get_qps_max; - int64_t put_qps_max = config::s3_put_qps_max; + int64_t get_per_core = config::s3_get_requests_per_second_per_core; + int64_t put_per_core = config::s3_put_requests_per_second_per_core; + int64_t get_qps_max = config::s3_get_requests_per_second_max; + int64_t put_qps_max = config::s3_put_requests_per_second_max; int64_t get_bytes_per_core = config::s3_get_bytes_per_second_per_core; int64_t put_bytes_per_core = config::s3_put_bytes_per_second_per_core; int64_t get_bytes_max = config::s3_get_bytes_per_second_max; @@ -186,10 +186,10 @@ struct RateLimiterConfigGuard { config::s3_put_token_per_second = put_tps; config::s3_put_bucket_tokens = put_bucket; config::s3_put_token_limit = put_limit; - config::s3_get_qps_per_core = get_per_core; - config::s3_put_qps_per_core = put_per_core; - config::s3_get_qps_max = get_qps_max; - config::s3_put_qps_max = put_qps_max; + config::s3_get_requests_per_second_per_core = get_per_core; + config::s3_put_requests_per_second_per_core = put_per_core; + config::s3_get_requests_per_second_max = get_qps_max; + config::s3_put_requests_per_second_max = put_qps_max; config::s3_get_bytes_per_second_per_core = get_bytes_per_core; config::s3_put_bytes_per_second_per_core = put_bytes_per_core; config::s3_get_bytes_per_second_max = get_bytes_max; @@ -206,8 +206,8 @@ constexpr size_t kNoThrottle = 1ULL << 40; TEST(S3RateLimiterResolveTest, registered_defaults_preserve_legacy_compatibility) { const auto& fields = *config::Register::_s_field_map; - EXPECT_STREQ("-1", fields.at("s3_get_qps_per_core").defval); - EXPECT_STREQ("-1", fields.at("s3_put_qps_per_core").defval); + EXPECT_STREQ("-1", fields.at("s3_get_requests_per_second_per_core").defval); + EXPECT_STREQ("-1", fields.at("s3_put_requests_per_second_per_core").defval); EXPECT_STREQ("-1", fields.at("s3_get_bytes_per_second_per_core").defval); EXPECT_STREQ("-1", fields.at("s3_put_bytes_per_second_per_core").defval); EXPECT_STREQ("0", fields.at("s3_rate_limiter_cpu_cores_override").defval); @@ -215,16 +215,16 @@ TEST(S3RateLimiterResolveTest, registered_defaults_preserve_legacy_compatibility TEST(S3RateLimiterResolveTest, legacy_config_wins_when_per_core_unset) { RateLimiterConfigGuard guard; - config::s3_get_qps_per_core = -1; + config::s3_get_requests_per_second_per_core = -1; config::s3_get_token_per_second = 123; config::s3_get_bucket_tokens = 456; config::s3_get_token_limit = 789; - config::s3_get_qps_max = 9; // must be ignored on the legacy path - config::s3_put_qps_per_core = -1; + config::s3_get_requests_per_second_max = 9; // must be ignored on the legacy path + config::s3_put_requests_per_second_per_core = -1; config::s3_put_token_per_second = 321; config::s3_put_bucket_tokens = 654; config::s3_put_token_limit = 987; - config::s3_put_qps_max = 8; // must be ignored on the legacy path + config::s3_put_requests_per_second_max = 8; // must be ignored on the legacy path config::s3_get_bytes_per_second_per_core = -1; config::s3_put_bytes_per_second_per_core = -1; @@ -243,13 +243,13 @@ TEST(S3RateLimiterResolveTest, legacy_config_wins_when_per_core_unset) { TEST(S3RateLimiterResolveTest, get_and_put_per_core_configs_override_legacy_independently) { RateLimiterConfigGuard guard; - config::s3_get_qps_per_core = 3; - config::s3_get_qps_max = 0; + config::s3_get_requests_per_second_per_core = 3; + config::s3_get_requests_per_second_max = 0; config::s3_get_token_per_second = 321; config::s3_get_bucket_tokens = 654; config::s3_get_token_limit = 987; - config::s3_put_qps_per_core = 7; - config::s3_put_qps_max = 0; + config::s3_put_requests_per_second_per_core = 7; + config::s3_put_requests_per_second_max = 0; config::s3_put_token_per_second = 123; config::s3_put_bucket_tokens = 456; config::s3_put_token_limit = 789; @@ -267,10 +267,10 @@ TEST(S3RateLimiterResolveTest, get_and_put_per_core_configs_override_legacy_inde TEST(S3RateLimiterResolveTest, per_core_zero_disables_qps_limiting) { RateLimiterConfigGuard guard; - config::s3_get_qps_per_core = 0; + config::s3_get_requests_per_second_per_core = 0; config::s3_get_token_per_second = 123; config::s3_get_token_limit = 789; - config::s3_put_qps_per_core = 0; + config::s3_put_requests_per_second_per_core = 0; config::s3_put_token_per_second = 321; config::s3_put_token_limit = 987; @@ -287,10 +287,10 @@ TEST(S3RateLimiterResolveTest, per_core_zero_disables_qps_limiting) { TEST(S3RateLimiterResolveTest, get_and_put_qps_max_cap_per_core_results_independently) { RateLimiterConfigGuard guard; - config::s3_get_qps_per_core = 1000000; - config::s3_get_qps_max = 256; - config::s3_put_qps_per_core = 2000000; - config::s3_put_qps_max = 512; + config::s3_get_requests_per_second_per_core = 1000000; + config::s3_get_requests_per_second_max = 256; + config::s3_put_requests_per_second_per_core = 2000000; + config::s3_put_requests_per_second_max = 512; auto get_limit = resolve_s3_rate_limit(S3RateLimitType::GET, kCores); EXPECT_EQ(256, get_limit.qps); @@ -303,14 +303,14 @@ TEST(S3RateLimiterResolveTest, get_and_put_qps_max_cap_per_core_results_independ TEST(S3RateLimiterResolveTest, overflowing_multiplication_is_capped) { RateLimiterConfigGuard guard; - config::s3_get_qps_per_core = std::numeric_limits::max(); - config::s3_get_qps_max = 1024; + config::s3_get_requests_per_second_per_core = std::numeric_limits::max(); + config::s3_get_requests_per_second_max = 1024; auto limit = resolve_s3_rate_limit(S3RateLimitType::GET, kCores); EXPECT_EQ(1024, limit.qps); // Without a cap the overflowing product saturates instead of wrapping around. - config::s3_get_qps_max = 0; + config::s3_get_requests_per_second_max = 0; limit = resolve_s3_rate_limit(S3RateLimitType::GET, kCores); EXPECT_EQ(std::numeric_limits::max(), limit.qps); } @@ -355,7 +355,7 @@ TEST(S3RateLimiterManagerTest, refresh_with_same_parameters_keeps_consumed_state auto& manager = S3RateLimiterManager::instance(); auto* get_qps = manager.qps_limiter(S3RateLimitType::GET); - config::s3_get_qps_per_core = -1; + config::s3_get_requests_per_second_per_core = -1; config::s3_get_token_per_second = kNoThrottle; config::s3_get_bucket_tokens = kNoThrottle; config::s3_get_token_limit = 2; @@ -375,7 +375,7 @@ TEST(S3RateLimiterManagerTest, legacy_get_config_throttles_when_per_core_is_unse auto& manager = S3RateLimiterManager::instance(); auto* get_qps = manager.qps_limiter(S3RateLimitType::GET); - config::s3_get_qps_per_core = -1; + config::s3_get_requests_per_second_per_core = -1; config::s3_get_token_per_second = 100; config::s3_get_bucket_tokens = 1; config::s3_get_token_limit = 0; @@ -394,7 +394,7 @@ TEST(S3RateLimiterManagerTest, refresh_applies_qps_speed_burst_and_limit_changes auto& manager = S3RateLimiterManager::instance(); auto* get_qps = manager.qps_limiter(S3RateLimitType::GET); - config::s3_get_qps_per_core = -1; + config::s3_get_requests_per_second_per_core = -1; config::s3_get_token_per_second = 100; config::s3_get_bucket_tokens = 200; config::s3_get_token_limit = 300; @@ -404,8 +404,8 @@ TEST(S3RateLimiterManagerTest, refresh_applies_qps_speed_burst_and_limit_changes EXPECT_EQ(300, get_qps->get_limit()); config::s3_rate_limiter_cpu_cores_override = kCores; - config::s3_get_qps_per_core = 100; - config::s3_get_qps_max = 0; + config::s3_get_requests_per_second_per_core = 100; + config::s3_get_requests_per_second_max = 0; manager.refresh(); EXPECT_EQ(100 * kCores, get_qps->get_max_speed()); EXPECT_EQ(100 * kCores, get_qps->get_max_burst()); @@ -550,14 +550,14 @@ TEST(S3RateLimiterMetricsTest, bytes_wait_and_rejection_have_distinct_counters) TEST(S3RateLimiterMetricsTest, get_cpu_aware_bytes_and_legacy_qps_update_separate_metrics) { RateLimiterConfigGuard guard; verify_cpu_aware_bytes_and_legacy_qps_metrics(S3RateLimitType::GET, S3RateLimitType::PUT, - config::s3_get_qps_per_core, + config::s3_get_requests_per_second_per_core, config::s3_get_bytes_per_second_per_core); } TEST(S3RateLimiterMetricsTest, put_cpu_aware_bytes_and_legacy_qps_update_separate_metrics) { RateLimiterConfigGuard guard; verify_cpu_aware_bytes_and_legacy_qps_metrics(S3RateLimitType::PUT, S3RateLimitType::GET, - config::s3_put_qps_per_core, + config::s3_put_requests_per_second_per_core, config::s3_put_bytes_per_second_per_core); } diff --git a/be/test/util/s3_util_test.cpp b/be/test/util/s3_util_test.cpp index eeeb1cec9e5340..fb5cc3c92d20a2 100644 --- a/be/test/util/s3_util_test.cpp +++ b/be/test/util/s3_util_test.cpp @@ -90,10 +90,10 @@ TEST_F(S3UTILTest, refresh_rebuilds_limiter_on_config_change) { const int64_t orig_tps = config::s3_get_token_per_second; const int64_t orig_bucket = config::s3_get_bucket_tokens; const int64_t orig_limit = config::s3_get_token_limit; - const int64_t orig_per_core = config::s3_get_qps_per_core; + const int64_t orig_per_core = config::s3_get_requests_per_second_per_core; // Establish a known baseline (legacy path, no count limit, no throttling). - config::s3_get_qps_per_core = -1; + config::s3_get_requests_per_second_per_core = -1; config::s3_get_token_per_second = 1000000000; config::s3_get_bucket_tokens = 1000000000; config::s3_get_token_limit = 0; @@ -121,7 +121,7 @@ TEST_F(S3UTILTest, refresh_rebuilds_limiter_on_config_change) { config::s3_get_token_per_second = orig_tps; config::s3_get_bucket_tokens = orig_bucket; config::s3_get_token_limit = orig_limit; - config::s3_get_qps_per_core = orig_per_core; + config::s3_get_requests_per_second_per_core = orig_per_core; manager.refresh(); } diff --git a/regression-test/suites/cloud_p0/s3/test_s3_rate_limiter.groovy b/regression-test/suites/cloud_p0/s3/test_s3_rate_limiter.groovy index ecd743e5a0323d..efbf3c3c4e11d8 100644 --- a/regression-test/suites/cloud_p0/s3/test_s3_rate_limiter.groovy +++ b/regression-test/suites/cloud_p0/s3/test_s3_rate_limiter.groovy @@ -90,8 +90,8 @@ suite("test_s3_rate_limiter", "p0,nonConcurrent") { def applyLimiterPhase = { long getQps, long putQps, long getBytes, long putBytes -> Map phase = [ - "s3_get_qps_per_core": getQps, - "s3_put_qps_per_core": putQps, + "s3_get_requests_per_second_per_core": getQps, + "s3_put_requests_per_second_per_core": putQps, "s3_get_bytes_per_second_per_core": getBytes, "s3_put_bytes_per_second_per_core": putBytes ] @@ -108,8 +108,8 @@ suite("test_s3_rate_limiter", "p0,nonConcurrent") { def applyLegacyQpsPhase = { long getQps, long putQps -> Map phase = [ - "s3_get_qps_per_core": getQps > 0 ? -1 : 0, - "s3_put_qps_per_core": putQps > 0 ? -1 : 0, + "s3_get_requests_per_second_per_core": getQps > 0 ? -1 : 0, + "s3_put_requests_per_second_per_core": putQps > 0 ? -1 : 0, "s3_get_token_per_second": getQps > 0 ? getQps : 1, "s3_put_token_per_second": putQps > 0 ? putQps : 1, "s3_get_bucket_tokens": 1, @@ -181,12 +181,12 @@ suite("test_s3_rate_limiter", "p0,nonConcurrent") { // would otherwise add GET traffic to an internal write. "enable_s3_object_check_after_upload": false, "s3_rate_limiter_cpu_cores_override": 1, - "s3_get_qps_per_core": 0, - "s3_put_qps_per_core": 0, - "s3_get_qps_max": 0, - "s3_put_qps_max": 0, + "s3_get_requests_per_second_per_core": 0, + "s3_put_requests_per_second_per_core": 0, + "s3_get_requests_per_second_max": 0, + "s3_put_requests_per_second_max": 0, // Keep the legacy QPS settings low. CPU-aware phases must ignore them - // unless qps_per_core is explicitly switched back to -1. + // unless requests_per_second_per_core is explicitly switched back to -1. "s3_get_token_per_second": 1, "s3_put_token_per_second": 1, "s3_get_bucket_tokens": 1, @@ -276,7 +276,7 @@ suite("test_s3_rate_limiter", "p0,nonConcurrent") { "s3_get_bytes_rate_limit_sleep_count" ]) - // Disable CPU-aware GET bytes and switch qps_per_core to -1. The low + // Disable CPU-aware GET bytes and switch requests_per_second_per_core to -1. The low // legacy QPS config that was ignored above must now throttle the same // internal-vault read. Only the original QPS metrics may grow. applyLegacyQpsPhase(1, 0) @@ -319,7 +319,7 @@ suite("test_s3_rate_limiter", "p0,nonConcurrent") { "s3_put_bytes_rate_limit_sleep_count" ]) - // Disable CPU-aware PUT bytes and switch qps_per_core to -1. Concurrent + // Disable CPU-aware PUT bytes and switch requests_per_second_per_core to -1. Concurrent // internal-vault writes must now update only the original PUT QPS metrics. applyLegacyQpsPhase(0, 1) before = snapshotMetrics() diff --git a/regression-test/suites/cloud_p0/s3/test_s3_rate_limiter_cgroup_cpu_resize.groovy b/regression-test/suites/cloud_p0/s3/test_s3_rate_limiter_cgroup_cpu_resize.groovy index 3c6e7a82906573..7afd282bc64857 100644 --- a/regression-test/suites/cloud_p0/s3/test_s3_rate_limiter_cgroup_cpu_resize.groovy +++ b/regression-test/suites/cloud_p0/s3/test_s3_rate_limiter_cgroup_cpu_resize.groovy @@ -32,10 +32,10 @@ suite("test_s3_rate_limiter_cgroup_cpu_resize", "docker") { options.beConfigs += [ "enable_s3_rate_limiter=true", "s3_rate_limiter_cpu_cores_override=0", - "s3_get_qps_per_core=${getQpsPerCore}", - "s3_put_qps_per_core=${putQpsPerCore}", - "s3_get_qps_max=0", - "s3_put_qps_max=0", + "s3_get_requests_per_second_per_core=${getQpsPerCore}", + "s3_put_requests_per_second_per_core=${putQpsPerCore}", + "s3_get_requests_per_second_max=0", + "s3_put_requests_per_second_max=0", "s3_get_bytes_per_second_per_core=${getBytesPerCore}", "s3_put_bytes_per_second_per_core=${putBytesPerCore}", "s3_get_bytes_per_second_max=0",