[improvement](filecache) Adapt file cache queue consumption

be/src/common/config.cpp

@@ -1246,6 +1246,7 @@ DEFINE_mInt64(file_cache_background_lru_dump_interval_ms, "60000");
 DEFINE_mInt64(file_cache_background_lru_dump_update_cnt_threshold, "1000");
 DEFINE_mInt64(file_cache_background_lru_dump_tail_record_num, "5000000");
 DEFINE_mInt64(file_cache_background_lru_log_replay_interval_ms, "1000");
+DEFINE_mInt64(file_cache_lru_recorder_log_queue_hard_cap, "500000");
 DEFINE_mBool(enable_evaluate_shadow_queue_diff, "false");
 
 DEFINE_mBool(file_cache_enable_only_warm_up_idx, "false");

be/src/common/config.h

@@ -1290,6 +1290,7 @@ DECLARE_mInt64(file_cache_background_lru_dump_interval_ms);
 DECLARE_mInt64(file_cache_background_lru_dump_update_cnt_threshold);
 DECLARE_mInt64(file_cache_background_lru_dump_tail_record_num);
 DECLARE_mInt64(file_cache_background_lru_log_replay_interval_ms);
+DECLARE_mInt64(file_cache_lru_recorder_log_queue_hard_cap);
 DECLARE_mBool(enable_evaluate_shadow_queue_diff);
 
 DECLARE_mBool(file_cache_enable_only_warm_up_idx);

be/src/io/cache/block_file_cache.cpp

@@ -61,25 +61,47 @@ namespace doris::io {
 
 // Insert a block pointer into one shard while swallowing allocation failures.
 bool NeedUpdateLRUBlocks::insert(FileBlockSPtr block) {
+ return insert_with_result(std::move(block)) == InsertResult::INSERTED;
+}
+
+NeedUpdateLRUBlocks::InsertResult NeedUpdateLRUBlocks::insert_with_result(FileBlockSPtr block) {
  if (!block) {
- return false;
+ return InsertResult::IGNORED;
  }
+ bool reserved = false;
  try {
  auto* raw_ptr = block.get();
  auto idx = shard_index(raw_ptr);
  auto& shard = _shards[idx];
  std::lock_guard lock(shard.mutex);
+ if (shard.entries.contains(raw_ptr)) {
+ return InsertResult::DUPLICATED;
+ }
+ if (!try_reserve_slot()) {
+ _dropped.fetch_add(1, std::memory_order_relaxed);
+ return InsertResult::DROPPED;
+ }
+ reserved = true;
  auto [_, inserted] = shard.entries.emplace(raw_ptr, std::move(block));
- if (inserted) {
- _size.fetch_add(1, std::memory_order_relaxed);
+ if (!inserted) {
+ _size.fetch_sub(1, std::memory_order_relaxed);
+ reserved = false;
+ return InsertResult::DUPLICATED;
  }
- return inserted;
+ reserved = false;
+ return InsertResult::INSERTED;
  } catch (const std::exception& e) {
+ if (reserved) {
+ _size.fetch_sub(1, std::memory_order_relaxed);
+ }
  LOG(WARNING) << "Failed to enqueue block for LRU update: " << e.what();
  } catch (...) {
+ if (reserved) {
+ _size.fetch_sub(1, std::memory_order_relaxed);
+ }
  LOG(WARNING) << "Failed to enqueue block for LRU update: unknown error";
  }
- return false;
+ return InsertResult::IGNORED;
 }
 
 // Drain up to `limit` unique blocks to the caller, keeping the structure consistent on failures.
@@ -138,11 +160,44 @@ size_t NeedUpdateLRUBlocks::shard_index(FileBlock* ptr) const {
  return std::hash<FileBlock*> {}(ptr)&kShardMask;
 }
 
+bool NeedUpdateLRUBlocks::try_reserve_slot() {
+ size_t cur_size = _size.load(std::memory_order_relaxed);
+ while (cur_size < _hard_cap) {
+ if (_size.compare_exchange_weak(cur_size, cur_size + 1, std::memory_order_relaxed,
+ std::memory_order_relaxed)) {
+ return true;
+ }
+ }
+ return false;
+}
+
+namespace {
+
+constexpr size_t kLruLogReplayBatchPerType = 25'000;
+constexpr size_t kBlockLruUpdateBatch = 10'000;
+constexpr size_t kBlockLruUpdateLockSliceBatch = 500;
+constexpr size_t kNeedUpdateLruBlocksHardCap = 100'000;
+constexpr size_t kDefaultLruRecorderLogQueueHardCap = 500'000;
+
+int64_t positive_or_default(int64_t value, int64_t default_value) {
+ return value > 0 ? value : default_value;
+}
+
+size_t lru_recorder_log_queue_hard_cap() {
+ if (config::file_cache_lru_recorder_log_queue_hard_cap <= 0) {
+ return kDefaultLruRecorderLogQueueHardCap;
+ }
+ return static_cast<size_t>(config::file_cache_lru_recorder_log_queue_hard_cap);
+}
+
+} // namespace
+
 BlockFileCache::BlockFileCache(const std::string& cache_base_path,
  const FileCacheSettings& cache_settings)
  : _cache_base_path(cache_base_path),
  _capacity(cache_settings.capacity),
- _max_file_block_size(cache_settings.max_file_block_size) {
+ _max_file_block_size(cache_settings.max_file_block_size),
+ _need_update_lru_blocks(kNeedUpdateLruBlocksHardCap) {
  _cur_cache_size_metrics = std::make_shared<bvar::Status<size_t>>(_cache_base_path.c_str(),
  "file_cache_cache_size", 0);
  _cache_capacity_metrics = std::make_shared<bvar::Status<size_t>>(
@@ -344,10 +399,16 @@ BlockFileCache::BlockFileCache(const std::string& cache_base_path,
  _cache_base_path.c_str(), "file_cache_evict_in_advance_latency_us");
  _lru_dump_latency_us = std::make_shared<bvar::LatencyRecorder>(
  _cache_base_path.c_str(), "file_cache_lru_dump_latency_us");
- _recycle_keys_length_recorder = std::make_shared<bvar::LatencyRecorder>(
- _cache_base_path.c_str(), "file_cache_recycle_keys_length");
- _need_update_lru_blocks_length_recorder = std::make_shared<bvar::LatencyRecorder>(
- _cache_base_path.c_str(), "file_cache_need_update_lru_blocks_length");
+ _recycle_keys_length_metrics = std::make_shared<bvar::Status<size_t>>(
+ _cache_base_path.c_str(), "file_cache_recycle_keys_length", 0);
+ _need_update_lru_blocks_length_metrics = std::make_shared<bvar::Status<size_t>>(
+ _cache_base_path.c_str(), "file_cache_need_update_lru_blocks_length", 0);
+ _need_update_lru_blocks_dropped_metrics = std::make_shared<bvar::Adder<size_t>>(
+ _cache_base_path.c_str(), "file_cache_need_update_lru_blocks_dropped");
+ _lru_recorder_log_queue_length_metrics = std::make_shared<bvar::Status<size_t>>(
+ _cache_base_path.c_str(), "file_cache_lru_recorder_log_queue_length", 0);
+ _lru_recorder_log_queue_dropped_metrics = std::make_shared<bvar::Adder<size_t>>(
+ _cache_base_path.c_str(), "file_cache_lru_recorder_log_queue_dropped");
  _update_lru_blocks_latency_us = std::make_shared<bvar::LatencyRecorder>(
  _cache_base_path.c_str(), "file_cache_update_lru_blocks_latency_us");
  _ttl_gc_latency_us = std::make_shared<bvar::LatencyRecorder>(_cache_base_path.c_str(),
@@ -364,7 +425,7 @@ BlockFileCache::BlockFileCache(const std::string& cache_base_path,
  _ttl_queue = LRUQueue(cache_settings.ttl_queue_size, cache_settings.ttl_queue_elements,
  std::numeric_limits<int>::max());
 
- _lru_recorder = std::make_unique<LRUQueueRecorder>(this);
+ _lru_recorder = std::make_unique<LRUQueueRecorder>(this, lru_recorder_log_queue_hard_cap());
  _lru_dumper = std::make_unique<CacheLRUDumper>(this, _lru_recorder.get());
  if (cache_settings.storage == "memory") {
  _storage = std::make_unique<MemFileCacheStorage>();
@@ -382,6 +443,11 @@ UInt128Wrapper BlockFileCache::hash(const std::string& path) {
  return UInt128Wrapper(value);
 }
 
+bool BlockFileCache::is_memory_storage() const {
+ DCHECK(_storage != nullptr);
+ return _storage->get_type() == FileCacheStorageType::MEMORY;
+}
+
 BlockFileCache::QueryFileCacheContextHolderPtr BlockFileCache::get_query_context_holder(
  const TUniqueId& query_id, int file_cache_query_limit_percent) {
  SCOPED_CACHE_LOCK(_mutex, this);
@@ -647,8 +713,14 @@ FileBlocks BlockFileCache::get_impl(const UInt128Wrapper& hash, const CacheConte
 }
 
 void BlockFileCache::add_need_update_lru_block(FileBlockSPtr block) {
- if (_need_update_lru_blocks.insert(std::move(block))) {
- *_need_update_lru_blocks_length_recorder << _need_update_lru_blocks.size();
+ auto result = _need_update_lru_blocks.insert_with_result(std::move(block));
+ if (result == NeedUpdateLRUBlocks::InsertResult::INSERTED) {
+ _need_update_lru_blocks_length_metrics->set_value(_need_update_lru_blocks.size());
+ } else if (result == NeedUpdateLRUBlocks::InsertResult::DROPPED) {
+ *(_need_update_lru_blocks_dropped_metrics) << 1;
+ LOG_EVERY_N(WARNING, 60) << "Drop block LRU update because hard cap is reached, hard_cap="
+ << _need_update_lru_blocks.hard_cap()
+ << " queue_size=" << _need_update_lru_blocks.size();
  }
 }
 
@@ -1453,7 +1525,7 @@ void BlockFileCache::remove(FileBlockSPtr file_block, T& cache_lock, U& block_lo
  // but it's ok, because the rowset is stale already
  bool ret = _recycle_keys.enqueue(key);
  if (ret) [[likely]] {
-*_recycle_keys_length_recorder << _recycle_keys.size_approx();
+_recycle_keys_length_metrics->set_value(_recycle_keys.size_approx());
  } else {
  LOG_WARNING("Failed to push recycle key to queue, do it synchronously");
  int64_t duration_ns = 0;
@@ -2000,16 +2072,16 @@ void BlockFileCache::run_background_monitor() {
  (double)_num_read_blocks_1h->get_value());
  }
 
- if (_no_warmup_num_hit_blocks->get_value() > 0) {
+ if (_no_warmup_num_read_blocks->get_value() > 0) {
  _no_warmup_hit_ratio->set_value((double)_no_warmup_num_hit_blocks->get_value() /
  (double)_no_warmup_num_read_blocks->get_value());
  }
- if (_no_warmup_num_hit_blocks_5m && _no_warmup_num_hit_blocks_5m->get_value() > 0) {
+ if (_no_warmup_num_read_blocks_5m && _no_warmup_num_read_blocks_5m->get_value() > 0) {
  _no_warmup_hit_ratio_5m->set_value(
  (double)_no_warmup_num_hit_blocks_5m->get_value() /
  (double)_no_warmup_num_read_blocks_5m->get_value());
  }
- if (_no_warmup_num_hit_blocks_1h && _no_warmup_num_hit_blocks_1h->get_value() > 0) {
+ if (_no_warmup_num_read_blocks_1h && _no_warmup_num_read_blocks_1h->get_value() > 0) {
  _no_warmup_hit_ratio_1h->set_value(
  (double)_no_warmup_num_hit_blocks_1h->get_value() /
  (double)_no_warmup_num_read_blocks_1h->get_value());
@@ -2047,7 +2119,7 @@ void BlockFileCache::run_background_gc() {
  }
  batch_count++;
  }
-*_recycle_keys_length_recorder << _recycle_keys.size_approx();
+_recycle_keys_length_metrics->set_value(_recycle_keys.size_approx());
  batch_count = 0;
  }
 }
@@ -2090,35 +2162,40 @@ void BlockFileCache::run_background_block_lru_update() {
  Thread::set_self_name("run_background_block_lru_update");
  std::vector<FileBlockSPtr> batch;
  while (!_close) {
- int64_t interval_ms = config::file_cache_background_block_lru_update_interval_ms;
- size_t batch_limit =
- config::file_cache_background_block_lru_update_qps_limit * interval_ms / 1000;
- {
+ size_t backlog = _need_update_lru_blocks.size();
+ if (backlog == 0) {
  std::unique_lock close_lock(_close_mtx);
- _close_cv.wait_for(close_lock, std::chrono::milliseconds(interval_ms));
+ _close_cv.wait_for(
+ close_lock,
+ std::chrono::milliseconds(positive_or_default(
+ config::file_cache_background_block_lru_update_interval_ms, 1)));
  if (_close) {
  break;
  }
  }
 
  batch.clear();
- batch.reserve(batch_limit);
- size_t drained = _need_update_lru_blocks.drain(batch_limit, &batch);
+ batch.reserve(kBlockLruUpdateBatch);
+ size_t drained = _need_update_lru_blocks.drain(kBlockLruUpdateBatch, &batch);
  if (drained == 0) {
-*_need_update_lru_blocks_length_recorder << _need_update_lru_blocks.size();
+_need_update_lru_blocks_length_metrics->set_value(_need_update_lru_blocks.size());
  continue;
  }
 
  int64_t duration_ns = 0;
- {
- SCOPED_CACHE_LOCK(_mutex, this);
- SCOPED_RAW_TIMER(&duration_ns);
- for (auto& block : batch) {
- update_block_lru(block, cache_lock);
+ const size_t slice_batch = std::min(kBlockLruUpdateLockSliceBatch, drained);
+ for (size_t begin = 0; begin < batch.size(); begin += slice_batch) {
+ const size_t end = std::min(begin + slice_batch, batch.size());
+ {
+ SCOPED_CACHE_LOCK(_mutex, this);
+ SCOPED_RAW_TIMER(&duration_ns);
+ for (size_t i = begin; i < end; ++i) {
+ update_block_lru(batch[i], cache_lock);
+ }
  }
  }
  *_update_lru_blocks_latency_us << (duration_ns / 1000);
-*_need_update_lru_blocks_length_recorder << _need_update_lru_blocks.size();
+_need_update_lru_blocks_length_metrics->set_value(_need_update_lru_blocks.size());
  }
 }
 
@@ -2195,7 +2272,7 @@ bool BlockFileCache::try_reserve_during_async_load(size_t size,
 
 void BlockFileCache::clear_need_update_lru_blocks() {
  _need_update_lru_blocks.clear();
-*_need_update_lru_blocks_length_recorder << _need_update_lru_blocks.size();
+_need_update_lru_blocks_length_metrics->set_value(_need_update_lru_blocks.size());
 }
 
 void BlockFileCache::pause_ttl_manager() {
@@ -2314,19 +2391,24 @@ void BlockFileCache::update_ttl_atime(const UInt128Wrapper& hash) {
 void BlockFileCache::run_background_lru_log_replay() {
  Thread::set_self_name("run_background_lru_log_replay");
  while (!_close) {
-int64_t interval_ms = config::file_cache_background_lru_log_replay_interval_ms;
- {
+size_t backlog = _lru_recorder->get_total_lru_log_queue_size();
+if (backlog == 0) {
  std::unique_lock close_lock(_close_mtx);
- _close_cv.wait_for(close_lock, std::chrono::milliseconds(interval_ms));
+ _close_cv.wait_for(
+ close_lock,
+ std::chrono::milliseconds(positive_or_default(
+ config::file_cache_background_lru_log_replay_interval_ms, 1)));
  if (_close) {
  break;
  }
  }
 
- _lru_recorder->replay_queue_event(FileCacheType::TTL);
- _lru_recorder->replay_queue_event(FileCacheType::INDEX);
- _lru_recorder->replay_queue_event(FileCacheType::NORMAL);
- _lru_recorder->replay_queue_event(FileCacheType::DISPOSABLE);
+ record_lru_recorder_log_queue_length();
+ _lru_recorder->replay_queue_event(FileCacheType::TTL, kLruLogReplayBatchPerType);
+ _lru_recorder->replay_queue_event(FileCacheType::INDEX, kLruLogReplayBatchPerType);
+ _lru_recorder->replay_queue_event(FileCacheType::NORMAL, kLruLogReplayBatchPerType);
+ _lru_recorder->replay_queue_event(FileCacheType::DISPOSABLE, kLruLogReplayBatchPerType);
+ record_lru_recorder_log_queue_length();
 
  if (config::enable_evaluate_shadow_queue_diff) {
  SCOPED_CACHE_LOCK(_mutex, this);
@@ -2338,6 +2420,11 @@ void BlockFileCache::run_background_lru_log_replay() {
  }
 }
 
+void BlockFileCache::record_lru_recorder_log_queue_length() {
+ _lru_recorder_log_queue_length_metrics->set_value(
+ _lru_recorder->get_total_lru_log_queue_size());
+}
+
 void BlockFileCache::dump_lru_queues(bool force) {
  std::unique_lock dump_lock(_dump_lru_queues_mtx);
  if (config::file_cache_background_lru_dump_tail_record_num > 0 &&