[fix](segment cache) estimate momory consumed by segment (#35647)

be/src/olap/base_tablet.cpp

@@ -33,6 +33,8 @@ extern MetricPrototype METRIC_query_scan_count;
 DEFINE_COUNTER_METRIC_PROTOTYPE_2ARG(flush_bytes, MetricUnit::BYTES);
 DEFINE_COUNTER_METRIC_PROTOTYPE_2ARG(flush_finish_count, MetricUnit::OPERATIONS);
 
+static bvar::Adder<size_t> g_total_tablet_num("doris_total_tablet_num");
+
 BaseTablet::BaseTablet(TabletMetaSharedPtr tablet_meta) : _tablet_meta(std::move(tablet_meta)) {
  _metric_entity = DorisMetrics::instance()->metric_registry()->register_entity(
  fmt::format("Tablet.{}", tablet_id()), {{"tablet_id", std::to_string(tablet_id())}},
@@ -42,10 +44,12 @@ BaseTablet::BaseTablet(TabletMetaSharedPtr tablet_meta) : _tablet_meta(std::move
  INT_COUNTER_METRIC_REGISTER(_metric_entity, query_scan_count);
  INT_COUNTER_METRIC_REGISTER(_metric_entity, flush_bytes);
  INT_COUNTER_METRIC_REGISTER(_metric_entity, flush_finish_count);
+ g_total_tablet_num << 1;
 }
 
 BaseTablet::~BaseTablet() {
  DorisMetrics::instance()->metric_registry()->deregister_entity(_metric_entity);
+ g_total_tablet_num << -1;
 }
 
 Status BaseTablet::set_tablet_state(TabletState state) {

be/src/olap/primary_key_index.cpp

@@ -32,6 +32,8 @@
 
 namespace doris {
 
+static bvar::Adder<size_t> g_primary_key_index_memory_bytes("doris_primary_key_index_memory_bytes");
+
 Status PrimaryKeyIndexBuilder::init() {
  // TODO(liaoxin) using the column type directly if there's only one column in unique key columns
  const auto* type_info = get_scalar_type_info<FieldType::OLAP_FIELD_TYPE_VARCHAR>();

be/src/olap/rowset/rowset.cpp

@@ -26,6 +26,8 @@
 
 namespace doris {
 
+static bvar::Adder<size_t> g_total_rowset_num("doris_total_rowset_num");
+
 Rowset::Rowset(const TabletSchemaSPtr& schema, const RowsetMetaSharedPtr& rowset_meta)
  : _rowset_meta(rowset_meta), _refs_by_reader(0) {
  _is_pending = !_rowset_meta->has_version();
@@ -37,6 +39,11 @@ Rowset::Rowset(const TabletSchemaSPtr& schema, const RowsetMetaSharedPtr& rowset
  }
  // build schema from RowsetMeta.tablet_schema or Tablet.tablet_schema
  _schema = _rowset_meta->tablet_schema() ? _rowset_meta->tablet_schema() : schema;
+ g_total_rowset_num << 1;
+}
+
+Rowset::~Rowset() {
+ g_total_rowset_num << -1;
 }
 
 Status Rowset::load(bool use_cache) {

be/src/olap/rowset/rowset.h

@@ -118,7 +118,7 @@ class RowsetStateMachine {
 
 class Rowset : public std::enable_shared_from_this<Rowset> {
 public:
- virtual ~Rowset() = default;
+ virtual ~Rowset();
 
  // Open all segment files in this rowset and load necessary metadata.
  // - `use_cache` : whether to use fd cache, only applicable to alpha rowset now

be/src/olap/rowset/segment_v2/column_reader.cpp

@@ -76,6 +76,9 @@
 namespace doris {
 namespace segment_v2 {
 
+static bvar::Adder<size_t> g_column_reader_memory_bytes("doris_column_reader_memory_bytes");
+static bvar::Adder<size_t> g_column_reader_num("doris_column_reader_num");
+
 Status ColumnReader::create(const ColumnReaderOptions& opts, const ColumnMetaPB& meta,
  uint64_t num_rows, const io::FileReaderSPtr& file_reader,
  std::unique_ptr<ColumnReader>* reader) {
@@ -205,9 +208,15 @@ ColumnReader::ColumnReader(const ColumnReaderOptions& opts, const ColumnMetaPB&
  _meta_is_nullable = meta.is_nullable();
  _meta_dict_page = meta.dict_page();
  _meta_compression = meta.compression();
+
+ g_column_reader_memory_bytes << sizeof(*this);
+ g_column_reader_num << 1;
 }
 
-ColumnReader::~ColumnReader() = default;
+ColumnReader::~ColumnReader() {
+ g_column_reader_memory_bytes << -sizeof(*this);
+ g_column_reader_num << -1;
+}
 
 Status ColumnReader::init(const ColumnMetaPB* meta) {
  _type_info = get_type_info(meta);

be/src/olap/rowset/segment_v2/column_reader.h

@@ -116,7 +116,7 @@ class ColumnReader {
 
  enum DictEncodingType { UNKNOWN_DICT_ENCODING, PARTIAL_DICT_ENCODING, ALL_DICT_ENCODING };
 
- ~ColumnReader();
+virtual ~ColumnReader();
 
  // create a new column iterator. Client should delete returned iterator
  Status new_iterator(ColumnIterator** iterator);

be/src/olap/rowset/segment_v2/indexed_column_reader.cpp

@@ -56,6 +56,8 @@ static bvar::Adder<uint64_t> g_index_reader_pk_pages("doris_pk", "index_reader_p
 static bvar::PerSecond<bvar::Adder<uint64_t>> g_index_reader_pk_bytes_per_second(
  "doris_pk", "index_reader_pk_pages_per_second", &g_index_reader_pk_pages, 60);
 
+static bvar::Adder<uint64_t> g_index_reader_memory_bytes("doris_index_reader_memory_bytes");
+
 using strings::Substitute;
 
 Status IndexedColumnReader::load(bool use_page_cache, bool kept_in_memory) {
@@ -91,6 +93,8 @@ Status IndexedColumnReader::load(bool use_page_cache, bool kept_in_memory) {
  }
  }
  _num_values = _meta.num_values();
+
+ g_index_reader_memory_bytes << sizeof(*this);
  return Status::OK();
 }
 
@@ -134,6 +138,10 @@ Status IndexedColumnReader::read_page(const PagePointer& pp, PageHandle* handle,
  return st;
 }
 
+IndexedColumnReader::~IndexedColumnReader() {
+ g_index_reader_memory_bytes << -sizeof(*this);
+}
+
 ///////////////////////////////////////////////////////////////////////////////
 
 Status IndexedColumnIterator::_read_data_page(const PagePointer& pp) {

be/src/olap/rowset/segment_v2/indexed_column_reader.h

@@ -51,6 +51,8 @@ class IndexedColumnReader {
  explicit IndexedColumnReader(io::FileReaderSPtr file_reader, const IndexedColumnMetaPB& meta)
  : _file_reader(std::move(file_reader)), _meta(meta) {}
 
+ ~IndexedColumnReader();
+
  Status load(bool use_page_cache, bool kept_in_memory);
 
  // read a page specified by `pp' from `file' into `handle'

be/src/olap/rowset/segment_v2/ordinal_page_index.cpp

@@ -29,9 +29,13 @@
 #include "olap/olap_common.h"
 #include "olap/rowset/segment_v2/page_handle.h"
 #include "olap/rowset/segment_v2/page_io.h"
+#include "ordinal_page_index.h"
 #include "util/slice.h"
 
 namespace doris {
+
+static bvar::Adder<size_t> g_ordinal_index_memory_bytes("doris_ordinal_index_memory_bytes");
+
 namespace segment_v2 {
 
 void OrdinalIndexWriter::append_entry(ordinal_t ordinal, const PagePointer& data_pp) {
@@ -122,6 +126,10 @@ Status OrdinalIndexReader::_load(bool use_page_cache, bool kept_in_memory,
  _pages[i] = reader.get_value(i);
  }
  _ordinals[_num_pages] = _num_values;
+
+ g_ordinal_index_memory_bytes << sizeof(*this) + _ordinals.size() * sizeof(ordinal_t) +
+ _pages.size() * sizeof(PagePointer) +
+ sizeof(OrdinalIndexReader);
  return Status::OK();
 }
 
@@ -146,5 +154,11 @@ OrdinalPageIndexIterator OrdinalIndexReader::seek_at_or_before(ordinal_t ordinal
  return OrdinalPageIndexIterator(this, left);
 }
 
+OrdinalIndexReader::~OrdinalIndexReader() {
+ g_ordinal_index_memory_bytes << -sizeof(*this) - _ordinals.size() * sizeof(ordinal_t) -
+ _pages.size() * sizeof(PagePointer) -
+ sizeof(OrdinalIndexReader);
+}
+
 } // namespace segment_v2
 } // namespace doris

be/src/olap/rowset/segment_v2/ordinal_page_index.h

@@ -72,6 +72,8 @@ class OrdinalIndexReader {
  _meta_pb.reset(new OrdinalIndexPB(meta_pb));
  }
 
+ virtual ~OrdinalIndexReader();
+
  // load and parse the index page into memory
  Status load(bool use_page_cache, bool kept_in_memory);
 

be/src/olap/rowset/segment_v2/segment.cpp

@@ -113,6 +113,17 @@ Status Segment::_open() {
  // DCHECK(footer.has_short_key_index_page());
  _sk_index_page = _footer_pb->short_key_index_page();
  _num_rows = _footer_pb->num_rows();
+
+ // An estimated memory usage of a segment
+ _meta_mem_usage += _footer_pb->ByteSizeLong();
+ _meta_mem_usage += sizeof(*this);
+ _meta_mem_usage += _tablet_schema->num_columns() * config::estimated_mem_per_column_reader;
+
+ // 1024 comes from SegmentWriterOptions
+ _meta_mem_usage += (_num_rows + 1023) / 1024 * (36 + 4);
+ // 0.01 comes from PrimaryKeyIndexBuilder::init
+ _meta_mem_usage += BloomFilter::optimal_bit_num(_num_rows, 0.01) / 8;
+
  return Status::OK();
 }
 
@@ -301,7 +312,7 @@ Status Segment::_load_pk_bloom_filter() {
  auto status = [this]() {
  return _load_pk_bf_once.call([this] {
  RETURN_IF_ERROR(_pk_index_reader->parse_bf(_file_reader, *_pk_index_meta));
- _meta_mem_usage += _pk_index_reader->get_bf_memory_size();
+// _meta_mem_usage += _pk_index_reader->get_bf_memory_size();
  return Status::OK();
  });
  }();
@@ -338,7 +349,7 @@ Status Segment::_load_index_impl() {
  if (_tablet_schema->keys_type() == UNIQUE_KEYS && _pk_index_meta != nullptr) {
  _pk_index_reader.reset(new PrimaryKeyIndexReader());
  RETURN_IF_ERROR(_pk_index_reader->parse_index(_file_reader, *_pk_index_meta));
- _meta_mem_usage += _pk_index_reader->get_memory_size();
+// _meta_mem_usage += _pk_index_reader->get_memory_size();
  return Status::OK();
  } else {
  // read and parse short key index page
@@ -360,7 +371,7 @@ Status Segment::_load_index_impl() {
  DCHECK_EQ(footer.type(), SHORT_KEY_PAGE);
  DCHECK(footer.has_short_key_page_footer());
 
- _meta_mem_usage += body.get_size();
+// _meta_mem_usage += body.get_size();
  _sk_index_decoder.reset(new ShortKeyIndexDecoder);
  return _sk_index_decoder->parse(body, footer.short_key_page_footer());
  }
@@ -430,7 +441,6 @@ Status Segment::_create_column_readers(const SegmentFooterPB& footer) {
  RETURN_IF_ERROR(ColumnReader::create(opts, footer.columns(iter->second), footer.num_rows(),
  _file_reader, &reader));
  _column_readers.emplace(column.unique_id(), std::move(reader));
- _meta_mem_usage += config::estimated_mem_per_column_reader;
  }
 
  // init by column path

be/src/olap/rowset/segment_v2/segment.h

@@ -256,6 +256,7 @@ class Segment : public std::enable_shared_from_this<Segment> {
  // used to hold short key index page in memory
  PageHandle _sk_index_handle;
  // short key index decoder
+ // all content is in memory
  std::unique_ptr<ShortKeyIndexDecoder> _sk_index_decoder;
  // primary key index reader
  std::unique_ptr<PrimaryKeyIndexReader> _pk_index_reader;

be/src/olap/rowset/segment_v2/zone_map_index.cpp

@@ -39,6 +39,8 @@
 namespace doris {
 struct uint24_t;
 
+static bvar::Adder<size_t> g_zone_map_memory_bytes("doris_zone_map_memory_bytes");
+
 namespace segment_v2 {
 
 template <PrimitiveType Type>
@@ -173,9 +175,18 @@ Status ZoneMapIndexReader::_load(bool use_page_cache, bool kept_in_memory,
  return Status::Corruption("Failed to parse zone map");
  }
  }
+
+ g_zone_map_memory_bytes << sizeof(*this) + sizeof(ZoneMapPB) * _page_zone_maps.size() +
+ sizeof(IndexedColumnMetaPB);
+
  return Status::OK();
 }
 
+ZoneMapIndexReader::~ZoneMapIndexReader() {
+ // Maybe wrong due to load failures.
+ g_zone_map_memory_bytes << -sizeof(*this) - sizeof(ZoneMapPB) * _page_zone_maps.size() -
+ sizeof(IndexedColumnMetaPB);
+}
 #define APPLY_FOR_PRIMITITYPE(M) \
  M(TYPE_TINYINT) \
  M(TYPE_SMALLINT) \

be/src/olap/rowset/segment_v2/zone_map_index.h

@@ -151,6 +151,8 @@ class ZoneMapIndexReader {
  _page_zone_maps_meta.reset(new IndexedColumnMetaPB(page_zone_maps));
  }
 
+ virtual ~ZoneMapIndexReader();
+
  // load all page zone maps into memory
  Status load(bool use_page_cache, bool kept_in_memory);
 

be/src/olap/short_key_index.cpp

@@ -22,12 +22,16 @@
 #include <ostream>
 
 #include "gutil/strings/substitute.h"
+#include "short_key_index.h"
+#include "util/bvar_helper.h"
 #include "util/coding.h"
 
 using strings::Substitute;
 
 namespace doris {
 
+static bvar::Adder<size_t> g_short_key_index_memory_bytes("doris_short_key_index_memory_bytes");
+
 Status ShortKeyIndexBuilder::add_item(const Slice& key) {
  put_varint32(&_offset_buf, _key_buf.size());
  _key_buf.append(key.data, key.size);
@@ -85,7 +89,19 @@ Status ShortKeyIndexDecoder::parse(const Slice& body, const segment_v2::ShortKey
  return Status::Corruption("Still has data after parse all key offset");
  }
  _parsed = true;
+
+ g_short_key_index_memory_bytes << sizeof(_footer) + _key_data.size +
+ _offsets.size() * sizeof(uint32_t) + sizeof(*this);
+
  return Status::OK();
 }
 
+ShortKeyIndexDecoder::~ShortKeyIndexDecoder() {
+ if (_parsed) {
+ g_short_key_index_memory_bytes << -sizeof(_footer) - _key_data.size -
+ _offsets.size() * sizeof(uint32_t) -
+ sizeof(*this);
+ }
+}
+
 } // namespace doris

be/src/olap/short_key_index.h

@@ -135,6 +135,7 @@ class ShortKeyIndexIterator {
 class ShortKeyIndexDecoder {
 public:
  ShortKeyIndexDecoder() : _parsed(false) {}
+ virtual ~ShortKeyIndexDecoder();
 
  // client should assure that body is available when this class is used
  Status parse(const Slice& body, const segment_v2::ShortKeyFooterPB& footer);

be/src/olap/tablet_schema.cpp

@@ -53,6 +53,8 @@
 
 namespace doris {
 
+static bvar::Adder<size_t> g_total_tablet_schema_num("doris_total_tablet_schema_num");
+
 FieldType TabletColumn::get_field_type_by_type(PrimitiveType primitiveType) {
  switch (primitiveType) {
  case PrimitiveType::INVALID_TYPE:
@@ -809,6 +811,14 @@ void TabletIndex::to_schema_pb(TabletIndexPB* index) const {
  }
 }
 
+TabletSchema::TabletSchema() {
+ g_total_tablet_schema_num << 1;
+}
+
+TabletSchema::~TabletSchema() {
+ g_total_tablet_schema_num << -1;
+}
+
 void TabletSchema::append_column(TabletColumn column, ColumnType col_type) {
  if (column.is_key()) {
  _num_key_columns++;

be/src/olap/tablet_schema.h

@@ -269,7 +269,9 @@ class TabletSchema {
  // TODO(yingchun): better to make constructor as private to avoid
  // manually init members incorrectly, and define a new function like
  // void create_from_pb(const TabletSchemaPB& schema, TabletSchema* tablet_schema).
- TabletSchema() = default;
+ TabletSchema();
+ virtual ~TabletSchema();
+
  void init_from_pb(const TabletSchemaPB& schema, bool ignore_extracted_columns = false);
  // Notice: Use deterministic way to serialize protobuf,
  // since serialize Map in protobuf may could lead to un-deterministic by default