[feature](compaction) Add an http action for visibility of compaction score on each tablet (#38489)

be/src/http/action/compaction_score_action.cpp

@@ -0,0 +1,160 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+#include "http/action/compaction_score_action.h"
+
+#include <gen_cpp/FrontendService_types.h>
+#include <gen_cpp/Types_types.h>
+#include <glog/logging.h>
+#include <rapidjson/document.h>
+#include <rapidjson/prettywriter.h>
+#include <rapidjson/stringbuffer.h>
+
+#include <algorithm>
+#include <cstddef>
+#include <cstdint>
+#include <exception>
+#include <functional>
+#include <iterator>
+#include <limits>
+#include <memory>
+#include <span>
+#include <stdexcept>
+#include <string>
+#include <string_view>
+#include <vector>
+
+#include "common/status.h"
+#include "http/http_channel.h"
+#include "http/http_handler_with_auth.h"
+#include "http/http_headers.h"
+#include "http/http_request.h"
+#include "http/http_status.h"
+#include "olap/tablet_fwd.h"
+#include "olap/tablet_manager.h"
+#include "util/stopwatch.hpp"
+
+namespace doris {
+
+const std::string TOP_N = "top_n";
+const std::string COMPACTION_SCORE = "compaction_score";
+constexpr size_t DEFAULT_TOP_N = std::numeric_limits<size_t>::max();
+constexpr std::string_view TABLET_ID = "tablet_id";
+
+template <typename T>
+concept CompactionScoreAccessble = requires(T t) {
+ { t.get_real_compaction_score() } -> std::same_as<uint32_t>;
+};
+
+template <CompactionScoreAccessble T>
+std::vector<CompactionScoreResult> calculate_compaction_scores(
+ std::span<std::shared_ptr<T>> tablets) {
+ std::vector<CompactionScoreResult> result;
+ result.reserve(tablets.size());
+ std::ranges::transform(tablets, std::back_inserter(result),
+ [](const std::shared_ptr<T>& tablet) -> CompactionScoreResult {
+ return {.tablet_id = tablet->tablet_id(),
+ .compaction_score = tablet->get_real_compaction_score()};
+ });
+ return result;
+}
+
+struct LocalCompactionScoreAccessor final : CompactionScoresAccessor {
+ LocalCompactionScoreAccessor(TabletManager* tablet_mgr) : tablet_mgr(tablet_mgr) {}
+
+ std::vector<CompactionScoreResult> get_all_tablet_compaction_scores() override {
+ auto tablets = tablet_mgr->get_all_tablet();
+ std::span<TabletSharedPtr> s = {tablets.begin(), tablets.end()};
+ return calculate_compaction_scores(s);
+ }
+
+ TabletManager* tablet_mgr;
+};
+
+static rapidjson::Value jsonfy_tablet_compaction_score(
+ const CompactionScoreResult& result, rapidjson::MemoryPoolAllocator<>& allocator) {
+ rapidjson::Value node;
+ node.SetObject();
+
+ rapidjson::Value tablet_id_key;
+ tablet_id_key.SetString(TABLET_ID.data(), TABLET_ID.length(), allocator);
+ rapidjson::Value tablet_id_val;
+ auto tablet_id_str = std::to_string(result.tablet_id);
+ tablet_id_val.SetString(tablet_id_str.c_str(), tablet_id_str.length(), allocator);
+
+ rapidjson::Value score_key;
+ score_key.SetString(COMPACTION_SCORE.data(), COMPACTION_SCORE.size());
+ rapidjson::Value score_val;
+ auto score_str = std::to_string(result.compaction_score);
+ score_val.SetString(score_str.c_str(), score_str.length(), allocator);
+ node.AddMember(score_key, score_val, allocator);
+
+ node.AddMember(tablet_id_key, tablet_id_val, allocator);
+ return node;
+}
+
+CompactionScoreAction::CompactionScoreAction(ExecEnv* exec_env, TPrivilegeHier::type hier,
+ TPrivilegeType::type type, TabletManager* tablet_mgr)
+ : HttpHandlerWithAuth(exec_env, hier, type),
+ _accessor(std::make_unique<LocalCompactionScoreAccessor>(tablet_mgr)) {}
+
+void CompactionScoreAction::handle(HttpRequest* req) {
+ req->add_output_header(HttpHeaders::CONTENT_TYPE, HttpHeaders::JsonType.data());
+ auto top_n_param = req->param(TOP_N);
+
+ size_t top_n = DEFAULT_TOP_N;
+ if (!top_n_param.empty()) {
+ try {
+ auto tmp_top_n = std::stoll(top_n_param);
+ if (tmp_top_n < 0) {
+ throw std::invalid_argument("`top_n` cannot less than 0");
+ }
+ top_n = tmp_top_n;
+ } catch (const std::exception& e) {
+ LOG(WARNING) << "convert failed:" << e.what();
+ auto msg = fmt::format("invalid argument: top_n={}", top_n_param);
+ HttpChannel::send_reply(req, HttpStatus::BAD_REQUEST, msg);
+ return;
+ }
+ }
+
+ std::string result;
+ if (auto st = _handle(top_n, &result); !st) {
+ HttpChannel::send_reply(req, HttpStatus::INTERNAL_SERVER_ERROR, st.to_json());
+ return;
+ }
+ HttpChannel::send_reply(req, HttpStatus::OK, result);
+}
+
+Status CompactionScoreAction::_handle(size_t top_n, std::string* result) {
+ auto scores = _accessor->get_all_tablet_compaction_scores();
+ top_n = std::min(top_n, scores.size());
+ std::partial_sort(scores.begin(), scores.begin() + top_n, scores.end(), std::greater<>());
+
+ rapidjson::Document root;
+ root.SetArray();
+ auto& allocator = root.GetAllocator();
+ std::for_each(scores.begin(), scores.begin() + top_n, [&](const auto& score) {
+ root.PushBack(jsonfy_tablet_compaction_score(score, allocator), allocator);
+ });
+ rapidjson::StringBuffer str_buf;
+ rapidjson::PrettyWriter<rapidjson::StringBuffer> writer(str_buf);
+ root.Accept(writer);
+ *result = str_buf.GetString();
+ return Status::OK();
+}
+
+} // namespace doris

be/src/http/action/compaction_score_action.h

@@ -0,0 +1,62 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <gen_cpp/FrontendService_types.h>
+
+#include <cstddef>
+#include <memory>
+#include <string>
+
+#include "common/status.h"
+#include "http/http_handler_with_auth.h"
+#include "http/http_request.h"
+#include "olap/storage_engine.h"
+#include "runtime/exec_env.h"
+namespace doris {
+
+struct CompactionScoreResult {
+ int64_t tablet_id;
+ size_t compaction_score;
+};
+
+inline bool operator>(const CompactionScoreResult& lhs, const CompactionScoreResult& rhs) {
+ return lhs.compaction_score > rhs.compaction_score;
+}
+
+struct CompactionScoresAccessor {
+ virtual ~CompactionScoresAccessor() = default;
+
+ virtual std::vector<CompactionScoreResult> get_all_tablet_compaction_scores() = 0;
+};
+
+// topn, sync
+class CompactionScoreAction : public HttpHandlerWithAuth {
+public:
+ explicit CompactionScoreAction(ExecEnv* exec_env, TPrivilegeHier::type hier,
+ TPrivilegeType::type type, TabletManager* tablet_mgr);
+
+ void handle(HttpRequest* req) override;
+
+private:
+ Status _handle(size_t top_n, std::string* result);
+
+ std::unique_ptr<CompactionScoresAccessor> _accessor;
+};
+
+} // namespace doris

be/src/olap/base_tablet.cpp

@@ -83,4 +83,12 @@ Status BaseTablet::update_by_least_common_schema(const TabletSchemaSPtr& update_
  return Status::OK();
 }
 
+uint32_t BaseTablet::get_real_compaction_score() const {
+ const auto& rs_metas = _tablet_meta->all_rs_metas();
+ return std::accumulate(rs_metas.begin(), rs_metas.end(), 0,
+ [](uint32_t score, const RowsetMetaSharedPtr& rs_meta) {
+ return score + rs_meta->get_compaction_score();
+ });
+}
+
 } /* namespace doris */

be/src/olap/base_tablet.h

@@ -87,6 +87,10 @@ class BaseTablet {
 
  virtual size_t tablet_footprint() = 0;
 
+ // this method just return the compaction sum on each rowset
+ // note(tsy): we should unify the compaction score calculation finally
+ uint32_t get_real_compaction_score() const;
+
 protected:
  mutable std::shared_mutex _meta_lock;
  const TabletMetaSharedPtr _tablet_meta;

be/src/olap/tablet.cpp

@@ -1149,6 +1149,9 @@ uint32_t Tablet::calc_cold_data_compaction_score() const {
 
 uint32_t Tablet::_calc_cumulative_compaction_score(
  std::shared_ptr<CumulativeCompactionPolicy> cumulative_compaction_policy) {
+ if (cumulative_compaction_policy == nullptr) [[unlikely]] {
+ return 0;
+ }
 #ifndef BE_TEST
  if (_cumulative_compaction_policy == nullptr ||
  _cumulative_compaction_policy->name() != cumulative_compaction_policy->name()) {

be/src/service/http_service.cpp

@@ -19,6 +19,7 @@
 
 #include <event2/bufferevent.h>
 #include <event2/http.h>
+#include <gen_cpp/FrontendService_types.h>
 
 #include <string>
 #include <vector>
@@ -33,6 +34,7 @@
 #include "http/action/checksum_action.h"
 #include "http/action/clear_cache_action.h"
 #include "http/action/compaction_action.h"
+#include "http/action/compaction_score_action.h"
 #include "http/action/config_action.h"
 #include "http/action/debug_point_action.h"
 #include "http/action/download_action.h"
@@ -258,6 +260,12 @@ Status HttpService::start() {
  SnapshotAction* snapshot_action =
  _pool.add(new SnapshotAction(_env, TPrivilegeHier::GLOBAL, TPrivilegeType::ADMIN));
  _ev_http_server->register_handler(HttpMethod::GET, "/api/snapshot", snapshot_action);
+
+ CompactionScoreAction* compaction_score_action =
+ _pool.add(new CompactionScoreAction(_env, TPrivilegeHier::GLOBAL, TPrivilegeType::ADMIN,
+ _env->get_storage_engine()->tablet_manager()));
+ _ev_http_server->register_handler(HttpMethod::GET, "/api/compaction_score",
+ compaction_score_action);
 #endif
 
  // 2 compaction actions

regression-test/suites/compaction/test_compaction_score_action.groovy

@@ -0,0 +1,53 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_compaction_score_action") {
+ def tableName = "test_compaction_score_action";
+
+ sql """ DROP TABLE IF EXISTS ${tableName} """
+ sql """
+ CREATE TABLE IF NOT EXISTS ${tableName} (
+ id INT NOT NULL,
+ name STRING NOT NULL
+ ) DUPLICATE KEY (`id`)
+ PROPERTIES ("replication_num" = "1", "disable_auto_compaction" = "true");
+ """
+ for (i in 0..<30) {
+ sql """ INSERT INTO ${tableName} VALUES(1, "Vedal") """
+ sql """ INSERT INTO ${tableName} VALUES(2, "Neuro") """
+ sql """ INSERT INTO ${tableName} VALUES(3, "Evil") """
+ }
+
+ def backendId_to_backendIP = [:]
+ def backendId_to_backendHttpPort = [:]
+ getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort);
+
+ for (int i=0;i<backendId_to_backendIP.size();i++){
+ def beHttpAddress =backendId_to_backendIP.entrySet()[i].getValue()+":"+backendId_to_backendHttpPort.entrySet()[i].getValue()
+ if (isCloudMode()) {
+ def (code, text, err) = curl("GET",beHttpAddress+ "/api/compaction_score?top_n=1&sync_meta=true")
+ def score_str = parseJson(text).get(0).get("compaction_score")
+ def score = Integer.parseInt(score_str)
+ assertTrue(score >= 90)
+ } else {
+ def (code, text, err) = curl("GET",beHttpAddress+"/api/compaction_score?top_n=1")
+ def score_str = parseJson(text).get(0).get("compaction_score")
+ def score = Integer.parseInt(score_str)
+ assertTrue(score >= 90)
+ }
+ }
+}