Skip to content

[fix](partial update) duplicate key occur when BE restart after conflict concurrent partial update #35739

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
dataroaring merged 1 commit into from
Jun 1, 2024
Merged

[fix](partial update) duplicate key occur when BE restart after conflict concurrent partial update #35739

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion be/src/olap/base_tablet.cpp
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -1114,7 +1114,7 @@ void BaseTablet::_remove_sentinel_mark_from_delete_bitmap(DeleteBitmapPtr delete
}
}

Status BaseTablet::update_delete_bitmap(const BaseTabletSPtr& self, const TabletTxnInfo* txn_info,
Status BaseTablet::update_delete_bitmap(const BaseTabletSPtr& self, TabletTxnInfo* txn_info,

@github-actionsgithub-actionsBotMay 31, 2024

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

warning: function 'update_delete_bitmap' has cognitive complexity of 58 (threshold 50) [readability-function-cognitive-complexity]

Status BaseTablet::update_delete_bitmap(const BaseTabletSPtr& self, TabletTxnInfo* txn_info,
^
Additional context

be/src/olap/base_tablet.cpp:1127: +1, including nesting penalty of 0, nesting level increased to 1

if (txn_info->partial_update_info && txn_info->partial_update_info->is_partial_update) {
^

be/src/olap/base_tablet.cpp:1127: +1

if (txn_info->partial_update_info && txn_info->partial_update_info->is_partial_update) {
^

be/src/olap/base_tablet.cpp:1128: nesting level increased to 2

 transient_rs_writer = DORIS_TRY(self->create_transient_rowset_writer(
^

be/src/common/status.h:694: expanded from macro 'DORIS_TRY'

 ({ \
^

be/src/olap/base_tablet.cpp:1128: +3, including nesting penalty of 2, nesting level increased to 3

 transient_rs_writer = DORIS_TRY(self->create_transient_rowset_writer(
^

be/src/common/status.h:697: expanded from macro 'DORIS_TRY'

if (!res.has_value()) [[unlikely]] { \
^

be/src/olap/base_tablet.cpp:1140: +1, including nesting penalty of 0, nesting level increased to 1

RETURN_IF_ERROR(std::dynamic_pointer_cast<BetaRowset>(rowset)->load_segments(&segments));
^

be/src/common/status.h:612: expanded from macro 'RETURN_IF_ERROR'

do { \
^

be/src/olap/base_tablet.cpp:1140: +2, including nesting penalty of 1, nesting level increased to 2

RETURN_IF_ERROR(std::dynamic_pointer_cast<BetaRowset>(rowset)->load_segments(&segments));
^

be/src/common/status.h:614: expanded from macro 'RETURN_IF_ERROR'

if (UNLIKELY(!_status_.ok())) { \
^

be/src/olap/base_tablet.cpp:1146: +1, including nesting penalty of 0, nesting level increased to 1

if (self->tablet_state() == TABLET_NOTREADY) {
^

be/src/olap/base_tablet.cpp:1151: +1, including nesting penalty of 0, nesting level increased to 1

RETURN_IF_ERROR(self->get_all_rs_id_unlocked(cur_version - 1, &cur_rowset_ids));
^

be/src/common/status.h:612: expanded from macro 'RETURN_IF_ERROR'

do { \
^

be/src/olap/base_tablet.cpp:1151: +2, including nesting penalty of 1, nesting level increased to 2

RETURN_IF_ERROR(self->get_all_rs_id_unlocked(cur_version - 1, &cur_rowset_ids));
^

be/src/common/status.h:614: expanded from macro 'RETURN_IF_ERROR'

if (UNLIKELY(!_status_.ok())) { \
^

be/src/olap/base_tablet.cpp:1170: +1, including nesting penalty of 0, nesting level increased to 1

if (segments.size() <= 1) {
^

be/src/olap/base_tablet.cpp:1171: +2, including nesting penalty of 1, nesting level increased to 2

RETURN_IF_ERROR(calc_delete_bitmap(self, rowset, segments, specified_rowsets, delete_bitmap,
^

be/src/common/status.h:612: expanded from macro 'RETURN_IF_ERROR'

do { \
^

be/src/olap/base_tablet.cpp:1171: +3, including nesting penalty of 2, nesting level increased to 3

RETURN_IF_ERROR(calc_delete_bitmap(self, rowset, segments, specified_rowsets, delete_bitmap,
^

be/src/common/status.h:614: expanded from macro 'RETURN_IF_ERROR'

if (UNLIKELY(!_status_.ok())) { \
^

be/src/olap/base_tablet.cpp:1174: +1, nesting level increased to 1

 } else {
^

be/src/olap/base_tablet.cpp:1176: +2, including nesting penalty of 1, nesting level increased to 2

RETURN_IF_ERROR(calc_delete_bitmap(self, rowset, segments, specified_rowsets, delete_bitmap,
^

be/src/common/status.h:612: expanded from macro 'RETURN_IF_ERROR'

do { \
^

be/src/olap/base_tablet.cpp:1176: +3, including nesting penalty of 2, nesting level increased to 3

RETURN_IF_ERROR(calc_delete_bitmap(self, rowset, segments, specified_rowsets, delete_bitmap,
^

be/src/common/status.h:614: expanded from macro 'RETURN_IF_ERROR'

if (UNLIKELY(!_status_.ok())) { \
^

be/src/olap/base_tablet.cpp:1179: +2, including nesting penalty of 1, nesting level increased to 2

RETURN_IF_ERROR(token->wait());
^

be/src/common/status.h:612: expanded from macro 'RETURN_IF_ERROR'

do { \
^

be/src/olap/base_tablet.cpp:1179: +3, including nesting penalty of 2, nesting level increased to 3

RETURN_IF_ERROR(token->wait());
^

be/src/common/status.h:614: expanded from macro 'RETURN_IF_ERROR'

if (UNLIKELY(!_status_.ok())) { \
^

be/src/olap/base_tablet.cpp:1183: +1, including nesting penalty of 0, nesting level increased to 1

if (watch.get_elapse_time_us() < 1 * 1000 * 1000) {
^

be/src/olap/base_tablet.cpp:1185: +1, nesting level increased to 1

 } else {
^

be/src/olap/base_tablet.cpp:1193: nesting level increased to 1

 [](size_t sum, const segment_v2::SegmentSharedPtr& s) { return sum += s->num_rows(); });
^

be/src/olap/base_tablet.cpp:1200: +1, including nesting penalty of 0, nesting level increased to 1

if (config::enable_merge_on_write_correctness_check && rowset->num_rows() != 0) {
^

be/src/olap/base_tablet.cpp:1200: +1

if (config::enable_merge_on_write_correctness_check && rowset->num_rows() != 0) {
^

be/src/olap/base_tablet.cpp:1205: +2, including nesting penalty of 1, nesting level increased to 2

if (!st.ok()) {
^

be/src/olap/base_tablet.cpp:1210: +1, including nesting penalty of 0, nesting level increased to 1

if (transient_rs_writer) {
^

be/src/olap/base_tablet.cpp:1211: +2, including nesting penalty of 1, nesting level increased to 2

DBUG_EXECUTE_IF("Tablet.update_delete_bitmap.partial_update_write_rowset_fail", {
^

be/src/util/debug_points.h:36: expanded from macro 'DBUG_EXECUTE_IF'

if (UNLIKELY(config::enable_debug_points)) { \
^

be/src/olap/base_tablet.cpp:1211: +3, including nesting penalty of 2, nesting level increased to 3

DBUG_EXECUTE_IF("Tablet.update_delete_bitmap.partial_update_write_rowset_fail", {
^

be/src/util/debug_points.h:38: expanded from macro 'DBUG_EXECUTE_IF'

if (dp) { \
^

be/src/olap/base_tablet.cpp:1212: +4, including nesting penalty of 3, nesting level increased to 4

if (rand() % 100 < (100 * dp->param("percent", 0.5))) {
^

be/src/olap/base_tablet.cpp:1220: +2, including nesting penalty of 1, nesting level increased to 2

RETURN_IF_ERROR(transient_rs_writer->flush());
^

be/src/common/status.h:612: expanded from macro 'RETURN_IF_ERROR'

do { \
^

be/src/olap/base_tablet.cpp:1220: +3, including nesting penalty of 2, nesting level increased to 3

RETURN_IF_ERROR(transient_rs_writer->flush());
^

be/src/common/status.h:614: expanded from macro 'RETURN_IF_ERROR'

if (UNLIKELY(!_status_.ok())) { \
^

be/src/olap/base_tablet.cpp:1222: +2, including nesting penalty of 1, nesting level increased to 2

RETURN_IF_ERROR(transient_rs_writer->build(transient_rowset));
^

be/src/common/status.h:612: expanded from macro 'RETURN_IF_ERROR'

do { \
^

be/src/olap/base_tablet.cpp:1222: +3, including nesting penalty of 2, nesting level increased to 3

RETURN_IF_ERROR(transient_rs_writer->build(transient_rowset));
^

be/src/common/status.h:614: expanded from macro 'RETURN_IF_ERROR'

if (UNLIKELY(!_status_.ok())) { \
^

be/src/olap/base_tablet.cpp:1231: +1, including nesting penalty of 0, nesting level increased to 1

RETURN_IF_ERROR(self->save_delete_bitmap(txn_info, txn_id, delete_bitmap,
^

be/src/common/status.h:612: expanded from macro 'RETURN_IF_ERROR'

do { \
^

be/src/olap/base_tablet.cpp:1231: +2, including nesting penalty of 1, nesting level increased to 2

RETURN_IF_ERROR(self->save_delete_bitmap(txn_info, txn_id, delete_bitmap,
^

be/src/common/status.h:614: expanded from macro 'RETURN_IF_ERROR'

if (UNLIKELY(!_status_.ok())) { \
^

@github-actionsgithub-actionsBotMay 31, 2024

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

warning: function 'update_delete_bitmap' exceeds recommended size/complexity thresholds [readability-function-size]

Status BaseTablet::update_delete_bitmap(const BaseTabletSPtr& self, TabletTxnInfo* txn_info,
^
Additional context

be/src/olap/base_tablet.cpp:1116: 117 lines including whitespace and comments (threshold 80)

Status BaseTablet::update_delete_bitmap(const BaseTabletSPtr& self, TabletTxnInfo* txn_info,
^

int64_t txn_id, int64_t txn_expiration) {
SCOPED_BVAR_LATENCY(g_tablet_update_delete_bitmap_latency);
RowsetIdUnorderedSet cur_rowset_ids;
Expand DownExpand Up@@ -1222,6 +1222,8 @@ Status BaseTablet::update_delete_bitmap(const BaseTabletSPtr& self, const Tablet
RowsetSharedPtr transient_rowset;
RETURN_IF_ERROR(transient_rs_writer->build(transient_rowset));
rowset->rowset_meta()->merge_rowset_meta(*transient_rowset->rowset_meta());
// update the shared_ptr to new bitmap, which is consistent with current rowset.
txn_info->delete_bitmap = delete_bitmap;

// erase segment cache cause we will add a segment to rowset
SegmentLoader::instance()->erase_segments(rowset->rowset_id(), rowset->num_segments());
Expand Down
2 changes: 1 addition & 1 deletion be/src/olap/base_tablet.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -210,7 +210,7 @@ class BaseTablet {
const Rowset& rowset, std::shared_ptr<PartialUpdateInfo> partial_update_info,
int64_t txn_expiration = 0) = 0;

static Status update_delete_bitmap(const BaseTabletSPtr& self, const TabletTxnInfo* txn_info,
static Status update_delete_bitmap(const BaseTabletSPtr& self, TabletTxnInfo* txn_info,
int64_t txn_id, int64_t txn_expiration = 0);

virtual Status save_delete_bitmap(const TabletTxnInfo* txn_info, int64_t txn_id,
Expand Down