Skip to content

[Refactor](inverted index) refactor inverted index interface in tablet schema#43003

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
airborne12 merged 5 commits into from
Nov 5, 2024
Merged

[Refactor](inverted index) refactor inverted index interface in tablet schema #43003

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
bc8f2bd
[Refactor](inverted index) refactor inverted index interface in table…
csun5285 Oct 31, 2024
504fef2
fix rebase
csun5285 Oct 31, 2024
b6e22ec
fix
csun5285 Nov 4, 2024
5150ca7
fix comment
csun5285 Nov 4, 2024
1bd320d
fix
csun5285 Nov 4, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 2 additions & 6 deletions be/src/cloud/cloud_meta_mgr.cpp
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -1215,12 +1215,8 @@ int64_t CloudMetaMgr::get_inverted_index_file_szie(const RowsetMeta& rs_meta) {
}
if (rs_meta.tablet_schema()->get_inverted_index_storage_format() ==
InvertedIndexStorageFormatPB::V1) {
auto indices = rs_meta.tablet_schema()->indexes();
const auto& indices = rs_meta.tablet_schema()->inverted_indexes();
for (auto& index : indices) {
// only get file_size for inverted index
if (index.index_type() != IndexType::INVERTED) {
continue;
}
for (int seg_id = 0; seg_id < rs_meta.num_segments(); ++seg_id) {
std::string segment_path = StorageResource().remote_segment_path(
rs_meta.tablet_id(), rs_meta.rowset_id().to_string(), seg_id);
Expand All@@ -1229,7 +1225,7 @@ int64_t CloudMetaMgr::get_inverted_index_file_szie(const RowsetMeta& rs_meta) {
std::string inverted_index_file_path =
InvertedIndexDescriptor::get_index_file_path_v1(
InvertedIndexDescriptor::get_index_file_path_prefix(segment_path),
index.index_id(), index.get_index_suffix());
index->index_id(), index->get_index_suffix());
auto st = fs->file_size(inverted_index_file_path, &file_size);
if (!st.ok()) {
file_size = 0;
Expand Down
14 changes: 6 additions & 8 deletions be/src/cloud/cloud_tablet.cpp
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -288,15 +288,13 @@ void CloudTablet::add_rowsets(std::vector<RowsetSharedPtr> to_add, bool version_
auto schema_ptr = rowset_meta->tablet_schema();
auto idx_version = schema_ptr->get_inverted_index_storage_format();
if (idx_version == InvertedIndexStorageFormatPB::V1) {
for (const auto& index : schema_ptr->indexes()) {
if (index.index_type() == IndexType::INVERTED) {
auto idx_path = storage_resource.value()->remote_idx_v1_path(
*rowset_meta, seg_id, index.index_id(),
index.get_index_suffix());
download_idx_file(idx_path);
}
for (const auto& index : schema_ptr->inverted_indexes()) {
auto idx_path = storage_resource.value()->remote_idx_v1_path(
*rowset_meta, seg_id, index->index_id(),
index->get_index_suffix());
download_idx_file(idx_path);
}
} else if (idx_version == InvertedIndexStorageFormatPB::V2) {
} else {
if (schema_ptr->has_inverted_index()) {
auto idx_path = storage_resource.value()->remote_idx_v2_path(
*rowset_meta, seg_id);
Expand Down
14 changes: 6 additions & 8 deletions be/src/cloud/cloud_warm_up_manager.cpp
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -147,15 +147,13 @@ void CloudWarmUpManager::handle_jobs() {
auto schema_ptr = rs->tablet_schema();
auto idx_version = schema_ptr->get_inverted_index_storage_format();
if (idx_version == InvertedIndexStorageFormatPB::V1) {
for (const auto& index : schema_ptr->indexes()) {
if (index.index_type() == IndexType::INVERTED) {
Comment thread
airborne12 marked this conversation as resolved.
wait->add_count();
auto idx_path = storage_resource.value()->remote_idx_v1_path(
*rs, seg_id, index.index_id(), index.get_index_suffix());
download_idx_file(idx_path);
}
for (const auto& index : schema_ptr->inverted_indexes()) {
wait->add_count();
auto idx_path = storage_resource.value()->remote_idx_v1_path(
*rs, seg_id, index->index_id(), index->get_index_suffix());
download_idx_file(idx_path);
}
} else if (idx_version == InvertedIndexStorageFormatPB::V2) {
} else {
if (schema_ptr->has_inverted_index()) {
wait->add_count();
auto idx_path =
Expand Down
19 changes: 7 additions & 12 deletions be/src/olap/compaction.cpp
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -644,7 +644,7 @@ Status Compaction::do_inverted_index_compaction() {
Status status = Status::OK();
for (auto&& column_uniq_id : ctx.columns_to_do_index_compaction) {
auto col = _cur_tablet_schema->column_by_uid(column_uniq_id);
const auto* index_meta = _cur_tablet_schema->get_inverted_index(col);
const auto* index_meta = _cur_tablet_schema->inverted_index(col);
Comment thread
csun5285 marked this conversation as resolved.

std::vector<lucene::store::Directory*> dest_index_dirs(dest_segment_num);
try {
Expand DownExpand Up@@ -682,15 +682,11 @@ Status Compaction::do_inverted_index_compaction() {
}

void Compaction::construct_index_compaction_columns(RowsetWriterContext& ctx) {
for (const auto& index : _cur_tablet_schema->indexes()) {
if (index.index_type() != IndexType::INVERTED) {
continue;
}

auto col_unique_ids = index.col_unique_ids();
for (const auto& index : _cur_tablet_schema->inverted_indexes()) {
auto col_unique_ids = index->col_unique_ids();
// check if column unique ids is empty to avoid crash
if (col_unique_ids.empty()) {
LOG(WARNING) << "tablet[" << _tablet->tablet_id() << "] index[" << index.index_id()
LOG(WARNING) << "tablet[" << _tablet->tablet_id() << "] index[" << index->index_id()
<< "] has no column unique id, will skip index compaction."
<< " tablet_schema=" << _cur_tablet_schema->dump_full_schema();
continue;
Expand All@@ -705,10 +701,9 @@ void Compaction::construct_index_compaction_columns(RowsetWriterContext& ctx) {
bool is_continue = false;
std::optional<std::map<std::string, std::string>> first_properties;
for (const auto& rowset : _input_rowsets) {
const auto* tablet_index =
rowset->tablet_schema()->get_inverted_index(col_unique_id, "");
const auto* tablet_index = rowset->tablet_schema()->inverted_index(col_unique_id);
// no inverted index or index id is different from current index id
if (tablet_index == nullptr || tablet_index->index_id() != index.index_id()) {
if (tablet_index == nullptr || tablet_index->index_id() != index->index_id()) {
is_continue = true;
break;
}
Expand DownExpand Up@@ -741,7 +736,7 @@ void Compaction::construct_index_compaction_columns(RowsetWriterContext& ctx) {
return false;
}

const auto* index_meta = rowset->tablet_schema()->get_inverted_index(col_unique_id, "");
const auto* index_meta = rowset->tablet_schema()->inverted_index(col_unique_id);
if (index_meta == nullptr) {
LOG(WARNING) << "tablet[" << _tablet->tablet_id() << "] column_unique_id["
<< col_unique_id << "] index meta is null, will skip index compaction";
Expand Down
2 changes: 1 addition & 1 deletion be/src/olap/delta_writer.cpp
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -254,7 +254,7 @@ void DeltaWriter::_request_slave_tablet_pull_rowset(const PNodeInfo& node_info)
auto tablet_schema = cur_rowset->rowset_meta()->tablet_schema();
if (!tablet_schema->skip_write_index_on_load()) {
for (auto& column : tablet_schema->columns()) {
const TabletIndex* index_meta = tablet_schema->get_inverted_index(*column);
const TabletIndex* index_meta = tablet_schema->inverted_index(*column);
if (index_meta) {
indices_ids.emplace_back(index_meta->index_id(), index_meta->get_index_suffix());
}
Expand Down
41 changes: 15 additions & 26 deletions be/src/olap/rowset/beta_rowset.cpp
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -81,20 +81,15 @@ Status BetaRowset::get_inverted_index_size(size_t* index_size) {
}

if (_schema->get_inverted_index_storage_format() == InvertedIndexStorageFormatPB::V1) {
auto indices = _schema->indexes();
for (auto& index : indices) {
// only get file_size for inverted index
if (index.index_type() != IndexType::INVERTED) {
continue;
}
for (const auto& index : _schema->inverted_indexes()) {
for (int seg_id = 0; seg_id < num_segments(); ++seg_id) {
auto seg_path = DORIS_TRY(segment_path(seg_id));
int64_t file_size = 0;

std::string inverted_index_file_path =
InvertedIndexDescriptor::get_index_file_path_v1(
InvertedIndexDescriptor::get_index_file_path_prefix(seg_path),
index.index_id(), index.get_index_suffix());
index->index_id(), index->get_index_suffix());
RETURN_IF_ERROR(fs->file_size(inverted_index_file_path, &file_size));
*index_size += file_size;
}
Expand DownExpand Up@@ -122,7 +117,7 @@ void BetaRowset::clear_inverted_index_cache() {

auto index_path_prefix = InvertedIndexDescriptor::get_index_file_path_prefix(*seg_path);
for (const auto& column : tablet_schema()->columns()) {
const TabletIndex* index_meta = tablet_schema()->get_inverted_index(*column);
const TabletIndex* index_meta = tablet_schema()->inverted_index(*column);
if (index_meta) {
auto inverted_index_file_cache_key =
InvertedIndexDescriptor::get_index_file_cache_key(
Expand DownExpand Up@@ -227,7 +222,7 @@ Status BetaRowset::remove() {

if (_schema->get_inverted_index_storage_format() == InvertedIndexStorageFormatPB::V1) {
for (auto& column : _schema->columns()) {
const TabletIndex* index_meta = _schema->get_inverted_index(*column);
const TabletIndex* index_meta = _schema->inverted_index(*column);
if (index_meta) {
std::string inverted_index_file =
InvertedIndexDescriptor::get_index_file_path_v1(
Expand DownExpand Up@@ -311,22 +306,19 @@ Status BetaRowset::link_files_to(const std::string& dir, RowsetId new_rowset_id,
return status;
});
if (_schema->get_inverted_index_storage_format() == InvertedIndexStorageFormatPB::V1) {
for (const auto& index : _schema->indexes()) {
if (index.index_type() != IndexType::INVERTED) {
continue;
}
auto index_id = index.index_id();
for (const auto& index : _schema->inverted_indexes()) {
auto index_id = index->index_id();
if (without_index_uids != nullptr && without_index_uids->count(index_id)) {
continue;
}
std::string inverted_index_src_file_path =
InvertedIndexDescriptor::get_index_file_path_v1(
InvertedIndexDescriptor::get_index_file_path_prefix(src_path),
index_id, index.get_index_suffix());
index_id, index->get_index_suffix());
std::string inverted_index_dst_file_path =
InvertedIndexDescriptor::get_index_file_path_v1(
InvertedIndexDescriptor::get_index_file_path_prefix(dst_path),
index_id, index.get_index_suffix());
index_id, index->get_index_suffix());
bool index_file_exists = true;
RETURN_IF_ERROR(local_fs->exists(inverted_index_src_file_path, &index_file_exists));
if (index_file_exists) {
Expand DownExpand Up@@ -405,7 +397,7 @@ Status BetaRowset::copy_files_to(const std::string& dir, const RowsetId& new_row
if (_schema->get_inverted_index_storage_format() == InvertedIndexStorageFormatPB::V1) {
for (auto& column : _schema->columns()) {
// if (column.has_inverted_index()) {
const TabletIndex* index_meta = _schema->get_inverted_index(*column);
const TabletIndex* index_meta = _schema->inverted_index(*column);
if (index_meta) {
std::string inverted_index_src_file_path =
InvertedIndexDescriptor::get_index_file_path_v1(
Expand DownExpand Up@@ -464,7 +456,7 @@ Status BetaRowset::upload_to(const StorageResource& dest_fs, const RowsetId& new
if (_schema->get_inverted_index_storage_format() == InvertedIndexStorageFormatPB::V1) {
for (auto& column : _schema->columns()) {
// if (column.has_inverted_index()) {
const TabletIndex* index_meta = _schema->get_inverted_index(*column);
const TabletIndex* index_meta = _schema->inverted_index(*column);
if (index_meta) {
std::string remote_inverted_index_file =
InvertedIndexDescriptor::get_index_file_path_v1(
Expand DownExpand Up@@ -613,14 +605,11 @@ Status BetaRowset::add_to_binlog() {
linked_success_files.push_back(binlog_file);

if (_schema->get_inverted_index_storage_format() == InvertedIndexStorageFormatPB::V1) {
for (const auto& index : _schema->indexes()) {
if (index.index_type() != IndexType::INVERTED) {
continue;
}
auto index_id = index.index_id();
for (const auto& index : _schema->inverted_indexes()) {
auto index_id = index->index_id();
auto index_file = InvertedIndexDescriptor::get_index_file_path_v1(
InvertedIndexDescriptor::get_index_file_path_prefix(seg_file), index_id,
index.get_index_suffix());
index->get_index_suffix());
auto binlog_index_file = (std::filesystem::path(binlog_dir) /
std::filesystem::path(index_file).filename())
.string();
Expand DownExpand Up@@ -661,7 +650,7 @@ Status BetaRowset::calc_file_crc(uint32_t* crc_value, int64_t* file_count) {
file_paths.emplace_back(seg_path);
if (_schema->get_inverted_index_storage_format() == InvertedIndexStorageFormatPB::V1) {
for (auto& column : _schema->columns()) {
const TabletIndex* index_meta = _schema->get_inverted_index(*column);
const TabletIndex* index_meta = _schema->inverted_index(*column);
if (index_meta) {
std::string inverted_index_file =
InvertedIndexDescriptor::get_index_file_path_v1(
Expand DownExpand Up@@ -805,7 +794,7 @@ Status BetaRowset::show_nested_index_file(rapidjson::Value* rowset_value,
} else {
rapidjson::Value indices(rapidjson::kArrayType);
for (auto column : _rowset_meta->tablet_schema()->columns()) {
const auto* index_meta = _rowset_meta->tablet_schema()->get_inverted_index(*column);
const auto* index_meta = _rowset_meta->tablet_schema()->inverted_index(*column);
if (index_meta == nullptr) {
continue;
}
Expand Down
4 changes: 2 additions & 2 deletions be/src/olap/rowset/beta_rowset_writer.cpp
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -547,8 +547,8 @@ Status BetaRowsetWriter::_rename_compacted_indices(int64_t begin, int64_t end, u
}
// rename remaining inverted index files
for (auto column : _context.tablet_schema->columns()) {
if (_context.tablet_schema->has_inverted_index(*column)) {
const auto* index_info = _context.tablet_schema->get_inverted_index(*column);
if (const auto& index_info = _context.tablet_schema->inverted_index(*column);
index_info != nullptr) {
auto index_id = index_info->index_id();
if (_context.tablet_schema->get_inverted_index_storage_format() ==
InvertedIndexStorageFormatPB::V1) {
Expand Down
3 changes: 1 addition & 2 deletions be/src/olap/rowset/segcompaction.cpp
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -165,8 +165,7 @@ Status SegcompactionWorker::_delete_original_segments(uint32_t begin, uint32_t e
}
// Delete inverted index files
for (auto&& column : schema->columns()) {
if (schema->has_inverted_index(*column)) {
const auto* index_info = schema->get_inverted_index(*column);
if (const auto* index_info = schema->inverted_index(*column); index_info != nullptr) {
auto index_id = index_info->index_id();
if (schema->get_inverted_index_storage_format() ==
InvertedIndexStorageFormatPB::V1) {
Expand Down
2 changes: 1 addition & 1 deletion be/src/olap/rowset/segment_v2/column_writer.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -63,7 +63,7 @@ struct ColumnWriterOptions {
bool need_inverted_index = false;
uint8_t gram_size;
uint16_t gram_bf_size;
std::vector<const TabletIndex*> indexes;
std::vector<const TabletIndex*> indexes; // unused
Comment thread
csun5285 marked this conversation as resolved.
const TabletIndex* inverted_index = nullptr;
InvertedIndexFileWriter* inverted_index_file_writer;
std::string to_string() const {
Expand Down
17 changes: 9 additions & 8 deletions be/src/olap/rowset/segment_v2/segment_iterator.cpp
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -1057,16 +1057,17 @@ Status SegmentIterator::_init_inverted_index_iterators() {
return Status::OK();
}
for (auto cid : _schema->column_ids()) {
// Use segment’s own index_meta, for compatibility with future indexing needs to default to lowercase.
if (_inverted_index_iterators[cid] == nullptr) {
// Not check type valid, since we need to get inverted index for related variant type when reading the segment.
// If check type valid, we can not get inverted index for variant type, and result nullptr.The result for calling
// get_inverted_index with variant suffix should return corresponding inverted index meta.
bool check_inverted_index_by_type = false;
// Use segment’s own index_meta, for compatibility with future indexing needs to default to lowercase.
// In the _opts.tablet_schema, the sub-column type information for the variant is FieldType::OLAP_FIELD_TYPE_VARIANT.
// This is because the sub-column is created in create_materialized_variant_column.
// We use this column to locate the metadata for the inverted index, which requires a unique_id and path.
const auto& column = _opts.tablet_schema->column(cid);
Comment thread
csun5285 marked this conversation as resolved.
int32_t col_unique_id =
column.is_extracted_column() ? column.parent_unique_id() : column.unique_id();
RETURN_IF_ERROR(_segment->new_inverted_index_iterator(
_opts.tablet_schema->column(cid),
_segment->_tablet_schema->get_inverted_index(_opts.tablet_schema->column(cid),
check_inverted_index_by_type),
column,
_segment->_tablet_schema->inverted_index(col_unique_id, column.suffix_path()),
Comment thread
csun5285 marked this conversation as resolved.
_opts, &_inverted_index_iterators[cid]));
}
}
Expand Down
Loading