Skip to content

[Improve](inverted index) improve match performance without index#24751

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
xiaokang merged 2 commits into from
Sep 22, 2023
Merged

[Improve](inverted index) improve match performance without index #24751

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
352610b
[Improve](inverted index) improve match performance without index
airborne12 Sep 21, 2023
518f046
fix compile problem
airborne12 Sep 22, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions be/src/olap/inverted_index_parser.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -21,6 +21,12 @@
#include <memory>
#include <string>

namespace lucene {
namespace analysis {
class Analyzer;
}
} // namespace lucene

namespace doris {

enum class InvertedIndexParserType {
Expand All@@ -38,6 +44,7 @@ struct InvertedIndexCtx {
InvertedIndexParserType parser_type;
std::string parser_mode;
CharFilterMap char_filter_map;
lucene::analysis::Analyzer* analyzer;
};

using InvertedIndexCtxSPtr = std::shared_ptr<InvertedIndexCtx>;
Expand Down
10 changes: 5 additions & 5 deletions be/src/olap/rowset/segment_v2/inverted_index/query/conjunction_query.cpp
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -37,15 +37,15 @@ ConjunctionQuery::~ConjunctionQuery() {
}
}

void ConjunctionQuery::add(const std::wstring& field_name,
const std::vector<std::wstring>& wterms) {
if (wterms.size() < 1) {
void ConjunctionQuery::add(const std::wstring& field_name, const std::vector<std::string>& terms) {
if (terms.size() < 1) {
_CLTHROWA(CL_ERR_IllegalArgument, "ConjunctionQuery::add: terms.size() < 1");
}

std::vector<TermIterator> iterators;
for (auto& wterm : wterms) {
Term* t = _CLNEW Term(field_name.c_str(), wterm.c_str());
for (auto& term : terms) {
std::wstring ws_term = StringUtil::string_to_wstring(term);
Term* t = _CLNEW Term(field_name.c_str(), ws_term.c_str());
_terms.push_back(t);
TermDocs* term_doc = _reader->termDocs(t);
_term_docs.push_back(term_doc);
Expand Down
2 changes: 1 addition & 1 deletion be/src/olap/rowset/segment_v2/inverted_index/query/conjunction_query.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -38,7 +38,7 @@ class ConjunctionQuery {
_conjunction_ratio = conjunction_ratio;
}

void add(const std::wstring& field_name, const std::vector<std::wstring>& wterms);
void add(const std::wstring& field_name, const std::vector<std::string>& terms);
void search(roaring::Roaring& roaring);

private:
Expand Down
72 changes: 43 additions & 29 deletions be/src/olap/rowset/segment_v2/inverted_index_reader.cpp
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -99,54 +99,63 @@ bool InvertedIndexReader::indexExists(io::Path& index_file_path) {
return exists;
}

std::vector<std::wstring> InvertedIndexReader::get_analyse_result(
const std::string& field_name, const std::string& value, InvertedIndexQueryType query_type,
InvertedIndexCtx* inverted_index_ctx, bool drop_duplicates) {
std::vector<std::wstring> analyse_result;
std::shared_ptr<lucene::analysis::Analyzer> analyzer;
std::unique_ptr<lucene::util::Reader> reader;
std::unique_ptr<lucene::analysis::Analyzer> InvertedIndexReader::create_analyzer(
InvertedIndexCtx* inverted_index_ctx) {
std::unique_ptr<lucene::analysis::Analyzer> analyzer;
auto analyser_type = inverted_index_ctx->parser_type;
if (analyser_type == InvertedIndexParserType::PARSER_STANDARD ||
analyser_type == InvertedIndexParserType::PARSER_UNICODE) {
analyzer = std::make_shared<lucene::analysis::standard95::StandardAnalyzer>();
analyzer = std::make_unique<lucene::analysis::standard95::StandardAnalyzer>();
} else if (analyser_type == InvertedIndexParserType::PARSER_ENGLISH) {
analyzer = std::make_shared<lucene::analysis::SimpleAnalyzer<char>>();
analyzer = std::make_unique<lucene::analysis::SimpleAnalyzer<char>>();
} else if (analyser_type == InvertedIndexParserType::PARSER_CHINESE) {
auto chinese_analyzer =
std::make_shared<lucene::analysis::LanguageBasedAnalyzer>(L"chinese", false);
std::make_unique<lucene::analysis::LanguageBasedAnalyzer>(L"chinese", false);
chinese_analyzer->initDict(config::inverted_index_dict_path);
auto mode = inverted_index_ctx->parser_mode;
if (mode == INVERTED_INDEX_PARSER_COARSE_GRANULARITY) {
chinese_analyzer->setMode(lucene::analysis::AnalyzerMode::Default);
} else {
chinese_analyzer->setMode(lucene::analysis::AnalyzerMode::All);
}
analyzer = chinese_analyzer;
analyzer = std::move(chinese_analyzer);
} else {
// default
analyzer = std::make_shared<lucene::analysis::SimpleAnalyzer<char>>();
analyzer = std::make_unique<lucene::analysis::SimpleAnalyzer<char>>();
}
reader.reset(new lucene::util::SStringReader<char>());
return analyzer;
}

std::unique_ptr<lucene::util::Reader> InvertedIndexReader::create_reader(
InvertedIndexCtx* inverted_index_ctx, const std::string& value) {
std::unique_ptr<lucene::util::Reader> reader =
std::make_unique<lucene::util::SStringReader<char>>();
CharFilterMap& char_filter_map = inverted_index_ctx->char_filter_map;
if (!char_filter_map.empty()) {
reader.reset(CharFilterFactory::create(
reader = std::unique_ptr<lucene::util::Reader>(CharFilterFactory::create(
char_filter_map[INVERTED_INDEX_PARSER_CHAR_FILTER_TYPE], reader.release(),
char_filter_map[INVERTED_INDEX_PARSER_CHAR_FILTER_PATTERN],
char_filter_map[INVERTED_INDEX_PARSER_CHAR_FILTER_REPLACEMENT]));
}
reader->init(value.data(), value.size(), false);
reader->init(value.data(), value.size(), true);
return reader;
}

std::vector<std::string> InvertedIndexReader::get_analyse_result(
lucene::util::Reader* reader, lucene::analysis::Analyzer* analyzer,
const std::string& field_name, InvertedIndexQueryType query_type, bool drop_duplicates) {
std::vector<std::string> analyse_result;

std::wstring field_ws = std::wstring(field_name.begin(), field_name.end());
std::unique_ptr<lucene::analysis::TokenStream> token_stream(
analyzer->tokenStream(field_ws.c_str(), reader.get()));
analyzer->tokenStream(field_ws.c_str(), reader));

lucene::analysis::Token token;

while (token_stream->next(&token)) {
if (token.termLength<char>() != 0) {
std::string_view term(token.termBuffer<char>(), token.termLength<char>());
std::wstring ws_term = StringUtil::string_to_wstring(term);
analyse_result.emplace_back(ws_term);
analyse_result.emplace_back(
std::string(token.termBuffer<char>(), token.termLength<char>()));
}
}

Expand All@@ -156,7 +165,7 @@ std::vector<std::wstring> InvertedIndexReader::get_analyse_result(

if (drop_duplicates && (query_type == InvertedIndexQueryType::MATCH_ANY_QUERY ||
query_type == InvertedIndexQueryType::MATCH_ALL_QUERY)) {
std::set<std::wstring> unrepeated_result(analyse_result.begin(), analyse_result.end());
std::set<std::string> unrepeated_result(analyse_result.begin(), analyse_result.end());
analyse_result.assign(unrepeated_result.begin(), unrepeated_result.end());
}

Expand DownExpand Up@@ -245,8 +254,11 @@ Status FullTextIndexReader::query(OlapReaderStatistics* stats, RuntimeState* run
inverted_index_ctx->char_filter_map =
get_parser_char_filter_map_from_properties(_index_meta.properties());
try {
std::vector<std::wstring> analyse_result =
get_analyse_result(column_name, search_str, query_type, inverted_index_ctx.get());
auto analyzer = create_analyzer(inverted_index_ctx.get());
auto reader = create_reader(inverted_index_ctx.get(), search_str);
inverted_index_ctx->analyzer = analyzer.get();
std::vector<std::string> analyse_result =
get_analyse_result(reader.get(), analyzer.get(), column_name, query_type);

if (analyse_result.empty()) {
auto msg = fmt::format(
Expand DownExpand Up@@ -282,18 +294,18 @@ Status FullTextIndexReader::query(OlapReaderStatistics* stats, RuntimeState* run
if (query_type == InvertedIndexQueryType::MATCH_PHRASE_QUERY ||
query_type == InvertedIndexQueryType::MATCH_ALL_QUERY ||
query_type == InvertedIndexQueryType::EQUAL_QUERY) {
std::wstring wstr_tokens;
std::string str_tokens;

@github-actionsgithub-actionsBotSep 22, 2023

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

warning: variable 'str_tokens' is not initialized [cppcoreguidelines-init-variables]

Suggested change
std::string str_tokens;
std::string str_tokens = 0;

for (auto& token : analyse_result) {
wstr_tokens += token;
wstr_tokens += L" ";
str_tokens += token;
str_tokens += " ";
}

auto cache = InvertedIndexQueryCache::instance();
InvertedIndexQueryCache::CacheKey cache_key;
cache_key.index_path = index_file_path;
cache_key.column_name = column_name;
cache_key.query_type = query_type;
auto str_tokens = lucene_wcstoutf8string(wstr_tokens.c_str(), wstr_tokens.length());
//auto str_tokens = lucene_wcstoutf8string(wstr_tokens.c_str(), wstr_tokens.length());
cache_key.value.swap(str_tokens);
InvertedIndexQueryCacheHandle cache_handle;
std::shared_ptr<roaring::Roaring> term_match_bitmap = nullptr;
Expand All@@ -309,7 +321,8 @@ Status FullTextIndexReader::query(OlapReaderStatistics* stats, RuntimeState* run
if (query_type == InvertedIndexQueryType::MATCH_PHRASE_QUERY) {
auto* phrase_query = new lucene::search::PhraseQuery();
for (auto& token : analyse_result) {
auto* term = _CLNEW lucene::index::Term(field_ws.c_str(), token.c_str());
std::wstring wtoken = StringUtil::string_to_wstring(token);
auto* term = _CLNEW lucene::index::Term(field_ws.c_str(), wtoken.c_str());
phrase_query->add(term);
_CLDECDELETE(term);
}
Expand All@@ -331,13 +344,14 @@ Status FullTextIndexReader::query(OlapReaderStatistics* stats, RuntimeState* run
query_match_bitmap = *term_match_bitmap;
} else {
bool first = true;
for (auto token_ws : analyse_result) {
for (auto token : analyse_result) {
std::shared_ptr<roaring::Roaring> term_match_bitmap = nullptr;

// try to get term bitmap match result from cache to avoid query index on cache hit
auto cache = InvertedIndexQueryCache::instance();
// use EQUAL_QUERY type here since cache is for each term/token
auto token = lucene_wcstoutf8string(token_ws.c_str(), token_ws.length());
//auto token = lucene_wcstoutf8string(token_ws.c_str(), token_ws.length());
std::wstring token_ws = StringUtil::string_to_wstring(token);

InvertedIndexQueryCache::CacheKey cache_key {
index_file_path, column_name, InvertedIndexQueryType::EQUAL_QUERY, token};
Expand DownExpand Up@@ -435,7 +449,7 @@ Status FullTextIndexReader::normal_index_search(

Status FullTextIndexReader::match_all_index_search(
OlapReaderStatistics* stats, RuntimeState* runtime_state, const std::wstring& field_ws,
const std::vector<std::wstring>& analyse_result, const IndexSearcherPtr& index_searcher,
const std::vector<std::string>& analyse_result, const IndexSearcherPtr& index_searcher,
const std::shared_ptr<roaring::Roaring>& term_match_bitmap) {
TQueryOptions queryOptions = runtime_state->query_options();
try {
Expand Down
16 changes: 10 additions & 6 deletions be/src/olap/rowset/segment_v2/inverted_index_reader.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -97,11 +97,15 @@ class InvertedIndexReader : public std::enable_shared_from_this<InvertedIndexRea
return _index_meta.properties();
}

static std::vector<std::wstring> get_analyse_result(const std::string& field_name,
const std::string& value,
InvertedIndexQueryType query_type,
InvertedIndexCtx* inverted_index_ctx,
bool drop_duplicates = true);
static std::vector<std::string> get_analyse_result(lucene::util::Reader* reader,
lucene::analysis::Analyzer* analyzer,
const std::string& field_name,
InvertedIndexQueryType query_type,
bool drop_duplicates = true);
static std::unique_ptr<lucene::util::Reader> create_reader(InvertedIndexCtx* inverted_index_ctx,
const std::string& value);
static std::unique_ptr<lucene::analysis::Analyzer> create_analyzer(
InvertedIndexCtx* inverted_index_ctx);

protected:
bool _is_range_query(InvertedIndexQueryType query_type);
Expand DownExpand Up@@ -144,7 +148,7 @@ class FullTextIndexReader : public InvertedIndexReader {

Status match_all_index_search(OlapReaderStatistics* stats, RuntimeState* runtime_state,
const std::wstring& field_ws,
const std::vector<std::wstring>& analyse_result,
const std::vector<std::string>& analyse_result,
const IndexSearcherPtr& index_searcher,
const std::shared_ptr<roaring::Roaring>& term_match_bitmap);

Expand Down
6 changes: 6 additions & 0 deletions be/src/vec/exprs/vmatch_predicate.cpp
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -30,6 +30,7 @@
#include <vector>

#include "common/status.h"
#include "olap/rowset/segment_v2/inverted_index_reader.h"
#include "vec/core/block.h"
#include "vec/core/column_numbers.h"
#include "vec/core/column_with_type_and_name.h"
Expand All@@ -43,15 +44,20 @@ class RuntimeState;
} // namespace doris

namespace doris::vectorized {
using namespace doris::segment_v2;

VMatchPredicate::VMatchPredicate(const TExprNode& node) : VExpr(node) {
_inverted_index_ctx = std::make_shared<InvertedIndexCtx>();
_inverted_index_ctx->parser_type =
get_inverted_index_parser_type_from_string(node.match_predicate.parser_type);
_inverted_index_ctx->parser_mode = node.match_predicate.parser_mode;
_inverted_index_ctx->char_filter_map = node.match_predicate.char_filter_map;
_analyzer = InvertedIndexReader::create_analyzer(_inverted_index_ctx.get());
_inverted_index_ctx->analyzer = _analyzer.get();
}

VMatchPredicate::~VMatchPredicate() = default;

Status VMatchPredicate::prepare(RuntimeState* state, const RowDescriptor& desc,
VExprContext* context) {
RETURN_IF_ERROR_OR_PREPARED(VExpr::prepare(state, desc, context));
Expand Down
8 changes: 7 additions & 1 deletion be/src/vec/exprs/vmatch_predicate.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -26,6 +26,11 @@
#include "vec/exprs/vexpr.h"
#include "vec/functions/function.h"

namespace lucene {
namespace analysis {
class Analyzer;
}
} // namespace lucene
namespace doris {
class RowDescriptor;
class RuntimeState;
Expand All@@ -43,7 +48,7 @@ class VMatchPredicate final : public VExpr {

public:
VMatchPredicate(const TExprNode& node);
~VMatchPredicate() override = default;
~VMatchPredicate() override;
Status execute(VExprContext* context, Block* block, int* result_column_id) override;
Status prepare(RuntimeState* state, const RowDescriptor& desc, VExprContext* context) override;
Status open(RuntimeState* state, VExprContext* context,
Expand All@@ -61,5 +66,6 @@ class VMatchPredicate final : public VExpr {
std::string _expr_name;
std::string _function_name;
InvertedIndexCtxSPtr _inverted_index_ctx;
std::unique_ptr<lucene::analysis::Analyzer> _analyzer;
};
} // namespace doris::vectorized
16 changes: 10 additions & 6 deletions be/src/vec/functions/function_tokenize.cpp
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -95,13 +95,14 @@ void FunctionTokenize::_do_tokenize(const ColumnString& src_column_string,
dest_offsets.push_back(dest_pos);
continue;
}
std::vector<std::wstring> query_tokens =
auto reader = doris::segment_v2::InvertedIndexReader::create_reader(
&inverted_index_ctx, tokenize_str.to_string());

std::vector<std::string> query_tokens =

@github-actionsgithub-actionsBotSep 22, 2023

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

warning: variable 'query_tokens' is not initialized [cppcoreguidelines-init-variables]

Suggested change
std::vector<std::string> query_tokens =
std::vector<std::string> query_tokens =0 =

doris::segment_v2::InvertedIndexReader::get_analyse_result(
"tokenize", tokenize_str.to_string(),
doris::segment_v2::InvertedIndexQueryType::MATCH_PHRASE_QUERY,
&inverted_index_ctx);
for (auto token_ws : query_tokens) {
std::string token = lucene_wcstoutf8string(token_ws.data(), token_ws.length());
reader.get(), inverted_index_ctx.analyzer, "tokenize",
doris::segment_v2::InvertedIndexQueryType::MATCH_PHRASE_QUERY);
for (auto token : query_tokens) {
const size_t old_size = column_string_chars.size();
const size_t split_part_size = token.length();
if (split_part_size > 0) {
Expand DownExpand Up@@ -158,6 +159,9 @@ Status FunctionTokenize::execute_impl(FunctionContext* /*context*/, Block& block
inverted_index_ctx.parser_mode = get_parser_mode_string_from_properties(properties);
inverted_index_ctx.char_filter_map =
get_parser_char_filter_map_from_properties(properties);
auto analyzer =
doris::segment_v2::InvertedIndexReader::create_analyzer(&inverted_index_ctx);
inverted_index_ctx.analyzer = analyzer.get();
_do_tokenize(*col_left, inverted_index_ctx, *dest_nested_column, dest_offsets,
dest_nested_null_map);

Expand Down
Loading