[feature](invert index) match_phrase_prefix feature added

be/src/exec/olap_common.h

@@ -304,6 +304,8 @@ class ColumnValueRange {
  condition.__set_condition_op("match_all");
  } else if (value.first == MatchType::MATCH_PHRASE) {
  condition.__set_condition_op("match_phrase");
+ } else if (value.first == MatchType::MATCH_PHRASE_PREFIX) {
+ condition.__set_condition_op("match_phrase_prefix");
  } else if (value.first == MatchType::MATCH_ELEMENT_EQ) {
  condition.__set_condition_op("match_element_eq");
  } else if (value.first == MatchType::MATCH_ELEMENT_LT) {

be/src/exec/olap_utils.h

@@ -169,6 +169,7 @@ enum class MatchType {
  MATCH_ELEMENT_GT = 5,
  MATCH_ELEMENT_LE = 6,
  MATCH_ELEMENT_GE = 7,
+ MATCH_PHRASE_PREFIX = 8,
 };
 
 inline MatchType to_match_type(TExprOpcode::type type) {
@@ -182,6 +183,9 @@ inline MatchType to_match_type(TExprOpcode::type type) {
  case TExprOpcode::type::MATCH_PHRASE:
  return MatchType::MATCH_PHRASE;
  break;
+ case TExprOpcode::type::MATCH_PHRASE_PREFIX:
+ return MatchType::MATCH_PHRASE_PREFIX;
+ break;
  case TExprOpcode::type::MATCH_ELEMENT_EQ:
  return MatchType::MATCH_ELEMENT_EQ;
  break;
@@ -211,6 +215,8 @@ inline MatchType to_match_type(const std::string& condition_op) {
  return MatchType::MATCH_ALL;
  } else if (condition_op.compare("match_phrase") == 0) {
  return MatchType::MATCH_PHRASE;
+ } else if (condition_op.compare("match_phrase_prefix") == 0) {
+ return MatchType::MATCH_PHRASE_PREFIX;
  } else if (condition_op.compare("match_element_eq") == 0) {
  return MatchType::MATCH_ELEMENT_EQ;
  } else if (condition_op.compare("match_element_lt") == 0) {
@@ -228,6 +234,7 @@ inline MatchType to_match_type(const std::string& condition_op) {
 inline bool is_match_condition(const std::string& op) {
  if (0 == strcasecmp(op.c_str(), "match_any") || 0 == strcasecmp(op.c_str(), "match_all") ||
  0 == strcasecmp(op.c_str(), "match_phrase") ||
+ 0 == strcasecmp(op.c_str(), "match_phrase_prefix") ||
  0 == strcasecmp(op.c_str(), "match_element_eq") ||
  0 == strcasecmp(op.c_str(), "match_element_lt") ||
  0 == strcasecmp(op.c_str(), "match_element_gt") ||
@@ -240,9 +247,10 @@ inline bool is_match_condition(const std::string& op) {
 
 inline bool is_match_operator(const TExprOpcode::type& op_type) {
  return TExprOpcode::MATCH_ANY == op_type || TExprOpcode::MATCH_ALL == op_type ||
- TExprOpcode::MATCH_PHRASE == op_type || TExprOpcode::MATCH_ELEMENT_EQ == op_type ||
- TExprOpcode::MATCH_ELEMENT_LT == op_type || TExprOpcode::MATCH_ELEMENT_GT == op_type ||
- TExprOpcode::MATCH_ELEMENT_LE == op_type || TExprOpcode::MATCH_ELEMENT_GE == op_type;
+ TExprOpcode::MATCH_PHRASE == op_type || TExprOpcode::MATCH_PHRASE_PREFIX == op_type ||
+ TExprOpcode::MATCH_ELEMENT_EQ == op_type || TExprOpcode::MATCH_ELEMENT_LT == op_type ||
+ TExprOpcode::MATCH_ELEMENT_GT == op_type || TExprOpcode::MATCH_ELEMENT_LE == op_type ||
+ TExprOpcode::MATCH_ELEMENT_GE == op_type;
 }
 
 } // namespace doris

be/src/olap/match_predicate.cpp

@@ -107,6 +107,9 @@ InvertedIndexQueryType MatchPredicate::_to_inverted_index_query_type(MatchType m
  case MatchType::MATCH_PHRASE:
  ret = InvertedIndexQueryType::MATCH_PHRASE_QUERY;
  break;
+ case MatchType::MATCH_PHRASE_PREFIX:
+ ret = InvertedIndexQueryType::MATCH_PHRASE_PREFIX_QUERY;
+ break;
  case MatchType::MATCH_ELEMENT_EQ:
  ret = InvertedIndexQueryType::EQUAL_QUERY;
  break;
@@ -129,7 +132,7 @@ InvertedIndexQueryType MatchPredicate::_to_inverted_index_query_type(MatchType m
 }
 
 bool MatchPredicate::_skip_evaluate(InvertedIndexIterator* iterator) const {
- if (_match_type == MatchType::MATCH_PHRASE &&
+ if ((_match_type == MatchType::MATCH_PHRASE || _match_type == MatchType::MATCH_PHRASE_PREFIX) &&
  iterator->get_inverted_index_reader_type() == InvertedIndexReaderType::FULLTEXT &&
  get_parser_phrase_support_string_from_properties(iterator->get_index_properties()) ==
  INVERTED_INDEX_PARSER_PHRASE_SUPPORT_NO) {

be/src/olap/rowset/segment_v2/inverted_index/query/phrase_prefix_query.cpp

@@ -0,0 +1,63 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "phrase_prefix_query.h"
+
+#include "olap/rowset//segment_v2/inverted_index/query/prefix_query.h"
+
+namespace doris {
+
+namespace segment_v2 {
+
+PhrasePrefixQuery::PhrasePrefixQuery(const std::shared_ptr<lucene::search::IndexSearcher>& searcher)
+ : _searcher(searcher) {}
+
+void PhrasePrefixQuery::add(const std::wstring& field_name, const std::vector<std::string>& terms) {
+ if (terms.empty()) {
+ return;
+ }
+
+ for (size_t i = 0; i < terms.size(); i++) {
+ if (i < terms.size() - 1) {
+ std::wstring ws = StringUtil::string_to_wstring(terms[i]);
+ Term* t = _CLNEW Term(field_name.c_str(), ws.c_str());
+ _query.add(t);
+ _CLDECDELETE(t);
+ } else {
+ std::vector<CL_NS(index)::Term*> prefix_terms;
+ PrefixQuery::get_prefix_terms(_searcher->getReader(), field_name, terms[i],
+ prefix_terms, _max_expansions);
+ if (prefix_terms.empty()) {
+ continue;
+ }
+ _query.add(prefix_terms);
+ for (auto& t : prefix_terms) {
+ _CLDECDELETE(t);
+ }
+ }
+ }
+}
+
+void PhrasePrefixQuery::search(roaring::Roaring& roaring) {
+ _searcher->_search(&_query, [&roaring](const int32_t docid, const float_t /*score*/) {
+ roaring.add(docid);
+ });
+}
+
+} // namespace segment_v2
+
+} // namespace doris

be/src/olap/rowset/segment_v2/inverted_index/query/phrase_prefix_query.h

@@ -0,0 +1,54 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <CLucene.h>
+#include <CLucene/index/IndexReader.h>
+
+#include <memory>
+
+#include "CLucene/search/MultiPhraseQuery.h"
+#include "roaring/roaring.hh"
+
+CL_NS_USE(index)
+CL_NS_USE(search)
+
+namespace doris {
+
+namespace segment_v2 {
+
+class PhrasePrefixQuery {
+public:
+ PhrasePrefixQuery(const std::shared_ptr<lucene::search::IndexSearcher>& searcher);
+ ~PhrasePrefixQuery() = default;
+
+ void set_max_expansions(int32_t max_expansions) { _max_expansions = max_expansions; }
+
+ void add(const std::wstring& field_name, const std::vector<std::string>& terms);
+ void search(roaring::Roaring& roaring);
+
+private:
+ std::shared_ptr<lucene::search::IndexSearcher> _searcher;
+ MultiPhraseQuery _query;
+
+ int32_t _max_expansions = 50;
+};
+
+} // namespace segment_v2
+
+} // namespace doris

be/src/olap/rowset/segment_v2/inverted_index/query/prefix_query.cpp

@@ -0,0 +1,80 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "prefix_query.h"
+
+namespace doris {
+
+void PrefixQuery::get_prefix_terms(IndexReader* reader, const std::wstring& field_name,
+ const std::string& prefix,
+ std::vector<CL_NS(index)::Term*>& prefix_terms,
+ int32_t max_expansions) {
+ std::wstring ws_prefix = StringUtil::string_to_wstring(prefix);
+
+ Term* prefix_term = _CLNEW Term(field_name.c_str(), ws_prefix.c_str());
+ TermEnum* enumerator = reader->terms(prefix_term);
+
+ int32_t count = 0;
+ Term* lastTerm = nullptr;
+ try {
+ const TCHAR* prefixText = prefix_term->text();
+ const TCHAR* prefixField = prefix_term->field();
+ const TCHAR* tmp = nullptr;
+ size_t i = 0;
+ size_t prefixLen = prefix_term->textLength();
+ do {
+ lastTerm = enumerator->term();
+ if (lastTerm != nullptr && lastTerm->field() == prefixField) {
+ size_t termLen = lastTerm->textLength();
+ if (prefixLen > termLen) {
+ break;
+ }
+
+ tmp = lastTerm->text();
+
+ for (i = prefixLen - 1; i != -1; --i) {
+ if (tmp[i] != prefixText[i]) {
+ tmp = nullptr;
+ break;
+ }
+ }
+ if (tmp == nullptr) {
+ break;
+ }
+
+ if (max_expansions > 0 && count >= max_expansions) {
+ break;
+ }
+
+ Term* t = _CLNEW Term(field_name.c_str(), tmp);
+ prefix_terms.push_back(t);
+ count++;
+ } else {
+ break;
+ }
+ _CLDECDELETE(lastTerm);
+ } while (enumerator->next());
+ }
+ _CLFINALLY({
+ enumerator->close();
+ _CLDELETE(enumerator);
+ _CLDECDELETE(lastTerm);
+ _CLDECDELETE(prefix_term);
+ });
+}
+
+} // namespace doris

be/src/olap/rowset/segment_v2/inverted_index/query/prefix_query.h

@@ -0,0 +1,40 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <CLucene.h>
+#include <CLucene/index/IndexReader.h>
+
+#include <cstdint>
+
+CL_NS_USE(index)
+
+namespace doris {
+
+class PrefixQuery {
+public:
+ PrefixQuery() = default;
+ ~PrefixQuery() = default;
+
+ static void get_prefix_terms(IndexReader* reader, const std::wstring& field_name,
+ const std::string& prefix,
+ std::vector<CL_NS(index)::Term*>& prefix_terms,
+ int32_t max_expansions = 50);
+};
+
+} // namespace doris

be/src/olap/rowset/segment_v2/inverted_index_query_type.h

@@ -76,6 +76,7 @@ enum class InvertedIndexQueryType {
  MATCH_ANY_QUERY = 5,
  MATCH_ALL_QUERY = 6,
  MATCH_PHRASE_QUERY = 7,
+ MATCH_PHRASE_PREFIX_QUERY = 8,
 };
 
 inline std::string query_type_to_string(InvertedIndexQueryType query_type) {
@@ -107,6 +108,9 @@ inline std::string query_type_to_string(InvertedIndexQueryType query_type) {
  case InvertedIndexQueryType::MATCH_PHRASE_QUERY: {
  return "MPHRASE";
  }
+ case InvertedIndexQueryType::MATCH_PHRASE_PREFIX_QUERY: {
+ return "MPHRASEPREFIX";
+ }
  default:
  return "";
  }