[feature](inverted-index) Add Japanese (Kuromoji) morphological analyzer

.clang-format-ignore

@@ -9,4 +9,5 @@ be/src/util/sse2neon.h
 be/src/util/mustache/mustache.h
 be/src/util/mustache/mustache.cc
 be/src/util/utf8_check.cpp
+be/src/storage/index/inverted/analyzer/kuromoji/dict/darts.h
 cloud/src/common/defer.h

.gitignore

@@ -154,6 +154,9 @@ docker/runtime/be/resource/apache-doris/
 compile_commands.json
 .github
 
+# generated kuromoji dictionary binaries
+/be/dict/kuromoji/*.bin
+
 .worktrees/
 .worktree_initialized
 

.licenserc.yaml

@@ -112,6 +112,7 @@ header:
  - "be/src/util/sse2neo.h"
  - "be/src/util/sse2neon.h"
  - "be/src/util/utf8_check.cpp"
+ - "be/src/storage/index/inverted/analyzer/kuromoji/dict/darts.h"
  - "be/src/pch/*"
  - "be/test/data"
  - "be/test/expected_result"

LICENSE.txt

@@ -369,6 +369,14 @@ be/src/olap/skiplist.h : BSD-style license
 
 --------------------------------------------------------------------------------
 
+be/src/storage/index/inverted/analyzer/kuromoji/dict/darts.h : 2-Clause BSD license (darts-clone)
+
+ Copyright (c) 2008-2014 Susumu Yata. All rights reserved.
+ Vendored from https://github.com/s-yata/darts-clone
+ Full text: dist/licenses/LICENSE-darts-clone.txt
+
+--------------------------------------------------------------------------------
+
 be/src/util/murmur_hash3.h
 be/src/util/murmur_hash3.cpp
 

NOTICE.txt

@@ -73,6 +73,9 @@ This software includes third party software subject to the following copyrights:
 - Netty Reactive Streams - https://github.com/playframework/netty-reactive-streams
 - Jackson-core - https://github.com/FasterXML/jackson-core
 - Jackson-dataformat-cbor - https://github.com/FasterXML/jackson-dataformats-binary
+- Darts-clone (double-array trie) - Copyright 2008-2014 Susumu Yata - https://github.com/s-yata/darts-clone (BSD 2-clause)
+- mecab-ipadic (IPADIC) Japanese morphological dictionary - Copyright 2000-2003 Nara Institute of Science and Technology (NAIST) - licensed under NAIST-2003 (BSD-style); the kuromoji analyzer bundles the UTF-8 form from https://github.com/lindera/mecab-ipadic (content of mecab-ipadic-2.7.0-20070801).
+- Apache Lucene - https://github.com/apache/lucene (Apache-2.0): the kuromoji Japanese analyzer under be/src/storage/index/inverted/analyzer/kuromoji is an independent C++ implementation modeled on Lucene's kuromoji analyzer (JapaneseTokenizer), including its search-mode compound-decomposition cost model.
 
 The licenses for these third party components are included in LICENSE.txt
 

be/CMakeLists.txt

@@ -374,6 +374,26 @@ install(DIRECTORY
  ${BASE_DIR}/dict/pinyin
  DESTINATION ${OUTPUT_DIR}/dict)
 
+# Japanese kuromoji dictionary. A real build always requires it the
+# binary files. However, Unit-test builds (MAKE_TEST=ON) skip it entirely as they don't
+# package for real use.
+if (NOT MAKE_TEST)
+ install(CODE "
+ foreach(_kmj_f
+ \"${BASE_DIR}/dict/kuromoji/system.bin\"
+ \"${BASE_DIR}/dict/kuromoji/matrix.bin\"
+ \"${BASE_DIR}/dict/kuromoji/chardef.bin\"
+ \"${BASE_DIR}/dict/kuromoji/unkdict.bin\")
+ if(NOT EXISTS \"\${_kmj_f}\")
+ message(FATAL_ERROR \"kuromoji dictionary file \${_kmj_f} is missing; build the 'kuromoji_dict' target (stage mecab-ipadic via thirdparty) before packaging.\")
+ endif()
+ endforeach()
+ ")
+ install(DIRECTORY
+ ${BASE_DIR}/dict/kuromoji
+ DESTINATION ${OUTPUT_DIR}/dict)
+endif()
+
 # Check if functions are supported in this platform. All flags will generated
 # in gensrc/build/common/env_config.h.
 # You can check funcion here which depends on platform. Don't forget add this
@@ -1099,6 +1119,49 @@ if (BUILD_META_TOOL OR BUILD_INDEX_TOOL)
  add_subdirectory(${SRC_DIR}/tools)
 endif()
 
+if (NOT MAKE_TEST)
+ # Offline generator: compiles the UTF-8 mecab-ipadic source into binary files.
+ add_executable(kuromoji_build_dict EXCLUDE_FROM_ALL ${SRC_DIR}/tools/kuromoji_build_dict.cpp)
+ target_include_directories(kuromoji_build_dict PRIVATE ${PROJECT_SOURCE_DIR}/..)
+ pch_reuse(kuromoji_build_dict)
+ set_target_properties(kuromoji_build_dict PROPERTIES ENABLE_EXPORTS 1)
+ if (COMPILER_CLANG)
+ target_compile_options(kuromoji_build_dict PRIVATE
+ -Wno-implicit-int-conversion
+ -Wno-shorten-64-to-32)
+ endif()
+ target_link_libraries(kuromoji_build_dict ${DORIS_LINK_LIBS})
+
+ set(KUROMOJI_IPADIC_SRC "${THIRDPARTY_DIR}/share/mecab-ipadic-2.7.0-20250920"
+ CACHE PATH "UTF-8 mecab-ipadic source directory used to generate the kuromoji dictionary")
+ set(KUROMOJI_DICT_OUT "${BASE_DIR}/dict/kuromoji")
+ file(GLOB KUROMOJI_IPADIC_SRC_FILES CONFIGURE_DEPENDS
+ "${KUROMOJI_IPADIC_SRC}/*.csv"
+ "${KUROMOJI_IPADIC_SRC}/*.def")
+ get_filename_component(KUROMOJI_LIBJVM_DIR "${LIB_JVM}" DIRECTORY)
+ set(KUROMOJI_DYLD_PATH "${KUROMOJI_LIBJVM_DIR}")
+ set(KUROMOJI_LD_PATH "${KUROMOJI_LIBJVM_DIR}")
+ if (NOT "$ENV{DYLD_LIBRARY_PATH}" STREQUAL "")
+ set(KUROMOJI_DYLD_PATH "${KUROMOJI_LIBJVM_DIR}:$ENV{DYLD_LIBRARY_PATH}")
+ endif()
+ if (NOT "$ENV{LD_LIBRARY_PATH}" STREQUAL "")
+ set(KUROMOJI_LD_PATH "${KUROMOJI_LIBJVM_DIR}:$ENV{LD_LIBRARY_PATH}")
+ endif()
+ add_custom_command(
+ OUTPUT "${KUROMOJI_DICT_OUT}/system.bin" "${KUROMOJI_DICT_OUT}/matrix.bin"
+ "${KUROMOJI_DICT_OUT}/chardef.bin" "${KUROMOJI_DICT_OUT}/unkdict.bin"
+ COMMAND ${CMAKE_COMMAND} -E make_directory "${KUROMOJI_DICT_OUT}"
+ COMMAND ${CMAKE_COMMAND} -E env
+ "DYLD_LIBRARY_PATH=${KUROMOJI_DYLD_PATH}"
+ "LD_LIBRARY_PATH=${KUROMOJI_LD_PATH}"
+ $<TARGET_FILE:kuromoji_build_dict> "${KUROMOJI_IPADIC_SRC}" "${KUROMOJI_DICT_OUT}"
+ DEPENDS kuromoji_build_dict ${KUROMOJI_IPADIC_SRC_FILES}
+ COMMENT "Generating kuromoji IPADIC dictionary from ${KUROMOJI_IPADIC_SRC}"
+ VERBATIM)
+ # ALL so a real build generates the dictionary before the install step runs.
+ add_custom_target(kuromoji_dict ALL DEPENDS "${KUROMOJI_DICT_OUT}/system.bin")
+endif()
+
 option(BUILD_FILE_CACHE_MICROBENCH_TOOL "Build file cache mirobench Tool" OFF)
 if (BUILD_FILE_CACHE_MICROBENCH_TOOL)
  add_subdirectory(${SRC_DIR}/io/tools)

be/dict/kuromoji/README.md

@@ -0,0 +1,60 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# Kuromoji (Japanese) dictionary
+
+This directory holds the compiled IPADIC dictionary consumed at runtime by the
+`kuromoji` inverted-index analyzer (`KuromojiAnalyzer` → `KuromojiDictionary`):
+
+- `system.bin` — surface→word Darts trie + word entries + feature blob
+- `matrix.bin` — connection-cost matrix (1316×1316)
+- `chardef.bin` — character-category map + per-category flags
+- `unkdict.bin` — unknown-word entries per category
+
+These `*.bin` files are **generated** (not committed; see `.gitignore`). The
+runtime resolves them at `${inverted_index_dict_path}/kuromoji`
+(default `${DORIS_HOME}/dict/kuromoji`); `be/CMakeLists.txt` installs this
+directory into the BE package.
+
+## How it's (re)generated
+
+Source: the UTF-8 IPADIC from <https://github.com/lindera/mecab-ipadic>
+(tag `2.7.0-20250920`) — the original `mecab-ipadic-2.7.0-20070801` lexicon
+converted to UTF-8 (license: NAIST-2003, see `dist/licenses/LICENSE-ipadic.txt`).
+
+A normal BE build (`sh build.sh`) generates these `*.bin` automatically: the
+`kuromoji_dict` target is part of `ALL` and the `install` rule then ships this
+directory. The target is defined only for real (`MAKE_TEST=OFF`) builds, not for
+the unit-test tree.
+
+To regenerate manually:
+
+```bash
+# 1. thirdparty fetches + stages the UTF-8 IPADIC source into
+# ${DORIS_THIRDPARTY}/installed/share/mecab-ipadic-2.7.0-20250920
+sh thirdparty/build-thirdparty.sh mecab_ipadic
+
+# 2. run the target in a real (non-test) build tree, e.g. the one sh build.sh
+# creates under be/build_<BUILD_TYPE> (build_Release by default)
+ninja -C be/build_Release kuromoji_dict
+```
+
+Override the source dir with `-DKUROMOJI_IPADIC_SRC=<path>` at CMake configure
+time. (The tool can also be run directly:
+`kuromoji_build_dict <utf8_ipadic_src_dir> be/dict/kuromoji`.)

be/src/common/config.cpp

@@ -1420,6 +1420,8 @@ DEFINE_mInt64(snii_forced_spill_min_arena_bytes, "67108864");
 DEFINE_mInt32(snii_spill_max_run_files_per_buffer, "64");
 // dict path for chinese analyzer
 DEFINE_String(inverted_index_dict_path, "${DORIS_HOME}/dict");
+// The kuromoji (Japanese) analyzer
+DEFINE_mBool(enable_kuromoji_analyzer, "false");
 DEFINE_Int32(inverted_index_read_buffer_size, "4096");
 // tree depth for bkd index
 DEFINE_Int32(max_depth_in_bkd_tree, "32");

be/src/common/config.h

@@ -1460,6 +1460,8 @@ DECLARE_mInt64(snii_forced_spill_min_arena_bytes);
 DECLARE_mInt32(snii_spill_max_run_files_per_buffer);
 // dict path for chinese analyzer
 DECLARE_String(inverted_index_dict_path);
+// The kuromoji (Japanese) analyzer
+DECLARE_mBool(enable_kuromoji_analyzer);
 DECLARE_Int32(inverted_index_read_buffer_size);
 // tree depth for bkd index
 DECLARE_Int32(max_depth_in_bkd_tree);

be/src/exprs/function/function_tokenize.cpp

@@ -159,7 +159,7 @@ Status FunctionTokenize::execute_impl(FunctionContext* /*context*/, Block& block
  if (config.parser_type == InvertedIndexParserType::PARSER_UNKNOWN) {
  return Status::Error<doris::ErrorCode::INDEX_INVALID_PARAMETERS>(
  "unsupported parser type. currently, only 'english', 'chinese', "
- "'unicode', 'icu', 'basic'and 'ik' analyzers are supported.");
+ "'unicode', 'icu', 'basic', 'ik' and 'kuromoji' analyzers are supported.");
  }
 
  // Special handling for PARSER_NONE: return original string as single token

be/src/storage/index/inverted/analyzer/analyzer.cpp

@@ -39,6 +39,7 @@
 #include "storage/index/inverted/analyzer/basic/basic_analyzer.h"
 #include "storage/index/inverted/analyzer/icu/icu_analyzer.h"
 #include "storage/index/inverted/analyzer/ik/IKAnalyzer.h"
+#include "storage/index/inverted/analyzer/kuromoji/KuromojiAnalyzer.h"
 #include "storage/index/inverted/char_filter/char_replace_char_filter_factory.h"
 #include "storage/index/inverted/token_filter/common_grams_filter.h"
 
@@ -88,7 +89,8 @@ bool InvertedIndexAnalyzer::is_builtin_analyzer(const std::string& analyzer_name
  analyzer_name == INVERTED_INDEX_PARSER_CHINESE ||
  analyzer_name == INVERTED_INDEX_PARSER_ICU ||
  analyzer_name == INVERTED_INDEX_PARSER_BASIC ||
- analyzer_name == INVERTED_INDEX_PARSER_IK;
+ analyzer_name == INVERTED_INDEX_PARSER_IK ||
+ analyzer_name == INVERTED_INDEX_PARSER_KUROMOJI;
 }
 
 AnalyzerPtr InvertedIndexAnalyzer::create_builtin_analyzer(InvertedIndexParserType parser_type,
@@ -126,6 +128,22 @@ AnalyzerPtr InvertedIndexAnalyzer::create_builtin_analyzer(InvertedIndexParserTy
  ik_analyzer->setMode(false);
  }
  analyzer = std::move(ik_analyzer);
+ } else if (parser_type == InvertedIndexParserType::PARSER_KUROMOJI) {
+ if (!config::enable_kuromoji_analyzer) {
+ throw Exception(ErrorCode::INVERTED_INDEX_ANALYZER_ERROR,
+ "kuromoji analyzer is disabled by default. Set "
+ "enable_kuromoji_analyzer=true in "
+ "be.conf (or via the BE config HTTP API) to enable it.");
+ }
+
+ std::string kuromoji_mode = parser_mode;
+ if (kuromoji_mode.empty() || kuromoji_mode == INVERTED_INDEX_PARSER_COARSE_GRANULARITY) {
+ kuromoji_mode = INVERTED_INDEX_PARSER_KUROMOJI_SEARCH;
+ }
+ auto kuromoji_analyzer = std::make_shared<KuromojiAnalyzer>();
+ kuromoji_analyzer->setMode(kuromoji_mode_from_string(kuromoji_mode));
+ kuromoji_analyzer->initDict(config::inverted_index_dict_path + "/kuromoji");
+ analyzer = std::move(kuromoji_analyzer);
  } else {
  // default
  analyzer = std::make_shared<lucene::analysis::SimpleAnalyzer<char>>();

be/src/storage/index/inverted/analyzer/kuromoji/KuromojiAnalyzer.h

@@ -0,0 +1,75 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <memory>
+#include <string>
+
+#include "common/exception.h"
+#include "common/logging.h"
+#include "storage/index/inverted/analyzer/kuromoji/KuromojiTokenizer.h"
+#include "storage/index/inverted/analyzer/kuromoji/dict/kuromoji_dictionary.h"
+
+namespace doris::segment_v2 {
+
+class KuromojiAnalyzer : public Analyzer {
+public:
+ KuromojiAnalyzer() {
+ _lowercase = true;
+ _ownReader = false;
+ }
+ ~KuromojiAnalyzer() override = default;
+
+ bool isSDocOpt() override { return true; }
+
+ // Loads (once, process-wide) the IPADIC dictionary from `dictPath`.
+ void initDict(const std::string& dictPath) override {
+ dict_ = inverted_index::kuromoji::KuromojiDictionary::get_or_load(dictPath);
+ if (dict_ == nullptr) {
+ throw doris::Exception(
+ doris::ErrorCode::INVERTED_INDEX_ANALYZER_ERROR,
+ "kuromoji dictionary could not be loaded from {}; ensure system.bin, "
+ "matrix.bin, chardef.bin and unkdict.bin are present in the BE package",
+ dictPath);
+ }
+ }
+
+ void setMode(KuromojiMode mode) { mode_ = mode; }
+
+ TokenStream* tokenStream(const TCHAR* fieldName, lucene::util::Reader* reader) override {
+ auto* tokenizer = _CLNEW KuromojiTokenizer(mode_, _lowercase, _ownReader, dict_);
+ tokenizer->reset(reader);
+ return (TokenStream*)tokenizer;
+ }
+
+ TokenStream* reusableTokenStream(const TCHAR* fieldName,
+ lucene::util::Reader* reader) override {
+ if (tokenizer_ == nullptr) {
+ tokenizer_ = std::make_unique<KuromojiTokenizer>(mode_, _lowercase, _ownReader, dict_);
+ }
+ tokenizer_->reset(reader);
+ return (TokenStream*)tokenizer_.get();
+ }
+
+private:
+ const inverted_index::kuromoji::KuromojiDictionary* dict_ {nullptr};
+ KuromojiMode mode_ {KuromojiMode::Search};
+ std::unique_ptr<KuromojiTokenizer> tokenizer_;
+};
+
+} // namespace doris::segment_v2

be/src/storage/index/inverted/analyzer/kuromoji/KuromojiMode.h

@@ -0,0 +1,48 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <string>
+
+#include "common/exception.h"
+
+namespace doris::segment_v2 {
+
+// Segmentation mode, mirroring Lucene's JapaneseTokenizer.Mode. Normal returns
+// the minimum-cost segmentation. Search additionally decomposes long compounds
+// into their shorter parts (via a length-based cost penalty) for better search
+// recall. Extended applies the Search penalty and also splits unknown
+// (out-of-vocabulary) words into per-character unigrams.
+enum class KuromojiMode { Normal, Search, Extended };
+
+inline KuromojiMode kuromoji_mode_from_string(const std::string& mode) {
+ if (mode.empty() || mode == "search") {
+ return KuromojiMode::Search;
+ }
+ if (mode == "normal") {
+ return KuromojiMode::Normal;
+ }
+ if (mode == "extended") {
+ return KuromojiMode::Extended;
+ }
+ throw doris::Exception(doris::ErrorCode::INVERTED_INDEX_ANALYZER_ERROR,
+ "Invalid kuromoji parser_mode: '{}', must be search, normal or extended",
+ mode);
+}
+
+} // namespace doris::segment_v2