diff --git a/be/src/cloud/cloud_index_change_compaction.cpp b/be/src/cloud/cloud_index_change_compaction.cpp index 55654ccc958824..03a0e02aec6647 100644 --- a/be/src/cloud/cloud_index_change_compaction.cpp +++ b/be/src/cloud/cloud_index_change_compaction.cpp @@ -83,7 +83,12 @@ Status CloudIndexChangeCompaction::prepare_compact() { _input_rowsets_total_size += rs->total_disk_size(); } - _enable_inverted_index_compaction = false; + // Non-SNII compaction cannot reuse index files when the index schema changes. SNII validates + // each destination logical index independently and rebuilds ineligible columns from raw data. + _enable_inverted_index_compaction = + _enable_inverted_index_compaction && + input_rowset->tablet_schema()->get_inverted_index_storage_format() == + InvertedIndexStorageFormatPB::SNII; LOG_INFO("[index_change]prepare CloudIndexChangeCompaction, tablet_id={}, range=[{}-{}]", _tablet->tablet_id(), _input_rowsets.front()->start_version(), _input_rowsets.back()->end_version()) diff --git a/be/src/cloud/cloud_internal_service.cpp b/be/src/cloud/cloud_internal_service.cpp index 7dca1d1517f680..4fbcca3764bb59 100644 --- a/be/src/cloud/cloud_internal_service.cpp +++ b/be/src/cloud/cloud_internal_service.cpp @@ -983,7 +983,7 @@ void record_warmup_ed_fail_index(const std::string& job_id_str, int64_t idx_size void record_warmup_ed_skipped_rowset_as_finished(RowsetMeta& rs_meta, const std::string& job_id_str) { auto schema_ptr = rs_meta.tablet_schema(); - bool has_inverted_index = schema_ptr->has_inverted_index() || schema_ptr->has_ann_index(); + bool has_inverted_index = schema_ptr->has_inverted_or_ann_index(); auto idx_version = schema_ptr->get_inverted_index_storage_format(); for (int64_t segment_id = 0; segment_id < rs_meta.num_segments(); segment_id++) { record_warmup_ed_finish_segment(job_id_str, rs_meta.segment_file_size(segment_id)); @@ -1287,7 +1287,7 @@ void CloudInternalServiceImpl::warm_up_rowset(google::protobuf::RpcController* c auto schema_ptr = rs_meta.tablet_schema(); auto idx_version = schema_ptr->get_inverted_index_storage_format(); - if (schema_ptr->has_inverted_index() || schema_ptr->has_ann_index()) { + if (schema_ptr->has_inverted_or_ann_index()) { if (idx_version == InvertedIndexStorageFormatPB::V1) { auto&& inverted_index_info = rs_meta.inverted_index_file_info(segment_id); std::unordered_map index_size_map; diff --git a/be/src/cloud/cloud_rowset_writer.cpp b/be/src/cloud/cloud_rowset_writer.cpp index 6dd107a1131821..c8999c1003a5b8 100644 --- a/be/src/cloud/cloud_rowset_writer.cpp +++ b/be/src/cloud/cloud_rowset_writer.cpp @@ -148,7 +148,7 @@ Status CloudRowsetWriter::build(RowsetSharedPtr& rowset) { } else { _rowset_meta->add_segments_file_size(seg_file_size.value()); } - if (_context.tablet_schema->has_inverted_index() || _context.tablet_schema->has_ann_index()) { + if (_context.tablet_schema->has_inverted_or_ann_index()) { if (auto idx_files_info = _idx_files.inverted_index_file_info(_segment_start_id); !idx_files_info.has_value()) [[unlikely]] { LOG(ERROR) << "expected inverted index files info, but none presents: " diff --git a/be/src/cloud/cloud_snapshot_mgr.cpp b/be/src/cloud/cloud_snapshot_mgr.cpp index 8c676aa4f05201..bcb9510d50c23a 100644 --- a/be/src/cloud/cloud_snapshot_mgr.cpp +++ b/be/src/cloud/cloud_snapshot_mgr.cpp @@ -250,8 +250,7 @@ Status CloudSnapshotMgr::_create_rowset_meta( file_mapping[src_index_file] = dst_index_file; } } else { - if (context.tablet_schema->has_inverted_index() || - context.tablet_schema->has_ann_index()) { + if (context.tablet_schema->has_inverted_or_ann_index()) { std::string src_index_file = InvertedIndexDescriptor::get_index_file_path_v2( InvertedIndexDescriptor::get_index_file_path_prefix(src_segment_file)); std::string dst_index_file = InvertedIndexDescriptor::get_index_file_path_v2( diff --git a/be/src/cloud/cloud_tablet.cpp b/be/src/cloud/cloud_tablet.cpp index 940e46331248ef..2cedfdf56e269f 100644 --- a/be/src/cloud/cloud_tablet.cpp +++ b/be/src/cloud/cloud_tablet.cpp @@ -1968,7 +1968,7 @@ void CloudTablet::_add_rowsets_directly(std::vector& rowsets, expiration_time); } } else { - if (schema_ptr->has_inverted_index() || schema_ptr->has_ann_index()) { + if (schema_ptr->has_inverted_or_ann_index()) { auto&& inverted_index_info = rowset_meta->inverted_index_file_info(seg_id); int64_t idx_size = 0; if (inverted_index_info.has_index_size()) { diff --git a/be/src/cloud/cloud_warm_up_manager.cpp b/be/src/cloud/cloud_warm_up_manager.cpp index 4590d96fb4ca80..408049bfa04099 100644 --- a/be/src/cloud/cloud_warm_up_manager.cpp +++ b/be/src/cloud/cloud_warm_up_manager.cpp @@ -380,7 +380,7 @@ void CloudWarmUpManager::handle_jobs() { tablet_id); } } else { - if (schema_ptr->has_inverted_index() || schema_ptr->has_ann_index()) { + if (schema_ptr->has_inverted_or_ann_index()) { auto idx_path = storage_resource.value()->remote_idx_v2_path(*rs, seg_id); file_size = idx_file_info.has_index_size() ? idx_file_info.index_size() @@ -853,7 +853,7 @@ Status CloudWarmUpManager::_do_warm_up_rowset(RowsetMeta& rs_meta, int64_t table g_file_cache_event_driven_warm_up_requested_segment_size << seg_size; g_warmup_ed_requested_segment_size.put({job_id_str}, seg_size); - if (schema_ptr->has_inverted_index() || schema_ptr->has_ann_index()) { + if (schema_ptr->has_inverted_or_ann_index()) { if (idx_version == InvertedIndexStorageFormatPB::V1) { auto&& inverted_index_info = rs_meta.inverted_index_file_info(cast_set(segment_id)); diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index d27cf607a36b24..bafbfa2cf1b6e1 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -30,6 +30,7 @@ #include // IWYU pragma: keep #include #include +#include #include #include #include @@ -1296,6 +1297,25 @@ DEFINE_Int32(inverted_index_query_cache_shards, "256"); // inverted index match bitmap cache size DEFINE_String(inverted_index_query_cache_limit, "10%"); +namespace { + +bool valid_common_grams_cost_ratio(int32_t value) { + return value >= 0 && value <= 100; +} + +bool valid_common_grams_verify_factor(int32_t value) { + return value >= 0; +} + +} // namespace + +DEFINE_mBool(enable_common_grams_query_plan, "false"); +DEFINE_mBool(enable_common_grams_index_build, "true"); +DEFINE_mInt32(common_grams_plan_cost_ratio_percent, "85"); +DEFINE_Validator(common_grams_plan_cost_ratio_percent, valid_common_grams_cost_ratio); +DEFINE_mInt32(common_grams_position_verify_factor, "0"); +DEFINE_Validator(common_grams_position_verify_factor, valid_common_grams_verify_factor); + // condition cache limit DEFINE_Int16(condition_cache_limit, "512"); @@ -1309,6 +1329,69 @@ DEFINE_mDouble(inverted_index_ram_buffer_size, "512"); // -1 indicates not working. // Normally we should not change this, it's useful for testing. DEFINE_mInt32(inverted_index_max_buffered_docs, "-1"); +// G16-c: whether plain positions-tier (non-scoring) SNII indexes lay out freq +// regions. Freq bytes serve ONLY BM25 scoring, which the Doris integration +// does not reach yet (scoring_query has no production caller), so the default +// drops them (textbench: -2.2 GB index). Scoring-config indexes always write +// freq regardless. Applies at segment build (write side only); existing +// segments keep whatever layout they were written with (self-describing). +DEFINE_mBool(snii_positions_index_write_freq, "false"); +// G16-h: zstd levels for the SNII dict-block compression and the .prx window +// auto mode. Level 9 (vs the historical 3) shrinks the two largest compressed +// sections -- textbench: index -457 MB (0.918x -> 0.891x V3) -- for an import +// CPU cost inside the run-to-run variance band; zstd decode speed does not +// depend on the level, and warm/cold benches measured no query change. +// Write side only; segments self-describe their compression. +// Default 3 since the all-level-3 evaluation (2026-07-11, 4 corpora): vs +// level 9 the settled index grows only +0.6%..+6.3% (whole table +// +0.3%..+1.9%) while import index CPU drops 17-24% and full-compaction CPU +// 8-24%; settled cold-query latency is unchanged (interleaved A/B). The +// delta+varint-encoded payloads are high-entropy, so level 9's extra search +// buys almost no ratio. Raise only for size-critical deployments. +DEFINE_mInt32(snii_dict_block_zstd_level, "3"); +DEFINE_mInt32(snii_prx_zstd_level, "3"); +// Patch C prx tiering: zstd level for the prx region of DIRECT-LOAD segments +// only (stream/broker load, see IndexColumnWriter::set_direct_load). Inert at +// the defaults (both levels 3); it exists for size-critical deployments that +// RAISE snii_prx_zstd_level (e.g. 9) and still want cheap loads: compaction +// rewrites every segment at snii_prx_zstd_level, so SETTLED data (and the +// cold-query path over it) is unaffected by the load tier -- measured -290s +// (httplogs) / -204s (agentlogs) of import index CPU at 3 vs 9. Same clamp +// [3, 19]. Read at index flush like snii_prx_zstd_level (a mid-load change +// lands on in-flight segments); the direct-load BIT itself is captured once. +DEFINE_mInt32(snii_prx_zstd_level_direct_load, "3"); +// G16-d: target SNII dict block size in bytes; 0 uses the format default +// (64 KiB). Larger blocks compress better under the per-block zstd (the dict +// is the dominant physical section on high-cardinality corpora) at the cost +// of a larger fetch+decompress unit per cold dict-block miss. Write side +// only; the block size is self-described by the on-disk directory. +DEFINE_mInt32(snii_target_dict_block_bytes, "0"); +// SNII's index-build share of the process memory limit, as a percent -- the +// index-build analogue of load_process_max_memory_limit_percent. Once live SNII +// index-build memory crosses this share, the largest reclaimable posting arenas +// are asked to spill early. Derived from the process limit rather than an +// absolute number so it scales with the BE. Only the RECLAIMABLE population +// counts against it: index-merge compaction charges the same observation +// tracker but registers no spillable writer, so its bytes are excluded from the +// comparison (its own hard reservation cap bounds them instead). +// +// 0 disables SNII's OWN share trigger; the process-level backstops (system +// available memory below its warning water mark, process usage above the soft +// limit) still apply. The share is deliberately well below those backstops so +// SNII sheds its own memory before the global valve -- the global valve trips +// late by design and would be a worse trigger than none. The derived share is +// floored at four times inverted_index_ram_buffer_size so a small BE is not +// permanently over it the moment two writers exist. +DEFINE_mInt32(snii_index_build_max_memory_limit_percent, "10"); +// Minimum reclaimable posting-arena bytes before a G09 forced spill is honored +// (and before a writer is eligible as a spill victim): forced spills reclaim +// ONLY the arena, so smaller triggers cut tiny runs for near-zero relief. +// Default 64 MiB. +DEFINE_mInt64(snii_forced_spill_min_arena_bytes, "67108864"); +// Max spill-run files one SNII writer accumulates before its runs are +// merge-compacted into one (bounds the k-way merge fan-in and its open fds; +// every run is held open for the whole merge). 0 = uncapped. Default 64. +DEFINE_mInt32(snii_spill_max_run_files_per_buffer, "64"); // dict path for chinese analyzer DEFINE_String(inverted_index_dict_path, "${DORIS_HOME}/dict"); DEFINE_Int32(inverted_index_read_buffer_size, "4096"); @@ -2270,6 +2353,33 @@ bool init(const char* conf_file, bool fill_conf_map, bool must_exist, bool set_t return Status::OK(); \ } +namespace { + +// UPDATE_FIELD invokes registered validators before assigning the candidate value. Validate the two +// mutable planner coefficients explicitly so their startup and runtime constraints stay identical. +Status validate_common_grams_runtime_config(const std::string& field, const std::string& value) { + bool (*validator)(int32_t) = nullptr; + if (field == "common_grams_plan_cost_ratio_percent") { + validator = valid_common_grams_cost_ratio; + } else if (field == "common_grams_position_verify_factor") { + validator = valid_common_grams_verify_factor; + } else { + return Status::OK(); + } + + int32_t candidate = 0; + if (!convert(value, candidate)) { + return Status::OK(); + } + if (!validator(candidate)) { + return Status::Error("validate {}={} failed", field, + candidate); + } + return Status::OK(); +} + +} // namespace + // write config to be_custom.conf // the caller need to make sure that the given config is valid Status persist_config(const std::string& field, const std::string& value) { @@ -2300,6 +2410,8 @@ Status set_config(const std::string& field, const std::string& value, bool need_ "'{}' is not support to modify", field); } + RETURN_IF_ERROR(validate_common_grams_runtime_config(field, value)); + UPDATE_FIELD(it->second, value, bool, need_persist); UPDATE_FIELD(it->second, value, int16_t, need_persist); UPDATE_FIELD(it->second, value, int32_t, need_persist); diff --git a/be/src/common/config.h b/be/src/common/config.h index 470d88819703de..1f53b060c3e88f 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -1363,6 +1363,15 @@ DECLARE_Int32(inverted_index_query_cache_shards); // inverted index match bitmap cache size DECLARE_String(inverted_index_query_cache_limit); +// Process-wide emergency switch for CommonGrams query plans. +DECLARE_mBool(enable_common_grams_query_plan); +// Build-only CommonGrams kill switch. Logical index writers snapshot it at construction; changing +// it affects only writers created after the transition and never changes query/cache semantics. +DECLARE_mBool(enable_common_grams_index_build); +// Release-calibrated query-planner coefficients. Both remain mutable for controlled recalibration. +DECLARE_mInt32(common_grams_plan_cost_ratio_percent); +DECLARE_mInt32(common_grams_position_verify_factor); + // condition cache limit DECLARE_Int16(condition_cache_limit); @@ -1374,6 +1383,68 @@ DECLARE_Int32(ann_index_result_cache_stale_sweep_time_sec); // inverted index DECLARE_mDouble(inverted_index_ram_buffer_size); DECLARE_mInt32(inverted_index_max_buffered_docs); +// G16-c: whether plain positions-tier (non-scoring) SNII indexes lay out freq +// regions. Freq serves ONLY BM25 scoring (no production caller yet), so the +// default (false) drops the layout; scoring-config indexes always keep freq. +// Write-side only; segments are self-describing either way. +DECLARE_mBool(snii_positions_index_write_freq); +// G16-h: zstd levels for SNII dict blocks / prx windows. Default 3 (the +// all-level-3 evaluation showed level 9 buys <=6.3% index size for 17-24% +// import CPU; see the DEFINEs in config.cpp). +DECLARE_mInt32(snii_dict_block_zstd_level); +DECLARE_mInt32(snii_prx_zstd_level); +// Patch C: prx zstd level for DIRECT-LOAD segments only (default 3, cheaper +// import); compaction rewrites at snii_prx_zstd_level so settled segments are +// unaffected. Full contract at the DEFINE in config.cpp. +DECLARE_mInt32(snii_prx_zstd_level_direct_load); +// G16-d: target SNII dict block size in bytes; 0 = format default (64 KiB). +// Bigger blocks -> better per-block zstd on the dict region, larger cold +// fetch+decompress unit per dict-block miss. Write side only. +DECLARE_mInt32(snii_target_dict_block_bytes); +// PROCESS-WIDE share for SNII index-build RAM, as a PERCENT of the process +// memory limit -- the index-build analogue of +// load_process_max_memory_limit_percent. The per-writer +// inverted_index_ram_buffer_size is a reclaimable-buffer spill threshold, not a +// hard cap on persistent vocabulary bytes: a concurrent load keeps (tablets x +// concurrency) writers alive at once, none of which may reach that threshold, +// while their SUM can still be large. Once live SNII index-build memory +// (ingestion plus index-merge compaction) crosses this share, the writers +// holding the largest reclaimable posting arenas are asked to spill early +// (async-safe advisory requests, honored on each writer's own thread; output +// stays byte-identical). Read at every decision, so a change takes effect +// immediately for writers that are already running. +// +// 0 disables SNII's own share trigger; the process-level backstops (system +// available memory below its warning water mark, process usage above the soft +// limit) still apply. +// +// FLOORED AGAINST inverted_index_ram_buffer_size: the share is never less than +// four writers' worth of the per-writer spill threshold. A smaller share would +// put a small BE permanently over it as soon as two writers exist -- unrelievable +// back-pressure rather than a limit -- because the per-writer threshold is what +// one writer may hold before it spills on its own. +DECLARE_mInt32(snii_index_build_max_memory_limit_percent); +// G09 forced-spill floor: minimum reclaimable posting-arena bytes a SNII +// writer must hold before a process-wide forced-spill request is honored, and +// before the global limiter selects it as a spill victim. A forced spill +// reclaims ONLY the posting arena -- the persistent vocab / pair-map +// structures survive it -- so honoring below a real floor degenerates into a +// storm of tiny runs whenever the memory over the share is dominated by +// persistent bytes (each run then costs a file, a sort and a merge-fd for +// near-zero memory relief). THIS FLOOR, not any judgement about whether the +// overage is reachable, is what bounds forced spilling: it caps the cost at one +// >= floor-sized run per floor of arena growth per writer. Forced spilling +// therefore reclaims SPILLABLE memory only, never persistent memory. +// Default 64 MiB. +DECLARE_mInt64(snii_forced_spill_min_arena_bytes); +// G09 run-file cap: maximum spill-run files one SNII writer may accumulate; +// on the next spill past the cap, the existing runs are merge-compacted into +// a single run first (term stream unchanged). Bounds the final k-way merge's +// fan-in and, decisively, its simultaneously-open file descriptors -- every +// run of a buffer is reopened and held open for the whole merge, so unbounded +// run counts across ~100 concurrent writers can exhaust the BE nofile rlimit +// ("Too many open files" at run reopen). 0 disables the cap. Default 64. +DECLARE_mInt32(snii_spill_max_run_files_per_buffer); // dict path for chinese analyzer DECLARE_String(inverted_index_dict_path); DECLARE_Int32(inverted_index_read_buffer_size); diff --git a/be/src/common/metrics/doris_metrics.h b/be/src/common/metrics/doris_metrics.h index 2973435305a693..d50354f24d548d 100644 --- a/be/src/common/metrics/doris_metrics.h +++ b/be/src/common/metrics/doris_metrics.h @@ -232,6 +232,7 @@ class DorisMetrics { UIntGauge* load_mem_consumption = nullptr; UIntGauge* load_channel_mem_consumption = nullptr; UIntGauge* memtable_memory_limiter_mem_consumption = nullptr; + UIntGauge* snii_index_build_mem_consumption = nullptr; UIntGauge* query_mem_consumption = nullptr; UIntGauge* schema_change_mem_consumption = nullptr; UIntGauge* storage_migration_mem_consumption = nullptr; diff --git a/be/src/common/status.h b/be/src/common/status.h index d29b66459a3832..37ff6917b4f567 100644 --- a/be/src/common/status.h +++ b/be/src/common/status.h @@ -299,6 +299,7 @@ namespace ErrorCode { E(INVERTED_INDEX_COMPACTION_ERROR, -6010, false); \ E(INVERTED_INDEX_ANALYZER_ERROR, -6011, false); \ E(INVERTED_INDEX_FILE_CORRUPTED, -6012, false); \ + E(INVERTED_INDEX_SNII_NOT_FOUND, -6013, false); \ E(KEY_NOT_FOUND, -7000, false); \ E(KEY_ALREADY_EXISTS, -7001, false); \ E(ENTRY_NOT_FOUND, -7002, false); \ diff --git a/be/src/exec/operator/olap_scan_operator.cpp b/be/src/exec/operator/olap_scan_operator.cpp index dda268de829252..3c8462c923dd3f 100644 --- a/be/src/exec/operator/olap_scan_operator.cpp +++ b/be/src/exec/operator/olap_scan_operator.cpp @@ -264,6 +264,10 @@ Status OlapScanLocalState::_init_profile() { ADD_COUNTER_WITH_LEVEL(_segment_profile, "InvertedIndexQueryCacheHit", TUnit::UNIT, 1); _inverted_index_query_cache_miss_counter = ADD_COUNTER_WITH_LEVEL(_segment_profile, "InvertedIndexQueryCacheMiss", TUnit::UNIT, 1); + _inverted_index_query_cache_lookup_counter = ADD_COUNTER_WITH_LEVEL( + _segment_profile, "InvertedIndexQueryCacheLookup", TUnit::UNIT, 1); + _inverted_index_query_cache_insert_counter = ADD_COUNTER_WITH_LEVEL( + _segment_profile, "InvertedIndexQueryCacheInsert", TUnit::UNIT, 1); _inverted_index_query_timer = ADD_TIMER_WITH_LEVEL(_segment_profile, "InvertedIndexQueryTime", 1); _inverted_index_query_null_bitmap_timer = @@ -348,6 +352,8 @@ Status OlapScanLocalState::_init_profile() { _index_filter_profile = std::make_unique("IndexFilter"); _scanner_profile->add_child(_index_filter_profile.get(), true, nullptr); + _snii_prx_profile_counters.initialize(_index_filter_profile.get()); + _snii_phrase_profile_counters.initialize(_index_filter_profile.get()); /* SegmentIterator: - AnnIndexLoadCosts: 102.262us diff --git a/be/src/exec/operator/olap_scan_operator.h b/be/src/exec/operator/olap_scan_operator.h index 9f7ddca05274a0..38aae4c0ce0d91 100644 --- a/be/src/exec/operator/olap_scan_operator.h +++ b/be/src/exec/operator/olap_scan_operator.h @@ -28,6 +28,7 @@ #include "exec/operator/operator.h" #include "exec/operator/scan_operator.h" #include "runtime/runtime_profile.h" +#include "storage/index/snii/snii_prx_profile.h" #include "storage/olap_scan_common.h" #include "storage/tablet/tablet_reader.h" @@ -149,6 +150,8 @@ class OlapScanLocalState final : public ScanLocalState { std::unique_ptr _segment_profile; std::unique_ptr _index_filter_profile; + snii::SniiPrxRuntimeProfileCounters _snii_prx_profile_counters; + snii::SniiPhraseRuntimeProfileCounters _snii_phrase_profile_counters; RuntimeProfile::Counter* _tablet_counter = nullptr; RuntimeProfile::Counter* _key_range_counter = nullptr; @@ -243,6 +246,8 @@ class OlapScanLocalState final : public ScanLocalState { RuntimeProfile::Counter* _inverted_index_query_null_bitmap_timer = nullptr; RuntimeProfile::Counter* _inverted_index_query_cache_hit_counter = nullptr; RuntimeProfile::Counter* _inverted_index_query_cache_miss_counter = nullptr; + RuntimeProfile::Counter* _inverted_index_query_cache_lookup_counter = nullptr; + RuntimeProfile::Counter* _inverted_index_query_cache_insert_counter = nullptr; RuntimeProfile::Counter* _inverted_index_query_timer = nullptr; RuntimeProfile::Counter* _inverted_index_query_bitmap_copy_timer = nullptr; RuntimeProfile::Counter* _inverted_index_searcher_open_timer = nullptr; diff --git a/be/src/exec/scan/olap_scanner.cpp b/be/src/exec/scan/olap_scanner.cpp index 9ffdb94e149c91..91aa9685f0c824 100644 --- a/be/src/exec/scan/olap_scanner.cpp +++ b/be/src/exec/scan/olap_scanner.cpp @@ -57,6 +57,7 @@ #include "storage/binlog.h" #include "storage/id_manager.h" #include "storage/index/inverted/inverted_index_profile.h" +#include "storage/index/inverted/similarity/collection_statistics.h" #include "storage/iterator/block_reader.h" #include "storage/olap_common.h" #include "storage/olap_tuple.h" @@ -156,7 +157,10 @@ static bool has_file_cache_statistics(const io::FileCacheStatistics& stats) { stats.inverted_index_bytes_read_from_remote != 0 || stats.inverted_index_bytes_read_from_peer != 0 || stats.inverted_index_local_io_timer != 0 || stats.inverted_index_remote_io_timer != 0 || - stats.inverted_index_peer_io_timer != 0 || stats.inverted_index_io_timer != 0; + stats.inverted_index_peer_io_timer != 0 || stats.inverted_index_io_timer != 0 || + stats.inverted_index_request_bytes != 0 || stats.inverted_index_read_bytes != 0 || + stats.inverted_index_range_read_count != 0 || + stats.inverted_index_serial_read_rounds != 0; } io::IOContext build_score_runtime_collection_io_context(RuntimeState* state, ReaderType reader_type, @@ -1001,6 +1005,10 @@ void OlapScanner::_collect_profile_before_close() { stats.inverted_index_query_cache_hit); COUNTER_UPDATE(local_state->_inverted_index_query_cache_miss_counter, stats.inverted_index_query_cache_miss); + COUNTER_UPDATE(local_state->_inverted_index_query_cache_lookup_counter, + stats.inverted_index_query_cache_lookup); + COUNTER_UPDATE(local_state->_inverted_index_query_cache_insert_counter, + stats.inverted_index_query_cache_insert); COUNTER_UPDATE(local_state->_inverted_index_query_timer, stats.inverted_index_query_timer); COUNTER_UPDATE(local_state->_inverted_index_query_null_bitmap_timer, stats.inverted_index_query_null_bitmap_timer); @@ -1023,6 +1031,8 @@ void OlapScanner::_collect_profile_before_close() { COUNTER_UPDATE(local_state->_inverted_index_analyzer_timer, stats.inverted_index_analyzer_timer); COUNTER_UPDATE(local_state->_inverted_index_lookup_timer, stats.inverted_index_lookup_timer); + local_state->_snii_prx_profile_counters.update(stats); + local_state->_snii_phrase_profile_counters.update(stats); COUNTER_UPDATE(local_state->_variant_scan_sparse_column_timer, stats.variant_scan_sparse_column_timer_ns); COUNTER_UPDATE(local_state->_variant_scan_sparse_column_bytes, diff --git a/be/src/exprs/function/function_search.cpp b/be/src/exprs/function/function_search.cpp index e87aff95ee075c..f4e9758a600b98 100644 --- a/be/src/exprs/function/function_search.cpp +++ b/be/src/exprs/function/function_search.cpp @@ -44,6 +44,7 @@ #include "runtime/runtime_profile.h" #include "storage/index/index_file_reader.h" #include "storage/index/index_query_context.h" +#include "storage/index/index_reader_helper.h" #include "storage/index/inverted/analyzer/analyzer.h" #include "storage/index/inverted/inverted_index_compound_reader.h" #include "storage/index/inverted/inverted_index_iterator.h" @@ -60,6 +61,7 @@ #include "storage/index/inverted/query_v2/phrase_query/multi_phrase_query.h" #include "storage/index/inverted/query_v2/phrase_query/phrase_query.h" #include "storage/index/inverted/query_v2/regexp_query/regexp_query.h" +#include "storage/index/inverted/query_v2/scored_bit_set_query/scored_bit_set_query.h" #include "storage/index/inverted/query_v2/term_query/term_query.h" #include "storage/index/inverted/query_v2/wildcard_query/wildcard_query.h" #include "storage/index/inverted/util/string_helper.h" @@ -115,6 +117,50 @@ static std::string extract_segment_prefix( return ""; } +static void collect_referenced_fields(const TSearchClause& clause, + std::unordered_set* fields) { + DORIS_CHECK(fields != nullptr); + if (clause.__isset.field_name && !clause.field_name.empty()) { + fields->insert(clause.field_name); + } + for (const auto& child : clause.children) { + collect_referenced_fields(child, fields); + } +} + +static bool referenced_fields_contain_snii_reader( + const TSearchClause& root, + const std::unordered_map& iterators) { + std::unordered_set referenced_fields; + collect_referenced_fields(root, &referenced_fields); + for (const auto& field_name : referenced_fields) { + auto iterator_it = iterators.find(field_name); + if (iterator_it == iterators.end()) { + continue; + } + auto* inv_iter = dynamic_cast(iterator_it->second); + if (inv_iter == nullptr) { + continue; + } + for (auto type : {InvertedIndexReaderType::FULLTEXT, InvertedIndexReaderType::STRING_TYPE, + InvertedIndexReaderType::BKD}) { + IndexReaderType reader_type = type; + auto reader = inv_iter->get_reader(reader_type); + if (reader == nullptr) { + continue; + } + auto inv_reader = std::dynamic_pointer_cast(reader); + DORIS_CHECK(inv_reader != nullptr); + auto file_reader = inv_reader->get_index_file_reader(); + DORIS_CHECK(file_reader != nullptr); + if (file_reader->get_storage_format() == InvertedIndexStorageFormatPB::SNII) { + return true; + } + } + } + return false; +} + namespace { bool is_nested_group_search_supported() { @@ -216,6 +262,14 @@ InvertedIndexQueryType direct_index_query_type_for_clause(const std::string& cla return InvertedIndexQueryType::UNKNOWN_QUERY; } +std::string normalize_wildcard_pattern(const std::string& value, + const std::map& index_properties) { + const bool has_parser = + inverted_index::InvertedIndexAnalyzer::should_analyzer(index_properties); + const std::string lowercase_setting = get_parser_lowercase_from_properties(index_properties); + return has_parser && lowercase_setting == INVERTED_INDEX_PARSER_TRUE ? to_lower(value) : value; +} + } // namespace Status FunctionSearch::execute_impl(FunctionContext* /*context*/, Block& /*block*/, @@ -274,13 +328,16 @@ Status FunctionSearch::evaluate_inverted_index_with_search_param( OlapReaderStatistics* outer_stats = index_query_context ? index_query_context->stats : nullptr; SCOPED_RAW_TIMER(outer_stats ? &outer_stats->inverted_index_query_timer : &query_timer_dummy); - const bool need_similarity_score = - index_query_context && index_query_context->collection_similarity; - // DSL result cache only stores bitmap/null bitmap. It does not store BM25 scores, // so score() queries must execute scorers to populate CollectionSimilarity. - auto* dsl_cache = (enable_cache && !need_similarity_score) ? InvertedIndexQueryCache::instance() - : nullptr; + const bool enable_scoring = + index_query_context != nullptr && index_query_context->collection_similarity != nullptr; + // Also bypass the DSL cache when any referenced field is served by an SNII reader. + auto* dsl_cache = + enable_cache && !enable_scoring && + !referenced_fields_contain_snii_reader(search_param.root, iterators) + ? InvertedIndexQueryCache::instance() + : nullptr; std::string seg_prefix; std::string dsl_sig; InvertedIndexQueryCache::CacheKey dsl_cache_key; @@ -394,11 +451,9 @@ Status FunctionSearch::evaluate_inverted_index_with_search_param( query_v2::QueryExecutionContext exec_ctx = build_variant_search_query_execution_context(num_rows, resolver, &null_resolver); - bool enable_scoring = false; bool is_asc = false; size_t top_k = 0; if (index_query_context) { - enable_scoring = index_query_context->collection_similarity != nullptr; is_asc = index_query_context->is_asc; top_k = index_query_context->query_limit; } @@ -734,6 +789,168 @@ Status FunctionSearch::build_leaf_query(const TSearchClause& clause, *binding_key = binding.binding_key; } + if (binding.use_snii_native_reader()) { + DORIS_CHECK(binding.inverted_reader != nullptr); + // The SNII reader answers a clause directly from a query type: it tokenizes the value + // itself and owns the matching operator, so unlike the CLucene path below there is no + // query tree to assemble here. RANGE and LIST reach the same TERM fallback the CLucene + // path uses, because neither is implemented there either. + InvertedIndexQueryType snii_query_type = (clause_type == "RANGE" || clause_type == "LIST") + ? InvertedIndexQueryType::EQUAL_QUERY + : clause_type_to_query_type(clause_type); + + if (clause_type == "TERM") { + // minimum_should_match ("at least N of M terms") has no SNII query type: the reader + // only knows AND-all (MATCH_ALL_QUERY) or OR-all (EQUAL_QUERY/MATCH_ANY_QUERY) of the + // terms it tokenizes internally, never a partial threshold. The CLucene TERM handling + // below builds an OccurBooleanQuery for this, but only when the value actually + // tokenizes to MORE THAN ONE term: its `term_infos.size() == 1` short-circuit returns + // a plain TermQuery first and never looks at msm, because msm is meaningless when + // there is only one term to select "at least N of" from. SNII must draw the line in + // the same place: tokenize the value up front and refuse only the genuinely + // unsupported multi-token case, instead of rejecting every analysed field the instant + // msm is set regardless of how many tokens the value produces. + if (minimum_should_match > 0 && + inverted_index::InvertedIndexAnalyzer::should_analyzer(binding.index_properties)) { + auto term_infos = inverted_index::InvertedIndexAnalyzer::get_analyse_result( + value, binding.index_properties, + inverted_index::AnalysisPurpose::kPlainQuery); + if (term_infos.size() > 1) { + return Status::NotSupported( + "SNII native SEARCH does not support minimum_should_match for TERM " + "clauses (got {})", + minimum_should_match); + } + if (term_infos.empty()) { + // Zero tokens (e.g. an all-stopword or empty value): mirror the CLucene TERM + // handling's own `term_infos.empty()` -> empty BitSetQuery short-circuit + // below, instead of falling through to binding.inverted_reader->query() + // below. That reader's own empty-term_infos short-circuit + // (snii_index_reader.cpp:722-731) only returns an empty bitmap when + // is_match_query() is true (inverted_index_query_type.h:99-106). TERM's + // "or"/default-operator query type is EQUAL_QUERY, which is not in that list, + // so it would instead return Status::Error. The + // "and" operator's MATCH_ALL_QUERY (assigned below) IS in that list, but this + // branch returns unconditionally before that assignment ever runs -- same as + // the CLucene reference path, which is unconditional too -- so the outcome + // here does not depend on default_operator. For SEARCH() specifically that + // error would be a hard query failure, not a slower row-scan fallback: + // VSearchExpr has no downgrade path of its own (vsearch.cpp:234/265), and + // prevent_search_row_fallback (vsearch.cpp:169-183) does not admit + // INVERTED_INDEX_NO_TERMS as a status that may fall back either. + return finish_leaf_query( + std::make_shared(roaring::Roaring())); + } + // size() == 1: msm is meaningless for a single token, matching V3 -- fall through. + } + // default_operator selects how a multi-token TERM value combines: "and" requires + // every term (MATCH_ALL_QUERY), "or" -- the default -- requires any term, which is + // already snii_query_type above (EQUAL_QUERY). A single-token value is unaffected + // either way, since the reader special-cases terms.size() == 1 for both query types. + if (default_operator == "and") { + snii_query_type = InvertedIndexQueryType::MATCH_ALL_QUERY; + } + } else if (clause_type == "PREFIX" && + !inverted_index::InvertedIndexAnalyzer::should_analyzer( + binding.index_properties)) { + // FE keeps the trailing '*' in the PREFIX value unstripped (SearchDslParser.java). + // On an analysed field the tokenizer drops it, leaving a clean single prefix term, + // so the default clause_type_to_query_type mapping (MATCH_PHRASE_PREFIX_QUERY) is + // correct as-is. On a keyword (non-analysed) field the whole string -- '*' included + // -- becomes one literal term (InvertedIndexAnalyzer::get_analyse_result), so + // MATCH_PHRASE_PREFIX_QUERY would search for a term that can never exist. Route + // those to WILDCARD_QUERY instead, exactly like the CLucene path's + // WildcardQuery(value) for PREFIX (function_search.cpp:1075-1076): the reader + // forwards a WILDCARD_QUERY value unanalysed, so the trailing '*' works the same way. + snii_query_type = InvertedIndexQueryType::WILDCARD_QUERY; + } + + // The SNII reader has no way to hand a clause's BM25 values back through query(): it + // publishes them into whatever CollectionSimilarity the context carries. If that were the + // query's own similarity, the reader and the collector -- which also calls collect() with + // the scorer's score, and whose collect() accumulates rather than overwrites -- would both + // write, so every document would end up with its BM25 plus the scorer's constant, and the + // early top-k path would rank by that constant instead of by relevance. Redirect the + // reader into a private sink and let the score reach the collector the normal way, through + // the scorer built below. + // + // The sink is created exactly when the reader is going to score, which is the same pair + // of conditions the reader itself uses (its actual_similarity): the caller supplied a + // similarity at all, and this query type scores on this index. Deciding it up front beats + // inferring it afterwards from "the sink came back non-empty", and it keeps every clause + // that cannot be scored -- wildcard, regexp, EQUAL_QUERY, and the WILDCARD "*" shortcut + // that never calls the reader -- from allocating a CollectionSimilarity that reserves + // 1024 entries in its constructor. + const bool reader_will_score = + context->collection_similarity != nullptr && + IndexReaderHelper::is_need_similarity_score( + snii_query_type, &binding.inverted_reader->get_index_meta()); + std::shared_ptr reader_context = context; + std::shared_ptr score_sink; + if (reader_will_score) { + score_sink = std::make_shared(); + reader_context = std::make_shared(*context); + reader_context->collection_similarity = score_sink; + } + + auto data_bitmap = std::make_shared(); + if (clause_type == "WILDCARD" && value == "*") { + data_bitmap->addRange(0, num_rows); + } else { + // Wildcard patterns carry the analyzer's lower_case semantics; every other clause + // passes its value through untouched, since the reader analyses it. + std::string pattern = + clause_type == "WILDCARD" + ? normalize_wildcard_pattern(value, binding.index_properties) + : value; + Field query_value = Field::create_field(pattern); + RETURN_IF_ERROR(binding.inverted_reader->query(reader_context, + binding.stored_field_name, query_value, + snii_query_type, data_bitmap, nullptr)); + // Reply-direction fields land on the copy the reader was given, so they have to be + // folded back. Today this is unreachable rather than load-bearing: the count-only + // fast path requires the scan to have no score runtime, while the similarity that + // creates the copy exists only when there IS one, so the two never coexist. It stays + // because the copy must remain honest if that ever changes -- a dropped reply would + // be silent. + if (reader_context != context) { + context->merge_reader_outputs(*reader_context); + } + // Restore the pre-normalization value for WILDCARD so the trace still shows what the + // caller actually asked for, not just what was sent to the reader. + std::string log_suffix = + clause_type == "WILDCARD" ? (" (original='" + value + "')") : std::string(); + VLOG_DEBUG << "search: SNII clause processed, type=" << clause_type + << ", field=" << field_name << ", value='" << pattern << "'" << log_suffix; + } + + auto null_bitmap = std::make_shared(); + if (binding.inverted_reader->has_null()) { + segment_v2::InvertedIndexQueryCacheHandle null_bitmap_cache_handle; + RETURN_IF_ERROR(binding.inverted_reader->read_null_bitmap( + context, &null_bitmap_cache_handle, nullptr)); + auto cached_null_bitmap = null_bitmap_cache_handle.get_bitmap(); + DORIS_CHECK(cached_null_bitmap != nullptr); + null_bitmap = std::move(cached_null_bitmap); + } + *data_bitmap -= *null_bitmap; + // Only clauses the reader actually scored get a scored query. The rest -- wildcard, + // regexp, and any query type is_need_similarity_score rejects -- have no per-document + // value to expose, and keep the constant-score BitSetQuery that the CLucene path also + // gives its unscored leaves, so their contribution to a compound query is unchanged. + // The emptiness check is not redundant with reader_will_score above: that gate cannot see + // the analysed term count, and the reader publishes nothing for shapes such as a + // MATCH_PHRASE_PREFIX_QUERY that tokenizes to a single term. + auto sink_scores = score_sink != nullptr ? score_sink->release_scores() : ScoreMap {}; + if (!sink_scores.empty()) { + return finish_leaf_query(std::make_shared( + std::move(data_bitmap), std::move(null_bitmap), + std::make_shared(std::move(sink_scores)))); + } + return finish_leaf_query(std::make_shared(std::move(data_bitmap), + std::move(null_bitmap))); + } + if (binding.use_direct_index_reader()) { auto direct_query_type = direct_index_query_type_for_clause(clause_type); if (direct_query_type == InvertedIndexQueryType::UNKNOWN_QUERY) { @@ -800,7 +1017,8 @@ Status FunctionSearch::build_leaf_query(const TSearchClause& clause, std::vector term_infos = inverted_index::InvertedIndexAnalyzer::get_analyse_result( - value, binding.index_properties); + value, binding.index_properties, + inverted_index::AnalysisPurpose::kPlainQuery); if (term_infos.empty()) { LOG(WARNING) << "search: No terms found after tokenization for TERM query, field=" << field_name << ", value='" << value @@ -864,7 +1082,8 @@ Status FunctionSearch::build_leaf_query(const TSearchClause& clause, std::vector term_infos = inverted_index::InvertedIndexAnalyzer::get_analyse_result( - value, binding.index_properties); + value, binding.index_properties, + inverted_index::AnalysisPurpose::kPlainQuery); if (term_infos.empty()) { LOG(WARNING) << "search: No terms found after tokenization for PHRASE query, field=" << field_name << ", value='" << value @@ -879,8 +1098,7 @@ Status FunctionSearch::build_leaf_query(const TSearchClause& clause, const auto& term_info = phrase_term_infos[0]; if (term_info.is_single_term()) { std::wstring term_wstr = StringHelper::to_wstring(term_info.get_single_term()); - return finish_leaf_query( - std::make_shared(context, field_wstr, term_wstr)); + return finish_leaf_query(make_term_query(term_wstr)); } else { auto builder = create_operator_boolean_query_builder(query_v2::OperatorType::OP_OR); @@ -922,7 +1140,8 @@ Status FunctionSearch::build_leaf_query(const TSearchClause& clause, std::vector term_infos = inverted_index::InvertedIndexAnalyzer::get_analyse_result( - value, binding.index_properties); + value, binding.index_properties, + inverted_index::AnalysisPurpose::kPlainQuery); if (term_infos.empty()) { LOG(WARNING) << "search: tokenization yielded no terms for clause '" << clause_type << "', field=" << field_name << ", returning empty BitSetQuery"; @@ -994,8 +1213,7 @@ Status FunctionSearch::build_leaf_query(const TSearchClause& clause, binding.index_properties); std::string lowercase_setting = get_parser_lowercase_from_properties(binding.index_properties); - bool should_lowercase = has_parser && (lowercase_setting == INVERTED_INDEX_PARSER_TRUE); - std::string pattern = should_lowercase ? to_lower(value) : value; + std::string pattern = normalize_wildcard_pattern(value, binding.index_properties); VLOG_DEBUG << "search: WILDCARD clause processed, field=" << field_name << ", pattern='" << pattern << "' (original='" << value << "', has_parser=" << has_parser << ", lower_case=" << lowercase_setting << ")"; diff --git a/be/src/exprs/function/function_tokenize.cpp b/be/src/exprs/function/function_tokenize.cpp index a3d616d635763c..b7aae728cfe504 100644 --- a/be/src/exprs/function/function_tokenize.cpp +++ b/be/src/exprs/function/function_tokenize.cpp @@ -187,7 +187,7 @@ Status FunctionTokenize::execute_impl(FunctionContext* /*context*/, Block& block try { analyzer_holder = doris::segment_v2::inverted_index::InvertedIndexAnalyzer::create_analyzer( - &config); + &config, AnalysisPurpose::kPlainQuery); } catch (CLuceneError& e) { return Status::Error( "inverted index create analyzer failed: {}", e.what()); diff --git a/be/src/exprs/function/match.cpp b/be/src/exprs/function/match.cpp index a280dd035e25b4..28ee41974ada50 100644 --- a/be/src/exprs/function/match.cpp +++ b/be/src/exprs/function/match.cpp @@ -34,7 +34,7 @@ const InvertedIndexAnalyzerCtx* get_match_analyzer_ctx(FunctionContext* context) if (context == nullptr) { return nullptr; } - auto* analyzer_ctx = reinterpret_cast( + const auto* analyzer_ctx = reinterpret_cast( context->get_function_state(FunctionContext::THREAD_LOCAL)); if (analyzer_ctx == nullptr) { analyzer_ctx = reinterpret_cast( @@ -87,6 +87,8 @@ Status FunctionMatchBase::evaluate_inverted_index( param.query_type = get_query_type_from_fn_name(); param.num_rows = num_rows; param.roaring = std::make_shared(); + segment_v2::InvertedIndexQueryCacheHandle null_bitmap_cache_handle; + param.null_bitmap_cache_handle = &null_bitmap_cache_handle; param.analyzer_ctx = analyzer_ctx; if (is_string_type(param_type)) { RETURN_IF_ERROR(iter->read_from_index(¶m)); @@ -95,11 +97,11 @@ Status FunctionMatchBase::evaluate_inverted_index( "invalid params type for FunctionMatchBase::evaluate_inverted_index {}", param_type); } - std::shared_ptr null_bitmap = std::make_shared(); - if (iter->has_null()) { - segment_v2::InvertedIndexQueryCacheHandle null_bitmap_cache_handle; - RETURN_IF_ERROR(iter->read_null_bitmap(&null_bitmap_cache_handle)); - null_bitmap = null_bitmap_cache_handle.get_bitmap(); + std::shared_ptr null_bitmap = null_bitmap_cache_handle.get_bitmap(); + if (null_bitmap == nullptr) { + // query_with_null_bitmap leaves the handle empty only when the selected reader proves that + // the index has no null rows. + null_bitmap = std::make_shared(); } segment_v2::InvertedIndexResultBitmap result(param.roaring, null_bitmap); bitmap_result = result; @@ -122,7 +124,7 @@ Status FunctionMatchBase::execute_impl(FunctionContext* context, Block& block, std::string column_name = block.get_by_position(arguments[0]).name; VLOG_DEBUG << "begin to execute match directly, column_name=" << column_name << ", match_query_str=" << match_query_str; - auto* analyzer_ctx = get_match_analyzer_ctx(context); + const auto* analyzer_ctx = get_match_analyzer_ctx(context); const ColumnPtr source_col = block.get_by_position(arguments[0]).column->convert_to_full_column_if_const(); const auto* values = check_and_get_column(source_col.get()); @@ -197,10 +199,8 @@ std::vector FunctionMatchBase::analyse_query_str_token( VLOG_DEBUG << "begin to run " << get_name() << ", parser_type: " << inverted_index_parser_type_to_string(analyzer_ctx->parser_type); - // Decision is based on parser_type (from index properties): - // - PARSER_NONE: no tokenization (keyword/exact match) - // - Other parsers: tokenize using the analyzer - if (!analyzer_ctx->should_tokenize()) { + // Raw execution is valid only when neither a named analyzer nor a builtin parser is active. + if (!analyzer_ctx->requires_analysis()) { // Keyword index: all strings (including empty) are valid tokens for exact match. // Empty string is a valid value in keyword index and should be matchable. query_tokens.emplace_back(match_query_str); @@ -233,15 +233,14 @@ inline std::vector FunctionMatchBase::analyse_data_token( return data_tokens; } - // Determine tokenization strategy based on parser_type - const bool should_tokenize = - analyzer_ctx->should_tokenize() && analyzer_ctx->analyzer != nullptr; + const bool requires_analysis = + analyzer_ctx->requires_analysis() && analyzer_ctx->analyzer != nullptr; if (array_offsets) { for (auto next_src_array_offset = (*array_offsets)[current_block_row_idx]; current_src_array_offset < next_src_array_offset; ++current_src_array_offset) { const auto& str_ref = string_col->get_data_at(current_src_array_offset); - if (!should_tokenize) { + if (!requires_analysis) { data_tokens.emplace_back(str_ref.to_string()); continue; } @@ -254,7 +253,7 @@ inline std::vector FunctionMatchBase::analyse_data_token( } } else { const auto& str_ref = string_col->get_data_at(current_block_row_idx); - if (!should_tokenize) { + if (!requires_analysis) { data_tokens.emplace_back(str_ref.to_string()); } else { auto reader = doris::segment_v2::inverted_index::InvertedIndexAnalyzer::create_reader( diff --git a/be/src/exprs/function/variant_inverted_index_search.cpp b/be/src/exprs/function/variant_inverted_index_search.cpp index cf3fc0505188c6..1e5d8f244898d4 100644 --- a/be/src/exprs/function/variant_inverted_index_search.cpp +++ b/be/src/exprs/function/variant_inverted_index_search.cpp @@ -36,6 +36,7 @@ #include "storage/index/inverted/inverted_index_compound_reader.h" #include "storage/index/inverted/inverted_index_parser.h" #include "storage/index/inverted/inverted_index_searcher.h" +#include "storage/index/inverted/inverted_index_selector.h" #include "storage/index/inverted/query_v2/bit_set_query/bit_set_scorer.h" #include "storage/index/inverted/query_v2/doc_set.h" #include "storage/index/inverted/query_v2/scorer.h" @@ -138,18 +139,18 @@ Status FieldReaderResolver::resolve(const std::string& field_name, InvertedIndexQueryType effective_query_type = query_type; const auto& column_type = data_it->second.second; - const bool is_text_field = - column_type != nullptr && is_string_type(column_type->get_storage_field_type()); + const bool is_text_field = column_type != nullptr && + is_string_type(get_inverted_index_leaf_field_type(column_type)); auto fb_it = _field_binding_map.find(field_name); std::string analyzer_key; - if (is_text_field && is_variant_sub && fb_it != _field_binding_map.end() && - fb_it->second->__isset.index_properties && !fb_it->second->index_properties.empty()) { + if (is_text_field && effective_query_type != InvertedIndexQueryType::EQUAL_QUERY && + fb_it != _field_binding_map.end() && fb_it->second->__isset.index_properties && + !fb_it->second->index_properties.empty()) { analyzer_key = normalize_analyzer_key( build_analyzer_key_from_properties(fb_it->second->index_properties)); if (inverted_index::InvertedIndexAnalyzer::should_analyzer( fb_it->second->index_properties) && - (effective_query_type == InvertedIndexQueryType::EQUAL_QUERY || - effective_query_type == InvertedIndexQueryType::WILDCARD_QUERY)) { + effective_query_type == InvertedIndexQueryType::WILDCARD_QUERY) { effective_query_type = InvertedIndexQueryType::MATCH_ANY_QUERY; } } @@ -200,12 +201,7 @@ Status FieldReaderResolver::resolve(const std::string& field_name, resolved.inverted_reader = inverted_reader; resolved.binding_key = binding_key; resolved.state = SearchFieldBindingState::BOUND; - if (fb_it != _field_binding_map.end() && fb_it->second->__isset.index_properties && - !fb_it->second->index_properties.empty()) { - resolved.index_properties = fb_it->second->index_properties; - } else { - resolved.index_properties = inverted_reader->get_index_properties(); - } + resolved.index_properties = inverted_reader->get_index_properties(); resolved.analyzer_key = normalize_analyzer_key(build_analyzer_key_from_properties(resolved.index_properties)); @@ -225,6 +221,7 @@ Status FieldReaderResolver::resolve(const std::string& field_name, } if (inverted_reader->type() == InvertedIndexReaderType::BKD) { + resolved.execution_mode = SearchFieldExecutionMode::DIRECT_INDEX; _cache.emplace(binding_key, resolved); if (is_variant_sub) { bool index_file_exists = false; @@ -249,6 +246,29 @@ Status FieldReaderResolver::resolve(const std::string& field_name, return Status::OK(); } + if (index_file_reader->get_storage_format() == InvertedIndexStorageFormatPB::SNII) { + resolved.execution_mode = SearchFieldExecutionMode::SNII_NATIVE; + _cache.emplace(binding_key, resolved); + if (is_variant_sub) { + add_search_binding_diagnostic( + _context, + fmt::format("[VariantSearchBinding] phase=field_resolve " + "result=selected_snii_native logical_field={} stored_field={} " + "query_type={} effective_query_type={} index_id={} suffix={} " + "reader_type={} analyzer_key={} index_file={}", + field_name, stored_field_name, query_type_to_string(query_type), + query_type_to_string(effective_query_type), + inverted_reader->get_index_id(), + inverted_reader->get_index_meta().get_index_suffix(), + reader_type_to_string(inverted_reader->type()), + resolved.analyzer_key, + index_file_reader->get_index_file_path( + &inverted_reader->get_index_meta()))); + } + *binding = resolved; + return Status::OK(); + } + auto index_file_key = index_file_reader->get_index_file_cache_key(&inverted_reader->get_index_meta()); InvertedIndexSearcherCache::CacheKey searcher_cache_key(index_file_key); @@ -291,7 +311,6 @@ Status FieldReaderResolver::resolve(const std::string& field_name, index_file_reader->init(config::inverted_index_read_buffer_size, _context->io_ctx)); auto directory = DORIS_TRY( index_file_reader->open(&inverted_reader->get_index_meta(), _context->io_ctx)); - auto index_searcher_builder = DORIS_TRY( IndexSearcherBuilder::create_index_searcher_builder(inverted_reader->type())); auto searcher_result = @@ -334,6 +353,7 @@ Status FieldReaderResolver::resolve(const std::string& field_name, _searcher_cache_handles.push_back(std::move(searcher_cache_handle)); resolved.lucene_reader = reader_holder; + resolved.execution_mode = SearchFieldExecutionMode::CLUCENE; _binding_readers[binding_key] = reader_holder; _field_readers[resolved.stored_field_wstr] = reader_holder; _readers.emplace_back(reader_holder); diff --git a/be/src/exprs/function/variant_inverted_index_search.h b/be/src/exprs/function/variant_inverted_index_search.h index 973c9c8c826c55..d03dbc486d8e18 100644 --- a/be/src/exprs/function/variant_inverted_index_search.h +++ b/be/src/exprs/function/variant_inverted_index_search.h @@ -68,6 +68,13 @@ enum class SearchFieldBindingState { MISSING_IN_SEGMENT, }; +enum class SearchFieldExecutionMode { + UNBOUND, + CLUCENE, + DIRECT_INDEX, + SNII_NATIVE, +}; + struct FieldReaderBinding { std::string logical_field_name; std::string stored_field_name; @@ -80,13 +87,17 @@ struct FieldReaderBinding { std::string binding_key; std::string analyzer_key; SearchFieldBindingState state = SearchFieldBindingState::MISSING_IN_SEGMENT; + SearchFieldExecutionMode execution_mode = SearchFieldExecutionMode::UNBOUND; bool is_bound() const { return state == SearchFieldBindingState::BOUND || inverted_reader != nullptr || lucene_reader != nullptr; } bool use_direct_index_reader() const { - return is_bound() && inverted_reader != nullptr && lucene_reader == nullptr; + return is_bound() && execution_mode == SearchFieldExecutionMode::DIRECT_INDEX; + } + bool use_snii_native_reader() const { + return is_bound() && execution_mode == SearchFieldExecutionMode::SNII_NATIVE; } }; diff --git a/be/src/exprs/vcompound_pred.h b/be/src/exprs/vcompound_pred.h index 8a153ef8e88615..d44346ee114cbd 100644 --- a/be/src/exprs/vcompound_pred.h +++ b/be/src/exprs/vcompound_pred.h @@ -44,6 +44,15 @@ inline std::string compound_operator_to_string(TExprOpcode::type op) { } } +inline bool inverted_index_status_allows_row_fallback(const Status& status) { + DORIS_CHECK(!status.ok()); + return status.is() || + status.is() || + status.is() || + status.is() || + status.is(); +} + class VCompoundPred : public VectorizedFnCall { ENABLE_FACTORY_CREATOR(VCompoundPred); @@ -272,6 +281,9 @@ class VCompoundPred : public VectorizedFnCall { !st.ok()) { LOG(ERROR) << "expr:" << child->expr_name() << " evaluate_inverted_index error:" << st.to_string(); + if (!inverted_index_status_allows_row_fallback(st)) { + return st; + } all_pass = false; continue; } @@ -301,6 +313,9 @@ class VCompoundPred : public VectorizedFnCall { !st.ok()) { LOG(ERROR) << "expr:" << child->expr_name() << " evaluate_inverted_index error:" << st.to_string(); + if (!inverted_index_status_allows_row_fallback(st)) { + return st; + } all_pass = false; continue; } diff --git a/be/src/exprs/vexpr.h b/be/src/exprs/vexpr.h index c0b99ab7a8f4dc..de5ad018d3be09 100644 --- a/be/src/exprs/vexpr.h +++ b/be/src/exprs/vexpr.h @@ -267,6 +267,10 @@ class VExpr { return empty; } + [[nodiscard]] virtual const InvertedIndexAnalyzerCtx* query_analyzer_ctx() const { + return nullptr; + } + Status _evaluate_inverted_index(VExprContext* context, const FunctionBasePtr& function, uint32_t segment_num_rows); diff --git a/be/src/exprs/vmatch_predicate.cpp b/be/src/exprs/vmatch_predicate.cpp index cf4f87e45d3b7e..d7a593f2ba4832 100644 --- a/be/src/exprs/vmatch_predicate.cpp +++ b/be/src/exprs/vmatch_predicate.cpp @@ -57,11 +57,12 @@ namespace doris { using namespace doris::segment_v2; VMatchPredicate::VMatchPredicate(const TExprNode& node) : VExpr(node) { - // Step 1: Create configuration (stack-allocated temporary, follows SRP) + const auto resolved = AnalyzerConfigParser::parse(node.match_predicate.analyzer_name, + node.match_predicate.parser_type); + InvertedIndexAnalyzerConfig config; - config.analyzer_name = node.match_predicate.analyzer_name; - config.parser_type = - get_inverted_index_parser_type_from_string(node.match_predicate.parser_type); + config.analyzer_name = resolved.provider_name; + config.parser_type = resolved.parser_type; config.parser_mode = node.match_predicate.parser_mode; config.char_filter_map = node.match_predicate.char_filter_map; if (node.match_predicate.parser_lowercase) { @@ -73,18 +74,20 @@ VMatchPredicate::VMatchPredicate(const TExprNode& node) : VExpr(node) { { config.lower_case = ""; }) config.stop_words = node.match_predicate.parser_stopwords; - // Step 2: Use config to create analyzer (factory method). - // Always create analyzer based on parser_type for slow path (tables without index). - // For index path, FullTextIndexReader will check analyzer_name to decide whether - // to use this analyzer or fallback to index's own analyzer. - _analyzer = inverted_index::InvertedIndexAnalyzer::create_analyzer(&config); - - // Step 3: Create runtime context (only extract runtime-needed info) _analyzer_ctx = std::make_shared(); - _analyzer_ctx->analyzer_name = config.analyzer_name; - _analyzer_ctx->parser_type = config.parser_type; + _analyzer_ctx->analyzer_key = resolved.analyzer_key; + _analyzer_ctx->analyzer_name = resolved.provider_name; + _analyzer_ctx->parser_type = resolved.parser_type; + + if (_analyzer_ctx->requires_analysis()) { + _analyzer_provider = + inverted_index::InvertedIndexAnalyzer::create_analyzer_provider(&config); + _analyzer = _analyzer_provider->get_analyzer(inverted_index::AnalysisPurpose::kPlainQuery); + } + _analyzer_ctx->char_filter_map = std::move(config.char_filter_map); _analyzer_ctx->analyzer = _analyzer; + _analyzer_ctx->analyzer_provider = _analyzer_provider; } VMatchPredicate::~VMatchPredicate() = default; @@ -152,7 +155,7 @@ Status VMatchPredicate::evaluate_inverted_index(VExprContext* context, uint32_t } const std::string& VMatchPredicate::get_analyzer_key() const { - return _analyzer_ctx->analyzer_name; + return _analyzer_ctx->analyzer_key; } Status VMatchPredicate::execute_column_impl(VExprContext* context, const Block* block, @@ -222,4 +225,4 @@ std::string VMatchPredicate::debug_string() const { return out.str(); } -} // namespace doris \ No newline at end of file +} // namespace doris diff --git a/be/src/exprs/vmatch_predicate.h b/be/src/exprs/vmatch_predicate.h index e36b695e3cfb1a..4b6792c44ed88d 100644 --- a/be/src/exprs/vmatch_predicate.h +++ b/be/src/exprs/vmatch_predicate.h @@ -57,6 +57,9 @@ class VMatchPredicate final : public VExpr { const std::string& expr_name() const override; const std::string& function_name() const; [[nodiscard]] const std::string& get_analyzer_key() const override; + [[nodiscard]] const InvertedIndexAnalyzerCtx* query_analyzer_ctx() const override { + return _analyzer_ctx.get(); + } std::string debug_string() const override; @@ -69,6 +72,7 @@ class VMatchPredicate final : public VExpr { // Lifecycle management: holds ownership of the analyzer std::shared_ptr _analyzer; + segment_v2::inverted_index::AnalyzerProviderPtr _analyzer_provider; // Runtime context: holds raw pointer to analyzer and necessary runtime info InvertedIndexAnalyzerCtxSPtr _analyzer_ctx; diff --git a/be/src/exprs/vsearch.cpp b/be/src/exprs/vsearch.cpp index ecaef392db87b2..75a3b48c67fad2 100644 --- a/be/src/exprs/vsearch.cpp +++ b/be/src/exprs/vsearch.cpp @@ -166,6 +166,24 @@ Status collect_search_inputs(const VSearchExpr& expr, VExprContext* context, return Status::OK(); } +bool search_status_allows_row_fallback(const Status& status) { + DORIS_CHECK(!status.ok()); + return status.is() || + status.is() || + status.is() || + status.is() || + status.is(); +} + +Status prevent_search_row_fallback(Status status) { + DORIS_CHECK(!status.ok()); + if (!search_status_allows_row_fallback(status)) { + return status; + } + return Status::Error( + "SEARCH cannot fall back to row execution: {}", status.to_string()); +} + } // namespace VSearchExpr::VSearchExpr(const TExprNode& node) : VExpr(node) { @@ -202,7 +220,7 @@ Status VSearchExpr::execute_column_impl(VExprContext* context, const Block* bloc Status VSearchExpr::evaluate_inverted_index(VExprContext* context, uint32_t segment_num_rows) { if (_search_param.original_dsl.empty()) { - return Status::InvalidArgument("search DSL is empty"); + return prevent_search_row_fallback(Status::InvalidArgument("search DSL is empty")); } auto index_context = context->get_index_context(); @@ -212,7 +230,9 @@ Status VSearchExpr::evaluate_inverted_index(VExprContext* context, uint32_t segm } SearchInputBundle bundle; - RETURN_IF_ERROR(collect_search_inputs(*this, context, &bundle)); + if (auto status = collect_search_inputs(*this, context, &bundle); !status.ok()) { + return prevent_search_row_fallback(std::move(status)); + } VLOG_DEBUG << "VSearchExpr: bundle.iterators.size()=" << bundle.iterators.size(); @@ -242,7 +262,7 @@ Status VSearchExpr::evaluate_inverted_index(VExprContext* context, uint32_t segm if (!status.ok()) { LOG(WARNING) << "VSearchExpr: Function evaluation failed: " << status.to_string(); - return status; + return prevent_search_row_fallback(std::move(status)); } index_context->set_index_result_for_expr(this, result_bitmap); diff --git a/be/src/io/cache/block_file_cache_profile.cpp b/be/src/io/cache/block_file_cache_profile.cpp index 7aed33ccaa6e07..40a031f7bcb658 100644 --- a/be/src/io/cache/block_file_cache_profile.cpp +++ b/be/src/io/cache/block_file_cache_profile.cpp @@ -17,6 +17,7 @@ #include "io/cache/block_file_cache_profile.h" +#include #include #include #include @@ -106,12 +107,17 @@ FileCacheStatistics diff_file_cache_statistics(const FileCacheStatistics& curren SUBTRACT_FIELD(inverted_index_bytes_read_from_local); SUBTRACT_FIELD(inverted_index_bytes_read_from_remote); SUBTRACT_FIELD(inverted_index_bytes_read_from_peer); + SUBTRACT_FIELD(inverted_index_remote_physical_read_bytes); + SUBTRACT_FIELD(inverted_index_bytes_write_into_cache); SUBTRACT_FIELD(inverted_index_local_io_timer); SUBTRACT_FIELD(inverted_index_remote_io_timer); SUBTRACT_FIELD(inverted_index_peer_io_timer); SUBTRACT_FIELD(inverted_index_io_timer); SUBTRACT_FIELD(inverted_index_write_cache_io_timer); - SUBTRACT_FIELD(inverted_index_bytes_write_into_cache); + SUBTRACT_FIELD(inverted_index_request_bytes); + SUBTRACT_FIELD(inverted_index_read_bytes); + SUBTRACT_FIELD(inverted_index_range_read_count); + SUBTRACT_FIELD(inverted_index_serial_read_rounds); SUBTRACT_FIELD(segment_footer_index_num_local_io_total); SUBTRACT_FIELD(segment_footer_index_num_remote_io_total); @@ -193,6 +199,10 @@ FileCacheProfileReporter::FileCacheProfileReporter(RuntimeProfile* profile, profile, "InvertedIndexBytesScannedFromRemote", TUnit::BYTES, cache_profile, 1); inverted_index_bytes_scanned_from_peer = ADD_CHILD_COUNTER_WITH_LEVEL( profile, "InvertedIndexBytesScannedFromPeer", TUnit::BYTES, cache_profile, 1); + inverted_index_remote_physical_read_bytes = ADD_CHILD_COUNTER_WITH_LEVEL( + profile, "InvertedIndexRemotePhysicalReadBytes", TUnit::BYTES, cache_profile, 1); + inverted_index_bytes_write_into_cache = ADD_CHILD_COUNTER_WITH_LEVEL( + profile, "InvertedIndexBytesWriteIntoCache", TUnit::BYTES, cache_profile, 1); inverted_index_local_io_timer = ADD_CHILD_TIMER_WITH_LEVEL(profile, "InvertedIndexLocalIOUseTimer", cache_profile, 1); inverted_index_remote_io_timer = @@ -203,8 +213,14 @@ FileCacheProfileReporter::FileCacheProfileReporter(RuntimeProfile* profile, ADD_CHILD_TIMER_WITH_LEVEL(profile, "InvertedIndexIOTimer", cache_profile, 1); inverted_index_write_cache_io_timer = ADD_CHILD_TIMER_WITH_LEVEL( profile, "InvertedIndexWriteCacheIOUseTimer", cache_profile, 1); - inverted_index_bytes_write_into_cache = ADD_CHILD_COUNTER_WITH_LEVEL( - profile, "InvertedIndexBytesWriteIntoCache", TUnit::BYTES, cache_profile, 1); + inverted_index_request_bytes = ADD_CHILD_COUNTER_WITH_LEVEL( + profile, "InvertedIndexRequestBytes", TUnit::BYTES, cache_profile, 1); + inverted_index_read_bytes = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "InvertedIndexReadBytes", + TUnit::BYTES, cache_profile, 1); + inverted_index_range_read_count = ADD_CHILD_COUNTER_WITH_LEVEL( + profile, "InvertedIndexRangeReadCount", TUnit::UNIT, cache_profile, 1); + inverted_index_serial_read_rounds = ADD_CHILD_COUNTER_WITH_LEVEL( + profile, "InvertedIndexSerialReadRounds", TUnit::UNIT, cache_profile, 1); segment_footer_index_num_local_io_total = ADD_CHILD_COUNTER_WITH_LEVEL( profile, "SegmentFooterIndexNumLocalIOTotal", TUnit::UNIT, cache_profile, 1); @@ -290,14 +306,16 @@ void FileCacheProfileReporter::update(const FileCacheStatistics* statistics) con statistics->inverted_index_bytes_read_from_remote); COUNTER_UPDATE(inverted_index_bytes_scanned_from_peer, statistics->inverted_index_bytes_read_from_peer); + COUNTER_UPDATE(inverted_index_remote_physical_read_bytes, + statistics->inverted_index_remote_physical_read_bytes); + COUNTER_UPDATE(inverted_index_bytes_write_into_cache, + statistics->inverted_index_bytes_write_into_cache); COUNTER_UPDATE(inverted_index_local_io_timer, statistics->inverted_index_local_io_timer); COUNTER_UPDATE(inverted_index_remote_io_timer, statistics->inverted_index_remote_io_timer); COUNTER_UPDATE(inverted_index_peer_io_timer, statistics->inverted_index_peer_io_timer); COUNTER_UPDATE(inverted_index_io_timer, statistics->inverted_index_io_timer); COUNTER_UPDATE(inverted_index_write_cache_io_timer, statistics->inverted_index_write_cache_io_timer); - COUNTER_UPDATE(inverted_index_bytes_write_into_cache, - statistics->inverted_index_bytes_write_into_cache); COUNTER_UPDATE(segment_footer_index_num_local_io_total, statistics->segment_footer_index_num_local_io_total); @@ -343,6 +361,11 @@ void FileCacheProfileReporter::update(const FileCacheStatistics* statistics) con } _profile->add_info_string("PeerCacheNodes", peer_nodes); } + COUNTER_UPDATE(inverted_index_request_bytes, statistics->inverted_index_request_bytes); + COUNTER_UPDATE(inverted_index_read_bytes, statistics->inverted_index_read_bytes); + COUNTER_UPDATE(inverted_index_range_read_count, statistics->inverted_index_range_read_count); + COUNTER_UPDATE(inverted_index_serial_read_rounds, + statistics->inverted_index_serial_read_rounds); } } // namespace doris::io diff --git a/be/src/io/cache/block_file_cache_profile.h b/be/src/io/cache/block_file_cache_profile.h index d3fd31033649c8..5e7a9e2a267f1a 100644 --- a/be/src/io/cache/block_file_cache_profile.h +++ b/be/src/io/cache/block_file_cache_profile.h @@ -19,6 +19,7 @@ #include +#include #include #include #include @@ -60,7 +61,6 @@ class FileCacheMetrics { void register_entity(); void update_metrics_callback(); -private: std::mutex _mtx; // use shared_ptr for concurrent std::shared_ptr _statistics; @@ -99,12 +99,17 @@ struct FileCacheProfileReporter { RuntimeProfile::Counter* inverted_index_bytes_scanned_from_cache = nullptr; RuntimeProfile::Counter* inverted_index_bytes_scanned_from_remote = nullptr; RuntimeProfile::Counter* inverted_index_bytes_scanned_from_peer = nullptr; + RuntimeProfile::Counter* inverted_index_remote_physical_read_bytes = nullptr; + RuntimeProfile::Counter* inverted_index_bytes_write_into_cache = nullptr; RuntimeProfile::Counter* inverted_index_local_io_timer = nullptr; RuntimeProfile::Counter* inverted_index_remote_io_timer = nullptr; RuntimeProfile::Counter* inverted_index_peer_io_timer = nullptr; RuntimeProfile::Counter* inverted_index_io_timer = nullptr; RuntimeProfile::Counter* inverted_index_write_cache_io_timer = nullptr; - RuntimeProfile::Counter* inverted_index_bytes_write_into_cache = nullptr; + RuntimeProfile::Counter* inverted_index_request_bytes = nullptr; + RuntimeProfile::Counter* inverted_index_read_bytes = nullptr; + RuntimeProfile::Counter* inverted_index_range_read_count = nullptr; + RuntimeProfile::Counter* inverted_index_serial_read_rounds = nullptr; RuntimeProfile::Counter* segment_footer_index_num_local_io_total = nullptr; RuntimeProfile::Counter* segment_footer_index_num_remote_io_total = nullptr; diff --git a/be/src/io/cache/cached_remote_file_reader.cpp b/be/src/io/cache/cached_remote_file_reader.cpp index 47fe02ee2c3e92..dcd8dde92946b3 100644 --- a/be/src/io/cache/cached_remote_file_reader.cpp +++ b/be/src/io/cache/cached_remote_file_reader.cpp @@ -325,7 +325,11 @@ Status execute_s3_read(size_t empty_start, size_t& size, std::unique_ptr s3_read_counter << 1; SCOPED_RAW_TIMER(&stats.remote_read_timer); stats.from_peer_cache = false; - return remote_file_reader->read_at(empty_start, Slice(buffer.get(), size), &size, io_ctx); + auto st = remote_file_reader->read_at(empty_start, Slice(buffer.get(), size), &size, io_ctx); + if (st.ok()) { + stats.remote_physical_read_bytes += size; + } + return st; } CloudWarmUpManager& get_warm_up_manager() { @@ -343,6 +347,10 @@ struct RaceState { Status peer_status; Status s3_status; std::unique_ptr s3_buf; + // Actual bytes fetched by the winning S3 leg; merged into the caller's + // ReadStatistics only in the winner==1 branch of collect_race_result (the + // losing S3 leg may outlive the caller's stack, so it must not touch stats). + size_t s3_read_size = 0; PeerFetchResult peer_res; std::string peer_winner_cg_id; // compute_group_id of the winning peer candidate std::string peer_winner_host; // host of the winning peer candidate @@ -466,6 +474,7 @@ void launch_s3_race(std::shared_ptr race, size_t empty_start, size_t if (st.ok() && race->winner < 0) { race->winner = 1; race->s3_buf = std::move(s3_buf); + race->s3_read_size = read_size; } race->cv.notify_all(); }; @@ -548,9 +557,11 @@ Status collect_race_result(std::shared_ptr race, size_t span_size, } return Status::OK(); } else if (race->winner == 1) { - // S3 won. + // S3 won: this was a real storage GET, so account it as physical remote IO + // exactly like the non-race download path does. buffer = std::move(race->s3_buf); stats.from_peer_cache = false; + stats.remote_physical_read_bytes += race->s3_read_size; g_peer_race_s3_win << 1; if (io_ctx != nullptr && io_ctx->file_cache_stats != nullptr) { io_ctx->file_cache_stats->num_peer_race_s3_win++; @@ -1038,6 +1049,8 @@ Status CachedRemoteFileReader::_read_remaining_blocks_from_cache( &remote_bytes_read, io_ctx)); indirect_read_bytes += read_size; source_read_breakdown.remote_bytes += remote_bytes_read; + // Self-heal fell back to a real storage GET; count it as physical remote IO. + stats.remote_physical_read_bytes += remote_bytes_read; DCHECK(remote_bytes_read == read_size); } @@ -1112,6 +1125,10 @@ Status CachedRemoteFileReader::_read_remote_only_on_cache_miss( *bytes_read = remote_bytes_read; DCHECK_EQ(*bytes_read, bytes_req); source_read_breakdown.remote_bytes += remote_bytes_read; + // This is a real storage GET, so it must count as physical remote IO just like + // the block-download path; otherwise profiles show scanned-from-remote bytes + // with zero physical reads whenever remote-only-on-miss is active. + stats.remote_physical_read_bytes += remote_bytes_read; g_read_cache_indirect_bytes << remote_bytes_read; g_read_cache_indirect_total_bytes << remote_bytes_read; return Status::OK(); @@ -1413,6 +1430,7 @@ void CachedRemoteFileReader::_update_stats(const ReadStatistics& read_stats, statis->inverted_index_remote_io_timer, statis->inverted_index_peer_io_timer, statis->inverted_index_write_cache_io_timer, statis->inverted_index_bytes_write_into_cache); + statis->inverted_index_remote_physical_read_bytes += read_stats.remote_physical_read_bytes; break; case FileCacheReadType::SEGMENT_FOOTER_INDEX: update_index_stats(statis->segment_footer_index_num_local_io_total, diff --git a/be/src/io/cache/file_cache_common.h b/be/src/io/cache/file_cache_common.h index 8b52af8d161b9a..7357f1a431acc2 100644 --- a/be/src/io/cache/file_cache_common.h +++ b/be/src/io/cache/file_cache_common.h @@ -75,6 +75,7 @@ struct ReadStatistics { int64_t bytes_read_from_local = 0; int64_t bytes_read_from_remote = 0; int64_t bytes_read_from_peer = 0; + int64_t remote_physical_read_bytes = 0; int64_t bytes_write_into_file_cache = 0; int64_t remote_read_timer = 0; int64_t peer_read_timer = 0; diff --git a/be/src/io/io_common.h b/be/src/io/io_common.h index c085f93347850f..3e71ed66c0992a 100644 --- a/be/src/io/io_common.h +++ b/be/src/io/io_common.h @@ -19,6 +19,9 @@ #include +#include +#include +#include #include #include @@ -78,12 +81,17 @@ struct FileCacheStatistics { int64_t inverted_index_bytes_read_from_local = 0; int64_t inverted_index_bytes_read_from_remote = 0; int64_t inverted_index_bytes_read_from_peer = 0; + int64_t inverted_index_remote_physical_read_bytes = 0; + int64_t inverted_index_bytes_write_into_cache = 0; int64_t inverted_index_local_io_timer = 0; int64_t inverted_index_remote_io_timer = 0; int64_t inverted_index_peer_io_timer = 0; int64_t inverted_index_io_timer = 0; int64_t inverted_index_write_cache_io_timer = 0; - int64_t inverted_index_bytes_write_into_cache = 0; + int64_t inverted_index_request_bytes = 0; + int64_t inverted_index_read_bytes = 0; + int64_t inverted_index_range_read_count = 0; + int64_t inverted_index_serial_read_rounds = 0; int64_t segment_footer_index_num_local_io_total = 0; int64_t segment_footer_index_num_remote_io_total = 0; @@ -139,12 +147,18 @@ struct FileCacheStatistics { inverted_index_bytes_read_from_local += other.inverted_index_bytes_read_from_local; inverted_index_bytes_read_from_remote += other.inverted_index_bytes_read_from_remote; inverted_index_bytes_read_from_peer += other.inverted_index_bytes_read_from_peer; + inverted_index_remote_physical_read_bytes += + other.inverted_index_remote_physical_read_bytes; inverted_index_local_io_timer += other.inverted_index_local_io_timer; inverted_index_remote_io_timer += other.inverted_index_remote_io_timer; inverted_index_peer_io_timer += other.inverted_index_peer_io_timer; inverted_index_io_timer += other.inverted_index_io_timer; inverted_index_write_cache_io_timer += other.inverted_index_write_cache_io_timer; inverted_index_bytes_write_into_cache += other.inverted_index_bytes_write_into_cache; + inverted_index_request_bytes += other.inverted_index_request_bytes; + inverted_index_read_bytes += other.inverted_index_read_bytes; + inverted_index_range_read_count += other.inverted_index_range_read_count; + inverted_index_serial_read_rounds += other.inverted_index_serial_read_rounds; segment_footer_index_num_local_io_total += other.segment_footer_index_num_local_io_total; segment_footer_index_num_remote_io_total += other.segment_footer_index_num_remote_io_total; @@ -211,6 +225,12 @@ struct IOContext { // if true, bypass peer read / peer-vs-S3 race and read directly from remote storage bool bypass_peer_read {false}; FileCacheMissPolicy file_cache_miss_policy = FileCacheMissPolicy::READ_THROUGH_AND_WRITE_BACK; + // From session variable inverted_index_snii_read_no_write_file_cache: SNII index + // reads of this query take REMOTE_ONLY_ON_MISS (hit served, miss reads remote + // and skips the cache write-back). Carried down to the SNII adapter, which is + // the sole place that turns it into a file_cache_miss_policy -- keeping CLucene + // index reads and data reads on the normal write-back path. + bool inverted_index_snii_read_no_write_file_cache = false; RemoteScanCacheWriteLimiter* remote_scan_cache_write_limiter = nullptr; // Ref }; diff --git a/be/src/runtime/index_policy/index_policy_mgr.cpp b/be/src/runtime/index_policy/index_policy_mgr.cpp index 5ab41a409ba921..e7ec8dfd2b9d0e 100644 --- a/be/src/runtime/index_policy/index_policy_mgr.cpp +++ b/be/src/runtime/index_policy/index_policy_mgr.cpp @@ -17,6 +17,7 @@ #include "runtime/index_policy/index_policy_mgr.h" +#include #include #include #include @@ -24,6 +25,24 @@ #include namespace doris { +namespace { + +class PurposeInsensitiveAnalyzerProvider final + : public segment_v2::inverted_index::AnalyzerProvider { +public: + explicit PurposeInsensitiveAnalyzerProvider(AnalyzerPtr analyzer) + : _analyzer(std::move(analyzer)) {} + + AnalyzerPtr get_analyzer( + segment_v2::inverted_index::AnalysisPurpose /*purpose*/) const override { + return _analyzer; + } + +private: + const AnalyzerPtr _analyzer; +}; + +} // namespace const std::unordered_set IndexPolicyMgr::BUILTIN_NORMALIZERS = {"lowercase"}; @@ -49,11 +68,9 @@ void IndexPolicyMgr::apply_policy_changes(const std::vector& polic LOG(INFO) << "Deleting policy - " << "ID: " << id << ", " << "Name: " << it->second.name; - - // Use normalized name for deletion _name_to_id.erase(normalize_name(it->second.name)); _policys.erase(it); - success_deletes++; + ++success_deletes; } else { LOG(WARNING) << "Delete failed - Policy ID not found: " << id; } @@ -66,8 +83,6 @@ void IndexPolicyMgr::apply_policy_changes(const std::vector& polic << " | New name: " << policy.name; continue; } - - // Use normalized name for case-insensitive lookup std::string normalized_name = normalize_name(policy.name); if (_name_to_id.contains(normalized_name)) { LOG(ERROR) << "Reject update - Duplicate policy name: " << policy.name @@ -77,10 +92,8 @@ void IndexPolicyMgr::apply_policy_changes(const std::vector& polic } _policys.emplace(policy.id, policy); - // Store with normalized key for case-insensitive lookup _name_to_id.emplace(normalized_name, policy.id); - success_updates++; - + ++success_updates; LOG(INFO) << "Successfully applied policy - " << "ID: " << policy.id << ", " << "Name: " << policy.name << ", " @@ -131,7 +144,79 @@ AnalyzerPtr IndexPolicyMgr::get_policy_by_name(const std::string& name) { throw Exception(ErrorCode::INVALID_ARGUMENT, "Policy not found with type: " + name); } -AnalyzerPtr IndexPolicyMgr::build_analyzer_from_policy(const TIndexPolicy& index_policy_analyzer) { +AnalyzerPtr IndexPolicyMgr::get_analyzer_by_name( + const std::string& name, segment_v2::inverted_index::AnalysisPurpose purpose) { + std::shared_lock lock(_mutex); + const std::string normalized_name = normalize_name(name); + auto name_it = _name_to_id.find(normalized_name); + if (name_it == _name_to_id.end()) { + if (is_builtin_normalizer(normalized_name)) { + return build_builtin_normalizer(name); + } + throw Exception(ErrorCode::INVALID_ARGUMENT, "Policy not found with name: " + name); + } + auto policy_it = _policys.find(name_it->second); + if (policy_it == _policys.end()) { + throw Exception(ErrorCode::INVALID_ARGUMENT, "Policy not found with id: " + name); + } + if (policy_it->second.type == TIndexPolicyType::ANALYZER) { + return build_analyzer_provider_from_config( + build_analyzer_config_from_policy(policy_it->second), {}) + ->get_analyzer(purpose); + } + if (policy_it->second.type == TIndexPolicyType::NORMALIZER) { + return build_normalizer_from_policy(policy_it->second); + } + throw Exception(ErrorCode::INVALID_ARGUMENT, "Analyzer policy not found: " + name); +} + +AnalyzerProviderPtr IndexPolicyMgr::get_analyzer_provider_by_name( + const std::string& name, const std::map& outer_char_filter_map) { + std::shared_lock lock(_mutex); + const std::string normalized_name = normalize_name(name); + auto name_it = _name_to_id.find(normalized_name); + if (name_it == _name_to_id.end()) { + if (is_builtin_normalizer(normalized_name)) { + return std::make_shared( + build_builtin_normalizer(name)); + } + throw Exception(ErrorCode::INVALID_ARGUMENT, "Policy not found with name: " + name); + } + auto policy_it = _policys.find(name_it->second); + if (policy_it == _policys.end()) { + throw Exception(ErrorCode::INVALID_ARGUMENT, "Policy not found with id: " + name); + } + if (policy_it->second.type == TIndexPolicyType::ANALYZER) { + return build_analyzer_provider_from_config( + build_analyzer_config_from_policy(policy_it->second), outer_char_filter_map); + } + if (policy_it->second.type == TIndexPolicyType::NORMALIZER) { + return std::make_shared( + build_normalizer_from_policy(policy_it->second)); + } + throw Exception(ErrorCode::INVALID_ARGUMENT, "Analyzer policy not found: " + name); +} + +AnalyzerProviderPtr IndexPolicyMgr::get_analyzer_provider_by_base_fingerprint( + std::string_view base_analyzer_fingerprint, + const std::map& outer_char_filter_map) { + std::shared_lock lock(_mutex); + for (const auto& [_, policy] : _policys) { + if (policy.type != TIndexPolicyType::ANALYZER) { + continue; + } + auto config = build_analyzer_config_from_policy(policy); + if (segment_v2::inverted_index::CustomAnalyzerProvider::calculate_base_analyzer_fingerprint( + config, outer_char_filter_map) != base_analyzer_fingerprint) { + continue; + } + return build_analyzer_provider_from_config(std::move(config), outer_char_filter_map); + } + return nullptr; +} + +segment_v2::inverted_index::CustomAnalyzerConfigPtr +IndexPolicyMgr::build_analyzer_config_from_policy(const TIndexPolicy& index_policy_analyzer) { segment_v2::inverted_index::CustomAnalyzerConfig::Builder builder; auto tokenizer_it = index_policy_analyzer.properties.find(PROP_TOKENIZER); @@ -175,9 +260,23 @@ AnalyzerPtr IndexPolicyMgr::build_analyzer_from_policy(const TIndexPolicy& index builder.add_token_filter_config(name, settings); }); - auto custom_analyzer_config = builder.build(); - return segment_v2::inverted_index::CustomAnalyzer::build_custom_analyzer( - custom_analyzer_config); + return builder.build(); +} + +AnalyzerProviderPtr IndexPolicyMgr::build_analyzer_provider_from_config( + segment_v2::inverted_index::CustomAnalyzerConfigPtr config, + const std::map& outer_char_filter_map) { + // One shape for every policy: the provider sources its CommonGrams word list from the + // BE-local default, so there is no per-policy word set to look up and no "not yet prepared" + // state to represent. + return std::make_shared( + std::move(config), outer_char_filter_map); +} + +AnalyzerPtr IndexPolicyMgr::build_analyzer_from_policy(const TIndexPolicy& index_policy_analyzer) { + return build_analyzer_provider_from_config( + build_analyzer_config_from_policy(index_policy_analyzer), {}) + ->get_analyzer(segment_v2::inverted_index::AnalysisPurpose::kIndex); } AnalyzerPtr IndexPolicyMgr::build_normalizer_from_policy( @@ -224,7 +323,8 @@ void IndexPolicyMgr::process_filter_configs( std::string normalized_filter_name = normalize_name(filter_name); if (_name_to_id.contains(normalized_filter_name)) { // Nested filter policy - const auto& filter_policy = _policys[_name_to_id[normalized_filter_name]]; + const int64_t filter_policy_id = _name_to_id.at(normalized_filter_name); + const auto& filter_policy = _policys.at(filter_policy_id); auto type_it = filter_policy.properties.find(PROP_TYPE); if (type_it == filter_policy.properties.end()) { throw Exception( diff --git a/be/src/runtime/index_policy/index_policy_mgr.h b/be/src/runtime/index_policy/index_policy_mgr.h index edbdee938b86ad..0fb1dc3b3a87c1 100644 --- a/be/src/runtime/index_policy/index_policy_mgr.h +++ b/be/src/runtime/index_policy/index_policy_mgr.h @@ -19,7 +19,13 @@ #include +#include +#include +#include #include +#include +#include +#include #include #include "storage/index/inverted/analyzer/custom_analyzer.h" @@ -29,6 +35,7 @@ namespace doris { using Policys = std::unordered_map; using AnalyzerPtr = std::shared_ptr; +using AnalyzerProviderPtr = segment_v2::inverted_index::AnalyzerProviderPtr; class IndexPolicyMgr { public: @@ -40,8 +47,21 @@ class IndexPolicyMgr { Policys get_index_policys(); AnalyzerPtr get_policy_by_name(const std::string& name); + AnalyzerPtr get_analyzer_by_name(const std::string& name, + segment_v2::inverted_index::AnalysisPurpose purpose); + AnalyzerProviderPtr get_analyzer_provider_by_name( + const std::string& name, + const std::map& outer_char_filter_map = {}); + AnalyzerProviderPtr get_analyzer_provider_by_base_fingerprint( + std::string_view base_analyzer_fingerprint, + const std::map& outer_char_filter_map = {}); private: + segment_v2::inverted_index::CustomAnalyzerConfigPtr build_analyzer_config_from_policy( + const TIndexPolicy& index_policy_analyzer); + AnalyzerProviderPtr build_analyzer_provider_from_config( + segment_v2::inverted_index::CustomAnalyzerConfigPtr config, + const std::map& outer_char_filter_map); AnalyzerPtr build_analyzer_from_policy(const TIndexPolicy& index_policy_analyzer); AnalyzerPtr build_normalizer_from_policy(const TIndexPolicy& index_policy_normalizer); diff --git a/be/src/service/backend_service.cpp b/be/src/service/backend_service.cpp index b4bbb435abdf0f..33c2b90c67e79a 100644 --- a/be/src/service/backend_service.cpp +++ b/be/src/service/backend_service.cpp @@ -510,7 +510,7 @@ void _ingest_binlog(StorageEngine& engine, IngestBinlogArg* arg) { } } else { for (int64_t segment_index = 0; segment_index < num_segments; ++segment_index) { - if (tablet_schema->has_inverted_index() || tablet_schema->has_ann_index()) { + if (tablet_schema->has_inverted_or_ann_index()) { auto get_segment_index_file_size_url = fmt::format( "{}?method={}&tablet_id={}&rowset_id={}&segment_index={}&segment_index_id={" "}", diff --git a/be/src/storage/compaction/collection_similarity.h b/be/src/storage/compaction/collection_similarity.h index f3ed818be4f142..482b1682ab4169 100644 --- a/be/src/storage/compaction/collection_similarity.h +++ b/be/src/storage/compaction/collection_similarity.h @@ -49,6 +49,19 @@ class CollectionSimilarity { void collect(segment_v2::rowid_t row_id, float score); + // Hands the collected scores over to the caller and leaves this instance empty. + // A reader that computes per-document scores inside its own query() cannot return them + // through the query API, so it publishes them into a throwaway CollectionSimilarity that the + // caller then relocates into a scorer. Moving instead of copying keeps that hand-off free of + // a full rehash of a map that can hold one entry per matched row. + ScoreMap release_scores() { + ScoreMap released = std::move(_bm25_scores); + // A moved-from flat_hash_map is valid but unspecified, not guaranteed empty, so make the + // "leaves this instance empty" half of the contract true rather than merely likely. + _bm25_scores.clear(); + return released; + } + void get_bm25_scores(roaring::Roaring* row_bitmap, IColumn::MutablePtr& scores, std::unique_ptr>& row_ids, const ScoreRangeFilterPtr& filter = nullptr) const; diff --git a/be/src/storage/compaction/collection_statistics.cpp b/be/src/storage/compaction/collection_statistics.cpp deleted file mode 100644 index 9afbe5f0944d43..00000000000000 --- a/be/src/storage/compaction/collection_statistics.cpp +++ /dev/null @@ -1,290 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "storage/compaction/collection_statistics.h" - -#include -#include - -#include "common/exception.h" -#include "exprs/vexpr.h" -#include "exprs/vexpr_context.h" -#include "exprs/vliteral.h" -#include "exprs/vslot_ref.h" -#include "storage/index/index_file_reader.h" -#include "storage/index/index_reader_helper.h" -#include "storage/index/inverted/analyzer/analyzer.h" -#include "storage/index/inverted/util/string_helper.h" -#include "storage/index/inverted/util/term_iterator.h" -#include "storage/rowset/rowset.h" -#include "storage/rowset/rowset_reader.h" -#include "util/uid_util.h" - -namespace doris { - -Status CollectionStatistics::collect(RuntimeState* state, - const std::vector& rs_splits, - const TabletSchemaSPtr& tablet_schema, - const VExprContextSPtrs& common_expr_ctxs_push_down, - io::IOContext* io_ctx) { - std::unordered_map collect_infos; - RETURN_IF_ERROR( - extract_collect_info(state, common_expr_ctxs_push_down, tablet_schema, &collect_infos)); - if (collect_infos.empty()) { - LOG(WARNING) << "Index statistics collection: no collect info extracted."; - return Status::OK(); - } - - for (const auto& rs_split : rs_splits) { - const auto& rs_reader = rs_split.rs_reader; - auto rowset = rs_reader->rowset(); - auto num_segments = rowset->num_segments(); - for (int32_t seg_id = 0; seg_id < num_segments; ++seg_id) { - auto status = - process_segment(rowset, seg_id, tablet_schema.get(), collect_infos, io_ctx); - if (!status.ok()) { - if (status.code() == ErrorCode::INVERTED_INDEX_FILE_NOT_FOUND || - status.code() == ErrorCode::INVERTED_INDEX_BYPASS) { - LOG(ERROR) << "Index statistics collection failed: " << status.to_string(); - } else { - return status; - } - } - } - } - - // Build a single-line log with query_id, tablet_ids, and per-field term statistics - if (VLOG_IS_ON(1)) { - std::set tablet_ids; - for (const auto& rs_split : rs_splits) { - if (rs_split.rs_reader && rs_split.rs_reader->rowset()) { - tablet_ids.insert(rs_split.rs_reader->rowset()->rowset_meta()->tablet_id()); - } - } - - std::ostringstream oss; - oss << "CollectionStatistics: query_id=" << print_id(state->query_id()); - - oss << ", tablet_ids=["; - bool first_tablet = true; - for (int64_t tid : tablet_ids) { - if (!first_tablet) oss << ","; - oss << tid; - first_tablet = false; - } - oss << "]"; - - oss << ", total_num_docs=" << _total_num_docs; - - for (const auto& [ws_field_name, num_tokens] : _total_num_tokens) { - oss << ", {field=" << StringHelper::to_string(ws_field_name) - << ", num_tokens=" << num_tokens << ", terms=["; - - bool first_term = true; - for (const auto& [term, doc_freq] : _term_doc_freqs.at(ws_field_name)) { - if (!first_term) oss << ", "; - oss << "(" << StringHelper::to_string(term) << ":" << doc_freq << ")"; - first_term = false; - } - oss << "]}"; - } - - VLOG(1) << oss.str(); - } - - return Status::OK(); -} - -Status CollectionStatistics::extract_collect_info( - RuntimeState* state, const VExprContextSPtrs& common_expr_ctxs_push_down, - const TabletSchemaSPtr& tablet_schema, CollectInfoMap* collect_infos) { - DCHECK(collect_infos != nullptr); - - std::unordered_map collectors; - collectors[TExprNodeType::MATCH_PRED] = std::make_unique(); - collectors[TExprNodeType::SEARCH_EXPR] = std::make_unique(); - - for (const auto& root_expr_ctx : common_expr_ctxs_push_down) { - const auto& root_expr = root_expr_ctx->root(); - if (root_expr == nullptr) { - continue; - } - - std::stack stack; - stack.emplace(root_expr); - - while (!stack.empty()) { - auto expr = stack.top(); - stack.pop(); - - if (!expr) { - continue; - } - - auto collector_it = collectors.find(expr->node_type()); - if (collector_it != collectors.end()) { - RETURN_IF_ERROR( - collector_it->second->collect(state, tablet_schema, expr, collect_infos)); - } - - const auto& children = expr->children(); - for (const auto& child : children) { - stack.push(child); - } - } - } - - LOG(INFO) << "Extracted collect info for " << collect_infos->size() << " fields"; - - return Status::OK(); -} - -Status CollectionStatistics::process_segment(const RowsetSharedPtr& rowset, int32_t seg_id, - const TabletSchema* tablet_schema, - const CollectInfoMap& collect_infos, - io::IOContext* io_ctx) { - auto seg_path = DORIS_TRY(rowset->segment_path(seg_id)); - auto rowset_meta = rowset->rowset_meta(); - - auto idx_file_reader = std::make_unique( - rowset_meta->fs(), - std::string {InvertedIndexDescriptor::get_index_file_path_prefix(seg_path)}, - tablet_schema->get_inverted_index_storage_format(), - rowset_meta->inverted_index_file_info(seg_id), rowset_meta->tablet_id()); - RETURN_IF_ERROR(idx_file_reader->init(config::inverted_index_read_buffer_size, io_ctx)); - - int32_t total_seg_num_docs = 0; - - for (const auto& [ws_field_name, collect_info] : collect_infos) { - lucene::search::IndexSearcher* index_searcher = nullptr; - lucene::index::IndexReader* index_reader = nullptr; - -#ifdef BE_TEST - auto compound_reader = DORIS_TRY(idx_file_reader->open(collect_info.index_meta, io_ctx)); - auto* reader = lucene::index::IndexReader::open(compound_reader.get()); - auto searcher_ptr = std::make_shared(reader, true); - index_searcher = searcher_ptr.get(); - index_reader = index_searcher->getReader(); -#else - InvertedIndexCacheHandle inverted_index_cache_handle; - auto index_file_key = idx_file_reader->get_index_file_cache_key(collect_info.index_meta); - InvertedIndexSearcherCache::CacheKey searcher_cache_key(index_file_key); - - if (!InvertedIndexSearcherCache::instance()->lookup(searcher_cache_key, - &inverted_index_cache_handle)) { - auto compound_reader = - DORIS_TRY(idx_file_reader->open(collect_info.index_meta, io_ctx)); - auto* reader = lucene::index::IndexReader::open(compound_reader.get()); - size_t reader_size = reader->getTermInfosRAMUsed(); - auto searcher_ptr = std::make_shared(reader, true); - auto* cache_value = new InvertedIndexSearcherCache::CacheValue( - std::move(searcher_ptr), reader_size, UnixMillis()); - InvertedIndexSearcherCache::instance()->insert(searcher_cache_key, cache_value, - &inverted_index_cache_handle); - } - - auto searcher_variant = inverted_index_cache_handle.get_index_searcher(); - auto index_searcher_ptr = std::get(searcher_variant); - index_searcher = index_searcher_ptr.get(); - index_reader = index_searcher->getReader(); -#endif - total_seg_num_docs = std::max(total_seg_num_docs, index_reader->maxDoc()); - - _total_num_tokens[ws_field_name] += - index_reader->sumTotalTermFreq(ws_field_name.c_str()).value_or(0); - - for (const auto& term_info : collect_info.term_infos) { - auto iter = TermIterator::create(io_ctx, false, index_reader, ws_field_name, - term_info.get_single_term()); - _term_doc_freqs[ws_field_name][iter->term()] += iter->doc_freq(); - } - } - - _total_num_docs += total_seg_num_docs; - - return Status::OK(); -} - -uint64_t CollectionStatistics::get_term_doc_freq_by_col(const std::wstring& lucene_col_name, - const std::wstring& term) { - if (!_term_doc_freqs.contains(lucene_col_name)) { - throw Exception(ErrorCode::INVERTED_INDEX_CLUCENE_ERROR, - "Index statistics collection failed: Not such column {}", - StringHelper::to_string(lucene_col_name)); - } - - if (!_term_doc_freqs[lucene_col_name].contains(term)) { - throw Exception(ErrorCode::INVERTED_INDEX_CLUCENE_ERROR, - "Index statistics collection failed: Not such term {}", - StringHelper::to_string(term)); - } - - return _term_doc_freqs[lucene_col_name][term]; -} - -uint64_t CollectionStatistics::get_total_term_cnt_by_col(const std::wstring& lucene_col_name) { - if (!_total_num_tokens.contains(lucene_col_name)) { - throw Exception(ErrorCode::INVERTED_INDEX_CLUCENE_ERROR, - "Index statistics collection failed: Not such column {}", - StringHelper::to_string(lucene_col_name)); - } - - return _total_num_tokens[lucene_col_name]; -} - -uint64_t CollectionStatistics::get_doc_num() const { - if (_total_num_docs == 0) { - throw Exception( - ErrorCode::INVERTED_INDEX_CLUCENE_ERROR, - "Index statistics collection failed: No data available for SimilarityCollector"); - } - - return _total_num_docs; -} - -float CollectionStatistics::get_or_calculate_avg_dl(const std::wstring& lucene_col_name) { - auto iter = _avg_dl_by_col.find(lucene_col_name); - if (iter != _avg_dl_by_col.end()) { - return iter->second; - } - - const uint64_t total_term_cnt = get_total_term_cnt_by_col(lucene_col_name); - const uint64_t total_doc_cnt = get_doc_num(); - float avg_dl = total_doc_cnt > 0 ? float((double)total_term_cnt / (double)total_doc_cnt) : 0.0F; - _avg_dl_by_col[lucene_col_name] = avg_dl; - return avg_dl; -} - -float CollectionStatistics::get_or_calculate_idf(const std::wstring& lucene_col_name, - const std::wstring& term) { - auto iter = _idf_by_col_term.find(lucene_col_name); - if (iter != _idf_by_col_term.end()) { - auto term_iter = iter->second.find(term); - if (term_iter != iter->second.end()) { - return term_iter->second; - } - } - - const uint64_t doc_num = get_doc_num(); - const uint64_t doc_freq = get_term_doc_freq_by_col(lucene_col_name, term); - auto idf = (float)std::log(1 + ((double)doc_num - (double)doc_freq + (double)0.5) / - ((double)doc_freq + (double)0.5)); - _idf_by_col_term[lucene_col_name][term] = idf; - return idf; -} - -} // namespace doris \ No newline at end of file diff --git a/be/src/storage/compaction/compaction.cpp b/be/src/storage/compaction/compaction.cpp index 46dc32d2bcd31f..c726947cf9046d 100644 --- a/be/src/storage/compaction/compaction.cpp +++ b/be/src/storage/compaction/compaction.cpp @@ -22,9 +22,12 @@ #include #include +#include #include #include #include +#include +#include #include #include #include @@ -41,6 +44,7 @@ #include "cloud/cloud_tablet.h" #include "cloud/config.h" #include "cloud/pb_convert.h" +#include "common/check.h" #include "common/config.h" #include "common/metrics/doris_metrics.h" #include "common/status.h" @@ -53,7 +57,6 @@ #include "io/io_common.h" #include "runtime/memory/mem_tracker_limiter.h" #include "runtime/thread_context.h" -#include "storage/compaction/collection_statistics.h" #include "storage/compaction/cumulative_compaction.h" #include "storage/compaction/cumulative_compaction_binlog_policy.h" #include "storage/compaction/cumulative_compaction_policy.h" @@ -65,6 +68,11 @@ #include "storage/index/inverted/inverted_index_compaction.h" #include "storage/index/inverted/inverted_index_desc.h" #include "storage/index/inverted/inverted_index_fs_directory.h" +#include "storage/index/inverted/similarity/collection_statistics.h" +#include "storage/index/snii/compaction/eligibility.h" +#include "storage/index/snii/compaction/snii_index_compaction.h" +#include "storage/index/snii/writer/memory_reporter.h" +#include "storage/index/snii/writer/snii_build_memory_tracker.h" #include "storage/olap_common.h" #include "storage/olap_define.h" #include "storage/rowset/beta_rowset.h" @@ -84,6 +92,7 @@ #include "storage/txn/txn_manager.h" #include "storage/utils.h" #include "util/pretty_printer.h" +#include "util/stopwatch.hpp" #include "util/time.h" #include "util/trace.h" @@ -119,6 +128,8 @@ bool should_enable_compaction_cache_index_only(bool write_file_cache, ReaderType namespace { +constexpr size_t kSniiCompactionReadAheadBudgetBytes = 64ULL << 20; + bool is_rowset_tidy(std::string& pre_max_key, bool& pre_rs_key_bounds_truncated, const RowsetSharedPtr& rhs) { size_t min_tidy_size = config::ordered_data_compaction_min_segment_size; @@ -284,9 +295,10 @@ Status Compaction::merge_input_rowsets() { // write merged rows to output rowset // The test results show that merger is low-memory-footprint, there is no need to tracker its mem pool - // if ctx.columns_to_do_index_compaction.size() > 0, it means we need to do inverted index compaction. - // the row ID conversion matrix needs to be used for inverted index compaction. - if (!ctx.columns_to_do_index_compaction.empty() || + // A non-empty index compaction set (per-column for V2/V3, per-index for + // SNII) means inverted index compaction runs and needs the row ID + // conversion matrix. + if (!ctx.columns_to_do_index_compaction.empty() || !ctx.snii_indexes_to_do_compaction.empty() || (_tablet->keys_type() == KeysType::UNIQUE_KEYS && _tablet->enable_unique_key_merge_on_write())) { _stats.rowid_conversion = _rowid_conversion.get(); @@ -800,10 +812,36 @@ Status CompactionMixin::execute_compact_impl(int64_t permits) { return Status::OK(); } +// Iteration domain of inverted index compaction: V2/V3 merge EVERY index of +// each column in columns_to_do_index_compaction (their per-column CLucene +// directories move as a unit); SNII merges exactly the (column, index) pairs +// the preflight proved eligible -- the segment writer already raw-built the +// rest. +static std::map> collect_index_compaction_domain( + const TabletSchema& schema, const RowsetWriterContext& ctx) { + std::map> column_index_metas; + if (schema.get_inverted_index_storage_format() == InvertedIndexStorageFormatPB::SNII) { + for (const auto& [column_uniq_id, index_id] : ctx.snii_indexes_to_do_compaction) { + const auto& col = schema.column_by_uid(column_uniq_id); + for (const TabletIndex* index_meta : schema.inverted_indexs(col)) { + if (index_meta->index_id() == index_id) { + column_index_metas[column_uniq_id].push_back(index_meta); + } + } + } + return column_index_metas; + } + for (auto&& column_uniq_id : ctx.columns_to_do_index_compaction) { + const auto& col = schema.column_by_uid(column_uniq_id); + column_index_metas.emplace(column_uniq_id, schema.inverted_indexs(col)); + } + return column_index_metas; +} + Status Compaction::do_inverted_index_compaction() { const auto& ctx = _output_rs_writer->context(); if (!_enable_inverted_index_compaction || _input_row_num <= 0 || - ctx.columns_to_do_index_compaction.empty()) { + (ctx.columns_to_do_index_compaction.empty() && ctx.snii_indexes_to_do_compaction.empty())) { return Status::OK(); } @@ -833,6 +871,7 @@ Status Compaction::do_inverted_index_compaction() { } OlapStopWatch inverted_watch; + ThreadCpuStopWatch inverted_cpu_watch(true); // translation vec // <> @@ -858,7 +897,8 @@ Status Compaction::do_inverted_index_compaction() { if (dest_segment_num <= 0) { LOG(INFO) << "skip doing index compaction due to no output segments" << ". tablet=" << _tablet->tablet_id() << ", input row number=" << _input_row_num - << ". elapsed time=" << inverted_watch.get_elapse_second() << "s."; + << ". elapsed time=" << inverted_watch.get_elapse_second() + << "s. thread cpu time=" << inverted_cpu_watch.elapsed_time() / 1e9 << "s."; return Status::OK(); } @@ -924,6 +964,7 @@ Status Compaction::do_inverted_index_compaction() { // src index dirs std::vector> index_file_readers(src_segment_num); + std::vector source_rowsets(src_segment_num, nullptr); for (const auto& m : src_seg_to_id_map) { const auto& [rowset_id, seg_id] = m.first; @@ -988,6 +1029,7 @@ Status Compaction::do_inverted_index_compaction() { _tablet->tablet_id(), rowset_id.to_string(), seg_id); } index_file_readers[m.second] = std::move(index_file_reader); + source_rowsets[m.second] = rowset; } // dest index files @@ -1017,9 +1059,28 @@ Status Compaction::do_inverted_index_compaction() { << ", destination index size=" << dest_segment_num << "."; Status status = Status::OK(); - for (auto&& column_uniq_id : ctx.columns_to_do_index_compaction) { + std::shared_ptr snii_merge_memory_reporter; + std::unique_ptr validated_snii_rowid_conversion; + if (_cur_tablet_schema->get_inverted_index_storage_format() == + InvertedIndexStorageFormatPB::SNII) { + const size_t spill_threshold = + static_cast(config::inverted_index_ram_buffer_size * 1024 * 1024); + // Mirror the merge's live build bytes into the process-wide SNII + // index-build observation tracker, the same line ingestion feeds: index + // merge builds the same structures and must be visible in the same + // place. Classified kUnregistered: this path holds Reservation scratch + // only and never registers a SpimiTermBuffer, so no forced spill can + // reclaim any of it -- the decision layer must not charge these bytes + // against ingestion writers' arenas. The kHardLimit cap policy is what + // bounds them instead; the tracker only observes. + snii_merge_memory_reporter = std::make_shared( + snii::writer::snii_build_consume_release( + snii::writer::BuildMemoryPopulation::kUnregistered), + spill_threshold, snii::writer::MemoryReporter::CapPolicy::kHardLimit); + } + for (auto&& [column_uniq_id, index_metas] : + collect_index_compaction_domain(*_cur_tablet_schema, ctx)) { auto col = _cur_tablet_schema->column_by_uid(column_uniq_id); - auto index_metas = _cur_tablet_schema->inverted_indexs(col); DBUG_EXECUTE_IF("Compaction::do_inverted_index_compaction_can_not_find_index_meta", { index_metas.clear(); }) if (index_metas.empty()) { @@ -1032,6 +1093,152 @@ Status Compaction::do_inverted_index_compaction() { break; } for (const auto& index_meta : index_metas) { + if (_cur_tablet_schema->get_inverted_index_storage_format() == + InvertedIndexStorageFormatPB::SNII) { + std::vector> source_indexes; + std::vector plan_sources; + std::vector eligibility_sources; + source_indexes.reserve(src_segment_num); + plan_sources.reserve(src_segment_num); + eligibility_sources.reserve(src_segment_num); + Status merge_status = Status::OK(); + for (size_t source_ordinal = 0; source_ordinal < src_segment_num; + ++source_ordinal) { + DORIS_CHECK(source_rowsets[source_ordinal] != nullptr); + const auto source_index_metas = + source_rowsets[source_ordinal]->tablet_schema()->inverted_indexs( + column_uniq_id); + const auto source_index_it = std::find_if( + source_index_metas.begin(), source_index_metas.end(), + [&index_meta](const TabletIndex* source_index) { + return source_index->index_id() == index_meta->index_id(); + }); + if (source_index_it == source_index_metas.end()) { + merge_status = Status::Error( + "source SNII index metadata disappeared after eligibility"); + break; + } + const TabletIndex* source_index_meta = *source_index_it; + auto source_index = index_file_readers[source_ordinal]->open_snii_index( + source_index_meta, nullptr, + snii::reader::LogicalIndexOpenMode::kCompaction); + if (!source_index.has_value()) { + merge_status = source_index.error(); + break; + } + source_indexes.push_back(std::move(source_index.value())); + plan_sources.push_back(source_indexes.back().get()); + eligibility_sources.push_back({.reader = std::cref(*source_indexes.back()), + .index_meta = std::cref(*source_index_meta)}); + } + + snii::compaction::SniiCompactionEligibility merge_eligibility; + if (merge_status.ok()) { + merge_status = snii::compaction::validate_snii_compaction_eligibility( + eligibility_sources, *index_meta, &merge_eligibility); + } + + if (merge_status.ok() && validated_snii_rowid_conversion == nullptr) { + std::vector source_segment_doc_counts; + source_segment_doc_counts.reserve(plan_sources.size()); + for (const snii::reader::LogicalIndexReader* source : plan_sources) { + DORIS_CHECK(source != nullptr); + if (source->stats().doc_count > std::numeric_limits::max()) { + merge_status = Status::Error( + "source doc count exceeds the SNII uint32 docid domain"); + break; + } + source_segment_doc_counts.push_back( + static_cast(source->stats().doc_count)); + } + if (merge_status.ok()) { + merge_status = snii::compaction::ValidatedRowIdConversion::create( + &trans_vec, source_segment_doc_counts, dest_segment_num_rows, + &validated_snii_rowid_conversion); + if (merge_status.ok()) { + DBUG_EXECUTE_IF("Compaction::snii_validated_rowid_conversion_created", + DBUG_RUN_CALLBACK()); + } + } + } + + std::unique_ptr merge_plan; + if (merge_status.ok()) { + DORIS_CHECK(validated_snii_rowid_conversion != nullptr); + merge_status = snii::compaction::SniiPlainT2MergePlan::prepare( + std::move(plan_sources), *validated_snii_rowid_conversion, + merge_eligibility, kSniiCompactionReadAheadBudgetBytes, + snii_merge_memory_reporter, &merge_plan); + } + + std::vector destination_sessions( + dest_segment_num, nullptr); + if (merge_status.ok()) { + DORIS_CHECK(snii_merge_memory_reporter != nullptr); + for (size_t destination_ordinal = 0; destination_ordinal < dest_segment_num; + ++destination_ordinal) { + DBUG_EXECUTE_IF("Compaction::before_add_snii_destination_session", + DBUG_RUN_CALLBACK(destination_ordinal, &merge_status)); + if (!merge_status.ok()) { + break; + } + auto* destination_writer = + inverted_index_file_writers[cast_set(destination_ordinal)] + .get(); + if (merge_eligibility.kind == + snii::compaction::SniiStreamedMergeKind::kCommonGramsT3) { + merge_status = destination_writer->add_snii_index_streamed( + index_meta, dest_segment_num_rows[destination_ordinal], + merge_plan->take_destination_null_docids(destination_ordinal), + merge_plan->take_destination_encoded_norms(destination_ordinal), + merge_plan->destination_common_grams_metadata( + destination_ordinal), + merge_plan->destination_common_grams_posting_policy(), + merge_plan->destination_index_config(), + snii_merge_memory_reporter, + &destination_sessions[destination_ordinal]); + } else { + merge_status = destination_writer->add_snii_index_streamed( + index_meta, dest_segment_num_rows[destination_ordinal], + merge_plan->take_destination_null_docids(destination_ordinal), + merge_plan->destination_index_config(), + snii_merge_memory_reporter, + &destination_sessions[destination_ordinal]); + } + if (!merge_status.ok()) { + break; + } + } + } + if (merge_status.ok()) { + DBUG_EXECUTE_IF("Compaction::before_execute_snii_merge", + DBUG_RUN_CALLBACK(&merge_status)); + } + if (!merge_status.ok()) { + for (size_t destination_ordinal = 0; + destination_ordinal < destination_sessions.size(); ++destination_ordinal) { + snii::writer::SniiStreamedIndexSession* session = + destination_sessions[destination_ordinal]; + if (session != nullptr) { + session->abort(merge_status); + DBUG_EXECUTE_IF("Compaction::snii_destination_session_aborted", + DBUG_RUN_CALLBACK(destination_ordinal)); + } + } + } else { + merge_status = merge_plan->execute(destination_sessions); + } + if (!merge_status.ok()) { + if (merge_status.is() || + merge_status.is() || + merge_status.is()) { + error_handler(index_meta->index_id(), column_uniq_id); + } + return merge_status; + } + continue; + } + std::vector dest_index_dirs(dest_segment_num); try { std::vector> src_idx_dirs( @@ -1095,7 +1302,8 @@ Status Compaction::do_inverted_index_compaction() { } LOG(INFO) << "succeed to do index compaction" << ". tablet=" << _tablet->tablet_id() - << ". elapsed time=" << inverted_watch.get_elapse_second() << "s."; + << ". elapsed time=" << inverted_watch.get_elapse_second() + << "s. thread cpu time=" << inverted_cpu_watch.elapsed_time() / 1e9 << "s."; return Status::OK(); } @@ -1103,6 +1311,9 @@ Status Compaction::do_inverted_index_compaction() { void Compaction::mark_skip_index_compaction( const RowsetWriterContext& context, const std::function& error_handler) { + for (const auto& [column_uniq_id, index_id] : context.snii_indexes_to_do_compaction) { + error_handler(index_id, column_uniq_id); + } for (auto&& column_uniq_id : context.columns_to_do_index_compaction) { auto col = _cur_tablet_schema->column_by_uid(column_uniq_id); auto index_metas = _cur_tablet_schema->inverted_indexs(col); @@ -1233,6 +1444,154 @@ static bool check_rowset_has_inverted_index(const RowsetSharedPtr& src_rs, int32 } void Compaction::construct_index_compaction_columns(RowsetWriterContext& ctx) { + if (_cur_tablet_schema->get_inverted_index_storage_format() == + InvertedIndexStorageFormatPB::SNII) { + std::map> source_file_readers; + for (const auto& destination_index : _cur_tablet_schema->inverted_indexes()) { + const auto& col_unique_ids = destination_index->col_unique_ids(); + if (col_unique_ids.empty()) { + LOG(WARNING) << "tablet[" << _tablet->tablet_id() << "] index[" + << destination_index->index_id() + << "] has no column unique id, will rebuild its SNII index"; + continue; + } + const int32_t col_unique_id = col_unique_ids[0]; + if (!_cur_tablet_schema->has_column_unique_id(col_unique_id) || + !field_is_slice_type(_cur_tablet_schema->column_by_uid(col_unique_id).type())) { + continue; + } + + bool eligible = true; + size_t source_segment_count = 0; + for (const auto& rowset : _input_rowsets) { + source_segment_count += rowset->num_segments(); + } + if (source_segment_count == 0 || + source_segment_count > kSniiCompactionReadAheadBudgetBytes / + snii::compaction::SniiPlainT2MergePlan:: + kMinReadAheadBudgetPerSource) { + eligible = false; + } + + Status eligibility_status = + eligible ? Status::OK() + : Status::Error( + "source index unavailable or read-ahead budget gate failed"); + std::optional merge_eligibility; + size_t source_ordinal = 0; + + for (const auto& rowset : _input_rowsets) { + if (!eligible) { + break; + } + auto* beta_rowset = static_cast(rowset.get()); + if (beta_rowset->is_skip_index_compaction(col_unique_id)) { + eligible = false; + break; + } + const auto source_index_metas = + rowset->tablet_schema()->inverted_indexs(col_unique_id); + const auto source_index_it = std::find_if( + source_index_metas.begin(), source_index_metas.end(), + [&destination_index](const TabletIndex* source_index) { + return source_index->index_id() == destination_index->index_id(); + }); + if (source_index_it == source_index_metas.end()) { + eligible = false; + break; + } + const TabletIndex* source_index_meta = *source_index_it; + const auto fs = rowset->rowset_meta()->fs(); + if (fs == nullptr) { + eligible = false; + break; + } + + for (uint32_t segment_id = 0; segment_id < rowset->num_segments(); ++segment_id) { + auto segment_path = rowset->segment_path(segment_id); + if (!segment_path.has_value()) { + eligible = false; + break; + } + const std::string index_file_path_prefix = + std::string {InvertedIndexDescriptor::get_index_file_path_prefix( + segment_path.value())}; + auto source_file_reader_it = source_file_readers.find(index_file_path_prefix); + if (source_file_reader_it == source_file_readers.end()) { + auto source_file_reader = std::make_unique( + fs, index_file_path_prefix, InvertedIndexStorageFormatPB::SNII, + rowset->rowset_meta()->inverted_index_file_info(segment_id), + _tablet->tablet_id()); + const Status init_status = + source_file_reader->init(config::inverted_index_read_buffer_size); + if (!init_status.ok()) { + eligible = false; + break; + } + DBUG_EXECUTE_IF("Compaction::snii_eligibility_reader_initialized", + DBUG_RUN_CALLBACK()); + source_file_reader_it = source_file_readers + .emplace(index_file_path_prefix, + std::move(source_file_reader)) + .first; + } + auto source_index = source_file_reader_it->second->open_snii_index( + source_index_meta, nullptr, + snii::reader::LogicalIndexOpenMode::kCompaction); + if (!source_index.has_value()) { + eligible = false; + break; + } + if (!merge_eligibility.has_value()) { + const std::array source = { + snii::compaction::PlainT2CompactionSource { + .reader = std::cref(*source_index.value()), + .index_meta = std::cref(*source_index_meta)}}; + snii::compaction::SniiCompactionEligibility eligibility; + eligibility_status = snii::compaction::validate_snii_compaction_eligibility( + source, *destination_index, &eligibility); + if (eligibility_status.ok()) { + merge_eligibility = std::move(eligibility); + } + } else if (source_index_meta->index_id() != destination_index->index_id() || + source_index_meta->get_index_suffix() != + destination_index->get_index_suffix() || + source_index_meta->properties() != destination_index->properties()) { + eligibility_status = Status::Error( + "source SNII index identity or properties differ from destination"); + } else { + eligibility_status = snii::compaction::validate_snii_source_eligibility( + *source_index.value(), source_ordinal, *merge_eligibility); + } + if (!eligibility_status.ok()) { + eligible = false; + break; + } + ++source_ordinal; + } + } + + if (!eligible && eligibility_status.ok()) { + eligibility_status = Status::Error( + "source SNII index file or metadata is unavailable"); + } + // Per-(column, index) granularity: an eligible index merges natively + // even when a sibling on the SAME column must be rebuilt from the + // raw column -- eligibility is a property of the logical index, not + // of the column. + if (eligible) { + ctx.snii_indexes_to_do_compaction.emplace(col_unique_id, + destination_index->index_id()); + } else { + LOG(INFO) << "tablet[" << _tablet->tablet_id() << "] index[" + << destination_index->index_id() + << "] is not eligible for SNII postings compaction; rebuild from raw " + "column. reason=" + << eligibility_status; + } + } + return; + } for (const auto& index : _cur_tablet_schema->inverted_indexes()) { auto col_unique_ids = index->col_unique_ids(); // check if column unique ids is empty to avoid crash diff --git a/be/src/storage/index/analyzer_key_matcher.cpp b/be/src/storage/index/analyzer_key_matcher.cpp index ef0bdddf75ddb1..81373fa529327f 100644 --- a/be/src/storage/index/analyzer_key_matcher.cpp +++ b/be/src/storage/index/analyzer_key_matcher.cpp @@ -17,8 +17,6 @@ #include "storage/index/analyzer_key_matcher.h" -#include "storage/index/inverted/inverted_index_iterator.h" - namespace doris::segment_v2 { AnalyzerMatchResult AnalyzerKeyMatcher::match( diff --git a/be/src/storage/index/analyzer_key_matcher.h b/be/src/storage/index/analyzer_key_matcher.h index 833193b259d592..e683c2d4dbb480 100644 --- a/be/src/storage/index/analyzer_key_matcher.h +++ b/be/src/storage/index/analyzer_key_matcher.h @@ -23,11 +23,15 @@ #include #include "storage/index/inverted/inverted_index_parser.h" +#include "storage/index/inverted/inverted_index_reader.h" namespace doris::segment_v2 { -// Forward declaration -struct ReaderEntry; +struct ReaderEntry { + InvertedIndexReaderType type; + std::string analyzer_key; + InvertedIndexReaderPtr reader; +}; // Result of analyzer key matching operation. // Contains candidate readers that match the requested analyzer key. diff --git a/be/src/storage/index/ann/ann_index_writer.cpp b/be/src/storage/index/ann/ann_index_writer.cpp index 21911417c4f9a1..d041eb7900d5e9 100644 --- a/be/src/storage/index/ann/ann_index_writer.cpp +++ b/be/src/storage/index/ann/ann_index_writer.cpp @@ -44,7 +44,11 @@ AnnIndexColumnWriter::AnnIndexColumnWriter(IndexFileWriter* index_file_writer, AnnIndexColumnWriter::~AnnIndexColumnWriter() = default; Status AnnIndexColumnWriter::init() { - Result> compound_dir = _index_file_writer->open(_index_meta); + // The staging directory, not necessarily a filesystem one: under SNII the + // faiss output is held in memory until begin_close() seals it into the + // container as a blob. Only the write side is used either way. + Result> compound_dir = + _index_file_writer->open_ann_directory(_index_meta); if (!compound_dir.has_value()) { return Status::IOError("Failed to open index file: {}", compound_dir.error().to_string()); diff --git a/be/src/storage/index/ann/ann_index_writer.h b/be/src/storage/index/ann/ann_index_writer.h index 67061bef9219a8..df07e3c386f641 100644 --- a/be/src/storage/index/ann/ann_index_writer.h +++ b/be/src/storage/index/ann/ann_index_writer.h @@ -77,6 +77,6 @@ class AnnIndexColumnWriter : public IndexColumnWriter { int64_t _total_rows = 0; IndexFileWriter* _index_file_writer; const TabletIndex* _index_meta; - std::shared_ptr _dir; + std::shared_ptr _dir; }; } // namespace doris::segment_v2 diff --git a/be/src/storage/index/bkd_field_encoding.cpp b/be/src/storage/index/bkd_field_encoding.cpp new file mode 100644 index 00000000000000..ba50871897e1ea --- /dev/null +++ b/be/src/storage/index/bkd_field_encoding.cpp @@ -0,0 +1,62 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/bkd_field_encoding.h" + +#include "core/data_type/primitive_type.h" + +namespace doris { + +Status encode_bkd_field_ascending(FieldType ft, const Field& field, const KeyCoder* coder, + std::string* out) { + // `actual` is the primitive type of the query Field from the caller; `PrimitiveType::PT` is the + // scalar type the BKD index stores (e.g. INT for an INT column or ARRAY index). + // Normally they match: `int_col = 1` -> both INT; `array_contains(int_arr, 2)` -> both INT. + // Mismatch happens when the query Field carries a non-scalar while BKD records the inner scalar: + // `arr = []` reaches here via `FunctionComparison` with the entire const ARRAY literal + // as the query Field, so `actual = TYPE_ARRAY` while PT is the inner scalar -- the predicate + // cannot be answered by BKD. Return INVERTED_INDEX_EVALUATE_SKIPPED so `_apply_index_expr` + // downgrades to scalar evaluation instead of crashing on `Field::get()` DCHECK below. +#define CASE(FT, PT) \ + case FieldType::FT: { \ + const auto actual = field.get_type(); \ + if (actual != PrimitiveType::PT && actual != PrimitiveType::TYPE_NULL && \ + !(is_string_type(actual) && is_string_type(PrimitiveType::PT))) { \ + return Status::Error( \ + "BKD query value type {} does not match index type {}", \ + static_cast(actual), static_cast(ft)); \ + } \ + full_encode_field_as_key(field, coder, out); \ + return Status::OK(); \ + } + switch (ft) { + DORIS_APPLY_FOR_KEY_ENCODABLE_NON_STRING_TYPES(CASE) + default: + break; + } +#undef CASE + // NOT InternalError. Every caller reaches this only with a field type that + // cannot be encoded, and for the SNII reader that type comes from the index + // HEADER -- i.e. from disk, i.e. it is reachable by corruption. Doris keys + // its scalar-evaluation fallback on specific codes (SegmentIterator:: + // _downgrade_without_index and friends); InternalError is not one of them, + // so a damaged byte would fail the query instead of downgrading it. + return Status::Error( + "unsupported BKD field type {}", static_cast(ft)); +} + +} // namespace doris diff --git a/be/src/storage/index/bkd_field_encoding.h b/be/src/storage/index/bkd_field_encoding.h new file mode 100644 index 00000000000000..d89ba3a8d7422d --- /dev/null +++ b/be/src/storage/index/bkd_field_encoding.h @@ -0,0 +1,44 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include "common/status.h" +#include "core/field.h" +#include "storage/key_coder.h" +#include "storage/olap_common.h" + +// Query-value encoding shared by every BKD reader, CLucene-backed or SNII-native. +// +// It lives in its own header because BOTH readers must encode a query value with +// the key coder of the INDEX's own field type (INV-1). An index encoded with one +// coder and probed with another is self-consistent -- every byte round-trips -- +// but compares in the wrong order, and no round-trip test can see it. One +// definition is the only way the two readers cannot drift. +// +// The +/- infinity sentinels are deliberately NOT here: they are an artifact of +// the CLucene visitor, whose bounds are always closed and whose strictness lives +// in matches(). The SNII-native reader carries strictness on the interval itself +// and leaves an open side unbounded, so it never needs them. +namespace doris { + +Status encode_bkd_field_ascending(FieldType ft, const Field& field, const KeyCoder* coder, + std::string* out); + +} // namespace doris diff --git a/be/src/storage/index/index_file_reader.cpp b/be/src/storage/index/index_file_reader.cpp index 348e1399421e5a..ac2f17db2521ea 100644 --- a/be/src/storage/index/index_file_reader.cpp +++ b/be/src/storage/index/index_file_reader.cpp @@ -20,6 +20,8 @@ #include #include +#include "common/cast_set.h" +#include "common/config.h" #include "storage/index/inverted/inverted_index_compound_reader.h" #include "storage/index/inverted/inverted_index_fs_directory.h" #include "storage/tablet/tablet_schema.h" @@ -31,7 +33,9 @@ Status IndexFileReader::init(int32_t read_buffer_size, const io::IOContext* io_c std::unique_lock lock(_mutex); // Lock for writing if (!_inited) { _read_buffer_size = read_buffer_size; - if (_storage_format >= InvertedIndexStorageFormatPB::V2) { + if (_storage_format == InvertedIndexStorageFormatPB::SNII) { + RETURN_IF_ERROR(_init_snii(io_ctx)); + } else if (_storage_format >= InvertedIndexStorageFormatPB::V2) { RETURN_IF_ERROR(_init_from(read_buffer_size, io_ctx)); } _inited = true; @@ -136,7 +140,58 @@ Status IndexFileReader::_init_from(int32_t read_buffer_size, const io::IOContext return Status::OK(); } +Status IndexFileReader::_init_snii(const io::IOContext* io_ctx) { + auto index_file_full_path = InvertedIndexDescriptor::get_index_file_path_v2(_index_path_prefix); + int64_t file_size = -1; + if (_idx_file_info.has_index_size()) { + file_size = _idx_file_info.index_size(); + } + file_size = file_size == 0 ? -1 : file_size; + + io::FileReaderOptions opts; + opts.cache_type = config::enable_file_cache ? io::FileCachePolicy::FILE_BLOCK_CACHE + : io::FileCachePolicy::NO_CACHE; + opts.is_doris_table = true; + opts.file_size = file_size; + opts.tablet_id = _tablet_id; + io::FileReaderSPtr reader; + // A rowset written before any index existed has no container at all. The + // filesystem reports that as a plain NOT_FOUND; translate it to the + // index-specific code the way the V1/V2 path does, because callers + // (IndexBuilder's BUILD INDEX rewrite) distinguish "no container yet, build + // everything fresh" from a real IO failure by exactly that code. + if (const Status open_status = _fs->open_file(index_file_full_path, &reader, &opts); + !open_status.ok()) { + if (open_status.is()) { + return Status::Error( + "inverted index file {} is not found.", index_file_full_path); + } + return open_status; + } + // With NO_CACHE on a remote filesystem there is no CachedRemoteFileReader to + // account physical remote bytes, so the adapter must count its own reads. + const bool direct_remote_io = opts.cache_type == io::FileCachePolicy::NO_CACHE && + _fs->type() != io::FileSystemType::LOCAL; + _snii_file_reader = std::make_shared( + std::move(reader), /*io_ctx=*/nullptr, direct_remote_io); + _snii_segment_reader = std::make_unique(); + io::IOContext meta_io_ctx; + if (io_ctx != nullptr) { + meta_io_ctx = *io_ctx; + } + meta_io_ctx.is_inverted_index = true; + meta_io_ctx.is_index_data = true; + snii_doris::DorisSniiFileReader::ScopedIOContext io_context_scope(&meta_io_ctx); + RETURN_IF_ERROR(doris::snii::reader::SniiSegmentReader::open(_snii_file_reader.get(), + _snii_segment_reader.get())); + return Status::OK(); +} + Result IndexFileReader::get_all_directories() { + if (_storage_format == InvertedIndexStorageFormatPB::SNII) { + return ResultError(Status::Error( + "SNII format does not expose CLucene directories")); + } InvertedIndexDirectoryMap res; std::shared_lock lock(_mutex); // Lock for reading for (auto& [index, _] : _indices_entries) { @@ -155,6 +210,69 @@ Result> IndexFileReader:: int64_t index_id, const std::string& index_suffix, const io::IOContext* io_ctx) const { std::unique_ptr compound_reader; + if (_storage_format == InvertedIndexStorageFormatPB::SNII) { + // A blob logical index is a named-sub-file table over the container, and + // a compound reader is a named-sub-file table over a stream -- the same + // shape. The offsets recorded in the directory are ABSOLUTE container + // offsets, exactly like a V2 compound entry, so the sub-files need no + // rebasing and DorisCompoundReader is reused unchanged. + const auto index_file_path = + InvertedIndexDescriptor::get_index_file_path_v2(_index_path_prefix); + EntriesType entries; + int64_t container_size = 0; + // The lock spans every use of _snii_segment_reader state, not just the + // lookup: `entry` points into the reader's decoded directory, and every + // other SNII accessor on this class holds the lock across the whole use. + // Narrowing it here would make this the one site whose safety rests on + // "the segment reader is never reset after init" rather than on the lock. + { + std::shared_lock lock(_mutex); + if (_snii_segment_reader == nullptr) { + return ResultError(Status::Error( + "SNII index file {} is not opened", index_file_path)); + } + const doris::snii::format::LogicalIndexMetadataRef* entry = nullptr; + RETURN_IF_ERROR_RESULT(_snii_segment_reader->blob_entry(cast_set(index_id), + index_suffix, &entry)); + DORIS_CHECK(entry != nullptr); + // Only an ANN index is served through a CLucene directory. A BKD blob + // has its own reader and must not be reachable this way, or a caller + // would get a directory over bytes no CLucene code can parse. + if (entry->kind != doris::snii::format::LogicalIndexKind::kAnn) { + return ResultError(Status::Error( + "SNII logical index {} is not an ANN blob; it has no CLucene directory", + index_id)); + } + // Blob extents were bounded against the container at open time + // (SniiSegmentReader::validate_blob_files), so they are safe to hand + // to the compound reader as-is. + for (const auto& blob : entry->files) { + auto file_entry = std::make_unique(); + file_entry->file_name = blob.name; + file_entry->offset = cast_set(blob.offset); + file_entry->length = cast_set(blob.length); + entries.emplace(blob.name, std::move(file_entry)); + } + container_size = get_inverted_file_size(); + } + + CLuceneError err; + CL_NS(store)::IndexInput* index_input = nullptr; + // The container size is already resident -- init() opened the file to read + // its directory. Passing -1 here would make the filesystem re-discover it, + // which is a stat() locally and a HeadObject round trip on S3, on every + // cold ANN index load. + if (!DorisFSDirectory::FSIndexInput::open(_fs, index_file_path.c_str(), index_input, err, + _read_buffer_size, container_size, _tablet_id)) { + return ResultError(Status::Error( + "CLuceneError occur when open SNII container {}, error msg: {}", + index_file_path, err.what())); + } + compound_reader.reset( + new DorisCompoundReader(index_input, entries, _read_buffer_size, io_ctx)); + return compound_reader; + } + if (_storage_format == InvertedIndexStorageFormatPB::V1) { auto index_file_path = InvertedIndexDescriptor::get_index_file_path_v1( _index_path_prefix, index_id, index_suffix); @@ -229,6 +347,90 @@ Result> IndexFileReader:: return compound_reader; } +Result> IndexFileReader::open_snii_index( + const TabletIndex* index_meta, const io::IOContext* io_ctx, + doris::snii::reader::LogicalIndexOpenMode open_mode) const { + DCHECK(_storage_format == InvertedIndexStorageFormatPB::SNII); + std::shared_lock lock(_mutex); + if (_snii_segment_reader == nullptr) { + return ResultError(Status::Error( + "SNII index file {} is not opened", + InvertedIndexDescriptor::get_index_file_path_v2(_index_path_prefix))); + } + io::IOContext meta_io_ctx; + if (io_ctx != nullptr) { + meta_io_ctx = *io_ctx; + } + meta_io_ctx.is_inverted_index = true; + meta_io_ctx.is_index_data = true; + snii_doris::DorisSniiFileReader::ScopedIOContext io_context_scope(&meta_io_ctx); + + auto logical_reader = std::make_unique(); + auto status = _snii_segment_reader->open_index(cast_set(index_meta->index_id()), + index_meta->get_index_suffix(), + logical_reader.get(), open_mode); + auto doris_status = status; + if (!doris_status.ok()) { + return ResultError(doris_status); + } + return logical_reader; +} + +Result> IndexFileReader::open_snii_bkd_index( + const TabletIndex* index_meta, const io::IOContext* io_ctx) const { + DCHECK(_storage_format == InvertedIndexStorageFormatPB::SNII); + std::shared_lock lock(_mutex); + if (_snii_segment_reader == nullptr) { + return ResultError(Status::Error( + "SNII index file {} is not opened", + InvertedIndexDescriptor::get_index_file_path_v2(_index_path_prefix))); + } + io::IOContext meta_io_ctx; + if (io_ctx != nullptr) { + meta_io_ctx = *io_ctx; + } + meta_io_ctx.is_inverted_index = true; + meta_io_ctx.is_index_data = true; + snii_doris::DorisSniiFileReader::ScopedIOContext io_context_scope(&meta_io_ctx); + + const doris::snii::format::LogicalIndexMetadataRef* entry = nullptr; + RETURN_IF_ERROR_RESULT(_snii_segment_reader->blob_entry( + cast_set(index_meta->index_id()), index_meta->get_index_suffix(), &entry)); + + // Placement is the container's decision, so every extent comes from the + // sealed directory rather than from anything the producer remembered. + doris::snii::bkd::BkdSections sections; + auto searcher = std::make_unique(); + for (const doris::snii::format::NamedBlobFileRef& blob : entry->files) { + if (blob.name == "bkd_data") { + sections.data_offset = blob.offset; + sections.data_length = blob.length; + } else if (blob.name == "bkd_index") { + sections.index_offset = blob.offset; + sections.index_length = blob.length; + } else if (blob.name == "bkd_nulls") { + searcher->null_bitmap_offset = blob.offset; + searcher->null_bitmap_length = blob.length; + } + } + RETURN_IF_ERROR_RESULT(doris::snii::bkd::BkdReader::open(_snii_segment_reader->reader(), + sections, &searcher->reader)); + return searcher; +} + +Status IndexFileReader::prepare_snii_rewrite_snapshot( + const std::vector& keep, uint64_t segment_doc_count, + doris::snii::reader::SniiRewriteSnapshot* out) const { + DCHECK(_storage_format == InvertedIndexStorageFormatPB::SNII); + std::shared_lock lock(_mutex); + if (_snii_segment_reader == nullptr) { + return Status::Error( + "SNII index file {} is not opened", + InvertedIndexDescriptor::get_index_file_path_v2(_index_path_prefix)); + } + return _snii_segment_reader->prepare_rewrite_snapshot(keep, segment_doc_count, out); +} + Result> IndexFileReader::open( const TabletIndex* index_meta, const io::IOContext* io_ctx) const { auto index_id = index_meta->index_id(); @@ -254,6 +456,25 @@ Status IndexFileReader::index_file_exist(const TabletIndex* index_meta, bool* re auto index_file_path = InvertedIndexDescriptor::get_index_file_path_v1( _index_path_prefix, index_meta->index_id(), index_meta->get_index_suffix()); return _fs->exists(index_file_path, res); + } else if (_storage_format == InvertedIndexStorageFormatPB::SNII) { + auto index_file_path = InvertedIndexDescriptor::get_index_file_path_v2(_index_path_prefix); + RETURN_IF_ERROR(_fs->exists(index_file_path, res)); + if (!*res) { + return Status::OK(); + } + std::shared_lock lock(_mutex); + if (_snii_segment_reader == nullptr) { + // The container is on disk but this reader never opened it, so we + // cannot tell whether the index is in there. Answering "absent" + // would make a BUILD INDEX rewrite drop it silently; report the real + // condition instead, as the V2 branch below does. + *res = false; + return Status::Error( + "SNII idx file {} exists but is not opened", + InvertedIndexDescriptor::get_index_file_path_v2(_index_path_prefix)); + } + return _snii_segment_reader->index_exists(cast_set(index_meta->index_id()), + index_meta->get_index_suffix(), res); } else { std::shared_lock lock(_mutex); // Lock for reading if (_stream == nullptr) { @@ -279,6 +500,25 @@ Status IndexFileReader::has_null(const TabletIndex* index_meta, bool* res) const *res = true; return Status::OK(); } + if (_storage_format == InvertedIndexStorageFormatPB::SNII) { + std::shared_lock lock(_mutex); + if (_snii_segment_reader == nullptr) { + return Status::Error( + "SNII index file {} is not opened", + InvertedIndexDescriptor::get_index_file_path_v2(_index_path_prefix)); + } + io::IOContext meta_io_ctx; + meta_io_ctx.is_inverted_index = true; + meta_io_ctx.is_index_data = true; + snii_doris::DorisSniiFileReader::ScopedIOContext io_context_scope(&meta_io_ctx); + + doris::snii::format::SectionRefs section_refs; + RETURN_IF_ERROR(_snii_segment_reader->section_refs_for_index( + cast_set(index_meta->index_id()), index_meta->get_index_suffix(), + §ion_refs)); + *res = section_refs.null_bitmap.length > 0; + return Status::OK(); + } std::shared_lock lock(_mutex); // Lock for reading if (_stream == nullptr) { return Status::Error( diff --git a/be/src/storage/index/index_file_reader.h b/be/src/storage/index/index_file_reader.h index fb4ec2b9a62fe3..c04a8d6ec207c4 100644 --- a/be/src/storage/index/index_file_reader.h +++ b/be/src/storage/index/index_file_reader.h @@ -35,6 +35,10 @@ #include "io/fs/file_system.h" #include "storage/index/index_file_writer.h" #include "storage/index/inverted/inverted_index_desc.h" +#include "storage/index/snii/reader/logical_index_reader.h" +#include "storage/index/snii/reader/snii_segment_reader.h" +#include "storage/index/snii/snii_bkd_searcher.h" +#include "storage/index/snii/snii_doris_adapter.h" namespace doris { class TabletIndex; @@ -60,7 +64,7 @@ class IndexFileReader { : _fs(std::move(fs)), _index_path_prefix(std::move(index_path_prefix)), _storage_format(storage_format), - _idx_file_info(idx_file_info), + _idx_file_info(std::move(idx_file_info)), _tablet_id(tablet_id) {} virtual ~IndexFileReader() = default; @@ -68,6 +72,32 @@ class IndexFileReader { const io::IOContext* io_ctx = nullptr); MOCK_FUNCTION Result> open( const TabletIndex* index_meta, const io::IOContext* io_ctx = nullptr) const; + // Opens one BLOB logical index of kind kBkd: resolves its named sub-files + // (bkd_data / bkd_index / bkd_nulls) into extents through the CONTAINER's own + // directory and hands back a reader bound to this IndexFileReader's file. + // The caller must keep this IndexFileReader alive for the reader's lifetime, + // exactly as open_snii_index requires. + Result> open_snii_bkd_index( + const TabletIndex* index_meta, const io::IOContext* io_ctx) const; + Result> open_snii_index( + const TabletIndex* index_meta, const io::IOContext* io_ctx = nullptr, + doris::snii::reader::LogicalIndexOpenMode open_mode = + doris::snii::reader::LogicalIndexOpenMode::kQuery) const; + // SNII only: builds the fully validated inheritance view of this container + // for a BUILD INDEX rewrite. `segment_doc_count` is the segment's row count; + // every kept logical index must agree with it. + Status prepare_snii_rewrite_snapshot( + const std::vector& keep, + uint64_t segment_doc_count, doris::snii::reader::SniiRewriteSnapshot* out) const; + // SNII only: the raw byte source backing this container, handed to + // SniiCompoundWriter::inherit for the sequential prefix copy. + doris::snii::io::FileReader* snii_io_reader() const { return _snii_file_reader.get(); } + // SNII only: true when the opened container holds a blob logical index + // (BKD / ANN). False when this reader opened no SNII container at all. + bool snii_has_blob_index() const { + std::shared_lock lock(_mutex); + return _snii_segment_reader != nullptr && _snii_segment_reader->has_blob_index(); + } void debug_file_entries(); std::string get_index_file_cache_key(const TabletIndex* index_meta) const; std::string get_index_file_path(const TabletIndex* index_meta) const; @@ -75,12 +105,19 @@ class IndexFileReader { Status has_null(const TabletIndex* index_meta, bool* res) const; Result get_all_directories(); // open file v2, init _stream - int64_t get_inverted_file_size() const { return _stream == nullptr ? 0 : _stream->length(); } + int64_t get_inverted_file_size() const { + if (_storage_format == InvertedIndexStorageFormatPB::SNII) { + return _snii_file_reader == nullptr ? 0 : _snii_file_reader->size(); + } + return _stream == nullptr ? 0 : _stream->length(); + } const std::string& get_index_path_prefix() const { return _index_path_prefix; } + InvertedIndexStorageFormatPB get_storage_format() const { return _storage_format; } friend IndexFileWriter; protected: Status _init_from(int32_t read_buffer_size, const io::IOContext* io_ctx); + Status _init_snii(const io::IOContext* io_ctx); Result> _open( int64_t index_id, const std::string& index_suffix, const io::IOContext* io_ctx = nullptr) const; @@ -88,6 +125,8 @@ class IndexFileReader { private: IndicesEntriesMap _indices_entries; std::unique_ptr _stream = nullptr; + std::shared_ptr _snii_file_reader; + std::unique_ptr _snii_segment_reader; const io::FileSystemSPtr _fs; std::string _index_path_prefix; int32_t _read_buffer_size = -1; diff --git a/be/src/storage/index/index_file_writer.cpp b/be/src/storage/index/index_file_writer.cpp index fcaded4bd7a4a9..e00dc047efcc9c 100644 --- a/be/src/storage/index/index_file_writer.cpp +++ b/be/src/storage/index/index_file_writer.cpp @@ -19,9 +19,12 @@ #include +#include #include #include +#include "common/cast_set.h" +#include "common/config.h" #include "common/status.h" #include "io/fs/packed_file_writer.h" #include "io/fs/s3_file_writer.h" @@ -35,10 +38,55 @@ #include "storage/index/inverted/inverted_index_desc.h" #include "storage/index/inverted/inverted_index_fs_directory.h" #include "storage/index/inverted/inverted_index_reader.h" +#include "storage/index/snii/snii_blob_staging_directory.h" +#include "storage/index/snii/snii_doris_adapter.h" #include "storage/tablet/tablet_schema.h" +#include "util/defer_op.h" namespace doris::segment_v2 { +// Resolves whether one segment index lays out freq regions (G16-c). Freq +// serves ONLY BM25 scoring: a scoring config always keeps it; a plain +// positions config keeps it only when the escape-hatch config asks for the +// full T2 layout. NOT in the anonymous namespace on purpose -- the UT covers +// this production policy line directly (a flipped operator or inverted flag +// here would otherwise stay green: no BE test drives add_snii_index). +bool snii_effective_write_freq(doris::snii::format::IndexConfig index_config) { + return doris::snii::format::has_scoring(index_config) || + config::snii_positions_index_write_freq; +} + +// Shared write-parameter resolution for one SNII index flush; `input->config` +// must already be set. BOTH the build path (add_snii_index) and the T2.2 +// compaction-merge streamed session resolve through this single helper, so the +// merge fast path can never drift from the rebuild contract (the T2 semantic +// golden invariant depends on parameter parity). NOT in the anonymous +// namespace on purpose -- the UT pins the resolved values directly. +void snii_resolve_index_write_params(bool is_direct_load, + doris::snii::writer::SniiIndexInput* input) { + // G16-c: freq regions serve only BM25 scoring; a plain positions index + // drops them unless the escape hatch asks for the full T2 layout. + input->write_freq = snii_effective_write_freq(input->config); + // G16-h: zstd levels. dict blocks accept zstd's full sane range; the prx + // level floor is 3 because the writer passes -level into the prx builders + // and -1 is the historic "auto at default level 3" sentinel -- a + // configured level 1 would silently resolve to 3 anyway (levels 1-2 buy + // nothing over 3 on these payloads). + input->dict_block_zstd_level = std::clamp(config::snii_dict_block_zstd_level, 1, 19); + // Patch C prx tiering: a direct load compresses prx at the cheaper load + // level; compaction / schema change / ADD INDEX keep snii_prx_zstd_level + // and compaction rewrites every segment with it, so settled segments (and + // the cold-query path over them) are byte-for-byte unaffected. + input->prx_zstd_level = std::clamp( + is_direct_load ? config::snii_prx_zstd_level_direct_load : config::snii_prx_zstd_level, + 3, 19); + // G16-d: dict block size experiment knob; <= 0 keeps the format default. + if (config::snii_target_dict_block_bytes > 0) { + input->target_dict_block_bytes = + static_cast(config::snii_target_dict_block_bytes); + } +} + IndexFileWriter::IndexFileWriter(io::FileSystemSPtr fs, std::string index_path_prefix, std::string rowset_id, int64_t seg_id, InvertedIndexStorageFormatPB storage_format, @@ -57,7 +105,7 @@ IndexFileWriter::IndexFileWriter(io::FileSystemSPtr fs, std::string index_path_p _tmp_dir = tmp_file_dir.native(); if (_storage_format == InvertedIndexStorageFormatPB::V1) { _index_storage_format = std::make_unique(this); - } else { + } else if (_storage_format != InvertedIndexStorageFormatPB::SNII) { _index_storage_format = std::make_unique(this); } } @@ -69,7 +117,7 @@ Status IndexFileWriter::initialize(InvertedIndexDirectoryMap& indices_dirs) { Status IndexFileWriter::_insert_directory_into_map(int64_t index_id, const std::string& index_suffix, - std::shared_ptr dir) { + std::shared_ptr dir) { auto key = std::make_pair(index_id, index_suffix); auto [it, inserted] = _indices_dirs.emplace(key, std::move(dir)); if (!inserted) { @@ -85,6 +133,13 @@ Status IndexFileWriter::_insert_directory_into_map(int64_t index_id, } Result> IndexFileWriter::open(const TabletIndex* index_meta) { + // No index under SNII writes through a CLucene filesystem directory: text + // postings go through the SPIMI writer, and an ANN index stages into memory + // (see open_ann_directory) so that nothing has to be cleaned off disk. + if (_storage_format == InvertedIndexStorageFormatPB::SNII) { + return ResultError(Status::Error( + "SNII format does not open CLucene filesystem directories")); + } auto local_fs_index_path = InvertedIndexDescriptor::get_temporary_index_path( _tmp_dir, _rowset_id, _seg_id, index_meta->index_id(), index_meta->get_index_suffix()); auto dir = std::shared_ptr(DorisFSDirectoryFactory::getDirectory( @@ -94,10 +149,234 @@ Result> IndexFileWriter::open(const TabletInde if (!st.ok()) { return ResultError(st); } + return dir; +} +Result> IndexFileWriter::open_ann_directory( + const TabletIndex* index_meta) { + if (_storage_format != InvertedIndexStorageFormatPB::SNII) { + // V1/V2 stage ANN output exactly like every other index. + auto dir = open(index_meta); + if (!dir.has_value()) { + return ResultError(dir.error()); + } + return std::shared_ptr(std::move(dir.value())); + } + return _open_snii_ann_staging_directory(index_meta); +} + +Result> IndexFileWriter::_open_snii_ann_staging_directory( + const TabletIndex* index_meta) { + // The container stores an ANN index as a blob logical index (kAnn), exactly + // as it stores the BKD, and the kind stamped at seal time depends on this + // gate -- so refusing anything else here is what keeps the seal honest. + if (!index_meta->is_ann_index()) { + return ResultError(Status::Error( + "SNII format only stages ANN indexes through a directory")); + } + auto dir = std::make_shared(); + RETURN_IF_ERROR_RESULT(_insert_directory_into_map(index_meta->index_id(), + index_meta->get_index_suffix(), dir)); + // Copied, not borrowed: the staged bytes are not harvested until + // begin_close(), and nothing promises the caller's TabletIndex is still alive + // by then. + _snii_blob_dir_metas.emplace( + std::make_pair(index_meta->index_id(), index_meta->get_index_suffix()), + std::make_shared(*index_meta)); return dir; } +Status IndexFileWriter::_seal_snii_blob_directories() { + DORIS_CHECK(_storage_format == InvertedIndexStorageFormatPB::SNII); + for (const auto& [key, dir] : _indices_dirs) { + const auto meta_it = _snii_blob_dir_metas.find(key); + // Every SNII directory is registered with its metadata by open(), which + // is the only way one gets into this map. + DORIS_CHECK(meta_it != _snii_blob_dir_metas.end()); + // The staging gate admits ONLY ann indexes under SNII, and the kind + // stamped below depends on it. Asserted here, where it is relied upon, so + // that widening that gate cannot silently mislabel another index kind. + DORIS_CHECK(meta_it->second->is_ann_index()); + // ... and the only thing that ever enters the map under SNII is a staging + // directory, so this is a type assertion, not a runtime branch. + DORIS_CHECK(std::strcmp(dir->getObjectName(), + snii_doris::SniiBlobStagingDirectory::getClassName()) == 0); + + // Nothing here can throw: the staged bytes are plain buffers, and each + // source keeps its own alive, so finish() may pull them after this + // directory is gone. + auto* staging = static_cast(dir.get()); + // All cold: a faiss index is read at QUERY time, never at container open, + // so nothing here belongs in the hot area the text metadata groups share. + RETURN_IF_ERROR(add_snii_blob_index(meta_it->second.get(), + doris::snii::format::LogicalIndexKind::kAnn, + staging->blob_sources(), {})); + } + return Status::OK(); +} + +void IndexFileWriter::_release_snii_blob_directories() { + // Dropping the map is the whole release: a staging directory holds its bytes + // in memory and owns no file, so there is nothing on disk to remove and -- + // unlike DorisFSDirectory::deleteDirectory() -- no throwing call to make from + // a Status-returning close path. Any buffer a registered blob source still + // needs stays alive through that source until finish() has pulled it. + _indices_dirs.clear(); + _snii_blob_dir_metas.clear(); +} + +Status IndexFileWriter::add_snii_index(const TabletIndex* index_meta, uint32_t doc_count, + std::vector null_docids, + doris::snii::writer::SpimiTermBuffer* const term_buffer, + doris::snii::format::IndexConfig index_config, + SniiAddIndexOptions options, + doris::snii::writer::MemoryReporter* const mem_reporter) { + DCHECK(_storage_format == InvertedIndexStorageFormatPB::SNII); + DCHECK(index_meta != nullptr); + DCHECK(term_buffer != nullptr); + if (_idx_v2_writer == nullptr) { + return Status::Error( + "SNII index file writer is null for {}", _index_path_prefix); + } + if (_snii_file_writer == nullptr) { + _snii_file_writer = std::make_unique(_idx_v2_writer.get()); + _snii_compound_writer = + std::make_unique(_snii_file_writer.get()); + } + + doris::snii::writer::SniiIndexInput input; + input.index_id = cast_set(index_meta->index_id()); + input.index_suffix = index_meta->get_index_suffix(); + input.config = index_config; + input.doc_count = doc_count; + input.null_docids = std::move(null_docids); + input.encoded_norms = std::move(options.encoded_norms); + input.common_grams_metadata = std::move(options.common_grams_metadata); + input.common_grams_posting_policy = options.common_grams_posting_policy; + input.term_source = term_buffer; + input.mem_reporter = mem_reporter; + snii_resolve_index_write_params(options.is_direct_load, &input); + RETURN_IF_ERROR(_snii_compound_writer->add_logical_index(input)); + ++_snii_index_count; + return Status::OK(); +} + +Status IndexFileWriter::add_snii_blob_index( + const TabletIndex* index_meta, doris::snii::format::LogicalIndexKind kind, + std::vector cold_files, + std::vector hot_files) { + DCHECK(_storage_format == InvertedIndexStorageFormatPB::SNII); + DCHECK(index_meta != nullptr); + if (_idx_v2_writer == nullptr) { + return Status::Error( + "SNII index file writer is null for {}", _index_path_prefix); + } + if (_snii_file_writer == nullptr) { + _snii_file_writer = std::make_unique(_idx_v2_writer.get()); + _snii_compound_writer = + std::make_unique(_snii_file_writer.get()); + } + RETURN_IF_ERROR(_snii_compound_writer->add_blob_index( + cast_set(index_meta->index_id()), index_meta->get_index_suffix(), kind, + std::move(cold_files), std::move(hot_files))); + ++_snii_index_count; + return Status::OK(); +} + +Status IndexFileWriter::add_snii_index_streamed( + const TabletIndex* index_meta, uint32_t doc_count, + doris::snii::writer::TrackedNullDocids null_docids, + doris::snii::format::IndexConfig index_config, + std::shared_ptr mem_reporter, + doris::snii::writer::SniiStreamedIndexSession** session) { + return add_snii_index_streamed(index_meta, doc_count, std::move(null_docids), + doris::snii::writer::TrackedEncodedNorms(std::vector()), + std::nullopt, + doris::snii::format::CommonGramsPostingPolicy::kNone, + index_config, std::move(mem_reporter), session); +} + +Status IndexFileWriter::add_snii_index_streamed( + const TabletIndex* index_meta, uint32_t doc_count, + doris::snii::writer::TrackedNullDocids null_docids, + doris::snii::writer::TrackedEncodedNorms encoded_norms, + std::optional common_grams_metadata, + doris::snii::format::CommonGramsPostingPolicy common_grams_posting_policy, + doris::snii::format::IndexConfig index_config, + std::shared_ptr mem_reporter, + doris::snii::writer::SniiStreamedIndexSession** session) { + DCHECK(_storage_format == InvertedIndexStorageFormatPB::SNII); + DCHECK(index_meta != nullptr); + if (session == nullptr) { + return Status::Error( + "SNII streamed session out parameter is null for {}", _index_path_prefix); + } + *session = nullptr; + if (_idx_v2_writer == nullptr) { + return Status::Error( + "SNII index file writer is null for {}", _index_path_prefix); + } + const bool has_scoring = doris::snii::format::has_scoring(index_config); + const bool valid_scoring_shape = + has_scoring ? common_grams_metadata.has_value() && encoded_norms.size() == doc_count + : !common_grams_metadata.has_value() && encoded_norms.empty(); + if (!valid_scoring_shape) { + return Status::InternalError( + "SNII streamed merge scoring shape disagrees with eligibility for {}", + _index_path_prefix); + } + if (_snii_file_writer == nullptr) { + _snii_file_writer = std::make_unique(_idx_v2_writer.get()); + _snii_compound_writer = + std::make_unique(_snii_file_writer.get()); + } + + doris::snii::writer::SniiIndexInput input; + input.index_id = cast_set(index_meta->index_id()); + input.index_suffix = index_meta->get_index_suffix(); + input.config = index_config; + input.doc_count = doc_count; + input.mem_reporter = mem_reporter.get(); + input.common_grams_metadata = std::move(common_grams_metadata); + input.common_grams_posting_policy = common_grams_posting_policy; + // Merge output is always the settled-segment shape: COMPACTION prx level. + snii_resolve_index_write_params(/*is_direct_load=*/false, &input); + if (mem_reporter != nullptr) { + constexpr uint64_t kMaxStreamedDictResidentBytes = 64ULL << 20; + DORIS_CHECK_GE(mem_reporter->cap_bytes(), 8); + input.dict_resident_cap_bytes = + std::min(kMaxStreamedDictResidentBytes, mem_reporter->cap_bytes() / 8); + } + RETURN_IF_ERROR(_snii_compound_writer->begin_streamed_index( + std::move(input), std::move(null_docids), std::move(encoded_norms), session)); + if (mem_reporter != nullptr) { + _snii_memory_reporters.push_back(std::move(mem_reporter)); + } + ++_snii_index_count; + return Status::OK(); +} + +void IndexFileWriter::retain_snii_memory_reporter( + std::unique_ptr mem_reporter) { + DCHECK(mem_reporter != nullptr); + _snii_memory_reporters.emplace_back(std::move(mem_reporter)); +} + +Status IndexFileWriter::inherit_snii(const doris::snii::reader::SniiRewriteSnapshot& snapshot, + doris::snii::io::FileReader* source) { + DCHECK(_storage_format == InvertedIndexStorageFormatPB::SNII); + if (_idx_v2_writer == nullptr) { + return Status::Error( + "SNII index file writer is null for {}", _index_path_prefix); + } + if (_snii_file_writer == nullptr) { + _snii_file_writer = std::make_unique(_idx_v2_writer.get()); + _snii_compound_writer = + std::make_unique(_snii_file_writer.get()); + } + return _snii_compound_writer->inherit(snapshot, source); +} + Status IndexFileWriter::delete_index(const TabletIndex* index_meta) { DBUG_EXECUTE_IF("IndexFileWriter::delete_index_index_meta_nullptr", { index_meta = nullptr; }); if (!index_meta) { @@ -124,6 +403,9 @@ Status IndexFileWriter::delete_index(const TabletIndex* index_meta) { } Status IndexFileWriter::add_into_searcher_cache() { + if (_storage_format == InvertedIndexStorageFormatPB::SNII) { + return Status::OK(); + } auto index_file_reader = std::make_unique( _fs, _index_path_prefix, _storage_format, InvertedIndexFileInfo(), _tablet_id); auto st = index_file_reader->init(); @@ -197,6 +479,28 @@ Result> IndexFileWriter::_construct_index_ Status IndexFileWriter::begin_close() { DCHECK(!_closed) << debug_string(); _closed = true; + if (_storage_format == InvertedIndexStorageFormatPB::SNII) { + if (_snii_compound_writer == nullptr) { + if (_idx_v2_writer == nullptr) { + return Status::OK(); + } + _snii_file_writer = + std::make_unique(_idx_v2_writer.get()); + _snii_compound_writer = std::make_unique( + _snii_file_writer.get()); + } + // The staging directories are dead either way: finish() has copied every + // staged byte into the container, or sealing failed and nobody will ever + // read them. Released on BOTH paths -- unlike the non-SNII branch below, + // finish_close() returns before _indices_dirs is cleared, so a failed + // close would otherwise pin an ANN-sized buffer for the writer's life. + Defer release_staging([this] { _release_snii_blob_directories(); }); + RETURN_IF_ERROR(_seal_snii_blob_directories()); + RETURN_IF_ERROR(_snii_compound_writer->finish()); + _total_file_size = _idx_v2_writer->bytes_appended(); + _file_info.set_index_size(_total_file_size); + return _idx_v2_writer->close(true); + } if (_indices_dirs.empty()) { // An empty file must still be created even if there are no indexes to write if (dynamic_cast(_idx_v2_writer.get()) != nullptr || @@ -239,6 +543,12 @@ Status IndexFileWriter::begin_close() { Status IndexFileWriter::finish_close() { DCHECK(_closed) << debug_string(); + if (_storage_format == InvertedIndexStorageFormatPB::SNII) { + if (_idx_v2_writer != nullptr && _idx_v2_writer->state() != io::FileWriter::State::CLOSED) { + RETURN_IF_ERROR(_idx_v2_writer->close(false)); + } + return Status::OK(); + } if (_indices_dirs.empty()) { // An empty file must still be created even if there are no indexes to write if (dynamic_cast(_idx_v2_writer.get()) != nullptr || diff --git a/be/src/storage/index/index_file_writer.h b/be/src/storage/index/index_file_writer.h index a303de8b68c156..4db100e8489442 100644 --- a/be/src/storage/index/index_file_writer.h +++ b/be/src/storage/index/index_file_writer.h @@ -22,23 +22,38 @@ #include #include +#include #include #include +#include #include "common/be_mock_util.h" #include "io/fs/file_system.h" #include "io/fs/file_writer.h" #include "io/fs/local_file_system.h" #include "storage/index/index_storage_format.h" +#include "storage/index/inverted/common_grams/common_grams_segment_metadata.h" #include "storage/index/inverted/inverted_index_common.h" #include "storage/index/inverted/inverted_index_compound_reader.h" #include "storage/index/inverted/inverted_index_searcher.h" +#include "storage/index/snii/format/format_constants.h" +#include "storage/index/snii/snii_doris_adapter.h" +#include "storage/index/snii/writer/snii_compound_writer.h" + +namespace doris::snii::writer { +class MemoryReporter; +class SpimiTermBuffer; +class SniiCompoundWriter; +} // namespace doris::snii::writer namespace doris { class TabletIndex; namespace segment_v2 { class DorisFSDirectory; +namespace snii_doris { +class DorisSniiFileWriter; +} // namespace snii_doris using InvertedIndexDirectoryMap = std::map, std::shared_ptr>; @@ -55,6 +70,82 @@ class IndexFileWriter { virtual ~IndexFileWriter() = default; MOCK_FUNCTION Result> open(const TabletIndex* index_meta); + // The directory an ANN index is built into. Separate from open() because the + // two formats stage ANN output in different places: V1/V2 hand faiss the same + // CLucene filesystem directory every other index gets, while SNII hands it a + // memory-backed staging directory whose bytes begin_close() seals into a blob + // logical index. Callers only ever write through it, so the return type is + // the lucene::store::Directory base -- widening open() itself would push that + // base type onto the CLucene inverted writer and index_tool, which genuinely + // need the DorisFSDirectory subclass. + Result> open_ann_directory( + const TabletIndex* index_meta); + // Write-path facts for one SNII index flush. + struct SniiAddIndexOptions { + // This flush serves a stream/broker load (DataWriteType::TYPE_DIRECT): + // the prx region compresses at snii_prx_zstd_level_direct_load; + // compaction / schema change / ADD INDEX keep snii_prx_zstd_level. + bool is_direct_load = false; + // Present only for a CommonGrams writer that has a complete immutable + // capability identity. These are semantic BM25 inputs; physical TTF is + // still derived from every emitted unigram and gram posting. + std::vector encoded_norms; + std::optional common_grams_metadata; + snii::format::CommonGramsPostingPolicy common_grams_posting_policy = + snii::format::CommonGramsPostingPolicy::kNone; + }; + Status add_snii_index(const TabletIndex* index_meta, uint32_t doc_count, + std::vector null_docids, + doris::snii::writer::SpimiTermBuffer* const term_buffer, + doris::snii::format::IndexConfig index_config, + SniiAddIndexOptions options, + doris::snii::writer::MemoryReporter* const mem_reporter); + // T2.2 compaction index merge fast path: begins a STREAMED SNII index + // session on this compound. Unlike add_snii_index (which drains a SPIMI + // term buffer), the caller pushes pre-merged, lexicographically sorted + // terms through *session and seals the index with (*session)->finish(). + // Write parameters resolve through the SAME helper as add_snii_index + // (write_freq / zstd levels / dict block size), always at the COMPACTION + // prx tier (a merge is never a direct load). CommonGrams T3 callers transfer + // a precharged destination norm vector and a validated static metadata seed; + // the streamed session late-binds semantic token_count before finish. Only ONE + // session may be active per compound at a time, and begin_close() with an + // unfinished session fails instead of sealing a half-fed container. The + // handle is owned by this writer and valid until it is destroyed. + Status add_snii_index_streamed( + const TabletIndex* index_meta, uint32_t doc_count, + doris::snii::writer::TrackedNullDocids null_docids, + doris::snii::format::IndexConfig index_config, + std::shared_ptr mem_reporter, + doris::snii::writer::SniiStreamedIndexSession** session); + Status add_snii_index_streamed( + const TabletIndex* index_meta, uint32_t doc_count, + doris::snii::writer::TrackedNullDocids null_docids, + doris::snii::writer::TrackedEncodedNorms encoded_norms, + std::optional common_grams_metadata, + doris::snii::format::CommonGramsPostingPolicy common_grams_posting_policy, + doris::snii::format::IndexConfig index_config, + std::shared_ptr mem_reporter, + doris::snii::writer::SniiStreamedIndexSession** session); + // Registers one opaque BLOB logical index (a numeric BKD, an ANN graph, ...) + // on this SNII compound. Unlike add_snii_index it feeds the writer no terms: + // the sub-file bytes are pulled through the BlobFileSource callbacks at + // finish(), which is what lets the container -- not the producer -- decide + // cold/hot placement. Registration writes no byte, so a rejected call leaves + // the writer clean. + Status add_snii_blob_index(const TabletIndex* index_meta, + doris::snii::format::LogicalIndexKind kind, + std::vector cold_files, + std::vector hot_files); + void retain_snii_memory_reporter( + std::unique_ptr mem_reporter); + // SNII only, BUILD INDEX rewrite: copies the source container's valid + // physical prefix and registers the inherited metadata groups so begin_close + // re-emits them without decoding a posting. Must precede every + // add_snii_index on this writer (the copied prefix owns the container + // front). + Status inherit_snii(const doris::snii::reader::SniiRewriteSnapshot& snapshot, + doris::snii::io::FileReader* source); Status delete_index(const TabletIndex* index_meta); Status initialize(InvertedIndexDirectoryMap& indices_dirs); Status add_into_searcher_cache(); @@ -84,12 +175,31 @@ class IndexFileWriter { private: Status _insert_directory_into_map(int64_t index_id, const std::string& index_suffix, - std::shared_ptr dir); + std::shared_ptr dir); + // SNII only: registers a memory-backed staging directory for one ANN index, + // together with the metadata begin_close() needs to seal it. + Result> _open_snii_ann_staging_directory( + const TabletIndex* index_meta); virtual Result> _construct_index_searcher_builder( const DorisCompoundReader* dir); + // SNII only: turns every ANN staging directory into a blob logical index in + // the container. Runs once, from begin_close(), before the compound writer is + // sealed. Registration copies no byte -- the staged buffers are pulled by + // finish() through the blob sources. + Status _seal_snii_blob_directories(); + // Drops the staging directories once the container owns their bytes, or once + // sealing has failed and they are dead either way. Only the SNII path needs + // this: the V1/V2 branch of begin_close() releases its own directories + // inline. + void _release_snii_blob_directories(); // Member variables... InvertedIndexDirectoryMap _indices_dirs; + // SNII only: the index metadata behind each entry of _indices_dirs. Owned a + // copy rather than borrowed, because the harvest happens in begin_close(), + // long after open() returned. Held by shared_ptr so this header keeps + // TabletIndex incomplete -- it is included nearly everywhere. + std::map, std::shared_ptr> _snii_blob_dir_metas; const io::FileSystemSPtr _fs; std::string _index_path_prefix; std::string _rowset_id; @@ -113,6 +223,10 @@ class IndexFileWriter { IndexStorageFormatPtr _index_storage_format; int64_t _tablet_id = -1; + std::unique_ptr _snii_file_writer; + std::vector> _snii_memory_reporters; + std::unique_ptr _snii_compound_writer; + size_t _snii_index_count = 0; friend class IndexStorageFormatV1; friend class IndexStorageFormatV2; diff --git a/be/src/storage/index/index_query_context.h b/be/src/storage/index/index_query_context.h index 92fb698f4d3767..d0e8c23492cf0d 100644 --- a/be/src/storage/index/index_query_context.h +++ b/be/src/storage/index/index_query_context.h @@ -18,7 +18,7 @@ #pragma once #include "storage/compaction/collection_similarity.h" -#include "storage/compaction/collection_statistics.h" +#include "storage/index/inverted/similarity/collection_statistics.h" namespace doris::segment_v2 { @@ -32,6 +32,41 @@ struct IndexQueryContext { size_t query_limit = 0; bool is_asc = false; + + // G02 count-only fast-path handshake. Set by SegmentIterator ONLY while it + // evaluates the single pushed-down MATCH predicate of a COUNT_ON_INDEX scan + // whose row space is provably unfiltered (no deletes, no other conjuncts, + // full row bitmap, no row-id consumers -- see count_on_index_fastpath.h), + // and reset immediately after. When set, an index reader MAY answer the + // query with a bitmap whose CARDINALITY equals the match count without the + // row ids being real (SNII returns [0, df) straight from dict-entry df, + // skipping the posting decode). Readers must never cache such a bitmap + // under a key a row-accurate query could hit. + bool count_on_index_fastpath = false; + + // ---- Reply direction: fields a READER writes and the CALLER reads back ---- + // + // A caller that hands a reader a COPY of this context rather than the context itself must + // fold the copy back with merge_reader_outputs(), or the reader's reply is dropped in + // silence: nothing fails to compile, no test goes red, the query simply takes the wrong plan. + // FunctionSearch's SNII leaf builder is such a caller -- it copies the context so the reader + // publishes its BM25 into a throwaway CollectionSimilarity instead of the query's own. + // + // Every field added below this line must also be merged in merge_reader_outputs(). + + // G03 reply direction of the same handshake. Set by a reader iff it DID + // answer with such a fabricated count bitmap (never on a query-cache hit, + // a single-flight shared result, or any row-accurate decode). Read and + // reset by SegmentIterator right after the index apply; a true value is + // the precondition for the count-emission shortcut that materializes the + // remaining count as default rows without iterating the row bitmap. + bool count_on_index_fastpath_hit = false; + + // Folds the reply-direction fields a reader wrote on a copy of this context back into it. + // Latching (never clearing) is what makes this safe to call for each of several readers. + void merge_reader_outputs(const IndexQueryContext& reader_context) { + count_on_index_fastpath_hit |= reader_context.count_on_index_fastpath_hit; + } }; using IndexQueryContextPtr = std::shared_ptr; diff --git a/be/src/storage/index/index_writer.cpp b/be/src/storage/index/index_writer.cpp index 2325d280471337..64b1813622a373 100644 --- a/be/src/storage/index/index_writer.cpp +++ b/be/src/storage/index/index_writer.cpp @@ -18,6 +18,8 @@ #include "common/exception.h" #include "storage/index/ann/ann_index_writer.h" #include "storage/index/inverted/inverted_index_writer.h" +#include "storage/index/snii/snii_bkd_index_writer.h" +#include "storage/index/snii/snii_index_writer.h" #include "storage/tablet/tablet_schema.h" #include "storage/types.h" @@ -80,6 +82,28 @@ Status IndexColumnWriter::create(const TabletColumn* column, } } + if (storage_format == InvertedIndexStorageFormatPB::SNII) { + // The two SNII column writers split the same way the two readers do: + // text goes to the SPIMI term dictionary, numerics to the native BKD. + // Anything else has no representation in either and is refused here + // rather than producing an index no query path can serve. + if (is_string_type(type)) { + *res = std::make_unique(index_file_writer, index_meta, type); + } else if (field_is_numeric_type(type)) { + *res = std::make_unique(index_file_writer, index_meta, + type); + } else { + return Status::Error( + "SNII inverted index storage format does not support index type {}", type); + } + auto st = (*res)->init(); + if (!st.ok()) { + (*res)->close_on_error(); + return st; + } + return Status::OK(); + } + DBUG_EXECUTE_IF("InvertedIndexColumnWriter::create_unsupported_type_for_inverted_index", { type = FieldType::OLAP_FIELD_TYPE_JSONB; }) switch (type) { diff --git a/be/src/storage/index/index_writer.h b/be/src/storage/index/index_writer.h index a0760f99000fcb..def7a8fc4c72cc 100644 --- a/be/src/storage/index/index_writer.h +++ b/be/src/storage/index/index_writer.h @@ -59,6 +59,15 @@ class IndexColumnWriter { virtual Status add_nulls(uint32_t count) = 0; virtual Status add_array_nulls(const uint8_t* null_map, size_t num_rows) = 0; + // Write-path hint from the segment writer: this writer serves a direct load + // (stream/broker load, DataWriteType::TYPE_DIRECT) as opposed to compaction + // / schema change / index build. The column writer forwards it + // unconditionally right after create() succeeds (i.e. after init(), before + // any value is added); creation paths outside the column writer (e.g. ADD + // INDEX in index_builder.cpp) never call it and keep the non-direct default. + // Default no-op: SNII uses it to select the direct-load PRX zstd level. + virtual void set_direct_load(bool /*is_direct_load*/) {} + virtual Status finish() = 0; virtual int64_t size() const = 0; diff --git a/be/src/storage/index/inverted/abstract_analysis_factory.h b/be/src/storage/index/inverted/abstract_analysis_factory.h index f9afc42f19db93..3452f93fef7cdf 100644 --- a/be/src/storage/index/inverted/abstract_analysis_factory.h +++ b/be/src/storage/index/inverted/abstract_analysis_factory.h @@ -21,6 +21,19 @@ namespace doris::segment_v2::inverted_index { +enum class AnalysisPurpose { + kIndex, + kSniiTransientIndex, + kPlainQuery, + kExactPhraseQuery, + kPhrasePrefixQuery, +}; + +enum class PositionCapability { + kUnknown, + kAlwaysUnitIncrement, +}; + class AbstractAnalysisFactory { public: virtual ~AbstractAnalysisFactory() = default; diff --git a/be/src/storage/index/inverted/analysis_factory_mgr.cpp b/be/src/storage/index/inverted/analysis_factory_mgr.cpp index a208a275bda640..d41960fad206a9 100644 --- a/be/src/storage/index/inverted/analysis_factory_mgr.cpp +++ b/be/src/storage/index/inverted/analysis_factory_mgr.cpp @@ -21,6 +21,7 @@ #include "storage/index/inverted/char_filter/empty_char_filter_factory.h" #include "storage/index/inverted/char_filter/icu_normalizer_char_filter_factory.h" #include "storage/index/inverted/token_filter/ascii_folding_filter_factory.h" +#include "storage/index/inverted/token_filter/common_grams_filter_factory.h" #include "storage/index/inverted/token_filter/empty_token_filter_factory.h" #include "storage/index/inverted/token_filter/icu_normalizer_filter_factory.h" #include "storage/index/inverted/token_filter/lower_case_filter_factory.h" @@ -82,6 +83,8 @@ void AnalysisFactoryMgr::initialise() { "pinyin", []() { return std::make_shared(); }); registerFactory( "icu_normalizer", []() { return std::make_shared(); }); + registerFactory( + "common_grams", []() { return std::make_shared(); }); }); } diff --git a/be/src/storage/index/inverted/analyzer/analyzer.cpp b/be/src/storage/index/inverted/analyzer/analyzer.cpp index 6d16613bcd9b00..c9735ee032b4f7 100644 --- a/be/src/storage/index/inverted/analyzer/analyzer.cpp +++ b/be/src/storage/index/inverted/analyzer/analyzer.cpp @@ -40,8 +40,27 @@ #include "storage/index/inverted/analyzer/icu/icu_analyzer.h" #include "storage/index/inverted/analyzer/ik/IKAnalyzer.h" #include "storage/index/inverted/char_filter/char_replace_char_filter_factory.h" +#include "storage/index/inverted/token_filter/common_grams_filter.h" namespace doris::segment_v2::inverted_index { +namespace { + +class BuiltinAnalyzerProvider final : public AnalyzerProvider { +public: + explicit BuiltinAnalyzerProvider(InvertedIndexAnalyzerConfig config) + : _analyzer(InvertedIndexAnalyzer::create_builtin_analyzer( + config.analyzer_name.empty() + ? config.parser_type + : get_inverted_index_parser_type_from_string(config.analyzer_name), + config.parser_mode, config.lower_case, config.stop_words)) {} + + AnalyzerPtr get_analyzer(AnalysisPurpose) const override { return _analyzer; } + +private: + const AnalyzerPtr _analyzer; +}; + +} // namespace ReaderPtr InvertedIndexAnalyzer::create_reader(const CharFilterMap& char_filter_map) { ReaderPtr reader = std::make_shared>(); @@ -130,32 +149,43 @@ AnalyzerPtr InvertedIndexAnalyzer::create_builtin_analyzer(InvertedIndexParserTy } AnalyzerPtr InvertedIndexAnalyzer::create_analyzer(const InvertedIndexAnalyzerConfig* config) { - DCHECK(config != nullptr); - const std::string& analyzer_name = config->analyzer_name; - - // Handle empty analyzer name - use builtin analyzer based on parser_type. - // This is the common case when user does not specify USING ANALYZER. - if (analyzer_name.empty()) { - return create_builtin_analyzer(config->parser_type, config->parser_mode, config->lower_case, - config->stop_words); - } + return create_analyzer(config, AnalysisPurpose::kPlainQuery); +} - // Check if it's a builtin analyzer name (english, chinese, standard, etc.) - if (is_builtin_analyzer(analyzer_name)) { - InvertedIndexParserType parser_type = - get_inverted_index_parser_type_from_string(analyzer_name); +AnalyzerPtr InvertedIndexAnalyzer::create_analyzer(const InvertedIndexAnalyzerConfig* config, + AnalysisPurpose purpose) { + DCHECK(config != nullptr); + if (config->analyzer_name.empty() || is_builtin_analyzer(config->analyzer_name)) { + const InvertedIndexParserType parser_type = + config->analyzer_name.empty() + ? config->parser_type + : get_inverted_index_parser_type_from_string(config->analyzer_name); return create_builtin_analyzer(parser_type, config->parser_mode, config->lower_case, config->stop_words); } - // Custom analyzer - look up in policy manager auto* index_policy_mgr = doris::ExecEnv::GetInstance()->index_policy_mgr(); - if (!index_policy_mgr) { + if (index_policy_mgr == nullptr) { throw Exception(ErrorCode::INVERTED_INDEX_ANALYZER_ERROR, "Index policy manager is not initialized"); } + return index_policy_mgr->get_analyzer_by_name(config->analyzer_name, purpose); +} + +AnalyzerProviderPtr InvertedIndexAnalyzer::create_analyzer_provider( + const InvertedIndexAnalyzerConfig* config) { + DCHECK(config != nullptr); + if (config->analyzer_name.empty() || is_builtin_analyzer(config->analyzer_name)) { + return std::make_shared(*config); + } - return index_policy_mgr->get_policy_by_name(analyzer_name); + auto* index_policy_mgr = doris::ExecEnv::GetInstance()->index_policy_mgr(); + if (index_policy_mgr == nullptr) { + throw Exception(ErrorCode::INVERTED_INDEX_ANALYZER_ERROR, + "Index policy manager is not initialized"); + } + return index_policy_mgr->get_analyzer_provider_by_name(config->analyzer_name, + config->char_filter_map); } std::vector InvertedIndexAnalyzer::get_analyse_result( @@ -172,6 +202,8 @@ std::vector InvertedIndexAnalyzer::get_analyse_result( t.term = std::string(token.termBuffer(), token.termLength()); position += token.getPositionIncrement(); t.position = position; + t.key_kind = is_common_gram_token_type(token.type()) ? TermKeyKind::kCommonGram + : TermKeyKind::kPlain; analyse_result.emplace_back(std::move(t)); } } @@ -185,6 +217,12 @@ std::vector InvertedIndexAnalyzer::get_analyse_result( std::vector InvertedIndexAnalyzer::get_analyse_result( const std::string& search_str, const std::map& properties) { + return get_analyse_result(search_str, properties, AnalysisPurpose::kPlainQuery); +} + +std::vector InvertedIndexAnalyzer::get_analyse_result( + const std::string& search_str, const std::map& properties, + AnalysisPurpose purpose) { if (!should_analyzer(properties)) { // Keyword index: all strings (including empty) are valid tokens for exact match. // Empty string is a valid value in keyword index and should be matchable. @@ -200,12 +238,26 @@ std::vector InvertedIndexAnalyzer::get_analyse_result( config.lower_case = get_parser_lowercase_from_properties(properties); config.stop_words = get_parser_stopwords_from_properties(properties); config.char_filter_map = get_parser_char_filter_map_from_properties(properties); - auto analyzer = create_analyzer(&config); + auto analyzer = create_analyzer(&config, purpose); auto reader = create_reader(config.char_filter_map); reader->init(search_str.data(), static_cast(search_str.size()), true); return get_analyse_result(reader, analyzer.get()); } +AnalysisPurpose select_analysis_purpose(InvertedIndexQueryType query_type, int32_t slop, + bool is_similarity) { + if (is_similarity) { + return AnalysisPurpose::kPlainQuery; + } + if (query_type == InvertedIndexQueryType::MATCH_PHRASE_QUERY && slop == 0) { + return AnalysisPurpose::kExactPhraseQuery; + } + if (query_type == InvertedIndexQueryType::MATCH_PHRASE_PREFIX_QUERY) { + return AnalysisPurpose::kPhrasePrefixQuery; + } + return AnalysisPurpose::kPlainQuery; +} + bool InvertedIndexAnalyzer::should_analyzer(const std::map& properties) { auto parser_type = get_inverted_index_parser_type_from_string( get_parser_string_from_properties(properties)); diff --git a/be/src/storage/index/inverted/analyzer/analyzer.h b/be/src/storage/index/inverted/analyzer/analyzer.h index 98588a251bf047..4378b2e6c24b94 100644 --- a/be/src/storage/index/inverted/analyzer/analyzer.h +++ b/be/src/storage/index/inverted/analyzer/analyzer.h @@ -20,6 +20,7 @@ #include #include +#include "storage/index/inverted/analyzer/analyzer_provider.h" #include "storage/index/inverted/inverted_index_parser.h" #include "storage/index/inverted/inverted_index_query_type.h" #include "storage/index/inverted/query/query.h" @@ -49,14 +50,23 @@ class InvertedIndexAnalyzer { const std::string& lower_case, const std::string& stop_words); static AnalyzerPtr create_analyzer(const InvertedIndexAnalyzerConfig* config); + static AnalyzerPtr create_analyzer(const InvertedIndexAnalyzerConfig* config, + AnalysisPurpose purpose); + static AnalyzerProviderPtr create_analyzer_provider(const InvertedIndexAnalyzerConfig* config); static std::vector get_analyse_result(ReaderPtr reader, lucene::analysis::Analyzer* analyzer); static std::vector get_analyse_result( const std::string& search_str, const std::map& properties); + static std::vector get_analyse_result( + const std::string& search_str, const std::map& properties, + AnalysisPurpose purpose); static bool should_analyzer(const std::map& properties); }; +AnalysisPurpose select_analysis_purpose(InvertedIndexQueryType query_type, int32_t slop, + bool is_similarity); + } // namespace doris::segment_v2::inverted_index \ No newline at end of file diff --git a/be/src/storage/index/inverted/analyzer/analyzer_provider.h b/be/src/storage/index/inverted/analyzer/analyzer_provider.h new file mode 100644 index 00000000000000..60fe8250b00a85 --- /dev/null +++ b/be/src/storage/index/inverted/analyzer/analyzer_provider.h @@ -0,0 +1,46 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include "storage/index/inverted/abstract_analysis_factory.h" +#include "storage/index/inverted/common_grams/common_grams_segment_metadata.h" + +namespace lucene::analysis { +class Analyzer; +} + +namespace doris::segment_v2::inverted_index { + +class CommonWordSet; + +class AnalyzerProvider { +public: + virtual ~AnalyzerProvider() = default; + virtual std::shared_ptr get_analyzer( + AnalysisPurpose purpose) const = 0; + virtual std::string_view base_analyzer_fingerprint() const { return {}; } + virtual bool uses_common_grams() const { return false; } + virtual const CommonGramsQueryIdentity* common_grams_identity() const { return nullptr; } + virtual const CommonWordSet* common_grams_word_set() const { return nullptr; } +}; +using AnalyzerProviderPtr = std::shared_ptr; + +} // namespace doris::segment_v2::inverted_index diff --git a/be/src/storage/index/inverted/analyzer/custom_analyzer.cpp b/be/src/storage/index/inverted/analyzer/custom_analyzer.cpp index 3970d2e3cac74d..469f5ee2d6921a 100644 --- a/be/src/storage/index/inverted/analyzer/custom_analyzer.cpp +++ b/be/src/storage/index/inverted/analyzer/custom_analyzer.cpp @@ -17,12 +17,116 @@ #include "storage/index/inverted/analyzer/custom_analyzer.h" +#include +#include + #include "common/status.h" #include "runtime/exec_env.h" #include "storage/index/inverted/analysis_factory_mgr.h" +#include "storage/index/inverted/common_grams/common_word_set.h" +#include "storage/index/inverted/token_filter/common_grams_filter_factory.h" #include "storage/index/inverted/token_stream.h" +#include "util/sha.h" namespace doris::segment_v2::inverted_index { +namespace { + +bool config_uses_common_grams(const ImmutableCustomAnalyzerConfigPtr& config) { + DORIS_CHECK(config != nullptr); + const auto& filter_configs = config->get_token_filter_configs(); + return std::any_of(filter_configs.begin(), filter_configs.end(), + [](const auto& entry) { return entry->get_name() == "common_grams"; }); +} + +void append_canonical_value(std::string_view value, std::string* output) { + output->append(std::to_string(value.size())); + output->push_back(':'); + output->append(value); +} + +void append_component(std::string_view role, const ComponentConfigPtr& component, + std::string* output) { + append_canonical_value(role, output); + append_canonical_value(component->get_name(), output); + const auto entries = component->get_params().sorted_entries(); + append_canonical_value(std::to_string(entries.size()), output); + for (const auto& [key, value] : entries) { + append_canonical_value(key, output); + append_canonical_value(value, output); + } +} + +std::string sha256(std::string_view value) { + SHA256Digest digest; + digest.reset(value.data(), value.size()); + return std::string(digest.digest()); +} + +std::string calculate_base_analyzer_fingerprint_impl( + const ImmutableCustomAnalyzerConfigPtr& config, + const std::map& outer_char_filter_map) { + DORIS_CHECK(config != nullptr); + std::string base; + append_canonical_value("doris-common-grams-base-analyzer:v1", &base); + append_canonical_value("outer_char_filter", &base); + append_canonical_value(std::to_string(outer_char_filter_map.size()), &base); + for (const auto& [key, value] : outer_char_filter_map) { + append_canonical_value(key, &base); + append_canonical_value(value, &base); + } + append_component("tokenizer", config->get_tokenizer_config(), &base); + const auto char_filters = config->get_char_filter_configs(); + append_canonical_value(std::to_string(char_filters.size()), &base); + for (const auto& char_filter : char_filters) { + append_component("char_filter", char_filter, &base); + } + const auto token_filters = config->get_token_filter_configs(); + const auto base_token_filter_count = std::count_if( + token_filters.begin(), token_filters.end(), + [](const auto& token_filter) { return token_filter->get_name() != "common_grams"; }); + append_canonical_value(std::to_string(base_token_filter_count), &base); + for (const auto& token_filter : token_filters) { + if (token_filter->get_name() != "common_grams") { + append_component("token_filter", token_filter, &base); + } + } + return sha256(base); +} + +CommonGramsQueryIdentity build_common_grams_identity(std::string dictionary_identity, + std::string base_analyzer_fingerprint) { + std::string common_grams; + append_canonical_value("doris-common-grams:v1", &common_grams); + append_canonical_value(std::to_string(COMMON_GRAMS_SEMANTICS_VERSION_V1), &common_grams); + append_canonical_value(std::to_string(COMMON_GRAMS_KEY_VERSION_V1), &common_grams); + append_canonical_value(WORDSET_FORMAT_V1, &common_grams); + append_canonical_value(dictionary_identity, &common_grams); + return {.common_grams_dictionary_identity = std::move(dictionary_identity), + .base_analyzer_fingerprint = std::move(base_analyzer_fingerprint), + .common_grams_fingerprint = sha256(common_grams)}; +} + +std::array, 5> build_purpose_analyzers( + const ImmutableCustomAnalyzerConfigPtr& config, + const std::shared_ptr& common_words) { + DORIS_CHECK(config != nullptr); + const bool has_common_grams = config_uses_common_grams(config); + if (!has_common_grams) { + auto analyzer = CustomAnalyzer::build_custom_analyzer(config); + return {analyzer, analyzer, analyzer, analyzer, analyzer}; + } + return {CustomAnalyzer::build_custom_analyzer(config, AnalysisPurpose::kIndex, common_words), + CustomAnalyzer::build_custom_analyzer(config, AnalysisPurpose::kSniiTransientIndex, + common_words), + CustomAnalyzer::build_custom_analyzer(config, AnalysisPurpose::kPlainQuery, + common_words), + CustomAnalyzer::build_custom_analyzer(config, AnalysisPurpose::kExactPhraseQuery, + common_words), + CustomAnalyzer::build_custom_analyzer(config, AnalysisPurpose::kPhrasePrefixQuery, + common_words)}; +} + +} // namespace CustomAnalyzer::CustomAnalyzer(Builder* builder) { _tokenizer = builder->_tokenizer; @@ -74,7 +178,8 @@ TokenStreamComponentsPtr CustomAnalyzer::create_components() { return std::make_shared(tk, ts); } -CustomAnalyzerPtr CustomAnalyzer::build_custom_analyzer(const CustomAnalyzerConfigPtr& config) { +CustomAnalyzerPtr CustomAnalyzer::build_custom_analyzer( + const ImmutableCustomAnalyzerConfigPtr& config) { if (config == nullptr) { throw Exception(ErrorCode::ILLEGAL_STATE, "Null configuration detected."); } @@ -90,6 +195,134 @@ CustomAnalyzerPtr CustomAnalyzer::build_custom_analyzer(const CustomAnalyzerConf return builder.build(); } +CustomAnalyzerPtr CustomAnalyzer::build_custom_analyzer( + const ImmutableCustomAnalyzerConfigPtr& config, AnalysisPurpose purpose) { + return build_custom_analyzer(config, purpose, CommonWordSet::default_word_set()); +} + +CustomAnalyzerPtr CustomAnalyzer::build_custom_analyzer( + const ImmutableCustomAnalyzerConfigPtr& config, AnalysisPurpose purpose, + const std::shared_ptr& common_words) { + if (config == nullptr) { + throw Exception(ErrorCode::ILLEGAL_STATE, "Null configuration detected."); + } + + CustomAnalyzer::Builder builder; + for (const auto& filter_config : config->get_char_filter_configs()) { + builder.add_char_filter(filter_config->get_name(), filter_config->get_params()); + } + builder.with_tokenizer(config->get_tokenizer_config()->get_name(), + config->get_tokenizer_config()->get_params()); + + const auto filter_configs = config->get_token_filter_configs(); + const size_t common_grams_count = + std::count_if(filter_configs.begin(), filter_configs.end(), + [](const auto& entry) { return entry->get_name() == "common_grams"; }); + if (common_grams_count == 0) { + for (const auto& filter_config : filter_configs) { + builder.add_token_filter(filter_config->get_name(), filter_config->get_params()); + } + return builder.build(); + } + if (common_grams_count != 1 || filter_configs.back()->get_name() != "common_grams") { + throw Exception(ErrorCode::INVERTED_INDEX_ANALYZER_ERROR, + "common_grams must appear exactly once as the terminal token filter"); + } + if (builder._tokenizer->position_capability() != PositionCapability::kAlwaysUnitIncrement) { + throw Exception(ErrorCode::INVERTED_INDEX_ANALYZER_ERROR, + "CommonGrams tokenizer does not guarantee unit position increments"); + } + + for (size_t i = 0; i + 1 < filter_configs.size(); ++i) { + auto factory = AnalysisFactoryMgr::instance().create( + filter_configs[i]->get_name(), filter_configs[i]->get_params()); + if (factory->position_capability() != PositionCapability::kAlwaysUnitIncrement) { + throw Exception(ErrorCode::INVERTED_INDEX_ANALYZER_ERROR, + "CommonGrams token filter '{}' does not guarantee unit position " + "increments", + filter_configs[i]->get_name()); + } + builder._token_filters.push_back(std::move(factory)); + } + + auto common_grams = AnalysisFactoryMgr::instance().create( + filter_configs.back()->get_name(), filter_configs.back()->get_params()); + auto common_grams_factory = std::dynamic_pointer_cast(common_grams); + DORIS_CHECK(common_grams_factory != nullptr); + common_grams_factory->set_common_words(common_words); + switch (purpose) { + case AnalysisPurpose::kIndex: + common_grams_factory->set_output_mode(CommonGramsOutputMode::kEscapedV1Index); + builder._token_filters.push_back(std::move(common_grams)); + break; + case AnalysisPurpose::kSniiTransientIndex: + common_grams_factory->set_output_mode(CommonGramsOutputMode::kEscapedV1SpimiIndex); + builder._token_filters.push_back(std::move(common_grams)); + break; + case AnalysisPurpose::kPlainQuery: { + auto factory = std::make_shared(); + factory->initialize({}); + builder._token_filters.push_back(std::move(factory)); + break; + } + case AnalysisPurpose::kExactPhraseQuery: { + builder._token_filters.push_back(std::move(common_grams)); + auto factory = std::make_shared(common_words); + factory->initialize({}); + builder._token_filters.push_back(std::move(factory)); + break; + } + case AnalysisPurpose::kPhrasePrefixQuery: { + builder._token_filters.push_back(std::move(common_grams)); + auto factory = std::make_shared(common_words); + factory->initialize({}); + builder._token_filters.push_back(std::move(factory)); + break; + } + } + return builder.build(); +} + +CustomAnalyzerProvider::CustomAnalyzerProvider( + ImmutableCustomAnalyzerConfigPtr config, + std::map outer_char_filter_map) + : _config(std::move(config)), + _base_analyzer_fingerprint( + calculate_base_analyzer_fingerprint(_config, outer_char_filter_map)), + _uses_common_grams(config_uses_common_grams(_config)) { + _common_words = CommonWordSet::default_word_set(); + _analyzers = build_purpose_analyzers(_config, _common_words); + if (_uses_common_grams) { + // Content-derived, so a BE reading a segment grammed against a different word list sees a + // mismatched identity and falls back to the plain plan instead of trusting its grams. + _common_grams_identity = + build_common_grams_identity(_common_words->identity(), _base_analyzer_fingerprint); + } +} + +std::string CustomAnalyzerProvider::calculate_base_analyzer_fingerprint( + const ImmutableCustomAnalyzerConfigPtr& config, + const std::map& outer_char_filter_map) { + return calculate_base_analyzer_fingerprint_impl(config, outer_char_filter_map); +} + +std::shared_ptr CustomAnalyzerProvider::get_analyzer( + AnalysisPurpose purpose) const { + switch (purpose) { + case AnalysisPurpose::kIndex: + return _analyzers[0]; + case AnalysisPurpose::kSniiTransientIndex: + return _analyzers[1]; + case AnalysisPurpose::kPlainQuery: + return _analyzers[2]; + case AnalysisPurpose::kExactPhraseQuery: + return _analyzers[3]; + case AnalysisPurpose::kPhrasePrefixQuery: + return _analyzers[4]; + } + __builtin_unreachable(); +} + void CustomAnalyzer::Builder::with_tokenizer(const std::string& name, const Settings& params) { _tokenizer = AnalysisFactoryMgr::instance().create(name, params); } diff --git a/be/src/storage/index/inverted/analyzer/custom_analyzer.h b/be/src/storage/index/inverted/analyzer/custom_analyzer.h index 68565355496d68..9f89a442912222 100644 --- a/be/src/storage/index/inverted/analyzer/custom_analyzer.h +++ b/be/src/storage/index/inverted/analyzer/custom_analyzer.h @@ -17,6 +17,11 @@ #pragma once +#include +#include +#include + +#include "storage/index/inverted/analyzer/analyzer_provider.h" #include "storage/index/inverted/analyzer/custom_analyzer_config.h" #include "storage/index/inverted/char_filter/char_filter_factory.h" #include "storage/index/inverted/setting.h" @@ -25,6 +30,7 @@ namespace doris::segment_v2::inverted_index { +class CommonWordSet; class CustomAnalyzer; using CustomAnalyzerPtr = std::shared_ptr; @@ -60,7 +66,12 @@ class CustomAnalyzer : public Analyzer { TokenStream* tokenStream(const TCHAR* fieldName, const ReaderPtr& reader) override; TokenStream* reusableTokenStream(const TCHAR* fieldName, const ReaderPtr& reader) override; - static CustomAnalyzerPtr build_custom_analyzer(const CustomAnalyzerConfigPtr& config); + static CustomAnalyzerPtr build_custom_analyzer(const ImmutableCustomAnalyzerConfigPtr& config); + static CustomAnalyzerPtr build_custom_analyzer(const ImmutableCustomAnalyzerConfigPtr& config, + AnalysisPurpose purpose); + static CustomAnalyzerPtr build_custom_analyzer( + const ImmutableCustomAnalyzerConfigPtr& config, AnalysisPurpose purpose, + const std::shared_ptr& common_words); private: ReaderPtr init_reader(ReaderPtr reader); @@ -73,4 +84,39 @@ class CustomAnalyzer : public Analyzer { TokenStreamComponentsPtr _reuse_token_stream; }; +class CustomAnalyzerProvider final : public AnalyzerProvider { +public: + // The CommonGrams word list is not a parameter: it is the BE-local + // CommonWordSet::default_word_set(), and the dictionary identity stamped into segments comes + // from that set's content. An index policy cannot choose either one. + explicit CustomAnalyzerProvider(ImmutableCustomAnalyzerConfigPtr config, + std::map outer_char_filter_map = {}); + + std::shared_ptr get_analyzer( + AnalysisPurpose purpose) const override; + std::string_view base_analyzer_fingerprint() const override { + return _base_analyzer_fingerprint; + } + bool uses_common_grams() const override { return _uses_common_grams; } + const CommonGramsQueryIdentity* common_grams_identity() const override { + return _common_grams_identity ? &*_common_grams_identity : nullptr; + } + const CommonWordSet* common_grams_word_set() const override { + return _uses_common_grams ? _common_words.get() : nullptr; + } + const std::shared_ptr& common_words() const { return _common_words; } + + static std::string calculate_base_analyzer_fingerprint( + const ImmutableCustomAnalyzerConfigPtr& config, + const std::map& outer_char_filter_map = {}); + +private: + ImmutableCustomAnalyzerConfigPtr _config; + const std::string _base_analyzer_fingerprint; + std::shared_ptr _common_words; + bool _uses_common_grams = false; + std::optional _common_grams_identity; + std::array, 5> _analyzers; +}; + } // namespace doris::segment_v2::inverted_index \ No newline at end of file diff --git a/be/src/storage/index/inverted/analyzer/custom_analyzer_config.cpp b/be/src/storage/index/inverted/analyzer/custom_analyzer_config.cpp index b946c13e17d207..0a4c6a5da3c09e 100644 --- a/be/src/storage/index/inverted/analyzer/custom_analyzer_config.cpp +++ b/be/src/storage/index/inverted/analyzer/custom_analyzer_config.cpp @@ -27,15 +27,15 @@ CustomAnalyzerConfig::CustomAnalyzerConfig(Builder* builder) { _token_filters = builder->_token_filters; } -ComponentConfigPtr CustomAnalyzerConfig::get_tokenizer_config() { +ComponentConfigPtr CustomAnalyzerConfig::get_tokenizer_config() const { return _tokenizer_config; } -std::vector CustomAnalyzerConfig::get_char_filter_configs() { +std::vector CustomAnalyzerConfig::get_char_filter_configs() const { return _char_filters; } -std::vector CustomAnalyzerConfig::get_token_filter_configs() { +std::vector CustomAnalyzerConfig::get_token_filter_configs() const { return _token_filters; } diff --git a/be/src/storage/index/inverted/analyzer/custom_analyzer_config.h b/be/src/storage/index/inverted/analyzer/custom_analyzer_config.h index 619d9ae78c9dac..7f6d55e8d2d11a 100644 --- a/be/src/storage/index/inverted/analyzer/custom_analyzer_config.h +++ b/be/src/storage/index/inverted/analyzer/custom_analyzer_config.h @@ -29,6 +29,7 @@ using ComponentConfigPtr = std::shared_ptr; class CustomAnalyzerConfig; using CustomAnalyzerConfigPtr = std::shared_ptr; +using ImmutableCustomAnalyzerConfigPtr = std::shared_ptr; class CustomAnalyzerConfig { public: @@ -53,9 +54,9 @@ class CustomAnalyzerConfig { CustomAnalyzerConfig(Builder* builder); ~CustomAnalyzerConfig() = default; - ComponentConfigPtr get_tokenizer_config(); - std::vector get_char_filter_configs(); - std::vector get_token_filter_configs(); + ComponentConfigPtr get_tokenizer_config() const; + std::vector get_char_filter_configs() const; + std::vector get_token_filter_configs() const; private: ComponentConfigPtr _tokenizer_config; diff --git a/be/src/storage/index/inverted/analyzer/segment_analyzer_context.cpp b/be/src/storage/index/inverted/analyzer/segment_analyzer_context.cpp new file mode 100644 index 00000000000000..804f4646f69177 --- /dev/null +++ b/be/src/storage/index/inverted/analyzer/segment_analyzer_context.cpp @@ -0,0 +1,110 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/inverted/analyzer/segment_analyzer_context.h" + +#include + +#include "common/exception.h" +#include "runtime/index_policy/index_policy_mgr.h" + +namespace doris::segment_v2::inverted_index { +namespace { + +Result> analyzer_bypass(std::string_view reason) { + return ResultError(Status::Error( + "segment analyzer unavailable: {}", reason)); +} + +} // namespace + +Result> maybe_rebuild_segment_analyzer_context( + const InvertedIndexAnalyzerCtx* request_context, + const std::optional& segment_metadata, + const std::map& physical_index_properties, + IndexPolicyMgr* index_policy_mgr) { + return maybe_rebuild_segment_analyzer_context(request_context, + segment_metadata ? &*segment_metadata : nullptr, + physical_index_properties, index_policy_mgr); +} + +Result> maybe_rebuild_segment_analyzer_context( + const InvertedIndexAnalyzerCtx* request_context, + const CommonGramsSegmentMetadata* segment_metadata, + const std::map& physical_index_properties, + IndexPolicyMgr* index_policy_mgr) { + if (segment_metadata == nullptr) { + return std::optional {}; + } + if (segment_metadata->base_analyzer_fingerprint.empty()) { + return analyzer_bypass("typed metadata has no base analyzer fingerprint"); + } + return maybe_rebuild_segment_analyzer_context(request_context, + segment_metadata->base_analyzer_fingerprint, + physical_index_properties, index_policy_mgr); +} + +Result> maybe_rebuild_segment_analyzer_context( + const InvertedIndexAnalyzerCtx* request_context, std::string_view segment_base_fingerprint, + const std::map& physical_index_properties, + IndexPolicyMgr* index_policy_mgr) { + DORIS_CHECK(!segment_base_fingerprint.empty()); + if (request_context == nullptr || request_context->analyzer_provider == nullptr || + !request_context->requires_analysis()) { + return analyzer_bypass("query context cannot reconstruct the physical token stream"); + } + + std::string_view request_base_fingerprint = + request_context->analyzer_provider->base_analyzer_fingerprint(); + if (request_base_fingerprint.empty()) { + const auto* identity = request_context->get_common_grams_identity(); + if (identity != nullptr) { + request_base_fingerprint = identity->base_analyzer_fingerprint; + } + } + if (request_base_fingerprint == segment_base_fingerprint) { + return std::optional {}; + } + if (index_policy_mgr == nullptr) { + return analyzer_bypass("index policy manager is not initialized"); + } + + const CharFilterMap physical_char_filter_map = + get_parser_char_filter_map_from_properties(physical_index_properties); + AnalyzerProviderPtr provider; + try { + provider = index_policy_mgr->get_analyzer_provider_by_base_fingerprint( + segment_base_fingerprint, physical_char_filter_map); + } catch (const CLuceneError& error) { + return analyzer_bypass(error.what()); + } catch (const Exception& error) { + return analyzer_bypass(error.what()); + } + if (provider == nullptr) { + return analyzer_bypass("no installed policy matches the segment base fingerprint"); + } + DORIS_CHECK(provider->base_analyzer_fingerprint() == segment_base_fingerprint); + + InvertedIndexAnalyzerCtx effective_context = *request_context; + effective_context.char_filter_map = physical_char_filter_map; + effective_context.analyzer.reset(); + effective_context.analyzer_provider = std::move(provider); + effective_context.common_grams_identity.reset(); + return std::optional(std::move(effective_context)); +} + +} // namespace doris::segment_v2::inverted_index diff --git a/be/src/storage/index/inverted/analyzer/segment_analyzer_context.h b/be/src/storage/index/inverted/analyzer/segment_analyzer_context.h new file mode 100644 index 00000000000000..06be7eeb9e4811 --- /dev/null +++ b/be/src/storage/index/inverted/analyzer/segment_analyzer_context.h @@ -0,0 +1,57 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/inverted/common_grams/common_grams_segment_metadata.h" +#include "storage/index/inverted/inverted_index_parser.h" + +namespace doris { + +class IndexPolicyMgr; + +namespace segment_v2::inverted_index { + +// A missing metadata record is a legacy segment and keeps the request analyzer. A present record +// must identify its base analyzer; otherwise the caller must bypass the inverted index. +Result> maybe_rebuild_segment_analyzer_context( + const InvertedIndexAnalyzerCtx* request_context, + const std::optional& segment_metadata, + const std::map& physical_index_properties, + IndexPolicyMgr* index_policy_mgr); + +Result> maybe_rebuild_segment_analyzer_context( + const InvertedIndexAnalyzerCtx* request_context, + const CommonGramsSegmentMetadata* segment_metadata, + const std::map& physical_index_properties, + IndexPolicyMgr* index_policy_mgr); + +// Returns nullopt when the request analyzer already matches the persisted segment analyzer. +// A returned context owns a fresh provider and must remain local to one query execution. +Result> maybe_rebuild_segment_analyzer_context( + const InvertedIndexAnalyzerCtx* request_context, std::string_view segment_base_fingerprint, + const std::map& physical_index_properties, + IndexPolicyMgr* index_policy_mgr); + +} // namespace segment_v2::inverted_index +} // namespace doris diff --git a/be/src/storage/index/inverted/common/single_flight.h b/be/src/storage/index/inverted/common/single_flight.h new file mode 100644 index 00000000000000..3d360afc4a2b26 --- /dev/null +++ b/be/src/storage/index/inverted/common/single_flight.h @@ -0,0 +1,109 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/check.h" + +namespace doris::segment_v2::inverted_index { + +// Collapses concurrent operations with the same key into one execution. No work +// or follower wake-up runs while the selected shard mutex is held. +template +class SingleFlight { +public: + using ResultFuture = std::shared_future; + + std::optional join_or_lead(const std::string& key) { + auto& shard = _shard_for(key); + std::lock_guard guard(shard.mutex); + if (auto it = shard.inflight.find(key); it != shard.inflight.end()) { + return it->second->future; + } + auto flight = std::make_shared(); + flight->future = flight->promise.get_future().share(); + shard.inflight.emplace(key, std::move(flight)); + return std::nullopt; + } + + void publish(const std::string& key, Result result) { + auto& shard = _shard_for(key); + std::shared_ptr flight; + { + std::lock_guard guard(shard.mutex); + auto it = shard.inflight.find(key); + if (it == shard.inflight.end() || it->second->publishing) { + return; + } + flight = it->second; + flight->publishing = true; + } + flight->promise.set_value(std::move(result)); + { + std::lock_guard guard(shard.mutex); + auto it = shard.inflight.find(key); + DORIS_CHECK(it != shard.inflight.end()); + DORIS_CHECK(it->second == flight); + shard.inflight.erase(it); + } + } + + size_t inflight_size() const { + std::array, kShardCount> guards; + for (size_t i = 0; i < kShardCount; ++i) { + guards[i] = std::unique_lock(_shards[i].mutex); + } + + size_t size = 0; + for (const auto& shard : _shards) { + size += shard.inflight.size(); + } + return size; + } + +private: + struct Flight { + std::promise promise; + ResultFuture future; + bool publishing = false; + }; + + struct Shard { + mutable std::mutex mutex; + std::unordered_map> inflight; + }; + + static constexpr size_t kShardCount = 64; + + Shard& _shard_for(const std::string& key) { + return _shards[std::hash {}(key) % kShardCount]; + } + + std::array _shards; +}; + +} // namespace doris::segment_v2::inverted_index diff --git a/be/src/storage/index/inverted/common_grams/common_grams_key_codec.cpp b/be/src/storage/index/inverted/common_grams/common_grams_key_codec.cpp new file mode 100644 index 00000000000000..98f0e3ac918643 --- /dev/null +++ b/be/src/storage/index/inverted/common_grams/common_grams_key_codec.cpp @@ -0,0 +1,279 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/inverted/common_grams/common_grams_key_codec.h" + +#include +#include + +#include "util/utf8_check.h" + +namespace doris::segment_v2::inverted_index { +namespace { + +constexpr size_t COMMON_GRAM_LENGTH_HEX_BYTES = 8; +constexpr size_t COMMON_GRAM_SEPARATOR_BYTES = 1; +constexpr size_t COMMON_GRAM_FIXED_BYTES = + CG_V1_MARKER.size() + COMMON_GRAM_LENGTH_HEX_BYTES + COMMON_GRAM_SEPARATOR_BYTES; +constexpr std::string_view HEX_DIGITS = "0123456789abcdef"; +constexpr std::string_view LEGACY_PHRASE_BIGRAM_MARKER = + "\x1f" + "SNII_PHRASE_BIGRAM" + "\x1f"; + +ResultError analyzer_error(std::string_view message) { + return ResultError(Status::Error("{}", message)); +} + +bool prefixes_overlap(std::string_view left, std::string_view right) { + return left.starts_with(right) || right.starts_with(left); +} + +} // namespace + +Status validate_common_grams_logical_term(std::string_view term, std::string_view component) { + if (term.find('\0') != std::string_view::npos) { + return Status::Error( + "CommonGrams {} contains NUL", component); + } + if (!validate_utf8(term.data(), term.size())) { + return Status::Error( + "CommonGrams {} is not valid UTF-8", component); + } + if (term.size() > COMMON_GRAM_MAX_ENCODED_BYTES) { + return Status::Error( + "CommonGrams {} has {} UTF-8 bytes, exceeding {}", component, term.size(), + COMMON_GRAM_MAX_ENCODED_BYTES); + } + return Status::OK(); +} + +Result encode_plain_term(std::string_view term, PlainTermKeyVersion version) { + std::string encoded; + auto encoded_result = try_encode_plain_term(term, version, &encoded); + if (!encoded_result.has_value()) { + return ResultError(std::move(encoded_result.error())); + } + if (!encoded_result.value()) { + return analyzer_error("escaped plain term would exceed the 16383-byte key limit"); + } + return encoded; +} + +Result try_encode_plain_term(std::string_view term, PlainTermKeyVersion version, + std::string* output) { + DORIS_CHECK(output != nullptr); + output->clear(); + auto status = validate_common_grams_logical_term(term, "plain term"); + if (!status.ok()) { + return ResultError(std::move(status)); + } + + switch (version) { + case PlainTermKeyVersion::kLegacyRaw: + case PlainTermKeyVersion::kRawNoInternal: + output->assign(term); + return true; + case PlainTermKeyVersion::kEscapedV1: + if (term.empty() || (term.front() != PLAIN_ESCAPE_PREFIX && term.front() != '\x1f')) { + output->assign(term); + return true; + } + return try_encode_escaped_plain_term_prevalidated(term, *output); + } + return analyzer_error("unknown plain-term key version"); +} + +Result> try_encode_plain_term_view(std::string_view term, + PlainTermKeyVersion version, + std::string* scratch) { + DORIS_CHECK(scratch != nullptr); + scratch->clear(); + auto status = validate_common_grams_logical_term(term, "plain term"); + if (!status.ok()) { + return ResultError(std::move(status)); + } + + switch (version) { + case PlainTermKeyVersion::kLegacyRaw: + case PlainTermKeyVersion::kRawNoInternal: + return std::optional(term); + case PlainTermKeyVersion::kEscapedV1: + if (term.empty() || (term.front() != PLAIN_ESCAPE_PREFIX && term.front() != '\x1f')) { + return std::optional(term); + } + if (!try_encode_escaped_plain_term_prevalidated(term, *scratch)) { + return std::optional(); + } + return std::optional(*scratch); + } + return analyzer_error("unknown plain-term key version"); +} + +bool try_encode_escaped_plain_term_prevalidated(std::string_view logical_term, + std::string& output) { + DCHECK(!logical_term.empty()); + DCHECK(logical_term.front() == PLAIN_ESCAPE_PREFIX || logical_term.front() == '\x1f'); + DCHECK(validate_common_grams_logical_term(logical_term, "plain term").ok()); + output.clear(); + if (logical_term.size() == COMMON_GRAM_MAX_ENCODED_BYTES) { + return false; + } + output.reserve(logical_term.size() + 1); + output.push_back(PLAIN_ESCAPE_PREFIX); + output.push_back(logical_term.front() == PLAIN_ESCAPE_PREFIX ? 'E' : 'G'); + output.append(logical_term.substr(1)); + return true; +} + +Result decode_plain_term_view(std::string_view term, PlainTermKeyVersion version, + std::string* scratch) { + DORIS_CHECK(scratch != nullptr); + scratch->clear(); + if (term.size() > COMMON_GRAM_MAX_ENCODED_BYTES) { + return analyzer_error("encoded plain term exceeds the 16383-byte key limit"); + } + + switch (version) { + case PlainTermKeyVersion::kLegacyRaw: + case PlainTermKeyVersion::kRawNoInternal: { + auto status = validate_common_grams_logical_term(term, "plain term"); + if (!status.ok()) { + return ResultError(std::move(status)); + } + return term; + } + case PlainTermKeyVersion::kEscapedV1: { + if (term.empty() || term.front() != PLAIN_ESCAPE_PREFIX) { + if (!term.empty() && term.front() == '\x1f') { + return analyzer_error("escaped plain term enters the internal namespace"); + } + auto status = validate_common_grams_logical_term(term, "plain term"); + if (!status.ok()) { + return ResultError(std::move(status)); + } + return term; + } + if (term.size() < 2 || (term[1] != 'E' && term[1] != 'G')) { + return analyzer_error("invalid plain_term_escape:v1 key"); + } + scratch->reserve(term.size() - 1); + scratch->push_back(term[1] == 'E' ? PLAIN_ESCAPE_PREFIX : '\x1f'); + scratch->append(term.substr(2)); + auto status = validate_common_grams_logical_term(*scratch, "plain term"); + if (!status.ok()) { + return ResultError(std::move(status)); + } + return std::string_view(*scratch); + } + } + return analyzer_error("unknown plain-term key version"); +} + +Result decode_plain_term(std::string_view term, PlainTermKeyVersion version) { + std::string scratch; + auto decoded = decode_plain_term_view(term, version, &scratch); + if (!decoded.has_value()) { + return ResultError(std::move(decoded.error())); + } + return std::string(*decoded); +} + +bool is_internal_term_key(std::string_view physical_term) { + return physical_term.starts_with(INTERNAL_TERM_NAMESPACE_BEGIN); +} + +bool legacy_raw_exact_requires_bypass(std::string_view logical_term) { + return logical_term.starts_with(CG_V1_MARKER) || + logical_term.starts_with(LEGACY_PHRASE_BIGRAM_MARKER); +} + +bool legacy_raw_prefix_requires_bypass(std::string_view logical_prefix) { + return prefixes_overlap(logical_prefix, CG_V1_MARKER) || + prefixes_overlap(logical_prefix, LEGACY_PHRASE_BIGRAM_MARKER); +} + +bool is_common_gram_encodable(std::string_view left, std::string_view right) { + if (!validate_common_grams_logical_term(left, "left term").ok() || + !validate_common_grams_logical_term(right, "right term").ok()) { + return false; + } + return is_common_gram_encodable_prevalidated(left, right); +} + +bool is_common_gram_encodable_prevalidated(std::string_view left, std::string_view right) { + DCHECK(validate_common_grams_logical_term(left, "left term").ok()); + DCHECK(validate_common_grams_logical_term(right, "right term").ok()); + return common_gram_component_sizes_encodable(left.size(), right.size()); +} + +bool common_gram_component_sizes_encodable(size_t left_size, size_t right_size) { + return left_size <= COMMON_GRAM_MAX_ENCODED_BYTES - COMMON_GRAM_FIXED_BYTES && + right_size <= COMMON_GRAM_MAX_ENCODED_BYTES - COMMON_GRAM_FIXED_BYTES - left_size; +} + +Result try_encode_common_gram(std::string_view left, std::string_view right, + std::string* output) { + DORIS_CHECK(output != nullptr); + output->clear(); + auto left_status = validate_common_grams_logical_term(left, "left term"); + if (!left_status.ok()) { + return ResultError(std::move(left_status)); + } + auto right_status = validate_common_grams_logical_term(right, "right term"); + if (!right_status.ok()) { + return ResultError(std::move(right_status)); + } + return try_encode_common_gram_prevalidated(left, right, *output); +} + +bool try_encode_common_gram_prevalidated(std::string_view left, std::string_view right, + std::string& output) { + DCHECK(validate_common_grams_logical_term(left, "left term").ok()); + DCHECK(validate_common_grams_logical_term(right, "right term").ok()); + output.clear(); + if (COMMON_GRAM_FIXED_BYTES + left.size() + right.size() > COMMON_GRAM_MAX_ENCODED_BYTES) { + return false; + } + + output.reserve(COMMON_GRAM_FIXED_BYTES + left.size() + right.size()); + output.append(CG_V1_MARKER); + char length_and_separator[COMMON_GRAM_LENGTH_HEX_BYTES + COMMON_GRAM_SEPARATOR_BYTES]; + for (size_t i = 0; i < COMMON_GRAM_LENGTH_HEX_BYTES; ++i) { + const size_t shift = (COMMON_GRAM_LENGTH_HEX_BYTES - i - 1) * 4; + length_and_separator[i] = HEX_DIGITS[(left.size() >> shift) & 0xf]; + } + length_and_separator[COMMON_GRAM_LENGTH_HEX_BYTES] = ':'; + output.append(length_and_separator, sizeof(length_and_separator)); + output.append(left); + output.append(right); + return true; +} + +Result encode_common_gram(std::string_view left, std::string_view right) { + std::string encoded; + auto encoded_result = try_encode_common_gram(left, right, &encoded); + if (!encoded_result.has_value()) { + return ResultError(std::move(encoded_result.error())); + } + if (!encoded_result.value()) { + return analyzer_error("encoded common gram would exceed the 16383-byte key limit"); + } + return encoded; +} + +} // namespace doris::segment_v2::inverted_index diff --git a/be/src/storage/index/inverted/common_grams/common_grams_key_codec.h b/be/src/storage/index/inverted/common_grams/common_grams_key_codec.h new file mode 100644 index 00000000000000..07869877a55d01 --- /dev/null +++ b/be/src/storage/index/inverted/common_grams/common_grams_key_codec.h @@ -0,0 +1,72 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include + +#include "common/status.h" + +namespace doris::segment_v2::inverted_index { + +inline constexpr size_t COMMON_GRAM_MAX_ENCODED_BYTES = 16383; +inline constexpr char PLAIN_ESCAPE_PREFIX = '\x1e'; +inline constexpr std::string_view INTERNAL_TERM_NAMESPACE_BEGIN {"\x1f", 1}; +inline constexpr std::string_view INTERNAL_TERM_NAMESPACE_END {"\x20", 1}; +inline constexpr std::string_view CG_V1_MARKER = + "\x1f" + "DORIS_COMMON_GRAM_V1" + "\x1f"; +inline constexpr std::string_view CG_V1_MARKER_END = + "\x1f" + "DORIS_COMMON_GRAM_V1" + "\x20"; + +enum class PlainTermKeyVersion : uint8_t { + kLegacyRaw = 0, + kEscapedV1 = 1, + kRawNoInternal = 2, +}; + +Result encode_plain_term(std::string_view term, PlainTermKeyVersion version); +Result try_encode_plain_term(std::string_view term, PlainTermKeyVersion version, + std::string* output); +Result> try_encode_plain_term_view(std::string_view term, + PlainTermKeyVersion version, + std::string* scratch); +bool try_encode_escaped_plain_term_prevalidated(std::string_view logical_term, std::string& output); +Result decode_plain_term_view(std::string_view term, PlainTermKeyVersion version, + std::string* scratch); +Result decode_plain_term(std::string_view term, PlainTermKeyVersion version); +bool is_internal_term_key(std::string_view physical_term); +bool legacy_raw_exact_requires_bypass(std::string_view logical_term); +bool legacy_raw_prefix_requires_bypass(std::string_view logical_prefix); +Status validate_common_grams_logical_term(std::string_view term, std::string_view component); +Result try_encode_common_gram(std::string_view left, std::string_view right, + std::string* output); +bool try_encode_common_gram_prevalidated(std::string_view left, std::string_view right, + std::string& output); +bool common_gram_component_sizes_encodable(size_t left_size, size_t right_size); +bool is_common_gram_encodable_prevalidated(std::string_view left, std::string_view right); +Result encode_common_gram(std::string_view left, std::string_view right); +bool is_common_gram_encodable(std::string_view left, std::string_view right); + +} // namespace doris::segment_v2::inverted_index diff --git a/be/src/storage/index/inverted/common_grams/common_grams_query_cost.h b/be/src/storage/index/inverted/common_grams/common_grams_query_cost.h new file mode 100644 index 00000000000000..eee4be344c3661 --- /dev/null +++ b/be/src/storage/index/inverted/common_grams/common_grams_query_cost.h @@ -0,0 +1,52 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +namespace doris::segment_v2::inverted_index { + +struct CommonGramsPlanRawCost { + uint64_t posting_bytes_or_df_sum = 0; + uint64_t estimated_candidate_df = 0; + uint32_t clause_count = 0; +}; + +struct CommonGramsPlanCostModel { + uint32_t position_verify_factor = 0; + uint32_t common_grams_cost_ratio_percent = 85; +}; + +inline uint64_t estimate_common_grams_plan_cost(const CommonGramsPlanRawCost& input, + uint32_t position_verify_factor) { + const unsigned __int128 estimate = + static_cast(input.posting_bytes_or_df_sum) + + static_cast(input.estimated_candidate_df) * position_verify_factor * + input.clause_count; + return estimate > std::numeric_limits::max() ? std::numeric_limits::max() + : static_cast(estimate); +} + +inline bool common_grams_plan_cost_wins(uint64_t plain_cost, uint64_t common_grams_cost, + uint32_t common_grams_cost_ratio_percent) { + return static_cast(common_grams_cost) * 100 <= + static_cast(plain_cost) * common_grams_cost_ratio_percent; +} + +} // namespace doris::segment_v2::inverted_index diff --git a/be/src/storage/index/inverted/common_grams/common_grams_segment_metadata.cpp b/be/src/storage/index/inverted/common_grams/common_grams_segment_metadata.cpp new file mode 100644 index 00000000000000..c8dcddecf70608 --- /dev/null +++ b/be/src/storage/index/inverted/common_grams/common_grams_segment_metadata.cpp @@ -0,0 +1,158 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/inverted/common_grams/common_grams_segment_metadata.h" + +namespace doris::segment_v2::inverted_index { + +CommonGramsSegmentMetadata make_common_grams_segment_metadata( + const CommonGramsQueryIdentity& identity) { + CommonGramsSegmentMetadata metadata; + metadata.plain_term_key_version = PlainTermKeyVersion::kEscapedV1; + metadata.common_grams_coverage = CommonGramsCoverage::kComplete; + metadata.common_grams_semantics_version = COMMON_GRAMS_SEMANTICS_VERSION_V1; + metadata.common_grams_key_version = COMMON_GRAMS_KEY_VERSION_V1; + metadata.common_grams_dictionary_identity = identity.common_grams_dictionary_identity; + metadata.base_analyzer_fingerprint = identity.base_analyzer_fingerprint; + metadata.common_grams_fingerprint = identity.common_grams_fingerprint; + metadata.scoring_coverage = ScoringCoverage::kComplete; + metadata.scoring_stats_version = COMMON_GRAMS_SCORING_STATS_VERSION_V1; + metadata.norm_semantics_version = COMMON_GRAMS_NORM_SEMANTICS_VERSION_V1; + return metadata; +} + +bool common_grams_identity_matches(const CommonGramsSegmentMetadata& metadata, + const CommonGramsQueryIdentity& identity) { + return metadata.common_grams_dictionary_identity == identity.common_grams_dictionary_identity && + metadata.base_analyzer_fingerprint == identity.base_analyzer_fingerprint && + metadata.common_grams_fingerprint == identity.common_grams_fingerprint; +} + +Status validate_common_grams_segment_metadata(const CommonGramsSegmentMetadata& metadata) { + switch (metadata.plain_term_key_version) { + case PlainTermKeyVersion::kLegacyRaw: + case PlainTermKeyVersion::kEscapedV1: + case PlainTermKeyVersion::kRawNoInternal: + break; + default: + return Status::Error( + "common_grams_metadata: invalid plain-term key version"); + } + + switch (metadata.common_grams_coverage) { + case CommonGramsCoverage::kNone: + case CommonGramsCoverage::kComplete: + case CommonGramsCoverage::kMixed: + break; + default: + return Status::Error( + "common_grams_metadata: invalid coverage"); + } + + switch (metadata.scoring_coverage) { + case ScoringCoverage::kNone: + case ScoringCoverage::kComplete: + break; + default: + return Status::Error( + "common_grams_metadata: invalid scoring coverage"); + } + + if (metadata.plain_term_key_version == PlainTermKeyVersion::kRawNoInternal && + metadata.common_grams_coverage != CommonGramsCoverage::kNone) { + return Status::Error( + "common_grams_metadata: raw-no-internal segment has gram coverage"); + } + + if (metadata.common_grams_coverage == CommonGramsCoverage::kComplete && + (metadata.plain_term_key_version != PlainTermKeyVersion::kEscapedV1 || + metadata.common_grams_semantics_version == 0 || metadata.common_grams_key_version == 0 || + metadata.common_grams_dictionary_identity.empty() || + metadata.base_analyzer_fingerprint.empty() || metadata.common_grams_fingerprint.empty())) { + return Status::Error( + "common_grams_metadata: incomplete complete-coverage identity"); + } + + if (metadata.scoring_coverage == ScoringCoverage::kComplete && + (metadata.scoring_stats_version == 0 || metadata.norm_semantics_version == 0 || + metadata.base_analyzer_fingerprint.empty())) { + return Status::Error( + "common_grams_metadata: incomplete scoring identity"); + } + return Status::OK(); +} + +Status validate_snii_scoring_metadata(const CommonGramsSegmentMetadata* metadata, + uint64_t physical_doc_count, + uint64_t physical_sum_total_term_freq, bool has_scoring_tier, + bool has_positions, bool has_norms) { + if (metadata == nullptr) { + return Status::Error( + "SNII semantic scoring metadata is missing"); + } + RETURN_IF_ERROR(validate_common_grams_segment_metadata(*metadata)); + if (metadata->scoring_coverage != ScoringCoverage::kComplete || + metadata->scoring_stats_version != COMMON_GRAMS_SCORING_STATS_VERSION_V1 || + metadata->norm_semantics_version != COMMON_GRAMS_NORM_SEMANTICS_VERSION_V1) { + return Status::Error( + "SNII scoring metadata uses unsupported semantics"); + } + if (!has_scoring_tier || !has_positions || !has_norms) { + return Status::Error( + "SNII complete scoring metadata requires the scoring tier, positions, and norms"); + } + if (metadata->scoring_doc_count != physical_doc_count) { + return Status::Error( + "SNII semantic scoring document count {} differs from physical document count {}", + metadata->scoring_doc_count, physical_doc_count); + } + if (metadata->scoring_token_count > physical_sum_total_term_freq) { + return Status::Error( + "SNII semantic scoring token count {} exceeds physical term frequency {}", + metadata->scoring_token_count, physical_sum_total_term_freq); + } + if (metadata->plain_term_key_version == PlainTermKeyVersion::kRawNoInternal && + metadata->common_grams_coverage == CommonGramsCoverage::kNone && + metadata->scoring_token_count != physical_sum_total_term_freq) { + return Status::Error( + "SNII semantic plain token count {} differs from physical term frequency {}", + metadata->scoring_token_count, physical_sum_total_term_freq); + } + if (physical_sum_total_term_freq != 0 && metadata->scoring_token_count == 0) { + return Status::Error( + "SNII non-empty physical postings have zero semantic scoring tokens"); + } + return Status::OK(); +} + +bool is_common_grams_query_compatible(const CommonGramsSegmentMetadata& metadata, + const CommonGramsQueryIdentity& identity) { + return is_common_grams_query_compatible(metadata, identity, CommonGramsCoverage::kComplete); +} + +bool is_common_grams_query_compatible(const CommonGramsSegmentMetadata& metadata, + const CommonGramsQueryIdentity& identity, + CommonGramsCoverage required_coverage) { + return metadata.plain_term_key_version == PlainTermKeyVersion::kEscapedV1 && + metadata.common_grams_coverage == required_coverage && + required_coverage != CommonGramsCoverage::kNone && + metadata.common_grams_semantics_version == COMMON_GRAMS_SEMANTICS_VERSION_V1 && + metadata.common_grams_key_version == COMMON_GRAMS_KEY_VERSION_V1 && + common_grams_identity_matches(metadata, identity); +} + +} // namespace doris::segment_v2::inverted_index diff --git a/be/src/storage/index/inverted/common_grams/common_grams_segment_metadata.h b/be/src/storage/index/inverted/common_grams/common_grams_segment_metadata.h new file mode 100644 index 00000000000000..b5864bd340fc87 --- /dev/null +++ b/be/src/storage/index/inverted/common_grams/common_grams_segment_metadata.h @@ -0,0 +1,87 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include "common/status.h" +#include "storage/index/inverted/common_grams/common_grams_key_codec.h" + +namespace doris::segment_v2::inverted_index { + +inline constexpr uint32_t COMMON_GRAMS_SEGMENT_METADATA_VERSION = 1; +inline constexpr uint32_t COMMON_GRAMS_SEMANTICS_VERSION_V1 = 1; +inline constexpr uint32_t COMMON_GRAMS_KEY_VERSION_V1 = 1; +inline constexpr uint32_t COMMON_GRAMS_SCORING_STATS_VERSION_V1 = 1; +inline constexpr uint32_t COMMON_GRAMS_NORM_SEMANTICS_VERSION_V1 = 1; + +enum class CommonGramsCoverage : uint8_t { + kNone = 0, + kComplete = 1, + kMixed = 2, +}; + +enum class ScoringCoverage : uint8_t { + kNone = 0, + kComplete = 1, +}; + +// SNII metadata. A missing record is a legacy segment; a present record must +// validate before any capability is used. +struct CommonGramsSegmentMetadata { + PlainTermKeyVersion plain_term_key_version = PlainTermKeyVersion::kLegacyRaw; + CommonGramsCoverage common_grams_coverage = CommonGramsCoverage::kNone; + uint32_t common_grams_semantics_version = 0; + uint32_t common_grams_key_version = 0; + std::string common_grams_dictionary_identity; + std::string base_analyzer_fingerprint; + std::string common_grams_fingerprint; + ScoringCoverage scoring_coverage = ScoringCoverage::kNone; + uint32_t scoring_stats_version = 0; + uint32_t norm_semantics_version = 0; + uint64_t scoring_doc_count = 0; + uint64_t scoring_token_count = 0; + + bool operator==(const CommonGramsSegmentMetadata&) const = default; +}; + +struct CommonGramsQueryIdentity { + std::string common_grams_dictionary_identity; + std::string base_analyzer_fingerprint; + std::string common_grams_fingerprint; + + bool operator==(const CommonGramsQueryIdentity&) const = default; +}; + +CommonGramsSegmentMetadata make_common_grams_segment_metadata( + const CommonGramsQueryIdentity& identity); +bool common_grams_identity_matches(const CommonGramsSegmentMetadata& metadata, + const CommonGramsQueryIdentity& identity); +Status validate_common_grams_segment_metadata(const CommonGramsSegmentMetadata& metadata); +Status validate_snii_scoring_metadata(const CommonGramsSegmentMetadata* metadata, + uint64_t physical_doc_count, + uint64_t physical_sum_total_term_freq, bool has_scoring_tier, + bool has_positions, bool has_norms); +bool is_common_grams_query_compatible(const CommonGramsSegmentMetadata& metadata, + const CommonGramsQueryIdentity& identity); +bool is_common_grams_query_compatible(const CommonGramsSegmentMetadata& metadata, + const CommonGramsQueryIdentity& identity, + CommonGramsCoverage required_coverage); + +} // namespace doris::segment_v2::inverted_index diff --git a/be/src/storage/index/inverted/common_grams/common_word_set.cpp b/be/src/storage/index/inverted/common_grams/common_word_set.cpp new file mode 100644 index 00000000000000..4e74cac590778d --- /dev/null +++ b/be/src/storage/index/inverted/common_grams/common_word_set.cpp @@ -0,0 +1,197 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/inverted/common_grams/common_word_set.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/config.h" +#include "common/logging.h" +#include "storage/index/inverted/common_grams/common_grams_key_codec.h" +#include "util/md5.h" +#include "util/utf8_check.h" + +namespace doris::segment_v2::inverted_index { +namespace { + +#ifdef BE_TEST +std::atomic g_common_word_membership_lookups {0}; +std::atomic g_common_word_hash_lookups {0}; +#endif + +ResultError wordset_error(std::string_view message) { + return ResultError(Status::Error("{}", message)); +} + +} // namespace + +CommonWordSet::CommonWordSet(WordContainer words, std::string identity) + : _words(std::move(words)), _identity(std::move(identity)) { + DORIS_CHECK(!_identity.empty()); + for (const std::string& word : _words) { + DORIS_CHECK(!word.empty()); + DORIS_CHECK_LE(word.size(), std::numeric_limits::max()); + const auto bytes = static_cast(word.size()); + _min_word_bytes = std::min(_min_word_bytes, bytes); + _max_word_bytes = std::max(_max_word_bytes, bytes); + const uint8_t first = static_cast(word.front()); + _first_byte_mask[first >> 6] |= uint64_t {1} << (first & 63); + } +} + +const CommonWordSet& CommonWordSet::builtin_english_stop_words_v1() { + static const CommonWordSet words( + WordContainer { + "a", "an", "and", "are", "as", "at", "be", "but", "by", + "for", "if", "in", "into", "is", "it", "no", "not", "of", + "on", "or", "such", "that", "the", "their", "then", "there", "these", + "they", "this", "to", "was", "will", "with", + }, + std::string(BUILTIN_COMMON_WORDS_RESOURCE)); + return words; +} + +Result CommonWordSet::parse_words(std::string_view content) { + // Digest first: the parse loop below advances `content`, so hashing it afterwards would hash + // an empty view. Content-derived so a segment records exactly which list grammed it, and + // digesting the raw bytes (not the parsed set) means a comment-only edit also yields a new + // identity -- the safe direction, since that re-plans instead of risking a stale match. + Md5Digest digest; + digest.update(content.data(), content.size()); + digest.digest(); + const std::string identity = "wordset:md5:" + digest.hex(); + + if (content.find('\0') != std::string_view::npos) { + return wordset_error("CommonGrams word list contains NUL"); + } + if (!validate_utf8(content.data(), content.size())) { + return wordset_error("CommonGrams word list is not valid UTF-8"); + } + + WordContainer words; + while (!content.empty()) { + const size_t newline = content.find('\n'); + std::string_view term = content.substr(0, newline); + if (newline != std::string_view::npos && !term.empty() && term.back() == '\r') { + term.remove_suffix(1); + } + if (!term.empty() && term.front() != '#') { + if (term.size() > COMMON_GRAM_MAX_ENCODED_BYTES) { + return wordset_error("wordset:v1 term exceeds the 16383-byte token limit"); + } + words.emplace(term); + } + if (newline == std::string_view::npos) { + break; + } + content.remove_prefix(newline + 1); + } + return CommonWordSet(std::move(words), identity); +} + +std::string CommonWordSet::default_word_set_path() { + return config::inverted_index_dict_path + "/common_grams/default_words.txt"; +} + +std::shared_ptr CommonWordSet::default_word_set() { + // Lazy singleton: the word list is immutable for the process, so every analyzer shares one + // copy and the fallback warning below is emitted at most once no matter how many analyzers + // are built. A later edit to inverted_index_dict_path is therefore ignored, which is intended + // -- swapping the list under live segments would change what their grams mean. + static const std::shared_ptr instance = [] { + auto builtin = [] { + return std::shared_ptr(&builtin_english_stop_words_v1(), + [](const CommonWordSet*) {}); + }; + const std::string path = default_word_set_path(); + std::ifstream input(path, std::ios::binary); + if (!input.is_open()) { + // Expected on a stock install: shipping no file means "use the built-in list", so this + // is INFO. A file that exists but cannot be read or parsed is an operator mistake and + // warns below. + LOG(INFO) << "No CommonGrams word list at " << path + << ", using the built-in English stop words"; + return builtin(); + } + std::ostringstream buffer; + buffer << input.rdbuf(); + if (input.bad()) { + LOG(WARNING) << "CommonGrams word list at " << path + << " could not be read, falling back to the built-in English stop words"; + return builtin(); + } + const std::string content = buffer.str(); + auto parsed = parse_words(content); + if (!parsed.has_value()) { + LOG(WARNING) << "CommonGrams word list at " << path << " is invalid: " << parsed.error() + << ", falling back to the built-in English stop words"; + return builtin(); + } + LOG(INFO) << "Loaded " << parsed.value().size() << " CommonGrams words from " << path; + return std::shared_ptr( + std::make_shared(std::move(parsed.value()))); + }(); + return instance; +} + +bool CommonWordSet::contains(std::string_view term) const { +#ifdef BE_TEST + g_common_word_membership_lookups.fetch_add(1, std::memory_order_relaxed); +#endif + if (term.size() < _min_word_bytes || term.size() > _max_word_bytes) { + return false; + } + const auto first = static_cast(term.front()); + if ((_first_byte_mask[first >> 6] & (uint64_t {1} << (first & 63))) == 0) { + return false; + } +#ifdef BE_TEST + g_common_word_hash_lookups.fetch_add(1, std::memory_order_relaxed); +#endif + return _words.contains(term); +} + +#ifdef BE_TEST +namespace common_grams_testing { + +uint64_t common_word_membership_lookup_count() { + return g_common_word_membership_lookups.load(std::memory_order_relaxed); +} + +void reset_common_word_membership_lookup_count() { + g_common_word_membership_lookups.store(0, std::memory_order_relaxed); +} + +uint64_t common_word_hash_lookup_count() { + return g_common_word_hash_lookups.load(std::memory_order_relaxed); +} + +void reset_common_word_hash_lookup_count() { + g_common_word_hash_lookups.store(0, std::memory_order_relaxed); +} + +} // namespace common_grams_testing +#endif + +} // namespace doris::segment_v2::inverted_index diff --git a/be/src/storage/index/inverted/common_grams/common_word_set.h b/be/src/storage/index/inverted/common_grams/common_word_set.h new file mode 100644 index 00000000000000..b6aac21f48b501 --- /dev/null +++ b/be/src/storage/index/inverted/common_grams/common_word_set.h @@ -0,0 +1,94 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/status.h" + +namespace doris::segment_v2::inverted_index { + +inline constexpr std::string_view BUILTIN_COMMON_WORDS_RESOURCE = "builtin:lucene_english_stop:v1"; +inline constexpr std::string_view WORDSET_FORMAT_V1 = "wordset:v1"; + +class CommonWordSet { +public: + static const CommonWordSet& builtin_english_stop_words_v1(); + + // Parses a newline-separated word list. Blank lines and '#' comments are skipped. + static Result parse_words(std::string_view content); + + // The BE-local word list every CommonGrams analyzer grams against, read once from + // /common_grams/default_words.txt -- the same layout the icu, ik and + // pinyin dictionaries use. Falls back to builtin_english_stop_words_v1() when the file is + // absent or unparseable. The set is deliberately not selectable per index policy: every replica + // of a tablet must gram identically, and a policy-supplied list would have to be distributed + // and acknowledged before any index could use it. + static std::shared_ptr default_word_set(); + + // Exposed so tests can assert the layout without duplicating the literal. + static std::string default_word_set_path(); + + // Stamped into a segment's CommonGrams metadata and compared against the querying analyzer's + // identity, so a segment is only read with gram expectations that match how it was written. + // Because the word list is now a BE-local file, this MUST derive from the content: two BEs + // pointed at different files, or one BE after the file is edited, would otherwise stamp the + // same identity onto incompatible segments and silently mis-plan phrase queries. + const std::string& identity() const { return _identity; } + + bool contains(std::string_view term) const; + size_t size() const { return _words.size(); } + +private: + struct TransparentStringHash { + using is_transparent = void; + + size_t operator()(std::string_view value) const { + return std::hash {}(value); + } + }; + + using WordContainer = phmap::flat_hash_set>; + + CommonWordSet(WordContainer words, std::string identity); + + WordContainer _words; + // Never empty. See identity() for why this is content-derived rather than a fixed constant. + std::string _identity; + std::array _first_byte_mask {}; + uint16_t _min_word_bytes = std::numeric_limits::max(); + uint16_t _max_word_bytes = 0; +}; + +#ifdef BE_TEST +namespace common_grams_testing { +uint64_t common_word_membership_lookup_count(); +void reset_common_word_membership_lookup_count(); +} // namespace common_grams_testing +#endif + +} // namespace doris::segment_v2::inverted_index diff --git a/be/src/storage/index/inverted/inverted_index_cache.cpp b/be/src/storage/index/inverted/inverted_index_cache.cpp index e759a0163ac7d3..ee3b6ec6d614ec 100644 --- a/be/src/storage/index/inverted/inverted_index_cache.cpp +++ b/be/src/storage/index/inverted/inverted_index_cache.cpp @@ -27,10 +27,50 @@ #include "runtime/exec_env.h" #include "runtime/thread_context.h" +#include "util/coding.h" #include "util/defer_op.h" namespace doris::segment_v2 { +namespace { + +void append_length_prefixed(std::string_view value, std::string* output) { + put_fixed64_le(output, value.size()); + output->append(value); +} + +} // namespace + +std::string InvertedIndexRawQuerySemantic::encode() const { + std::string output; + output.reserve(sizeof(cache_semantics_version) + sizeof(uint64_t) + raw_query_bytes.size() + + sizeof(query_type) + sizeof(slop) + sizeof(ordered) + sizeof(max_expansions) + + sizeof(common_grams_query_plan_enabled)); + put_fixed32_le(&output, cache_semantics_version); + append_length_prefixed(raw_query_bytes, &output); + put_fixed32_le(&output, static_cast(query_type)); + put_fixed32_le(&output, static_cast(slop)); + output.push_back(static_cast(ordered)); + put_fixed32_le(&output, static_cast(max_expansions)); + output.push_back(static_cast(common_grams_query_plan_enabled)); + return output; +} + +std::string InvertedIndexQueryCache::CacheKey::encode() const { + if (query_type_to_string(query_type).empty()) { + return {}; + } + std::string output; + const std::string index_path_string = index_path.string(); + output.reserve(3 * sizeof(uint64_t) + index_path_string.size() + column_name.size() + + sizeof(query_type) + value.size()); + append_length_prefixed(index_path_string, &output); + append_length_prefixed(column_name, &output); + put_fixed32_le(&output, static_cast(query_type)); + append_length_prefixed(value, &output); + return output; +} + InvertedIndexSearcherCache* InvertedIndexSearcherCache::create_global_instance( size_t capacity, uint32_t num_shards) { return new InvertedIndexSearcherCache(capacity, num_shards); diff --git a/be/src/storage/index/inverted/inverted_index_cache.h b/be/src/storage/index/inverted/inverted_index_cache.h index 0e33fe747b3523..4aaa41fbc5e14b 100644 --- a/be/src/storage/index/inverted/inverted_index_cache.h +++ b/be/src/storage/index/inverted/inverted_index_cache.h @@ -26,6 +26,7 @@ #include #include #include +#include #include "common/config.h" #include "common/status.h" @@ -35,6 +36,8 @@ #include "runtime/memory/lru_cache_policy.h" #include "runtime/memory/mem_tracker.h" #include "storage/index/inverted/inverted_index_searcher.h" +#include "storage/index/snii/reader/logical_index_reader.h" +#include "storage/index/snii/snii_bkd_searcher.h" #include "util/lru_cache.h" #include "util/slice.h" #include "util/time.h" @@ -42,6 +45,7 @@ namespace doris { namespace segment_v2 { class InvertedIndexCacheHandle; +class IndexFileReader; class InvertedIndexSearcherCache { public: @@ -56,6 +60,12 @@ class InvertedIndexSearcherCache { class CacheValue : public LRUCacheValueBase { public: IndexSearcherPtr index_searcher; + std::shared_ptr snii_index_file_reader; + std::unique_ptr snii_logical_reader; + // The numeric counterpart of snii_logical_reader: an opened SNII-native + // BKD blob index. A cache entry holds exactly one of the two, decided by + // which constructor ran. + std::unique_ptr snii_bkd_searcher; size_t size = 0; int64_t last_visit_time; @@ -65,6 +75,22 @@ class InvertedIndexSearcherCache { size = mem_size; last_visit_time = visit_time; } + explicit CacheValue(std::unique_ptr logical_reader, + size_t mem_size, int64_t visit_time, + std::shared_ptr index_file_reader) + : snii_index_file_reader(std::move(index_file_reader)), + snii_logical_reader(std::move(logical_reader)) { + size = mem_size; + last_visit_time = visit_time; + } + explicit CacheValue(std::unique_ptr bkd_searcher, + size_t mem_size, int64_t visit_time, + std::shared_ptr index_file_reader) + : snii_index_file_reader(std::move(index_file_reader)), + snii_bkd_searcher(std::move(bkd_searcher)) { + size = mem_size; + last_visit_time = visit_time; + } }; // Create global instance of this class. // "capacity" is the capacity of lru cache. @@ -166,6 +192,16 @@ class InvertedIndexCacheHandle { return ((InvertedIndexSearcherCache::CacheValue*)_cache->value(_handle))->index_searcher; } + doris::snii::reader::LogicalIndexReader* get_snii_logical_reader() { + return ((InvertedIndexSearcherCache::CacheValue*)_cache->value(_handle)) + ->snii_logical_reader.get(); + } + + doris::snii::bkd::BkdSearcher* get_snii_bkd_searcher() { + return ((InvertedIndexSearcherCache::CacheValue*)_cache->value(_handle)) + ->snii_bkd_searcher.get(); + } + InvertedIndexSearcherCache::CacheValue* get_index_cache_value() { return ((InvertedIndexSearcherCache::CacheValue*)_cache->value(_handle)); } @@ -180,6 +216,23 @@ class InvertedIndexCacheHandle { class InvertedIndexQueryCacheHandle; +inline constexpr uint32_t INVERTED_INDEX_QUERY_CACHE_SEMANTICS_VERSION = 1; + +// Stable identity shared by result-cache and row-accurate single-flight. It intentionally contains +// no analyzer output or internal plan kind: those are segment-local implementation details below +// the cache lookup. +struct InvertedIndexRawQuerySemantic { + std::string_view raw_query_bytes; + InvertedIndexQueryType query_type; + int32_t slop = 0; + bool ordered = false; + int32_t max_expansions = 0; + uint32_t cache_semantics_version = INVERTED_INDEX_QUERY_CACHE_SEMANTICS_VERSION; + bool common_grams_query_plan_enabled = false; + + std::string encode() const; +}; + class InvertedIndexQueryCache : public LRUCachePolicy { public: using LRUCachePolicy::insert; @@ -191,21 +244,8 @@ class InvertedIndexQueryCache : public LRUCachePolicy { InvertedIndexQueryType query_type; // query type std::string value; // query value - // Encode to a flat binary which can be used as LRUCache's key - std::string encode() const { - std::string key_buf(index_path.string()); - key_buf.append("/"); - key_buf.append(column_name); - key_buf.append("/"); - auto query_type_str = query_type_to_string(query_type); - if (query_type_str.empty()) { - return ""; - } - key_buf.append(query_type_str); - key_buf.append("/"); - key_buf.append(value); - return key_buf; - } + // Encode to an unambiguous flat binary which can be used as LRUCache's key. + std::string encode() const; }; class CacheValue : public LRUCacheValueBase { diff --git a/be/src/storage/index/inverted/inverted_index_fs_directory.cpp b/be/src/storage/index/inverted/inverted_index_fs_directory.cpp index 30f168e8b14e02..dcb20b734453d1 100644 --- a/be/src/storage/index/inverted/inverted_index_fs_directory.cpp +++ b/be/src/storage/index/inverted/inverted_index_fs_directory.cpp @@ -102,6 +102,11 @@ bool DorisFSDirectory::FSIndexInput::open(const io::FileSystemSPtr& fs, const ch reader_options.is_doris_table = true; reader_options.file_size = file_size; reader_options.tablet_id = tablet_id; + // NO_CACHE on a remote fs means every range GET bypasses CachedRemoteFileReader, + // the only layer that normally accounts physical remote bytes; readInternal then + // counts them itself. Local NO_CACHE reads must stay excluded. + h->_direct_remote_io = reader_options.cache_type == io::FileCachePolicy::NO_CACHE && + fs->type() != io::FileSystemType::LOCAL; Status st = fs->open_file(path, &h->_reader, &reader_options); DBUG_EXECUTE_IF("inverted file read error: index file not found", { st = Status::Error("index file not found"); }) @@ -179,20 +184,19 @@ void DorisFSDirectory::FSIndexInput::close() { } void DorisFSDirectory::FSIndexInput::setIoContext(const void* io_ctx) { + // Copy the caller's full IOContext (expiration_time for TTL cache classification, + // is_warmup, is_disposable, read_file_cache, bypass_peer_read, the miss policy and + // the remote-scan cache-write limiter all propagate); a field whitelist here silently + // drops newly added flags. Only the per-stream identity bits are re-stamped below. + const bool is_index_data = _io_ctx.is_index_data; if (io_ctx) { const auto& ctx = static_cast(io_ctx); - _io_ctx.reader_type = ctx->reader_type; - _io_ctx.query_id = ctx->query_id; - _io_ctx.file_cache_stats = ctx->file_cache_stats; - _io_ctx.file_cache_miss_policy = ctx->file_cache_miss_policy; - _io_ctx.remote_scan_cache_write_limiter = ctx->remote_scan_cache_write_limiter; + _io_ctx = *ctx; } else { - _io_ctx.reader_type = ReaderType::UNKNOWN; - _io_ctx.query_id = nullptr; - _io_ctx.file_cache_stats = nullptr; - _io_ctx.file_cache_miss_policy = io::FileCacheMissPolicy::READ_THROUGH_AND_WRITE_BACK; - _io_ctx.remote_scan_cache_write_limiter = nullptr; + _io_ctx = io::IOContext {}; } + _io_ctx.is_index_data = is_index_data; + _io_ctx.is_inverted_index = true; } const void* DorisFSDirectory::FSIndexInput::getIoContext() { @@ -251,6 +255,15 @@ void DorisFSDirectory::FSIndexInput::readInternal(uint8_t* b, const int32_t len) if (_io_ctx.file_cache_stats != nullptr) { _io_ctx.file_cache_stats->inverted_index_io_timer += inverted_index_io_timer; + _io_ctx.file_cache_stats->inverted_index_request_bytes += len; + _io_ctx.file_cache_stats->inverted_index_read_bytes += len; + ++_io_ctx.file_cache_stats->inverted_index_range_read_count; + ++_io_ctx.file_cache_stats->inverted_index_serial_read_rounds; + if (_handle->_direct_remote_io) { + // Cache-bypassed remote read: no CachedRemoteFileReader below to count + // the GET, so account the physical remote bytes here. + _io_ctx.file_cache_stats->inverted_index_remote_physical_read_bytes += len; + } } } diff --git a/be/src/storage/index/inverted/inverted_index_fs_directory.h b/be/src/storage/index/inverted/inverted_index_fs_directory.h index 79854df88d235d..64719880e7caa9 100644 --- a/be/src/storage/index/inverted/inverted_index_fs_directory.h +++ b/be/src/storage/index/inverted/inverted_index_fs_directory.h @@ -170,6 +170,11 @@ class DorisFSDirectory::FSIndexInput : public lucene::store::BufferedIndexInput std::mutex _shared_lock; //std::mutex* _shared_lock = nullptr; char path[4096]; + // True when _reader serves ranges straight from remote storage with no + // CachedRemoteFileReader in between (NO_CACHE on a non-local fs): only + // that layer would normally account physical remote bytes, so + // readInternal then counts them itself. + bool _direct_remote_io = false; SharedHandle(const char* path); ~SharedHandle() override; }; diff --git a/be/src/storage/index/inverted/inverted_index_iterator.cpp b/be/src/storage/index/inverted/inverted_index_iterator.cpp index 6e93d70d025964..9bd7800e7e0ccc 100644 --- a/be/src/storage/index/inverted/inverted_index_iterator.cpp +++ b/be/src/storage/index/inverted/inverted_index_iterator.cpp @@ -31,9 +31,6 @@ namespace doris::segment_v2 { InvertedIndexIterator::InvertedIndexIterator() = default; std::string InvertedIndexIterator::ensure_normalized_key(const std::string& analyzer_key) { - // Simple normalization: lowercase, empty stays empty. - // Empty means "user did not specify" (auto-select mode). - // Non-empty means "user specified this analyzer" (exact match mode). return normalize_analyzer_key(analyzer_key); } @@ -46,12 +43,13 @@ void InvertedIndexIterator::add_reader(InvertedIndexReaderType type, VLOG_DEBUG << "InvertedIndexIterator add_reader: type=" << static_cast(type) << ", analyzer_key=" << analyzer_key; - const size_t entry_index = _reader_entries.size(); - _reader_entries.push_back( - ReaderEntry {.type = type, .analyzer_key = std::move(analyzer_key), .reader = reader}); - - // Update index for O(1) lookup - _key_to_entries[_reader_entries.back().analyzer_key].push_back(entry_index); + auto status = add_inverted_index_selection_candidate( + InvertedIndexSelectionCandidate {.index_id = cast_set(reader->get_index_id()), + .reader_type = type, + .analyzer_key = std::move(analyzer_key)}, + &_selection_candidates, &_key_to_entries); + DORIS_CHECK(status.ok()) << status; + _readers.push_back(reader); } Status InvertedIndexIterator::read_from_index(const IndexParam& param) { @@ -68,13 +66,11 @@ Status InvertedIndexIterator::read_from_index(const IndexParam& param) { return Status::Error("inverted index bypass"); }); - // analyzer_name from analyzer_ctx: what user specified in USING ANALYZER clause. - // Empty means "user did not specify" (BE auto-selects index). - // Non-empty means "user specified this analyzer" (BE exact matches). - const std::string& analyzer_name = - (i_param->analyzer_ctx != nullptr) ? i_param->analyzer_ctx->analyzer_name : ""; + // The execution context carries reader selection separately from analyzer execution. + const std::string& analyzer_key = + (i_param->analyzer_ctx != nullptr) ? i_param->analyzer_ctx->analyzer_key : ""; auto reader = - DORIS_TRY(select_best_reader(i_param->column_type, i_param->query_type, analyzer_name)); + DORIS_TRY(select_best_reader(i_param->column_type, i_param->query_type, analyzer_key)); if (UNLIKELY(reader == nullptr)) { return Status::Error( "inverted index reader is null"); @@ -101,6 +97,11 @@ Status InvertedIndexIterator::read_from_index(const IndexParam& param) { // Note: analyzer_ctx is now passed via i_param->analyzer_ctx auto execute_query = [&]() { + if (i_param->null_bitmap_cache_handle != nullptr) { + return reader->query_with_null_bitmap( + _context, i_param->column_name, i_param->query_value, i_param->query_type, + i_param->roaring, i_param->null_bitmap_cache_handle, i_param->analyzer_ctx); + } return reader->query(_context, i_param->column_name, i_param->query_value, i_param->query_type, i_param->roaring, i_param->analyzer_ctx); }; @@ -122,14 +123,12 @@ Status InvertedIndexIterator::read_from_index(const IndexParam& param) { } Status InvertedIndexIterator::read_null_bitmap(InvertedIndexQueryCacheHandle* cache_handle) { - // For null bitmap, use any available reader (empty = auto-select) - auto reader = DORIS_TRY(select_best_reader("")); + auto reader = DORIS_TRY(select_any_reader()); return reader->read_null_bitmap(_context, cache_handle, nullptr); } Result InvertedIndexIterator::has_null() { - // For has_null check, use any available reader (empty = auto-select) - auto reader = DORIS_TRY(select_best_reader("")); + auto reader = DORIS_TRY(select_any_reader()); return reader->has_null(); } @@ -149,177 +148,59 @@ Status InvertedIndexIterator::try_read_from_inverted_index(const InvertedIndexRe return Status::OK(); } -// When multiple candidates of the preferred type exist, pick the one with -// the smallest index_id so that the choice is deterministic regardless of -// the order indexes appear in the rowset schema. Different segments may -// have different index orderings (e.g. after sequential BUILD INDEX -// operations), and relying on iteration order would cause inconsistent -// query results across segments. -static const ReaderEntry* pick_preferred(const std::vector& candidates, - InvertedIndexReaderType preferred_type) { - const ReaderEntry* best = nullptr; - for (const auto* entry : candidates) { - if (entry->type == preferred_type) { - if (best == nullptr || entry->reader->get_index_id() < best->reader->get_index_id()) { - best = entry; - } - } - } - return best; -} - -static const ReaderEntry* pick_smallest_index_id( - const std::vector& candidates) { - const ReaderEntry* best = candidates.front(); - for (const auto* entry : candidates) { - if (entry->reader->get_index_id() < best->reader->get_index_id()) { - best = entry; - } - } - return best; -} - -Result InvertedIndexIterator::select_for_text( - const AnalyzerMatchResult& match, InvertedIndexQueryType query_type, - const std::string& analyzer_key) { - // Bypass: explicit analyzer specified but not found - if (match.empty() && AnalyzerKeyMatcher::is_explicit(analyzer_key)) { - return ResultError(Status::Error( - "No inverted index reader found for analyzer '{}'. " - "The index for this analyzer may not be built yet.", - analyzer_key)); - } - - if (match.empty()) { - return ResultError(Status::Error( - "No available inverted index readers for text column.")); - } - - // MATCH queries prefer FULLTEXT - if (is_match_query(query_type)) { - if (auto* best = pick_preferred(match.candidates, InvertedIndexReaderType::FULLTEXT)) { - return best->reader; - } - } - - // EQUAL queries prefer STRING_TYPE for exact match - if (is_equal_query(query_type)) { - if (auto* best = pick_preferred(match.candidates, InvertedIndexReaderType::STRING_TYPE)) { - return best->reader; - } - } - - // Default: smallest index_id for deterministic selection - return pick_smallest_index_id(match.candidates)->reader; -} - -Result InvertedIndexIterator::select_for_numeric( - const AnalyzerMatchResult& match, InvertedIndexQueryType query_type) { - if (match.empty()) { - return ResultError(Status::Error( - "No available inverted index readers for numeric column.")); - } - - // RANGE queries prefer BKD - if (is_range_query(query_type)) { - if (const auto* best = pick_preferred(match.candidates, InvertedIndexReaderType::BKD)) { - return best->reader; - } - } - - // Fallback priority: BKD > STRING_TYPE > smallest index_id - if (const auto* best = pick_preferred(match.candidates, InvertedIndexReaderType::BKD)) { - return best->reader; - } - if (const auto* best = pick_preferred(match.candidates, InvertedIndexReaderType::STRING_TYPE)) { - return best->reader; - } - - // Last resort: smallest index_id for deterministic selection - return pick_smallest_index_id(match.candidates)->reader; -} - Result InvertedIndexIterator::select_best_reader( const DataTypePtr& column_type, InvertedIndexQueryType query_type, const std::string& analyzer_key) { - if (_reader_entries.empty()) { - return ResultError(Status::Error( - "No available inverted index readers. Check if index is properly initialized.")); - } - - // Normalize once at entry point const std::string normalized_key = ensure_normalized_key(analyzer_key); - - // Single reader optimization - if (_reader_entries.size() == 1) { - const auto& entry = _reader_entries.front(); - if (AnalyzerKeyMatcher::is_explicit(normalized_key) && - entry.analyzer_key != normalized_key) { - return ResultError(Status::Error( - "No inverted index reader found for analyzer '{}'. " - "Available analyzer: '{}'.", - normalized_key, entry.analyzer_key)); + // The column type only disambiguates between several indexes on the same field; with a + // single candidate the selection is already determined. Callers that have no runtime type + // binding therefore leave column_type null, so resolve the leaf type only when it matters. + FieldType field_type = FieldType::OLAP_FIELD_TYPE_UNKNOWN; + if (_selection_candidates.size() > 1) { + if (column_type == nullptr) { + return ResultError(Status::Error( + "column_type is required to select among {} inverted indexes", + _selection_candidates.size())); } - return entry.reader; - } - - // Match analyzer key using AnalyzerKeyMatcher - auto match = AnalyzerKeyMatcher::match(normalized_key, _reader_entries, _key_to_entries); - - // Dispatch by column type - const auto field_type = column_type->get_storage_field_type(); - - if (is_string_type(field_type)) { - return select_for_text(match, query_type, normalized_key); + field_type = get_inverted_index_leaf_field_type(column_type); } - - if (field_is_numeric_type(field_type)) { - return select_for_numeric(match, query_type); + auto selection = select_best_inverted_index_candidate(_selection_candidates, _key_to_entries, + field_type, query_type, normalized_key); + if (!selection.has_value()) { + return ResultError(std::move(selection.error())); } + const size_t selected = *selection; + DORIS_CHECK(selected < _readers.size()); + return _readers[selected]; +} - // Default: return deterministic candidate or error - if (match.empty()) { - return ResultError(Status::Error( - "No available inverted index readers for column type.")); +Result InvertedIndexIterator::select_any_reader() { + auto selection = select_best_inverted_index_candidate( + _selection_candidates, _key_to_entries, FieldType::OLAP_FIELD_TYPE_UNKNOWN, + InvertedIndexQueryType::UNKNOWN_QUERY, ""); + if (!selection.has_value()) { + return ResultError(std::move(selection.error())); } - return pick_smallest_index_id(match.candidates)->reader; + const size_t selected = *selection; + DORIS_CHECK(selected < _readers.size()); + return _readers[selected]; } Result InvertedIndexIterator::select_best_reader( const std::string& analyzer_key) { - if (_reader_entries.empty()) { - return ResultError(Status::Error( - "No available inverted index readers. Check if index is properly initialized.")); + if (analyzer_key.empty()) { + return select_any_reader(); } - const std::string normalized_key = ensure_normalized_key(analyzer_key); - - // Single reader optimization - if (_reader_entries.size() == 1) { - const auto& entry = _reader_entries.front(); - if (AnalyzerKeyMatcher::is_explicit(normalized_key) && - entry.analyzer_key != normalized_key) { - return ResultError(Status::Error( - "No inverted index reader found for analyzer '{}'. " - "Available analyzer: '{}'.", - normalized_key, entry.analyzer_key)); - } - return entry.reader; - } - - // Match and return deterministic candidate - auto match = AnalyzerKeyMatcher::match(normalized_key, _reader_entries, _key_to_entries); - - if (match.empty()) { - if (AnalyzerKeyMatcher::is_explicit(normalized_key)) { - return ResultError(Status::Error( - "No inverted index reader found for analyzer '{}'.", normalized_key)); - } - return ResultError(Status::Error( - "No available inverted index readers.")); - } - - return pick_smallest_index_id(match.candidates)->reader; + auto selection = select_best_inverted_index_candidate( + _selection_candidates, _key_to_entries, FieldType::OLAP_FIELD_TYPE_UNKNOWN, + InvertedIndexQueryType::UNKNOWN_QUERY, normalized_key); + if (!selection.has_value()) { + return ResultError(std::move(selection.error())); + } + const size_t selected = *selection; + DORIS_CHECK(selected < _readers.size()); + return _readers[selected]; } IndexReaderPtr InvertedIndexIterator::get_reader(IndexReaderType type) const { @@ -327,9 +208,10 @@ IndexReaderPtr InvertedIndexIterator::get_reader(IndexReaderType type) const { if (inverted_type == nullptr) { return nullptr; } - for (const auto& entry : _reader_entries) { - if (entry.type == *inverted_type) { - return entry.reader; + for (size_t i = 0; i < _selection_candidates.size(); ++i) { + if (_selection_candidates[i].reader_type == *inverted_type) { + DORIS_CHECK(i < _readers.size()); + return _readers[i]; } } return nullptr; diff --git a/be/src/storage/index/inverted/inverted_index_iterator.h b/be/src/storage/index/inverted/inverted_index_iterator.h index afc4a663670633..5f7c2250cc0836 100644 --- a/be/src/storage/index/inverted/inverted_index_iterator.h +++ b/be/src/storage/index/inverted/inverted_index_iterator.h @@ -20,10 +20,10 @@ #include #include "core/field.h" -#include "storage/index/analyzer_key_matcher.h" #include "storage/index/index_iterator.h" #include "storage/index/inverted/inverted_index_parser.h" #include "storage/index/inverted/inverted_index_reader.h" +#include "storage/index/inverted/inverted_index_selector.h" namespace doris::segment_v2 { @@ -35,19 +35,13 @@ struct InvertedIndexParam { uint32_t num_rows; std::shared_ptr roaring; bool skip_try = false; + // Non-null only when the caller consumes both the query result and this reader's null bitmap. + InvertedIndexQueryCacheHandle* null_bitmap_cache_handle = nullptr; // Pointer to analyzer context (can be nullptr if not needed) // Used by FullTextIndexReader for tokenization const InvertedIndexAnalyzerCtx* analyzer_ctx = nullptr; }; -// Entry representing an inverted index reader with its type and analyzer key. -// Used by InvertedIndexIterator and AnalyzerKeyMatcher for reader selection. -struct ReaderEntry { - InvertedIndexReaderType type; - std::string analyzer_key; - InvertedIndexReaderPtr reader; -}; - class InvertedIndexIterator : public IndexIterator { public: InvertedIndexIterator(); @@ -67,6 +61,10 @@ class InvertedIndexIterator : public IndexIterator { [[nodiscard]] Result select_best_reader( const DataTypePtr& column_type, InvertedIndexQueryType query_type, const std::string& analyzer_key); + + [[nodiscard]] Result select_any_reader(); + + // Temporary compatibility for variant fields whose runtime binding has no type. [[nodiscard]] Result select_best_reader( const std::string& analyzer_key); @@ -81,27 +79,16 @@ class InvertedIndexIterator : public IndexIterator { // Empty input stays empty (means "user did not specify"). static std::string ensure_normalized_key(const std::string& analyzer_key); - // Select best reader for text (string) columns. - // Handles FULLTEXT vs STRING_TYPE priority based on query type. - // Returns BYPASS error if explicit analyzer not found. - [[nodiscard]] Result select_for_text(const AnalyzerMatchResult& match, - InvertedIndexQueryType query_type, - const std::string& analyzer_key); - - // Select best reader for numeric columns. - // Handles BKD priority for range queries. - [[nodiscard]] Result select_for_numeric( - const AnalyzerMatchResult& match, InvertedIndexQueryType query_type); - - // THREAD SAFETY: _reader_entries and _key_to_entries are populated during initialization + // THREAD SAFETY: reader metadata and _key_to_entries are populated during initialization // phase (via add_reader) and only read during query phase (via read_from_index/select_best_reader). // These two phases are guaranteed not to overlap, so no synchronization is needed. // Do NOT call add_reader() after any read_from_index() call on the same iterator. - std::vector _reader_entries; + std::vector _selection_candidates; + std::vector _readers; - // Index for O(1) lookup by analyzer_key. Maps normalized key to indices in _reader_entries. + // Index for O(1) lookup by analyzer_key. Maps normalized key to candidate indices. // Built incrementally in add_reader(). - std::unordered_map> _key_to_entries; + InvertedIndexSelectionKeyIndex _key_to_entries; }; } // namespace doris::segment_v2 \ No newline at end of file diff --git a/be/src/storage/index/inverted/inverted_index_parser.cpp b/be/src/storage/index/inverted/inverted_index_parser.cpp index 47819cc62f6397..c228a3a14ca948 100644 --- a/be/src/storage/index/inverted/inverted_index_parser.cpp +++ b/be/src/storage/index/inverted/inverted_index_parser.cpp @@ -179,9 +179,6 @@ std::string get_analyzer_name_from_properties( } std::string normalize_analyzer_key(std::string_view analyzer) { - // Simple normalization: lowercase, or empty if input is empty. - // Empty string means "user did not specify" - BE will auto-select. - // Non-empty string means "user specified this analyzer" - BE will exact match. if (analyzer.empty()) { return ""; } @@ -190,16 +187,11 @@ std::string normalize_analyzer_key(std::string_view analyzer) { std::string build_analyzer_key_from_properties( const std::map& properties) { - // Build analyzer key from index properties for reader registration. - // This determines how the index is stored/identified. - - // 1. Check for custom analyzer name - auto custom_it = properties.find(INVERTED_INDEX_ANALYZER_NAME_KEY); - if (custom_it != properties.end() && !custom_it->second.empty()) { - return to_lower(custom_it->second); + const auto analyzer_name = get_analyzer_name_from_properties(properties); + if (!analyzer_name.empty()) { + return normalize_analyzer_key(analyzer_name); } - // 2. Fall back to parser type std::string parser; auto parser_it = properties.find(INVERTED_INDEX_PARSER_KEY); if (parser_it != properties.end()) { @@ -211,11 +203,10 @@ std::string build_analyzer_key_from_properties( } } - // 3. Return normalized parser or "" for no explicit configuration if (parser.empty()) { - return ""; // No explicit parser - empty key means "no configuration" + return INVERTED_INDEX_PARSER_NONE; } - return to_lower(parser); + return normalize_analyzer_key(parser); } // ============================================================================ @@ -234,52 +225,25 @@ bool AnalyzerConfigParser::is_builtin_analyzer(const std::string& normalized_nam return parser_type != InvertedIndexParserType::PARSER_UNKNOWN; } -std::string AnalyzerConfigParser::compute_analyzer_key(const std::string& value) { - // Simple: just lowercase, empty stays empty - return normalize_analyzer_key(value); -} - AnalyzerConfig AnalyzerConfigParser::parse(const std::string& analyzer_name, const std::string& parser_type_str) { AnalyzerConfig config; - - // Determine parser type from parser_type_str (from index properties) - auto parser_type = get_inverted_index_parser_type_from_string(parser_type_str); const std::string normalized_analyzer = normalize_to_lower(analyzer_name); - - // If parser_type_str didn't yield a valid type, try analyzer_name - if (parser_type == InvertedIndexParserType::PARSER_UNKNOWN && !normalized_analyzer.empty()) { - parser_type = get_inverted_index_parser_type_from_string(normalized_analyzer); - } - const bool analyzer_is_builtin = is_builtin_analyzer(normalized_analyzer); - // Case 1: analyzer_name is non-empty and NOT a builtin type => custom analyzer - if (!analyzer_name.empty() && !analyzer_is_builtin) { - config.custom_analyzer = analyzer_name; - config.parser_type = InvertedIndexParserType::PARSER_NONE; + if (!normalized_analyzer.empty()) { config.analyzer_key = normalize_to_lower(analyzer_name); - } else { - // Case 2: builtin analyzer or user did not specify analyzer - config.custom_analyzer.clear(); - - // Use parser_type from index properties for slow path tokenization - if (parser_type == InvertedIndexParserType::PARSER_UNKNOWN) { - config.parser_type = InvertedIndexParserType::PARSER_NONE; - } else { - config.parser_type = parser_type; - } - - // analyzer_key: what user specified (for index selection) - // Empty means "user did not specify", BE will auto-select - if (normalized_analyzer.empty() && parser_type != InvertedIndexParserType::PARSER_UNKNOWN) { - // No analyzer name but valid parser type - use parser type as key - config.analyzer_key = inverted_index_parser_type_to_string(parser_type); + if (analyzer_is_builtin) { + config.parser_type = get_inverted_index_parser_type_from_string(normalized_analyzer); } else { - config.analyzer_key = normalized_analyzer; + config.provider_name = analyzer_name; + config.parser_type = InvertedIndexParserType::PARSER_NONE; } + return config; } + config.parser_type = get_inverted_index_parser_type_from_string(parser_type_str); + return config; } diff --git a/be/src/storage/index/inverted/inverted_index_parser.h b/be/src/storage/index/inverted/inverted_index_parser.h index d2d3df47abd0a3..3a48c533470c3f 100644 --- a/be/src/storage/index/inverted/inverted_index_parser.h +++ b/be/src/storage/index/inverted/inverted_index_parser.h @@ -19,9 +19,12 @@ #include #include +#include #include #include +#include "storage/index/inverted/analyzer/analyzer_provider.h" +#include "storage/index/inverted/common_grams/common_grams_segment_metadata.h" #include "util/debug_points.h" namespace lucene { @@ -100,31 +103,60 @@ const std::string INVERTED_INDEX_ANALYZER_NAME_KEY = "analyzer"; const std::string INVERTED_INDEX_NORMALIZER_NAME_KEY = "normalizer"; const std::string INVERTED_INDEX_PARSER_FIELD_PATTERN_KEY = "field_pattern"; -// Normalize an analyzer name to a standardized key format (lowercase). -// Empty string stays empty (means "user did not specify"). -// Non-empty string is lowercased (means "user specified this analyzer"). +// Normalize a physical analyzer selection key to lowercase. Empty stays empty. std::string normalize_analyzer_key(std::string_view analyzer); // Runtime context for analyzer // Contains only the fields needed at runtime struct InvertedIndexAnalyzerCtx { - // analyzer_name: what user specified in USING ANALYZER clause - // Empty means user did not specify (BE auto-selects index) - // Non-empty means user explicitly specified (BE exact matches) + // Physical reader selection key from Thrift. Empty allows fallback selection; + // non-empty requires an exact match. + std::string analyzer_key; + + // Named custom analyzer or normalizer used to execute the predicate. std::string analyzer_name; - // parser_type: determined from index properties, used for slow path tokenization + // Builtin parser used to execute the predicate. InvertedIndexParserType parser_type = InvertedIndexParserType::PARSER_UNKNOWN; // Used for creating reader and tokenization CharFilterMap char_filter_map; std::shared_ptr analyzer; + segment_v2::inverted_index::AnalyzerProviderPtr analyzer_provider; + std::optional common_grams_identity; + + std::shared_ptr get_analyzer( + segment_v2::inverted_index::AnalysisPurpose purpose) const { + if (analyzer_provider != nullptr) { + return analyzer_provider->get_analyzer(purpose); + } + return analyzer; + } + + const segment_v2::inverted_index::CommonGramsQueryIdentity* get_common_grams_identity() const { + if (common_grams_identity.has_value()) { + return &*common_grams_identity; + } + return analyzer_provider == nullptr ? nullptr : analyzer_provider->common_grams_identity(); + } - // Returns true if tokenization should be performed. - // Decision is based on parser_type (from index properties): - // - PARSER_NONE: no tokenization (keyword/exact match) - // - Other parsers: tokenize using that parser - bool should_tokenize() const { return parser_type != InvertedIndexParserType::PARSER_NONE; } + bool has_complete_common_grams_identity() const { + const auto* identity = get_common_grams_identity(); + return identity != nullptr && !identity->common_grams_dictionary_identity.empty() && + !identity->base_analyzer_fingerprint.empty() && + !identity->common_grams_fingerprint.empty(); + } + + // Raw-query cache and single-flight keys intentionally exclude analyzer output. A tokenizing + // provider therefore needs a complete immutable identity before those results may be shared. + bool can_share_raw_query_semantics() const { + return !requires_analysis() || has_complete_common_grams_identity(); + } + + // This controls analyzer execution, not the number of emitted terms. + bool requires_analysis() const { + return !analyzer_name.empty() || parser_type != InvertedIndexParserType::PARSER_NONE; + } }; using InvertedIndexAnalyzerCtxSPtr = std::shared_ptr; @@ -170,24 +202,20 @@ std::string get_parser_dict_compression_from_properties( std::string get_analyzer_name_from_properties(const std::map& properties); // Build a normalized analyzer key from index properties. -// Checks custom_analyzer first, then falls back to parser type. +// Precedence is analyzer, normalizer, then parser type. A raw index uses "none". std::string build_analyzer_key_from_properties( const std::map& properties); // Result structure for analyzer config parsing struct AnalyzerConfig { - std::string custom_analyzer; + std::string provider_name; InvertedIndexParserType parser_type = InvertedIndexParserType::PARSER_NONE; - // analyzer_key: what user specified in USING ANALYZER clause - // Empty means "user did not specify" (BE auto-selects) - // Non-empty means "user specified this analyzer" (BE exact matches) + // Physical reader selection key from the Thrift analyzer name. + // Empty allows fallback selection; non-empty requires an exact match. std::string analyzer_key; - // Check if this is a custom analyzer (not builtin) - bool is_custom() const { return !custom_analyzer.empty(); } - - // Check if user explicitly specified an analyzer - bool is_user_specified() const { return !analyzer_key.empty(); } + // Check if execution uses a named analyzer or normalizer provider. + bool uses_provider() const { return !provider_name.empty(); } }; // Parser for analyzer configuration from Thrift TMatchPredicate. @@ -196,7 +224,7 @@ struct AnalyzerConfig { class AnalyzerConfigParser { public: // Parse from raw analyzer name and parser type string (extracted from Thrift). - // @param analyzer_name: User-specified analyzer name (may be custom or builtin, or empty). + // @param analyzer_name: Analyzer selection name from Thrift (custom, builtin, or empty). // @param parser_type_str: Parser type string like "chinese", "standard", etc. [[nodiscard]] static AnalyzerConfig parse(const std::string& analyzer_name, const std::string& parser_type_str); @@ -206,9 +234,6 @@ class AnalyzerConfigParser { private: static std::string normalize_to_lower(const std::string& value); - - // Compute normalized analyzer_key from raw value. - static std::string compute_analyzer_key(const std::string& value); }; } // namespace doris diff --git a/be/src/storage/index/inverted/inverted_index_reader.cpp b/be/src/storage/index/inverted/inverted_index_reader.cpp index 0e83238064297e..a119992538b5e5 100644 --- a/be/src/storage/index/inverted/inverted_index_reader.cpp +++ b/be/src/storage/index/inverted/inverted_index_reader.cpp @@ -44,6 +44,7 @@ #include "core/type_limit.h" #include "runtime/runtime_profile.h" #include "runtime/runtime_state.h" +#include "storage/index/bkd_field_encoding.h" #include "storage/index/index_file_reader.h" #include "storage/index/index_reader_helper.h" #include "storage/index/inverted/analyzer/analyzer.h" @@ -85,36 +86,11 @@ static void bkd_encode_max(const doris::KeyCoder* coder, std::string* out) { coder->full_encode_ascending(&v, out); } -static doris::Status encode_bkd_field_ascending(doris::FieldType ft, const doris::Field& field, - const doris::KeyCoder* coder, std::string* out) { - // `actual` is the primitive type of the query Field from the caller; `PrimitiveType::PT` is the - // scalar type the BKD index stores (e.g. INT for an INT column or ARRAY index). - // Normally they match: `int_col = 1` -> both INT; `array_contains(int_arr, 2)` -> both INT. - // Mismatch happens when the query Field carries a non-scalar while BKD records the inner scalar: - // `arr = []` reaches here via `FunctionComparison` with the entire const ARRAY literal - // as the query Field, so `actual = TYPE_ARRAY` while PT is the inner scalar -- the predicate - // cannot be answered by BKD. Return INVERTED_INDEX_EVALUATE_SKIPPED so `_apply_index_expr` - // downgrades to scalar evaluation instead of crashing on `Field::get()` DCHECK below. -#define CASE(FT, PT) \ - case doris::FieldType::FT: { \ - const auto actual = field.get_type(); \ - if (actual != doris::PrimitiveType::PT && actual != doris::PrimitiveType::TYPE_NULL && \ - !(doris::is_string_type(actual) && doris::is_string_type(doris::PrimitiveType::PT))) { \ - return doris::Status::Error( \ - "BKD query value type {} does not match index type {}", \ - static_cast(actual), static_cast(ft)); \ - } \ - doris::full_encode_field_as_key(field, coder, out); \ - return doris::Status::OK(); \ - } - switch (ft) { - DORIS_APPLY_FOR_KEY_ENCODABLE_NON_STRING_TYPES(CASE) - default: - break; - } -#undef CASE - return doris::Status::InternalError("unsupported BKD field type {}", static_cast(ft)); -} +// encode_bkd_field_ascending now lives in storage/index/bkd_field_encoding.h so +// the SNII-native BKD reader encodes query values through the exact same +// definition (INV-1); only the +/- infinity sentinels below stay here, being an +// artifact of this visitor's always-closed bounds. +using doris::encode_bkd_field_ascending; static doris::Status encode_bkd_min_ascending(doris::FieldType ft, const doris::KeyCoder* coder, std::string* out) { @@ -154,6 +130,20 @@ std::string InvertedIndexReader::get_index_file_path() { return _index_file_reader->get_index_file_path(&_index_meta); } +Status InvertedIndexReader::query_with_null_bitmap( + const IndexQueryContextPtr& context, const std::string& column_name, + const Field& query_value, InvertedIndexQueryType query_type, + std::shared_ptr& bit_map, + InvertedIndexQueryCacheHandle* null_bitmap_cache_handle, + const InvertedIndexAnalyzerCtx* analyzer_ctx) { + DORIS_CHECK(null_bitmap_cache_handle != nullptr); + RETURN_IF_ERROR(query(context, column_name, query_value, query_type, bit_map, analyzer_ctx)); + if (!has_null()) { + return Status::OK(); + } + return read_null_bitmap(context, null_bitmap_cache_handle); +} + Status InvertedIndexReader::read_null_bitmap(const IndexQueryContextPtr& context, InvertedIndexQueryCacheHandle* cache_handle, lucene::store::Directory* dir) { @@ -219,15 +209,16 @@ bool InvertedIndexReader::handle_query_cache(const IndexQueryContextPtr& context InvertedIndexQueryCache* cache, const InvertedIndexQueryCache::CacheKey& cache_key, InvertedIndexQueryCacheHandle* cache_handler, - std::shared_ptr& bit_map) { + std::shared_ptr& bit_map, + bool enabled) { const auto& query_options = context->runtime_state->query_options(); - - bool cache_hit = false; - if (query_options.enable_inverted_index_query_cache) { - SCOPED_RAW_TIMER(&context->stats->inverted_index_lookup_timer); - cache_hit = cache->lookup(cache_key, cache_handler); + if (!enabled || !query_options.enable_inverted_index_query_cache) { + return false; } + context->stats->inverted_index_query_cache_lookup++; + SCOPED_RAW_TIMER(&context->stats->inverted_index_lookup_timer); + const bool cache_hit = cache->lookup(cache_key, cache_handler); if (cache_hit) { DBUG_EXECUTE_IF("InvertedIndexReader.handle_query_cache_hit", { return Status::Error("handle query cache hit"); @@ -245,6 +236,19 @@ bool InvertedIndexReader::handle_query_cache(const IndexQueryContextPtr& context return false; } +void InvertedIndexReader::insert_query_cache(const IndexQueryContextPtr& context, + InvertedIndexQueryCache* cache, + const InvertedIndexQueryCache::CacheKey& cache_key, + std::shared_ptr bit_map, + InvertedIndexQueryCacheHandle* cache_handler, + bool enabled) { + if (!enabled || !context->runtime_state->query_options().enable_inverted_index_query_cache) { + return; + } + cache->insert(cache_key, std::move(bit_map), cache_handler); + context->stats->inverted_index_query_cache_insert++; +} + Status InvertedIndexReader::handle_searcher_cache( const IndexQueryContextPtr& context, InvertedIndexCacheHandle* inverted_index_cache_handle) { @@ -403,7 +407,7 @@ Status FullTextIndexReader::query(const IndexQueryContextPtr& context, query_info); } else { SCOPED_RAW_TIMER(&context->stats->inverted_index_analyzer_timer); - if (analyzer_ctx != nullptr && !analyzer_ctx->should_tokenize()) { + if (analyzer_ctx != nullptr && !analyzer_ctx->requires_analysis()) { // Keyword index: all strings (including empty) are valid tokens for exact match. // Empty string is a valid value in keyword index and should be matchable. query_info.term_infos.emplace_back(search_str); diff --git a/be/src/storage/index/inverted/inverted_index_reader.h b/be/src/storage/index/inverted/inverted_index_reader.h index 0e2f6a120d41e3..6960e135e572b5 100644 --- a/be/src/storage/index/inverted/inverted_index_reader.h +++ b/be/src/storage/index/inverted/inverted_index_reader.h @@ -226,13 +226,19 @@ class InvertedIndexReader : public IndexReader { const Field& query_value, InvertedIndexQueryType query_type, std::shared_ptr& bit_map, const InvertedIndexAnalyzerCtx* analyzer_ctx = nullptr) = 0; + virtual Status query_with_null_bitmap(const IndexQueryContextPtr& context, + const std::string& column_name, const Field& query_value, + InvertedIndexQueryType query_type, + std::shared_ptr& bit_map, + InvertedIndexQueryCacheHandle* null_bitmap_cache_handle, + const InvertedIndexAnalyzerCtx* analyzer_ctx = nullptr); virtual Status try_query(const IndexQueryContextPtr& context, const std::string& column_name, const Field& query_value, InvertedIndexQueryType query_type, size_t* count) = 0; - Status read_null_bitmap(const IndexQueryContextPtr& context, - InvertedIndexQueryCacheHandle* cache_handle, - lucene::store::Directory* dir = nullptr); + virtual Status read_null_bitmap(const IndexQueryContextPtr& context, + InvertedIndexQueryCacheHandle* cache_handle, + lucene::store::Directory* dir = nullptr); virtual InvertedIndexReaderType type() = 0; @@ -249,7 +255,11 @@ class InvertedIndexReader : public IndexReader { bool handle_query_cache(const IndexQueryContextPtr& context, InvertedIndexQueryCache* cache, const InvertedIndexQueryCache::CacheKey& cache_key, InvertedIndexQueryCacheHandle* cache_handler, - std::shared_ptr& bit_map); + std::shared_ptr& bit_map, bool enabled = true); + void insert_query_cache(const IndexQueryContextPtr& context, InvertedIndexQueryCache* cache, + const InvertedIndexQueryCache::CacheKey& cache_key, + std::shared_ptr bit_map, + InvertedIndexQueryCacheHandle* cache_handler, bool enabled = true); virtual Status handle_searcher_cache(const IndexQueryContextPtr& context, InvertedIndexCacheHandle* inverted_index_cache_handle); @@ -335,7 +345,6 @@ class InvertedIndexVisitor : public lucene::util::bkd::bkd_reader::intersect_vis std::string query_min; std::string query_max; -public: InvertedIndexVisitor(const void* io_ctx, lucene::util::bkd::bkd_reader* r, roaring::Roaring* hits, bool only_count = false); ~InvertedIndexVisitor() override = default; diff --git a/be/src/storage/index/inverted/inverted_index_selector.cpp b/be/src/storage/index/inverted/inverted_index_selector.cpp new file mode 100644 index 00000000000000..dac0ec9b41b1cb --- /dev/null +++ b/be/src/storage/index/inverted/inverted_index_selector.cpp @@ -0,0 +1,145 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/inverted/inverted_index_selector.h" + +#include + +#include "common/logging.h" +#include "core/data_type/data_type_array.h" +#include "core/data_type/data_type_nullable.h" +#include "storage/index/inverted/analyzer/analyzer.h" +#include "storage/utils.h" + +namespace doris::segment_v2 { + +Status add_inverted_index_selection_candidate( + InvertedIndexSelectionCandidate candidate, + std::vector* candidates, + InvertedIndexSelectionKeyIndex* key_index) { + DORIS_CHECK(candidates != nullptr); + DORIS_CHECK(key_index != nullptr); + for (const auto& existing : *candidates) { + if (existing.index_id == candidate.index_id) { + return Status::Error( + "Duplicate inverted index id {} in one field", candidate.index_id); + } + } + + const size_t candidate_index = candidates->size(); + candidates->push_back(std::move(candidate)); + (*key_index)[candidates->back().analyzer_key].push_back(candidate_index); + return Status::OK(); +} + +Result select_best_inverted_index_candidate( + const std::vector& candidates, + const InvertedIndexSelectionKeyIndex& key_index, FieldType field_type, + InvertedIndexQueryType query_type, std::string_view normalized_analyzer_key) { + if (candidates.empty()) { + return ResultError(Status::Error( + "No available inverted index candidates")); + } + + const std::vector* exact_candidates = nullptr; + if (!normalized_analyzer_key.empty()) { + const auto exact = key_index.find(std::string(normalized_analyzer_key)); + if (exact == key_index.end() || exact->second.empty()) { + return ResultError(Status::Error( + "No inverted index found for analyzer '{}'", normalized_analyzer_key)); + } + exact_candidates = &exact->second; + } + + const size_t candidate_count = + exact_candidates == nullptr ? candidates.size() : exact_candidates->size(); + auto candidate_at = + [&](size_t ordinal) -> std::pair { + const size_t index = exact_candidates == nullptr ? ordinal : (*exact_candidates)[ordinal]; + DORIS_CHECK(index < candidates.size()); + return {index, candidates[index]}; + }; + auto pick = + [&](std::optional preferred_type) -> std::optional { + std::optional best; + for (size_t ordinal = 0; ordinal < candidate_count; ++ordinal) { + const auto [index, candidate] = candidate_at(ordinal); + if (preferred_type.has_value() && candidate.reader_type != *preferred_type) { + continue; + } + if (!best.has_value() || candidate.index_id < candidates[*best].index_id) { + best = index; + } + } + return best; + }; + + if (is_string_type(field_type)) { + if (is_match_query(query_type)) { + if (auto best = pick(InvertedIndexReaderType::FULLTEXT); best.has_value()) { + return *best; + } + } + if (is_equal_query(query_type)) { + if (auto best = pick(InvertedIndexReaderType::STRING_TYPE); best.has_value()) { + return *best; + } + } + } else if (field_is_numeric_type(field_type)) { + if (is_range_query(query_type)) { + if (auto best = pick(InvertedIndexReaderType::BKD); best.has_value()) { + return *best; + } + } + if (auto best = pick(InvertedIndexReaderType::BKD); best.has_value()) { + return *best; + } + if (auto best = pick(InvertedIndexReaderType::STRING_TYPE); best.has_value()) { + return *best; + } + } + + auto best = pick(std::nullopt); + DORIS_CHECK(best.has_value()); + return *best; +} + +FieldType get_inverted_index_leaf_field_type(const DataTypePtr& column_type) { + DORIS_CHECK(column_type != nullptr); + DataTypePtr leaf_type = remove_nullable(column_type); + while (leaf_type->get_storage_field_type() == FieldType::OLAP_FIELD_TYPE_ARRAY) { + const auto* array_type = dynamic_cast(leaf_type.get()); + DORIS_CHECK(array_type != nullptr); + leaf_type = remove_nullable(array_type->get_nested_type()); + } + return leaf_type->get_storage_field_type(); +} + +InvertedIndexReaderType infer_inverted_index_reader_type( + FieldType field_type, const std::map& properties) { + if (is_string_type(field_type)) { + return inverted_index::InvertedIndexAnalyzer::should_analyzer(properties) + ? InvertedIndexReaderType::FULLTEXT + : InvertedIndexReaderType::STRING_TYPE; + } + if (field_is_numeric_type(field_type)) { + return InvertedIndexReaderType::BKD; + } + return InvertedIndexReaderType::UNKNOWN; +} + +} // namespace doris::segment_v2 diff --git a/be/src/storage/index/inverted/inverted_index_selector.h b/be/src/storage/index/inverted/inverted_index_selector.h new file mode 100644 index 00000000000000..c295a9db3bc33e --- /dev/null +++ b/be/src/storage/index/inverted/inverted_index_selector.h @@ -0,0 +1,57 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "core/data_type/data_type.h" +#include "storage/index/inverted/inverted_index_query_type.h" +#include "storage/olap_common.h" + +namespace doris::segment_v2 { + +struct InvertedIndexSelectionCandidate { + int64_t index_id; + InvertedIndexReaderType reader_type; + std::string analyzer_key; +}; + +using InvertedIndexSelectionKeyIndex = std::unordered_map>; + +Status add_inverted_index_selection_candidate( + InvertedIndexSelectionCandidate candidate, + std::vector* candidates, + InvertedIndexSelectionKeyIndex* key_index); + +[[nodiscard]] Result select_best_inverted_index_candidate( + const std::vector& candidates, + const InvertedIndexSelectionKeyIndex& key_index, FieldType field_type, + InvertedIndexQueryType query_type, std::string_view normalized_analyzer_key); + +FieldType get_inverted_index_leaf_field_type(const DataTypePtr& column_type); + +InvertedIndexReaderType infer_inverted_index_reader_type( + FieldType field_type, const std::map& properties); + +} // namespace doris::segment_v2 diff --git a/be/src/storage/index/inverted/query/conjunction_query.cpp b/be/src/storage/index/inverted/query/conjunction_query.cpp index 8fc3ba0b16148d..eb25605d938803 100644 --- a/be/src/storage/index/inverted/query/conjunction_query.cpp +++ b/be/src/storage/index/inverted/query/conjunction_query.cpp @@ -17,8 +17,8 @@ #include "storage/index/inverted/query/conjunction_query.h" -#include "storage/compaction/collection_statistics.h" #include "storage/index/inverted/query/query_helper.h" +#include "storage/index/inverted/similarity/collection_statistics.h" #include "storage/index/inverted/util/mock_iterator.h" #include "storage/index/inverted/util/string_helper.h" diff --git a/be/src/storage/index/inverted/query/query_info.h b/be/src/storage/index/inverted/query/query_info.h index 829faaca4d7f24..685d3eee951b84 100644 --- a/be/src/storage/index/inverted/query/query_info.h +++ b/be/src/storage/index/inverted/query/query_info.h @@ -17,17 +17,25 @@ #pragma once +#include +#include #include #include namespace doris::segment_v2 { +enum class TermKeyKind : uint8_t { + kPlain = 0, + kCommonGram = 1, +}; + class TermInfo { public: using Term = std::variant>; Term term; int32_t position = 0; + TermKeyKind key_kind = TermKeyKind::kPlain; bool is_single_term() const { return std::holds_alternative(term); } bool is_multi_terms() const { return std::holds_alternative>(term); } @@ -49,6 +57,15 @@ class InvertedIndexQueryInfo { // for test bool use_mock_iter = false; + bool has_common_gram() const { + for (const auto& term_info : term_infos) { + if (term_info.key_kind == TermKeyKind::kCommonGram) { + return true; + } + } + return false; + } + std::string generate_tokens_key() const { std::string key; for (const auto& token : term_infos) { diff --git a/be/src/storage/index/inverted/query_v2/scored_bit_set_query/scored_bit_set_query.h b/be/src/storage/index/inverted/query_v2/scored_bit_set_query/scored_bit_set_query.h new file mode 100644 index 00000000000000..2b4a5846899ab5 --- /dev/null +++ b/be/src/storage/index/inverted/query_v2/scored_bit_set_query/scored_bit_set_query.h @@ -0,0 +1,67 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include "roaring/roaring.hh" +#include "storage/index/inverted/query_v2/bit_set_query/bit_set_weight.h" +#include "storage/index/inverted/query_v2/query.h" +#include "storage/index/inverted/query_v2/scored_bit_set_query/scored_bit_set_weight.h" + +namespace doris::segment_v2::inverted_index::query_v2 { + +// A pre-resolved doc set that also carries a relevance score per document. +// +// This is BitSetQuery plus scores, and exists separately rather than as an option on BitSetQuery +// because BitSetQuery's constant 1.0 is depended on by the CLucene/V3 leaves and by non-scoring +// uses; widening its contract would change their behaviour silently. +// +// Built by the SEARCH leaf builder for clauses the SNII native reader answers with BM25: that +// reader scores inside its own query() call, so by the time the query tree is assembled the +// scores already exist and only need carrying to the scorer. +class ScoredBitSetQuery : public Query { +public: + ScoredBitSetQuery(std::shared_ptr bitmap, + std::shared_ptr null_bitmap, ScoredBitSetMapPtr scores) + : _bitmap(std::move(bitmap)), + _null_bitmap(std::move(null_bitmap)), + _scores(std::move(scores)) { + DCHECK(_scores != nullptr); + } + ~ScoredBitSetQuery() override = default; + + WeightPtr weight(bool enable_scoring) override { + if (!enable_scoring) { + // A non-scoring execution never calls score(), so give it the plain doc-set weight + // instead of making it carry the score map through every scorer it builds. + return std::make_shared(_bitmap, _null_bitmap); + } + return std::make_shared(_bitmap, _null_bitmap, _scores); + } + +private: + std::shared_ptr _bitmap; + std::shared_ptr _null_bitmap; + ScoredBitSetMapPtr _scores; +}; + +using ScoredBitSetQueryPtr = std::shared_ptr; + +} // namespace doris::segment_v2::inverted_index::query_v2 diff --git a/be/src/storage/index/inverted/query_v2/scored_bit_set_query/scored_bit_set_scorer.h b/be/src/storage/index/inverted/query_v2/scored_bit_set_query/scored_bit_set_scorer.h new file mode 100644 index 00000000000000..59aae9a09f1eec --- /dev/null +++ b/be/src/storage/index/inverted/query_v2/scored_bit_set_query/scored_bit_set_scorer.h @@ -0,0 +1,86 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include "roaring/roaring.hh" +#include "storage/compaction/collection_similarity.h" +#include "storage/index/inverted/query_v2/bit_set_query/bit_set_scorer.h" +#include "storage/index/inverted/query_v2/scorer.h" + +namespace doris::segment_v2::inverted_index::query_v2 { + +using ScoredBitSetMap = doris::ScoreMap; +using ScoredBitSetMapPtr = std::shared_ptr; + +// A doc set whose relevance scores were computed before the query tree was assembled. +// +// The SNII native reader answers a whole clause inside its own query() call and produces the +// per-document BM25 values as a side effect there, long before this scorer exists. It therefore +// cannot participate in the incremental term/norm scoring the CLucene-backed scorers do; the +// values are simply looked up per document as the collector walks the doc set. +// +// Iteration is delegated to BitSetScorer so the two stay identical by construction: only score() +// differs, and duplicating the roaring iteration would be the thing most likely to drift. +class ScoredBitSetScorer final : public Scorer { +public: + ScoredBitSetScorer(std::shared_ptr bitmap, + std::shared_ptr null_bitmap, ScoredBitSetMapPtr scores) + : _doc_set(std::move(bitmap), std::move(null_bitmap)), _scores(std::move(scores)) { + DCHECK(_scores != nullptr); + } + ~ScoredBitSetScorer() override = default; + + uint32_t advance() override { return _doc_set.advance(); } + + uint32_t seek(uint32_t target) override { return _doc_set.seek(target); } + + uint32_t doc() const override { return _doc_set.doc(); } + + uint32_t size_hint() const override { return _doc_set.size_hint(); } + + float score() override { + const uint32_t current = _doc_set.doc(); + if (current == TERMINATED) { + return 0.0F; + } + auto it = _scores->find(current); + // A matched document without a score means the producer scored only part of the doc set. + // Zero matches how CollectionSimilarity itself reports an unscored row, so an unexpected + // gap degrades to "ranks last" rather than to a fabricated constant. + DCHECK(it != _scores->end()); + return it != _scores->end() ? it->second : 0.0F; + } + + bool has_null_bitmap(const NullBitmapResolver* resolver = nullptr) override { + return _doc_set.has_null_bitmap(resolver); + } + + const roaring::Roaring* get_null_bitmap(const NullBitmapResolver* resolver = nullptr) override { + return _doc_set.get_null_bitmap(resolver); + } + +private: + BitSetScorer _doc_set; + ScoredBitSetMapPtr _scores; +}; +using ScoredBitSetScorerPtr = std::shared_ptr; + +} // namespace doris::segment_v2::inverted_index::query_v2 diff --git a/be/src/storage/index/inverted/query_v2/scored_bit_set_query/scored_bit_set_weight.h b/be/src/storage/index/inverted/query_v2/scored_bit_set_query/scored_bit_set_weight.h new file mode 100644 index 00000000000000..14dffcf01d2260 --- /dev/null +++ b/be/src/storage/index/inverted/query_v2/scored_bit_set_query/scored_bit_set_weight.h @@ -0,0 +1,54 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include "roaring/roaring.hh" +#include "storage/index/inverted/query_v2/scored_bit_set_query/scored_bit_set_scorer.h" +#include "storage/index/inverted/query_v2/weight.h" + +namespace doris::segment_v2::inverted_index::query_v2 { + +class ScoredBitSetWeight final : public Weight { +public: + ScoredBitSetWeight(std::shared_ptr bitmap, + std::shared_ptr null_bitmap, ScoredBitSetMapPtr scores) + : _bitmap(std::move(bitmap)), + _null_bitmap(std::move(null_bitmap)), + _scores(std::move(scores)) {} + ~ScoredBitSetWeight() override = default; + + ScorerPtr scorer(const QueryExecutionContext& /*context*/) override { + if ((_bitmap == nullptr || _bitmap->isEmpty()) && + (_null_bitmap == nullptr || _null_bitmap->isEmpty())) { + return std::make_shared(); + } + auto bitmap = _bitmap ? _bitmap : std::make_shared(); + return std::make_shared(std::move(bitmap), _null_bitmap, _scores); + } + +private: + std::shared_ptr _bitmap; + std::shared_ptr _null_bitmap; + ScoredBitSetMapPtr _scores; +}; +using ScoredBitSetWeightPtr = std::shared_ptr; + +} // namespace doris::segment_v2::inverted_index::query_v2 diff --git a/be/src/storage/index/inverted/setting.h b/be/src/storage/index/inverted/setting.h index 51782ab0b2de5d..8e2a7ce1a72dc4 100644 --- a/be/src/storage/index/inverted/setting.h +++ b/be/src/storage/index/inverted/setting.h @@ -19,12 +19,14 @@ #include +#include #include #include #include #include #include #include +#include #include "common/exception.h" @@ -168,6 +170,12 @@ class Settings { return result; } + std::vector> sorted_entries() const { + std::vector> entries(_args.begin(), _args.end()); + std::sort(entries.begin(), entries.end()); + return entries; + } + private: std::unordered_map _args; }; diff --git a/be/src/storage/index/inverted/similarity/collection_statistics.cpp b/be/src/storage/index/inverted/similarity/collection_statistics.cpp new file mode 100644 index 00000000000000..77a3e918462897 --- /dev/null +++ b/be/src/storage/index/inverted/similarity/collection_statistics.cpp @@ -0,0 +1,523 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/inverted/similarity/collection_statistics.h" + +#include +#include +#include + +#include "common/exception.h" +#include "exprs/vexpr.h" +#include "exprs/vexpr_context.h" +#include "exprs/vliteral.h" +#include "exprs/vslot_ref.h" +#include "storage/index/index_file_reader.h" +#include "storage/index/index_reader_helper.h" +#include "storage/index/inverted/analyzer/analyzer.h" +#include "storage/index/inverted/util/string_helper.h" +#include "storage/index/inverted/util/term_iterator.h" +#include "storage/index/snii/format/dict_entry.h" +#include "storage/index/snii/reader/dict_block_cache.h" +#include "storage/index/snii/snii_doris_adapter.h" +#include "storage/rowset/rowset.h" +#include "storage/rowset/rowset_reader.h" +#include "util/uid_util.h" + +namespace doris { +namespace collection_statistics_detail { + +Result resolve_snii_scoring_segment( + const std::optional& metadata, + uint64_t index_doc_count, uint64_t physical_sum_total_term_freq, bool has_scoring_tier, + bool has_positions, bool has_semantic_norms) { + using namespace segment_v2::inverted_index; + const auto* metadata_ptr = metadata ? &*metadata : nullptr; + auto validation_status = validate_snii_scoring_metadata( + metadata_ptr, index_doc_count, physical_sum_total_term_freq, has_scoring_tier, + has_positions, has_semantic_norms); + if (!validation_status.ok()) { + return ResultError(std::move(validation_status)); + } + DORIS_CHECK(metadata_ptr != nullptr); + + return SniiScoringSegmentStats { + .doc_count = metadata_ptr->scoring_doc_count, + .token_count = metadata_ptr->scoring_token_count, + .plain_term_key_version = metadata_ptr->plain_term_key_version, + .base_analyzer_fingerprint = metadata_ptr->base_analyzer_fingerprint}; +} + +void add_term_doc_frequency( + std::unordered_map>* + logical_frequencies, + const std::wstring& field, const std::wstring& logical_term, uint64_t doc_frequency) { + DORIS_CHECK(logical_frequencies != nullptr); + (*logical_frequencies)[field][logical_term] += doc_frequency; +} + +} // namespace collection_statistics_detail + +Status CollectionStatistics::collect(RuntimeState* state, + const std::vector& rs_splits, + const TabletSchemaSPtr& tablet_schema, + const VExprContextSPtrs& common_expr_ctxs_push_down, + io::IOContext* io_ctx) { + std::vector rowsets; + rowsets.reserve(rs_splits.size()); + for (const auto& rs_split : rs_splits) { + DORIS_CHECK(rs_split.rs_reader != nullptr); + auto rowset = rs_split.rs_reader->rowset(); + DORIS_CHECK(rowset != nullptr); + rowsets.emplace_back(std::move(rowset)); + } + return collect_full_collection(state, rowsets, tablet_schema, common_expr_ctxs_push_down, + io_ctx); +} + +Status CollectionStatistics::collect_full_collection( + RuntimeState* state, const std::vector& rowsets, + const TabletSchemaSPtr& tablet_schema, const VExprContextSPtrs& common_expr_ctxs_push_down, + io::IOContext* io_ctx) { + clear(); + std::unordered_map collect_infos; + RETURN_IF_ERROR( + extract_collect_info(state, common_expr_ctxs_push_down, tablet_schema, &collect_infos)); + if (collect_infos.empty()) { + LOG(WARNING) << "Index statistics collection: no collect info extracted."; + return Status::OK(); + } + + for (const auto& rowset : rowsets) { + DORIS_CHECK(rowset != nullptr); + const auto num_segments = rowset->num_segments(); + for (int64_t seg_id = 0; seg_id < num_segments; ++seg_id) { + auto status = + process_segment(rowset, seg_id, tablet_schema.get(), collect_infos, io_ctx); + if (!status.ok()) { + if (tablet_schema->get_inverted_index_storage_format() != + InvertedIndexStorageFormatPB::SNII && + (status.code() == ErrorCode::INVERTED_INDEX_FILE_NOT_FOUND || + status.code() == ErrorCode::INVERTED_INDEX_BYPASS)) { + LOG(ERROR) << "Index statistics collection failed: " << status.to_string(); + continue; + } + clear(); + return status; + } + } + } + + // Build a single-line log with query_id, tablet_ids, and per-field term statistics + if (VLOG_IS_ON(1)) { + std::set tablet_ids; + for (const auto& rowset : rowsets) { + DORIS_CHECK(rowset != nullptr); + tablet_ids.insert(rowset->rowset_meta()->tablet_id()); + } + + std::ostringstream oss; + oss << "CollectionStatistics: query_id=" << print_id(state->query_id()); + + oss << ", tablet_ids=["; + bool first_tablet = true; + for (int64_t tid : tablet_ids) { + if (!first_tablet) oss << ","; + oss << tid; + first_tablet = false; + } + oss << "]"; + + oss << ", total_num_docs=" << _total_num_docs; + + for (const auto& [ws_field_name, num_tokens] : _total_num_tokens) { + oss << ", {field=" << StringHelper::to_string(ws_field_name) + << ", num_tokens=" << num_tokens << ", terms=["; + + auto field_term_doc_freqs = _term_doc_freqs.find(ws_field_name); + if (field_term_doc_freqs != _term_doc_freqs.end()) { + bool first_term = true; + for (const auto& [term, doc_freq] : field_term_doc_freqs->second) { + if (!first_term) oss << ", "; + oss << "(" << StringHelper::to_string(term) << ":" << doc_freq << ")"; + first_term = false; + } + } + oss << "]}"; + } + + VLOG(1) << oss.str(); + } + + return Status::OK(); +} + +Status CollectionStatistics::extract_collect_info( + RuntimeState* state, const VExprContextSPtrs& common_expr_ctxs_push_down, + const TabletSchemaSPtr& tablet_schema, CollectInfoMap* collect_infos) { + DCHECK(collect_infos != nullptr); + + std::unordered_map collectors; + collectors[TExprNodeType::MATCH_PRED] = std::make_unique(); + collectors[TExprNodeType::SEARCH_EXPR] = std::make_unique(); + + for (const auto& root_expr_ctx : common_expr_ctxs_push_down) { + const auto& root_expr = root_expr_ctx->root(); + if (root_expr == nullptr) { + continue; + } + + std::stack stack; + stack.emplace(root_expr); + + while (!stack.empty()) { + auto expr = stack.top(); + stack.pop(); + + if (!expr) { + continue; + } + + auto collector_it = collectors.find(expr->node_type()); + if (collector_it != collectors.end()) { + RETURN_IF_ERROR( + collector_it->second->collect(state, tablet_schema, expr, collect_infos)); + } + + const auto& children = expr->children(); + for (auto child = children.rbegin(); child != children.rend(); ++child) { + stack.push(*child); + } + } + } + + LOG(INFO) << "Extracted collect info for " << collect_infos->size() << " fields"; + + return Status::OK(); +} + +Status CollectionStatistics::process_segment(const RowsetSharedPtr& rowset, int64_t seg_id, + const TabletSchema* tablet_schema, + const CollectInfoMap& collect_infos, + io::IOContext* io_ctx) { + auto seg_path = DORIS_TRY(rowset->segment_path(seg_id)); + auto rowset_meta = rowset->rowset_meta(); + + auto idx_file_reader = std::make_unique( + rowset_meta->fs(), + std::string {InvertedIndexDescriptor::get_index_file_path_prefix(seg_path)}, + tablet_schema->get_inverted_index_storage_format(), + rowset_meta->inverted_index_file_info(static_cast(seg_id)), + rowset_meta->tablet_id()); + const bool is_snii = + idx_file_reader->get_storage_format() == InvertedIndexStorageFormatPB::SNII; + auto init_status = idx_file_reader->init(config::inverted_index_read_buffer_size, io_ctx); + if (!init_status.ok()) { + if (is_snii && (init_status.code() == ErrorCode::NOT_FOUND || + init_status.code() == ErrorCode::INVERTED_INDEX_FILE_NOT_FOUND || + init_status.code() == ErrorCode::INVERTED_INDEX_BYPASS)) { + return Status::Error( + "SNII scoring requires every collection segment: {}", init_status.msg()); + } + return init_status; + } + + if (is_snii) { + segment_v2::snii_doris::DorisSniiFileReader::ScopedIOContext io_context_scope(io_ctx); + SniiScoringSegmentAccumulator segment_accumulator; + for (const auto& [ws_field_name, collect_info] : collect_infos) { + auto logical_reader_result = + idx_file_reader->open_snii_index(collect_info.index_meta, io_ctx); + if (!logical_reader_result.has_value()) { + auto status = std::move(logical_reader_result.error()); + if (status.code() == ErrorCode::INVERTED_INDEX_SNII_NOT_FOUND || + status.code() == ErrorCode::INVERTED_INDEX_FILE_NOT_FOUND || + status.code() == ErrorCode::INVERTED_INDEX_BYPASS) { + return Status::Error( + "SNII scoring requires every logical index in the collection: {}", + status.msg()); + } + return status; + } + auto logical_reader = std::move(logical_reader_result.value()); + const auto* common_grams_metadata = logical_reader->common_grams_metadata(); + segment_v2::inverted_index::PlainTermKeyVersion key_version; + RETURN_IF_ERROR(admit_snii_scoring_segment( + ws_field_name, + common_grams_metadata == nullptr + ? std::nullopt + : std::optional( + *common_grams_metadata), + collect_info.expected_base_analyzer_fingerprint, + logical_reader->stats().doc_count, logical_reader->stats().sum_total_term_freq, + logical_reader->tier() == ::doris::snii::format::IndexTier::kT3, + logical_reader->has_positions(), + logical_reader->section_refs().norms.length != 0, &key_version, + &segment_accumulator)); + DORIS_CHECK(common_grams_metadata != nullptr); + + ::doris::snii::reader::DictBlockCache dict_block_cache; + for (const auto& logical_term_bytes : collect_info.unique_terms) { + std::string physical_term; + const bool term_present = + DORIS_TRY(segment_v2::inverted_index::try_encode_plain_term( + logical_term_bytes, key_version, &physical_term)); + const auto logical_term = + segment_v2::inverted_index::StringHelper::to_wstring(logical_term_bytes); + if (!term_present) { + collection_statistics_detail::add_term_doc_frequency( + &segment_accumulator.term_doc_freqs, ws_field_name, logical_term, 0); + continue; + } + + bool found = false; + ::doris::snii::format::DictEntry entry; + uint64_t frq_base = 0; + uint64_t prx_base = 0; + RETURN_IF_ERROR(logical_reader->lookup(physical_term, &found, &entry, &frq_base, + &prx_base, &dict_block_cache)); + if (found && entry.df > common_grams_metadata->scoring_doc_count) { + return Status::Error( + "SNII term document frequency {} exceeds scoring document count {}", + entry.df, common_grams_metadata->scoring_doc_count); + } + collection_statistics_detail::add_term_doc_frequency( + &segment_accumulator.term_doc_freqs, ws_field_name, logical_term, + found ? entry.df : 0); + } + } + + commit_snii_scoring_segment(std::move(segment_accumulator)); + return Status::OK(); + } + + int32_t total_segment_docs = 0; + + for (const auto& [ws_field_name, collect_info] : collect_infos) { + lucene::search::IndexSearcher* index_searcher = nullptr; + lucene::index::IndexReader* index_reader = nullptr; + +#ifdef BE_TEST + auto compound_reader = DORIS_TRY(idx_file_reader->open(collect_info.index_meta, io_ctx)); + auto* reader = lucene::index::IndexReader::open(compound_reader.get()); + auto owned_index_searcher = std::make_shared(reader, true); + index_searcher = owned_index_searcher.get(); + index_reader = index_searcher->getReader(); +#else + InvertedIndexCacheHandle inverted_index_cache_handle; + auto index_file_key = idx_file_reader->get_index_file_cache_key(collect_info.index_meta); + InvertedIndexSearcherCache::CacheKey searcher_cache_key(index_file_key); + + if (!InvertedIndexSearcherCache::instance()->lookup(searcher_cache_key, + &inverted_index_cache_handle)) { + auto compound_reader = + DORIS_TRY(idx_file_reader->open(collect_info.index_meta, io_ctx)); + auto* reader = lucene::index::IndexReader::open(compound_reader.get()); + size_t reader_size = reader->getTermInfosRAMUsed(); + auto searcher_ptr = std::make_shared(reader, true); + auto* cache_value = new InvertedIndexSearcherCache::CacheValue( + std::move(searcher_ptr), reader_size, UnixMillis()); + InvertedIndexSearcherCache::instance()->insert(searcher_cache_key, cache_value, + &inverted_index_cache_handle); + } + + auto searcher_variant = inverted_index_cache_handle.get_index_searcher(); + auto index_searcher_ptr = std::get(searcher_variant); + index_searcher = index_searcher_ptr.get(); + index_reader = index_searcher->getReader(); +#endif + total_segment_docs = std::max(total_segment_docs, index_reader->maxDoc()); + _total_num_tokens[ws_field_name] += + index_reader->sumTotalTermFreq(ws_field_name.c_str()).value_or(0); + + for (const auto& logical_term_bytes : collect_info.unique_terms) { + const auto logical_term = + segment_v2::inverted_index::StringHelper::to_wstring(logical_term_bytes); + auto iter = TermIterator::create(io_ctx, false, index_reader, ws_field_name, + logical_term_bytes); + collection_statistics_detail::add_term_doc_frequency(&_term_doc_freqs, ws_field_name, + logical_term, iter->doc_freq()); + } + } + + _total_num_docs += static_cast(total_segment_docs); + _avg_dl_by_col.clear(); + _idf_by_col_term.clear(); + + return Status::OK(); +} + +Status CollectionStatistics::admit_snii_scoring_segment( + const std::wstring& field_name, + const std::optional& metadata, + std::string_view expected_base_analyzer_fingerprint, uint64_t index_doc_count, + uint64_t physical_sum_total_term_freq, bool has_scoring_tier, bool has_positions, + bool has_semantic_norms, + segment_v2::inverted_index::PlainTermKeyVersion* plain_term_key_version, + SniiScoringSegmentAccumulator* segment_accumulator) { + DORIS_CHECK(plain_term_key_version != nullptr); + DORIS_CHECK(segment_accumulator != nullptr); + auto segment_stats = collection_statistics_detail::resolve_snii_scoring_segment( + metadata, index_doc_count, physical_sum_total_term_freq, has_scoring_tier, + has_positions, has_semantic_norms); + if (!segment_stats.has_value()) { + clear(); + return segment_stats.error(); + } + + const auto& base_analyzer_fingerprint = segment_stats->base_analyzer_fingerprint; + if (base_analyzer_fingerprint != expected_base_analyzer_fingerprint) { + clear(); + return Status::Error( + "SNII scoring segment base analyzer does not match the request analyzer for field " + "{}", + StringHelper::to_string(field_name)); + } + auto staged_fingerprint = segment_accumulator->base_analyzer_fingerprints.find(field_name); + if (staged_fingerprint != segment_accumulator->base_analyzer_fingerprints.end() && + staged_fingerprint->second != base_analyzer_fingerprint) { + clear(); + return Status::Error( + "SNII scoring cannot combine segments with different base analyzers for field {}", + StringHelper::to_string(field_name)); + } + auto collected_fingerprint = _snii_base_analyzer_fingerprints.find(field_name); + if (collected_fingerprint != _snii_base_analyzer_fingerprints.end() && + collected_fingerprint->second != base_analyzer_fingerprint) { + clear(); + return Status::Error( + "SNII scoring cannot combine segments with different base analyzers for field {}", + StringHelper::to_string(field_name)); + } + segment_accumulator->base_analyzer_fingerprints.insert_or_assign(field_name, + base_analyzer_fingerprint); + + if (!segment_accumulator->token_counts.empty() && + segment_accumulator->doc_count != segment_stats->doc_count) { + clear(); + return Status::Error( + "SNII scoring fields in one segment have different document counts: {} and {}", + segment_accumulator->doc_count, segment_stats->doc_count); + } + segment_accumulator->doc_count = segment_stats->doc_count; + segment_accumulator->token_counts[field_name] += segment_stats->token_count; + *plain_term_key_version = segment_stats->plain_term_key_version; + return Status::OK(); +} + +void CollectionStatistics::commit_snii_scoring_segment( + SniiScoringSegmentAccumulator&& segment_accumulator) { + for (const auto& [field_name, base_analyzer_fingerprint] : + segment_accumulator.base_analyzer_fingerprints) { + auto [collected_fingerprint, inserted] = + _snii_base_analyzer_fingerprints.try_emplace(field_name, base_analyzer_fingerprint); + DORIS_CHECK(inserted || collected_fingerprint->second == base_analyzer_fingerprint); + } + for (const auto& [field_name, token_count] : segment_accumulator.token_counts) { + _total_num_tokens[field_name] += token_count; + } + for (const auto& [field_name, term_doc_freqs] : segment_accumulator.term_doc_freqs) { + for (const auto& [term, doc_freq] : term_doc_freqs) { + _term_doc_freqs[field_name][term] += doc_freq; + } + } + _total_num_docs += segment_accumulator.doc_count; + _avg_dl_by_col.clear(); + _idf_by_col_term.clear(); +} + +void CollectionStatistics::clear() { + _total_num_docs = 0; + _total_num_tokens.clear(); + _term_doc_freqs.clear(); + _snii_base_analyzer_fingerprints.clear(); + _avg_dl_by_col.clear(); + _idf_by_col_term.clear(); +} + +uint64_t CollectionStatistics::get_term_doc_freq_by_col(const std::wstring& lucene_col_name, + const std::wstring& term) { + const auto field = _term_doc_freqs.find(lucene_col_name); + if (field == _term_doc_freqs.end()) { + throw Exception(ErrorCode::INVERTED_INDEX_CLUCENE_ERROR, + "Index statistics collection failed: Not such column {}", + StringHelper::to_string(lucene_col_name)); + } + + const auto term_frequency = field->second.find(term); + if (term_frequency == field->second.end()) { + throw Exception(ErrorCode::INVERTED_INDEX_CLUCENE_ERROR, + "Index statistics collection failed: Not such term {}", + StringHelper::to_string(term)); + } + + return term_frequency->second; +} + +uint64_t CollectionStatistics::get_total_term_cnt_by_col(const std::wstring& lucene_col_name) { + const auto token_count = _total_num_tokens.find(lucene_col_name); + if (token_count == _total_num_tokens.end()) { + throw Exception(ErrorCode::INVERTED_INDEX_CLUCENE_ERROR, + "Index statistics collection failed: Not such column {}", + StringHelper::to_string(lucene_col_name)); + } + + return token_count->second; +} + +uint64_t CollectionStatistics::get_doc_num() const { + if (_total_num_docs == 0) { + throw Exception( + ErrorCode::INVERTED_INDEX_CLUCENE_ERROR, + "Index statistics collection failed: No data available for SimilarityCollector"); + } + + return _total_num_docs; +} + +float CollectionStatistics::get_or_calculate_avg_dl(const std::wstring& lucene_col_name) { + auto iter = _avg_dl_by_col.find(lucene_col_name); + if (iter != _avg_dl_by_col.end()) { + return iter->second; + } + + const uint64_t total_term_cnt = get_total_term_cnt_by_col(lucene_col_name); + const uint64_t total_doc_cnt = get_doc_num(); + float avg_dl = total_doc_cnt > 0 ? float((double)total_term_cnt / (double)total_doc_cnt) : 0.0F; + _avg_dl_by_col[lucene_col_name] = avg_dl; + return avg_dl; +} + +float CollectionStatistics::get_or_calculate_idf(const std::wstring& lucene_col_name, + const std::wstring& term) { + auto iter = _idf_by_col_term.find(lucene_col_name); + if (iter != _idf_by_col_term.end()) { + auto term_iter = iter->second.find(term); + if (term_iter != iter->second.end()) { + return term_iter->second; + } + } + + const uint64_t doc_num = get_doc_num(); + const uint64_t doc_freq = get_term_doc_freq_by_col(lucene_col_name, term); + auto idf = (float)std::log(1 + ((double)doc_num - (double)doc_freq + (double)0.5) / + ((double)doc_freq + (double)0.5)); + _idf_by_col_term[lucene_col_name][term] = idf; + return idf; +} + +} // namespace doris diff --git a/be/src/storage/compaction/collection_statistics.h b/be/src/storage/index/inverted/similarity/collection_statistics.h similarity index 53% rename from be/src/storage/compaction/collection_statistics.h rename to be/src/storage/index/inverted/similarity/collection_statistics.h index b93d5a424ae1ab..fda35b368af402 100644 --- a/be/src/storage/compaction/collection_statistics.h +++ b/be/src/storage/index/inverted/similarity/collection_statistics.h @@ -16,16 +16,24 @@ // under the License. #pragma once +#include #include +#include +#include +#include #include +#include #include +#include +#include #include "common/be_mock_util.h" #include "exprs/vexpr_fwd.h" #include "runtime/runtime_state.h" +#include "storage/index/inverted/common_grams/common_grams_segment_metadata.h" #include "storage/index/inverted/query/query_info.h" +#include "storage/index/inverted/similarity/predicate_collector.h" #include "storage/olap_common.h" -#include "storage/predicate_collector.h" namespace doris { @@ -52,19 +60,40 @@ class CollectionStatistics { Status collect(RuntimeState* state, const std::vector& rs_splits, const TabletSchemaSPtr& tablet_schema, const VExprContextSPtrs& common_expr_ctxs_push_down, io::IOContext* io_ctx); + Status collect_full_collection(RuntimeState* state, const std::vector& rowsets, + const TabletSchemaSPtr& tablet_schema, + const VExprContextSPtrs& common_expr_ctxs_push_down, + io::IOContext* io_ctx); MOCK_FUNCTION float get_or_calculate_idf(const std::wstring& lucene_col_name, const std::wstring& term); MOCK_FUNCTION float get_or_calculate_avg_dl(const std::wstring& lucene_col_name); private: + struct SniiScoringSegmentAccumulator { + uint64_t doc_count = 0; + std::unordered_map token_counts; + std::unordered_map> term_doc_freqs; + std::unordered_map base_analyzer_fingerprints; + }; + Status extract_collect_info(RuntimeState* state, const VExprContextSPtrs& common_expr_ctxs_push_down, const TabletSchemaSPtr& tablet_schema, CollectInfoMap* collect_infos); - Status process_segment(const RowsetSharedPtr& rowset, int32_t seg_id, + Status process_segment(const RowsetSharedPtr& rowset, int64_t seg_id, const TabletSchema* tablet_schema, const CollectInfoMap& collect_infos, io::IOContext* io_ctx); + Status admit_snii_scoring_segment( + const std::wstring& field_name, + const std::optional& metadata, + std::string_view expected_base_analyzer_fingerprint, uint64_t index_doc_count, + uint64_t physical_sum_total_term_freq, bool has_scoring_tier, bool has_positions, + bool has_semantic_norms, + segment_v2::inverted_index::PlainTermKeyVersion* plain_term_key_version, + SniiScoringSegmentAccumulator* segment_accumulator); + void commit_snii_scoring_segment(SniiScoringSegmentAccumulator&& segment_accumulator); + void clear(); uint64_t get_term_doc_freq_by_col(const std::wstring& lucene_col_name, const std::wstring& term); @@ -74,6 +103,7 @@ class CollectionStatistics { uint64_t _total_num_docs = 0; std::unordered_map _total_num_tokens; std::unordered_map> _term_doc_freqs; + std::unordered_map _snii_base_analyzer_fingerprints; std::unordered_map _avg_dl_by_col; std::unordered_map> _idf_by_col_term; @@ -85,4 +115,29 @@ class CollectionStatistics { }; using CollectionStatisticsPtr = std::shared_ptr; +// Implementation details of the SNII scoring-segment admission math, surfaced so +// collection_statistics_test.cpp can exercise them without compiling the .cpp a +// second time via #include. +namespace collection_statistics_detail { + +struct SniiScoringSegmentStats { + uint64_t doc_count = 0; + uint64_t token_count = 0; + segment_v2::inverted_index::PlainTermKeyVersion plain_term_key_version = + segment_v2::inverted_index::PlainTermKeyVersion::kLegacyRaw; + std::string base_analyzer_fingerprint; +}; + +Result resolve_snii_scoring_segment( + const std::optional& metadata, + uint64_t index_doc_count, uint64_t physical_sum_total_term_freq, bool has_scoring_tier, + bool has_positions, bool has_semantic_norms); + +void add_term_doc_frequency( + std::unordered_map>* + logical_frequencies, + const std::wstring& field, const std::wstring& logical_term, uint64_t doc_frequency); + +} // namespace collection_statistics_detail + } // namespace doris diff --git a/be/src/storage/index/inverted/similarity/predicate_collector.cpp b/be/src/storage/index/inverted/similarity/predicate_collector.cpp new file mode 100644 index 00000000000000..29f4a5b33a66db --- /dev/null +++ b/be/src/storage/index/inverted/similarity/predicate_collector.cpp @@ -0,0 +1,593 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/inverted/similarity/predicate_collector.h" + +#include + +#include + +#include "exec/common/variant_util.h" +#include "exprs/vexpr.h" +#include "exprs/vexpr_context.h" +#include "exprs/vliteral.h" +#include "exprs/vsearch.h" +#include "exprs/vslot_ref.h" +#include "gen_cpp/Exprs_types.h" +#include "storage/index/index_reader_helper.h" +#include "storage/index/inverted/analyzer/analyzer.h" +#include "storage/index/inverted/inverted_index_selector.h" +#include "storage/index/inverted/util/string_helper.h" +#include "storage/tablet/tablet_schema.h" +#include "storage/utils.h" + +namespace doris { + +using namespace segment_v2; + +namespace { + +InvertedIndexAnalyzerCtx analyzer_context_from_properties( + const std::map& properties) { + InvertedIndexAnalyzerConfig config; + config.analyzer_name = get_analyzer_name_from_properties(properties); + config.parser_type = get_inverted_index_parser_type_from_string( + get_parser_string_from_properties(properties)); + config.parser_mode = get_parser_mode_string_from_properties(properties); + config.lower_case = get_parser_lowercase_from_properties(properties); + config.stop_words = get_parser_stopwords_from_properties(properties); + config.char_filter_map = get_parser_char_filter_map_from_properties(properties); + + InvertedIndexAnalyzerCtx analyzer_ctx; + analyzer_ctx.analyzer_name = config.analyzer_name; + analyzer_ctx.parser_type = config.parser_type; + analyzer_ctx.char_filter_map = config.char_filter_map; + analyzer_ctx.analyzer_provider = + inverted_index::InvertedIndexAnalyzer::create_analyzer_provider(&config); + return analyzer_ctx; +} + +std::vector analyze_plain_query(const std::string& value, + const InvertedIndexAnalyzerCtx& analyzer_ctx) { + DORIS_CHECK(analyzer_ctx.analyzer_provider != nullptr); + auto analyzer = analyzer_ctx.analyzer_provider->get_analyzer( + inverted_index::AnalysisPurpose::kPlainQuery); + auto reader = + inverted_index::InvertedIndexAnalyzer::create_reader(analyzer_ctx.char_filter_map); + reader->init(value.data(), static_cast(value.size()), true); + return inverted_index::InvertedIndexAnalyzer::get_analyse_result(reader, analyzer.get()); +} + +Status append_scoring_leaf(CollectInfo* collect_info, const std::vector& term_infos, + std::string_view base_analyzer_fingerprint) { + DORIS_CHECK(collect_info != nullptr); + if (!collect_info->logical_scoring_leaves.empty() && + collect_info->expected_base_analyzer_fingerprint != base_analyzer_fingerprint) { + return Status::Error( + "Scoring predicates for one field use different base analyzers"); + } + if (collect_info->logical_scoring_leaves.empty()) { + collect_info->expected_base_analyzer_fingerprint = base_analyzer_fingerprint; + } + + LogicalScoringLeaf leaf; + leaf.clauses.reserve(term_infos.size()); + for (const auto& term_info : term_infos) { + DORIS_CHECK(term_info.is_single_term()); + DORIS_CHECK(term_info.key_kind == TermKeyKind::kPlain); + const auto& term = term_info.get_single_term(); + auto [slot, inserted] = collect_info->unique_term_slots.try_emplace( + term, static_cast(collect_info->unique_terms.size())); + if (inserted) { + collect_info->unique_terms.push_back(term); + } + leaf.clauses.emplace_back( + LogicalScoringClause {.df_slot = slot->second, .position = term_info.position}); + } + collect_info->logical_scoring_leaves.emplace_back(std::move(leaf)); + return Status::OK(); +} + +InvertedIndexQueryType match_query_type(TExprOpcode::type opcode) { + switch (opcode) { + case TExprOpcode::MATCH_ANY: + return InvertedIndexQueryType::MATCH_ANY_QUERY; + case TExprOpcode::MATCH_ALL: + return InvertedIndexQueryType::MATCH_ALL_QUERY; + case TExprOpcode::MATCH_PHRASE: + return InvertedIndexQueryType::MATCH_PHRASE_QUERY; + case TExprOpcode::MATCH_PHRASE_PREFIX: + return InvertedIndexQueryType::MATCH_PHRASE_PREFIX_QUERY; + case TExprOpcode::MATCH_REGEXP: + return InvertedIndexQueryType::MATCH_REGEXP_QUERY; + case TExprOpcode::MATCH_PHRASE_EDGE: + return InvertedIndexQueryType::MATCH_PHRASE_EDGE_QUERY; + default: + return InvertedIndexQueryType::UNKNOWN_QUERY; + } +} + +InvertedIndexQueryType search_query_type(std::string_view clause_type) { + if (clause_type == "EXACT") { + return InvertedIndexQueryType::EQUAL_QUERY; + } + if (clause_type == "PHRASE") { + return InvertedIndexQueryType::MATCH_PHRASE_QUERY; + } + if (clause_type == "ALL") { + return InvertedIndexQueryType::MATCH_ALL_QUERY; + } + return InvertedIndexQueryType::MATCH_ANY_QUERY; +} + +Result select_index_meta(const std::vector& index_metas, + FieldType field_type, + InvertedIndexQueryType query_type, + std::string_view analyzer_key) { + std::vector candidates; + candidates.reserve(index_metas.size()); + InvertedIndexSelectionKeyIndex key_index; + for (const auto* index_meta : index_metas) { + auto status = add_inverted_index_selection_candidate( + InvertedIndexSelectionCandidate {.index_id = index_meta->index_id(), + .reader_type = infer_inverted_index_reader_type( + field_type, index_meta->properties()), + .analyzer_key = build_analyzer_key_from_properties( + index_meta->properties())}, + &candidates, &key_index); + if (!status.ok()) { + return ResultError(std::move(status)); + } + } + + auto selection = select_best_inverted_index_candidate( + candidates, key_index, field_type, query_type, normalize_analyzer_key(analyzer_key)); + if (!selection.has_value()) { + return ResultError(std::move(selection.error())); + } + const size_t selected = *selection; + DORIS_CHECK(selected < index_metas.size()); + return index_metas[selected]; +} + +Status validate_same_physical_index(const CollectInfo& collect_info, + const TabletIndex& selected_index) { + DORIS_CHECK(collect_info.index_meta != nullptr); + if (collect_info.index_meta->index_id() != selected_index.index_id() || + collect_info.index_meta->get_index_suffix() != selected_index.get_index_suffix()) { + return Status::Error( + "Scoring predicates for one field select different inverted indexes: {} and {}", + collect_info.index_meta->index_id(), selected_index.index_id()); + } + return Status::OK(); +} + +struct ScoringIndexCandidates { + FieldType field_type = FieldType::OLAP_FIELD_TYPE_UNKNOWN; + std::string index_suffix_path; + std::vector index_metas; + std::vector> owned_index_metas; +}; + +FieldType scoring_leaf_type(const TabletColumn& column) { + const TabletColumn* leaf = &column; + while (leaf->is_array_type()) { + DORIS_CHECK_EQ(leaf->get_subtype_count(), 1); + leaf = &leaf->get_sub_column(0); + } + return leaf->type(); +} + +ScoringIndexCandidates resolve_text_scoring_index_candidates(const TabletSchemaSPtr& tablet_schema, + const TabletColumn& column) { + ScoringIndexCandidates candidates {.field_type = scoring_leaf_type(column), + .index_suffix_path = column.suffix_path(), + .index_metas = tablet_schema->inverted_indexs(column), + .owned_index_metas = {}}; + + // The collector has tablet-schema context but no segment-side variant + // inference. Resolve only shapes that are deterministic from schema: + // typed/materialized paths, field-pattern templates, and a plain parent + // index inherited by the dynamic VARIANT placeholder. + if (!candidates.index_metas.empty() || !column.is_extracted_column()) { + return candidates; + } + + TabletSchema::SubColumnInfo sub_column_info; + const std::string relative_path = column.path_info_ptr()->copy_pop_front().get_path(); + if (variant_util::generate_sub_column_info(*tablet_schema, column.parent_unique_id(), + relative_path, &sub_column_info) && + !sub_column_info.indexes.empty()) { + candidates.field_type = scoring_leaf_type(sub_column_info.column); + candidates.index_suffix_path = sub_column_info.column.suffix_path(); + for (auto& index : sub_column_info.indexes) { + candidates.index_metas.push_back(index.get()); + candidates.owned_index_metas.emplace_back(std::move(index)); + } + return candidates; + } + + if (!column.is_variant_type()) { + return candidates; + } + + // MATCH and score-bearing SEARCH clauses have text semantics. When a dynamic + // VARIANT path has no materialized type, those semantics provide the missing + // leaf-type proof for selecting its plain parent full-text index. Typed paths + // returned above keep their schema type, so numeric BKD leaves remain rejected. + candidates.field_type = FieldType::OLAP_FIELD_TYPE_STRING; + const auto parent_indexes = tablet_schema->inverted_indexs(column.parent_unique_id()); + for (const auto* index : parent_indexes) { + if (!index->field_pattern().empty()) { + continue; + } + auto owned_index = std::make_shared(*index); + owned_index->set_escaped_escaped_index_suffix_path(column.path_info_ptr()->get_path()); + candidates.index_metas.push_back(owned_index.get()); + candidates.owned_index_metas.emplace_back(std::move(owned_index)); + } + return candidates; +} + +Status validate_scoring_leaf_type(const ScoringIndexCandidates& candidates, + std::string_view field_name) { + if (candidates.field_type == FieldType::OLAP_FIELD_TYPE_VARIANT) { + return Status::Error( + "Index statistics collection failed: Cannot prove scoring leaf type for field={}", + field_name); + } + return Status::OK(); +} + +void preserve_selected_index_metadata(const ScoringIndexCandidates& candidates, + const TabletIndex* selected_index, + CollectInfo* collect_info) { + DORIS_CHECK(selected_index != nullptr); + DORIS_CHECK(collect_info != nullptr); + collect_info->index_meta = selected_index; + for (const auto& owned_index : candidates.owned_index_metas) { + if (owned_index.get() == selected_index) { + collect_info->owned_index_meta = owned_index; + return; + } + } +} + +Result resolve_search_scoring_index_candidates( + const TabletSchemaSPtr& tablet_schema, const std::string& field_name, + const TSearchFieldBinding* field_binding) { + const int32_t column_index = tablet_schema->field_index(field_name); + if (column_index >= 0) { + return resolve_text_scoring_index_candidates(tablet_schema, + tablet_schema->column(column_index)); + } + + if (field_binding == nullptr || !field_binding->__isset.is_variant_subcolumn || + !field_binding->is_variant_subcolumn || !field_binding->__isset.parent_field_name || + field_binding->parent_field_name.empty() || !field_binding->__isset.subcolumn_path || + field_binding->subcolumn_path.empty()) { + return ResultError(Status::Error( + "Index statistics collection failed: Cannot resolve search field={}", field_name)); + } + + const int32_t parent_column_index = + tablet_schema->field_index(field_binding->parent_field_name); + if (parent_column_index < 0) { + return ResultError(Status::Error( + "Index statistics collection failed: Cannot resolve parent={} for search field={}", + field_binding->parent_field_name, field_name)); + } + const auto& parent_column = tablet_schema->column(parent_column_index); + if (!parent_column.is_variant_type()) { + return ResultError(Status::Error( + "Index statistics collection failed: Search field={} parent={} is not VARIANT", + field_name, field_binding->parent_field_name)); + } + + TabletColumn dynamic_column; + dynamic_column.set_unique_id(-1); + dynamic_column.set_name(field_name); + dynamic_column.set_type(FieldType::OLAP_FIELD_TYPE_VARIANT); + dynamic_column.set_parent_unique_id(parent_column.unique_id()); + dynamic_column.set_path_info( + PathInData(field_binding->parent_field_name + "." + field_binding->subcolumn_path)); + return resolve_text_scoring_index_candidates(tablet_schema, dynamic_column); +} + +} // namespace + +VSlotRef* PredicateCollector::find_slot_ref(const VExprSPtr& expr) const { + if (!expr) { + return nullptr; + } + + auto cur = VExpr::expr_without_cast(expr); + if (cur->node_type() == TExprNodeType::SLOT_REF) { + return static_cast(cur.get()); + } + + for (const auto& ch : cur->children()) { + if (auto* s = find_slot_ref(ch)) { + return s; + } + } + + return nullptr; +} + +std::string PredicateCollector::build_field_name(int32_t col_unique_id, + const std::string& suffix_path) const { + std::string field_name = std::to_string(col_unique_id); + if (!suffix_path.empty()) { + field_name += "." + suffix_path; + } + return field_name; +} + +Status MatchPredicateCollector::collect(RuntimeState* state, const TabletSchemaSPtr& tablet_schema, + const VExprSPtr& expr, CollectInfoMap* collect_infos) { + DCHECK(collect_infos != nullptr); + + auto* left_slot_ref = find_slot_ref(expr->children()[0]); + if (left_slot_ref == nullptr) { + return Status::Error( + "Index statistics collection failed: Cannot find slot reference in match predicate " + "left expression"); + } + + auto* right_literal = static_cast(expr->children()[1].get()); + DCHECK(right_literal != nullptr); + + const auto* sd = state->desc_tbl().get_slot_descriptor(left_slot_ref->slot_id()); + if (sd == nullptr) { + return Status::Error( + "Index statistics collection failed: Cannot find slot descriptor for slot_id={}", + left_slot_ref->slot_id()); + } + + int32_t col_idx = tablet_schema->field_index(left_slot_ref->column_name()); + if (col_idx == -1) { + return Status::Error( + "Index statistics collection failed: Cannot find column index for column={}", + left_slot_ref->column_name()); + } + + const auto& column = tablet_schema->column(col_idx); + auto candidates = resolve_text_scoring_index_candidates(tablet_schema, column); + RETURN_IF_ERROR(validate_scoring_leaf_type(candidates, left_slot_ref->column_name())); + +#ifndef BE_TEST + if (candidates.index_metas.empty()) { + return Status::Error( + "Index statistics collection failed: Score query is not supported without inverted " + "index for column={}", + left_slot_ref->column_name()); + } +#else + if (candidates.index_metas.empty()) { + return Status::OK(); + } +#endif + + const auto* analyzer_ctx = expr->query_analyzer_ctx(); + DORIS_CHECK(analyzer_ctx != nullptr); + const auto query_type = match_query_type(expr->op()); + DORIS_CHECK(query_type != InvertedIndexQueryType::UNKNOWN_QUERY); + const auto* index_meta = DORIS_TRY(select_index_meta( + candidates.index_metas, candidates.field_type, query_type, analyzer_ctx->analyzer_key)); + if (!InvertedIndexAnalyzer::should_analyzer(index_meta->properties()) || + !IndexReaderHelper::is_need_similarity_score(expr->op(), index_meta)) { + return Status::OK(); + } + + DORIS_CHECK(analyzer_ctx->analyzer_provider != nullptr); + auto options = DataTypeSerDe::get_default_format_options(); + options.timezone = &state->timezone_obj(); + auto term_infos = analyze_plain_query(right_literal->value(options), *analyzer_ctx); + if (expr->op() == TExprOpcode::MATCH_PHRASE_PREFIX && !term_infos.empty()) { + term_infos.pop_back(); + } + const auto base_analyzer_fingerprint = + analyzer_ctx->analyzer_provider->base_analyzer_fingerprint(); + + std::string field_name = + build_field_name(index_meta->col_unique_ids()[0], candidates.index_suffix_path); + std::wstring ws_field_name = StringHelper::to_wstring(field_name); + + auto iter = collect_infos->find(ws_field_name); + if (iter == collect_infos->end()) { + CollectInfo collect_info; + RETURN_IF_ERROR(append_scoring_leaf(&collect_info, term_infos, base_analyzer_fingerprint)); + preserve_selected_index_metadata(candidates, index_meta, &collect_info); + (*collect_infos)[ws_field_name] = std::move(collect_info); + } else { + RETURN_IF_ERROR(validate_same_physical_index(iter->second, *index_meta)); + RETURN_IF_ERROR(append_scoring_leaf(&iter->second, term_infos, base_analyzer_fingerprint)); + } + + return Status::OK(); +} + +Status SearchPredicateCollector::collect(RuntimeState* state, const TabletSchemaSPtr& tablet_schema, + const VExprSPtr& expr, CollectInfoMap* collect_infos) { + DCHECK(collect_infos != nullptr); + + auto* search_expr = dynamic_cast(expr.get()); + if (search_expr == nullptr) { + return Status::InternalError("SearchPredicateCollector: expr is not VSearchExpr type"); + } + + const TSearchParam& search_param = search_expr->get_search_param(); + FieldBindingMap field_bindings; + field_bindings.reserve(search_param.field_bindings.size()); + for (const auto& field_binding : search_param.field_bindings) { + field_bindings[field_binding.field_name] = &field_binding; + } + + RETURN_IF_ERROR(collect_from_clause(search_param.root, state, tablet_schema, field_bindings, + collect_infos)); + + return Status::OK(); +} + +Status SearchPredicateCollector::collect_from_clause(const TSearchClause& clause, + RuntimeState* state, + const TabletSchemaSPtr& tablet_schema, + const FieldBindingMap& field_bindings, + CollectInfoMap* collect_infos) { + const std::string& clause_type = clause.clause_type; + if (clause_type == "NESTED") { + return Status::Error( + "Scoring nested search clauses is not supported"); + } + ClauseTypeCategory category = get_clause_type_category(clause_type); + + if (category == ClauseTypeCategory::COMPOUND) { + if (clause.__isset.children) { + for (const auto& child_clause : clause.children) { + RETURN_IF_ERROR(collect_from_clause(child_clause, state, tablet_schema, + field_bindings, collect_infos)); + } + } + return Status::OK(); + } + + return collect_from_leaf(clause, state, tablet_schema, field_bindings, collect_infos); +} + +Status SearchPredicateCollector::collect_from_leaf(const TSearchClause& clause, RuntimeState* state, + const TabletSchemaSPtr& tablet_schema, + const FieldBindingMap& field_bindings, + CollectInfoMap* collect_infos) { + if (!clause.__isset.field_name || !clause.__isset.value) { + return Status::InvalidArgument("Search clause missing field_name or value"); + } + + const std::string& field_name = clause.field_name; + const std::string& value = clause.value; + const std::string& clause_type = clause.clause_type; + + if (!is_score_query_type(clause_type)) { + return Status::OK(); + } + + const auto field_binding_iter = field_bindings.find(field_name); + const auto* field_binding = + field_binding_iter == field_bindings.end() ? nullptr : field_binding_iter->second; + auto candidates = DORIS_TRY( + resolve_search_scoring_index_candidates(tablet_schema, field_name, field_binding)); + RETURN_IF_ERROR(validate_scoring_leaf_type(candidates, field_name)); + if (candidates.index_metas.empty()) { + return Status::Error( + "Index statistics collection failed: Score query is not supported without " + "inverted index for search field={}", + field_name); + } + + ClauseTypeCategory category = get_clause_type_category(clause_type); + auto query_type = search_query_type(clause_type); + std::string analyzer_key; + if (query_type != InvertedIndexQueryType::EQUAL_QUERY && field_binding != nullptr && + field_binding->__isset.index_properties && !field_binding->index_properties.empty() && + is_string_type(candidates.field_type)) { + analyzer_key = build_analyzer_key_from_properties(field_binding->index_properties); + } + + auto selected_index = select_index_meta(candidates.index_metas, candidates.field_type, + query_type, analyzer_key); + if (!selected_index.has_value()) { + return Status::Error( + "Index statistics collection failed: Cannot select scoring index for search " + "field={}: {}", + field_name, selected_index.error().to_string()); + } + const auto* index_meta = *selected_index; + if (infer_inverted_index_reader_type(candidates.field_type, index_meta->properties()) == + InvertedIndexReaderType::BKD) { + return Status::Error( + "Index statistics collection failed: BM25 scoring does not support numeric BKD " + "search field={}", + field_name); + } + + const auto& analysis_properties = index_meta->properties(); + + std::vector term_infos; + std::string_view base_analyzer_fingerprint; + std::optional analyzer_ctx; + if (InvertedIndexAnalyzer::should_analyzer(analysis_properties)) { + analyzer_ctx.emplace(analyzer_context_from_properties(analysis_properties)); + base_analyzer_fingerprint = analyzer_ctx->analyzer_provider->base_analyzer_fingerprint(); + } + + if (clause_type == "MATCH") { + term_infos.emplace_back(value); + } else if (category == ClauseTypeCategory::TOKENIZED) { + if (analyzer_ctx.has_value()) { + term_infos = analyze_plain_query(value, *analyzer_ctx); + } else { + term_infos.emplace_back(value); + } + } else if (category == ClauseTypeCategory::NON_TOKENIZED) { + if (clause_type == "TERM" && analyzer_ctx.has_value()) { + term_infos = analyze_plain_query(value, *analyzer_ctx); + } else { + term_infos.emplace_back(value); + } + } + + std::string lucene_field_name = + build_field_name(index_meta->col_unique_ids()[0], candidates.index_suffix_path); + std::wstring ws_field_name = StringHelper::to_wstring(lucene_field_name); + + auto iter = collect_infos->find(ws_field_name); + if (iter == collect_infos->end()) { + CollectInfo collect_info; + RETURN_IF_ERROR(append_scoring_leaf(&collect_info, term_infos, base_analyzer_fingerprint)); + preserve_selected_index_metadata(candidates, index_meta, &collect_info); + (*collect_infos)[ws_field_name] = std::move(collect_info); + } else { + RETURN_IF_ERROR(validate_same_physical_index(iter->second, *index_meta)); + RETURN_IF_ERROR(append_scoring_leaf(&iter->second, term_infos, base_analyzer_fingerprint)); + } + + return Status::OK(); +} + +bool SearchPredicateCollector::is_score_query_type(const std::string& clause_type) const { + return clause_type == "TERM" || clause_type == "EXACT" || clause_type == "PHRASE" || + clause_type == "MATCH" || clause_type == "ANY" || clause_type == "ALL"; +} + +SearchPredicateCollector::ClauseTypeCategory SearchPredicateCollector::get_clause_type_category( + const std::string& clause_type) const { + if (clause_type == "AND" || clause_type == "OR" || clause_type == "NOT" || + clause_type == "OCCUR_BOOLEAN") { + return ClauseTypeCategory::COMPOUND; + } else if (clause_type == "TERM" || clause_type == "EXACT") { + return ClauseTypeCategory::NON_TOKENIZED; + } else if (clause_type == "PHRASE" || clause_type == "MATCH" || clause_type == "ANY" || + clause_type == "ALL") { + return ClauseTypeCategory::TOKENIZED; + } else { + LOG(WARNING) << "Unknown clause type '" << clause_type + << "', defaulting to NON_TOKENIZED category"; + return ClauseTypeCategory::NON_TOKENIZED; + } +} + +} // namespace doris diff --git a/be/src/storage/predicate_collector.h b/be/src/storage/index/inverted/similarity/predicate_collector.h similarity index 80% rename from be/src/storage/predicate_collector.h rename to be/src/storage/index/inverted/similarity/predicate_collector.h index c96e0af9c45ed5..c43865a63a52d3 100644 --- a/be/src/storage/predicate_collector.h +++ b/be/src/storage/index/inverted/similarity/predicate_collector.h @@ -17,10 +17,11 @@ #pragma once +#include #include -#include #include #include +#include #include "common/status.h" #include "exprs/vexpr_fwd.h" @@ -35,14 +36,20 @@ class TabletIndex; class TabletSchema; using TabletSchemaSPtr = std::shared_ptr; -struct TermInfoComparer { - bool operator()(const segment_v2::TermInfo& lhs, const segment_v2::TermInfo& rhs) const { - return lhs.term < rhs.term; - } +struct LogicalScoringClause { + uint32_t df_slot = 0; + int32_t position = 0; +}; + +struct LogicalScoringLeaf { + std::vector clauses; }; struct CollectInfo { - std::set term_infos; + std::vector unique_terms; + std::unordered_map unique_term_slots; + std::vector logical_scoring_leaves; + std::string expected_base_analyzer_fingerprint; std::shared_ptr owned_index_meta; const TabletIndex* index_meta = nullptr; }; @@ -73,16 +80,19 @@ class SearchPredicateCollector : public PredicateCollector { private: enum class ClauseTypeCategory { NON_TOKENIZED, TOKENIZED, COMPOUND }; + using FieldBindingMap = std::unordered_map; Status collect_from_clause(const TSearchClause& clause, RuntimeState* state, const TabletSchemaSPtr& tablet_schema, + const FieldBindingMap& field_bindings, CollectInfoMap* collect_infos); Status collect_from_leaf(const TSearchClause& clause, RuntimeState* state, - const TabletSchemaSPtr& tablet_schema, CollectInfoMap* collect_infos); + const TabletSchemaSPtr& tablet_schema, + const FieldBindingMap& field_bindings, CollectInfoMap* collect_infos); bool is_score_query_type(const std::string& clause_type) const; ClauseTypeCategory get_clause_type_category(const std::string& clause_type) const; }; using PredicateCollectorPtr = std::unique_ptr; -} // namespace doris \ No newline at end of file +} // namespace doris diff --git a/be/src/storage/index/inverted/token_filter/ascii_folding_filter_factory.h b/be/src/storage/index/inverted/token_filter/ascii_folding_filter_factory.h index 9710f0f3cadede..ee3321d96a27c7 100644 --- a/be/src/storage/index/inverted/token_filter/ascii_folding_filter_factory.h +++ b/be/src/storage/index/inverted/token_filter/ascii_folding_filter_factory.h @@ -35,6 +35,11 @@ class ASCIIFoldingFilterFactory : public TokenFilterFactory { return std::make_shared(in, _preserve_original); } + PositionCapability position_capability() const override { + return _preserve_original ? PositionCapability::kUnknown + : PositionCapability::kAlwaysUnitIncrement; + } + private: bool _preserve_original = false; }; diff --git a/be/src/storage/index/inverted/token_filter/common_grams_filter.cpp b/be/src/storage/index/inverted/token_filter/common_grams_filter.cpp new file mode 100644 index 00000000000000..2781ea58842975 --- /dev/null +++ b/be/src/storage/index/inverted/token_filter/common_grams_filter.cpp @@ -0,0 +1,384 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/inverted/token_filter/common_grams_filter.h" + +#include +#include + +#include "common/exception.h" +#include "common/logging.h" +#include "storage/index/inverted/common_grams/common_grams_key_codec.h" + +namespace doris::segment_v2::inverted_index { +namespace { + +void validate_input_token_shape(const Token& token, std::string_view term) { + if (token.getPositionIncrement() != 1) { + throw Exception(ErrorCode::INVERTED_INDEX_ANALYZER_ERROR, + "CommonGrams requires position increment 1, got {}", + token.getPositionIncrement()); + } + if (term.empty()) { + throw Exception(ErrorCode::INVERTED_INDEX_ANALYZER_ERROR, + "CommonGrams requires non-empty input tokens"); + } +} + +void validate_input_token(const Token& token, std::string_view term) { + validate_input_token_shape(token, term); + auto status = validate_common_grams_logical_term(term, "input token"); + if (!status.ok()) { + throw Exception(status); + } +} + +void set_output_token(Token* token, std::string_view term, int32_t position_increment, + int32_t start_offset, int32_t end_offset, const TCHAR* type) { + token->clear(); + token->setTextNoCopy(term.data(), static_cast(term.size())); + token->setPositionIncrement(position_increment); + token->setStartOffset(start_offset); + token->setEndOffset(end_offset); + token->setType(type); +} + +bool is_common_word(const CommonGramsBufferedToken& token, const CommonWordSet& common_words, + std::optional* cached_membership) { + if (!cached_membership->has_value()) { + cached_membership->emplace(common_words.contains(token.term)); + } + return cached_membership->value(); +} + +} // namespace + +bool common_grams_query_may_use_gram(std::span terms, CommonGramsQueryMode mode, + const CommonWordSet& common_words) { + if (terms.size() < 2) { + return false; + } + const size_t relevant_term_count = + mode == CommonGramsQueryMode::kPhrasePrefix ? terms.size() - 1 : terms.size(); + for (size_t i = 0; i < relevant_term_count; ++i) { + if (common_words.contains(terms[i])) { + return true; + } + } + return false; +} + +CommonGramsFilter::CommonGramsFilter(TokenStreamPtr in, + std::shared_ptr common_words, + CommonGramsOutputMode output_mode) + : DorisTokenFilter(std::move(in)), + _common_words(std::move(common_words)), + _output_mode(output_mode) { + DORIS_CHECK(_common_words != nullptr); +} + +bool CommonGramsFilter::read_input(CommonGramsBufferedToken* token, + std::optional* is_common) { + if (_in->next(&_input_token) == nullptr) { + return false; + } + std::string_view term(_input_token.termBuffer(), _input_token.termLength()); + validate_input_token(_input_token, term); + token->term.assign(term); + token->start_offset = _input_token.startOffset(); + token->end_offset = _input_token.endOffset(); + token->type = _input_token.type(); + is_common->reset(); + return true; +} + +Token* CommonGramsFilter::emit_unigram(Token* token, const CommonGramsBufferedToken& buffered) { + const std::string_view output_term = encode_plain_term(buffered.term); + set_output_token(token, output_term, 1, buffered.start_offset, buffered.end_offset, + buffered.type); + return token; +} + +std::string_view CommonGramsFilter::encode_plain_term(std::string_view term) { + if (_output_mode != CommonGramsOutputMode::kLogical && !term.empty() && + (term.front() == PLAIN_ESCAPE_PREFIX || term.front() == '\x1f')) { + if (!try_encode_escaped_plain_term_prevalidated(term, _encoded_plain_term)) { + throw Exception(Status::Error( + "CommonGrams escaped plain term would exceed the 16383-byte key limit; " + "set enable_common_grams_index_build=false and retry the import in a new " + "transaction")); + } + return _encoded_plain_term; + } + return term; +} + +std::string_view CommonGramsFilter::encode_snii_plain_term(std::string_view term) { + DORIS_CHECK(_output_mode == CommonGramsOutputMode::kEscapedV1SpimiIndex); + DORIS_CHECK(!term.empty()); + if (term.front() != PLAIN_ESCAPE_PREFIX && term.front() != '\x1f') { + return term; + } + if (term.size() == COMMON_GRAM_MAX_ENCODED_BYTES) { + throw Exception(Status::Error( + "CommonGrams escaped plain term would exceed the 16383-byte key limit; " + "set enable_common_grams_index_build=false and retry the import in a new " + "transaction")); + } + DORIS_CHECK_LT(term.size(), COMMON_GRAM_MAX_ENCODED_BYTES); + _encoded_plain_term.clear(); + _encoded_plain_term.reserve(term.size() + 1); + _encoded_plain_term.push_back(PLAIN_ESCAPE_PREFIX); + _encoded_plain_term.push_back(term.front() == PLAIN_ESCAPE_PREFIX ? 'E' : 'G'); + _encoded_plain_term.append(term.substr(1)); + return _encoded_plain_term; +} + +Token* CommonGramsFilter::emit_gram(Token* token, int32_t start_offset, int32_t end_offset) { + set_output_token(token, _gram, 0, start_offset, end_offset, COMMON_GRAM_TOKEN_TYPE); + return token; +} + +Token* CommonGramsFilter::next(Token* token) { + if (_emit_current) { + _emit_current = false; + return emit_unigram(token, _current); + } + + if (!_has_current) { + if (!read_input(&_current, &_current_is_common)) { + return nullptr; + } + _has_current = true; + return emit_unigram(token, _current); + } + + if (!read_input(&_lookahead, &_lookahead_is_common)) { + _has_current = false; + return nullptr; + } + + const bool uses_gram = is_common_word(_current, *_common_words, &_current_is_common) || + is_common_word(_lookahead, *_common_words, &_lookahead_is_common); + if (uses_gram) { + const bool encoded = + try_encode_common_gram_prevalidated(_current.term, _lookahead.term, _gram); + if (encoded) { + const int32_t start_offset = _current.start_offset; + const int32_t end_offset = _lookahead.end_offset; + std::swap(_current, _lookahead); + std::swap(_current_is_common, _lookahead_is_common); + _emit_current = true; + return emit_gram(token, start_offset, end_offset); + } + } + + std::swap(_current, _lookahead); + std::swap(_current_is_common, _lookahead_is_common); + return emit_unigram(token, _current); +} + +bool CommonGramsFilter::next_snii_index_event(SniiCommonGramsIndexEvent* event) { + DORIS_CHECK(event != nullptr); + DORIS_CHECK(_output_mode == CommonGramsOutputMode::kEscapedV1SpimiIndex); + if (_in->next(&_input_token) == nullptr) { + return false; + } + + const std::string_view logical_term(_input_token.termBuffer(), + _input_token.termLength()); + validate_input_token_shape(_input_token, logical_term); + const bool requires_prevalidation = + logical_term.size() > COMMON_GRAM_MAX_ENCODED_BYTES || + (logical_term.size() == COMMON_GRAM_MAX_ENCODED_BYTES && + (logical_term.front() == PLAIN_ESCAPE_PREFIX || logical_term.front() == '\x1f')); + if (requires_prevalidation) { + auto status = validate_common_grams_logical_term(logical_term, "input token"); + if (!status.ok()) { + throw Exception(status); + } + } + event->logical_term = logical_term; + event->plain_term = encode_snii_plain_term(logical_term); + return true; +} + +void CommonGramsFilter::reset() { + DorisTokenFilter::reset(); + _current.term.clear(); + _current.start_offset = 0; + _current.end_offset = 0; + _current.type = Token::getDefaultType(); + _lookahead.term.clear(); + _lookahead.start_offset = 0; + _lookahead.end_offset = 0; + _lookahead.type = Token::getDefaultType(); + _current_is_common.reset(); + _lookahead_is_common.reset(); + _has_current = false; + _emit_current = false; + _gram.clear(); + _encoded_plain_term.clear(); +} + +Token* CommonGramsPositionFilter::next(Token* token) { + if (_in->next(token) == nullptr) { + return nullptr; + } + const std::string_view term(token->termBuffer(), token->termLength()); + validate_input_token(*token, term); + return token; +} + +CommonGramsQueryFilterBase::CommonGramsQueryFilterBase( + TokenStreamPtr in, std::shared_ptr common_words, + CommonGramsQueryMode mode) + : DorisTokenFilter(std::move(in)), _common_words(std::move(common_words)), _mode(mode) { + DORIS_CHECK(_common_words != nullptr); +} + +bool CommonGramsQueryFilterBase::pair_uses_gram( + const std::vector& unigrams, std::optional* left_is_common, + size_t pair_index) const { + const bool is_prefix_boundary = + _mode == CommonGramsQueryMode::kPhrasePrefix && pair_index + 2 == unigrams.size(); + if (is_prefix_boundary) { + return is_common_word(unigrams[pair_index], *_common_words, left_is_common); + } + + std::optional right_is_common; + const bool uses_gram = + is_common_word(unigrams[pair_index], *_common_words, left_is_common) || + is_common_word(unigrams[pair_index + 1], *_common_words, &right_is_common); + *left_is_common = right_is_common; + return uses_gram; +} + +void CommonGramsQueryFilterBase::append_plain_output(std::vector* output, + const CommonGramsBufferedToken& token) { + output->push_back(token); +} + +bool CommonGramsQueryFilterBase::append_gram_output( + std::vector* output, + const std::optional& indexed_gram, + const CommonGramsBufferedToken& left, const CommonGramsBufferedToken& right) { + if (!indexed_gram.has_value()) { + DCHECK(!is_common_gram_encodable(left.term, right.term)); + return false; + } + CommonGramsBufferedToken query_gram = *indexed_gram; + query_gram.type = COMMON_GRAM_TOKEN_TYPE; + output->push_back(std::move(query_gram)); + return true; +} + +void CommonGramsQueryFilterBase::prepare_output() { + std::vector unigrams; + std::vector> indexed_grams; + Token token; + while (_in->next(&token) != nullptr) { + const std::string_view term(token.termBuffer(), token.termLength()); + if (token.getPositionIncrement() == 0) { + if (!is_common_gram_token_type(token.type()) || unigrams.empty() || + indexed_grams.back().has_value()) { + throw Exception(ErrorCode::INVERTED_INDEX_ANALYZER_ERROR, + "Invalid indexed CommonGrams token sequence"); + } + auto status = validate_common_grams_logical_term(term, "indexed gram"); + if (!status.ok()) { + throw Exception(status); + } + indexed_grams.back() = CommonGramsBufferedToken {.term = std::string(term), + .start_offset = token.startOffset(), + .end_offset = token.endOffset(), + .type = token.type()}; + continue; + } + validate_input_token(token, term); + unigrams.push_back({.term = std::string(term), + .start_offset = token.startOffset(), + .end_offset = token.endOffset(), + .type = token.type()}); + indexed_grams.emplace_back(); + } + + std::vector output; + if (unigrams.size() < 2) { + _output = std::move(unigrams); + _prepared = true; + return; + } + + bool last_pair_used_gram = false; + std::optional left_is_common; + for (size_t i = 0; i + 1 < unigrams.size(); ++i) { + last_pair_used_gram = pair_uses_gram(unigrams, &left_is_common, i); + if (last_pair_used_gram) { + if (!append_gram_output(&output, indexed_grams[i], unigrams[i], unigrams[i + 1])) { + output.clear(); + for (const auto& unigram : unigrams) { + append_plain_output(&output, unigram); + } + _output = std::move(output); + _prepared = true; + return; + } + } else { + append_plain_output(&output, unigrams[i]); + } + } + if (!last_pair_used_gram) { + append_plain_output(&output, unigrams.back()); + } + _output = std::move(output); + _prepared = true; +} + +Token* CommonGramsQueryFilterBase::emit(Token* token, const CommonGramsBufferedToken& buffered) { + set_output_token(token, buffered.term, 1, buffered.start_offset, buffered.end_offset, + buffered.type); + return token; +} + +Token* CommonGramsQueryFilterBase::next(Token* token) { + if (_failure != nullptr) { + std::rethrow_exception(_failure); + } + if (!_prepared) { + try { + prepare_output(); + } catch (...) { + _failure = std::current_exception(); + throw; + } + } + if (_next_output == _output.size()) { + return nullptr; + } + return emit(token, _output[_next_output++]); +} + +void CommonGramsQueryFilterBase::reset() { + DorisTokenFilter::reset(); + _output.clear(); + _next_output = 0; + _prepared = false; + _failure = nullptr; +} + +} // namespace doris::segment_v2::inverted_index diff --git a/be/src/storage/index/inverted/token_filter/common_grams_filter.h b/be/src/storage/index/inverted/token_filter/common_grams_filter.h new file mode 100644 index 00000000000000..5fae9ce79890ac --- /dev/null +++ b/be/src/storage/index/inverted/token_filter/common_grams_filter.h @@ -0,0 +1,152 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include "storage/index/inverted/common_grams/common_word_set.h" +#include "storage/index/inverted/token_filter/token_filter.h" + +namespace doris::segment_v2::inverted_index { + +inline constexpr const TCHAR* COMMON_GRAM_TOKEN_TYPE = L"common_gram"; + +inline bool is_common_gram_token_type(const TCHAR* type) { + return std::wstring_view(type) == COMMON_GRAM_TOKEN_TYPE; +} + +struct CommonGramsBufferedToken { + std::string term; + int32_t start_offset = 0; + int32_t end_offset = 0; + const TCHAR* type = Token::getDefaultType(); +}; + +enum class CommonGramsOutputMode { + kLogical, + kEscapedV1Index, + kEscapedV1SpimiIndex, +}; + +struct SniiCommonGramsIndexEvent { + // Analyzer output before the EscapedV1 namespace transform. The view has the + // same lifetime as plain_term and is used for validate-on-first-intern. + std::string_view logical_term; + // EscapedV1 physical plain key. The view remains valid until the next event + // call or reset; the SNII writer interns it synchronously. + std::string_view plain_term; +}; + +class CommonGramsFilter final : public DorisTokenFilter { +public: + CommonGramsFilter(TokenStreamPtr in, std::shared_ptr common_words, + CommonGramsOutputMode output_mode = CommonGramsOutputMode::kLogical); + + Token* next(Token* token) override; + bool next_snii_index_event(SniiCommonGramsIndexEvent* event); + void reset() override; + const CommonWordSet& common_words() const { return *_common_words; } + +private: + bool read_input(CommonGramsBufferedToken* token, std::optional* is_common); + std::string_view encode_plain_term(std::string_view term); + std::string_view encode_snii_plain_term(std::string_view term); + Token* emit_unigram(Token* token, const CommonGramsBufferedToken& buffered); + Token* emit_gram(Token* token, int32_t start_offset, int32_t end_offset); + + std::shared_ptr _common_words; + CommonGramsOutputMode _output_mode; + Token _input_token; + CommonGramsBufferedToken _current; + CommonGramsBufferedToken _lookahead; + std::optional _current_is_common; + std::optional _lookahead_is_common; + bool _has_current = false; + bool _emit_current = false; + std::string _gram; + std::string _encoded_plain_term; +}; + +class CommonGramsPositionFilter final : public DorisTokenFilter { +public: + explicit CommonGramsPositionFilter(TokenStreamPtr in) : DorisTokenFilter(std::move(in)) {} + + Token* next(Token* token) override; +}; + +enum class CommonGramsQueryMode { + kExact, + kPhrasePrefix, +}; + +// A false result proves that the purpose-specific query filter cannot select a +// gram. A true result stays conservative because key encoding may still force +// the filter to replay the complete plain stream. +bool common_grams_query_may_use_gram(std::span terms, CommonGramsQueryMode mode, + const CommonWordSet& common_words); + +class CommonGramsQueryFilterBase : public DorisTokenFilter { +public: + CommonGramsQueryFilterBase(TokenStreamPtr in, std::shared_ptr common_words, + CommonGramsQueryMode mode); + + Token* next(Token* token) override; + void reset() override; + +private: + void prepare_output(); + bool pair_uses_gram(const std::vector& unigrams, + std::optional* left_is_common, size_t pair_index) const; + static void append_plain_output(std::vector* output, + const CommonGramsBufferedToken& token); + static bool append_gram_output(std::vector* output, + const std::optional& indexed_gram, + const CommonGramsBufferedToken& left, + const CommonGramsBufferedToken& right); + static Token* emit(Token* token, const CommonGramsBufferedToken& buffered); + + std::shared_ptr _common_words; + CommonGramsQueryMode _mode; + std::vector _output; + size_t _next_output = 0; + bool _prepared = false; + std::exception_ptr _failure; +}; + +class CommonGramsQueryFilter final : public CommonGramsQueryFilterBase { +public: + CommonGramsQueryFilter(TokenStreamPtr in, std::shared_ptr common_words) + : CommonGramsQueryFilterBase(std::move(in), std::move(common_words), + CommonGramsQueryMode::kExact) {} +}; + +class CommonGramsPhrasePrefixFilter final : public CommonGramsQueryFilterBase { +public: + CommonGramsPhrasePrefixFilter(TokenStreamPtr in, + std::shared_ptr common_words) + : CommonGramsQueryFilterBase(std::move(in), std::move(common_words), + CommonGramsQueryMode::kPhrasePrefix) {} +}; + +} // namespace doris::segment_v2::inverted_index diff --git a/be/src/storage/index/inverted/token_filter/common_grams_filter_factory.h b/be/src/storage/index/inverted/token_filter/common_grams_filter_factory.h new file mode 100644 index 00000000000000..7803c7f00b6004 --- /dev/null +++ b/be/src/storage/index/inverted/token_filter/common_grams_filter_factory.h @@ -0,0 +1,107 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include "common/exception.h" +#include "storage/index/inverted/token_filter/common_grams_filter.h" +#include "storage/index/inverted/token_filter/token_filter_factory.h" + +namespace doris::segment_v2::inverted_index { + +class CommonGramsFilterFactory final : public TokenFilterFactory { +public: + explicit CommonGramsFilterFactory(std::shared_ptr common_words = nullptr) + : _common_words(std::move(common_words)) {} + + void initialize(const Settings& settings) override { + if (!settings.empty()) { + throw Exception(ErrorCode::INVERTED_INDEX_ANALYZER_ERROR, + "common_grams does not accept settings"); + } + if (_common_words == nullptr) { + _common_words = CommonWordSet::default_word_set(); + } + } + + TokenFilterPtr create(const TokenStreamPtr& in) override { + return std::make_shared(in, _common_words, _output_mode); + } + + void set_common_words(std::shared_ptr common_words) { + _common_words = std::move(common_words); + } + + void set_output_mode(CommonGramsOutputMode output_mode) { _output_mode = output_mode; } + + const std::shared_ptr& common_words() const { return _common_words; } + +private: + std::shared_ptr _common_words; + CommonGramsOutputMode _output_mode = CommonGramsOutputMode::kLogical; +}; + +class CommonGramsPositionFilterFactory final : public TokenFilterFactory { +public: + void initialize(const Settings&) override {} + + TokenFilterPtr create(const TokenStreamPtr& in) override { + return std::make_shared(in); + } + + PositionCapability position_capability() const override { + return PositionCapability::kAlwaysUnitIncrement; + } +}; + +class CommonGramsQueryFilterFactory final : public TokenFilterFactory { +public: + explicit CommonGramsQueryFilterFactory(std::shared_ptr common_words) + : _common_words(std::move(common_words)) {} + + void initialize(const Settings&) override {} + + TokenFilterPtr create(const TokenStreamPtr& in) override { + return std::make_shared(in, _common_words); + } + + const std::shared_ptr& common_words() const { return _common_words; } + +private: + std::shared_ptr _common_words; +}; + +class CommonGramsPhrasePrefixFilterFactory final : public TokenFilterFactory { +public: + explicit CommonGramsPhrasePrefixFilterFactory(std::shared_ptr common_words) + : _common_words(std::move(common_words)) {} + + void initialize(const Settings&) override {} + + TokenFilterPtr create(const TokenStreamPtr& in) override { + return std::make_shared(in, _common_words); + } + + const std::shared_ptr& common_words() const { return _common_words; } + +private: + std::shared_ptr _common_words; +}; + +} // namespace doris::segment_v2::inverted_index diff --git a/be/src/storage/index/inverted/token_filter/empty_token_filter_factory.h b/be/src/storage/index/inverted/token_filter/empty_token_filter_factory.h index 2855546e78bdb1..d1c42966f14a11 100644 --- a/be/src/storage/index/inverted/token_filter/empty_token_filter_factory.h +++ b/be/src/storage/index/inverted/token_filter/empty_token_filter_factory.h @@ -46,6 +46,10 @@ class EmptyTokenFilterFactory : public TokenFilterFactory { token_filter->initialize(); return token_filter; } + + PositionCapability position_capability() const override { + return PositionCapability::kAlwaysUnitIncrement; + } }; } // namespace doris::segment_v2::inverted_index \ No newline at end of file diff --git a/be/src/storage/index/inverted/token_filter/icu_normalizer_filter_factory.h b/be/src/storage/index/inverted/token_filter/icu_normalizer_filter_factory.h index c0cedb8c483cbd..27dc897e818a34 100644 --- a/be/src/storage/index/inverted/token_filter/icu_normalizer_filter_factory.h +++ b/be/src/storage/index/inverted/token_filter/icu_normalizer_filter_factory.h @@ -82,6 +82,10 @@ class ICUNormalizerFilterFactory : public TokenFilterFactory { return std::make_shared(in, _normalizer); } + PositionCapability position_capability() const override { + return PositionCapability::kAlwaysUnitIncrement; + } + private: static const icu::Normalizer2* get_normalizer(const std::string& name, UErrorCode& status) { std::string lower_name = to_lower_copy(trim_copy(name)); diff --git a/be/src/storage/index/inverted/token_filter/lower_case_filter.h b/be/src/storage/index/inverted/token_filter/lower_case_filter.h index dd55d48060d03a..ff737209a1ea09 100644 --- a/be/src/storage/index/inverted/token_filter/lower_case_filter.h +++ b/be/src/storage/index/inverted/token_filter/lower_case_filter.h @@ -19,10 +19,40 @@ #include +#ifdef BE_TEST +#include +#endif + +#include "common/cast_set.h" +#include "common/exception.h" #include "storage/index/inverted/token_filter/token_filter.h" +#include "util/utf8_check.h" namespace doris::segment_v2::inverted_index { +#ifdef BE_TEST +namespace lower_case_testing { + +inline std::atomic& unicode_path_counter() { + static std::atomic counter {0}; + return counter; +} + +inline uint64_t unicode_path_count() { + return unicode_path_counter().load(std::memory_order_relaxed); +} + +inline void reset_unicode_path_count() { + unicode_path_counter().store(0, std::memory_order_relaxed); +} + +inline void note_unicode_path() { + unicode_path_counter().fetch_add(1, std::memory_order_relaxed); +} + +} // namespace lower_case_testing +#endif + /** * @brief A token filter that converts Unicode text to lowercase using ICU library. * @@ -40,7 +70,7 @@ class LowerCaseFilter : public DorisTokenFilter { UErrorCode status = U_ZERO_ERROR; auto* ucsm = ucasemap_open("", 0, &status); if (U_FAILURE(status)) { - throw Exception(ErrorCode::RUNTIME_ERROR, + throw Exception(ErrorCode::INVERTED_INDEX_ANALYZER_ERROR, "Failed to open UCaseMap. ICU Error: " + std::to_string(status) + " - " + u_errorName(status)); } @@ -52,21 +82,53 @@ class LowerCaseFilter : public DorisTokenFilter { return nullptr; } std::string_view term(t->termBuffer(), t->termLength()); - - size_t max_len = term.size() * 2; - if (_lower_term.size() < max_len) { - _lower_term.resize(max_len); + bool has_ascii_upper = false; + bool all_ascii = true; + for (const char value : term) { + const auto byte = static_cast(value); + if (byte >= 0x80) { + all_ascii = false; + break; + } + has_ascii_upper |= byte >= 'A' && byte <= 'Z'; + } + if (all_ascii) { + if (!has_ascii_upper) { + return t; + } + _lower_term.resize(term.size()); + for (size_t i = 0; i < term.size(); ++i) { + const auto byte = static_cast(term[i]); + _lower_term[i] = + static_cast(byte >= 'A' && byte <= 'Z' ? byte + ('a' - 'A') : byte); + } + set_text(t, _lower_term); + return t; + } +#ifdef BE_TEST + lower_case_testing::note_unicode_path(); +#endif + if (!validate_utf8(term.data(), term.size())) { + throw Exception(ErrorCode::INVERTED_INDEX_ANALYZER_ERROR, + "Failed to lowercase token: invalid UTF-8"); } + _lower_term.resize(term.size()); UErrorCode status = U_ZERO_ERROR; - int32_t result_len = ucasemap_utf8ToLower(_ucsm.get(), _lower_term.data(), max_len, - term.data(), term.size(), &status); + int32_t result_len = ucasemap_utf8ToLower( + _ucsm.get(), _lower_term.data(), cast_set(_lower_term.size()), term.data(), + cast_set(term.size()), &status); + if (status == U_BUFFER_OVERFLOW_ERROR) { + _lower_term.resize(cast_set(result_len)); + status = U_ZERO_ERROR; + result_len = ucasemap_utf8ToLower(_ucsm.get(), _lower_term.data(), + cast_set(_lower_term.size()), term.data(), + cast_set(term.size()), &status); + } if (U_FAILURE(status)) { - LOG(WARNING) << "Failed to convert to lowercase. " - << "Term: '" << term << "', " - << "ICU Error: " << status << " - " << u_errorName(status) - << ", Buffer size: " << max_len << std::endl; - return nullptr; + throw Exception(ErrorCode::INVERTED_INDEX_ANALYZER_ERROR, + "Failed to convert token to lowercase. ICU Error: {} - {}", + static_cast(status), u_errorName(status)); } set_text(t, std::string_view(_lower_term.data(), result_len)); diff --git a/be/src/storage/index/inverted/token_filter/lower_case_filter_factory.h b/be/src/storage/index/inverted/token_filter/lower_case_filter_factory.h index 03467c5114a6b4..b24ad4b0822944 100644 --- a/be/src/storage/index/inverted/token_filter/lower_case_filter_factory.h +++ b/be/src/storage/index/inverted/token_filter/lower_case_filter_factory.h @@ -34,6 +34,10 @@ class LowerCaseFilterFactory : public TokenFilterFactory { token_filter->initialize(); return token_filter; } + + PositionCapability position_capability() const override { + return PositionCapability::kAlwaysUnitIncrement; + } }; } // namespace doris::segment_v2::inverted_index \ No newline at end of file diff --git a/be/src/storage/index/inverted/token_filter/token_filter_factory.h b/be/src/storage/index/inverted/token_filter/token_filter_factory.h index ebbd8836715e8d..5e1bd93236977a 100644 --- a/be/src/storage/index/inverted/token_filter/token_filter_factory.h +++ b/be/src/storage/index/inverted/token_filter/token_filter_factory.h @@ -28,6 +28,7 @@ class TokenFilterFactory : public AbstractAnalysisFactory { ~TokenFilterFactory() override = default; virtual TokenFilterPtr create(const TokenStreamPtr& in) = 0; + virtual PositionCapability position_capability() const { return PositionCapability::kUnknown; } }; using TokenFilterFactoryPtr = std::shared_ptr; diff --git a/be/src/storage/index/inverted/tokenizer/char/char_group_tokenizer_factory.h b/be/src/storage/index/inverted/tokenizer/char/char_group_tokenizer_factory.h index 030c0c2512f850..68273ad1366783 100644 --- a/be/src/storage/index/inverted/tokenizer/char/char_group_tokenizer_factory.h +++ b/be/src/storage/index/inverted/tokenizer/char/char_group_tokenizer_factory.h @@ -30,6 +30,10 @@ class CharGroupTokenizerFactory : public TokenizerFactory { TokenizerPtr create() override; + PositionCapability position_capability() const override { + return PositionCapability::kAlwaysUnitIncrement; + } + private: static UChar32 parse_escaped_char(const icu::UnicodeString& unicode_str); diff --git a/be/src/storage/index/inverted/tokenizer/char/char_tokenizer.cpp b/be/src/storage/index/inverted/tokenizer/char/char_tokenizer.cpp index d7b774ef0966a4..54ac1d72c9de82 100644 --- a/be/src/storage/index/inverted/tokenizer/char/char_tokenizer.cpp +++ b/be/src/storage/index/inverted/tokenizer/char/char_tokenizer.cpp @@ -17,10 +17,32 @@ #include "storage/index/inverted/tokenizer/char/char_tokenizer.h" +#ifdef BE_TEST +#include +#endif + #include "common/exception.h" namespace doris::segment_v2::inverted_index { +#ifdef BE_TEST +namespace { +std::atomic g_non_ascii_decode_count {0}; +} // namespace + +namespace char_tokenizer_testing { + +uint64_t non_ascii_decode_count() { + return g_non_ascii_decode_count.load(std::memory_order_relaxed); +} + +void reset_non_ascii_decode_count() { + g_non_ascii_decode_count.store(0, std::memory_order_relaxed); +} + +} // namespace char_tokenizer_testing +#endif + void CharTokenizer::initialize(int32_t max_token_len) { if (max_token_len > MAX_TOKEN_LENGTH_LIMIT || max_token_len <= 0) { throw Exception(ErrorCode::INVALID_ARGUMENT, @@ -29,6 +51,33 @@ void CharTokenizer::initialize(int32_t max_token_len) { " passed: " + std::to_string(max_token_len)); } _max_token_len = max_token_len; + for (size_t value = 0; value < _ascii_char_classes.size(); ++value) { + const auto c = static_cast(value); + _ascii_char_classes[value] = + is_cjk_char(c) + ? AsciiCharClass::kCjk + : (is_token_char(c) ? AsciiCharClass::kToken : AsciiCharClass::kDelimiter); + } +} + +CharTokenizer::AsciiCharClass CharTokenizer::read_next_char_class() { + const auto first_byte = static_cast(_char_buffer[_buffer_index]); + if (first_byte < _ascii_char_classes.size()) { + ++_buffer_index; + return _ascii_char_classes[first_byte]; + } +#ifdef BE_TEST + g_non_ascii_decode_count.fetch_add(1, std::memory_order_relaxed); +#endif + UChar32 c = U_UNASSIGNED; + U8_NEXT(_char_buffer, _buffer_index, _data_len, c); + if (c < 0) { + return AsciiCharClass::kInvalid; + } + if (is_cjk_char(c)) { + return AsciiCharClass::kCjk; + } + return is_token_char(c) ? AsciiCharClass::kToken : AsciiCharClass::kDelimiter; } Token* CharTokenizer::next(Token* token) { @@ -46,14 +95,13 @@ Token* CharTokenizer::next(Token* token) { break; } - UChar32 c = U_UNASSIGNED; const int32_t prev_i = _buffer_index; - U8_NEXT(_char_buffer, _buffer_index, _data_len, c); - if (c < 0) { + const AsciiCharClass char_class = read_next_char_class(); + if (char_class == AsciiCharClass::kInvalid) { continue; } - if (is_cjk_char(c)) { + if (char_class == AsciiCharClass::kCjk) { if (start == -1) { start = prev_i; end = _buffer_index - 1; @@ -61,7 +109,7 @@ Token* CharTokenizer::next(Token* token) { _buffer_index = prev_i; } break; - } else if (is_token_char(c)) { + } else if (char_class == AsciiCharClass::kToken) { if (start == -1) { start = prev_i; } diff --git a/be/src/storage/index/inverted/tokenizer/char/char_tokenizer.h b/be/src/storage/index/inverted/tokenizer/char/char_tokenizer.h index 66a9979a9fc97b..bae2f6c55d734f 100644 --- a/be/src/storage/index/inverted/tokenizer/char/char_tokenizer.h +++ b/be/src/storage/index/inverted/tokenizer/char/char_tokenizer.h @@ -17,6 +17,9 @@ #pragma once +#include +#include + #include "storage/index/inverted/tokenizer/tokenizer.h" namespace doris::segment_v2::inverted_index { @@ -36,8 +39,13 @@ class CharTokenizer : public DorisTokenizer { static constexpr int32_t DEFAULT_MAX_WORD_LEN = 255; private: + enum class AsciiCharClass : uint8_t { kDelimiter, kToken, kCjk, kInvalid }; + static constexpr int32_t MAX_TOKEN_LENGTH_LIMIT = 16383; + AsciiCharClass read_next_char_class(); + + std::array _ascii_char_classes {}; int32_t _max_token_len = 0; int32_t _buffer_index = 0; diff --git a/be/src/storage/index/inverted/tokenizer/empty/empty_tokenizer_factory.h b/be/src/storage/index/inverted/tokenizer/empty/empty_tokenizer_factory.h index 27229ba99f1184..b540dfa45a3f61 100644 --- a/be/src/storage/index/inverted/tokenizer/empty/empty_tokenizer_factory.h +++ b/be/src/storage/index/inverted/tokenizer/empty/empty_tokenizer_factory.h @@ -67,6 +67,10 @@ class EmptyTokenizerFactory : public TokenizerFactory { tokenizer->initialize(); return tokenizer; } + + PositionCapability position_capability() const override { + return PositionCapability::kAlwaysUnitIncrement; + } }; } // namespace doris::segment_v2::inverted_index \ No newline at end of file diff --git a/be/src/storage/index/inverted/tokenizer/ngram/edge_ngram_tokenizer_factory.h b/be/src/storage/index/inverted/tokenizer/ngram/edge_ngram_tokenizer_factory.h index 8d63b5cd230582..a0b88153a7a11f 100644 --- a/be/src/storage/index/inverted/tokenizer/ngram/edge_ngram_tokenizer_factory.h +++ b/be/src/storage/index/inverted/tokenizer/ngram/edge_ngram_tokenizer_factory.h @@ -56,6 +56,10 @@ class EdgeNGramTokenizerFactory : public TokenizerFactory { } } + PositionCapability position_capability() const override { + return PositionCapability::kAlwaysUnitIncrement; + } + private: int32_t _min_gram = 0; int32_t _max_gram = 0; diff --git a/be/src/storage/index/inverted/tokenizer/ngram/ngram_tokenizer_factory.h b/be/src/storage/index/inverted/tokenizer/ngram/ngram_tokenizer_factory.h index d064749d9d51a5..2ee428e32ff77f 100644 --- a/be/src/storage/index/inverted/tokenizer/ngram/ngram_tokenizer_factory.h +++ b/be/src/storage/index/inverted/tokenizer/ngram/ngram_tokenizer_factory.h @@ -50,6 +50,10 @@ class NGramTokenizerFactory : public TokenizerFactory { } } + PositionCapability position_capability() const override { + return PositionCapability::kAlwaysUnitIncrement; + } + static void initialize_matchers(); static CharMatcherPtr parse_token_chars(const Settings& settings); diff --git a/be/src/storage/index/inverted/tokenizer/tokenizer_factory.h b/be/src/storage/index/inverted/tokenizer/tokenizer_factory.h index b788990ecf09be..654726a9b976bb 100644 --- a/be/src/storage/index/inverted/tokenizer/tokenizer_factory.h +++ b/be/src/storage/index/inverted/tokenizer/tokenizer_factory.h @@ -28,6 +28,7 @@ class TokenizerFactory : public AbstractAnalysisFactory { ~TokenizerFactory() override = default; virtual TokenizerPtr create() = 0; + virtual PositionCapability position_capability() const { return PositionCapability::kUnknown; } }; using TokenizerFactoryPtr = std::shared_ptr; diff --git a/be/src/storage/index/snii/bkd/bkd_builder.cpp b/be/src/storage/index/snii/bkd/bkd_builder.cpp new file mode 100644 index 00000000000000..5fbf38897f1b86 --- /dev/null +++ b/be/src/storage/index/snii/bkd/bkd_builder.cpp @@ -0,0 +1,452 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/bkd/bkd_builder.h" + +#include + +#include +#include +#include +#include +#include +#include + +#include "common/cast_set.h" +#include "common/check.h" +#include "storage/index/snii/bkd/bkd_format.h" +#include "storage/index/snii/bkd/bkd_index_block.h" +#include "storage/index/snii/bkd/leaf_codec.h" +#include "storage/index/snii/bkd/point_merger.h" +#include "storage/index/snii/bkd/point_run.h" +#include "storage/index/snii/bkd/point_sorter.h" +#include "storage/index/snii/writer/temp_dir.h" +#include "storage/types.h" + +namespace doris::snii::bkd { + +namespace { + +// Points the record buffer allocates for on its first growth. Large enough that a +// small column does not walk the geometric ladder from one record, small enough that +// a column with a handful of points does not hold a page it will never fill. +constexpr size_t kInitialRecordBufferPoints = 1024; + +// The Phase 1 point stream: the builder's own resident run, already sorted in place, +// handed out in leaf-sized windows. It copies nothing -- every block is a view into +// the record buffer -- which is why sorting has to be in place (design 6.2): a +// second buffer would double the bound build_buffer_bytes is supposed to set. +// +// Phase 2's k-way merge over spilled runs implements this same interface, so the +// leaf-cutting loop in write_index() stays untouched. +class ResidentPointSource final : public PointSource { +public: + ResidentPointSource(Slice records, size_t record_size) + : records_(records), record_size_(record_size) { + DORIS_CHECK_GT(record_size, 0UL); + DORIS_CHECK_EQ(records.size() % record_size, 0UL); + } + + Status next_block(uint32_t max_points, Slice* records) override { + DORIS_CHECK(records != nullptr); + DORIS_CHECK_GT(max_points, 0U); + const size_t wanted = std::min(records_.size() - consumed_, + static_cast(max_points) * record_size_); + *records = Slice(records_.data() + consumed_, wanted); + consumed_ += wanted; + return Status::OK(); + } + +private: + const Slice records_; + const size_t record_size_; + size_t consumed_ = 0; +}; + +} // namespace + +BkdBuilder::BkdBuilder(const BkdBuilderOptions& options) + : options_(options), + record_size_(static_cast(options.bytes_per_dim) + kPointDocIdBytes), + max_points_(static_cast(options.build_buffer_bytes / record_size_)), + reservation_(options.reporter == nullptr ? writer::MemoryReporter::Reservation() + : options.reporter->make_reservation()) {} + +Status BkdBuilder::create(const BkdBuilderOptions& options, std::unique_ptr* out) { + DORIS_CHECK(out != nullptr); + // Design 6.1: everything the builder needs is settled BEFORE the object exists, + // so there is no constructed-but-invalid state for later code to defend against. + // These options come from Doris's own writer layer in this same process -- they + // are internal invariants, not untrusted bytes, hence DORIS_CHECK (design 8). + DORIS_CHECK_GT(options.bytes_per_dim, 0U); + // INV-2. The membership test for "is this field type indexable at all" belongs to + // encode_bkd_index_block, which owns the on-disk field_type vocabulary and asserts + // it on the header this builder hands over in finish(). + DORIS_CHECK_EQ(static_cast(options.bytes_per_dim), field_type_size(options.field_type)); + DORIS_CHECK_GT(options.points_per_leaf, 0U); + // A buffer that cannot hold one record would make every add() fail. + DORIS_CHECK_GE(options.build_buffer_bytes, + static_cast(options.bytes_per_dim) + kPointDocIdBytes); + *out = std::unique_ptr(new BkdBuilder(options)); + return Status::OK(); +} + +Status BkdBuilder::add(uint32_t doc_id, Slice sortable_value) { + DORIS_CHECK(!finished_); + // DORIS_CHECK rather than DCHECK even though this is the per-row path: + // bytes_per_dim bytes are copied out of this Slice below, so a short value would + // read past its end in a release build too. + DORIS_CHECK_EQ(sortable_value.size(), static_cast(options_.bytes_per_dim)); + + if (records_.size() / record_size_ == max_points_) { + // The ceiling is a spill trigger, not a refusal (design 6.2). The old + // implementation had no offline sort at all and simply grew until the + // process died; here the resident footprint stays flat and the excess + // goes to a run. + RETURN_IF_ERROR(spill_current_run()); + } + const size_t point_count = records_.size() / record_size_; + RETURN_IF_ERROR(reserve_points(point_count + 1)); + + // doc_count is counted HERE (design 6.1), never pushed in from outside. Doris + // appends in ascending row order and an array column repeats one row's doc id + // consecutively, so a doc id that differs from the previous one starts a new + // document. That ordering is what makes the running counter exact; it is a + // per-point property, hence DCHECK. + // + // "First point ever" is doc_count_ == 0, NOT an empty resident buffer: a + // spill empties that buffer mid-stream, and testing it here would restart the + // run of equal doc ids and count one document twice. + DCHECK(doc_count_ == 0 || doc_id >= last_doc_id_); + if (doc_count_ == 0 || doc_id != last_doc_id_) { + ++doc_count_; + } + last_doc_id_ = doc_id; + + records_.insert(records_.end(), sortable_value.data(), + sortable_value.data() + sortable_value.size()); + // BIG-endian doc id tail: the memcmp of the whole record is then exactly + // (value, doc_id) order, which is what point_sorter sorts by and what + // leaf_codec's "doc ids ascend inside a run" relies on (design 6.2). + for (uint32_t i = 0; i < kPointDocIdBytes; ++i) { + records_.push_back(static_cast(doc_id >> (8 * (kPointDocIdBytes - 1 - i)))); + } + return Status::OK(); +} + +Status BkdBuilder::finish(io::FileWriter* data_out, ByteSink* index_out, BkdStats* stats) { + DORIS_CHECK(data_out != nullptr); + DORIS_CHECK(index_out != nullptr); + DORIS_CHECK(stats != nullptr); + DORIS_CHECK(!finished_); + finished_ = true; + + Status status; + if (run_paths_.empty()) { + // FAST PATH: everything stayed resident. One in-place pass over the run. + // The whole record is the key, so this single sort establishes + // (value, doc_id) order without a separate tie-break (design 6.3). + point_sorter::sort(records_.data(), records_.size() / record_size_, + static_cast(record_size_)); + // Scoped so the source -- which is nothing but a view into records_ -- is + // gone before the buffer it views is. + ResidentPointSource source(Slice(records_), record_size_); + status = write_index(&source, data_out, index_out, stats); + release_records(); + return status; + } + + // MERGE PATH. The residual becomes one more run rather than a special case, so + // the merge sees a uniform set of inputs and the leaf-cutting loop below still + // cannot tell the two build modes apart (design 6.2). + status = spill_current_run(); + if (status.ok()) { + // The resident buffer is dead weight from here on: the merge's own + // footprint is (runs x per-run window + one leaf block), and holding the + // old buffer on top of it would breach the very bound this path exists to + // keep. + release_records(); + + // Fan-in the resident allowance can actually window: every cursor gets at + // least kMinMergeCursorRecords. Above this many runs a single merge would + // fall back to one record per cursor and hold run_count records -- + // total_points / max_points_, a footprint that GROWS with the input -- + // so the runs are folded in groups first. The leaf block sits on top of + // the cursor windows because a leaf has to be materialized contiguously + // no matter how the points arrived. + const size_t fan_in = std::max(2, max_points_ / kMinMergeCursorRecords); + status = fold_runs_to_fan_in(fan_in, stats); + if (status.ok()) { + const size_t per_run = records_per_cursor(run_paths_.size()); + ++stats->merge_passes; + std::unique_ptr source; + status = MergingPointSource::create(run_paths_, static_cast(record_size_), + options_.points_per_leaf, + static_cast(per_run), &source); + if (status.ok()) { + // Measured from the cursors that were actually opened, not + // recomputed from what they were asked for. + stats->peak_merge_buffer_bytes = std::max( + stats->peak_merge_buffer_bytes, source->resident_buffer_bytes()); + status = write_index(source.get(), data_out, index_out, stats); + } + } + } + release_records(); + remove_runs(); + if (status.ok()) { + stats->built_with_spill = true; + } + return status; +} + +Status BkdBuilder::spill_current_run() { + point_sorter::sort(records_.data(), records_.size() / record_size_, + static_cast(record_size_)); + + // pid plus a process-wide counter: two builders running concurrently in one BE + // must not collide, and a stale file from a previous process must not be + // mistaken for one of ours. + static std::atomic sequence {0}; + const std::string path = writer::resolve_temp_dir() + "/snii_bkd_" + + std::to_string(::getpid()) + "_" + + std::to_string(sequence.fetch_add(1)) + ".run"; + + PointRunWriter run; + RETURN_IF_ERROR(run.open(path)); + // Recorded BEFORE the first write: a run that fails halfway still has to be + // unlinked, and remove_runs() can only clean up what it knows about. + run_paths_.push_back(path); + if (!records_.empty()) { + RETURN_IF_ERROR(run.append(Slice(records_))); + } + RETURN_IF_ERROR(run.close()); + + // Keep the capacity: the ceiling is meant to hold the footprint flat, not to + // make it oscillate between empty and full. + records_.clear(); + return Status::OK(); +} + +size_t BkdBuilder::records_per_cursor(size_t run_count) const { + DORIS_CHECK_GT(run_count, 0U); + // run_count x this <= max_points_ by construction, except at the degenerate + // floor of one record per cursor -- which is reached only when max_points_ is + // smaller than the fan-in itself. The overshoot there is a CONSTANT couple of + // records, not the run_count-proportional growth the fold exists to remove. + return std::max(1, max_points_ / run_count); +} + +Status BkdBuilder::merge_group_into_run(const std::vector& group, + uint32_t cursor_records, BkdStats* stats, + std::string* out) { + DORIS_CHECK_GE(group.size(), 2U); + static std::atomic sequence {0}; + const std::string path = writer::resolve_temp_dir() + "/snii_bkd_fold_" + + std::to_string(::getpid()) + "_" + + std::to_string(sequence.fetch_add(1)) + ".run"; + // Handed back BEFORE a byte is written: a fold that fails halfway still has + // to leave a path its caller can register for removal. + *out = path; + + std::unique_ptr source; + RETURN_IF_ERROR(MergingPointSource::create(group, static_cast(record_size_), + options_.points_per_leaf, cursor_records, &source)); + // Measured from the cursors this fold actually opened. + stats->peak_merge_buffer_bytes = + std::max(stats->peak_merge_buffer_bytes, source->resident_buffer_bytes()); + PointRunWriter run; + RETURN_IF_ERROR(run.open(path)); + while (true) { + Slice records; + RETURN_IF_ERROR(source->next_block(options_.points_per_leaf, &records)); + if (records.empty()) { + break; + } + RETURN_IF_ERROR(run.append(records)); + } + return run.close(); +} + +Status BkdBuilder::fold_runs_to_fan_in(size_t fan_in, BkdStats* stats) { + DORIS_CHECK_GE(fan_in, 2U); + while (run_paths_.size() > fan_in) { + const size_t per_run = records_per_cursor(fan_in); + std::vector folded; + Status status; + size_t begin = 0; + for (; begin < run_paths_.size(); begin += fan_in) { + const size_t end = std::min(begin + fan_in, run_paths_.size()); + if (end - begin == 1) { + // An odd tail carries forward untouched; rewriting it would cost + // a full copy to achieve nothing. + folded.push_back(run_paths_[begin]); + continue; + } + const std::vector group(run_paths_.begin() + cast_set(begin), + run_paths_.begin() + cast_set(end)); + std::string merged; + status = merge_group_into_run(group, static_cast(per_run), stats, &merged); + if (!merged.empty()) { + folded.push_back(merged); + } + if (!status.ok()) { + break; + } + // Unlinked only once its replacement is complete, so disk holds one + // extra copy of a GROUP at a time -- never one extra copy per pass. + for (const std::string& path : group) { + ::unlink(path.c_str()); + } + } + if (!status.ok()) { + // Everything from `begin` on is still on disk and unmerged. It must + // stay in run_paths_ or remove_runs() would leak it. + folded.insert(folded.end(), run_paths_.begin() + cast_set(begin), + run_paths_.end()); + run_paths_ = std::move(folded); + return status; + } + run_paths_ = std::move(folded); + ++stats->merge_passes; + } + return Status::OK(); +} + +void BkdBuilder::remove_runs() { + for (const std::string& path : run_paths_) { + ::unlink(path.c_str()); + } + run_paths_.clear(); +} + +BkdBuilder::~BkdBuilder() { + // An abandoned build (an error between add() and finish(), or a caller that + // simply drops the builder) must not leave runs in the temp dir. + remove_runs(); +} + +Status BkdBuilder::reserve_points(size_t point_count) { + const size_t needed = point_count * record_size_; + if (needed <= records_.capacity()) { + return Status::OK(); + } + // Geometric growth, then clamped to the configured ceiling: the buffer must not + // overshoot build_buffer_bytes even transiently, or the bound design 6.2 promises + // would only hold for the logical size and not for the RSS. + size_t target = std::max(needed, records_.capacity() * 2); + target = std::max(target, kInitialRecordBufferPoints * record_size_); + target = std::min(target, max_points_ * record_size_); + // add() rejects a point beyond max_points_ before calling here, so the clamp + // above can never land below what this point needs. + DCHECK_GE(target, needed); + + if (options_.reporter != nullptr) { + // Charges the new buffer WHILE the old one is still charged, which is exactly + // the transient double residency of a vector growth. Only after the physical + // move succeeds does the old charge go away. + writer::MemoryReporter::Reservation replacement; + RETURN_IF_ERROR(reservation_.prepare_replacement(target, &replacement)); + records_.reserve(target); + reservation_ = std::move(replacement); + return Status::OK(); + } + records_.reserve(target); + return Status::OK(); +} + +Status BkdBuilder::write_index(PointSource* source, io::FileWriter* data_out, ByteSink* index_out, + BkdStats* stats) { + // bytes_written() is the offset truth (io/file_writer.h), so the leaf directory is + // anchored to wherever this build's bkd_data begins inside the container. + const uint64_t data_start = data_out->bytes_written(); + const size_t index_start = index_out->size(); + const size_t bytes_per_dim = options_.bytes_per_dim; + + std::vector leaves; + std::vector min_value; + std::vector max_value; + std::vector split_values; + // Reused across leaves: one buffer for the whole build instead of one per leaf. + ByteSink leaf_block; + uint64_t leaf_offset = 0; + uint64_t point_count = 0; + + // Design 6.4: cut every points_per_leaf points, let the last leaf keep the + // remainder, and do NOT round the leaf count up to a power of two -- an ordered + // split array has no complete-binary-tree requirement, so the configured capacity + // is the real capacity instead of an upper bound that repeated halving dilutes. + while (true) { + Slice block; + RETURN_IF_ERROR(source->next_block(options_.points_per_leaf, &block)); + if (block.empty()) { + break; + } + DCHECK_EQ(block.size() % record_size_, 0UL); + const uint32_t count = static_cast(block.size() / record_size_); + const uint8_t* first_value = block.data(); + const uint8_t* last_value = block.data() + (count - 1) * record_size_; + + if (leaves.empty()) { + min_value.assign(first_value, first_value + bytes_per_dim); + } else { + // The boundary between leaf i and leaf i + 1 is leaf i + 1's FIRST value, + // i.e. leaf i + 1 covers [split(i), split(i + 1)). + split_values.insert(split_values.end(), first_value, first_value + bytes_per_dim); + } + // The stream is ordered, so whichever leaf turns out to be last leaves the + // global maximum behind. + max_value.assign(last_value, last_value + bytes_per_dim); + + leaf_block.clear(); + encode_leaf_block(block, options_.bytes_per_dim, &leaf_block); + RETURN_IF_ERROR(data_out->append(leaf_block.view())); + leaves.push_back(LeafRef {.offset = leaf_offset, .count = count}); + leaf_offset += leaf_block.size(); + point_count += count; + } + DORIS_CHECK_LE(leaves.size(), static_cast(std::numeric_limits::max())); + + BkdIndexHeader header; + header.format_version = kFormatVersion; + // Phase 1 finishes entirely in RAM, so index_flags::kBuiltWithSpill stays clear. + header.flags = 0; + header.bytes_per_dim = options_.bytes_per_dim; + header.field_type = options_.field_type; + header.point_count = point_count; + header.doc_count = doc_count_; + // 0 leaves is the empty index (design 5.3): header only, zero-length bkd_data. + header.leaf_count = static_cast(leaves.size()); + header.points_per_leaf = options_.points_per_leaf; + encode_bkd_index_block(header, Slice(min_value), Slice(max_value), Slice(split_values), leaves, + index_out); + + stats->point_count = point_count; + stats->doc_count = doc_count_; + stats->leaf_count = header.leaf_count; + stats->index_bytes = index_out->size() - index_start; + stats->data_bytes = data_out->bytes_written() - data_start; + stats->built_with_spill = false; + return Status::OK(); +} + +void BkdBuilder::release_records() { + std::vector().swap(records_); + reservation_.reset(); +} + +} // namespace doris::snii::bkd diff --git a/be/src/storage/index/snii/bkd/bkd_builder.h b/be/src/storage/index/snii/bkd/bkd_builder.h new file mode 100644 index 00000000000000..2970e1bcbb0241 --- /dev/null +++ b/be/src/storage/index/snii/bkd/bkd_builder.h @@ -0,0 +1,163 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/bkd/bkd_types.h" +#include "storage/index/snii/bkd/point_source.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/encoding/byte_sink.h" +#include "storage/index/snii/io/file_writer.h" +#include "storage/index/snii/writer/memory_reporter.h" + +// Write-side orchestration for the SNII-native BKD index (design 6): buffer the +// points, order them, cut leaves, emit bkd_data and bkd_index. +// +// This header pulls in NOTHING from the read side (design 4). The old writer TU +// transitively included the entire reader through docids_writer.h, because that +// type declared both directions at once; here the two directions meet only in +// bkd_format.h's constants. +namespace doris::snii::bkd { + +// One-shot builder: create -> add* -> finish. +// +// PHASE 1 -- FAST PATH ONLY. Every point stays resident, is sorted once in +// finish(), and is cut into leaves directly. Crossing +// BkdBuilderOptions::build_buffer_bytes makes add() return a MEM_LIMIT_EXCEEDED +// Status. That refusal IS the improvement over the old implementation, which had +// no offline sort at all and silently held every point until finish(), i.e. grew +// unbounded until the process died. Phase 2 replaces the refusal with a spill by +// adding a PointSource implementation; the leaf-cutting loop below does not change. +class BkdBuilder { +public: + // The ONLY way to obtain a builder. Options are checked BEFORE the object + // exists, so there is no such thing as a constructed-but-invalid builder -- the + // old bkd_writer instead threw from its constructor and left docs_seen_ + // uninitialized for the caller to fill in from outside. + static Status create(const BkdBuilderOptions& options, std::unique_ptr* out); + + ~BkdBuilder(); + + BkdBuilder(const BkdBuilder&) = delete; + BkdBuilder& operator=(const BkdBuilder&) = delete; + + // Appends one point. `sortable_value` is exactly bytes_per_dim unsigned + // big-endian sortable bytes from KeyCoder::full_encode_ascending (INV-1/INV-2); + // a wrong length is a caller bug and trips DORIS_CHECK, not a Status. + // + // NULL rows do not call this at all -- they live in the SNII-native null bitmap + // section (design 9 / D9). + // + // doc_count is counted HERE, from doc_id changing between consecutive calls + // (Doris calls in ascending row order; an array column calls several times for + // one row). It is an implementation detail, not the undocumented "push + // docs_seen_ in before finish()" contract the old writer relied on. + // + // Returns MEM_LIMIT_EXCEEDED once the resident buffer is full (see the class + // comment) or once the shared MemoryReporter cap refuses the growth. + Status add(uint32_t doc_id, Slice sortable_value); + + // Orders the points, cuts leaves, APPENDS the leaf blocks to `data_out` and the + // framed bkd_index section to `index_out`, and reports what was written. + // + // Leaf offsets are relative to the START of this build's bkd_data, i.e. to + // data_out->bytes_written() on entry, so the writer may already carry other + // sub-files of the same container. `data_out` is NOT finalized here: it belongs + // to the container writer, which keeps appending after this returns. + // + // Consumes the builder: it releases its point buffer here and calling add() or + // finish() again is a caller bug (DORIS_CHECK). No Directory, no IndexOutput -- + // a FileWriter and a ByteSink are the whole output surface (D2). + Status finish(io::FileWriter* data_out, ByteSink* index_out, BkdStats* stats); + +private: + explicit BkdBuilder(const BkdBuilderOptions& options); + + // Grows the record buffer to hold `point_count` points, pre-charging the + // MemoryReporter before the allocation happens. + Status reserve_points(size_t point_count); + + // Sorts the resident buffer and writes it out as one more run, then empties + // the buffer while KEEPING its capacity -- the point of the ceiling is that + // the resident footprint stays flat, not that it oscillates. + Status spill_current_run(); + + // Unlinks every run written so far. Called on every exit from finish() and + // again from the destructor, so an abandoned build leaves nothing behind. + void remove_runs(); + + // Runs a merge that a single pass can window, given the resident ceiling. + // Above it, folds run_paths_ in groups of that many until the remainder + // fits, replacing the group with its merged output and unlinking the inputs + // as it goes -- so disk holds one extra copy of the data at most, never one + // per pass. Records what it did in *stats. + Status fold_runs_to_fan_in(size_t fan_in, BkdStats* stats); + + // Merges `group` into one new run file and appends its path to *out. The + // inputs are NOT unlinked here; the caller owns that, because a failure + // partway through still has to leave every path it knows about removable. + Status merge_group_into_run(const std::vector& group, uint32_t records_per_cursor, + BkdStats* stats, std::string* out); + + // Cursor-window bytes one merge over `run_count` runs would hold, and the + // per-cursor record window that produces it. Single-sourced so the bound + // reported in BkdStats cannot drift from the one actually configured. + size_t records_per_cursor(size_t run_count) const; + + // The leaf-cutting loop of design 6.4, shared by every build mode: it sees only + // an ordered PointSource and never learns whether the points came from RAM or + // from a merge of spilled runs. + Status write_index(PointSource* source, io::FileWriter* data_out, ByteSink* index_out, + BkdStats* stats); + + // Drops the point buffer and its memory charge. Called once the points have been + // consumed, so a builder awaiting destruction holds nothing. + void release_records(); + + const BkdBuilderOptions options_; + // bytes_per_dim + kPointDocIdBytes: the fixed record width whose whole-record + // memcmp is (value, doc_id) order (design 6.2). + const size_t record_size_; + // build_buffer_bytes expressed in whole records -- the resident ceiling add() + // enforces. Rounded DOWN, so the buffer never exceeds the configured bound. + const size_t max_points_; + + // point_count * record_size_ bytes of [value][doc_id big-endian] records, in + // insertion order until finish() sorts them in place. This IS the point count: + // no second counter can drift away from it. + std::vector records_; + // Charge held against options_.reporter for records_.capacity(). Default (owner + // null) when no reporter was supplied, which is legal off-Doris. + writer::MemoryReporter::Reservation reservation_; + + // Paths of the runs spilled so far, in spill order. Empty means the build + // never crossed the ceiling and finish() takes the resident fast path. + std::vector run_paths_; + + uint32_t doc_count_ = 0; + uint32_t last_doc_id_ = 0; + bool finished_ = false; +}; + +} // namespace doris::snii::bkd diff --git a/be/src/storage/index/snii/bkd/bkd_format.h b/be/src/storage/index/snii/bkd/bkd_format.h new file mode 100644 index 00000000000000..63f6861c9ed11a --- /dev/null +++ b/be/src/storage/index/snii/bkd/bkd_format.h @@ -0,0 +1,168 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +// On-disk contract constants for the SNII-native one-dimensional BKD numeric +// index. This header is CONSTANTS ONLY -- no logic, no includes beyond the +// standard library -- so both the write side and the read side can depend on it +// without pulling each other in (in the third-party implementation this replaces, +// the writer TU transitively included the whole read side through its shared +// doc-id codec header). +// +// Once published these values are format semantics: changing any of them +// requires bumping kFormatVersion and defining a compatibility policy. The +// format deliberately does NOT interoperate with the third-party BKD bytes; +// V1/V2/V3 tables keep using the old implementation and SNII tables use this one. +// +// Structural integers use the SNII encoding vocabulary (little-endian +// put_fixed*, LEB128 put_varint*, see snii/encoding/byte_sink.h). Point VALUES +// are exempt: they stay unsigned big-endian sortable bytes as produced by +// KeyCoder::full_encode_ascending, because every BKD comparison is a byte-wise +// unsigned compare from offset 0 (INV-1). The two are fully decoupled -- values +// only ever travel through put_bytes / memcpy, never through an integer codec. +namespace doris::snii::bkd { + +// ---- bkd_index header magic / version ---- +// The big-endian reading of the ASCII bytes "BKD1" ('B'=0x42 'K'=0x4B 'D'=0x44 +// '1'=0x31). Written with ByteSink::put_fixed32, i.e. little-endian on disk; +// what is pinned is this numeric value, which is all a reader compares. (Note +// this differs from format::kContainerMagic, whose constant is spelled as the +// little-endian reading of "SNII" -- the two are independent magics and neither +// convention constrains the other.) +inline constexpr uint32_t kBkdIndexMagic = 0x424B4431U; + +// Version written by this binary. +inline constexpr uint32_t kFormatVersion = 1; +// Highest version this binary can read. A file above it is a CAPABILITY +// boundary, not corruption: readers must return INVERTED_INDEX_NOT_SUPPORTED so +// the caller falls back to "index unavailable" rather than reporting a damaged +// segment. +inline constexpr uint32_t kSupportedVersion = 1; + +// ---- SectionFramer type byte for the whole bkd_index payload ---- +// The framer type byte is one flat namespace across the container, and there is +// no shared SectionType enum for blob logical indexes, so -- exactly as +// format::kNullBitmapSectionType (0x20) does -- this is a documented literal +// picked outside the ranges already taken by the inverted-index sections +// (format::SectionType, currently 1..14) and the null-bitmap POD (0x20). +// Framing the payload is what gives bkd_index its checksum; no section here +// hand-rolls a crc. +inline constexpr uint8_t kBkdIndexSectionType = 0x30; + +// ---- Leaf block value encoding (bkd_data, one byte per leaf) ---- +// A closed, disjoint, exhaustive 3-value enum, replacing the old +// `-1 / -2 / sorted_dim` encoding in which -1 conflated "all values equal" with +// "the common prefix covers the whole value", and sorted_dim burned a byte to +// carry a value that is always 0 in one dimension. +enum class LeafValueMode : uint8_t { + // Every point in the leaf has the same value; the common prefix IS the + // value and no suffix data follows. + kAllEqual = 0, + // Run-length: run_count varint32, then run_count pairs of + // { suffix bytes[S], run_len varint32 }. + kRle = 1, + // point_count fixed-width suffixes of S = bytes_per_dim - common_prefix_len + // bytes each. + kRaw = 2, +}; +// Largest legal value_mode byte. A leaf decoder reads an untrusted byte from +// disk and must reject anything above this as INVERTED_INDEX_FILE_CORRUPTED. +inline constexpr LeafValueMode kMaxLeafValueMode = LeafValueMode::kRaw; + +// ---- bkd_index header `flags` bits ---- +namespace index_flags { +// The build spilled at least one run to disk instead of finishing entirely in +// the resident buffer. DIAGNOSTIC ONLY: the produced bytes are identical either +// way, so no read path may branch on it. +inline constexpr uint32_t kBuiltWithSpill = 1U << 0; +// bit1-31 reserved. +} // namespace index_flags + +// ---- Blob sub-file names in the SNII named-file table ---- +// Two files, not the old implementation's three: bkd_meta is folded into the +// bkd_index header, where it gains a magic, a version and a framer checksum. +// bkd_index is HOT (read in full at open and kept resident); bkd_data is COLD +// (one positioned read per touched leaf). +inline constexpr std::string_view kBkdIndexFileName = "bkd_index"; +inline constexpr std::string_view kBkdDataFileName = "bkd_data"; + +// ---- Build-time parameters (NOT format semantics) ---- +// A reader derives everything it needs from the header, so these may be retuned +// against real measurements without a version bump. + +// Width of the doc_id tail of a build-time point record +// ([value: bytes_per_dim][doc_id: 4 bytes BIG-endian]). Big-endian is +// deliberate: a memcmp of the whole record then equals lexicographic +// (value, doc_id) order, so the sorter and the merger never split the record +// into fields and the sort key is (value, doc_id) by construction -- which is +// also what makes docids ascending inside a single-value run. +inline constexpr uint32_t kPointDocIdBytes = 4; + +// Points a leaf holds unless the caller overrides it. +// +// 128, not the 1024 inherited from the third-party implementation this +// replaces. Measured at 2M BIGINT points (the BKD comparison benchmark under +// be/test/.../bench, 2026-08-02): 1024 was never calibrated, and is the worst +// reasonable choice for selective queries -- a boundary leaf is scanned in full +// however few values match, so narrow-range cost rises monotonically with leaf +// size (12x from 128 to 4096). It also costs index size: 128 emits 8.73 MB +// against 1024's 9.34 MB, cutting the size regression against the baseline +// implementation from +15.5% to +7.9%. +// +// The size curve is NOT monotonic -- 256 is the largest of the six values +// swept -- because below roughly 256 the per-leaf prefix compression gains +// more than the larger leaf directory and split array cost. +// +// The trade is wide ranges: 136 ms at 128 against 111 ms at 1024, +23%. That +// is the deliberate call -- selective predicates and index size are the common +// case, and a range wide enough to feel the difference is already reading a +// million rows. +inline constexpr uint32_t kDefaultPointsPerLeaf = 128; + +// Hard ceiling on the recorded points_per_leaf, enforced at open. This is not a +// tuning knob: points_per_leaf is the ONLY quantity that bounds a leaf's count, +// and a leaf's count is what sizes the doc id vector during leaf decode. Left +// unbounded, a self-consistent but hostile bkd_index (inflate point_count and +// the leaf counts together and the sum identity still holds) would drive an +// arbitrarily large allocation from a ~25-byte leaf block, and the resulting +// bad_alloc would escape a module that has no catch anywhere -- turning a +// recoverable downgrade into a node crash, which is exactly what design 8 +// exists to prevent. 1 Mi points caps one leaf's doc id vector at 4 MiB. +inline constexpr uint32_t kMaxPointsPerLeaf = 1U << 20; + +// Resident point-buffer ceiling before a run is sorted and spilled. Bounds +// build-time RSS independently of the segment's row count. +inline constexpr uint64_t kDefaultBuildBufferBytes = 256ULL << 20; + +// Smallest read window a merge cursor is given, in records. It sets the merge's +// FAN-IN: a pass folds at most build_buffer_bytes / (record_size x this) runs at +// once, and a build with more runs than that folds them in groups over several +// passes rather than opening every run at once. +// +// Without a fan-in cap the "one record per cursor" floor makes a single merge +// hold run_count records, i.e. total_points / max_points -- a footprint that +// GROWS with the input, which is exactly what the build_buffer_bytes ceiling +// exists to prevent. 32 records is small enough that a real 256 MiB buffer +// still folds ~700k runs in one pass (so multi-pass never triggers in +// practice), and large enough that a cursor read is not one record per syscall. +inline constexpr uint32_t kMinMergeCursorRecords = 32; + +} // namespace doris::snii::bkd diff --git a/be/src/storage/index/snii/bkd/bkd_index_block.cpp b/be/src/storage/index/snii/bkd/bkd_index_block.cpp new file mode 100644 index 00000000000000..40735c8b36358f --- /dev/null +++ b/be/src/storage/index/snii/bkd/bkd_index_block.cpp @@ -0,0 +1,357 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/bkd/bkd_index_block.h" + +#include +#include +#include + +#include "storage/index/snii/bkd/bkd_format.h" +#include "storage/index/snii/encoding/byte_source.h" +#include "storage/index/snii/encoding/section_framer.h" +#include "storage/types.h" + +namespace doris::snii::bkd { + +namespace { + +// The field types a native BKD index can be built for: exactly the non-string +// instantiations of InvertedIndexColumnWriter. A string type is excluded because +// it has no fixed-width sortable-bytes representation (INV-2). +// +// field_type is read from disk, so an unrecognised value must be rejected HERE +// and never cast into FieldType and passed to field_type_size(), which +// LOG(FATAL)s on anything outside its own switch -- that would turn a +// recoverable index downgrade into a node crash (design 8). +constexpr FieldType kIndexableFieldTypes[] = { + FieldType::OLAP_FIELD_TYPE_BOOL, FieldType::OLAP_FIELD_TYPE_TINYINT, + FieldType::OLAP_FIELD_TYPE_SMALLINT, FieldType::OLAP_FIELD_TYPE_INT, + FieldType::OLAP_FIELD_TYPE_UNSIGNED_INT, FieldType::OLAP_FIELD_TYPE_BIGINT, + FieldType::OLAP_FIELD_TYPE_LARGEINT, FieldType::OLAP_FIELD_TYPE_FLOAT, + FieldType::OLAP_FIELD_TYPE_DOUBLE, FieldType::OLAP_FIELD_TYPE_DECIMAL, + FieldType::OLAP_FIELD_TYPE_DECIMAL32, FieldType::OLAP_FIELD_TYPE_DECIMAL64, + FieldType::OLAP_FIELD_TYPE_DECIMAL128I, FieldType::OLAP_FIELD_TYPE_DECIMAL256, + FieldType::OLAP_FIELD_TYPE_DATE, FieldType::OLAP_FIELD_TYPE_DATETIME, + FieldType::OLAP_FIELD_TYPE_DATEV2, FieldType::OLAP_FIELD_TYPE_DATETIMEV2, + FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ, FieldType::OLAP_FIELD_TYPE_IPV4, + FieldType::OLAP_FIELD_TYPE_IPV6, +}; + +// Resolves a raw on-disk field_type code and the bytes_per_dim it implies. +// Compares against the enumerators as integers rather than casting the untrusted +// code into the enum first. +bool resolve_field_type(uint32_t raw, FieldType* type, uint32_t* bytes_per_dim) { + const auto* match = std::ranges::find(kIndexableFieldTypes, raw, [](FieldType candidate) { + return static_cast(candidate); + }); + if (match == std::ranges::end(kIndexableFieldTypes)) { + return false; + } + *type = *match; + *bytes_per_dim = static_cast(field_type_size(*match)); + return true; +} + +// Every rejection of untrusted bytes funnels through here. Disk data is NOT an +// invariant, so none of these may be a DORIS_CHECK: the caller downgrades to a +// scan, it does not abort the process (design 8). +Status corrupted(std::string_view what) { + return Status::Error("bkd_index: {}", what); +} + +// Upper bound on the encoded size of one leaf-directory row: a varint64 offset +// delta plus a varint32 count. Used only to size the write buffer. +constexpr size_t kMaxLeafDirectoryRowBytes = 10 + 5; +// Upper bound on the encoded header: fixed32 magic plus eight varints. +constexpr size_t kMaxHeaderBytes = 4 + 8 * 10; + +// Decodes the fixed header and establishes its self-consistency (design 5.1). +// Everything downstream -- array strides, allocation sizes, the KeyCoder the +// query side resolves -- is derived from these fields, so they are checked before +// a single array byte is touched. +Status decode_header(ByteSource* src, BkdIndexHeader* header) { + uint32_t magic = 0; + RETURN_IF_ERROR(src->get_fixed32(&magic)); + if (magic != kBkdIndexMagic) { + return corrupted("bad magic"); + } + + // Read before anything else is interpreted: a future layout may differ field + // by field, so the capability answer must not depend on the rest parsing. + uint32_t format_version = 0; + RETURN_IF_ERROR(src->get_varint32(&format_version)); + if (format_version > kSupportedVersion) { + // A capability boundary, NOT damage: the caller reports "index + // unavailable" and falls back to a scan (design 3). + return Status::Error( + "bkd_index: format_version {} is above the supported {}", format_version, + kSupportedVersion); + } + if (format_version == 0) { + // No binary ever wrote version 0, so this is damage rather than a format + // from the future. + return corrupted("format_version is zero"); + } + header->format_version = format_version; + + RETURN_IF_ERROR(src->get_varint32(&header->flags)); + RETURN_IF_ERROR(src->get_varint32(&header->bytes_per_dim)); + uint32_t raw_field_type = 0; + RETURN_IF_ERROR(src->get_varint32(&raw_field_type)); + RETURN_IF_ERROR(src->get_varint64(&header->point_count)); + RETURN_IF_ERROR(src->get_varint32(&header->doc_count)); + RETURN_IF_ERROR(src->get_varint32(&header->leaf_count)); + RETURN_IF_ERROR(src->get_varint32(&header->points_per_leaf)); + + uint32_t expected_bytes_per_dim = 0; + if (!resolve_field_type(raw_field_type, &header->field_type, &expected_bytes_per_dim)) { + return corrupted("field_type is not an indexable numeric type"); + } + // INV-2: the fixed width is what makes the split array binary-searchable and + // the build-time record memcmp-comparable. A width that disagrees with the + // recorded type would read every array at the wrong stride. + if (header->bytes_per_dim != expected_bytes_per_dim) { + return corrupted("bytes_per_dim disagrees with field_type"); + } + // points_per_leaf is the only quantity that can bound a leaf's count, and a + // leaf's count is what sizes the doc id vector in leaf_codec. It must itself + // be bounded before anything downstream leans on it. The builder always + // records a non-zero value (bkd_builder.cpp DORIS_CHECK_GT in create, and + // finish writes it unconditionally -- including for the empty index), so + // rejecting zero cannot reject a legitimate file. + if (header->points_per_leaf == 0 || header->points_per_leaf > kMaxPointsPerLeaf) { + return corrupted("points_per_leaf is outside the supported range"); + } + return Status::OK(); +} + +// Non-decreasing rather than strictly increasing: a value spanning several leaves +// makes consecutive leaves start at the same value. Comparison is unsigned +// byte-wise from offset 0 (INV-1), i.e. plain memcmp over the sortable bytes. +// An unordered array would silently route the binary search to the wrong leaf -- +// wrong results with no error -- so it must never survive open. +Status validate_split_order(Slice splits, size_t bytes_per_dim, uint32_t leaf_count) { + for (uint32_t i = 1; i + 1 < leaf_count; ++i) { + const uint8_t* previous = splits.data() + (i - 1) * bytes_per_dim; + const uint8_t* current = splits.data() + i * bytes_per_dim; + if (std::memcmp(previous, current, bytes_per_dim) > 0) { + return corrupted("split_values are not in ascending order"); + } + } + return Status::OK(); +} + +// Leaf directory: delta-varint64 offsets, then varint32 counts (design 5.1). +// Both arrays are validated against the header here so the query path can index +// them unchecked. +Status decode_leaf_directory(ByteSource* src, const BkdIndexHeader& header, uint64_t data_length, + std::vector* leaves) { + std::vector decoded(header.leaf_count); + + uint64_t offset = 0; + for (uint32_t i = 0; i < header.leaf_count; ++i) { + uint64_t delta = 0; + RETURN_IF_ERROR(src->get_varint64(&delta)); + // The first delta is the absolute offset of leaf 0; from the second leaf + // on a zero delta would mean two leaves sharing a start. + if (i != 0 && delta == 0) { + return corrupted("leaf offsets are not strictly increasing"); + } + if (delta > std::numeric_limits::max() - offset) { + return corrupted("leaf offsets overflow"); + } + offset += delta; + decoded[i].offset = offset; + } + // `offset` is now the last leaf's offset. Bounding it against the companion + // sub-file here is what lets a leaf read skip re-validating its offset. + if (offset > data_length) { + return corrupted("last leaf offset is beyond the bkd_data length"); + } + + uint64_t total_points = 0; + for (uint32_t i = 0; i < header.leaf_count; ++i) { + uint32_t count = 0; + RETURN_IF_ERROR(src->get_varint32(&count)); + // The sum identity below is not enough on its own: point_count and the + // counts can be inflated TOGETHER and still agree, leaving the leaf + // decode allocation sized by a number that came straight off disk. This + // is the invariant bkd_types.h documents on LeafRef::count; enforcing it + // here is what makes it true rather than aspirational. + if (count > header.points_per_leaf) { + return corrupted("a leaf count exceeds points_per_leaf"); + } + decoded[i].count = count; + // leaf_count and each count are uint32, so the sum cannot overflow 64 bits. + total_points += count; + } + if (total_points != header.point_count) { + return corrupted("leaf counts do not sum to point_count"); + } + *leaves = std::move(decoded); + return Status::OK(); +} + +} // namespace + +void encode_bkd_index_block(const BkdIndexHeader& header, Slice min_value, Slice max_value, + Slice split_values, std::span leaves, ByteSink* sink) { + DORIS_CHECK(sink != nullptr); + DORIS_CHECK_EQ(header.format_version, kFormatVersion); + DORIS_CHECK_EQ(leaves.size(), static_cast(header.leaf_count)); + + FieldType field_type {}; + uint32_t expected_bytes_per_dim = 0; + DORIS_CHECK(resolve_field_type(static_cast(header.field_type), &field_type, + &expected_bytes_per_dim)); + DORIS_CHECK_EQ(header.bytes_per_dim, expected_bytes_per_dim); + + ByteSink payload; + payload.reserve(kMaxHeaderBytes + min_value.size() + max_value.size() + split_values.size() + + leaves.size() * kMaxLeafDirectoryRowBytes); + payload.put_fixed32(kBkdIndexMagic); + payload.put_varint32(header.format_version); + payload.put_varint32(header.flags); + payload.put_varint32(header.bytes_per_dim); + payload.put_varint32(static_cast(header.field_type)); + payload.put_varint64(header.point_count); + payload.put_varint32(header.doc_count); + payload.put_varint32(header.leaf_count); + payload.put_varint32(header.points_per_leaf); + + if (header.leaf_count == 0) { + // The empty index is header-only (design 5.3). + DORIS_CHECK_EQ(header.point_count, 0); + DORIS_CHECK(min_value.empty()); + DORIS_CHECK(max_value.empty()); + DORIS_CHECK(split_values.empty()); + } else { + const size_t bytes_per_dim = header.bytes_per_dim; + DORIS_CHECK_EQ(min_value.size(), bytes_per_dim); + DORIS_CHECK_EQ(max_value.size(), bytes_per_dim); + DORIS_CHECK_EQ(split_values.size(), (header.leaf_count - 1) * bytes_per_dim); + payload.put_bytes(min_value); + payload.put_bytes(max_value); + payload.put_bytes(split_values); + + // The reader binary-searches this array and indexes the leaf directory + // without re-checking, so both orderings are asserted where they are + // produced. Comparison is unsigned byte-wise from offset 0 (INV-1), which + // is exactly memcmp over the sortable bytes. + for (uint32_t i = 1; i + 1 < header.leaf_count; ++i) { + DORIS_CHECK_LE(std::memcmp(split_values.data() + (i - 1) * bytes_per_dim, + split_values.data() + i * bytes_per_dim, bytes_per_dim), + 0); + } + + uint64_t previous_offset = 0; + uint64_t total_points = 0; + for (size_t i = 0; i < leaves.size(); ++i) { + const uint64_t offset = leaves[i].offset; + // Strictly increasing from the second leaf on; the first delta is the + // absolute offset of leaf 0 inside bkd_data. + DORIS_CHECK(i == 0 || offset > previous_offset); + payload.put_varint64(offset - previous_offset); + previous_offset = offset; + total_points += leaves[i].count; + } + DORIS_CHECK_EQ(total_points, header.point_count); + for (const LeafRef& leaf : leaves) { + payload.put_varint32(leaf.count); + } + } + + // The type + length + crc32c envelope comes from the framer; nothing here + // hand-rolls a checksum. + SectionFramer::write(*sink, kBkdIndexSectionType, payload.view()); +} + +Status BkdIndexBlockReader::open(Slice framed, uint64_t data_length, BkdIndexBlockReader* out) { + DORIS_CHECK(out != nullptr); + + ByteSource src(framed); + FramedSection section; + RETURN_IF_ERROR(SectionFramer::read(src, §ion)); + if (!src.eof()) { + return corrupted("trailing bytes after the framed section"); + } + if (section.type != kBkdIndexSectionType) { + return corrupted("section type is not kBkdIndexSectionType"); + } + return out->decode_payload(section.payload, data_length); +} + +Status BkdIndexBlockReader::decode_payload(Slice payload, uint64_t data_length) { + ByteSource src(payload); + BkdIndexHeader header; + RETURN_IF_ERROR(decode_header(&src, &header)); + + if (header.leaf_count == 0) { + // The empty index (design 5.3): legal, explicit, header-only. + if (header.point_count != 0) { + return corrupted("empty index carries a non-zero point_count"); + } + if (!src.eof()) { + return corrupted("empty index carries trailing payload bytes"); + } + header_ = header; + bounds_.clear(); + split_values_.clear(); + leaves_.clear(); + return Status::OK(); + } + + const uint64_t bytes_per_dim = header.bytes_per_dim; + const uint64_t leaf_count = header.leaf_count; + // Bound leaf_count against the bytes actually present BEFORE anything is + // sized by it: min + max + one split value per gap, plus at least one byte + // for each leaf's offset delta and count. A damaged leaf_count would + // otherwise drive a multi-gigabyte reservation. + const uint64_t minimum_remaining = (leaf_count + 1) * bytes_per_dim + 2 * leaf_count; + if (minimum_remaining > src.remaining()) { + return corrupted("leaf_count exceeds what the payload can describe"); + } + + Slice bounds; + RETURN_IF_ERROR(src.get_bytes(2 * static_cast(bytes_per_dim), &bounds)); + + Slice splits; + RETURN_IF_ERROR(src.get_bytes(static_cast((leaf_count - 1) * bytes_per_dim), &splits)); + RETURN_IF_ERROR( + validate_split_order(splits, static_cast(bytes_per_dim), header.leaf_count)); + + std::vector leaves; + RETURN_IF_ERROR(decode_leaf_directory(&src, header, data_length, &leaves)); + if (!src.eof()) { + return corrupted("trailing payload bytes after the leaf directory"); + } + + // Everything validated: commit. Members are only written once the whole + // payload is known good, so a failed open leaves no half-decoded state. + header_ = header; + bounds_.assign(bounds.data(), bounds.data() + bounds.size()); + split_values_.assign(splits.data(), splits.data() + splits.size()); + leaves_ = std::move(leaves); + return Status::OK(); +} + +size_t BkdIndexBlockReader::heap_bytes() const { + return bounds_.capacity() + split_values_.capacity() + leaves_.capacity() * sizeof(LeafRef); +} + +} // namespace doris::snii::bkd diff --git a/be/src/storage/index/snii/bkd/bkd_index_block.h b/be/src/storage/index/snii/bkd/bkd_index_block.h new file mode 100644 index 00000000000000..d4c82fd016642d --- /dev/null +++ b/be/src/storage/index/snii/bkd/bkd_index_block.h @@ -0,0 +1,160 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include + +#include "common/check.h" +#include "common/status.h" +#include "storage/index/snii/bkd/bkd_types.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/encoding/byte_sink.h" + +// bkd_index -- the HOT sub-file of the SNII-native BKD index (design 5.1). +// +// One SectionFramer section (type kBkdIndexSectionType), so the checksum and the +// length envelope come from the framer and nothing here hand-rolls a crc. Payload: +// +// --- header --- +// magic fixed32 kBkdIndexMagic +// format_version varint32 +// flags varint32 +// bytes_per_dim varint32 +// field_type varint32 +// point_count varint64 +// doc_count varint32 +// leaf_count varint32 +// points_per_leaf varint32 +// --- present only when leaf_count > 0 --- +// min_value bytes[bytes_per_dim] +// max_value bytes[bytes_per_dim] +// split_values bytes[(leaf_count - 1) * bytes_per_dim] ascending, fixed width +// leaf_offsets delta-varint64[leaf_count] strictly increasing +// leaf_counts varint32[leaf_count] +// +// leaf_count == 0 is the EMPTY index (design 5.3): header only, and bkd_data has +// length 0. It is a legal state, never corruption -- unlike the old +// implementation's implicit `indexFP == 0` sentinel over an unchecked bkd_meta. +// +// There is no internal node tree. In one dimension an inner node only routes a +// value to a leaf, which is exactly what an ordered array of split values does: +// leaf i covers [split_value(i - 1), split_value(i)). The old recursive packed +// tree existed to carry a split DIMENSION and an FP delta per level; with +// multi-dimensional support out of scope it collapses to a binary-searchable +// fixed-width array (design 5.1). +namespace doris::snii::bkd { + +// Serializes the bkd_index payload and APPENDS the framed section to `sink` +// (`sink` is not cleared). +// +// Every argument is a BUILD-TIME INVARIANT -- the builder produced all of it in +// this same run -- so violations are programming errors and trip DORIS_CHECK +// rather than returning a Status (design 8). Untrusted bytes only ever enter +// through BkdIndexBlockReader::open. +// +// min_value / max_value : exactly bytes_per_dim bytes, empty iff leaf_count == 0 +// split_values : (leaf_count - 1) * bytes_per_dim bytes, non-decreasing +// leaves : leaf_count entries, offsets strictly increasing, +// counts summing to header.point_count +void encode_bkd_index_block(const BkdIndexHeader& header, Slice min_value, Slice max_value, + Slice split_values, std::span leaves, ByteSink* sink); + +// Decoded bkd_index. Immutable once open() returns, owns its arrays, and holds no +// cursor -- so one instance can serve concurrent queries with no locking and no +// per-query copy (design 9), unlike the packed index the old reader deep-copied on +// every query. +// +// open() runs the ENTIRE structural validation up front (design 8.2): after it +// succeeds the invariants hold by construction and the query hot path may index +// the arrays without re-checking. Disk bytes are NOT invariants, so every one of +// those checks is a Status, never a DORIS_CHECK -- asserting on them would turn a +// recoverable index downgrade into a node crash. +class BkdIndexBlockReader { +public: + BkdIndexBlockReader() = default; + + // Parses and fully validates a framed bkd_index section. + // + // `data_length` is the byte length of the companion bkd_data sub-file; leaf + // offsets are bounded against it HERE so leaf reads later need no bound check. + // + // format_version above kSupportedVersion -> INVERTED_INDEX_NOT_SUPPORTED: a + // capability boundary, so the caller reports "index unavailable" instead of a + // damaged segment. Every other rejection (bad magic, unknown field_type, + // bytes_per_dim disagreeing with field_type, array lengths disagreeing with + // leaf_count, unordered split values, non-increasing or out-of-range leaf + // offsets, leaf counts not summing to point_count, trailing bytes, crc + // mismatch, truncation) -> INVERTED_INDEX_FILE_CORRUPTED. + static Status open(Slice framed, uint64_t data_length, BkdIndexBlockReader* out); + + const BkdIndexHeader& header() const { return header_; } + uint32_t leaf_count() const { return header_.leaf_count; } + // The empty index (design 5.3). Callers must branch on this before asking for + // bounds, split values or leaves -- an empty index has none. + bool empty() const { return header_.leaf_count == 0; } + + // Smallest / largest indexed value, as unsigned big-endian sortable bytes. + // Once per query (the global-bounds fast reject), hence DORIS_CHECK. + Slice min_value() const { + DORIS_CHECK(!empty()); + return Slice(bounds_.data(), header_.bytes_per_dim); + } + Slice max_value() const { + DORIS_CHECK(!empty()); + return Slice(bounds_.data() + header_.bytes_per_dim, header_.bytes_per_dim); + } + + // Boundary between leaf i and leaf i + 1: leaf i + 1 covers + // [split_value(i), split_value(i + 1)). i < leaf_count - 1. Read inside the + // binary search, hence DCHECK. + Slice split_value(uint32_t i) const { + DCHECK_LT(static_cast(i) + 1, header_.leaf_count); + return Slice(split_values_.data() + static_cast(i) * header_.bytes_per_dim, + header_.bytes_per_dim); + } + + // The whole fixed-width split array, for a search that walks it directly + // instead of going through split_value(). Empty when leaf_count <= 1. + Slice split_values() const { return Slice(split_values_); } + + LeafRef leaf(uint32_t i) const { + DCHECK_LT(i, header_.leaf_count); + return leaves_[i]; + } + + // Resident heap held beyond sizeof(*this). This is the REAL cost -- the arrays + // are decoded once and never re-materialized -- so a searcher-cache charge + // built on it does not under-count the way the old ram_bytes_used() did by + // omitting the packed index entirely. + size_t heap_bytes() const; + +private: + Status decode_payload(Slice payload, uint64_t data_length); + + BkdIndexHeader header_; + // min_value followed by max_value, one allocation. Empty for an empty index. + std::vector bounds_; + // (leaf_count - 1) * bytes_per_dim bytes, non-decreasing, fixed width. + std::vector split_values_; + std::vector leaves_; +}; + +} // namespace doris::snii::bkd diff --git a/be/src/storage/index/snii/bkd/bkd_reader.cpp b/be/src/storage/index/snii/bkd/bkd_reader.cpp new file mode 100644 index 00000000000000..7c3073daed046b --- /dev/null +++ b/be/src/storage/index/snii/bkd/bkd_reader.cpp @@ -0,0 +1,601 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/bkd/bkd_reader.h" + +#include +#include +#include +#include + +#include "common/check.h" +#include "roaring/roaring.hh" + +namespace doris::snii::bkd { + +namespace { + +// LSD radix sort over 32-bit doc ids: four 8-bit passes, counting sort each. +// +// std::sort is the wrong tool here and was measured to be: sorting 1M doc ids +// with it costs more than the roaring insertion it was meant to cheapen (the +// wide-range case went from 0.886x to 1.588x against the baseline). Radix is +// O(n) with a 256-entry histogram per pass, which is what makes "sort, then +// insert ascending" cheaper than inserting in leaf order. +void radix_sort_u32(std::vector* values, std::vector* scratch) { + const size_t n = values->size(); + if (n < 2) { + return; + } + scratch->resize(n); + uint32_t* src = values->data(); + uint32_t* dst = scratch->data(); + for (int shift = 0; shift < 32; shift += 8) { + size_t count[257] = {}; + for (size_t i = 0; i < n; ++i) { + ++count[((src[i] >> shift) & 0xFFU) + 1]; + } + // A pass whose key byte is constant reorders nothing; skipping it also + // keeps the src/dst parity correct without an extra copy. + bool uniform = false; + for (size_t b = 1; b < 257; ++b) { + if (count[b] == n) { + uniform = true; + break; + } + } + if (uniform) { + continue; + } + for (size_t b = 1; b < 257; ++b) { + count[b] += count[b - 1]; + } + for (size_t i = 0; i < n; ++i) { + dst[count[(src[i] >> shift) & 0xFFU]++] = src[i]; + } + std::swap(src, dst); + } + // Odd number of executed passes leaves the result in the scratch buffer. + if (src != values->data()) { + std::memcpy(values->data(), src, n * sizeof(uint32_t)); + } +} + +// The BkdSections extents come from the container's named-file table, i.e. from +// disk. Damage there is reported, never asserted (design 8) -- and it is caught +// BEFORE a length is handed to a read, so a corrupt one cannot drive a +// multi-gigabyte allocation on the way to failing. +Status bkd_reader_corrupted(std::string_view what) { + return Status::Error("bkd_reader: {}", what); +} + +Status check_extent(uint64_t offset, uint64_t length, uint64_t file_size, std::string_view name) { + // Written as "length first, then offset against what is left" so neither + // comparison can overflow the way offset + length would. + if (length > file_size || offset > file_size - length) { + return bkd_reader_corrupted(name); + } + return Status::OK(); +} + +// --------------------------------------------------------------------------- +// Split-value search (design 7.2) +// --------------------------------------------------------------------------- +// +// The split array replaces the old recursive descent entirely: routing a value to +// a leaf in one dimension is exactly a binary search over an ordered fixed-width +// array, with no per-level VLong / VInt / prefix decode to pay. +// +// Both searches are over an array that is NON-decreasing, not strictly +// increasing -- one value repeated across several leaves makes consecutive +// splits equal -- so the two bounds genuinely differ and neither may be +// substituted for the other. + +// Number of split values strictly less than `key`, i.e. the index of the first +// split >= key. +uint32_t split_lower_bound(Slice splits, uint32_t width, const uint8_t* key) { + uint32_t low = 0; + auto high = static_cast(splits.size() / width); + while (low < high) { + const uint32_t mid = low + (high - low) / 2; + if (std::memcmp(splits.data() + static_cast(mid) * width, key, width) < 0) { + low = mid + 1; + } else { + high = mid; + } + } + return low; +} + +// Number of split values <= `key`, i.e. the index of the first split > key. +uint32_t split_upper_bound(Slice splits, uint32_t width, const uint8_t* key) { + uint32_t low = 0; + auto high = static_cast(splits.size() / width); + while (low < high) { + const uint32_t mid = low + (high - low) / 2; + if (std::memcmp(splits.data() + static_cast(mid) * width, key, width) <= 0) { + low = mid + 1; + } else { + high = mid; + } + } + return low; +} + +// The whole zero-IO half of design 7.2: narrows a range down to the contiguous +// run of leaves [*first, *last] that can hold a match, from nothing but the +// resident header and split array. Returns false when nothing can match, in +// which case not a single leaf is touched. +// +// `block` must be non-empty -- the empty index is answered by the caller. +bool locate_leaf_window(const BkdIndexBlockReader& block, Slice lower, bool lower_inclusive, + Slice upper, bool upper_inclusive, uint32_t* first, uint32_t* last) { + const uint32_t width = block.header().bytes_per_dim; + + // Global-bounds fast reject. Every indexed value lies in + // [min_value, max_value], so if the extreme value on one side already fails + // its bound, nothing in the index can satisfy it. + if (!upper.empty()) { + const int order = std::memcmp(block.min_value().data(), upper.data(), width); + if (upper_inclusive ? order > 0 : order >= 0) { + return false; + } + } + if (!lower.empty()) { + const int order = std::memcmp(block.max_value().data(), lower.data(), width); + if (lower_inclusive ? order < 0 : order <= 0) { + return false; + } + } + + // Leaf i spans values [split(i-1), split(i)] inclusive at BOTH ends + // (split(-1) is min_value, split(leaf_count-1) is max_value): the builder + // makes split(i) the FIRST value of leaf i+1, and one value repeated across + // leaves makes the last value of leaf i equal split(i) too. + // + // first = the lowest leaf whose MAXIMUM can still satisfy the lower bound. + // Leaf j's maximum is at most split(j), so a leaf whose split(j) + // already fails the bound is skipped outright. + // last = the highest leaf whose MINIMUM can still satisfy the upper bound. + // Leaf j's minimum is exactly split(j-1), so the COUNT of splits + // still satisfying the bound is that leaf's index. + const Slice splits = block.split_values(); + *first = 0; + if (!lower.empty()) { + *first = lower_inclusive ? split_lower_bound(splits, width, lower.data()) + : split_upper_bound(splits, width, lower.data()); + } + *last = block.leaf_count() - 1; + if (!upper.empty()) { + *last = upper_inclusive ? split_upper_bound(splits, width, upper.data()) + : split_lower_bound(splits, width, upper.data()); + } + // An interval whose lower bound sits above its upper one is empty by + // definition -- a legal query (a planner fusing `a > 30 AND a < 10` produces + // one), and one the global reject cannot catch when both bounds lie inside + // [min_value, max_value]. + return *first <= *last; +} + +// --------------------------------------------------------------------------- +// Boundary-leaf value filter +// --------------------------------------------------------------------------- + +enum class BoundSide { kLower, kUpper }; + +// One side of the range predicate, specialized to ONE decoded leaf. +// +// Every value in a leaf is common_prefix ++ suffix, and the prefix is the same +// for all of them, so its comparison against the bound is done ONCE here; per run +// only the suffix is left. The boundary scan therefore never reassembles a whole +// value, and when the prefix alone already decides (the usual case for a narrow +// range over a wide type) the per-run cost is a single branch. +// +// An EMPTY `bound` is the unbounded side (design 7.1): satisfied_by() is then +// always true, which is also how the two boundary leaves of a multi-leaf range +// each test only the one bound that can still exclude something. +class LeafValueBound { +public: + LeafValueBound(const DecodedLeafBlock& leaf, Slice bound, bool inclusive, BoundSide side) + : bounded_(!bound.empty()), inclusive_(inclusive), side_(side) { + if (!bounded_) { + return; + } + const size_t prefix_length = leaf.common_prefix.size(); + // A zero-length prefix decides nothing, and memcmp over zero bytes would + // be handed the null data() of an empty Slice. + prefix_order_ = prefix_length == 0 ? 0 + : std::memcmp(leaf.common_prefix.data(), bound.data(), + prefix_length); + bound_suffix_ = bound.data() + prefix_length; + suffix_width_ = leaf.suffix_width; + } + + bool satisfied_by(const LeafValueRun& run) const { + if (!bounded_) { + return true; + } + const int order = compare(run); + if (side_ == BoundSide::kLower) { + return inclusive_ ? order >= 0 : order > 0; + } + return inclusive_ ? order <= 0 : order < 0; + } + +private: + // Sign of (run value) - (bound), as an unsigned byte-wise comparison (INV-1). + int compare(const LeafValueRun& run) const { + if (prefix_order_ != 0) { + return prefix_order_; + } + // kAllEqual: the prefix IS the whole value, so equal prefixes mean equal + // values and there is no suffix to look at. + if (suffix_width_ == 0) { + return 0; + } + return std::memcmp(run.suffix.data(), bound_suffix_, suffix_width_); + } + + const bool bounded_; + const bool inclusive_; + const BoundSide side_; + int prefix_order_ = 0; + const uint8_t* bound_suffix_ = nullptr; + uint32_t suffix_width_ = 0; +}; + +} // namespace + +BkdReader::BkdReader(io::FileReader* file, const BkdSections& sections) + : file_(file), sections_(sections) {} + +Status BkdReader::open(io::FileReader* file, const BkdSections& sections, + std::unique_ptr* out) { + DORIS_CHECK(file != nullptr); + DORIS_CHECK(out != nullptr); + + const uint64_t file_size = file->size(); + RETURN_IF_ERROR(check_extent(sections.index_offset, sections.index_length, file_size, + "the bkd_index extent does not fit the file")); + RETURN_IF_ERROR(check_extent(sections.data_offset, sections.data_length, file_size, + "the bkd_data extent does not fit the file")); + + // bkd_index is the HOT sub-file: read in full once, kept resident, never read + // again (design 5.1). A zero length falls through to the framer, which + // reports it as damage. + std::vector index_bytes; + index_bytes.resize(static_cast(sections.index_length)); + RETURN_IF_ERROR(file->read_into(sections.index_offset, index_bytes.data(), index_bytes.size())); + + auto reader = std::unique_ptr(new BkdReader(file, sections)); + // Runs the ENTIRE structural validation, including bounding the leaf offsets + // against the bkd_data length passed here -- which is what lets read_leaf() + // below compute a block extent without re-checking anything (design 8.2). + RETURN_IF_ERROR( + BkdIndexBlockReader::open(Slice(index_bytes), sections.data_length, &reader->block_)); + // Published only once everything is valid, so a failed open leaves the + // caller's unique_ptr untouched. + *out = std::move(reader); + return Status::OK(); +} + +Status BkdReader::range(Slice lower, bool lower_inclusive, Slice upper, bool upper_inclusive, + roaring::Roaring* hits) const { + BkdQueryScratch scratch; + return range(lower, lower_inclusive, upper, upper_inclusive, hits, &scratch); +} + +Status BkdReader::range(Slice lower, bool lower_inclusive, Slice upper, bool upper_inclusive, + roaring::Roaring* hits, BkdQueryScratch* scratch) const { + DORIS_CHECK(hits != nullptr); + DORIS_CHECK(scratch != nullptr); + const uint32_t width = block_.header().bytes_per_dim; + // Bounds come from the caller's KeyCoder for header().field_type, so a wrong + // width is a programming error, not damage. Empty is the unbounded side. + DORIS_CHECK(lower.empty() || lower.size() == width); + DORIS_CHECK(upper.empty() || upper.size() == width); + + // Whatever the caller's bitmap held is not part of this answer. + *hits = roaring::Roaring(); + + // The empty index (design 5.3 / 10.4): an empty result, NOT an error for the + // adapter to translate, and no I/O. + if (block_.empty()) { + return Status::OK(); + } + + uint32_t first = 0; + uint32_t last = 0; + // Answered entirely from the resident bkd_index. A range that cannot match -- + // outside the global bounds, or an empty interval -- therefore costs zero + // positioned reads (design 7.2). + if (!locate_leaf_window(block_, lower, lower_inclusive, upper, upper_inclusive, &first, + &last)) { + return Status::OK(); + } + + if (first == last) { + return scan_boundary_leaf(first, lower, lower_inclusive, upper, upper_inclusive, hits, + scratch); + } + + // first < last, so split(first) already satisfies the lower bound and + // split(last - 1) already satisfies the upper one. Everything in leaf `first` + // is therefore at most split(first) <= upper, and everything in leaf `last` is + // at least split(last-1) >= lower: each boundary leaf only has to test the one + // bound on its own side. + RETURN_IF_ERROR( + scan_boundary_leaf(first, lower, lower_inclusive, Slice(), true, hits, scratch)); + // Leaves strictly between them are bounded by those same two splits on both + // sides, so they are whole-leaf hits: doc ids only, values never decoded. + // Interior leaves are whole-leaf hits. Their doc ids are gathered, SORTED, + // and inserted once. + // + // The sort is the point, not the batching. Leaves are ordered by VALUE, so + // consecutive leaves carry unrelated doc ids and the insertion sequence is + // effectively random. Roaring pays far more for that than for an ascending + // run: measured on this benchmark, inserting 1M doc ids in leaf order costs + // ~63 ms while inserting the same ids ascending costs ~5.5 ms. Batching + // alone does not recover it -- an earlier attempt that gathered without + // sorting changed nothing. + // Flushed in bounded chunks, NOT accumulated across the whole range. + // + // Gathering every interior leaf's doc ids into one vector makes the + // allocation a function of the RANGE, and nothing in the format bounds + // leaf_count * points_per_leaf: a crafted index of a few tens of KB, whose + // leaves each declare the legal maximum count and encode as kAllEqual (zero + // bytes per point), drives billions of doc ids here. The per-leaf ceiling in + // bkd_index_block does not compose into an aggregate one. + // + // A chunk still sorts in large batches, which is where the win is -- the + // cost being avoided is random-order insertion, not the call count. + constexpr size_t kMaxGatheredDocIds = 1U << 20; + // The boundary-leaf decode holds Slices INTO scratch->leaf_bytes, which the + // interior loop is about to overwrite and may reallocate. Cleared so nothing + // can later read a struct that still looks populated but points at freed + // bytes. + scratch->decoded.clear(); + scratch->gathered.clear(); + const auto flush = [&] { + if (scratch->gathered.empty()) { + return; + } + radix_sort_u32(&scratch->gathered, &scratch->radix_scratch); + hits->addMany(scratch->gathered.size(), scratch->gathered.data()); + scratch->gathered.clear(); + }; + for (uint32_t leaf = first + 1; leaf < last; ++leaf) { + RETURN_IF_ERROR(read_leaf(leaf, &scratch->leaf_bytes)); + RETURN_IF_ERROR(decode_leaf_doc_ids(Slice(scratch->leaf_bytes), + block_.header().bytes_per_dim, block_.leaf(leaf).count, + &scratch->doc_ids)); + scratch->gathered.insert(scratch->gathered.end(), scratch->doc_ids.begin(), + scratch->doc_ids.end()); + if (scratch->gathered.size() >= kMaxGatheredDocIds) { + flush(); + } + } + flush(); + return scan_boundary_leaf(last, Slice(), true, upper, upper_inclusive, hits, scratch); +} + +Status BkdReader::lookup_many(const std::vector& values, roaring::Roaring* hits) const { + BkdQueryScratch scratch; + return lookup_many(values, hits, &scratch); +} + +Status BkdReader::lookup_many(const std::vector& values, roaring::Roaring* hits, + BkdQueryScratch* scratch) const { + DORIS_CHECK(hits != nullptr); + DORIS_CHECK(scratch != nullptr); + const uint32_t width = block_.header().bytes_per_dim; + + *hits = roaring::Roaring(); + if (values.empty() || block_.empty()) { + return Status::OK(); + } + for (const Slice& value : values) { + // Same width contract as range(): these come from the caller's KeyCoder. + DORIS_CHECK_EQ(value.size(), static_cast(width)); + } + + // Ordered HERE rather than demanded of the caller. + // + // This used to be a caller invariant, justified by "the caller holds the set + // and knows its order for free". That premise is wrong for the caller this + // exists to serve: InListPredicateBase iterates a HybridSet backed by + // phmap::flat_hash_set, which yields HASH order. It was also enforced with a + // bare glog DCHECK, which is a no-op under NDEBUG -- so in a release build an + // unsorted list silently lost rows instead of failing, because the watermark + // below skips every leaf under the high-water mark. + // + // Sorting N probe values is negligible against the leaf reads they cause, and + // it makes the entry point correct for any caller. Duplicates are dropped + // because two equal values would locate the same window twice. + scratch->probes.assign(values.begin(), values.end()); + std::sort(scratch->probes.begin(), scratch->probes.end(), + [width](const Slice& a, const Slice& b) { + return std::memcmp(a.data(), b.data(), width) < 0; + }); + scratch->probes.erase(std::unique(scratch->probes.begin(), scratch->probes.end(), + [width](const Slice& a, const Slice& b) { + return std::memcmp(a.data(), b.data(), width) == 0; + }), + scratch->probes.end()); + const std::vector& ordered = scratch->probes; + + // Every leaf that at least one value can live in, ascending and unique. + // Windows are non-decreasing because the values are, so the watermark alone + // deduplicates them -- a value repeated across leaves widens its own window, + // and two values in one leaf produce overlapping windows that collapse here. + std::vector leaves; + uint32_t watermark = 0; + for (const Slice& value : ordered) { + uint32_t first = 0; + uint32_t last = 0; + if (!locate_leaf_window(block_, value, true, value, true, &first, &last)) { + continue; + } + for (uint32_t leaf = std::max(first, watermark); leaf <= last; ++leaf) { + leaves.push_back(leaf); + } + watermark = std::max(watermark, last + 1); + } + + std::vector value_buffer(width); + for (const uint32_t leaf_index : leaves) { + RETURN_IF_ERROR(read_leaf(leaf_index, &scratch->leaf_bytes)); + RETURN_IF_ERROR(decode_leaf_block(Slice(scratch->leaf_bytes), width, + block_.leaf(leaf_index).count, &scratch->decoded)); + const DecodedLeafBlock& leaf = scratch->decoded; + std::memcpy(value_buffer.data(), leaf.common_prefix.data(), leaf.common_prefix.size()); + + // Runs, not points: a run is one distinct value, so one binary search + // over the query set answers for all of its doc ids at once. Both + // sequences ascend, but the search is kept per run rather than turned + // into a linear merge because a leaf typically holds far more runs than + // the query holds values. + for (const LeafValueRun& run : leaf.runs) { + std::memcpy(value_buffer.data() + leaf.common_prefix.size(), run.suffix.data(), + run.suffix.size()); + // `ordered`, never `values`: the caller's vector is in whatever order + // it was handed to us, and a binary search over it is meaningless. + const auto found = std::lower_bound( + ordered.begin(), ordered.end(), value_buffer, + [width](const Slice& candidate, const std::vector& target) { + return std::memcmp(candidate.data(), target.data(), width) < 0; + }); + if (found != ordered.end() && + std::memcmp(found->data(), value_buffer.data(), width) == 0) { + hits->addMany(run.count, leaf.doc_ids.data() + run.first_point); + } + } + } + return Status::OK(); +} + +Status BkdReader::estimate_cardinality(Slice lower, bool lower_inclusive, Slice upper, + bool upper_inclusive, uint64_t* out) const { + DORIS_CHECK(out != nullptr); + const uint32_t width = block_.header().bytes_per_dim; + DORIS_CHECK(lower.empty() || lower.size() == width); + DORIS_CHECK(upper.empty() || upper.size() == width); + + *out = 0; + if (block_.empty()) { + return Status::OK(); + } + uint32_t first = 0; + uint32_t last = 0; + if (!locate_leaf_window(block_, lower, lower_inclusive, upper, upper_inclusive, &first, + &last)) { + return Status::OK(); + } + + // A boundary leaf is only GUESSED at when the bound actually cuts into it. + // Halving it unconditionally would be wrong in the common case: an unbounded + // interval has no partial leaf at all, yet the outermost leaves are still + // "first" and "last", and halving them would under-count a full scan by a + // whole leaf on each side. + // + // leaf i spans [leaf_min(i), leaf_max(i)] where leaf_min(i) is split(i-1) -- + // exactly leaf i's first value, since the builder makes split(i) the first + // value of leaf i+1 -- and leaf_max(i) is split(i), which is an UPPER bound + // on leaf i's real maximum. Using it errs toward calling a leaf partial, so + // the estimate stays conservative rather than optimistic. + const Slice splits = block_.split_values(); + const auto leaf_min = [&](uint32_t i) { + return i == 0 ? block_.min_value() + : Slice(splits.data() + static_cast(i - 1) * width, width); + }; + const auto leaf_max = [&](uint32_t i) { + return i + 1 == block_.leaf_count() + ? block_.max_value() + : Slice(splits.data() + static_cast(i) * width, width); + }; + + bool first_whole = lower.empty(); + if (!first_whole) { + const int order = std::memcmp(leaf_min(first).data(), lower.data(), width); + first_whole = lower_inclusive ? order >= 0 : order > 0; + } + bool last_whole = upper.empty(); + if (!last_whole) { + const int order = std::memcmp(leaf_max(last).data(), upper.data(), width); + last_whole = upper_inclusive ? order <= 0 : order < 0; + } + + // The interior is EXACT: those leaves lie wholly inside the interval, so + // their recorded counts are the answer, not a guess. + uint64_t estimate = 0; + for (uint32_t leaf = first + 1; leaf + 1 <= last; ++leaf) { + estimate += block_.leaf(leaf).count; + } + if (first == last) { + const uint32_t count = block_.leaf(first).count; + estimate += (first_whole && last_whole) ? count : count / 2; + } else { + estimate += first_whole ? block_.leaf(first).count : block_.leaf(first).count / 2; + estimate += last_whole ? block_.leaf(last).count : block_.leaf(last).count / 2; + } + *out = estimate; + return Status::OK(); +} + +Status BkdReader::read_leaf(uint32_t index, std::vector* buffer) const { + const LeafRef leaf = block_.leaf(index); + // open() established that leaf offsets strictly increase and that the last one + // is within data_length, so the subtraction cannot wrap and the extent cannot + // leave the sub-file (design 8.2). A last leaf starting exactly at the end of + // bkd_data yields an empty block, which the leaf decoder rejects as damage. + const uint64_t end = (index + 1 < block_.leaf_count()) ? block_.leaf(index + 1).offset + : sections_.data_length; + const size_t length = static_cast(end - leaf.offset); + buffer->resize(length); + // One stateless positioned read per leaf -- no cursor, hence no clone() and no + // synchronization between concurrent queries (design 9). + return file_->read_into(sections_.data_offset + leaf.offset, buffer->data(), length); +} + +Status BkdReader::scan_boundary_leaf(uint32_t index, Slice lower, bool lower_inclusive, Slice upper, + bool upper_inclusive, roaring::Roaring* hits, + BkdQueryScratch* scratch) const { + RETURN_IF_ERROR(read_leaf(index, &scratch->leaf_bytes)); + RETURN_IF_ERROR(decode_leaf_block(Slice(scratch->leaf_bytes), block_.header().bytes_per_dim, + block_.leaf(index).count, &scratch->decoded)); + const DecodedLeafBlock& leaf = scratch->decoded; + + const LeafValueBound lower_bound(leaf, lower, lower_inclusive, BoundSide::kLower); + const LeafValueBound upper_bound(leaf, upper, upper_inclusive, BoundSide::kUpper); + // Runs, not points: a run of equal values is judged ONCE and then accepted or + // skipped whole, which is what makes a leaf of a heavily repeated value cost a + // handful of comparisons instead of one per point. + for (const LeafValueRun& run : leaf.runs) { + if (!lower_bound.satisfied_by(run)) { + continue; + } + // Runs ascend, so the first one past the upper bound ends the leaf: the + // early exit the old implementation had, kept. + if (!upper_bound.satisfied_by(run)) { + break; + } + hits->addMany(run.count, leaf.doc_ids.data() + run.first_point); + } + return Status::OK(); +} + +} // namespace doris::snii::bkd diff --git a/be/src/storage/index/snii/bkd/bkd_reader.h b/be/src/storage/index/snii/bkd/bkd_reader.h new file mode 100644 index 00000000000000..6f85fd55e21878 --- /dev/null +++ b/be/src/storage/index/snii/bkd/bkd_reader.h @@ -0,0 +1,223 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/bkd/bkd_index_block.h" +#include "storage/index/snii/bkd/bkd_types.h" +#include "storage/index/snii/bkd/leaf_codec.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/io/file_reader.h" + +// Forward-declare the CRoaring C++ bitmap so this header stays free of the +// (large) roaring include, exactly as format/null_bitmap.h does. +namespace roaring { +class Roaring; +} // namespace roaring + +// Read side of the SNII-native BKD index (design 7). This header includes only +// the DECODE half of the shared leaf codec plus the already-decoded bkd_index; +// nothing from bkd_builder.h reaches it (design 4). +namespace doris::snii::bkd { + +// Per-query working buffers, owned by the CALLER. +// +// Design 9 targets zero per-query heap allocation: a caller that runs many +// queries keeps one of these alive and every leaf read, every decoded leaf and +// every doc id array lands in buffers that are already the right size. The +// reader itself stays immutable and stateless, which is what makes it shareable +// across concurrent queries with no locking -- the state has to live somewhere, +// and this is that somewhere. +// +// A fresh scratch is always valid; reuse is an optimization, never a +// correctness requirement, and nothing carried over from a previous query can +// affect the next one's answer. +struct BkdQueryScratch { + // Raw bytes of the leaf currently being examined, as read from bkd_data. + std::vector leaf_bytes; + // The boundary-leaf decode (values + doc ids). Keeps its capacity across + // leaves. + DecodedLeafBlock decoded; + // The whole-leaf-hit decode (doc ids only). + std::vector doc_ids; + // Doc ids gathered across consecutive whole-leaf hits, SORTED before they + // reach the bitmap. Leaves are ordered by value, so their doc ids arrive in + // arbitrary order, and roaring inserts sorted input far more cheaply. + std::vector gathered; + // Ping-pong buffer for the radix sort of `gathered`. + std::vector radix_scratch; + // lookup_many's probe values, ordered and deduplicated internally. + std::vector probes; +}; + +// An opened BKD index: the whole hot bkd_index resident and validated, plus the +// FileReader the cold bkd_data leaves are read from. +// +// IMMUTABLE AFTER open(). Every query method is const and keeps all of its state +// on the stack or in the caller's BkdQueryScratch, so one instance serves +// concurrent queries with no locking, no clone() and no per-query copy of +// anything (design 9). The old reader needed clone() because IndexInput carries a +// cursor, deep-copied its packed index on every query, and wrote a shared field +// from query threads; positioned reads plus an immutable directory remove all +// three by construction. +// +// Lifetime: `file` is borrowed and must outlive the reader. There is no +// reference counting and no Directory -- the SNII segment reader owns the file +// and strictly outlives every index opened over it (D2). +class BkdReader { +public: + // Reads bkd_index in full, validates it (BkdIndexBlockReader::open runs the + // entire structural check, design 8.2), and publishes the reader. + // + // `sections` comes from the container's named-file table, i.e. from disk, so + // an extent that does not fit the file is damage to REPORT, not an invariant + // to assert -- and it is rejected before its length is handed to a read. + // + // A bkd_index above kSupportedVersion comes back as + // INVERTED_INDEX_NOT_SUPPORTED; every other rejection is + // INVERTED_INDEX_FILE_CORRUPTED. `*out` is left untouched on failure. + // + // An index with zero points opens successfully (design 5.3): emptiness is a + // legal state that answers queries with an empty bitmap, NOT an error the + // adapter has to translate. + static Status open(io::FileReader* file, const BkdSections& sections, + std::unique_ptr* out); + + ~BkdReader() = default; + + BkdReader(const BkdReader&) = delete; + BkdReader& operator=(const BkdReader&) = delete; + + // The single range primitive (design 7.1). An EMPTY `lower` / `upper` Slice + // means that side is unbounded, so <, <=, >, >= and BETWEEN are all one call + // and one pass. A non-empty bound must be exactly bytes_per_dim unsigned + // big-endian sortable bytes produced by the KeyCoder of header().field_type + // -- the index's OWN type, not the query's, or every comparison would run + // against a different byte order (INV-1). A wrong length is a caller bug and + // trips DORIS_CHECK. + // + // `hits` is CLEARED and then filled with exactly the matching doc ids; + // whatever it held before is discarded. + // + // An interval that is empty on its face (lower above upper, or an open + // interval over a single value) is answered with an empty bitmap and no I/O + // at all -- it is a legal query, not an error. + // + // The overload without a scratch allocates one on the stack for the duration + // of the call; pass one explicitly to reuse its buffers across queries. + Status range(Slice lower, bool lower_inclusive, Slice upper, bool upper_inclusive, + roaring::Roaring* hits) const; + Status range(Slice lower, bool lower_inclusive, Slice upper, bool upper_inclusive, + roaring::Roaring* hits, BkdQueryScratch* scratch) const; + + // Multi-value lookup in ONE pass (design 7.3). `values` may arrive in ANY + // order and may contain duplicates: they are sorted and deduplicated here. + // They were once a caller invariant, but the caller this serves -- + // InListPredicateBase over a hash-backed HybridSet -- cannot supply order + // cheaply, and the cost of sorting N probes is nothing against the leaf + // reads they cause. + // + // Equivalent to the union of range(v, true, v, true) over every value, but a + // leaf that several values land in is READ ONCE. That is the whole reason + // this exists: `IN (v1..vN)` currently runs N full traversals, one per value, + // because InListPredicateBase loops over its own value set (design 15 Q1b). + // + // `hits` is CLEARED first, exactly as range() does. + Status lookup_many(const std::vector& values, roaring::Roaring* hits) const; + Status lookup_many(const std::vector& values, roaring::Roaring* hits, + BkdQueryScratch* scratch) const; + + // How many POINTS the interval is expected to hold, from the resident leaf + // directory alone -- no leaf is read (design 7.4). + // + // Only the two boundary leaves are guessed at, each at half its recorded + // count, so the error never exceeds points_per_leaf and an interval that + // covers whole leaves only is EXACT. The old implementation returned + // max_points_in_leaf x subtree_leaves for an inside node, i.e. it assumed + // every leaf was full, and over-counted a sparse tail by multiples -- which + // matters because this number is what inverted_index_skip_threshold's bypass + // decision is made on. + // + // Bound semantics are range()'s: an empty Slice is an unbounded side, and an + // interval that cannot match anything estimates 0. + Status estimate_cardinality(Slice lower, bool lower_inclusive, Slice upper, + bool upper_inclusive, uint64_t* out) const; + + // Everything the validated bkd_index header records, including the + // field_type a caller resolves its KeyCoder from. + // The file this reader was opened against. Callers that resolved an extent + // from the SAME container (a blob index's null-bitmap sub-file) must read + // through THIS reader, not through whatever IndexFileReader they happen to + // hold: a searcher-cache hit can outlive the IndexFileReader that opened it, + // and the caller's own may never have been init()-ed. + io::FileReader* reader() const { return file_; } + + const BkdIndexHeader& header() const { return block_.header(); } + + uint64_t point_count() const { return block_.header().point_count; } + uint32_t doc_count() const { return block_.header().doc_count; } + uint32_t leaf_count() const { return block_.leaf_count(); } + // The empty index (design 5.3). Callers must branch on this before asking for + // bounds -- an empty index has none. + bool empty() const { return block_.empty(); } + + // Smallest / largest indexed value as sortable bytes. DORIS_CHECKs !empty(). + Slice min_value() const { return block_.min_value(); } + Slice max_value() const { return block_.max_value(); } + + // Real resident cost: this object plus the decoded leaf directory and split + // array it owns. There is no hidden per-query allocation for it to omit, so + // unlike the old ram_bytes_used() -- which left out the packed index that was + // then deep-copied per query -- this is the whole story. + size_t memory_usage() const { return sizeof(*this) + block_.heap_bytes(); } + +private: + BkdReader(io::FileReader* file, const BkdSections& sections); + + // Reads leaf `index` into `buffer`. The block's extent comes from the leaf + // directory: the next leaf's offset, or the bkd_data length for the last + // leaf. Both the strict ordering and the bound against data_length were + // established at open, so nothing is re-checked here (design 8.2). + Status read_leaf(uint32_t index, std::vector* buffer) const; + + // A leaf that may hold both matching and non-matching values: decode the + // values and filter run by run. At most two leaves per range see this + // (design 7.2). An empty bound means that side needs no test, which is how + // the middle-of-a-range boundary leaves skip half the work. + Status scan_boundary_leaf(uint32_t index, Slice lower, bool lower_inclusive, Slice upper, + bool upper_inclusive, roaring::Roaring* hits, + BkdQueryScratch* scratch) const; + + // A leaf entirely inside the range: take its doc ids and never look at a + // value. The leaf's trailing docid_block_offset is what makes this skip the + // value bytes outright (design 7.2) -- and this is the path with thousands of + // leaves on it, which is why the layout optimizes it over the boundary one. + + io::FileReader* const file_; + const BkdSections sections_; + // The whole hot sub-file, decoded once. Immutable, owns its arrays, holds no + // cursor. + BkdIndexBlockReader block_; +}; + +} // namespace doris::snii::bkd diff --git a/be/src/storage/index/snii/bkd/bkd_types.h b/be/src/storage/index/snii/bkd/bkd_types.h new file mode 100644 index 00000000000000..a21fafd11bb722 --- /dev/null +++ b/be/src/storage/index/snii/bkd/bkd_types.h @@ -0,0 +1,135 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include "storage/index/snii/bkd/bkd_format.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/writer/memory_reporter.h" +#include "storage/olap_common.h" + +// Plain data definitions shared by the SNII-native BKD write and read sides. +// DEFINITIONS ONLY -- no behaviour lives here, so neither side has to include +// the other's headers to name a parameter or a result. Every type is trivially +// copyable and owns nothing: a reader can hold them by value and share them +// across concurrent queries without locking or per-query copies. +namespace doris::snii::bkd { + +// One point as handed to / decoded from the builder. The sort key is the pair +// (value, doc_id): a doc contributing several points (array column) is a +// first-class case, not something a "single value per doc" flag has to promise +// away. +struct PointRef { + // Unsigned big-endian sortable bytes from KeyCoder::full_encode_ascending, + // exactly bytes_per_dim long (INV-1 / INV-2). A VIEW -- the referenced bytes + // are owned elsewhere and must outlive this struct. + Slice value; + // Segment-local row id (INV-3). + uint32_t doc_id = 0; +}; + +// One decoded leaf-directory row. The directory is stored column-wise in +// bkd_index as delta-varint64 offsets followed by varint32 counts (design 5.1); +// this is the row view a reader works with. +struct LeafRef { + // Byte offset of the leaf block within bkd_data. Directory offsets are + // strictly increasing and the last one is bounded by the bkd_data length -- + // both established once at open, so query-time access needs no re-checking. + uint64_t offset = 0; + // Points in this leaf; <= points_per_leaf, and only the last leaf may be + // short. + uint32_t count = 0; +}; + +// Decoded bkd_index header (design 5.1). A default-constructed value already +// reads as the EMPTY index: leaf_count == 0 states emptiness explicitly, +// unlike the old implementation's implicit indexFP == 0 sentinel over an +// unchecked bkd_meta. +struct BkdIndexHeader { + uint32_t format_version = kFormatVersion; + // index_flags bits; diagnostic only, never branched on while reading. + uint32_t flags = 0; + // == sizeof(CppType) for field_type (INV-2). 0 only for a header that has + // not been decoded yet. + uint32_t bytes_per_dim = 0; + // The type the index was BUILT with. The KeyCoder used at query time is + // resolved from this, not from the query's own type, or the comparison + // would silently run against a different byte order (INV-1). + FieldType field_type {}; + uint64_t point_count = 0; + // Distinct doc ids owning at least one point. Counted by the builder, not + // pushed in from outside before finish() as the old docs_seen_ was. + uint32_t doc_count = 0; + // 0 == empty index: no bounds, no split values, no leaf directory, and a + // zero-length bkd_data. + uint32_t leaf_count = 0; + uint32_t points_per_leaf = 0; +}; + +// Builder construction parameters (design 6.1). Validated once by +// BkdBuilder::create, so a constructed builder is always fully valid -- there +// is no half-initialized state to defend against later. +struct BkdBuilderOptions { + // REQUIRED, == sizeof(CppType). 0 is the unset sentinel. + uint32_t bytes_per_dim = 0; + // REQUIRED. FieldType has no enumerator 0, so the value-initialized state is + // an unambiguous unset sentinel. + FieldType field_type {}; + uint32_t points_per_leaf = kDefaultPointsPerLeaf; + // Resident point-buffer ceiling; crossing it sorts and spills a run. + uint64_t build_buffer_bytes = kDefaultBuildBufferBytes; + // Build-time RAM accounting. Legitimately null off-Doris (unit tests, + // benchmarks), where only the local buffer bound applies. + writer::MemoryReporter* reporter = nullptr; +}; + +// What a completed build reports back to its caller. +struct BkdStats { + uint64_t point_count = 0; + uint32_t doc_count = 0; + uint32_t leaf_count = 0; + // Encoded sizes of the two sub-files, for container bookkeeping and for + // reporting real resident/on-disk cost instead of an estimate. + uint64_t index_bytes = 0; + uint64_t data_bytes = 0; + // Mirrors index_flags::kBuiltWithSpill; diagnostic only, the emitted bytes + // are identical either way. + bool built_with_spill = false; + // How many merge passes the spill path ran. 0 = no spill; 1 = every run was + // folded in one k-way merge; >1 = the run count exceeded the fan-in the + // memory bound allows, so runs were folded in groups first. + uint32_t merge_passes = 0; + // Largest resident cursor-window footprint any single merge held, in bytes. + // This is the quantity the build_buffer_bytes ceiling is supposed to bound, + // and it is reported rather than assumed so a test can hold the bound to + // account instead of trusting the arithmetic that produced it. + uint64_t peak_merge_buffer_bytes = 0; +}; + +// Where the two sub-files live inside the SNII container, as resolved from the +// blob logical index's named-file table. length == 0 is LEGAL and means the +// empty index (design 5.3) -- it must never be treated as corruption. +struct BkdSections { + uint64_t index_offset = 0; + uint64_t index_length = 0; + uint64_t data_offset = 0; + uint64_t data_length = 0; +}; + +} // namespace doris::snii::bkd diff --git a/be/src/storage/index/snii/bkd/leaf_codec.cpp b/be/src/storage/index/snii/bkd/leaf_codec.cpp new file mode 100644 index 00000000000000..3ef0a40427ac4c --- /dev/null +++ b/be/src/storage/index/snii/bkd/leaf_codec.cpp @@ -0,0 +1,498 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/bkd/leaf_codec.h" + +#include +#include +#include +#include +#include + +#include "common/check.h" +#include "storage/index/snii/bkd/bkd_format.h" +#include "storage/index/snii/encoding/byte_source.h" +#include "storage/index/snii/encoding/pfor.h" +#include "storage/index/snii/encoding/varint.h" + +// The encode half and the decode half below share NOTHING but the constants in +// bkd_format.h (design 4): no helper, no struct, no constant is reused across the +// divider. That is the whole point -- the old docids_writer declared both +// directions on one type, which is how the writer TU came to include the entire +// read side. +namespace doris::snii::bkd { + +// =========================================================================== +// Encode +// =========================================================================== +namespace { + +static_assert(kPointDocIdBytes == 4, "the build-time record tail is a 4-byte big-endian doc id"); + +// The builder's own point buffer (design 6.2), addressed by point index: fixed +// width [value: bytes_per_dim][doc_id: 4 big-endian] records, sorted by the memcmp +// of the whole record, which IS (value, doc_id) order. +struct PointArray { + const uint8_t* records = nullptr; + size_t record_size = 0; + uint32_t bytes_per_dim = 0; + size_t point_count = 0; + + const uint8_t* value(size_t point) const { return records + point * record_size; } + + // The big-endian doc id tail. Big-endian is what makes the whole-record memcmp + // equal (value, doc_id) order, so it is assembled by hand here rather than read + // as a native integer. + uint32_t doc_id(size_t point) const { + const uint8_t* tail = records + point * record_size + bytes_per_dim; + return (static_cast(tail[0]) << 24) | (static_cast(tail[1]) << 16) | + (static_cast(tail[2]) << 8) | static_cast(tail[3]); + } +}; + +// Index of the first point of every maximal run of equal values. The input is +// sorted, so one memcmp pass finds them all. +std::vector scan_runs(const PointArray& points) { + std::vector run_starts; + run_starts.push_back(0); + for (size_t point = 1; point < points.point_count; ++point) { + const int order = + std::memcmp(points.value(point - 1), points.value(point), points.bytes_per_dim); + // Per-point, hence DCHECK. Equal adjacent records are legal (an array + // column repeating one value inside one row); descending ones mean the + // caller handed over an unsorted buffer. + DCHECK_LE(order, 0); + if (order != 0) { + run_starts.push_back(static_cast(point)); + } + } + return run_starts; +} + +uint32_t run_length_at(const std::vector& run_starts, size_t point_count, size_t run) { + const size_t end = (run + 1 < run_starts.size()) ? run_starts[run + 1] : point_count; + return static_cast(end - run_starts[run]); +} + +// Bytes the first and last value share. The points are sorted, so a prefix shared +// by those two is shared by every value in between. +uint32_t common_prefix_length(const PointArray& points) { + const uint8_t* first = points.value(0); + const uint8_t* last = points.value(points.point_count - 1); + uint32_t shared = 0; + while (shared < points.bytes_per_dim && first[shared] == last[shared]) { + ++shared; + } + return shared; +} + +LeafValueMode choose_value_mode(const PointArray& points, const std::vector& run_starts, + uint32_t suffix_width) { + if (run_starts.size() == 1) { + // One run is one value, so the shared prefix is the whole value and there + // is nothing left to store per point. + DORIS_CHECK_EQ(suffix_width, 0U); + return LeafValueMode::kAllEqual; + } + // Both candidate value areas are sized EXACTLY rather than estimated -- the + // decision only has to be right for this one leaf, and both sizes are already + // known here. + uint64_t rle_bytes = varint_len(run_starts.size()); + for (size_t run = 0; run < run_starts.size(); ++run) { + rle_bytes += suffix_width + varint_len(run_length_at(run_starts, points.point_count, run)); + } + const uint64_t raw_bytes = static_cast(points.point_count) * suffix_width; + // A tie goes to kRaw: same bytes, and its value area decodes as one + // bounds-checked get_bytes instead of a varint walk. + return (rle_bytes < raw_bytes) ? LeafValueMode::kRle : LeafValueMode::kRaw; +} + +void write_value_area(const PointArray& points, const std::vector& run_starts, + LeafValueMode mode, uint32_t common_prefix_len, ByteSink* sink) { + const uint32_t suffix_width = points.bytes_per_dim - common_prefix_len; + if (mode == LeafValueMode::kRle) { + sink->put_varint32(static_cast(run_starts.size())); + for (size_t run = 0; run < run_starts.size(); ++run) { + sink->put_bytes(Slice(points.value(run_starts[run]) + common_prefix_len, suffix_width)); + sink->put_varint32(run_length_at(run_starts, points.point_count, run)); + } + return; + } + if (mode == LeafValueMode::kRaw) { + for (size_t point = 0; point < points.point_count; ++point) { + sink->put_bytes(Slice(points.value(point) + common_prefix_len, suffix_width)); + } + return; + } + // kAllEqual carries no value area at all: the head's prefix is the value. + DCHECK(mode == LeafValueMode::kAllEqual); +} + +// Fills the PFOR codes. Doc ids ride on the (value, doc_id) sort key: inside a run +// they are non-decreasing, so a run's points collapse to small deltas. kRaw skips +// the delta -- its runs are length 1, so the deltas would be the doc ids. +void build_doc_id_codes(const PointArray& points, const std::vector& run_starts, + LeafValueMode mode, std::vector* codes) { + codes->resize(points.point_count); + if (mode == LeafValueMode::kRaw) { + for (size_t point = 0; point < points.point_count; ++point) { + (*codes)[point] = points.doc_id(point); + } + return; + } + for (size_t run = 0; run < run_starts.size(); ++run) { + const uint32_t first = run_starts[run]; + const uint32_t length = run_length_at(run_starts, points.point_count, run); + uint32_t previous = points.doc_id(first); + (*codes)[first] = previous; + for (uint32_t i = 1; i < length; ++i) { + const uint32_t doc_id = points.doc_id(first + i); + DCHECK_GE(doc_id, previous); + (*codes)[first + i] = doc_id - previous; + previous = doc_id; + } + } +} + +} // namespace + +void encode_leaf_block(Slice records, uint32_t bytes_per_dim, ByteSink* sink) { + DORIS_CHECK(sink != nullptr); + DORIS_CHECK_GT(bytes_per_dim, 0U); + const size_t record_size = static_cast(bytes_per_dim) + kPointDocIdBytes; + DORIS_CHECK_EQ(records.size() % record_size, 0UL); + const PointArray points {records.data(), record_size, bytes_per_dim, + records.size() / record_size}; + // The builder slices leaves off a non-empty sorted stream, so an empty leaf is + // a bug in the caller, not a shape this format has to express. + DORIS_CHECK_GT(points.point_count, 0UL); + DORIS_CHECK_LE(points.point_count, static_cast(std::numeric_limits::max())); + + const std::vector run_starts = scan_runs(points); + const uint32_t common_prefix_len = common_prefix_length(points); + const uint32_t suffix_width = bytes_per_dim - common_prefix_len; + const LeafValueMode mode = choose_value_mode(points, run_starts, suffix_width); + + const size_t block_start = sink->size(); + // Capacity only; a loose upper bound over head + value area + PFOR + tail. + sink->reserve(12 + bytes_per_dim + + points.point_count * (static_cast(bytes_per_dim) + 11)); + + sink->put_varint32(static_cast(points.point_count)); + sink->put_u8(static_cast(mode)); + sink->put_varint32(common_prefix_len); + sink->put_bytes(Slice(points.value(0), common_prefix_len)); + write_value_area(points, run_starts, mode, common_prefix_len, sink); + + const size_t docid_block_offset = sink->size() - block_start; + DORIS_CHECK_LE(docid_block_offset, static_cast(std::numeric_limits::max())); + + std::vector codes; + build_doc_id_codes(points, run_starts, mode, &codes); + pfor_encode(codes.data(), points.point_count, sink); + + // The trailing length byte is what makes the varint reachable from the end of + // the block; see the header for why a bare LEB128 varint is not. + const size_t offset_start = sink->size(); + sink->put_varint32(static_cast(docid_block_offset)); + sink->put_u8(static_cast(sink->size() - offset_start)); +} + +// =========================================================================== +// Decode +// =========================================================================== +namespace { + +// Longest LEB128 encoding of a uint32. +constexpr size_t kMaxVarint32Bytes = 5; + +// Every rejection of leaf bytes funnels through here. A leaf is read lazily and is +// therefore NOT covered by the open-time validation of bkd_index, so these are the +// checks that stand between a damaged file and the query. Disk data is not an +// invariant: none of them may be a DORIS_CHECK, or a recoverable downgrade would +// become a node crash (design 8). +Status leaf_codec_corrupted(std::string_view what) { + return Status::Error("bkd leaf: {}", what); +} + +// The fixed part of a leaf block, validated against the index header. +struct LeafHead { + uint32_t point_count = 0; + LeafValueMode value_mode = LeafValueMode::kAllEqual; + Slice common_prefix; + uint32_t suffix_width = 0; +}; + +Status decode_leaf_head(ByteSource* src, uint32_t bytes_per_dim, uint32_t expected_point_count, + LeafHead* head) { + uint32_t point_count = 0; + RETURN_IF_ERROR(src->get_varint32(&point_count)); + // Design 5.2: the leaf directory already said how many points are here, and it + // was validated at open. Pinning the two together both catches damage and + // bounds every allocation below by trusted metadata -- necessary because an + // all-equal leaf spends zero bytes per point, so block length alone bounds + // nothing. + if (point_count != expected_point_count) { + return leaf_codec_corrupted("point_count disagrees with the leaf directory"); + } + if (point_count == 0) { + return leaf_codec_corrupted("leaf carries no points"); + } + + uint8_t raw_mode = 0; + RETURN_IF_ERROR(src->get_u8(&raw_mode)); + if (raw_mode > static_cast(kMaxLeafValueMode)) { + return leaf_codec_corrupted("unknown value_mode"); + } + const auto value_mode = static_cast(raw_mode); + + uint32_t common_prefix_len = 0; + RETURN_IF_ERROR(src->get_varint32(&common_prefix_len)); + if (common_prefix_len > bytes_per_dim) { + return leaf_codec_corrupted("common_prefix_len exceeds bytes_per_dim"); + } + Slice common_prefix; + RETURN_IF_ERROR(src->get_bytes(common_prefix_len, &common_prefix)); + const uint32_t suffix_width = bytes_per_dim - common_prefix_len; + + // kAllEqual means the prefix IS the value, so it covers the whole width; the + // other two modes need at least one suffix byte to tell their values apart. + // Either mismatch would make the decoded values silently wrong rather than + // merely unparseable. + if ((value_mode == LeafValueMode::kAllEqual) != (suffix_width == 0)) { + return leaf_codec_corrupted("value_mode disagrees with common_prefix_len"); + } + + head->point_count = point_count; + head->value_mode = value_mode; + head->common_prefix = common_prefix; + head->suffix_width = suffix_width; + return Status::OK(); +} + +// Reads the trailing { docid_block_offset varint32, offset_length u8 }. +// `docid_block_end` comes back as the first tail byte, i.e. the doc id block is +// exactly [*docid_block_offset, *docid_block_end). +Status decode_leaf_tail(Slice block, uint32_t* docid_block_offset, size_t* docid_block_end) { + if (block.empty()) { + return leaf_codec_corrupted("leaf block is empty"); + } + ByteSource length_src(block.subslice(block.size() - 1, 1)); + uint8_t length = 0; + RETURN_IF_ERROR(length_src.get_u8(&length)); + if (length == 0 || length > kMaxVarint32Bytes || + static_cast(length) + 1 > block.size()) { + return leaf_codec_corrupted("docid_block_offset length byte is out of range"); + } + + const size_t offset_start = block.size() - 1 - length; + ByteSource offset_src(block.subslice(offset_start, length)); + RETURN_IF_ERROR(offset_src.get_varint32(docid_block_offset)); + if (!offset_src.eof()) { + // The length byte and the varint must describe the same bytes, or the two + // ends of the block disagree about where the doc ids start. + return leaf_codec_corrupted("docid_block_offset is not exactly its declared length"); + } + if (*docid_block_offset > offset_start) { + return leaf_codec_corrupted("docid_block_offset points past the leaf tail"); + } + *docid_block_end = offset_start; + return Status::OK(); +} + +// Decodes the value area into ascending runs of equal values. `runs` is appended +// to (the caller cleared it), so a reused DecodedLeafBlock keeps its capacity. +// +// `runs` is written through, by reserve() and push_back() -- a pointer-to-const +// would not compile. readability-non-const-parameter still claims otherwise +// because clang-tidy cannot locate stddef.h in this toolchain (see the +// clang-diagnostic-error it reports first), so 's members parse into +// recovery nodes and the modification becomes invisible to it. +// NOLINTNEXTLINE(readability-non-const-parameter) +Status read_value_area(ByteSource* src, const LeafHead& head, std::vector* runs) { + if (head.value_mode == LeafValueMode::kAllEqual) { + // No value area at all: the head's prefix is the value, and the whole leaf + // is one run. + runs->push_back(LeafValueRun {Slice(), 0, head.point_count}); + return Status::OK(); + } + + if (head.value_mode == LeafValueMode::kRle) { + uint32_t run_count = 0; + RETURN_IF_ERROR(src->get_varint32(&run_count)); + // Bounded by the directory-pinned point_count before it sizes anything. + if (run_count == 0 || run_count > head.point_count) { + return leaf_codec_corrupted("run_count is out of range"); + } + // Bounded by what is actually left to read, not by point_count alone: a + // ~10-byte kRle leaf can otherwise declare a million runs and reserve 25 MB + // (24 bytes each) before the first suffix read proves the bytes are not + // there. Every run costs at least a suffix plus one doc id byte. + if (run_count > src->remaining() / (static_cast(head.suffix_width) + 1)) { + return leaf_codec_corrupted("run count exceeds the bytes remaining in the leaf"); + } + runs->reserve(run_count); + uint32_t covered = 0; + Slice previous; + for (uint32_t run = 0; run < run_count; ++run) { + Slice suffix; + RETURN_IF_ERROR(src->get_bytes(head.suffix_width, &suffix)); + uint32_t run_length = 0; + RETURN_IF_ERROR(src->get_varint32(&run_length)); + if (run_length == 0) { + return leaf_codec_corrupted("run_len is zero"); + } + // Runs are maximal and ascending. An unordered leaf would not fail to + // parse -- it would silently break the boundary-leaf early exit, which + // stops at the first value past the range -- so the order is checked + // here rather than trusted. + if (run != 0 && std::memcmp(previous.data(), suffix.data(), head.suffix_width) >= 0) { + return leaf_codec_corrupted("run suffixes are not strictly ascending"); + } + if (run_length > head.point_count - covered) { + return leaf_codec_corrupted("run lengths overrun point_count"); + } + runs->push_back(LeafValueRun {suffix, covered, run_length}); + covered += run_length; + previous = suffix; + } + if (covered != head.point_count) { + return leaf_codec_corrupted("run lengths do not sum to point_count"); + } + return Status::OK(); + } + + // kRaw -- the only mode left, decode_leaf_head rejected everything else. + DCHECK(head.value_mode == LeafValueMode::kRaw); + Slice suffixes; + RETURN_IF_ERROR( + src->get_bytes(static_cast(head.point_count) * head.suffix_width, &suffixes)); + runs->reserve(head.point_count); + Slice previous; + for (uint32_t point = 0; point < head.point_count; ++point) { + const Slice suffix = suffixes.subslice(static_cast(point) * head.suffix_width, + head.suffix_width); + // Non-decreasing, not strictly: kRaw is picked on size, so a leaf with a + // few short runs can still land here. + if (point != 0 && std::memcmp(previous.data(), suffix.data(), head.suffix_width) > 0) { + return leaf_codec_corrupted("suffixes are not ascending"); + } + runs->push_back(LeafValueRun {suffix, point, 1}); + previous = suffix; + } + return Status::OK(); +} + +// The whole-leaf-hit variant: steps over the value area instead of decoding it. +// Only kRaw can be skipped outright -- kRle carries the run lengths its doc id +// deltas restart on, and kAllEqual has no value area to skip. +Status skip_value_area(ByteSource* src, const LeafHead& head, std::vector* runs) { + if (head.value_mode == LeafValueMode::kRaw) { + Slice unused; + return src->get_bytes(static_cast(head.point_count) * head.suffix_width, &unused); + } + return read_value_area(src, head, runs); +} + +// Decodes the PFOR block into absolute doc ids. `region` must be exactly the doc +// id bytes; `runs` is unused for kRaw, whose codes already are the doc ids. +Status decode_doc_id_block(Slice region, const LeafHead& head, std::span runs, + std::vector* doc_ids) { + ByteSource src(region); + doc_ids->resize(head.point_count); + RETURN_IF_ERROR(pfor_decode(&src, head.point_count, doc_ids->data())); + if (!src.eof()) { + return leaf_codec_corrupted("trailing bytes between the doc id block and the leaf tail"); + } + if (head.value_mode == LeafValueMode::kRaw) { + return Status::OK(); + } + + // kAllEqual is one run over the leaf, kRle one per value: in both the first + // code of a run is absolute and the rest are deltas off it. + for (const LeafValueRun& run : runs) { + uint64_t doc_id = (*doc_ids)[run.first_point]; + for (uint32_t i = 1; i < run.count; ++i) { + doc_id += (*doc_ids)[run.first_point + i]; + if (doc_id > std::numeric_limits::max()) { + return leaf_codec_corrupted("doc id deltas overflow a 32-bit row id"); + } + (*doc_ids)[run.first_point + i] = static_cast(doc_id); + } + } + return Status::OK(); +} + +} // namespace + +Status decode_leaf_block(Slice block, uint32_t bytes_per_dim, uint32_t expected_point_count, + DecodedLeafBlock* out) { + DORIS_CHECK(out != nullptr); + DORIS_CHECK_GT(bytes_per_dim, 0U); + out->clear(); + + ByteSource src(block); + LeafHead head; + RETURN_IF_ERROR(decode_leaf_head(&src, bytes_per_dim, expected_point_count, &head)); + + uint32_t docid_block_offset = 0; + size_t docid_block_end = 0; + RETURN_IF_ERROR(decode_leaf_tail(block, &docid_block_offset, &docid_block_end)); + + RETURN_IF_ERROR(read_value_area(&src, head, &out->runs)); + // The two ends of the block must agree on where the values stop. Without this + // a damaged offset would let the doc id decoder reinterpret value bytes. + if (src.position() != docid_block_offset) { + return leaf_codec_corrupted("docid_block_offset disagrees with the end of the value area"); + } + RETURN_IF_ERROR(decode_doc_id_block( + block.subslice(docid_block_offset, docid_block_end - docid_block_offset), head, + out->runs, &out->doc_ids)); + + out->value_mode = head.value_mode; + out->common_prefix = head.common_prefix; + out->suffix_width = head.suffix_width; + // Written LAST, so a decode that failed anywhere above leaves a leaf that + // reports zero points rather than one holding half-decoded scratch. + out->point_count = head.point_count; + return Status::OK(); +} + +Status decode_leaf_doc_ids(Slice block, uint32_t bytes_per_dim, uint32_t expected_point_count, + std::vector* doc_ids) { + DORIS_CHECK(doc_ids != nullptr); + DORIS_CHECK_GT(bytes_per_dim, 0U); + + ByteSource src(block); + LeafHead head; + RETURN_IF_ERROR(decode_leaf_head(&src, bytes_per_dim, expected_point_count, &head)); + + uint32_t docid_block_offset = 0; + size_t docid_block_end = 0; + RETURN_IF_ERROR(decode_leaf_tail(block, &docid_block_offset, &docid_block_end)); + + std::vector runs; + RETURN_IF_ERROR(skip_value_area(&src, head, &runs)); + if (src.position() != docid_block_offset) { + return leaf_codec_corrupted("docid_block_offset disagrees with the end of the value area"); + } + return decode_doc_id_block( + block.subslice(docid_block_offset, docid_block_end - docid_block_offset), head, runs, + doc_ids); +} + +} // namespace doris::snii::bkd diff --git a/be/src/storage/index/snii/bkd/leaf_codec.h b/be/src/storage/index/snii/bkd/leaf_codec.h new file mode 100644 index 00000000000000..b2e4f62340230e --- /dev/null +++ b/be/src/storage/index/snii/bkd/leaf_codec.h @@ -0,0 +1,179 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include "common/status.h" +#include "storage/index/snii/bkd/bkd_format.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/encoding/byte_sink.h" + +// bkd_data leaf blocks -- the COLD sub-file of the SNII-native BKD index +// (design 5.2). One block per leaf, self-contained, concatenated in leaf order; +// the bkd_index leaf directory says where each one starts and how many points it +// holds. +// +// point_count varint32 points in this leaf; must equal the count the +// leaf directory records, else the block is corrupt +// value_mode u8 LeafValueMode +// common_prefix_len varint32 0 .. bytes_per_dim +// common_prefix bytes[common_prefix_len] +// --- value area, with S = bytes_per_dim - common_prefix_len --- +// kAllEqual : nothing. common_prefix_len == bytes_per_dim, i.e. the prefix IS +// the one value every point in the leaf carries. +// kRle : run_count varint32, then run_count x { suffix bytes[S], run_len varint32 } +// kRaw : point_count x suffix bytes[S] +// --- doc ids --- +// docid_block one PFOR block (encoding/pfor.h) of point_count uint32 codes +// docid_block_offset varint32 block-relative offset of the docid block +// offset_length u8 byte length of the varint immediately above +// +// The doc id CODES exploit a property the old implementation had but never used: +// the build-time sort key is (value, doc_id) (see kPointDocIdBytes), so doc ids +// ascend inside every run of equal values. Hence +// kAllEqual -> one ascending delta run over the whole leaf (code 0 is absolute) +// kRle -> deltas restarting at each run (each run's first code is absolute) +// kRaw -> the doc ids themselves; values are nearly all distinct there, so +// runs are length 1 and a delta would only add work +// All three are ONE pfor block of point_count codes, so there is no equivalent of +// the old five-way bpv dispatch (one branch of which was write-never/read-only). +// +// WHY docid_block_offset SITS AT THE TAIL (design 7.2): a whole-leaf hit wants the +// doc ids without paying for the value area, and the value area comes first +// because the only readers that decode values are the at most two boundary leaves +// of a range, while whole-leaf hits can number in the thousands. The trailing +// offset_length byte is what makes the varint reachable from the end: a bare +// LEB128 varint cannot be scanned backwards (the last byte of the doc id block may +// itself have bit 7 set, so the varint's start is ambiguous). Note that a reader +// still has to parse the head for value_mode and point_count, and that a kRle leaf +// additionally has to walk the run lengths its doc id deltas restart on -- the +// offset removes the value BYTES from the whole-leaf path, not the head. +// +// ENCODE AND DECODE ARE TWO INDEPENDENT FREE FUNCTIONS sharing nothing but the +// constants in bkd_format.h (design 4). The old docids_writer declared both +// directions on one class, which is how the writer TU ended up transitively +// including the entire read side. +namespace doris::snii::bkd { + +// --------------------------------------------------------------------------- +// Encode +// --------------------------------------------------------------------------- + +// Encodes one leaf and APPENDS the block to `sink` (`sink` is not cleared). +// +// `records` is the build-time point array (design 6.2): point_count fixed-width +// records of [value: bytes_per_dim][doc_id: kPointDocIdBytes big-endian], already +// sorted by the memcmp of the whole record, which is exactly (value, doc_id) +// order. This is the builder's own buffer, so no PointRef array is materialized +// per leaf. +// +// Every argument is a BUILD-TIME INVARIANT -- the builder produced all of it in +// this same run -- so violations trip DORIS_CHECK (DCHECK for the per-point ones) +// rather than returning a Status (design 8). Untrusted bytes only ever enter +// through the decode functions below. +// +// sink : non-null +// bytes_per_dim: > 0, and records.size() a whole multiple of the record size +// point_count : >= 1 (the builder never emits an empty leaf) +// ordering : records non-decreasing under memcmp. Equal ADJACENT records are +// legal: an array column may repeat one value inside one row, and +// that pair encodes as a zero doc id delta. +void encode_leaf_block(Slice records, uint32_t bytes_per_dim, ByteSink* sink); + +// --------------------------------------------------------------------------- +// Decode +// --------------------------------------------------------------------------- + +// One maximal run of equal values inside a decoded leaf. kRle stores runs +// explicitly; kAllEqual is one run over the whole leaf; kRaw reports one run per +// point (its values are nearly all distinct, and merging the occasional pair would +// cost a memcmp per point on the boundary-leaf path to save nothing). +struct LeafValueRun { + // The run's value is common_prefix ++ suffix. A VIEW into the block bytes + // passed to the decoder -- it does not own them. Empty exactly when the common + // prefix already covers the whole value (kAllEqual). + Slice suffix; + // Index of the run's first point in the leaf. doc_ids[first_point, + // first_point + count) belong to this run and are non-decreasing. + uint32_t first_point = 0; + uint32_t count = 0; +}; + +// A decoded leaf. Reused across leaves by the query path: the vectors keep their +// capacity, so a scan over many leaves does not re-allocate per leaf. +// +// The Slices are VIEWS into the block handed to decode_leaf_block and are only +// valid while those bytes are. +struct DecodedLeafBlock { + // Set LAST, so a failed decode leaves it 0 and the partially filled arrays + // below are unusable scratch rather than plausible-looking data. + uint32_t point_count = 0; + LeafValueMode value_mode = LeafValueMode::kAllEqual; + // common_prefix_len bytes shared by every value in the leaf. + Slice common_prefix; + // bytes_per_dim - common_prefix.size(); the width of every suffix in `runs`. + uint32_t suffix_width = 0; + // Ascending by value, covering [0, point_count) with no gap and no overlap. + std::vector runs; + // point_count doc ids in point order (i.e. in (value, doc_id) order). + std::vector doc_ids; + + void clear() { + point_count = 0; + value_mode = LeafValueMode::kAllEqual; + common_prefix = Slice(); + suffix_width = 0; + runs.clear(); + doc_ids.clear(); + } +}; + +// Decodes one leaf block: values as prefix + runs of suffixes, plus every doc id. +// This is the boundary-leaf path of a range query, the only one that looks at +// values at all. +// +// `block` must be exactly the leaf's bytes, which the reader knows from the leaf +// directory (leaf i + 1's offset, or the bkd_data length for the last leaf). +// `bytes_per_dim` and `expected_point_count` come from the already-validated +// bkd_index: the count both implements design 5.2's "a point_count disagreeing +// with the directory is corruption" rule and bounds the decode allocation -- a +// leaf whose values are all equal spends zero bytes per point, so without that +// bound a damaged point_count could drive a multi-gigabyte resize. +// +// Leaf blocks are read lazily and are NOT covered by the open-time validation, so +// every field here is checked as it is decoded (design 8.3). Disk bytes are not +// invariants: every rejection is Status INVERTED_INDEX_FILE_CORRUPTED, never a +// DORIS_CHECK, so a damaged leaf downgrades the query instead of killing the node. +Status decode_leaf_block(Slice block, uint32_t bytes_per_dim, uint32_t expected_point_count, + DecodedLeafBlock* out); + +// Decodes ONLY the doc ids, stepping over the value area via the tail +// docid_block_offset. This is the whole-leaf-hit path (design 7.2): every leaf +// strictly between the two boundary leaves of a range contributes all of its doc +// ids and none of its values. +// +// Same arguments, same corruption contract, and the same doc ids as +// decode_leaf_block would produce for the same block -- including the validation +// that the tail offset agrees with where the value area actually ends, so a +// damaged offset cannot silently reinterpret value bytes as doc ids. +Status decode_leaf_doc_ids(Slice block, uint32_t bytes_per_dim, uint32_t expected_point_count, + std::vector* doc_ids); + +} // namespace doris::snii::bkd diff --git a/be/src/storage/index/snii/bkd/point_merger.cpp b/be/src/storage/index/snii/bkd/point_merger.cpp new file mode 100644 index 00000000000000..5a69176f5294e3 --- /dev/null +++ b/be/src/storage/index/snii/bkd/point_merger.cpp @@ -0,0 +1,97 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/bkd/point_merger.h" + +#include +#include + +#include "common/check.h" + +namespace doris::snii::bkd { + +MergingPointSource::MergingPointSource(uint32_t record_size, uint32_t block_records) + : record_size_(record_size), block_records_(block_records) {} + +Status MergingPointSource::create(const std::vector& run_paths, uint32_t record_size, + uint32_t block_records, uint32_t buffer_records_per_run, + std::unique_ptr* out) { + DORIS_CHECK(out != nullptr); + DORIS_CHECK_GT(record_size, 0U); + DORIS_CHECK_GT(block_records, 0U); + DORIS_CHECK_GT(buffer_records_per_run, 0U); + + std::unique_ptr source(new MergingPointSource(record_size, block_records)); + for (const std::string& path : run_paths) { + auto reader = std::make_unique(); + RETURN_IF_ERROR(reader->open(path, record_size, buffer_records_per_run)); + // An empty run never enters the heap. Doing that here rather than at pop + // time is what keeps the heap invariant "every member has a current + // record", so sorts_after never has to ask whether one exists. + if (!reader->exhausted()) { + source->heap_.push_back(static_cast(source->readers_.size())); + } + source->readers_.push_back(std::move(reader)); + } + + MergingPointSource* raw = source.get(); + std::make_heap(raw->heap_.begin(), raw->heap_.end(), + [raw](uint32_t a, uint32_t b) { return raw->sorts_after(a, b); }); + *out = std::move(source); + return Status::OK(); +} + +bool MergingPointSource::sorts_after(uint32_t a, uint32_t b) const { + const Slice left = readers_[a]->current(); + const Slice right = readers_[b]->current(); + // Fixed width, so one memcmp IS the (value, doc_id) order. Ties are records + // that are byte-identical, and which copy is emitted first cannot be + // observed -- so no tiebreak on the run index is needed for determinism of + // the OUTPUT, only for that of the heap, which nothing depends on. + return std::memcmp(left.data(), right.data(), record_size_) > 0; +} + +Status MergingPointSource::next_block(uint32_t max_points, Slice* records) { + DORIS_CHECK(records != nullptr); + DORIS_CHECK_GT(max_points, 0U); + + const auto after = [this](uint32_t a, uint32_t b) { return sorts_after(a, b); }; + const size_t wanted = std::min(max_points, block_records_); + block_.clear(); + block_.reserve(wanted * record_size_); + + while (block_.size() / record_size_ < wanted && !heap_.empty()) { + std::pop_heap(heap_.begin(), heap_.end(), after); + const uint32_t run = heap_.back(); + const Slice record = readers_[run]->current(); + block_.insert(block_.end(), record.data(), record.data() + record.size()); + + RETURN_IF_ERROR(readers_[run]->advance()); + if (readers_[run]->exhausted()) { + heap_.pop_back(); + } else { + // The cursor moved, so this run's key changed; re-establishing the + // heap from the back is what push_heap is for. + std::push_heap(heap_.begin(), heap_.end(), after); + } + } + + *records = Slice(block_.data(), block_.size()); + return Status::OK(); +} + +} // namespace doris::snii::bkd diff --git a/be/src/storage/index/snii/bkd/point_merger.h b/be/src/storage/index/snii/bkd/point_merger.h new file mode 100644 index 00000000000000..7b72c6c1f0bcb1 --- /dev/null +++ b/be/src/storage/index/snii/bkd/point_merger.h @@ -0,0 +1,82 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/bkd/point_run.h" +#include "storage/index/snii/bkd/point_source.h" +#include "storage/index/snii/common/slice.h" + +// The spilling half of design 6.2: a k-way merge over spilled runs, presented as +// an ordinary PointSource so the leaf-cutting loop cannot tell the two build +// modes apart. +namespace doris::snii::bkd { + +class MergingPointSource final : public PointSource { +public: + // Opens one cursor per run. The caller keeps ownership of the files, + // including their removal -- this type never unlinks anything, so a failed + // build leaves the runs where a human can look at them. + // + // Resident footprint is (runs x buffer_records_per_run + block_records) + // records, which is what keeps the merge bounded independently of how much + // was spilled. + static Status create(const std::vector& run_paths, uint32_t record_size, + uint32_t block_records, uint32_t buffer_records_per_run, + std::unique_ptr* out); + + ~MergingPointSource() override = default; + + Status next_block(uint32_t max_points, Slice* records) override; + + // Summed over every open cursor: the merge's actual resident footprint, + // excluding the leaf block which is accounted separately. This is a + // MEASUREMENT -- recomputing it from create()'s arguments would only restate + // the request, which is exactly what the bound needs to be checked against. + uint64_t resident_buffer_bytes() const { + uint64_t total = 0; + for (const auto& reader : readers_) { + total += reader->resident_buffer_bytes(); + } + return total; + } + +private: + MergingPointSource(uint32_t record_size, uint32_t block_records); + + // True when the record under cursor `a` sorts AFTER the one under `b`, which + // is what std::push_heap/pop_heap need to behave as a min-heap. Records are + // fixed width, so one memcmp IS the (value, doc_id) order. + bool sorts_after(uint32_t a, uint32_t b) const; + + uint32_t record_size_; + uint32_t block_records_; + std::vector> readers_; + // Indices into readers_, kept as a min-heap over the record each one is + // positioned on. An exhausted reader is simply absent. + std::vector heap_; + // The contiguous view handed out by next_block, valid until the next call. + std::vector block_; +}; + +} // namespace doris::snii::bkd diff --git a/be/src/storage/index/snii/bkd/point_run.cpp b/be/src/storage/index/snii/bkd/point_run.cpp new file mode 100644 index 00000000000000..3764ada5e81e02 --- /dev/null +++ b/be/src/storage/index/snii/bkd/point_run.cpp @@ -0,0 +1,153 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/bkd/point_run.h" + +#include +#include + +#include +#include + +#include "common/check.h" + +namespace doris::snii::bkd { + +PointRunWriter::~PointRunWriter() { + if (fd_ >= 0) { + // Nothing to report to: a run whose close failed is a run the merge will + // fail to read, and that failure is the one worth surfacing. + ::close(fd_); + fd_ = -1; + } +} + +Status PointRunWriter::open(const std::string& path) { + DORIS_CHECK_LT(fd_, 0); // one run per writer + const int fd = ::open(path.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0600); + if (fd < 0) { + return Status::IOError("failed to create spilled point run {}: {}", path, + std::strerror(errno)); + } + fd_ = fd; + return Status::OK(); +} + +Status PointRunWriter::append(Slice records) { + DORIS_CHECK_GE(fd_, 0); + const uint8_t* cursor = records.data(); + size_t remaining = records.size(); + while (remaining > 0) { + const ssize_t written = ::write(fd_, cursor, remaining); + if (written < 0) { + if (errno == EINTR) { + continue; + } + return Status::IOError("failed to write a spilled point run: {}", std::strerror(errno)); + } + // A short write is normal for a large buffer; only a zero-length one on a + // non-empty request would be a stall, and write(2) does not do that. + cursor += written; + remaining -= static_cast(written); + } + return Status::OK(); +} + +Status PointRunWriter::close() { + DORIS_CHECK_GE(fd_, 0); + const int fd = fd_; + fd_ = -1; + if (::close(fd) != 0) { + // Deferred write errors surface here, so this cannot be ignored: the run + // would read back short and the merge would silently drop points. + return Status::IOError("failed to close a spilled point run: {}", std::strerror(errno)); + } + return Status::OK(); +} + +PointRunReader::~PointRunReader() { + if (fd_ >= 0) { + ::close(fd_); + fd_ = -1; + } +} + +Status PointRunReader::open(const std::string& path, uint32_t record_size, + uint32_t buffer_records) { + DORIS_CHECK_LT(fd_, 0); + DORIS_CHECK_GT(record_size, 0U); + DORIS_CHECK_GT(buffer_records, 0U); + + const int fd = ::open(path.c_str(), O_RDONLY); + if (fd < 0) { + // Not a silently empty stream: swallowing this would drop a whole run + // from the merge and yield an index that is short with no error anywhere. + return Status::IOError("failed to open spilled point run {}: {}", path, + std::strerror(errno)); + } + fd_ = fd; + record_size_ = record_size; + buffer_.resize(static_cast(record_size) * buffer_records); + return fill(); +} + +Slice PointRunReader::current() const { + // Reading past the end is a caller bug -- exhausted() is right there -- so it + // crashes rather than returning something a merge would happily compare. + DORIS_CHECK_LE(cursor_ + record_size_, valid_bytes_); + return {buffer_.data() + cursor_, record_size_}; +} + +Status PointRunReader::advance() { + DORIS_CHECK_LE(cursor_ + record_size_, valid_bytes_); + cursor_ += record_size_; + if (cursor_ >= valid_bytes_ && !eof_) { + return fill(); + } + return Status::OK(); +} + +Status PointRunReader::fill() { + valid_bytes_ = 0; + cursor_ = 0; + while (valid_bytes_ < buffer_.size()) { + const ssize_t bytes = + ::read(fd_, buffer_.data() + valid_bytes_, buffer_.size() - valid_bytes_); + if (bytes < 0) { + if (errno == EINTR) { + continue; + } + return Status::IOError("failed to read a spilled point run: {}", std::strerror(errno)); + } + if (bytes == 0) { + eof_ = true; + break; + } + valid_bytes_ += static_cast(bytes); + } + // Records are fixed width and the writer only ever appends whole ones, so a + // partial tail means the file was truncated after it was written -- a temp + // dir swept from under us, or a full disk that surfaced late. It is reported + // rather than asserted because it is reachable without any bug of ours. + if (valid_bytes_ % record_size_ != 0) { + return Status::IOError("spilled point run ends mid-record ({} bytes, record size {})", + valid_bytes_, record_size_); + } + return Status::OK(); +} + +} // namespace doris::snii::bkd diff --git a/be/src/storage/index/snii/bkd/point_run.h b/be/src/storage/index/snii/bkd/point_run.h new file mode 100644 index 00000000000000..82353f6326330a --- /dev/null +++ b/be/src/storage/index/snii/bkd/point_run.h @@ -0,0 +1,99 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/common/slice.h" + +// One spilled RUN of build-time point records (design 6.2). +// +// A run is written once, in full, already sorted, and read back exactly once by +// the k-way merge. That is the whole lifecycle, and it is why these two types +// are deliberately smaller than a general file abstraction: no seeking, no +// random access, no re-reads. +// +// Records are FIXED WIDTH (value bytes followed by a big-endian doc id), so the +// run needs no framing of its own -- a record boundary is arithmetic, not a +// delimiter -- and the reader's buffer can be sized in whole records. +namespace doris::snii::bkd { + +// Append-only sink for one run. The caller owns the path and its removal; this +// type owns only the descriptor. +class PointRunWriter { +public: + PointRunWriter() = default; + ~PointRunWriter(); + + PointRunWriter(const PointRunWriter&) = delete; + PointRunWriter& operator=(const PointRunWriter&) = delete; + + Status open(const std::string& path); + // `records` is a whole number of records; partial writes are retried until + // the slice is on disk. + Status append(Slice records); + // Flushes and releases the descriptor. Safe to call once; the destructor + // closes an un-closed descriptor without reporting. + Status close(); + +private: + int fd_ = -1; +}; + +// Forward-only cursor over one run, holding at most `buffer_records` records +// resident. Sizing the cursor in records rather than bytes is what keeps the +// merge's total footprint a function of (run count x buffer_records), which is +// the bound design 6.2 promises. +class PointRunReader { +public: + PointRunReader() = default; + ~PointRunReader(); + + PointRunReader(const PointRunReader&) = delete; + PointRunReader& operator=(const PointRunReader&) = delete; + + // Positions the cursor on the first record, so current() is valid + // immediately unless the run is empty. + Status open(const std::string& path, uint32_t record_size, uint32_t buffer_records); + + bool exhausted() const { return cursor_ >= valid_bytes_ && eof_; } + // The record under the cursor. Valid until the next advance(). Calling this + // on an exhausted cursor is a caller bug, not a runtime condition. + Slice current() const; + Status advance(); + + // Bytes this cursor actually holds resident. Reported rather than recomputed + // from the open() arguments, so a memory bound can be checked against what + // was allocated instead of against the arithmetic that asked for it. + uint64_t resident_buffer_bytes() const { return buffer_.size(); } + +private: + Status fill(); + + int fd_ = -1; + uint32_t record_size_ = 0; + std::vector buffer_; + size_t valid_bytes_ = 0; + size_t cursor_ = 0; + bool eof_ = false; +}; + +} // namespace doris::snii::bkd diff --git a/be/src/storage/index/snii/bkd/point_sorter.cpp b/be/src/storage/index/snii/bkd/point_sorter.cpp new file mode 100644 index 00000000000000..6fe3cdc86e3fe0 --- /dev/null +++ b/be/src/storage/index/snii/bkd/point_sorter.cpp @@ -0,0 +1,170 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/bkd/point_sorter.h" + +#include +#include +#include + +#include "common/check.h" + +namespace doris::snii::bkd::point_sorter { +namespace { + +// Buckets at or below this size go to the comparison fallback: under it a radix +// pass spends more time zeroing and walking a 256-entry histogram than a +// comparison sort spends on the whole bucket. Purely a tuning knob -- the sorted +// output is byte-identical either way and nothing on disk depends on it. +constexpr size_t kIntroThreshold = 64; + +// One bucket per byte value. There is no extra "key exhausted" bucket the way a +// variable-length radix sort needs: every record is exactly record_size bytes +// (INV-2), so at any level every record still has a byte. +constexpr size_t kBucketCount = 256; + +uint8_t* record_at(uint8_t* records, size_t index, uint32_t record_size) { + return records + index * static_cast(record_size); +} + +// Comparison fallback for one small bucket. Everything in the bucket already +// shares bytes [0, level) -- that is what put it in the same bucket -- so the +// compare starts at `level` instead of re-reading the common prefix per compare. +void sort_small(uint8_t* records, size_t count, uint32_t record_size, uint32_t level) { + DCHECK_LE(count, kIntroThreshold); + // std::sort needs a movable element, which a fixed-width slice of a byte + // array is not, so what gets sorted is the bucket's index permutation. It is + // bounded by the threshold, hence a stack array rather than an allocation. + std::array order; + for (size_t i = 0; i < count; ++i) { + order[i] = static_cast(i); + } + const size_t width = record_size - level; + std::sort(order.begin(), order.begin() + count, [&](uint32_t lhs, uint32_t rhs) { + return std::memcmp(record_at(records, lhs, record_size) + level, + record_at(records, rhs, record_size) + level, width) < 0; + }); + + // Apply the permutation in place -- no scratch copy of the bucket, per the + // in-place requirement in the header. order[target] is where the record that + // belongs at `target` STARTED. Positions below `target` are already final, so + // a source below it has since been overwritten; following the chain through + // those settled positions lands on wherever that record was displaced to. + // Records that compare equal are byte-identical (the whole record is the key), + // so which of them the chain picks cannot be observed. + for (size_t target = 0; target < count; ++target) { + size_t source = order[target]; + while (source < target) { + source = order[source]; + } + if (source != target) { + std::swap_ranges(record_at(records, target, record_size), + record_at(records, target + 1, record_size), + record_at(records, source, record_size)); + } + } +} + +// MSB radix sort of one bucket, distinguishing records from byte `level` on. +// +// The single-bucket case loops instead of recursing, so a long common prefix (all +// eight value bytes equal, which is the norm for a leaf of one repeated value) +// descends without a stack frame per byte. Only a real split recurses, which caps +// the live frame count at record_size. +void radix_sort(uint8_t* records, size_t count, uint32_t record_size, uint32_t level) { + while (true) { + if (count <= 1) { + return; + } + if (level == record_size) { + // Every byte of every record here compared equal, so they are + // byte-identical and the arrangement they are in already IS sorted. + return; + } + if (count <= kIntroThreshold) { + sort_small(records, count, record_size, level); + return; + } + + // bucket_start[b] .. bucket_start[b + 1] is where bucket b ends up. + // Counting one slot to the right lets the prefix sum run in place, so the + // per-level stack cost is this array plus the cursors and nothing else. + size_t bucket_start[kBucketCount + 1] = {}; + for (size_t i = 0; i < count; ++i) { + ++bucket_start[record_at(records, i, record_size)[level] + 1]; + } + const uint8_t first_byte = record_at(records, 0, record_size)[level]; + if (bucket_start[first_byte + 1] == count) { + ++level; + continue; + } + for (size_t bucket = 0; bucket < kBucketCount; ++bucket) { + bucket_start[bucket + 1] += bucket_start[bucket]; + } + + // American flag sort. cursor[b] is the next unfilled slot of bucket b, so + // every swap puts one record where it will stay: the pass costs at most + // `count` swaps and, unlike a counting sort, no second copy of the array. + size_t cursor[kBucketCount]; + for (size_t bucket = 0; bucket < kBucketCount; ++bucket) { + cursor[bucket] = bucket_start[bucket]; + } + for (size_t bucket = 0; bucket < kBucketCount; ++bucket) { + const size_t bucket_end = bucket_start[bucket + 1]; + while (cursor[bucket] < bucket_end) { + uint8_t* record = record_at(records, cursor[bucket], record_size); + const uint8_t target = record[level]; + if (target == bucket) { + ++cursor[bucket]; + continue; + } + // This record belongs to `target` yet is sitting outside it, so + // bucket `target` cannot be full: the swap always has a slot. + DCHECK_LT(cursor[target], bucket_start[target + 1]); + std::swap_ranges(record, record + record_size, + record_at(records, cursor[target], record_size)); + ++cursor[target]; + // cursor[bucket] deliberately does not advance: the record just + // swapped in is unplaced and gets classified on the next turn. + } + } + + for (size_t bucket = 0; bucket < kBucketCount; ++bucket) { + const size_t bucket_size = bucket_start[bucket + 1] - bucket_start[bucket]; + if (bucket_size > 1) { + radix_sort(record_at(records, bucket_start[bucket], record_size), bucket_size, + record_size, level + 1); + } + } + return; + } +} + +} // namespace + +void sort(uint8_t* records, size_t count, uint32_t record_size) { + DORIS_CHECK_GT(record_size, 0U); + if (count <= 1) { + // Nothing to permute. This precedes the pointer assertion because an + // empty run legitimately has no buffer to point at. + return; + } + DORIS_CHECK(records != nullptr); + radix_sort(records, count, record_size, 0); +} + +} // namespace doris::snii::bkd::point_sorter diff --git a/be/src/storage/index/snii/bkd/point_sorter.h b/be/src/storage/index/snii/bkd/point_sorter.h new file mode 100644 index 00000000000000..cdbf1dbd667182 --- /dev/null +++ b/be/src/storage/index/snii/bkd/point_sorter.h @@ -0,0 +1,66 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +// Ordering for the builder's fixed-width point records (design 6.3). +// +// A build-time record is [value: bytes_per_dim][doc_id: kPointDocIdBytes +// BIG-endian] and the order is the memcmp of the WHOLE record. Because the doc id +// tail is big-endian, that memcmp IS lexicographic (value, doc_id) order, which is +// why this function takes a record WIDTH and never learns where the value ends: +// the field boundary is not needed to compare, and (value, doc_id) becomes the +// sort key by construction rather than by a separate tie-break rule. leaf_codec +// then depends on the consequence -- doc ids ascend inside every run of equal +// values -- so this file and that one agree through the record layout alone. +// +// A FREE FUNCTION WITH NO STATE, deliberately. The old MSBRadixSorter was an +// abstract class whose inner IntroSorter held a shared_ptr; that +// forced bkd_writer to derive from enable_shared_from_this, and the requirement +// propagated all the way out to InvertedIndexColumnWriter::_bkd_writer having to +// be a shared_ptr. Nothing here can impose an ownership model on a caller: there +// is no object to own. +// +// Everything the sorter sees is data the builder produced in this same run, never +// disk bytes, so its preconditions are internal invariants (DORIS_CHECK) and not +// Status returns -- the corruption contract of design 8 applies to the decode +// side, which this file is not part of. +namespace doris::snii::bkd::point_sorter { + +// Sorts `count` records of `record_size` bytes each, ascending by the unsigned +// byte-wise comparison of the whole record, IN PLACE. +// +// In place is a requirement, not an implementation note: a run buffer is sized by +// BkdBuilderOptions::build_buffer_bytes (256 MB by default), and design 6.2 bounds +// build RSS by that figure, so sorting must not need a second buffer of the same +// size. The implementation is an MSB radix sort permuting the array bucket by +// bucket, falling back to a comparison sort once a bucket is small; peak auxiliary +// memory is one histogram per byte position still being distinguished. +// +// NOT stable, and stability is unobservable here: the whole record is the key, so +// two records that compare equal are byte-identical and no permutation of them can +// be told apart. Exact duplicates are legal input (an array column may repeat one +// value inside one row) and come back with the same multiplicity. +// +// `records` must hold count * record_size bytes and `record_size` must be +// non-zero; both are guaranteed by the builder that owns the buffer. +void sort(uint8_t* records, size_t count, uint32_t record_size); + +} // namespace doris::snii::bkd::point_sorter diff --git a/be/src/storage/index/snii/bkd/point_source.h b/be/src/storage/index/snii/bkd/point_source.h new file mode 100644 index 00000000000000..80771c77649da0 --- /dev/null +++ b/be/src/storage/index/snii/bkd/point_source.h @@ -0,0 +1,62 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include "common/status.h" +#include "storage/index/snii/common/slice.h" + +// Where the builder's fully ordered point stream comes from (design 6.2). +// +// Leaf cutting is the SAME code in both build modes: consume the ordered stream, +// slice off points_per_leaf points at a time, encode a leaf (design 6.4). Only the +// upstream differs -- Phase 1 has one resident, freshly sorted run; Phase 2 adds a +// k-way merge over spilled runs. Naming that seam now is what lets Phase 2 be a NEW +// implementation of this interface rather than an edit to BkdBuilder::finish. +namespace doris::snii::bkd { + +// A forward-only cursor over build-time point records, ordered by the memcmp of the +// whole record, i.e. by (value, doc_id) (see kPointDocIdBytes). +// +// Everything a source produces was produced by the builder in this same run, so its +// preconditions are internal invariants (DORIS_CHECK). The Status return exists for +// the Phase 2 merge, whose refills read spilled runs back from disk and can fail on +// IO -- corruption of an index FILE is not in scope here, that contract belongs to +// the decode side (design 8). +class PointSource { +public: + virtual ~PointSource() = default; + + PointSource(const PointSource&) = delete; + PointSource& operator=(const PointSource&) = delete; + + // Hands back the next run of at most `max_points` CONSECUTIVE records as one + // contiguous view, which is exactly the shape encode_leaf_block consumes -- no + // per-leaf PointRef array is ever materialized. + // + // The view is owned by the source and stays valid only until the next call. + // Fewer than `max_points` records come back only when the stream runs out; an + // EMPTY slice means exhausted, and every later call returns empty again. + virtual Status next_block(uint32_t max_points, Slice* records) = 0; + +protected: + PointSource() = default; +}; + +} // namespace doris::snii::bkd diff --git a/be/src/storage/index/snii/bkd/staged_blob_file.cpp b/be/src/storage/index/snii/bkd/staged_blob_file.cpp new file mode 100644 index 00000000000000..7b63e024b1f8e7 --- /dev/null +++ b/be/src/storage/index/snii/bkd/staged_blob_file.cpp @@ -0,0 +1,142 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/bkd/staged_blob_file.h" + +#include +#include + +#include +#include +#include + +#include "common/check.h" +#include "storage/index/snii/writer/temp_dir.h" + +namespace doris::snii::bkd { + +Status StagedBlobFile::create(const std::string& tag, std::unique_ptr* out) { + DORIS_CHECK(out != nullptr); + // pid plus a process-wide counter: two builds in one BE must not collide, and + // a leftover from a dead process must not be mistaken for ours. + static std::atomic sequence {0}; + std::string path = writer::resolve_temp_dir() + "/snii_bkdstage_" + tag + "_" + + std::to_string(::getpid()) + "_" + std::to_string(sequence.fetch_add(1)) + + ".stage"; + + // O_EXCL: the name is supposed to be unique, so an existing file means the + // assumption is wrong and silently truncating someone else's staging file + // would be the worst possible response. + const int fd = ::open(path.c_str(), O_RDWR | O_CREAT | O_EXCL, 0600); + if (fd < 0) { + return Status::IOError("failed to create blob staging file {}: {}", path, + std::strerror(errno)); + } + std::unique_ptr file(new StagedBlobFile()); + file->fd_ = fd; + file->path_ = std::move(path); + *out = std::move(file); + return Status::OK(); +} + +StagedBlobFile::~StagedBlobFile() { + remove(); +} + +Status StagedBlobFile::append(Slice data) { + DORIS_CHECK_GE(fd_, 0); + DORIS_CHECK(!finalized_); + const uint8_t* cursor = data.data(); + size_t remaining = data.size(); + while (remaining > 0) { + const ssize_t written = ::write(fd_, cursor, remaining); + if (written < 0) { + if (errno == EINTR) { + continue; + } + return Status::IOError("failed to write a blob staging file: {}", std::strerror(errno)); + } + cursor += written; + remaining -= static_cast(written); + } + bytes_written_ += data.size(); + return Status::OK(); +} + +Status StagedBlobFile::finalize() { + DORIS_CHECK_GE(fd_, 0); + DORIS_CHECK(!finalized_); + // The descriptor stays open on purpose: read_at reads through this same one, + // so the file survives an unlink and cannot be swapped out from under us. + // A deferred write error would surface at close(), which happens in remove() + // after the container has already sealed -- so force it out here instead. + if (::fsync(fd_) != 0) { + return Status::IOError("failed to flush a blob staging file: {}", std::strerror(errno)); + } + finalized_ = true; + return Status::OK(); +} + +Status StagedBlobFile::read_at(uint64_t offset, size_t len, uint8_t* out) const { + DORIS_CHECK_GE(fd_, 0); + DORIS_CHECK(finalized_); + if (len == 0) { + return Status::OK(); + } + DORIS_CHECK(out != nullptr); + // Reported, not asserted: the extent comes from the blob file table the + // container is assembling, and a mismatch there must not take the process + // down. Written as a subtraction so a huge len cannot wrap the sum. + if (offset > bytes_written_ || len > bytes_written_ - offset) { + return Status::IOError("blob staging read [{}, +{}) is outside the staged {} bytes", offset, + len, bytes_written_); + } + + size_t filled = 0; + while (filled < len) { + const ssize_t bytes = + ::pread(fd_, out + filled, len - filled, static_cast(offset + filled)); + if (bytes < 0) { + if (errno == EINTR) { + continue; + } + return Status::IOError("failed to read a blob staging file: {}", std::strerror(errno)); + } + if (bytes == 0) { + // The bound above already proved these bytes exist, so EOF here means + // the file was truncated under us. Never a short success: the caller + // checksums this buffer. + return Status::IOError("blob staging file is shorter than the {} bytes it staged", + bytes_written_); + } + filled += static_cast(bytes); + } + return Status::OK(); +} + +void StagedBlobFile::remove() { + if (fd_ >= 0) { + ::close(fd_); + fd_ = -1; + } + if (!path_.empty()) { + ::unlink(path_.c_str()); + path_.clear(); + } +} + +} // namespace doris::snii::bkd diff --git a/be/src/storage/index/snii/bkd/staged_blob_file.h b/be/src/storage/index/snii/bkd/staged_blob_file.h new file mode 100644 index 00000000000000..8d7d67fc29a835 --- /dev/null +++ b/be/src/storage/index/snii/bkd/staged_blob_file.h @@ -0,0 +1,83 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/io/file_writer.h" + +// Build-time staging for one blob sub-file (design 10). +// +// The container is a PULL consumer: SniiCompoundWriter::add_blob_index registers +// a BlobFileSource and only asks for the bytes at finish(), because placement +// (cold before the metadata groups, hot after) is the container's decision, not +// the producer's. BkdBuilder, on the other hand, PUSHES into an io::FileWriter. +// Something has to sit between the two and hold the bytes meanwhile. +// +// It cannot be SpillableByteBuffer: that one exposes stream_into() -- a single +// sequential drain -- and read_fn is positional. It cannot be a plain vector +// either, because bkd_data is the COLD sub-file and is sized by the point count. +// So: a temp file, written once and then read positionally, which is the same +// shape the spilled point runs already use. +namespace doris::snii::bkd { + +class StagedBlobFile final : public io::FileWriter { +public: + // `tag` only makes the temp file recognizable to a human; uniqueness comes + // from the pid and a process-wide counter. + static Status create(const std::string& tag, std::unique_ptr* out); + + ~StagedBlobFile() override; + + StagedBlobFile(const StagedBlobFile&) = delete; + StagedBlobFile& operator=(const StagedBlobFile&) = delete; + + // io::FileWriter. append() is the producer side; finalize() flushes and + // switches the file to readable. + Status append(Slice data) override; + Status finalize() override; + uint64_t bytes_written() const override { return bytes_written_; } + + // Positional read for BlobFileSource::read_fn. Reads EXACTLY `len` bytes or + // fails -- a short read reported as OK would be checksummed and sealed as if + // it were the real payload. Only valid after finalize(). + Status read_at(uint64_t offset, size_t len, uint8_t* out) const; + + // Removes the temp file. Called by the destructor too, so an abandoned build + // leaves nothing behind. + void remove(); + + // Where the staging file lives, for diagnostics and for tests that need to + // assert on THIS file rather than on whatever a directory scan happens to + // find. Empty once remove() has run. + const std::string& path() const { return path_; } + +private: + StagedBlobFile() = default; + + int fd_ = -1; + std::string path_; + uint64_t bytes_written_ = 0; + bool finalized_ = false; +}; + +} // namespace doris::snii::bkd diff --git a/be/src/storage/index/snii/common/slice.h b/be/src/storage/index/snii/common/slice.h new file mode 100644 index 00000000000000..e5b80932944df3 --- /dev/null +++ b/be/src/storage/index/snii/common/slice.h @@ -0,0 +1,56 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include + +namespace doris::snii { + +// Read-only byte view (does not own memory). Lifetime is managed by the underlying buffer. +class Slice { +public: + Slice() = default; + Slice(const uint8_t* d, size_t n) : data_(d), size_(n) {} + explicit Slice(const std::vector& v) : data_(v.data()), size_(v.size()) {} + explicit Slice(std::string_view sv) + : data_(reinterpret_cast(sv.data())), size_(sv.size()) {} + + const uint8_t* data() const { return data_; } + size_t size() const { return size_; } + bool empty() const { return size_ == 0; } + + uint8_t operator[](size_t i) const { + assert(i < size_); + return data_[i]; + } + + Slice subslice(size_t off, size_t n) const { + assert(off + n <= size_); + return Slice(data_ + off, n); + } + +private: + const uint8_t* data_ = nullptr; + size_t size_ = 0; +}; + +} // namespace doris::snii diff --git a/be/src/storage/index/snii/compaction/eligibility.cpp b/be/src/storage/index/snii/compaction/eligibility.cpp new file mode 100644 index 00000000000000..92b8676e979ad7 --- /dev/null +++ b/be/src/storage/index/snii/compaction/eligibility.cpp @@ -0,0 +1,386 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/compaction/eligibility.h" + +#include +#include +#include +#include + +#include "CLucene.h" +#include "common/check.h" +#include "common/config.h" +#include "common/exception.h" +#include "storage/index/inverted/analyzer/analyzer.h" +#include "storage/index/inverted/common_grams/common_grams_segment_metadata.h" +#include "storage/index/snii/format/format_constants.h" +#include "storage/index/snii/format/phrase_bigram.h" +#include "storage/index/snii/reader/logical_index_reader.h" +#include "storage/tablet/tablet_schema.h" + +namespace doris::snii::compaction { + +namespace inverted_index = segment_v2::inverted_index; + +namespace { + +Status reject(std::string_view reason) { + return Status::Error( + "SNII streamed compaction is not eligible: {}", reason); +} + +InvertedIndexAnalyzerConfig analyzer_config_from_properties( + const std::map& properties) { + InvertedIndexAnalyzerConfig config; + config.analyzer_name = get_analyzer_name_from_properties(properties); + config.parser_type = get_inverted_index_parser_type_from_string( + get_parser_string_from_properties(properties)); + config.parser_mode = get_parser_mode_string_from_properties(properties); + config.char_filter_map = get_parser_char_filter_map_from_properties(properties); + config.lower_case = get_parser_lowercase_from_properties(properties); + config.stop_words = get_parser_stopwords_from_properties(properties); + return config; +} + +Status validate_source_shape(const reader::LogicalIndexReader& source, size_t source_ordinal) { + if (source.tier() != format::IndexTier::kT2) { + return reject(fmt::format("source {} is not T2", source_ordinal)); + } + if (!source.has_positions()) { + return reject(fmt::format("source {} has no positions", source_ordinal)); + } + const auto& norms = source.section_refs().norms; + if (norms.offset != 0 || norms.length != 0) { + return reject(fmt::format("source {} carries scoring norms", source_ordinal)); + } + if (source.common_grams_metadata() != nullptr) { + return reject( + fmt::format("source {} carries CommonGrams/scoring metadata", source_ordinal)); + } + + const auto& stats = source.stats(); + if (stats.indexed_doc_count > stats.doc_count || + stats.null_count != stats.doc_count - stats.indexed_doc_count) { + return reject(fmt::format("source {} document statistics violate indexed + null = doc", + source_ordinal)); + } + return Status::OK(); +} + +inverted_index::CommonGramsSegmentMetadata static_common_grams_metadata( + const inverted_index::CommonGramsSegmentMetadata& metadata) { + auto seed = metadata; + seed.scoring_doc_count = 0; + seed.scoring_token_count = 0; + return seed; +} + +Status validate_common_grams_source_shape(const reader::LogicalIndexReader& source, + size_t source_ordinal) { + if (source.tier() != format::IndexTier::kT3 || !source.has_positions()) { + return reject(fmt::format("source {} is not CommonGrams T3", source_ordinal)); + } + if (source.section_refs().norms.length == 0) { + return reject(fmt::format("source {} has no scoring norms", source_ordinal)); + } + const auto* metadata = source.common_grams_metadata(); + if (metadata == nullptr) { + return reject(fmt::format("source {} has no CommonGrams metadata", source_ordinal)); + } + const Status metadata_status = + inverted_index::validate_common_grams_segment_metadata(*metadata); + const bool complete_shape = + metadata->common_grams_coverage == inverted_index::CommonGramsCoverage::kComplete && + source.common_grams_posting_policy() == format::CommonGramsPostingPolicy::kNone; + const bool hybrid_shape = + metadata->common_grams_coverage == inverted_index::CommonGramsCoverage::kMixed && + source.common_grams_posting_policy() == format::CommonGramsPostingPolicy::kHybridV1; + if (!metadata_status.ok() || (!complete_shape && !hybrid_shape) || + metadata->plain_term_key_version != inverted_index::PlainTermKeyVersion::kEscapedV1 || + metadata->common_grams_semantics_version != + inverted_index::COMMON_GRAMS_SEMANTICS_VERSION_V1 || + metadata->common_grams_key_version != inverted_index::COMMON_GRAMS_KEY_VERSION_V1 || + metadata->scoring_coverage != inverted_index::ScoringCoverage::kComplete || + metadata->scoring_stats_version != inverted_index::COMMON_GRAMS_SCORING_STATS_VERSION_V1 || + metadata->norm_semantics_version != + inverted_index::COMMON_GRAMS_NORM_SEMANTICS_VERSION_V1) { + return reject(fmt::format("source {} has incomplete or unsupported CommonGrams metadata", + source_ordinal)); + } + const Status scoring_status = inverted_index::validate_snii_scoring_metadata( + metadata, source.stats().doc_count, source.stats().sum_total_term_freq, + /*has_scoring_tier=*/true, /*has_positions=*/true, /*has_norms=*/true); + if (!scoring_status.ok()) { + return reject(fmt::format("source {} has incomplete scoring metadata: {}", source_ordinal, + scoring_status.to_string())); + } + const auto& stats = source.stats(); + if (stats.indexed_doc_count > stats.doc_count || + stats.null_count != stats.doc_count - stats.indexed_doc_count) { + return reject(fmt::format("source {} document statistics violate indexed + null = doc", + source_ordinal)); + } + return Status::OK(); +} + +Status reject_legacy_bigram(const reader::LogicalIndexReader& source, size_t source_ordinal) { + bool found_legacy_bigram = false; + RETURN_IF_ERROR(source.visit_prefix_terms( + format::kPhraseBigramTermMarker, + [&found_legacy_bigram](reader::LogicalIndexReader::PrefixHit&&, bool* stop) { + found_legacy_bigram = true; + *stop = true; + return Status::OK(); + })); + if (found_legacy_bigram) { + return reject( + fmt::format("source {} contains a legacy phrase-bigram term", source_ordinal)); + } + return Status::OK(); +} + +Status resolve_destination_analyzer(const TabletIndex& destination_index, + const AnalyzerProviderFactory& analyzer_provider_factory, + inverted_index::AnalyzerProviderPtr* analyzer_provider) { + analyzer_provider->reset(); + const auto& properties = destination_index.properties(); + if (!inverted_index::InvertedIndexAnalyzer::should_analyzer(properties)) { + return Status::OK(); + } + const InvertedIndexAnalyzerConfig analyzer_config = analyzer_config_from_properties(properties); + try { + *analyzer_provider = + analyzer_provider_factory + ? analyzer_provider_factory(analyzer_config) + : inverted_index::InvertedIndexAnalyzer::create_analyzer_provider( + &analyzer_config); + } catch (const CLuceneError& error) { + return reject(fmt::format("destination analyzer resolution failed: {}", error.what())); + } catch (const Exception& error) { + return reject(fmt::format("destination analyzer resolution failed: {}", error.what())); + } + DORIS_CHECK(*analyzer_provider != nullptr); + return Status::OK(); +} + +Status validate_destination_policy(const TabletIndex& destination_index, + const AnalyzerProviderFactory& analyzer_provider_factory) { + const auto& properties = destination_index.properties(); + if (get_parser_phrase_support_string_from_properties(properties) != + INVERTED_INDEX_PARSER_PHRASE_SUPPORT_YES) { + return reject("destination does not request phrase positions"); + } + if (!inverted_index::InvertedIndexAnalyzer::should_analyzer(properties)) { + return Status::OK(); + } + + inverted_index::AnalyzerProviderPtr analyzer_provider; + RETURN_IF_ERROR(resolve_destination_analyzer(destination_index, analyzer_provider_factory, + &analyzer_provider)); + DORIS_CHECK(analyzer_provider != nullptr); + if (config::enable_common_grams_index_build && analyzer_provider->uses_common_grams()) { + return reject("destination analyzer policy would build CommonGrams"); + } + return Status::OK(); +} + +} // namespace + +Status validate_plain_t2_source(const reader::LogicalIndexReader& source, size_t source_ordinal) { + return validate_source_shape(source, source_ordinal); +} + +Status validate_plain_t2_source_eligibility(const reader::LogicalIndexReader& source, + size_t source_ordinal) { + RETURN_IF_ERROR(validate_source_shape(source, source_ordinal)); + return reject_legacy_bigram(source, source_ordinal); +} + +Status validate_snii_source_eligibility(const reader::LogicalIndexReader& source, + size_t source_ordinal, + const SniiCompactionEligibility& eligibility) { + if (eligibility.kind == SniiStreamedMergeKind::kPlainT2) { + return validate_plain_t2_source_eligibility(source, source_ordinal); + } + RETURN_IF_ERROR(validate_common_grams_source_shape(source, source_ordinal)); + RETURN_IF_ERROR(reject_legacy_bigram(source, source_ordinal)); + if (!eligibility.common_grams_metadata_seed.has_value()) { + return reject("CommonGrams eligibility has no metadata identity seed"); + } + if (source.common_grams_posting_policy() != eligibility.common_grams_posting_policy) { + return reject(fmt::format("source {} CommonGrams posting policy differs from eligibility", + source_ordinal)); + } + if (static_common_grams_metadata(*source.common_grams_metadata()) != + *eligibility.common_grams_metadata_seed) { + return reject(fmt::format("source {} CommonGrams static identity differs from eligibility", + source_ordinal)); + } + return Status::OK(); +} + +Status validate_plain_t2_compaction_eligibility( + std::span sources, const TabletIndex& destination_index, + const AnalyzerProviderFactory& analyzer_provider_factory) { + if (sources.empty()) { + return reject("no source logical indexes"); + } + if (!destination_index.is_inverted_index()) { + return reject("destination is not an inverted index"); + } + + const TabletIndex& first_index_meta = sources.front().index_meta.get(); + if (!first_index_meta.is_inverted_index()) { + return reject("source 0 is not an inverted index"); + } + const auto& source_properties = first_index_meta.properties(); + for (size_t source_ordinal = 0; source_ordinal < sources.size(); ++source_ordinal) { + const TabletIndex& index_meta = sources[source_ordinal].index_meta.get(); + if (!index_meta.is_inverted_index()) { + return reject(fmt::format("source {} is not an inverted index", source_ordinal)); + } + if (index_meta.properties() != source_properties) { + return reject(fmt::format("source {} properties differ from source 0", source_ordinal)); + } + if (index_meta.index_id() != destination_index.index_id()) { + return reject( + fmt::format("source {} index id differs from destination", source_ordinal)); + } + if (index_meta.get_index_suffix() != destination_index.get_index_suffix()) { + return reject( + fmt::format("source {} index suffix differs from destination", source_ordinal)); + } + } + if (destination_index.properties() != source_properties) { + return reject("destination properties differ from source properties"); + } + + for (size_t source_ordinal = 0; source_ordinal < sources.size(); ++source_ordinal) { + RETURN_IF_ERROR(validate_plain_t2_source_eligibility(sources[source_ordinal].reader.get(), + source_ordinal)); + } + return validate_destination_policy(destination_index, analyzer_provider_factory); +} + +Status validate_snii_compaction_eligibility( + std::span sources, const TabletIndex& destination_index, + SniiCompactionEligibility* out, const AnalyzerProviderFactory& analyzer_provider_factory) { + if (out == nullptr) { + return Status::InvalidArgument("SNII compaction eligibility has null output"); + } + *out = SniiCompactionEligibility {}; + if (sources.empty()) { + return reject("no source logical indexes"); + } + if (!destination_index.is_inverted_index()) { + return reject("destination is not an inverted index"); + } + if (get_parser_phrase_support_string_from_properties(destination_index.properties()) != + INVERTED_INDEX_PARSER_PHRASE_SUPPORT_YES) { + return reject("destination does not request phrase positions"); + } + + const TabletIndex& first_index_meta = sources.front().index_meta.get(); + if (!first_index_meta.is_inverted_index()) { + return reject("source 0 is not an inverted index"); + } + const auto& source_properties = first_index_meta.properties(); + for (size_t source_ordinal = 0; source_ordinal < sources.size(); ++source_ordinal) { + const TabletIndex& index_meta = sources[source_ordinal].index_meta.get(); + if (!index_meta.is_inverted_index()) { + return reject(fmt::format("source {} is not an inverted index", source_ordinal)); + } + if (index_meta.properties() != source_properties) { + return reject(fmt::format("source {} properties differ from source 0", source_ordinal)); + } + if (index_meta.index_id() != destination_index.index_id()) { + return reject( + fmt::format("source {} index id differs from destination", source_ordinal)); + } + if (index_meta.get_index_suffix() != destination_index.get_index_suffix()) { + return reject( + fmt::format("source {} index suffix differs from destination", source_ordinal)); + } + } + if (destination_index.properties() != source_properties) { + return reject("destination properties differ from source properties"); + } + + const format::IndexTier source_tier = sources.front().reader.get().tier(); + if (source_tier == format::IndexTier::kT2) { + for (size_t source_ordinal = 0; source_ordinal < sources.size(); ++source_ordinal) { + if (sources[source_ordinal].reader.get().tier() != format::IndexTier::kT2) { + return reject("source streamed-merge shapes are not homogeneous"); + } + RETURN_IF_ERROR(validate_plain_t2_source_eligibility( + sources[source_ordinal].reader.get(), source_ordinal)); + } + RETURN_IF_ERROR(validate_destination_policy(destination_index, analyzer_provider_factory)); + out->kind = SniiStreamedMergeKind::kPlainT2; + return Status::OK(); + } + if (source_tier != format::IndexTier::kT3) { + return reject("source streamed-merge shape is neither plain T2 nor CommonGrams T3"); + } + + std::optional source_seed; + std::optional source_policy; + for (size_t source_ordinal = 0; source_ordinal < sources.size(); ++source_ordinal) { + const auto& source = sources[source_ordinal].reader.get(); + if (source.tier() != format::IndexTier::kT3) { + return reject("source streamed-merge shapes are not homogeneous"); + } + RETURN_IF_ERROR(validate_common_grams_source_shape(source, source_ordinal)); + RETURN_IF_ERROR(reject_legacy_bigram(source, source_ordinal)); + const auto seed = static_common_grams_metadata(*source.common_grams_metadata()); + if (!source_seed.has_value()) { + source_seed = seed; + source_policy = source.common_grams_posting_policy(); + } else if (source.common_grams_posting_policy() != *source_policy) { + return reject("source CommonGrams posting policies are not homogeneous"); + } else if (*source_seed != seed) { + return reject(fmt::format("source {} CommonGrams static identity differs from source 0", + source_ordinal)); + } + } + + if (!config::enable_common_grams_index_build) { + return reject("destination CommonGrams index build is disabled"); + } + inverted_index::AnalyzerProviderPtr analyzer_provider; + RETURN_IF_ERROR(resolve_destination_analyzer(destination_index, analyzer_provider_factory, + &analyzer_provider)); + if (analyzer_provider == nullptr || !analyzer_provider->uses_common_grams()) { + return reject("destination analyzer does not build CommonGrams"); + } + const auto* destination_identity = analyzer_provider->common_grams_identity(); + if (destination_identity == nullptr) { + return reject("destination analyzer has no complete CommonGrams identity"); + } + if (!source_seed.has_value() || + !inverted_index::common_grams_identity_matches(*source_seed, *destination_identity)) { + return reject("destination CommonGrams identity differs from source identity"); + } + out->kind = SniiStreamedMergeKind::kCommonGramsT3; + out->common_grams_metadata_seed = std::move(source_seed); + DORIS_CHECK(source_policy.has_value()); + out->common_grams_posting_policy = *source_policy; + return Status::OK(); +} + +} // namespace doris::snii::compaction diff --git a/be/src/storage/index/snii/compaction/eligibility.h b/be/src/storage/index/snii/compaction/eligibility.h new file mode 100644 index 00000000000000..bbbfae59204d22 --- /dev/null +++ b/be/src/storage/index/snii/compaction/eligibility.h @@ -0,0 +1,114 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/inverted/analyzer/analyzer_provider.h" +#include "storage/index/inverted/common_grams/common_grams_segment_metadata.h" +#include "storage/index/inverted/inverted_index_parser.h" +#include "storage/index/snii/format/core_metadata.h" + +namespace doris { + +class TabletIndex; + +namespace snii::reader { +class LogicalIndexReader; +} + +namespace snii::compaction { + +// One already-opened SNII source logical index and the TabletIndex metadata +// that produced it. References make null inputs unrepresentable; both objects +// must outlive validate_plain_t2_compaction_eligibility(). +struct PlainT2CompactionSource { + std::reference_wrapper reader; + std::reference_wrapper index_meta; +}; + +// Injectable only at analyzer-provider construction. Production callers omit +// it and use InvertedIndexAnalyzer::create_analyzer_provider; focused tests can +// supply an immutable provider without mutating the process IndexPolicyMgr. +using AnalyzerProviderFactory = std::function; + +enum class SniiStreamedMergeKind : uint8_t { + kPlainT2, + kCommonGramsT3, +}; + +// Validated destination shape for one streamed merge. CommonGrams metadata is a +// static identity seed: destination doc_count is bound when its session starts, +// and semantic token_count is bound after the single postings pass. +struct SniiCompactionEligibility { + SniiStreamedMergeKind kind = SniiStreamedMergeKind::kPlainT2; + std::optional + common_grams_metadata_seed; + format::CommonGramsPostingPolicy common_grams_posting_policy = + format::CommonGramsPostingPolicy::kNone; +}; + +// O(1) physical/semantic validation for one source. The merge planner may call +// this while preparing opened sources; the aggregate validator below reuses the +// exact same predicate. +Status validate_plain_t2_source(const reader::LogicalIndexReader& source, size_t source_ordinal); + +// Gate-only validation. In addition to the O(1) physical shape checks above, +// performs one bounded DICT prefix seek for the legacy hidden-bigram namespace +// so the owner can select raw rebuild before creating streamed output. +Status validate_plain_t2_source_eligibility(const reader::LogicalIndexReader& source, + size_t source_ordinal); + +// Revalidates one opened source against an already selected streamed shape. +// CommonGrams checks include exact current versions and static identity equality +// with the eligibility seed. +Status validate_snii_source_eligibility(const reader::LogicalIndexReader& source, + size_t source_ordinal, + const SniiCompactionEligibility& eligibility); + +// O(source-count), O(1)-metadata validation for the SNII postings-merge fast +// path. OK means every source is a plain positions-only T2 index, all source +// properties are byte-for-byte equal to the current destination properties, +// source/destination index ids and escaped suffixes match, and the current +// destination analyzer policy will not build CommonGrams. +// Rejections use INVERTED_INDEX_NOT_SUPPORTED with a concrete reason so the +// caller can select raw-column rebuild before creating output. +// +// Each source also receives one bounded seek for the legacy hidden-bigram +// marker namespace. This avoids discovering an unsupported legacy segment only +// after the raw-column compaction path has already been skipped. +Status validate_plain_t2_compaction_eligibility( + std::span sources, const TabletIndex& destination_index, + const AnalyzerProviderFactory& analyzer_provider_factory = {}); + +// Accepts either the existing plain positions-only T2 shape or homogeneous, +// complete CommonGrams T3 sources. Mixed shapes and any CommonGrams identity or +// destination-build-policy mismatch return INVERTED_INDEX_NOT_SUPPORTED so the +// caller can rebuild from raw columns before creating streamed output. +Status validate_snii_compaction_eligibility( + std::span sources, const TabletIndex& destination_index, + SniiCompactionEligibility* out, + const AnalyzerProviderFactory& analyzer_provider_factory = {}); + +} // namespace snii::compaction +} // namespace doris diff --git a/be/src/storage/index/snii/compaction/indexed_winner_tree.h b/be/src/storage/index/snii/compaction/indexed_winner_tree.h new file mode 100644 index 00000000000000..e16205a484832c --- /dev/null +++ b/be/src/storage/index/snii/compaction/indexed_winner_tree.h @@ -0,0 +1,103 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include + +#include "common/check.h" + +namespace doris::snii::compaction { + +// Complete binary winner tree over dense source ordinals. Updating one source +// touches only its leaf-to-root path. runner_up() inspects the sibling winners +// on the current winner's path without changing the tree. +template +class IndexedWinnerTree { +public: + static constexpr size_t kNoSource = std::numeric_limits::max(); + + explicit IndexedWinnerTree(Before before) : before_(std::move(before)), nodes_(2, kNoSource) {} + + template + void build(size_t source_count, IsLive&& is_live) { + source_count_ = source_count; + leaf_base_ = 1; + while (leaf_base_ < source_count_) { + DORIS_CHECK_LE(leaf_base_, std::numeric_limits::max() / 2); + leaf_base_ *= 2; + } + nodes_.assign(leaf_base_ * 2, kNoSource); + for (size_t source = 0; source < source_count_; ++source) { + if (is_live(source)) { + nodes_[leaf_base_ + source] = source; + } + } + for (size_t node = leaf_base_; node-- > 1;) { + nodes_[node] = select(nodes_[node * 2], nodes_[node * 2 + 1]); + } + } + + bool empty() const noexcept { return nodes_[1] == kNoSource; } + + size_t winner() const { + DCHECK(!empty()); + return nodes_[1]; + } + + size_t runner_up() const { + const size_t current_winner = winner(); + size_t candidate = kNoSource; + size_t node = leaf_base_ + current_winner; + while (node > 1) { + candidate = select(candidate, nodes_[node ^ 1]); + node /= 2; + } + return candidate; + } + + void update(size_t source, bool live) { + DCHECK_LT(source, source_count_); + size_t node = leaf_base_ + source; + nodes_[node] = live ? source : kNoSource; + while (node > 1) { + node /= 2; + nodes_[node] = select(nodes_[node * 2], nodes_[node * 2 + 1]); + } + } + +private: + size_t select(size_t candidate, size_t challenger) const { + if (candidate == kNoSource) { + return challenger; + } + if (challenger == kNoSource) { + return candidate; + } + return before_(challenger, candidate) ? challenger : candidate; + } + + Before before_; + size_t source_count_ = 0; + size_t leaf_base_ = 1; + boost::container::small_vector nodes_; +}; + +} // namespace doris::snii::compaction diff --git a/be/src/storage/index/snii/compaction/posting_cursor.cpp b/be/src/storage/index/snii/compaction/posting_cursor.cpp new file mode 100644 index 00000000000000..5ca54eb7c1f517 --- /dev/null +++ b/be/src/storage/index/snii/compaction/posting_cursor.cpp @@ -0,0 +1,1022 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/compaction/posting_cursor.h" + +#include + +#include +#include +#include +#include + +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/encoding/byte_source.h" +#include "storage/index/snii/format/format_constants.h" +#include "storage/index/snii/format/frq_pod.h" +#include "storage/index/snii/format/prx_pod.h" +#include "storage/index/snii/reader/windowed_posting.h" + +namespace doris::snii::compaction { + +namespace { + +Status checked_add(uint64_t lhs, uint64_t rhs, const char* message, uint64_t* out) { + if (rhs > std::numeric_limits::max() - lhs) { + return Status::Error(message); + } + *out = lhs + rhs; + return Status::OK(); +} + +Status posting_corruption(const char* message, uint32_t source_ordinal) { + return Status::Error( + "posting_cursor: {} (src_ord={})", message, source_ordinal); +} + +} // namespace + +Status validate_posting_region(const format::RegionRef& region, uint64_t file_size) { + if (region.offset > file_size || region.length > file_size - region.offset) { + return Status::Error( + "posting_cursor: posting region outside source file"); + } + return Status::OK(); +} + +bool posting_entry_has_positions(const format::DictEntry& entry) { + return entry.kind == format::DictEntryKind::kInline ? !entry.prx_bytes.empty() + : entry.prx_len != 0; +} + +SniiPostingReadContext::TermLease::~TermLease() { + if (context_ != nullptr) { + context_->release_term(); + } +} + +size_t SniiPostingReadContext::DecoderWorkspace::capacity_bytes() const { + return docs_scratch.capacity() + prx_scratch.capacity() + decompressed.capacity() + + sizeof(uint32_t) * (docids.capacity() + positions_flat.capacity() + + position_offsets.capacity() + frequencies.capacity()) + + sizeof(DestinationPostingRun) * destination_runs.capacity(); +} + +void SniiPostingReadContext::DecoderWorkspace::init_memory_reporter( + writer::MemoryReporter* memory_reporter) { + if (memory_reporter == nullptr) return; + docs_scratch_reservation = memory_reporter->make_reservation(); + prx_scratch_reservation = memory_reporter->make_reservation(); + docids_reservation = memory_reporter->make_reservation(); + positions_reservation = memory_reporter->make_reservation(); + position_offsets_reservation = memory_reporter->make_reservation(); + destination_runs_reservation = memory_reporter->make_reservation(); + frequencies_reservation = memory_reporter->make_reservation(); + decompressed_reservation = memory_reporter->make_reservation(); + reservations_enabled = true; +} + +Status SniiPostingReadContext::DecoderWorkspace::reserve_remapped(size_t document_count, + size_t run_count, + bool retain_frequencies) { + const bool grow_runs = destination_runs.capacity() < run_count; + const bool grow_frequencies = retain_frequencies && frequencies.capacity() < document_count; + if (!reservations_enabled) { + destination_runs.reserve(run_count); + if (retain_frequencies) { + frequencies.reserve(document_count); + } + return Status::OK(); + } + if (!grow_runs && !grow_frequencies) { + DCHECK_EQ(destination_runs_reservation.bytes(), + destination_runs.capacity() * sizeof(DestinationPostingRun)); + DCHECK_EQ(frequencies_reservation.bytes(), frequencies.capacity() * sizeof(uint32_t)); + return Status::OK(); + } + if (run_count > std::numeric_limits::max() / sizeof(DestinationPostingRun) || + document_count > std::numeric_limits::max() / sizeof(uint32_t)) { + return Status::Error( + "posting cursor: remapped workspace size overflows size_t"); + } + writer::MemoryReporter::Reservation replacement_runs; + writer::MemoryReporter::Reservation replacement_frequencies; + if (grow_runs) { + RETURN_IF_ERROR(destination_runs_reservation.prepare_replacement( + run_count * sizeof(DestinationPostingRun), &replacement_runs)); + } + if (grow_frequencies) { + RETURN_IF_ERROR(frequencies_reservation.prepare_replacement( + document_count * sizeof(uint32_t), &replacement_frequencies)); + } + { + std::vector new_runs; + std::vector new_frequencies; + if (grow_runs) { + new_runs.reserve(run_count); + DCHECK_EQ(new_runs.capacity(), run_count); + destination_runs.swap(new_runs); + } + if (grow_frequencies) { + new_frequencies.reserve(document_count); + DCHECK_EQ(new_frequencies.capacity(), document_count); + frequencies.swap(new_frequencies); + } + } + if (grow_runs) { + destination_runs_reservation = std::move(replacement_runs); + } + if (grow_frequencies) { + frequencies_reservation = std::move(replacement_frequencies); + } + return Status::OK(); +} + +Status SniiPostingReadContext::DecoderWorkspace::reserve_docids(size_t count) { + if (!reservations_enabled || docids.capacity() >= count) { + if (reservations_enabled) { + DCHECK_EQ(docids_reservation.bytes(), docids.capacity() * sizeof(uint32_t)); + } + return Status::OK(); + } + const size_t target_bytes = count * sizeof(uint32_t); + writer::MemoryReporter::Reservation replacement; + RETURN_IF_ERROR(docids_reservation.prepare_replacement(target_bytes, &replacement)); + docids.reserve(count); + DCHECK_EQ(docids.capacity(), count); + docids_reservation = std::move(replacement); + return Status::OK(); +} + +Status SniiPostingReadContext::DecoderWorkspace::reserve_csr(std::vector* pos_flat, + size_t position_count, + std::vector* pos_off, + size_t offset_count) { + DCHECK(reservations_enabled); + DCHECK_EQ(pos_flat, &positions_flat); + DCHECK_EQ(pos_off, &position_offsets); + if (position_count > std::numeric_limits::max() / sizeof(uint32_t) || + offset_count > std::numeric_limits::max() / sizeof(uint32_t)) { + return Status::Error( + "posting cursor: position workspace size overflows size_t"); + } + + const bool grow_positions = positions_flat.capacity() < position_count; + const bool grow_offsets = position_offsets.capacity() < offset_count; + if (!grow_positions && !grow_offsets) { + DCHECK_EQ(positions_reservation.bytes(), positions_flat.capacity() * sizeof(uint32_t)); + DCHECK_EQ(position_offsets_reservation.bytes(), + position_offsets.capacity() * sizeof(uint32_t)); + return Status::OK(); + } + + writer::MemoryReporter::Reservation replacement_positions; + writer::MemoryReporter::Reservation replacement_offsets; + if (grow_positions) { + RETURN_IF_ERROR(positions_reservation.prepare_replacement(position_count * sizeof(uint32_t), + &replacement_positions)); + } + if (grow_offsets) { + RETURN_IF_ERROR(position_offsets_reservation.prepare_replacement( + offset_count * sizeof(uint32_t), &replacement_offsets)); + } + + { + std::vector new_positions; + std::vector new_offsets; + if (grow_positions) { + new_positions.reserve(position_count); + DCHECK_EQ(new_positions.capacity(), position_count); + positions_flat.swap(new_positions); + } + if (grow_offsets) { + new_offsets.reserve(offset_count); + DCHECK_EQ(new_offsets.capacity(), offset_count); + position_offsets.swap(new_offsets); + } + } + if (grow_positions) { + positions_reservation = std::move(replacement_positions); + } + if (grow_offsets) { + position_offsets_reservation = std::move(replacement_offsets); + } + return Status::OK(); +} + +Status SniiPostingReadContext::DecoderWorkspace::reserve_decompression( + size_t bytes, std::vector** buffer) { + DCHECK(reservations_enabled); + DCHECK(buffer != nullptr); + if (decompressed.capacity() < bytes) { + writer::MemoryReporter::Reservation replacement; + RETURN_IF_ERROR(decompressed_reservation.prepare_replacement(bytes, &replacement)); + { + std::vector new_decompressed; + new_decompressed.reserve(bytes); + DCHECK_EQ(new_decompressed.capacity(), bytes); + decompressed.swap(new_decompressed); + } + decompressed_reservation = std::move(replacement); + } else { + DCHECK_EQ(decompressed_reservation.bytes(), decompressed.capacity()); + } + *buffer = &decompressed; + return Status::OK(); +} + +void SniiPostingReadContext::DecoderWorkspace::release_large_buffers( + size_t retained_capacity_limit_bytes) { + prelude = format::FrqPreludeReader(); + if (capacity_bytes() <= retained_capacity_limit_bytes) { + return; + } + std::vector().swap(docs_scratch); + std::vector().swap(prx_scratch); + std::vector().swap(decompressed); + std::vector().swap(docids); + std::vector().swap(positions_flat); + std::vector().swap(position_offsets); + std::vector().swap(destination_runs); + std::vector().swap(frequencies); + docs_scratch_reservation.reset(); + prx_scratch_reservation.reset(); + docids_reservation.reset(); + positions_reservation.reset(); + position_offsets_reservation.reset(); + destination_runs_reservation.reset(); + frequencies_reservation.reset(); + decompressed_reservation.reset(); + DCHECK_LE(capacity_bytes(), retained_capacity_limit_bytes); +} + +Status SniiPostingReadContext::poison(Status status) { + DCHECK(!status.ok()); + if (failed_.ok()) { + failed_ = std::move(status); + } + return failed_; +} + +Status SniiPostingReadContext::init() { + if (initialized_) { + return Status::Error( + "posting_read_context: init called twice"); + } + if (index_ == nullptr || index_->reader() == nullptr) { + return Status::Error( + "posting_read_context: null source index"); + } + if (total_read_ahead_budget_bytes_ < 2 || + total_read_ahead_budget_bytes_ > kMaxReadAheadBudgetBytes) { + return Status::Error( + "posting_read_context: total read-ahead budget outside [2, {}]", + kMaxReadAheadBudgetBytes); + } + + posting_region_ = index_->section_refs().posting_region; + RETURN_IF_ERROR(validate_posting_region(posting_region_, index_->reader()->size())); + decoder_workspace_.init_memory_reporter(memory_reporter_); + posting_cache_ = std::make_unique( + index_->reader(), posting_region_.offset, posting_region_.length, + total_read_ahead_budget_bytes_, memory_reporter_); + RETURN_IF_ERROR(posting_cache_->init()); + initialized_ = true; + return Status::OK(); +} + +Status SniiPostingReadContext::validate_next_range(const format::RegionRef& range, + bool has_previous, uint64_t previous_end, + const char* stream, uint64_t* end) const { + DCHECK(end != nullptr); + if (range.length == 0 || range.offset < posting_region_.offset) { + return Status::Error( + "posting_read_context: invalid {} term range", stream); + } + const uint64_t relative_offset = range.offset - posting_region_.offset; + if (relative_offset > posting_region_.length || + range.length > posting_region_.length - relative_offset) { + return Status::Error( + "posting_read_context: {} term range outside posting region", stream); + } + if (has_previous && range.offset < previous_end) { + return Status::Error( + "posting_read_context: {} term ranges are not monotone", stream); + } + *end = range.offset + range.length; + return Status::OK(); +} + +Status SniiPostingReadContext::acquire_term(bool has_docs_range, bool has_prx_range, + const format::RegionRef& docs_range, + const format::RegionRef& prx_range, + std::unique_ptr* lease) { + DCHECK(lease != nullptr); + DCHECK(*lease == nullptr); + if (!initialized_) { + return Status::Error( + "posting_read_context: acquire before init"); + } + if (!failed_.ok()) { + return failed_; + } + if (term_active_) { + return Status::Error( + "posting_read_context: concurrent term cursor"); + } + + uint64_t docs_end = 0; + uint64_t prx_end = 0; + if (has_docs_range) { + Status status = + validate_next_range(docs_range, has_docs_range_, last_docs_end_, "docs", &docs_end); + if (!status.ok()) { + return poison(status); + } + last_docs_end_ = docs_end; + has_docs_range_ = true; + } + if (has_prx_range) { + Status status = + validate_next_range(prx_range, has_prx_range_, last_prx_end_, "prx", &prx_end); + if (!status.ok()) { + return poison(status); + } + last_prx_end_ = prx_end; + has_prx_range_ = true; + } + + term_active_ = true; + lease->reset(new TermLease(this)); + return Status::OK(); +} + +void SniiPostingReadContext::release_term() { + DCHECK(term_active_); + term_active_ = false; + decoder_workspace_.release_large_buffers(retained_decoder_workspace_limit_bytes()); +} + +Status SniiPostingReadContext::poison_active_term(Status status, TermLease* lease) { + DCHECK(lease != nullptr); + DCHECK_EQ(lease->context_, this); + DCHECK(term_active_); + Status first = poison(std::move(status)); + lease->context_ = nullptr; + release_term(); + return first; +} + +uint64_t SniiPostingReadContext::docs_read_calls() const { + DCHECK(initialized_); + return posting_cache_->read_calls(PostingStream::kDocs); +} + +uint64_t SniiPostingReadContext::prx_read_calls() const { + DCHECK(initialized_); + return posting_cache_->read_calls(PostingStream::kPrx); +} + +uint64_t SniiPostingReadContext::docs_buffer_hits() const { + DCHECK(initialized_); + return posting_cache_->buffer_hits(PostingStream::kDocs); +} + +uint64_t SniiPostingReadContext::prx_buffer_hits() const { + DCHECK(initialized_); + return posting_cache_->buffer_hits(PostingStream::kPrx); +} + +uint64_t SniiPostingReadContext::physical_read_ranges() const { + DCHECK(initialized_); + return posting_cache_->physical_read_ranges(); +} + +uint64_t SniiPostingReadContext::physical_read_bytes() const { + DCHECK(initialized_); + return posting_cache_->physical_read_bytes(); +} + +size_t SniiPostingReadContext::resident_read_ahead_capacity_bytes() const { + DCHECK(initialized_); + return posting_cache_->resident_capacity_bytes(); +} + +size_t SniiPostingReadContext::decoder_workspace_capacity_bytes() const { + DCHECK(initialized_); + return decoder_workspace_.capacity_bytes(); +} + +Status SniiPostingCursor::poison(Status status) { + DCHECK(!status.ok()); + if (failed_.ok()) { + if (term_lease_ != nullptr) { + failed_ = read_context_->poison_active_term(std::move(status), term_lease_.get()); + term_lease_.reset(); + } else { + failed_ = std::move(status); + } + } + return failed_; +} + +Status SniiPostingCursor::validate_entry_geometry() { + if (entry_.df == 0) { + return posting_corruption("zero-df dictionary entry", source_ordinal_); + } + if (entry_.kind == format::DictEntryKind::kInline) { + if (entry_.enc != format::DictEntryEnc::kSlim) { + return posting_corruption("inline entry is not slim", source_ordinal_); + } + if (entry_.inline_dd_disk_len != entry_.dd_meta.disk_len || + entry_.inline_dd_disk_len > entry_.frq_bytes.size()) { + return posting_corruption("inline dd geometry mismatch", source_ordinal_); + } + const uint64_t freq_len = entry_.frq_bytes.size() - entry_.inline_dd_disk_len; + if (entry_.freq_meta.disk_len != freq_len) { + return posting_corruption("inline freq geometry mismatch", source_ordinal_); + } + shape_ = Shape::kFlat; + return Status::OK(); + } + + if (entry_.kind != format::DictEntryKind::kPodRef) { + return posting_corruption("unknown dictionary entry kind", source_ordinal_); + } + if (entry_.enc == format::DictEntryEnc::kWindowed) { + if (entry_.prelude_len == 0 || entry_.prelude_len > entry_.frq_docs_len || + entry_.frq_docs_len > entry_.frq_len) { + return posting_corruption("invalid windowed frq geometry", source_ordinal_); + } + shape_ = Shape::kWindowed; + return Status::OK(); + } + if (entry_.enc != format::DictEntryEnc::kSlim) { + return posting_corruption("unknown dictionary entry encoding", source_ordinal_); + } + if (entry_.prelude_len != 0 || entry_.frq_docs_len != entry_.dd_meta.disk_len || + entry_.frq_docs_len > entry_.frq_len) { + return posting_corruption("invalid slim frq geometry", source_ordinal_); + } + if (entry_.freq_meta.disk_len != entry_.frq_len - entry_.frq_docs_len) { + return posting_corruption("slim freq geometry mismatch", source_ordinal_); + } + shape_ = Shape::kFlat; + return Status::OK(); +} + +Status SniiPostingCursor::prepare_flat_ranges() { + if (entry_.kind == format::DictEntryKind::kInline) { + flat_dd_len_ = entry_.inline_dd_disk_len; + flat_prx_len_ = entry_.prx_bytes.size(); + return Status::OK(); + } + + uint64_t frq_len = 0; + RETURN_IF_ERROR(index_->resolve_frq_window(entry_, frq_base_, &flat_dd_abs_, &frq_len)); + if (frq_len != entry_.frq_len) { + return posting_corruption("resolved slim frq length mismatch", source_ordinal_); + } + flat_dd_len_ = entry_.frq_docs_len; + if (term_has_positions_) { + RETURN_IF_ERROR( + index_->resolve_prx_window(entry_, prx_base_, &flat_prx_abs_, &flat_prx_len_)); + if (flat_prx_len_ != entry_.prx_len) { + return posting_corruption("resolved slim prx length mismatch", source_ordinal_); + } + } + return Status::OK(); +} + +Status SniiPostingCursor::prepare_windowed_ranges() { + RETURN_IF_ERROR(index_->resolve_frq_window(entry_, frq_base_, &flat_dd_abs_, &flat_dd_len_)); + if (flat_dd_len_ != entry_.frq_len - entry_.prelude_len || flat_dd_abs_ < entry_.prelude_len) { + return posting_corruption("resolved windowed frq geometry mismatch", source_ordinal_); + } + if (term_has_positions_) { + RETURN_IF_ERROR( + index_->resolve_prx_window(entry_, prx_base_, &flat_prx_abs_, &flat_prx_len_)); + if (flat_prx_len_ != entry_.prx_len) { + return posting_corruption("resolved windowed prx length mismatch", source_ordinal_); + } + } + return Status::OK(); +} + +Status SniiPostingCursor::prepare_windowed() { + DCHECK(workspace_ != nullptr); + Slice prelude_bytes; + RETURN_IF_ERROR(read_context_->posting_cache_->resolve( + PostingStream::kDocs, flat_dd_abs_ - entry_.prelude_len, entry_.prelude_len, + &workspace_->docs_scratch, &prelude_bytes, + read_context_->memory_reporter_ == nullptr ? nullptr + : &workspace_->docs_scratch_reservation)); + RETURN_IF_ERROR(format::FrqPreludeReader::open(prelude_bytes, &workspace_->prelude)); + if (workspace_->prelude.has_prx() != term_has_positions_) { + return posting_corruption("windowed prelude position shape differs from entry", + source_ordinal_); + } + + uint64_t docs_prefix_len = 0; + RETURN_IF_ERROR(checked_add(entry_.prelude_len, workspace_->prelude.dd_block_len(), + "posting_cursor: windowed docs prefix overflow", &docs_prefix_len)); + if (docs_prefix_len != entry_.frq_docs_len) { + return posting_corruption("windowed docs prefix mismatch", source_ordinal_); + } + uint64_t encoded_frq_len = 0; + RETURN_IF_ERROR(checked_add(docs_prefix_len, workspace_->prelude.freq_block_len(), + "posting_cursor: windowed frq length overflow", &encoded_frq_len)); + if (encoded_frq_len != entry_.frq_len) { + return posting_corruption("windowed frq blocks do not tile entry", source_ordinal_); + } + + uint64_t dd_bytes = 0; + uint64_t freq_bytes = 0; + uint64_t prx_bytes = 0; + uint64_t docs = 0; + uint32_t previous_last_docid = 0; + bool has_previous_window = false; + for (uint32_t window = 0; window < workspace_->prelude.n_windows(); ++window) { + format::WindowMeta meta; + RETURN_IF_ERROR(workspace_->prelude.window(window, &meta)); + if (meta.doc_count == 0 || meta.dd_off != dd_bytes || meta.prx_off != prx_bytes || + (workspace_->prelude.has_freq() && meta.freq_off != freq_bytes)) { + return posting_corruption("non-contiguous window metadata", source_ordinal_); + } + if ((!has_previous_window && meta.win_base != 0) || + (has_previous_window && + (meta.win_base != previous_last_docid || meta.last_docid <= previous_last_docid))) { + return posting_corruption("invalid window docid chain", source_ordinal_); + } + if (meta.last_docid >= index_->stats().doc_count) { + return posting_corruption("window last docid outside index", source_ordinal_); + } + RETURN_IF_ERROR(checked_add(dd_bytes, meta.dd_disk_len, + "posting_cursor: window dd bytes overflow", &dd_bytes)); + RETURN_IF_ERROR(checked_add(freq_bytes, meta.freq_disk_len, + "posting_cursor: window freq bytes overflow", &freq_bytes)); + RETURN_IF_ERROR(checked_add(prx_bytes, meta.prx_len, + "posting_cursor: window prx bytes overflow", &prx_bytes)); + RETURN_IF_ERROR(checked_add(docs, meta.doc_count, + "posting_cursor: window doc count overflow", &docs)); + previous_last_docid = meta.last_docid; + has_previous_window = true; + } + if (dd_bytes != workspace_->prelude.dd_block_len() || + freq_bytes != workspace_->prelude.freq_block_len() || prx_bytes != entry_.prx_len || + docs != entry_.df) { + return posting_corruption("window directory totals mismatch", source_ordinal_); + } + return Status::OK(); +} + +Status SniiPostingCursor::init() { + if (initialized_) { + return Status::Error( + "posting_cursor: init called twice"); + } + if (read_context_ == nullptr || index_ == nullptr || rowid_conversion_ == nullptr) { + return Status::Error( + "posting_cursor: null read context or rowid conversion"); + } + if (!read_context_->initialized()) { + return Status::Error( + "posting_cursor: source read context not initialized"); + } + if (!read_context_->failed_status().ok()) { + return read_context_->failed_status(); + } + if (index_->tier() == format::IndexTier::kT1) { + return Status::Error( + "posting_cursor: positions index is required"); + } + if (!index_->has_positions()) { + return posting_corruption("positions tier lacks positions capability", source_ordinal_); + } + if (source_ordinal_ >= rowid_conversion_->source_segment_count()) { + return Status::Error( + "posting_cursor: source ordinal outside rowid conversion"); + } + if (index_->stats().doc_count != + rowid_conversion_->source_segment_doc_counts()[source_ordinal_]) { + return posting_corruption("rowid conversion size differs from source doc count", + source_ordinal_); + } + source_mapping_ = rowid_conversion_->source_mapping(source_ordinal_); + source_has_deletions_ = rowid_conversion_->source_has_deletions(source_ordinal_); + + RETURN_IF_ERROR(validate_entry_geometry()); + const bool has_pod_ranges = entry_.kind == format::DictEntryKind::kPodRef; + format::RegionRef docs_range; + format::RegionRef prx_range; + if (shape_ == Shape::kWindowed) { + RETURN_IF_ERROR(prepare_windowed_ranges()); + docs_range.offset = flat_dd_abs_ - entry_.prelude_len; + docs_range.length = entry_.frq_docs_len; + prx_range.offset = flat_prx_abs_; + prx_range.length = flat_prx_len_; + } else { + RETURN_IF_ERROR(prepare_flat_ranges()); + if (has_pod_ranges) { + docs_range.offset = flat_dd_abs_; + docs_range.length = flat_dd_len_; + prx_range.offset = flat_prx_abs_; + prx_range.length = flat_prx_len_; + } + } + RETURN_IF_ERROR(read_context_->acquire_term(has_pod_ranges, + has_pod_ranges && term_has_positions_, docs_range, + prx_range, &term_lease_)); + workspace_ = &read_context_->decoder_workspace_; + workspace_->destination_runs.clear(); + workspace_->frequencies.clear(); + next_destination_run_ = 0; + if (shape_ == Shape::kWindowed) { + const Status status = prepare_windowed(); + if (!status.ok()) { + return poison(status); + } + } + initialized_ = true; + return Status::OK(); +} + +Status SniiPostingCursor::decode_prx(Slice bytes, format::PrxDecodedShape* shape) { + DCHECK(workspace_ != nullptr); + DCHECK(shape != nullptr); + ByteSource source(bytes); + format::PrxDecodeContext decode_context { + .shape = shape, + .allocation_gate = workspace_->reservations_enabled ? workspace_ : nullptr}; + RETURN_IF_ERROR(format::read_prx_window_csr(&source, &workspace_->positions_flat, + &workspace_->position_offsets, &decode_context)); + if (!source.eof()) { + return posting_corruption("trailing bytes after prx frame", source_ordinal_); + } + return Status::OK(); +} + +Status SniiPostingCursor::decode_dd(Slice bytes, const format::FrqRegionMeta& meta, + uint64_t win_base, uint32_t expected_doc_count) { + DCHECK(workspace_ != nullptr); + if (!workspace_->reservations_enabled) { + return format::decode_dd_region(bytes, meta, win_base, expected_doc_count, + &workspace_->docids); + } + return format::decode_dd_region(bytes, meta, win_base, expected_doc_count, workspace_, + &workspace_->docids); +} + +Status SniiPostingCursor::load_flat_chunk() { + DCHECK(workspace_ != nullptr); + Slice dd_bytes; + Slice prx_bytes; + if (entry_.kind == format::DictEntryKind::kInline) { + dd_bytes = Slice(entry_.frq_bytes.data(), static_cast(flat_dd_len_)); + prx_bytes = Slice(entry_.prx_bytes); + } else { + RETURN_IF_ERROR(read_context_->posting_cache_->resolve( + PostingStream::kDocs, flat_dd_abs_, flat_dd_len_, &workspace_->docs_scratch, + &dd_bytes, + read_context_->memory_reporter_ == nullptr + ? nullptr + : &workspace_->docs_scratch_reservation)); + if (term_has_positions_) { + RETURN_IF_ERROR(read_context_->posting_cache_->resolve( + PostingStream::kPrx, flat_prx_abs_, flat_prx_len_, &workspace_->prx_scratch, + &prx_bytes, + read_context_->memory_reporter_ == nullptr + ? nullptr + : &workspace_->prx_scratch_reservation)); + } + } + + RETURN_IF_ERROR(workspace_->reserve_docids(entry_.df)); + RETURN_IF_ERROR(decode_dd(dd_bytes, entry_.dd_meta, /*win_base=*/0, entry_.df)); + format::PrxDecodedShape prx_shape; + if (term_has_positions_) { + RETURN_IF_ERROR(decode_prx(prx_bytes, &prx_shape)); + } + if (workspace_->docids.size() != entry_.df || workspace_->docids.empty() || + workspace_->docids.back() >= index_->stats().doc_count) { + return posting_corruption("decoded docids differ from flat entry shape", source_ordinal_); + } + if (term_has_positions_ && + (prx_shape.total_docs != entry_.df || + prx_shape.total_positions != workspace_->positions_flat.size() || + prx_shape.has_zero_frequency || + workspace_->position_offsets.size() != static_cast(entry_.df) + 1 || + workspace_->position_offsets.empty() || workspace_->position_offsets.front() != 0 || + workspace_->position_offsets.back() != workspace_->positions_flat.size())) { + return posting_corruption("dd/prx document shape mismatch", source_ordinal_); + } + decoded_docs_ = entry_.df; + if (term_has_positions_) { + decoded_total_freq_ = prx_shape.total_positions; + decoded_max_freq_ = prx_shape.max_frequency; + } + flat_loaded_ = true; + return Status::OK(); +} + +Status SniiPostingCursor::load_windowed_chunk() { + DCHECK(workspace_ != nullptr); + format::WindowMeta meta; + RETURN_IF_ERROR(workspace_->prelude.window(next_window_, &meta)); + reader::WindowAbsRange range; + RETURN_IF_ERROR(reader::windowed_window_range( + *index_, entry_, frq_base_, prx_base_, workspace_->prelude, next_window_, + /*want_positions=*/term_has_positions_, /*want_freq=*/false, &range)); + + Slice dd_bytes; + Slice prx_bytes; + RETURN_IF_ERROR(read_context_->posting_cache_->resolve( + PostingStream::kDocs, range.dd_off, range.dd_len, &workspace_->docs_scratch, &dd_bytes, + read_context_->memory_reporter_ == nullptr ? nullptr + : &workspace_->docs_scratch_reservation)); + if (term_has_positions_) { + RETURN_IF_ERROR(read_context_->posting_cache_->resolve( + PostingStream::kPrx, range.prx_off, range.prx_len, &workspace_->prx_scratch, + &prx_bytes, + read_context_->memory_reporter_ == nullptr ? nullptr + : &workspace_->prx_scratch_reservation)); + } + RETURN_IF_ERROR(workspace_->reserve_docids(meta.doc_count)); + RETURN_IF_ERROR(decode_dd(dd_bytes, + format::FrqRegionMeta {.zstd = meta.dd_zstd, + .uncomp_len = meta.dd_uncomp_len, + .disk_len = meta.dd_disk_len, + .crc = meta.crc_dd, + .verify_crc = meta.verify_crc}, + meta.win_base, meta.doc_count)); + format::PrxDecodedShape prx_shape; + if (term_has_positions_) { + RETURN_IF_ERROR(decode_prx(prx_bytes, &prx_shape)); + } + if (workspace_->docids.size() != meta.doc_count || workspace_->docids.empty() || + workspace_->docids.back() != meta.last_docid || + (next_window_ != 0 && workspace_->docids.front() <= meta.win_base)) { + return posting_corruption("window docid shape or last docid mismatch", source_ordinal_); + } + if (term_has_positions_ && + (prx_shape.total_docs != meta.doc_count || + prx_shape.total_positions != workspace_->positions_flat.size() || + prx_shape.has_zero_frequency || + workspace_->position_offsets.size() != static_cast(meta.doc_count) + 1 || + workspace_->position_offsets.empty() || workspace_->position_offsets.front() != 0 || + workspace_->position_offsets.back() != workspace_->positions_flat.size())) { + return posting_corruption("dd/prx document shape mismatch", source_ordinal_); + } + if (term_has_positions_ && entry_.term_stats_present && + prx_shape.max_frequency != meta.max_freq) { + return posting_corruption("window max frequency mismatch", source_ordinal_); + } + if (decoded_docs_ > entry_.df || meta.doc_count > entry_.df - decoded_docs_) { + return posting_corruption("decoded document count exceeds df", source_ordinal_); + } + decoded_docs_ += meta.doc_count; + if (term_has_positions_) { + if (prx_shape.total_positions > + std::numeric_limits::max() - decoded_total_freq_) { + return posting_corruption("total term frequency overflow", source_ordinal_); + } + decoded_total_freq_ += prx_shape.total_positions; + decoded_max_freq_ = std::max(decoded_max_freq_, prx_shape.max_frequency); + } + ++next_window_; + return Status::OK(); +} + +Status SniiPostingCursor::load_next_chunk(bool* loaded) { + DCHECK(loaded != nullptr); + DCHECK(workspace_ != nullptr); + *loaded = false; + // Both decode paths resize and validate these buffers. Clearing them first would force a cold + // grow and zero-fill on every warm re-decode. Stale contents are never read when not loaded. + + if (shape_ == Shape::kFlat) { + if (flat_loaded_) { + return Status::OK(); + } + RETURN_IF_ERROR(load_flat_chunk()); + *loaded = true; + return Status::OK(); + } + if (next_window_ >= workspace_->prelude.n_windows()) { + return Status::OK(); + } + RETURN_IF_ERROR(load_windowed_chunk()); + *loaded = true; + return Status::OK(); +} + +Status SniiPostingCursor::finish_source() { + if (decoded_docs_ != entry_.df) { + return posting_corruption("decoded document count differs from df", source_ordinal_); + } + if (entry_.term_stats_present && + (decoded_total_freq_ != entry_.ttf_delta || decoded_max_freq_ != entry_.max_freq)) { + return posting_corruption("decoded term statistics mismatch", source_ordinal_); + } + exhausted_ = true; + term_lease_.reset(); + return Status::OK(); +} + +Status SniiPostingCursor::map_decoded_chunk() { + DCHECK(workspace_ != nullptr); + std::vector& runs = workspace_->destination_runs; + std::vector& docids = workspace_->docids; + std::vector& frequencies = workspace_->frequencies; + runs.clear(); + frequencies.clear(); + const size_t document_count = docids.size(); + const size_t max_run_count = + std::min(document_count, rowid_conversion_->destination_segment_doc_counts().size()); + RETURN_IF_ERROR( + workspace_->reserve_remapped(document_count, max_run_count, term_has_positions_)); + + auto append_live_document = [&](uint32_t segment, uint32_t docid, size_t live_docs) { + if (runs.empty() || runs.back().destination_segment != segment) { + if (!runs.empty()) { + runs.back().document_end = static_cast(live_docs); + } + runs.push_back({.destination_segment = segment}); + } + docids[live_docs] = docid; + }; + + if (!source_has_deletions_) { + if (term_has_positions_) { + // Frequencies are adjacent position-offset deltas; fill them in + // their own pass so the remap loop below stays a pure gather. + const uint32_t* offsets = workspace_->position_offsets.data(); + for (size_t ordinal = 0; ordinal < document_count; ++ordinal) { + frequencies.push_back(offsets[ordinal + 1] - offsets[ordinal]); + } + } + // The gather walks source_mapping_ at monotonically increasing but + // sparse indexes the hardware prefetcher cannot follow; the future + // lookup indexes are already decoded, so prefetch them explicitly. + // Without deletions every mapping entry is live, so the destination + // segment can never equal the uint32 sentinel. + constexpr size_t kMapPrefetchDistance = 16; + uint32_t current_segment = std::numeric_limits::max(); + for (size_t ordinal = 0; ordinal < document_count; ++ordinal) { + if (ordinal + kMapPrefetchDistance < document_count) { + __builtin_prefetch(&source_mapping_[docids[ordinal + kMapPrefetchDistance]]); + } + const auto [segment, docid] = source_mapping_[docids[ordinal]]; + if (segment != current_segment) { + if (!runs.empty()) { + runs.back().document_end = static_cast(ordinal); + } + runs.push_back({.destination_segment = segment}); + current_segment = segment; + } + docids[ordinal] = docid; + } + } else if (!term_has_positions_) { + constexpr uint32_t kDeleted = std::numeric_limits::max(); + size_t live_docs = 0; + for (uint32_t source_docid : docids) { + const auto [segment, docid] = source_mapping_[source_docid]; + if (segment != kDeleted) { + append_live_document(segment, docid, live_docs++); + } + } + docids.resize(live_docs); + } else { + constexpr uint32_t kDeleted = std::numeric_limits::max(); + size_t write_position = 0; + size_t live_docs = 0; + workspace_->position_offsets[0] = 0; + for (size_t ordinal = 0; ordinal < docids.size(); ++ordinal) { + const auto [segment, docid] = source_mapping_[docids[ordinal]]; + if (segment == kDeleted) { + continue; + } + const size_t begin = workspace_->position_offsets[ordinal]; + const size_t end = workspace_->position_offsets[ordinal + 1]; + const uint32_t frequency = static_cast(end - begin); + if (write_position != begin) { + std::copy(workspace_->positions_flat.begin() + begin, + workspace_->positions_flat.begin() + end, + workspace_->positions_flat.begin() + write_position); + } + write_position += frequency; + append_live_document(segment, docid, live_docs); + frequencies.push_back(frequency); + workspace_->position_offsets[++live_docs] = static_cast(write_position); + } + docids.resize(live_docs); + workspace_->positions_flat.resize(write_position); + workspace_->position_offsets.resize(live_docs + 1); + } + + if (!runs.empty()) { + runs.back().document_end = static_cast(docids.size()); + } + next_destination_run_ = 0; + return Status::OK(); +} + +void SniiPostingCursor::emit_next_mapped_run(RemappedPostingChunk* chunk) { + DCHECK(chunk != nullptr); + DCHECK(workspace_ != nullptr); + DCHECK_LT(next_destination_run_, workspace_->destination_runs.size()); + + const size_t run_ordinal = next_destination_run_++; + const DestinationPostingRun& run = workspace_->destination_runs[run_ordinal]; + const size_t document_begin = + run_ordinal == 0 ? 0 : workspace_->destination_runs[run_ordinal - 1].document_end; + const size_t document_end = run.document_end; + DCHECK_LT(document_begin, document_end); + DCHECK_LE(document_end, workspace_->docids.size()); + const size_t document_count = document_end - document_begin; + + chunk->destination_segment = run.destination_segment; + chunk->destination_docids = + std::span(workspace_->docids).subspan(document_begin, document_count); + if (!term_has_positions_) { + return; + } + + chunk->freqs = std::span(workspace_->frequencies) + .subspan(document_begin, document_count); + chunk->position_offsets = std::span(workspace_->position_offsets) + .subspan(document_begin, document_count + 1); + const size_t position_begin = chunk->position_offsets.front(); + const size_t position_end = chunk->position_offsets.back(); + DCHECK_LE(position_begin, position_end); + DCHECK_LE(position_end, workspace_->positions_flat.size()); + chunk->positions_flat = std::span(workspace_->positions_flat) + .subspan(position_begin, position_end - position_begin); +} + +Status SniiPostingCursor::next_chunk(RemappedPostingChunk* chunk, bool* has_chunk) { + if (chunk == nullptr || has_chunk == nullptr) { + return Status::Error( + "posting_cursor: null chunk output"); + } + *chunk = {}; + *has_chunk = false; + if (!failed_.ok()) { + return failed_; + } + if (!initialized_) { + return Status::Error( + "posting_cursor: next_chunk before init"); + } + if (exhausted_) { + return Status::OK(); + } + + if (next_destination_run_ < workspace_->destination_runs.size()) { + emit_next_mapped_run(chunk); + *has_chunk = true; + return Status::OK(); + } + + while (true) { + bool loaded = false; + const Status status = load_next_chunk(&loaded); + if (!status.ok()) { + return poison(status); + } + if (!loaded) { + *chunk = {}; + const Status finish_status = finish_source(); + if (!finish_status.ok()) { + return poison(finish_status); + } + return Status::OK(); + } + const Status map_status = map_decoded_chunk(); + if (!map_status.ok()) { + return poison(map_status); + } + if (!workspace_->destination_runs.empty()) { + emit_next_mapped_run(chunk); + *has_chunk = true; + return Status::OK(); + } + *chunk = {}; + } +} + +} // namespace doris::snii::compaction diff --git a/be/src/storage/index/snii/compaction/posting_cursor.h b/be/src/storage/index/snii/compaction/posting_cursor.h new file mode 100644 index 00000000000000..356f08d7a9326a --- /dev/null +++ b/be/src/storage/index/snii/compaction/posting_cursor.h @@ -0,0 +1,258 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/compaction/region_reader.h" +#include "storage/index/snii/compaction/rowid_conversion.h" +#include "storage/index/snii/format/core_metadata.h" +#include "storage/index/snii/format/dict_entry.h" +#include "storage/index/snii/format/frq_prelude.h" +#include "storage/index/snii/format/prx_decode_stats.h" +#include "storage/index/snii/reader/logical_index_reader.h" + +namespace doris::snii::compaction { + +struct DestinationPostingRun { + uint32_t destination_segment = 0; + uint32_t document_end = 0; +}; + +// One destination-homogeneous slice of a decoded physical chunk. Positioned +// runs retain the decoder's offset base; positions_flat covers exactly +// [position_offsets.front(), position_offsets.back()). +struct RemappedPostingChunk { + uint32_t destination_segment = 0; + std::span destination_docids; + std::span freqs; + std::span position_offsets; + std::span positions_flat; +}; + +// Checks [region.offset, region.offset + region.length) against the source +// file without ever evaluating the potentially-overflowing addition. +Status validate_posting_region(const format::RegionRef& region, uint64_t file_size); +bool posting_entry_has_positions(const format::DictEntry& entry); + +class SniiPostingCursor; + +// Persistent read/decode state for every term decoded from one source logical +// index. Docs and positions share one bounded physical chunk cache while their +// decoder vectors and prelude workspace survive term cursor destruction. +class SniiPostingReadContext { +public: + static constexpr size_t kMaxReadAheadBudgetBytes = + 2 * SequentialRegionReader::kDefaultChunkBytes; + static constexpr size_t kMaxRetainedDecoderWorkspaceBytes = 64ULL << 10; + + SniiPostingReadContext(const reader::LogicalIndexReader* index, + size_t total_read_ahead_budget_bytes, + writer::MemoryReporter* memory_reporter = nullptr) + : index_(index), + total_read_ahead_budget_bytes_(total_read_ahead_budget_bytes), + memory_reporter_(memory_reporter) {} + + SniiPostingReadContext(const SniiPostingReadContext&) = delete; + SniiPostingReadContext& operator=(const SniiPostingReadContext&) = delete; + SniiPostingReadContext(SniiPostingReadContext&&) = delete; + SniiPostingReadContext& operator=(SniiPostingReadContext&&) = delete; + + Status init(); + + const reader::LogicalIndexReader* index() const { return index_; } + bool initialized() const { return initialized_; } + size_t total_read_ahead_budget_bytes() const { return total_read_ahead_budget_bytes_; } + uint64_t docs_read_calls() const; + uint64_t prx_read_calls() const; + uint64_t docs_buffer_hits() const; + uint64_t prx_buffer_hits() const; + uint64_t physical_read_ranges() const; + uint64_t physical_read_bytes() const; + size_t resident_read_ahead_capacity_bytes() const; + size_t decoder_workspace_capacity_bytes() const; + size_t retained_decoder_workspace_limit_bytes() const { + return std::min(total_read_ahead_budget_bytes_, kMaxRetainedDecoderWorkspaceBytes); + } + +private: + friend class SniiPostingCursor; + + struct DecoderWorkspace final : public format::PrxCsrAllocationGate { + format::FrqPreludeReader prelude; + writer::MemoryReporter::Reservation docs_scratch_reservation; + writer::MemoryReporter::Reservation prx_scratch_reservation; + writer::MemoryReporter::Reservation docids_reservation; + writer::MemoryReporter::Reservation positions_reservation; + writer::MemoryReporter::Reservation position_offsets_reservation; + writer::MemoryReporter::Reservation destination_runs_reservation; + writer::MemoryReporter::Reservation frequencies_reservation; + writer::MemoryReporter::Reservation decompressed_reservation; + std::vector docs_scratch; + std::vector prx_scratch; + std::vector decompressed; + std::vector docids; + std::vector positions_flat; + std::vector position_offsets; + std::vector destination_runs; + std::vector frequencies; + bool reservations_enabled = false; + + size_t capacity_bytes() const; + void init_memory_reporter(writer::MemoryReporter* memory_reporter); + Status reserve_docids(size_t count); + Status reserve_remapped(size_t document_count, size_t run_count, bool retain_frequencies); + Status reserve_csr(std::vector* pos_flat, size_t position_count, + std::vector* pos_off, size_t offset_count) override; + Status reserve_decompression(size_t bytes, std::vector** buffer) override; + void release_large_buffers(size_t retained_capacity_limit_bytes); + }; + + class TermLease { + public: + ~TermLease(); + + TermLease(const TermLease&) = delete; + TermLease& operator=(const TermLease&) = delete; + + private: + friend class SniiPostingReadContext; + explicit TermLease(SniiPostingReadContext* context) : context_(context) {} + SniiPostingReadContext* context_ = nullptr; + }; + + Status acquire_term(bool has_docs_range, bool has_prx_range, + const format::RegionRef& docs_range, const format::RegionRef& prx_range, + std::unique_ptr* lease); + Status validate_next_range(const format::RegionRef& range, bool has_previous, + uint64_t previous_end, const char* stream, uint64_t* end) const; + Status poison(Status status); + Status poison_active_term(Status status, TermLease* lease); + void release_term(); + const Status& failed_status() const { return failed_; } + + const reader::LogicalIndexReader* index_ = nullptr; + size_t total_read_ahead_budget_bytes_ = 0; + writer::MemoryReporter* memory_reporter_ = nullptr; + format::RegionRef posting_region_; + std::unique_ptr posting_cache_; + DecoderWorkspace decoder_workspace_; + + uint64_t last_docs_end_ = 0; + uint64_t last_prx_end_ = 0; + bool has_docs_range_ = false; + bool has_prx_range_ = false; + bool term_active_ = false; + bool initialized_ = false; + Status failed_ = Status::OK(); +}; + +// Sequential positions-posting decoder for one term in one source SNII index. +// It accepts all v1 physical shapes (inline, slim POD-ref and windowed POD-ref), +// validates the decoded doc/frequency/position stream, applies a validated +// row-id conversion, and yields surviving rows in destination-order chunks. +// +// The borrowed read context and row-id capability must outlive the cursor. A +// cursor is single-use and single-threaded. Only one cursor may hold a context +// term lease at a time. +class SniiPostingCursor { +public: + SniiPostingCursor(SniiPostingReadContext* read_context, format::DictEntry entry, + uint64_t frq_base, uint64_t prx_base, uint32_t source_ordinal, + const ValidatedRowIdConversion* rowid_conversion) + : read_context_(read_context), + index_(read_context == nullptr ? nullptr : read_context->index()), + entry_(std::move(entry)), + frq_base_(frq_base), + prx_base_(prx_base), + source_ordinal_(source_ordinal), + rowid_conversion_(rowid_conversion), + term_has_positions_(posting_entry_has_positions(entry_)) {} + + // Validates index capability, posting locators and fixed entry geometry, + // then acquires the source context's exclusive term lease. Payload decoding + // is lazy: corrupt DD/PRX frames surface from next_chunk(). + Status init(); + + // Returns the next non-empty remapped chunk. Its spans, including an exact + // base-relative positions slice, remain valid until the next call. + // Decoder/corruption failures poison the cursor and are returned unchanged + // by subsequent calls. + Status next_chunk(RemappedPostingChunk* chunk, bool* has_chunk); + bool has_positions() const { return term_has_positions_; } + +private: + enum class Shape : uint8_t { kFlat, kWindowed }; + + Status validate_entry_geometry(); + Status prepare_flat_ranges(); + Status prepare_windowed_ranges(); + Status prepare_windowed(); + Status load_next_chunk(bool* loaded); + Status load_flat_chunk(); + Status load_windowed_chunk(); + Status decode_dd(Slice bytes, const format::FrqRegionMeta& meta, uint64_t win_base, + uint32_t expected_doc_count); + Status decode_prx(Slice bytes, format::PrxDecodedShape* shape); + Status map_decoded_chunk(); + void emit_next_mapped_run(RemappedPostingChunk* chunk); + Status finish_source(); + Status poison(Status status); + + SniiPostingReadContext* read_context_ = nullptr; + const reader::LogicalIndexReader* index_ = nullptr; + format::DictEntry entry_; + uint64_t frq_base_ = 0; + uint64_t prx_base_ = 0; + uint32_t source_ordinal_ = 0; + const ValidatedRowIdConversion* rowid_conversion_ = nullptr; + std::span> source_mapping_; + bool source_has_deletions_ = false; + bool term_has_positions_ = true; + + Shape shape_ = Shape::kFlat; + std::unique_ptr term_lease_; + SniiPostingReadContext::DecoderWorkspace* workspace_ = nullptr; + + uint64_t flat_dd_abs_ = 0; + uint64_t flat_dd_len_ = 0; + uint64_t flat_prx_abs_ = 0; + uint64_t flat_prx_len_ = 0; + uint32_t next_window_ = 0; + bool flat_loaded_ = false; + + uint64_t decoded_docs_ = 0; + uint64_t decoded_total_freq_ = 0; + uint32_t decoded_max_freq_ = 0; + size_t next_destination_run_ = 0; + + bool initialized_ = false; + bool exhausted_ = false; + Status failed_ = Status::OK(); +}; + +} // namespace doris::snii::compaction diff --git a/be/src/storage/index/snii/compaction/posting_run_merger.cpp b/be/src/storage/index/snii/compaction/posting_run_merger.cpp new file mode 100644 index 00000000000000..bdd5b31c8742cd --- /dev/null +++ b/be/src/storage/index/snii/compaction/posting_run_merger.cpp @@ -0,0 +1,463 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/compaction/posting_run_merger.h" + +#include +#include +#include +#include +#include + +#include "common/check.h" + +namespace doris::snii::compaction { + +namespace { + +Status invalid_source(std::string_view reason) { + return Status::Error("posting_run_merger: {}", reason); +} + +Status merge_corruption(std::string_view reason) { + return Status::Error("posting_run_merger: {}", + reason); +} + +bool posting_after(uint32_t lhs_segment, uint32_t lhs_docid, uint32_t rhs_segment, + uint32_t rhs_docid) { + return lhs_segment > rhs_segment || (lhs_segment == rhs_segment && lhs_docid > rhs_docid); +} + +#ifdef BE_TEST +std::atomic posting_run_frontier_update_counter {0}; +std::atomic posting_run_frontier_comparison_counter {0}; +std::atomic posting_run_document_counter {0}; +std::atomic posting_run_emitted_run_counter {0}; +std::atomic posting_run_boundary_search_counter {0}; +std::atomic posting_run_shape_scan_document_counter {0}; +std::atomic posting_run_legacy_fill_call_counter {0}; +std::atomic posting_run_copied_document_counter {0}; +#endif + +size_t lower_bound_docid(std::span docids, size_t begin, uint32_t target) { +#ifdef BE_TEST + posting_run_boundary_search_counter.fetch_add(1, std::memory_order_relaxed); +#endif + const size_t size = docids.size(); + if (begin >= size || docids[begin] >= target) { + return begin; + } + // Gallop before the binary search: when many sources interleave (a full + // compaction merging dozens of sorted segments), destination runs shrink to + // one or two documents, and a plain binary search over the whole remaining + // chunk costs O(log chunk) comparisons per emitted run. Doubling probes + // resolve those short runs in O(1) while keeping O(log run) for long runs. + size_t less = begin; + size_t probe = 1; + while (less + probe < size && docids[less + probe] < target) { + less += probe; + probe *= 2; + } + size_t low = less + 1; + size_t high = std::min(less + probe, size); + while (low < high) { + const size_t middle = low + (high - low) / 2; + if (docids[middle] < target) { + low = middle + 1; + } else { + high = middle; + } + } + return low; +} + +} // namespace + +void MergedPostingRuns::ActivePostingChunk::refresh_frontier() { + DCHECK_LT(ordinal, chunk.destination_docids.size()); + frontier_segment = chunk.destination_segment; + frontier_docid = chunk.destination_docids[ordinal]; +} + +Status MergedPostingRuns::ActivePostingChunk::validate_and_refresh_frontier( + bool retain_positions, std::span destination_doc_counts) { + if (chunk.destination_docids.empty() || ordinal >= chunk.destination_docids.size()) { + return merge_corruption("destination posting run is empty"); + } + if (chunk.destination_segment >= destination_doc_counts.size()) { + return merge_corruption("destination posting run segment is out of range"); + } + if (retain_positions) { + if (chunk.freqs.size() != chunk.destination_docids.size() || + chunk.position_offsets.size() != chunk.destination_docids.size() + 1) { + return merge_corruption("positioned posting run has an invalid shape"); + } + const uint32_t position_begin = chunk.position_offsets.front(); + const uint32_t position_end = chunk.position_offsets.back(); + if (position_end < position_begin || + position_end - position_begin != chunk.positions_flat.size()) { + return merge_corruption("positioned posting run has invalid offsets"); + } + } else if (!chunk.freqs.empty() || !chunk.position_offsets.empty() || + !chunk.positions_flat.empty()) { + return merge_corruption("docs-only posting run has positioned payload"); + } + + uint32_t previous_offset = retain_positions ? chunk.position_offsets.front() : 0; + for (size_t document = 0; document < chunk.destination_docids.size(); ++document) { + const uint32_t docid = chunk.destination_docids[document]; + if (docid >= destination_doc_counts[chunk.destination_segment]) { + return merge_corruption("destination posting run document is out of range"); + } + if (document > 0 && docid <= chunk.destination_docids[document - 1]) { + return merge_corruption("destination posting run is not strictly monotone"); + } + if (retain_positions) { + const uint32_t next_offset = chunk.position_offsets[document + 1]; + if (next_offset < previous_offset || + next_offset - previous_offset != chunk.freqs[document]) { + return merge_corruption("positioned posting run offsets differ from frequencies"); + } + previous_offset = next_offset; + } +#ifdef BE_TEST + posting_run_shape_scan_document_counter.fetch_add(1, std::memory_order_relaxed); +#endif + } + if (has_previous_chunk_posting && + !posting_after(chunk.destination_segment, chunk.destination_docids.front(), + previous_chunk_segment, previous_chunk_docid)) { + return merge_corruption("source posting chunks are not globally monotone"); + } + previous_chunk_segment = chunk.destination_segment; + previous_chunk_docid = chunk.destination_docids.back(); + has_previous_chunk_posting = true; + refresh_frontier(); + return Status::OK(); +} + +bool MergedPostingRuns::FrontierBefore::operator()(size_t lhs, size_t rhs) const { +#ifdef BE_TEST + posting_run_frontier_comparison_counter.fetch_add(1, std::memory_order_relaxed); +#endif + const ActivePostingChunk& lhs_chunk = (*active_chunks)[lhs]; + const ActivePostingChunk& rhs_chunk = (*active_chunks)[rhs]; + if (lhs_chunk.frontier_segment != rhs_chunk.frontier_segment) { + return lhs_chunk.frontier_segment < rhs_chunk.frontier_segment; + } + if (lhs_chunk.frontier_docid != rhs_chunk.frontier_docid) { + return lhs_chunk.frontier_docid < rhs_chunk.frontier_docid; + } + return lhs < rhs; +} + +MergedPostingRuns::MergedPostingRuns(std::vector> cursors, + bool retain_positions, bool counts_as_semantic_token, + std::span destination_doc_counts, + std::span destination_semantic_token_counts) + : cursors_(std::move(cursors)), + active_frontier_(FrontierBefore {.active_chunks = &active_chunks_}), + retain_positions_(retain_positions), + counts_as_semantic_token_(counts_as_semantic_token), + destination_doc_counts_(destination_doc_counts), + destination_semantic_token_counts_(destination_semantic_token_counts) {} + +Status MergedPostingRuns::init() { + if (initialized_) { + return invalid_source("source initialized twice"); + } + if (cursors_.empty() || destination_doc_counts_.empty()) { + return invalid_source("source or destination set is empty"); + } + if (counts_as_semantic_token_ && + destination_semantic_token_counts_.size() != destination_doc_counts_.size()) { + return invalid_source("semantic token counters differ from destination count"); + } + active_chunks_.resize(cursors_.size()); + for (size_t cursor_ordinal = 0; cursor_ordinal < cursors_.size(); ++cursor_ordinal) { + bool has_chunk = false; + RETURN_IF_ERROR(cursors_[cursor_ordinal]->next_chunk(&active_chunks_[cursor_ordinal].chunk, + &has_chunk)); + if (has_chunk) { + RETURN_IF_ERROR(active_chunks_[cursor_ordinal].validate_and_refresh_frontier( + retain_positions_, destination_doc_counts_)); + } + } + active_frontier_.build(cursors_.size(), [this](size_t source) { + return !active_chunks_[source].chunk.destination_docids.empty(); + }); + initialized_ = true; + return Status::OK(); +} + +bool MergedPostingRuns::empty() const { + DCHECK(initialized_); + return active_frontier_.empty(); +} + +uint32_t MergedPostingRuns::next_destination() const { + DCHECK(initialized_); + DCHECK(!active_destination_.has_value()); + DCHECK(!pending_source_.has_value()); + return front_segment(); +} + +Status MergedPostingRuns::begin_destination(uint32_t destination) { + if (!initialized_ || active_destination_.has_value() || pending_source_.has_value() || + active_frontier_.empty()) { + return invalid_source("cannot begin destination"); + } + if (front_segment() != destination) { + return invalid_source("destination differs from frontier"); + } + active_destination_ = destination; + return Status::OK(); +} + +Status MergedPostingRuns::next_run(uint32_t max_docs, writer::PostingRunView* run, bool* has_run) { + if (max_docs == 0 || run == nullptr || has_run == nullptr) { + return invalid_source("invalid next_run arguments"); + } + if (!initialized_ || !active_destination_.has_value()) { + return invalid_source("source has no active destination"); + } + *run = {}; + *has_run = false; + RETURN_IF_ERROR(settle_pending_run()); + if (active_frontier_.empty() || front_segment() != *active_destination_) { + active_destination_.reset(); + return Status::OK(); + } + RETURN_IF_ERROR(select_front_run(max_docs, run)); + *has_run = true; + return Status::OK(); +} + +Status MergedPostingRuns::fill(uint32_t target_docs, writer::TermPostingBuffer* out, + bool* exhausted) { + if (target_docs == 0 || out == nullptr || exhausted == nullptr) { + return invalid_source("invalid fill arguments"); + } + if (!initialized_ || !active_destination_.has_value()) { + return invalid_source("source has no active destination"); + } + if (!out->empty()) { + return invalid_source("output must be empty"); + } +#ifdef BE_TEST + posting_run_legacy_fill_call_counter.fetch_add(1, std::memory_order_relaxed); +#endif + + while (out->document_count() < target_docs) { + writer::PostingRunView run; + bool has_run = false; + RETURN_IF_ERROR(next_run(static_cast(target_docs - out->document_count()), &run, + &has_run)); + if (!has_run) { + *exhausted = true; + return Status::OK(); + } + const size_t position_count = run.positions_flat.size(); + writer::MutableTermPostingSpan destination; + RETURN_IF_ERROR(out->grow_uninitialized(run.docids.size(), retain_positions_, + position_count, &destination)); + std::ranges::copy(run.docids, destination.docids.begin()); + if (retain_positions_) { + std::ranges::copy(run.freqs, destination.freqs.begin()); + std::ranges::copy(run.positions_flat, destination.positions_flat.begin()); + } +#ifdef BE_TEST + posting_run_copied_document_counter.fetch_add(run.docids.size(), std::memory_order_relaxed); +#endif + } + + RETURN_IF_ERROR(settle_pending_run()); + *exhausted = active_frontier_.empty() || front_segment() != *active_destination_; + if (*exhausted) { + active_destination_.reset(); + } + return Status::OK(); +} + +uint32_t MergedPostingRuns::front_segment() const { + return active_chunks_[active_frontier_.winner()].frontier_segment; +} + +Status MergedPostingRuns::select_front_run(size_t max_docs, writer::PostingRunView* run) { + DCHECK_GT(max_docs, 0); + DCHECK(!pending_source_.has_value()); + const size_t cursor_ordinal = active_frontier_.winner(); + std::optional> next_frontier; + const size_t runner_up = active_frontier_.runner_up(); + if (runner_up != IndexedWinnerTree::kNoSource) { + const ActivePostingChunk& next = active_chunks_[runner_up]; + next_frontier = std::pair(next.frontier_segment, next.frontier_docid); + } + + ActivePostingChunk& active = active_chunks_[cursor_ordinal]; + RETURN_IF_ERROR(select_run(&active, max_docs, next_frontier, run)); + pending_source_ = cursor_ordinal; + return Status::OK(); +} + +Status MergedPostingRuns::select_run(ActivePostingChunk* active, size_t max_docs, + std::optional> next_frontier, + writer::PostingRunView* run) { + DCHECK(active != nullptr); + const auto docids = active->chunk.destination_docids; + if (active->chunk.destination_segment != *active_destination_) { + return merge_corruption("posting run differs from the active destination"); + } + const size_t begin = active->ordinal; + size_t end = begin + std::min(max_docs, docids.size() - begin); + if (next_frontier.has_value() && next_frontier->first == active->chunk.destination_segment) { + end = std::min(end, lower_bound_docid(docids, begin, next_frontier->second)); + } + if (end == begin) { + return merge_corruption("destination postings contain an equal merge frontier"); + } + if (has_previous_posting_ && !posting_after(active->frontier_segment, active->frontier_docid, + previous_segment_, previous_docid_)) { + return merge_corruption("destination postings are duplicated or not globally monotone"); + } + + const size_t document_count = end - begin; + size_t position_begin = 0; + size_t position_count = 0; + if (retain_positions_) { + const size_t position_base = active->chunk.position_offsets.front(); + const size_t absolute_position_begin = active->chunk.position_offsets[begin]; + const size_t absolute_position_end = active->chunk.position_offsets[end]; + DCHECK_GE(absolute_position_begin, position_base); + DCHECK_GE(absolute_position_end, absolute_position_begin); + DCHECK_LE(absolute_position_end - position_base, active->chunk.positions_flat.size()); + position_begin = absolute_position_begin - position_base; + position_count = absolute_position_end - absolute_position_begin; + } + + run->docids = docids.subspan(begin, document_count); + run->freqs = retain_positions_ ? active->chunk.freqs.subspan(begin, document_count) + : std::span {}; + run->position_offsets = + retain_positions_ ? active->chunk.position_offsets.subspan(begin, document_count + 1) + : std::span {}; + run->positions_flat = + retain_positions_ ? active->chunk.positions_flat.subspan(position_begin, position_count) + : std::span {}; + + if (counts_as_semantic_token_) { + uint64_t& token_count = destination_semantic_token_counts_[*active_destination_]; + if (position_count > std::numeric_limits::max() - token_count) { + return merge_corruption("semantic token count overflows uint64"); + } + token_count += position_count; + } + previous_segment_ = active->chunk.destination_segment; + previous_docid_ = docids[end - 1]; + has_previous_posting_ = true; + active->ordinal = end; +#ifdef BE_TEST + posting_run_document_counter.fetch_add(document_count, std::memory_order_relaxed); + posting_run_emitted_run_counter.fetch_add(1, std::memory_order_relaxed); +#endif + return Status::OK(); +} + +Status MergedPostingRuns::settle_pending_run() { + if (!pending_source_.has_value()) { + return Status::OK(); + } + const size_t cursor_ordinal = *pending_source_; + RETURN_IF_ERROR(advance_front_source(cursor_ordinal, &active_chunks_[cursor_ordinal])); + pending_source_.reset(); + return Status::OK(); +} + +Status MergedPostingRuns::advance_front_source(size_t cursor_ordinal, ActivePostingChunk* active) { + DCHECK(active != nullptr); + bool has_chunk = active->ordinal < active->chunk.destination_docids.size(); + if (!has_chunk) { + active->chunk = {}; + active->ordinal = 0; + RETURN_IF_ERROR(cursors_[cursor_ordinal]->next_chunk(&active->chunk, &has_chunk)); + } + if (has_chunk) { + if (active->ordinal == 0) { + RETURN_IF_ERROR(active->validate_and_refresh_frontier(retain_positions_, + destination_doc_counts_)); + } else { + active->refresh_frontier(); + } + } + active_frontier_.update(cursor_ordinal, has_chunk); +#ifdef BE_TEST + posting_run_frontier_update_counter.fetch_add(1, std::memory_order_relaxed); +#endif + return Status::OK(); +} + +#ifdef BE_TEST +namespace testing { + +void reset_posting_run_merge_counters() { + posting_run_frontier_update_counter.store(0, std::memory_order_relaxed); + posting_run_frontier_comparison_counter.store(0, std::memory_order_relaxed); + posting_run_document_counter.store(0, std::memory_order_relaxed); + posting_run_emitted_run_counter.store(0, std::memory_order_relaxed); + posting_run_boundary_search_counter.store(0, std::memory_order_relaxed); + posting_run_shape_scan_document_counter.store(0, std::memory_order_relaxed); + posting_run_legacy_fill_call_counter.store(0, std::memory_order_relaxed); + posting_run_copied_document_counter.store(0, std::memory_order_relaxed); +} + +uint64_t posting_run_frontier_updates() { + return posting_run_frontier_update_counter.load(std::memory_order_relaxed); +} + +uint64_t posting_run_frontier_comparisons() { + return posting_run_frontier_comparison_counter.load(std::memory_order_relaxed); +} + +uint64_t posting_run_documents() { + return posting_run_document_counter.load(std::memory_order_relaxed); +} + +uint64_t posting_run_emitted_runs() { + return posting_run_emitted_run_counter.load(std::memory_order_relaxed); +} + +uint64_t posting_run_boundary_searches() { + return posting_run_boundary_search_counter.load(std::memory_order_relaxed); +} + +uint64_t posting_run_shape_scan_documents() { + return posting_run_shape_scan_document_counter.load(std::memory_order_relaxed); +} + +uint64_t posting_run_legacy_fill_calls() { + return posting_run_legacy_fill_call_counter.load(std::memory_order_relaxed); +} + +uint64_t posting_run_copied_documents() { + return posting_run_copied_document_counter.load(std::memory_order_relaxed); +} + +} // namespace testing +#endif + +} // namespace doris::snii::compaction diff --git a/be/src/storage/index/snii/compaction/posting_run_merger.h b/be/src/storage/index/snii/compaction/posting_run_merger.h new file mode 100644 index 00000000000000..7792a862d35f22 --- /dev/null +++ b/be/src/storage/index/snii/compaction/posting_run_merger.h @@ -0,0 +1,116 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/compaction/indexed_winner_tree.h" +#include "storage/index/snii/compaction/posting_cursor.h" +#include "storage/index/snii/writer/posting_window_emitter.h" +#include "storage/index/snii/writer/term_posting_source.h" + +namespace doris::snii::compaction { + +// K-way merge of destination-homogeneous cursor chunks. next_run() borrows the +// selected cursor workspace without copying; the view remains valid until a +// later next_run() call advances that cursor. +class MergedPostingRuns final : public writer::TermPostingSource { + struct ActivePostingChunk { + RemappedPostingChunk chunk; + size_t ordinal = 0; + uint32_t frontier_segment = 0; + uint32_t frontier_docid = 0; + uint32_t previous_chunk_segment = 0; + uint32_t previous_chunk_docid = 0; + bool has_previous_chunk_posting = false; + + void refresh_frontier(); + Status validate_and_refresh_frontier(bool retain_positions, + std::span destination_doc_counts); + }; + + struct FrontierBefore { + const std::vector* active_chunks = nullptr; + + bool operator()(size_t lhs, size_t rhs) const; + }; + +public: + MergedPostingRuns(std::vector> cursors, + bool retain_positions, bool counts_as_semantic_token, + std::span destination_doc_counts, + std::span destination_semantic_token_counts); + + Status init(); + bool empty() const; + uint32_t next_destination() const; + Status begin_destination(uint32_t destination); + Status next_run(uint32_t max_docs, writer::PostingRunView* run, bool* has_run); + + // Transitional streamed-session adapter. Compaction tests use next_run() + // directly; the assembler integration removes this handoff in the next + // milestone. + Status fill(uint32_t target_docs, writer::TermPostingBuffer* out, bool* exhausted) override; + +private: + uint32_t front_segment() const; + Status select_front_run(size_t max_docs, writer::PostingRunView* run); + Status select_run(ActivePostingChunk* active, size_t max_docs, + std::optional> next_frontier, + writer::PostingRunView* run); + Status settle_pending_run(); + Status advance_front_source(size_t cursor_ordinal, ActivePostingChunk* active); + + std::vector> cursors_; + std::vector active_chunks_; + IndexedWinnerTree active_frontier_; + bool retain_positions_ = true; + bool counts_as_semantic_token_ = false; + std::span destination_doc_counts_; + std::span destination_semantic_token_counts_; + std::optional active_destination_; + std::optional pending_source_; + uint32_t previous_segment_ = 0; + uint32_t previous_docid_ = 0; + bool has_previous_posting_ = false; + bool initialized_ = false; +}; + +#ifdef BE_TEST +namespace testing { + +void reset_posting_run_merge_counters(); +uint64_t posting_run_frontier_updates(); +uint64_t posting_run_frontier_comparisons(); +uint64_t posting_run_documents(); +uint64_t posting_run_emitted_runs(); +uint64_t posting_run_boundary_searches(); +uint64_t posting_run_shape_scan_documents(); +uint64_t posting_run_legacy_fill_calls(); +uint64_t posting_run_copied_documents(); + +} // namespace testing +#endif + +} // namespace doris::snii::compaction diff --git a/be/src/storage/index/snii/compaction/region_reader.cpp b/be/src/storage/index/snii/compaction/region_reader.cpp new file mode 100644 index 00000000000000..3832b8505523ef --- /dev/null +++ b/be/src/storage/index/snii/compaction/region_reader.cpp @@ -0,0 +1,274 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/compaction/region_reader.h" + +#include +#include + +namespace doris::snii::compaction { + +namespace { + +Status reserve_read_buffer(size_t target, std::vector* buffer, + writer::MemoryReporter::Reservation* reservation) { + if (reservation == nullptr) return Status::OK(); + if (buffer->capacity() >= target) { + DCHECK_EQ(reservation->bytes(), buffer->capacity()); + return Status::OK(); + } + writer::MemoryReporter::Reservation replacement; + RETURN_IF_ERROR(reservation->prepare_replacement(target, &replacement)); + buffer->reserve(target); + DCHECK_EQ(buffer->capacity(), target); + *reservation = std::move(replacement); + return Status::OK(); +} + +} // namespace + +size_t SharedAlignedRegionCache::stream_index(PostingStream stream) { + const size_t index = static_cast(stream); + DCHECK_LT(index, kStreamCount); + return index; +} + +Status SharedAlignedRegionCache::init() { + if (initialized_) { + return Status::Error( + "shared_region_cache: init called twice"); + } + if (reader_ == nullptr) { + return Status::Error( + "shared_region_cache: null reader"); + } + if (total_budget_bytes_ < kSlotCount) { + return Status::Error( + "shared_region_cache: budget must hold two chunks"); + } + if (region_len_ > std::numeric_limits::max() - region_off_ || + region_off_ > reader_->size() || region_len_ > reader_->size() - region_off_) { + return Status::Error( + "shared_region_cache: region outside source file"); + } + + block_bytes_ = total_budget_bytes_ / kSlotCount; + for (size_t slot_index = 0; slot_index < slots_.size(); ++slot_index) { + Slot& slot = slots_[slot_index]; + if (memory_reporter_ != nullptr) { + slot_reservations_[slot_index] = memory_reporter_->make_reservation(); + RETURN_IF_ERROR(slot_reservations_[slot_index].set_bytes(block_bytes_)); + slot.bytes.reserve(block_bytes_); + DCHECK_EQ(slot.bytes.capacity(), block_bytes_); + } + slot.bytes.resize(block_bytes_); + } + if (resident_capacity_bytes() > total_budget_bytes_) { + return Status::Error( + "shared_region_cache: allocator exceeded cache budget"); + } + initialized_ = true; + return Status::OK(); +} + +size_t SharedAlignedRegionCache::resident_capacity_bytes() const { + size_t capacity = 0; + for (const Slot& slot : slots_) { + capacity += slot.bytes.capacity(); + } + return capacity; +} + +uint64_t SharedAlignedRegionCache::read_calls(PostingStream stream) const { + return read_calls_[stream_index(stream)]; +} + +uint64_t SharedAlignedRegionCache::buffer_hits(PostingStream stream) const { + return buffer_hits_[stream_index(stream)]; +} + +void SharedAlignedRegionCache::unpin(PostingStream stream) { + const size_t index = stream_index(stream); + const int8_t slot_index = stream_slots_[index]; + if (slot_index < 0) { + return; + } + Slot& slot = slots_[static_cast(slot_index)]; + DCHECK_GT(slot.pins, 0); + --slot.pins; + stream_slots_[index] = -1; +} + +Status SharedAlignedRegionCache::read_physical(PostingStream stream, uint64_t abs_off, size_t len, + std::vector* out) { + RETURN_IF_ERROR(reader_->read_at(abs_off, len, out)); + ++physical_read_ranges_; + physical_read_bytes_ += len; + ++read_calls_[stream_index(stream)]; + return Status::OK(); +} + +Status SharedAlignedRegionCache::resolve(PostingStream stream, uint64_t abs_off, uint64_t len, + std::vector* scratch, Slice* out, + writer::MemoryReporter::Reservation* scratch_reservation) { + if (scratch == nullptr || out == nullptr) { + return Status::Error("shared_region_cache: null out"); + } + if (!initialized_) { + return Status::Error( + "shared_region_cache: resolve before init"); + } + *out = Slice(); + if (abs_off < region_off_) { + return Status::Error( + "shared_region_cache: window outside region"); + } + const uint64_t relative_off = abs_off - region_off_; + if (relative_off > region_len_ || len > region_len_ - relative_off) { + return Status::Error( + "shared_region_cache: window outside region"); + } + if (len > std::numeric_limits::max()) { + return Status::Error( + "shared_region_cache: window length out of range"); + } + + unpin(stream); + const size_t want = static_cast(len); + if (want == 0) { + return Status::OK(); + } + + const uint64_t aligned_relative_off = (relative_off / block_bytes_) * block_bytes_; + const uint64_t block_off = region_off_ + aligned_relative_off; + const size_t block_len = static_cast( + std::min(block_bytes_, region_len_ - aligned_relative_off)); + const size_t offset_in_block = static_cast(relative_off - aligned_relative_off); + const bool fits_one_block = want <= block_len - offset_in_block; + if (!fits_one_block) { + RETURN_IF_ERROR(reserve_read_buffer(want, scratch, scratch_reservation)); + RETURN_IF_ERROR(read_physical(stream, abs_off, want, scratch)); + if (scratch->size() != want) { + return Status::Error( + "shared_region_cache: short exact read"); + } + *out = Slice(scratch->data(), want); + return Status::OK(); + } + + const size_t stream_id = stream_index(stream); + for (size_t slot_index = 0; slot_index < slots_.size(); ++slot_index) { + Slot& slot = slots_[slot_index]; + if (slot.valid && slot.offset == block_off && offset_in_block + want <= slot.bytes.size()) { + ++slot.pins; + stream_slots_[stream_id] = static_cast(slot_index); + ++buffer_hits_[stream_id]; + *out = Slice(slot.bytes.data() + offset_in_block, want); + return Status::OK(); + } + } + + size_t slot_index = 0; + while (slot_index < slots_.size() && slots_[slot_index].pins != 0) { + ++slot_index; + } + DCHECK_LT(slot_index, slots_.size()); + Slot& slot = slots_[slot_index]; + slot.valid = false; + RETURN_IF_ERROR(read_physical(stream, block_off, block_len, &slot.bytes)); + if (slot.bytes.size() != block_len) { + return Status::Error( + "shared_region_cache: short chunk read"); + } + slot.offset = block_off; + slot.pins = 1; + slot.valid = true; + stream_slots_[stream_id] = static_cast(slot_index); + *out = Slice(slot.bytes.data() + offset_in_block, want); + return Status::OK(); +} + +Status SequentialRegionReader::resolve(uint64_t abs_off, uint64_t len, + std::vector* scratch, Slice* out) { + if (scratch == nullptr || out == nullptr) { + return Status::Error("region_reader: null out"); + } + if (reader_ == nullptr) { + return Status::Error("region_reader: null reader"); + } + *out = Slice(); + // Subtraction-based bounds avoid wrapping region_offset+region_length or + // abs_off+len at UINT64_MAX. + if (region_len_ > std::numeric_limits::max() - region_off_ || abs_off < region_off_) { + return Status::Error( + "region_reader: window outside region"); + } + const uint64_t relative_off = abs_off - region_off_; + if (relative_off > region_len_ || len > region_len_ - relative_off) { + return Status::Error( + "region_reader: window outside region"); + } + if (len > std::numeric_limits::max()) { + return Status::Error( + "region_reader: window length out of range"); + } + const size_t want = static_cast(len); + if (want == 0) { + return Status::OK(); + } + + // 1. Buffered hit: zero-copy slice, no file read. + if (!buf_.empty() && abs_off >= buf_off_ && abs_off - buf_off_ + want <= buf_.size()) { + ++buffer_hits_; + *out = Slice(buf_.data() + (abs_off - buf_off_), want); + return Status::OK(); + } + + // 3. Oversized or backward miss: one exact range read into the caller's + // scratch, keeping the buffered chunk (and the forward stream position) + // intact. + const bool backward = !buf_.empty() && abs_off < buf_off_; + if (want > chunk_bytes_ || backward) { + RETURN_IF_ERROR(reader_->read_at(abs_off, want, scratch)); + ++read_calls_; + if (scratch->size() != want) { + return Status::Error( + "region_reader: short read"); + } + *out = Slice(scratch->data(), want); + return Status::OK(); + } + + // 2. Forward miss: refill the chunk starting at the window, clamped to the + // region end so read-ahead never reads past the region (whose tail may abut + // other file sections or EOF). want <= chunk and the range check above + // guarantee the window fits the refilled chunk. + const uint64_t remaining = region_len_ - relative_off; + const size_t fill = static_cast(std::min(chunk_bytes_, remaining)); + RETURN_IF_ERROR(reader_->read_at(abs_off, fill, &buf_)); + ++read_calls_; + if (buf_.size() != fill) { + buf_.clear(); + return Status::Error( + "region_reader: short chunk read"); + } + buf_off_ = abs_off; + *out = Slice(buf_.data(), want); + return Status::OK(); +} + +} // namespace doris::snii::compaction diff --git a/be/src/storage/index/snii/compaction/region_reader.h b/be/src/storage/index/snii/compaction/region_reader.h new file mode 100644 index 00000000000000..63304883cd8ed7 --- /dev/null +++ b/be/src/storage/index/snii/compaction/region_reader.h @@ -0,0 +1,156 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/io/file_reader.h" +#include "storage/index/snii/writer/memory_reporter.h" + +// SequentialRegionReader -- chunked sequential read-ahead over ONE contiguous +// byte region of a source file (T2.3, compaction index-merge fast path). +// +// The merge walks a source segment's posting region in ascending offset order +// (the writer laid the per-term [prx][frq] spans out in term order, and the +// term cursor replays terms in that same order), so per-window read_at calls +// would issue thousands of tiny reads over an already-sequential byte stream. +// This reader amortizes them: resolve() serves a window from the buffered +// chunk when possible and only touches the file on a miss. +// +// resolve() preference order (documented contract, pinned by UT): +// 1. window fully inside the buffered chunk -> zero-copy Slice into the +// buffer, NO file read; +// 2. forward miss with len <= chunk_bytes -> ONE chunk read starting at the +// window (clamped to the region end, never past it) and a slice of it; +// 3. oversized (len > chunk_bytes) or backward window -> ONE exact range +// read into *scratch, leaving the buffered chunk untouched (a rare +// backward probe must not thrash the forward stream). +// A window not fully inside [region_offset, region_offset+region_length) is +// Corruption -- posting locators were already validated against the region by +// LogicalIndexReader::resolve_*_window, so an out-of-region request here means +// a caller bug or corrupt state, never a legal miss. +// +// The returned Slice is valid until the NEXT resolve() call (buffer path) or +// until *scratch is next modified (fallback path); callers decode immediately. +// Single-threaded, borrowed FileReader must outlive the region reader. +namespace doris::snii::compaction { + +enum class PostingStream : uint8_t { kDocs = 0, kPrx = 1 }; + +// Two logical monotone posting streams share two aligned physical chunks. A +// stream pins at most its current chunk, so resolving the other stream cannot +// invalidate an outstanding Slice. Requests crossing an aligned chunk use the +// caller's per-stream scratch and do not evict either pinned chunk. +class SharedAlignedRegionCache { +public: + SharedAlignedRegionCache(io::FileReader* reader, uint64_t region_offset, uint64_t region_length, + size_t total_budget_bytes, + writer::MemoryReporter* memory_reporter = nullptr) + : reader_(reader), + region_off_(region_offset), + region_len_(region_length), + total_budget_bytes_(total_budget_bytes), + memory_reporter_(memory_reporter) {} + + Status init(); + Status resolve(PostingStream stream, uint64_t abs_off, uint64_t len, + std::vector* scratch, Slice* out, + writer::MemoryReporter::Reservation* scratch_reservation = nullptr); + + uint64_t physical_read_ranges() const { return physical_read_ranges_; } + uint64_t physical_read_bytes() const { return physical_read_bytes_; } + uint64_t read_calls(PostingStream stream) const; + uint64_t buffer_hits(PostingStream stream) const; + size_t resident_capacity_bytes() const; + +private: + static constexpr size_t kStreamCount = 2; + static constexpr size_t kSlotCount = 2; + + struct Slot { + std::vector bytes; + uint64_t offset = 0; + uint8_t pins = 0; + bool valid = false; + }; + + static size_t stream_index(PostingStream stream); + void unpin(PostingStream stream); + Status read_physical(PostingStream stream, uint64_t abs_off, size_t len, + std::vector* out); + + io::FileReader* reader_ = nullptr; + uint64_t region_off_ = 0; + uint64_t region_len_ = 0; + size_t total_budget_bytes_ = 0; + size_t block_bytes_ = 0; + writer::MemoryReporter* memory_reporter_ = nullptr; + std::array slot_reservations_; + std::array slots_; + std::array stream_slots_ {-1, -1}; + std::array read_calls_ {}; + std::array buffer_hits_ {}; + uint64_t physical_read_ranges_ = 0; + uint64_t physical_read_bytes_ = 0; + bool initialized_ = false; +}; + +class SequentialRegionReader { +public: + // Default chunk: large enough to amortize per-window read overhead over + // slim terms, small enough that k sources x 1 chunk stays negligible next + // to the merge's memory-precheck budget. Configurable per instance (the + // compaction wiring exposes a config knob in T2.6). + static constexpr size_t kDefaultChunkBytes = 4ULL << 20; + + SequentialRegionReader(io::FileReader* reader, uint64_t region_offset, uint64_t region_length, + size_t chunk_bytes = kDefaultChunkBytes) + : reader_(reader), + region_off_(region_offset), + region_len_(region_length), + chunk_bytes_(chunk_bytes == 0 ? kDefaultChunkBytes : chunk_bytes) {} + + // Resolves the absolute byte window [abs_off, abs_off+len) per the + // contract above. len == 0 yields an empty slice without touching the + // file. + Status resolve(uint64_t abs_off, uint64_t len, std::vector* scratch, Slice* out); + + // Observability for tests/profiling: physical reads issued vs windows + // served straight from the buffered chunk. + uint64_t read_calls() const { return read_calls_; } + uint64_t buffer_hits() const { return buffer_hits_; } + +private: + io::FileReader* reader_ = nullptr; + uint64_t region_off_ = 0; + uint64_t region_len_ = 0; + size_t chunk_bytes_ = kDefaultChunkBytes; + + std::vector buf_; // buffered chunk; empty until the first fill + uint64_t buf_off_ = 0; // absolute offset of buf_[0] (valid when buf_ non-empty) + + uint64_t read_calls_ = 0; + uint64_t buffer_hits_ = 0; +}; + +} // namespace doris::snii::compaction diff --git a/be/src/storage/index/snii/compaction/rowid_conversion.cpp b/be/src/storage/index/snii/compaction/rowid_conversion.cpp new file mode 100644 index 00000000000000..e05b80572d20c9 --- /dev/null +++ b/be/src/storage/index/snii/compaction/rowid_conversion.cpp @@ -0,0 +1,240 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/compaction/rowid_conversion.h" + +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "common/check.h" + +namespace doris::snii::compaction { +namespace { + +constexpr uint32_t kDeleted = std::numeric_limits::max(); + +struct HeapEntry { + uint64_t destination_ordinal = 0; + size_t source_ordinal = 0; + size_t source_docid = 0; +}; + +struct HeapEntryGreater { + bool operator()(const HeapEntry& lhs, const HeapEntry& rhs) const { + if (lhs.destination_ordinal != rhs.destination_ordinal) { + return lhs.destination_ordinal > rhs.destination_ordinal; + } + return lhs.source_ordinal > rhs.source_ordinal; + } +}; + +uint64_t destination_ordinal(const std::pair& destination, + const std::vector& destination_segment_prefixes) { + return destination_segment_prefixes[destination.first] + destination.second; +} + +} // namespace + +Status validate_rowid_conversion(const RowIdConversionMap& conversion, + const std::vector& source_segment_doc_counts, + const std::vector& destination_segment_doc_counts) { + if (conversion.size() != source_segment_doc_counts.size()) { + return Status::InvalidArgument( + fmt::format("SNII rowid conversion source segment count mismatch: conversion={}, " + "doc_counts={}", + conversion.size(), source_segment_doc_counts.size())); + } + if (destination_segment_doc_counts.size() > + static_cast(std::numeric_limits::max())) { + return Status::InvalidArgument(fmt::format( + "SNII rowid conversion destination segment count {} exceeds uint32 encoding", + destination_segment_doc_counts.size())); + } + + std::vector destination_segment_prefixes; + destination_segment_prefixes.reserve(destination_segment_doc_counts.size() + 1); + destination_segment_prefixes.push_back(0); + uint64_t destination_doc_count = 0; + for (size_t destination_segment = 0; + destination_segment < destination_segment_doc_counts.size(); ++destination_segment) { + const uint64_t segment_doc_count = destination_segment_doc_counts[destination_segment]; + if (segment_doc_count > std::numeric_limits::max() - destination_doc_count) { + return Status::InvalidArgument(fmt::format( + "SNII rowid conversion destination prefix sum overflows uint64 at segment {}", + destination_segment)); + } + destination_doc_count += segment_doc_count; + destination_segment_prefixes.push_back(destination_doc_count); + } + + for (size_t source = 0; source < conversion.size(); ++source) { + const auto& source_conversion = conversion[source]; + if (source_conversion.size() != source_segment_doc_counts[source]) { + return Status::InvalidArgument(fmt::format( + "SNII rowid conversion source doc count mismatch at source {}: " + "conversion={}, doc_count={}", + source, source_conversion.size(), source_segment_doc_counts[source])); + } + + bool has_previous = false; + uint64_t previous_ordinal = 0; + for (size_t source_docid = 0; source_docid < source_conversion.size(); ++source_docid) { + const auto& destination = source_conversion[source_docid]; + const bool segment_deleted = destination.first == kDeleted; + const bool row_deleted = destination.second == kDeleted; + if (segment_deleted != row_deleted) { + return Status::InvalidArgument(fmt::format( + "SNII rowid conversion entry is partially deleted at source {} doc {}: " + "destination=({}, {})", + source, source_docid, destination.first, destination.second)); + } + if (segment_deleted) { + continue; + } + if (destination.first >= destination_segment_doc_counts.size()) { + return Status::InvalidArgument(fmt::format( + "SNII rowid conversion destination segment {} is out of range at " + "source {} doc {} (segment_count={})", + destination.first, source, source_docid, + destination_segment_doc_counts.size())); + } + if (destination.second >= destination_segment_doc_counts[destination.first]) { + return Status::InvalidArgument(fmt::format( + "SNII rowid conversion destination row {} is out of range at source {} " + "doc {} (destination segment {} has {} docs)", + destination.second, source, source_docid, destination.first, + destination_segment_doc_counts[destination.first])); + } + + const uint64_t ordinal = destination_ordinal(destination, destination_segment_prefixes); + if (has_previous && ordinal <= previous_ordinal) { + return Status::InvalidArgument(fmt::format( + "SNII rowid conversion source {} is not strictly increasing at doc {}: " + "destination ordinal {} follows {}", + source, source_docid, ordinal, previous_ordinal)); + } + previous_ordinal = ordinal; + has_previous = true; + } + } + + std::priority_queue, HeapEntryGreater> heap; + auto push_next_live = [&](size_t source, size_t source_docid) { + const auto& source_conversion = conversion[source]; + while (source_docid < source_conversion.size() && + source_conversion[source_docid].first == kDeleted) { + ++source_docid; + } + if (source_docid < source_conversion.size()) { + heap.push({destination_ordinal(source_conversion[source_docid], + destination_segment_prefixes), + source, source_docid}); + } + }; + + for (size_t source = 0; source < conversion.size(); ++source) { + push_next_live(source, 0); + } + + uint64_t expected_ordinal = 0; + while (!heap.empty()) { + const HeapEntry entry = heap.top(); + heap.pop(); + if (entry.destination_ordinal < expected_ordinal) { + return Status::InvalidArgument(fmt::format( + "SNII rowid conversion has duplicate destination ordinal {} at source {} " + "doc {}", + entry.destination_ordinal, entry.source_ordinal, entry.source_docid)); + } + if (entry.destination_ordinal > expected_ordinal) { + return Status::InvalidArgument(fmt::format( + "SNII rowid conversion is missing destination ordinal {} before ordinal {} " + "at source {} doc {}", + expected_ordinal, entry.destination_ordinal, entry.source_ordinal, + entry.source_docid)); + } + ++expected_ordinal; + push_next_live(entry.source_ordinal, entry.source_docid + 1); + } + + if (expected_ordinal != destination_doc_count) { + return Status::InvalidArgument(fmt::format( + "SNII rowid conversion is missing destination ordinal {}: covered {} of {} " + "destination docs", + expected_ordinal, expected_ordinal, destination_doc_count)); + } + return Status::OK(); +} + +ValidatedRowIdConversion::ValidatedRowIdConversion( + const RowIdConversionMap* conversion, std::vector source_segment_doc_counts, + std::vector destination_segment_doc_counts) + : conversion_(conversion), + source_segment_doc_counts_(std::move(source_segment_doc_counts)), + destination_segment_doc_counts_(std::move(destination_segment_doc_counts)) { + DORIS_CHECK(conversion_ != nullptr); + source_has_deletions_.reserve(conversion_->size()); + for (const auto& source : *conversion_) { + const bool has_deletions = + std::ranges::any_of(source, [](const std::pair& mapping) { + return mapping.first == kDeleted; + }); + source_has_deletions_.push_back(static_cast(has_deletions)); + } +} + +Status ValidatedRowIdConversion::create(const RowIdConversionMap* conversion, + std::span source_segment_doc_counts, + std::span destination_segment_doc_counts, + std::unique_ptr* out) { + if (out == nullptr) { + return Status::InvalidArgument("SNII validated rowid conversion has null out parameter"); + } + out->reset(); + if (conversion == nullptr) { + return Status::InvalidArgument("SNII validated rowid conversion has null conversion"); + } + + std::vector source_counts(source_segment_doc_counts.begin(), + source_segment_doc_counts.end()); + std::vector destination_counts(destination_segment_doc_counts.begin(), + destination_segment_doc_counts.end()); + RETURN_IF_ERROR(validate_rowid_conversion(*conversion, source_counts, destination_counts)); + out->reset(new ValidatedRowIdConversion(conversion, std::move(source_counts), + std::move(destination_counts))); + return Status::OK(); +} + +std::span> ValidatedRowIdConversion::source_mapping( + size_t source_ordinal) const { + DCHECK_LT(source_ordinal, conversion_->size()); + return std::span>((*conversion_)[source_ordinal]); +} + +bool ValidatedRowIdConversion::source_has_deletions(size_t source_ordinal) const { + DCHECK_LT(source_ordinal, source_has_deletions_.size()); + return source_has_deletions_[source_ordinal] != 0; +} + +} // namespace doris::snii::compaction diff --git a/be/src/storage/index/snii/compaction/rowid_conversion.h b/be/src/storage/index/snii/compaction/rowid_conversion.h new file mode 100644 index 00000000000000..ac6a339ba01f4f --- /dev/null +++ b/be/src/storage/index/snii/compaction/rowid_conversion.h @@ -0,0 +1,85 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "common/status.h" + +namespace doris::snii::compaction { + +using RowIdConversionMap = std::vector>>; + +// Capability proving that a complete row-id conversion has passed the global +// shape, bounds, monotonicity and destination-coverage validation. Construction +// is restricted to create(), so merge plans cannot accidentally accept an +// unvalidated conversion. The conversion map is borrowed and must outlive this +// token and every merge plan prepared from it. +class ValidatedRowIdConversion { +public: + ValidatedRowIdConversion(const ValidatedRowIdConversion&) = delete; + ValidatedRowIdConversion& operator=(const ValidatedRowIdConversion&) = delete; + + static Status create(const RowIdConversionMap* conversion, + std::span source_segment_doc_counts, + std::span destination_segment_doc_counts, + std::unique_ptr* out); + + size_t source_segment_count() const { return source_segment_doc_counts_.size(); } + const std::vector& source_segment_doc_counts() const { + return source_segment_doc_counts_; + } + const std::vector& destination_segment_doc_counts() const { + return destination_segment_doc_counts_; + } + bool source_has_deletions(size_t source_ordinal) const; + std::span> source_mapping(size_t source_ordinal) const; + +private: + ValidatedRowIdConversion(const RowIdConversionMap* conversion, + std::vector source_segment_doc_counts, + std::vector destination_segment_doc_counts); + + const RowIdConversionMap* conversion_ = nullptr; + std::vector source_segment_doc_counts_; + std::vector destination_segment_doc_counts_; + std::vector source_has_deletions_; +}; + +// Validates the complete row-id conversion before an SNII index fast merge +// writes any destination bytes. Each inner conversion vector belongs to one +// source segment and is indexed by source docid. A deleted source doc must be +// represented by exactly (UINT32_MAX, UINT32_MAX); every other pair is a live +// (destination segment, destination docid). +// +// In addition to shape and bounds, the function proves that: +// * each source stream is strictly increasing in global destination order; +// * all source streams together contain every destination doc exactly once. +// +// Completeness is checked by a k-way merge of the monotonic source streams. +// Memory is O(source segments + destination segments), never O(output rows). +Status validate_rowid_conversion(const RowIdConversionMap& conversion, + const std::vector& source_segment_doc_counts, + const std::vector& destination_segment_doc_counts); + +} // namespace doris::snii::compaction diff --git a/be/src/storage/index/snii/compaction/snii_index_compaction.cpp b/be/src/storage/index/snii/compaction/snii_index_compaction.cpp new file mode 100644 index 00000000000000..91661d8de520df --- /dev/null +++ b/be/src/storage/index/snii/compaction/snii_index_compaction.cpp @@ -0,0 +1,561 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/compaction/snii_index_compaction.h" + +#include +#include +#include +#include +#include +#include + +#include "common/check.h" +#include "common/logging.h" +#include "storage/index/inverted/common_grams/common_grams_key_codec.h" +#include "storage/index/snii/compaction/eligibility.h" +#include "storage/index/snii/compaction/posting_run_merger.h" +#include "storage/index/snii/compaction/term_cursor.h" +#include "storage/index/snii/compaction/term_merge_frontier.h" +#include "storage/index/snii/format/norms_pod.h" +#include "storage/index/snii/writer/spimi_term_buffer.h" + +namespace doris::snii::compaction { + +namespace { + +Status invalid_plan(std::string_view reason) { + return Status::Error("snii_compaction: {}", reason); +} + +Status index_compaction_merge_corruption(std::string_view reason) { + return Status::Error("snii_compaction: {}", + reason); +} + +bool is_well_formed_common_gram(std::string_view term) { + namespace inverted_index = segment_v2::inverted_index; + if (!term.starts_with(inverted_index::CG_V1_MARKER) || + term.size() > inverted_index::COMMON_GRAM_MAX_ENCODED_BYTES) { + return false; + } + constexpr size_t kLengthBytes = 8; + const size_t length_offset = inverted_index::CG_V1_MARKER.size(); + if (term.size() < length_offset + kLengthBytes + 1) { + return false; + } + uint32_t left_length = 0; + for (size_t i = 0; i < kLengthBytes; ++i) { + const char digit = term[length_offset + i]; + if (!((digit >= '0' && digit <= '9') || (digit >= 'a' && digit <= 'f'))) { + return false; + } + left_length = (left_length << 4) | + static_cast(digit <= '9' ? digit - '0' : digit - 'a' + 10); + } + const size_t separator = length_offset + kLengthBytes; + if (term[separator] != ':') { + return false; + } + const std::string_view components = term.substr(separator + 1); + if (left_length > components.size()) { + return false; + } + return inverted_index::validate_common_grams_logical_term(components.substr(0, left_length), + "left term") + .ok() && + inverted_index::validate_common_grams_logical_term(components.substr(left_length), + "right term") + .ok(); +} + +template +Status reserve_tracked_vector(std::vector* values, size_t additional, + writer::MemoryReporter::Reservation* reservation) { + if (reservation == nullptr || additional == 0) return Status::OK(); + if (additional > std::numeric_limits::max() - values->size()) { + return Status::Error( + "snii_compaction: destination posting vector size overflows"); + } + const size_t required = values->size() + additional; + if (required <= values->capacity()) { + DCHECK_EQ(reservation->bytes(), values->capacity() * sizeof(T)); + return Status::OK(); + } + size_t target = std::max(64, required); + if (values->capacity() != 0 && values->capacity() <= std::numeric_limits::max() / 2) { + target = std::max(target, values->capacity() * 2); + } + if (target > static_cast(std::numeric_limits::max()) / sizeof(T)) { + return Status::Error( + "snii_compaction: destination posting reservation exceeds int64"); + } + writer::MemoryReporter::Reservation replacement; + RETURN_IF_ERROR(reservation->prepare_replacement(target * sizeof(T), &replacement)); + values->reserve(target); + DCHECK_EQ(values->capacity(), target); + *reservation = std::move(replacement); + return Status::OK(); +} +} // namespace + +SniiPlainT2MergePlan::SniiPlainT2MergePlan( + std::vector source_indexes, + const ValidatedRowIdConversion* rowid_conversion, + std::vector destination_segment_num_rows, + std::vector destination_null_reservations, + std::vector> destination_null_docids, + SniiCompactionEligibility eligibility, + std::vector destination_norm_reservations, + std::vector> destination_encoded_norms, + std::shared_ptr memory_reporter, + std::vector> read_contexts) + : source_indexes_(std::move(source_indexes)), + rowid_conversion_(rowid_conversion), + destination_segment_num_rows_(std::move(destination_segment_num_rows)), + memory_reporter_(std::move(memory_reporter)), + destination_null_reservations_(std::move(destination_null_reservations)), + destination_null_docids_(std::move(destination_null_docids)), + destination_null_docids_taken_(destination_null_docids_.size(), false), + eligibility_(std::move(eligibility)), + destination_norm_reservations_(std::move(destination_norm_reservations)), + destination_encoded_norms_(std::move(destination_encoded_norms)), + destination_encoded_norms_taken_(destination_encoded_norms_.size(), false), + destination_semantic_token_counts_(destination_segment_num_rows_.size(), 0), + read_contexts_(std::move(read_contexts)) {} + +Status SniiPlainT2MergePlan::prepare(std::vector source_indexes, + const ValidatedRowIdConversion& rowid_conversion, + size_t total_read_ahead_budget_bytes, + std::unique_ptr* out) { + return prepare(std::move(source_indexes), rowid_conversion, total_read_ahead_budget_bytes, + nullptr, out); +} + +Status SniiPlainT2MergePlan::prepare(std::vector source_indexes, + const ValidatedRowIdConversion& rowid_conversion, + size_t total_read_ahead_budget_bytes, + std::shared_ptr memory_reporter, + std::unique_ptr* out) { + SniiCompactionEligibility eligibility; + eligibility.kind = SniiStreamedMergeKind::kPlainT2; + return prepare(std::move(source_indexes), rowid_conversion, eligibility, + total_read_ahead_budget_bytes, std::move(memory_reporter), out); +} + +Status SniiPlainT2MergePlan::prepare(std::vector source_indexes, + const ValidatedRowIdConversion& rowid_conversion, + const SniiCompactionEligibility& eligibility, + size_t total_read_ahead_budget_bytes, + std::unique_ptr* out) { + return prepare(std::move(source_indexes), rowid_conversion, eligibility, + total_read_ahead_budget_bytes, nullptr, out); +} + +Status SniiPlainT2MergePlan::prepare(std::vector source_indexes, + const ValidatedRowIdConversion& rowid_conversion, + const SniiCompactionEligibility& eligibility, + size_t total_read_ahead_budget_bytes, + std::shared_ptr memory_reporter, + std::unique_ptr* out) { + if (out == nullptr) { + return invalid_plan("null plan out parameter"); + } + out->reset(); + if (source_indexes.empty()) { + return invalid_plan("no source indexes"); + } + const std::vector& destination_segment_num_rows = + rowid_conversion.destination_segment_doc_counts(); + if (destination_segment_num_rows.empty()) { + return invalid_plan("no destination segments"); + } + if (source_indexes.size() != rowid_conversion.source_segment_count()) { + return invalid_plan("source index count differs from row-id conversion"); + } + // Division first avoids source_count * minimum overflow. Rejecting tiny + // allocations is an IO gate: the raw rebuild is cheaper than issuing a + // range request for every small posting window. + if (source_indexes.size() > total_read_ahead_budget_bytes / kMinReadAheadBudgetPerSource) { + return invalid_plan("read-ahead budget is below the per-source IO floor"); + } + const size_t per_source_read_ahead_budget = + std::min(SniiPostingReadContext::kMaxReadAheadBudgetBytes, + total_read_ahead_budget_bytes / source_indexes.size()); + DORIS_CHECK_GE(per_source_read_ahead_budget, kMinReadAheadBudgetPerSource); + DORIS_CHECK_LE(per_source_read_ahead_budget, + total_read_ahead_budget_bytes / source_indexes.size()); + + for (size_t source_ordinal = 0; source_ordinal < source_indexes.size(); ++source_ordinal) { + const reader::LogicalIndexReader* source = source_indexes[source_ordinal]; + if (source == nullptr) { + return invalid_plan("null source index"); + } + RETURN_IF_ERROR(validate_snii_source_eligibility(*source, source_ordinal, eligibility)); + if (source->stats().doc_count > std::numeric_limits::max()) { + return index_compaction_merge_corruption( + "source doc count exceeds the SNII uint32 docid domain"); + } + if (source->stats().doc_count != + rowid_conversion.source_segment_doc_counts()[source_ordinal]) { + return invalid_plan("source doc count differs from validated row-id conversion"); + } + } + + std::vector destination_null_reservations; + destination_null_reservations.reserve(destination_segment_num_rows.size()); + for (size_t destination_ordinal = 0; destination_ordinal < destination_segment_num_rows.size(); + ++destination_ordinal) { + destination_null_reservations.push_back(memory_reporter == nullptr + ? writer::MemoryReporter::Reservation() + : memory_reporter->make_reservation()); + } + std::vector> destination_null_docids(destination_segment_num_rows.size()); + for (size_t source_ordinal = 0; source_ordinal < source_indexes.size(); ++source_ordinal) { + reader::NullDocidsScanMemory scan_memory; + RETURN_IF_ERROR(source_indexes[source_ordinal]->null_docids_scan_memory(&scan_memory)); + writer::MemoryReporter::Reservation source_output_reservation = + memory_reporter == nullptr ? writer::MemoryReporter::Reservation() + : memory_reporter->make_reservation(); + writer::MemoryReporter::Reservation source_frame_reservation = + memory_reporter == nullptr ? writer::MemoryReporter::Reservation() + : memory_reporter->make_reservation(); + writer::MemoryReporter::Reservation source_decode_reservation = + memory_reporter == nullptr ? writer::MemoryReporter::Reservation() + : memory_reporter->make_reservation(); + if (memory_reporter != nullptr) { + RETURN_IF_ERROR(source_output_reservation.set_bytes(scan_memory.output_bytes)); + RETURN_IF_ERROR(source_frame_reservation.set_bytes(scan_memory.frame_bytes)); + } + std::vector source_null_docids; + source_null_docids.reserve(source_indexes[source_ordinal]->stats().null_count); + if (memory_reporter != nullptr) { + DORIS_CHECK_EQ(source_null_docids.capacity() * sizeof(uint32_t), + source_output_reservation.bytes()); + } + RETURN_IF_ERROR(source_indexes[source_ordinal]->read_null_docids( + &source_null_docids, [&](uint64_t bytes) { + return memory_reporter == nullptr ? Status::OK() + : source_decode_reservation.set_bytes(bytes); + })); + source_frame_reservation.reset(); + source_decode_reservation.reset(); + const auto source_mapping = rowid_conversion.source_mapping(source_ordinal); + for (uint32_t source_docid : source_null_docids) { + DCHECK_LT(source_docid, source_mapping.size()); + const auto [destination_segment, destination_docid] = source_mapping[source_docid]; + const bool segment_deleted = + destination_segment == std::numeric_limits::max(); + const bool doc_deleted = destination_docid == std::numeric_limits::max(); + DCHECK_EQ(segment_deleted, doc_deleted); + if (!segment_deleted) { + DCHECK_LT(destination_segment, destination_null_docids.size()); + DCHECK_LT(destination_docid, destination_segment_num_rows[destination_segment]); + RETURN_IF_ERROR(reserve_tracked_vector( + &destination_null_docids[destination_segment], 1, + memory_reporter == nullptr + ? nullptr + : &destination_null_reservations[destination_segment])); + destination_null_docids[destination_segment].push_back(destination_docid); + } + } + } + for (auto& null_docids : destination_null_docids) { + std::ranges::sort(null_docids); + DCHECK(std::adjacent_find(null_docids.begin(), null_docids.end()) == null_docids.end()); + } + + std::vector destination_norm_reservations; + std::vector> destination_encoded_norms; + if (eligibility.kind == SniiStreamedMergeKind::kCommonGramsT3) { + DORIS_CHECK(eligibility.common_grams_metadata_seed.has_value()); + destination_norm_reservations.reserve(destination_segment_num_rows.size()); + destination_encoded_norms.resize(destination_segment_num_rows.size()); + for (size_t destination_ordinal = 0; + destination_ordinal < destination_segment_num_rows.size(); ++destination_ordinal) { + destination_norm_reservations.push_back(memory_reporter == nullptr + ? writer::MemoryReporter::Reservation() + : memory_reporter->make_reservation()); + const uint32_t doc_count = destination_segment_num_rows[destination_ordinal]; + auto& norms = destination_encoded_norms[destination_ordinal]; + RETURN_IF_ERROR(reserve_tracked_vector( + &norms, doc_count, + memory_reporter == nullptr ? nullptr : &destination_norm_reservations.back())); + norms.resize(doc_count); + if (memory_reporter != nullptr) { + DORIS_CHECK_EQ(destination_norm_reservations.back().bytes(), norms.capacity()); + } + } + for (size_t source_ordinal = 0; source_ordinal < source_indexes.size(); ++source_ordinal) { + writer::MemoryReporter::Reservation source_norms_reservation = + memory_reporter == nullptr ? writer::MemoryReporter::Reservation() + : memory_reporter->make_reservation(); + if (memory_reporter != nullptr) { + RETURN_IF_ERROR(source_norms_reservation.set_bytes( + source_indexes[source_ordinal]->compaction_norms_cache_charge())); + } + format::NormsPodReader source_norms; + RETURN_IF_ERROR(source_indexes[source_ordinal]->open_norms(&source_norms)); + const auto source_mapping = rowid_conversion.source_mapping(source_ordinal); + // The norms POD is only CRC-self-consistent; nothing upstream ties its + // doc_count to the validated conversion, and the loop below indexes + // source_mapping by it. Reconcile loudly (once per source, cold path). + if (source_norms.doc_count() != source_mapping.size()) { + return index_compaction_merge_corruption( + "norms doc count differs from validated row-id conversion"); + } + for (uint32_t source_docid = 0; source_docid < source_norms.doc_count(); + ++source_docid) { + const auto [destination_segment, destination_docid] = source_mapping[source_docid]; + const bool deleted = destination_segment == std::numeric_limits::max(); + DCHECK_EQ(deleted, destination_docid == std::numeric_limits::max()); + if (deleted) { + continue; + } + DCHECK_LT(destination_segment, destination_encoded_norms.size()); + DCHECK_LT(destination_docid, destination_encoded_norms[destination_segment].size()); + destination_encoded_norms[destination_segment][destination_docid] = + source_norms.encoded_norm(source_docid); + } + source_indexes[source_ordinal]->release_compaction_norms(); + source_norms_reservation.reset(); + } + } + + std::vector> read_contexts; + read_contexts.reserve(source_indexes.size()); + for (const reader::LogicalIndexReader* source : source_indexes) { + auto context = std::make_unique( + source, per_source_read_ahead_budget, memory_reporter.get()); + RETURN_IF_ERROR(context->init()); + read_contexts.push_back(std::move(context)); + } + + out->reset(new SniiPlainT2MergePlan( + std::move(source_indexes), &rowid_conversion, destination_segment_num_rows, + std::move(destination_null_reservations), std::move(destination_null_docids), + eligibility, std::move(destination_norm_reservations), + std::move(destination_encoded_norms), std::move(memory_reporter), + std::move(read_contexts))); + return Status::OK(); +} + +const std::vector& SniiPlainT2MergePlan::destination_null_docids( + size_t destination_segment) const { + DORIS_CHECK_LT(destination_segment, destination_null_docids_.size()); + return destination_null_docids_[destination_segment]; +} + +writer::TrackedNullDocids SniiPlainT2MergePlan::take_destination_null_docids( + size_t destination_segment) { + DORIS_CHECK_LT(destination_segment, destination_null_docids_.size()); + DORIS_CHECK(!destination_null_docids_taken_[destination_segment]); + destination_null_docids_taken_[destination_segment] = true; + return writer::TrackedNullDocids(std::move(destination_null_reservations_[destination_segment]), + std::move(destination_null_docids_[destination_segment])); +} + +const std::vector& SniiPlainT2MergePlan::destination_encoded_norms( + size_t destination_segment) const { + DORIS_CHECK(eligibility_.kind == SniiStreamedMergeKind::kCommonGramsT3); + DORIS_CHECK_LT(destination_segment, destination_encoded_norms_.size()); + return destination_encoded_norms_[destination_segment]; +} + +writer::TrackedEncodedNorms SniiPlainT2MergePlan::take_destination_encoded_norms( + size_t destination_segment) { + DORIS_CHECK(eligibility_.kind == SniiStreamedMergeKind::kCommonGramsT3); + DORIS_CHECK_LT(destination_segment, destination_encoded_norms_.size()); + DORIS_CHECK(!destination_encoded_norms_taken_[destination_segment]); + destination_encoded_norms_taken_[destination_segment] = true; + return writer::TrackedEncodedNorms( + std::move(destination_norm_reservations_[destination_segment]), + std::move(destination_encoded_norms_[destination_segment])); +} + +format::IndexConfig SniiPlainT2MergePlan::destination_index_config() const { + return eligibility_.kind == SniiStreamedMergeKind::kCommonGramsT3 + ? format::IndexConfig::kDocsPositionsScoring + : format::IndexConfig::kDocsPositions; +} + +std::optional +SniiPlainT2MergePlan::destination_common_grams_metadata(size_t destination_segment) const { + if (eligibility_.kind == SniiStreamedMergeKind::kPlainT2) { + return std::nullopt; + } + DORIS_CHECK(eligibility_.common_grams_metadata_seed.has_value()); + DORIS_CHECK_LT(destination_segment, destination_segment_num_rows_.size()); + auto metadata = *eligibility_.common_grams_metadata_seed; + metadata.scoring_doc_count = destination_segment_num_rows_[destination_segment]; + metadata.scoring_token_count = 0; + return metadata; +} + +Status SniiPlainT2MergePlan::poison(Status status) { + DORIS_CHECK(!status.ok()); + if (failed_.ok()) { + failed_ = std::move(status); + } + return failed_; +} + +Status SniiPlainT2MergePlan::take_front_source(TermMergeFrontier* frontier, CurrentTerm* current) { + SniiSegmentTermCursor* source = frontier->front(); + const uint32_t source_ordinal = source->source_ordinal(); + DCHECK_LT(source_ordinal, source_indexes_.size()); + const uint64_t frq_base = source->frq_base(); + const uint64_t prx_base = source->prx_base(); + format::DictEntry entry = source->take_entry(); + if (current->posting_cursors.empty()) { + current->term = std::move(entry.term); + } + + const bool source_has_positions = posting_entry_has_positions(entry); + if (!current->common_gram && !source_has_positions) { + return index_compaction_merge_corruption("ordinary term has docs-only posting shape"); + } + if (!current->has_positions.has_value()) { + current->has_positions = source_has_positions; + } else if (*current->has_positions != source_has_positions) { + return index_compaction_merge_corruption( + "same term has inconsistent position shape across sources"); + } + + auto cursor = std::make_unique(read_contexts_[source_ordinal].get(), + std::move(entry), frq_base, prx_base, + source_ordinal, rowid_conversion_); + RETURN_IF_ERROR(cursor->init()); + current->posting_cursors.push_back(std::move(cursor)); + return frontier->advance_front(); +} + +Status SniiPlainT2MergePlan::take_current_term(TermMergeFrontier* frontier, CurrentTerm* current) { + DCHECK(frontier != nullptr); + DCHECK(current != nullptr); + DCHECK(!frontier->empty()); + const std::string_view group_term = frontier->front()->term(); + current->posting_cursors.reserve(source_indexes_.size()); + if (eligibility_.kind == SniiStreamedMergeKind::kCommonGramsT3) { + current->common_gram = segment_v2::inverted_index::is_internal_term_key(group_term); + if (current->common_gram && !is_well_formed_common_gram(group_term)) { + return index_compaction_merge_corruption( + "CommonGrams source contains an unknown internal term marker"); + } + current->counts_as_semantic_token = !current->common_gram; + } + + do { + RETURN_IF_ERROR(take_front_source(frontier, current)); + } while (!frontier->empty() && frontier->front()->term() == current->term); + DCHECK(current->has_positions.has_value()); + return Status::OK(); +} + +Status SniiPlainT2MergePlan::write_current_term( + CurrentTerm current, std::span sessions) { + MergedPostingRuns posting_source(std::move(current.posting_cursors), *current.has_positions, + current.counts_as_semantic_token, + destination_segment_num_rows_, + destination_semantic_token_counts_); + RETURN_IF_ERROR(posting_source.init()); + while (!posting_source.empty()) { + const uint32_t destination = posting_source.next_destination(); + DCHECK_LT(destination, sessions.size()); + RETURN_IF_ERROR(posting_source.begin_destination(destination)); + writer::StreamedTermPostings postings {.term = current.term, + .retain_positions = *current.has_positions, + .source = &posting_source}; + RETURN_IF_ERROR(sessions[destination]->push_term(std::move(postings))); + } + return Status::OK(); +} + +Status SniiPlainT2MergePlan::merge_terms( + std::span sessions) { + std::vector> term_cursors; + std::vector term_cursor_ptrs; + term_cursors.reserve(source_indexes_.size()); + term_cursor_ptrs.reserve(source_indexes_.size()); + for (size_t source_ordinal = 0; source_ordinal < source_indexes_.size(); ++source_ordinal) { + term_cursors.push_back(std::make_unique( + source_indexes_[source_ordinal], static_cast(source_ordinal), + memory_reporter_.get())); + term_cursor_ptrs.push_back(term_cursors.back().get()); + } + TermMergeFrontier term_frontier; + RETURN_IF_ERROR(term_frontier.init(std::move(term_cursor_ptrs))); + + while (!term_frontier.empty()) { + CurrentTerm current; + RETURN_IF_ERROR(take_current_term(&term_frontier, ¤t)); + RETURN_IF_ERROR(write_current_term(std::move(current), sessions)); + } + + if (eligibility_.kind == SniiStreamedMergeKind::kCommonGramsT3) { + for (size_t destination_ordinal = 0; destination_ordinal < sessions.size(); + ++destination_ordinal) { + RETURN_IF_ERROR(sessions[destination_ordinal]->set_semantic_token_count( + destination_semantic_token_counts_[destination_ordinal])); + } + } + for (writer::SniiStreamedIndexSession* session : sessions) { + RETURN_IF_ERROR(session->finish()); + } + return Status::OK(); +} + +Status SniiPlainT2MergePlan::execute(std::span sessions) { + const auto abort_sessions = [&sessions](const Status& cause) { + DCHECK(!cause.ok()); + for (writer::SniiStreamedIndexSession* session : sessions) { + if (session != nullptr) { + session->abort(cause); + } + } + }; + if (!failed_.ok()) { + abort_sessions(failed_); + return failed_; + } + if (executed_) { + const Status status = invalid_plan("merge plan executed twice"); + abort_sessions(status); + return poison(status); + } + if (sessions.size() != destination_segment_num_rows_.size()) { + const Status status = invalid_plan("destination session count mismatch"); + abort_sessions(status); + return poison(status); + } + for (writer::SniiStreamedIndexSession* session : sessions) { + if (session == nullptr) { + const Status status = invalid_plan("null destination session"); + abort_sessions(status); + return poison(status); + } + } + executed_ = true; + const Status status = merge_terms(sessions); + if (!status.ok()) { + abort_sessions(status); + return poison(status); + } + return Status::OK(); +} + +} // namespace doris::snii::compaction diff --git a/be/src/storage/index/snii/compaction/snii_index_compaction.h b/be/src/storage/index/snii/compaction/snii_index_compaction.h new file mode 100644 index 00000000000000..e012ae4e294111 --- /dev/null +++ b/be/src/storage/index/snii/compaction/snii_index_compaction.h @@ -0,0 +1,146 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/compaction/eligibility.h" +#include "storage/index/snii/compaction/posting_cursor.h" +#include "storage/index/snii/compaction/rowid_conversion.h" +#include "storage/index/snii/reader/logical_index_reader.h" +#include "storage/index/snii/writer/memory_reporter.h" +#include "storage/index/snii/writer/snii_compound_writer.h" + +namespace doris::snii::compaction { + +class TermMergeFrontier; + +// Prepared, one-shot merge of one plain positions-only logical index across +// source segments. prepare() performs every O(1)-metadata, row-id and NULL +// preflight before the caller creates destination streamed sessions. execute() +// then performs exactly one dictionary/posting pass and seals every session. +// +// Source readers and the validated row-id conversion token are borrowed and +// must remain stable through execute(). The plan owns the per-source read +// contexts, destination row counts and remapped NULL docids. Its aggregate +// read-ahead allocation never exceeds the explicit prepare() budget. +class SniiPlainT2MergePlan { +public: + // Below this aggregate-per-source budget, two posting streams would issue + // tiny range reads and turn a memory fallback into an IO regression. + static constexpr size_t kMinReadAheadBudgetPerSource = 64U << 10; + + SniiPlainT2MergePlan(const SniiPlainT2MergePlan&) = delete; + SniiPlainT2MergePlan& operator=(const SniiPlainT2MergePlan&) = delete; + SniiPlainT2MergePlan(SniiPlainT2MergePlan&&) = delete; + SniiPlainT2MergePlan& operator=(SniiPlainT2MergePlan&&) = delete; + + static Status prepare(std::vector source_indexes, + const ValidatedRowIdConversion& rowid_conversion, + size_t total_read_ahead_budget_bytes, + std::unique_ptr* out); + static Status prepare(std::vector source_indexes, + const ValidatedRowIdConversion& rowid_conversion, + size_t total_read_ahead_budget_bytes, + std::shared_ptr memory_reporter, + std::unique_ptr* out); + static Status prepare(std::vector source_indexes, + const ValidatedRowIdConversion& rowid_conversion, + const SniiCompactionEligibility& eligibility, + size_t total_read_ahead_budget_bytes, + std::unique_ptr* out); + static Status prepare(std::vector source_indexes, + const ValidatedRowIdConversion& rowid_conversion, + const SniiCompactionEligibility& eligibility, + size_t total_read_ahead_budget_bytes, + std::shared_ptr memory_reporter, + std::unique_ptr* out); + + const std::vector& destination_null_docids(size_t destination_segment) const; + writer::TrackedNullDocids take_destination_null_docids(size_t destination_segment); + const std::vector& destination_encoded_norms(size_t destination_segment) const; + writer::TrackedEncodedNorms take_destination_encoded_norms(size_t destination_segment); + format::IndexConfig destination_index_config() const; + std::optional + destination_common_grams_metadata(size_t destination_segment) const; + format::CommonGramsPostingPolicy destination_common_grams_posting_policy() const { + return eligibility_.common_grams_posting_policy; + } + size_t destination_segment_count() const { return destination_segment_num_rows_.size(); } + + // Sessions must correspond one-for-one with destination segments and must + // already carry the doc_count and destination_null_docids() returned by this + // plan. Any failure is terminal: the first error is sticky, sessions remain + // unsealable, and the whole unpublished compaction output must be discarded. + Status execute(std::span sessions); + +private: + struct CurrentTerm { + std::string term; + std::vector> posting_cursors; + std::optional has_positions; + bool common_gram = false; + bool counts_as_semantic_token = false; + }; + + SniiPlainT2MergePlan( + std::vector source_indexes, + const ValidatedRowIdConversion* rowid_conversion, + std::vector destination_segment_num_rows, + std::vector destination_null_reservations, + std::vector> destination_null_docids, + SniiCompactionEligibility eligibility, + std::vector destination_norm_reservations, + std::vector> destination_encoded_norms, + std::shared_ptr memory_reporter, + std::vector> read_contexts); + + Status take_front_source(TermMergeFrontier* frontier, CurrentTerm* current); + Status take_current_term(TermMergeFrontier* frontier, CurrentTerm* current); + Status write_current_term(CurrentTerm current, + std::span sessions); + Status merge_terms(std::span sessions); + Status poison(Status status); + + std::vector source_indexes_; + const ValidatedRowIdConversion* rowid_conversion_ = nullptr; + std::vector destination_segment_num_rows_; + std::shared_ptr memory_reporter_; + // Reservations precede their vectors so physical memory is destroyed first. + std::vector destination_null_reservations_; + std::vector> destination_null_docids_; + std::vector destination_null_docids_taken_; + SniiCompactionEligibility eligibility_; + // Reservations precede their vectors so physical memory is destroyed first. + std::vector destination_norm_reservations_; + std::vector> destination_encoded_norms_; + std::vector destination_encoded_norms_taken_; + std::vector destination_semantic_token_counts_; + std::vector> read_contexts_; + bool executed_ = false; + Status failed_ = Status::OK(); +}; + +} // namespace doris::snii::compaction diff --git a/be/src/storage/index/snii/compaction/term_cursor.cpp b/be/src/storage/index/snii/compaction/term_cursor.cpp new file mode 100644 index 00000000000000..1c25f1b57ff06a --- /dev/null +++ b/be/src/storage/index/snii/compaction/term_cursor.cpp @@ -0,0 +1,164 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/compaction/term_cursor.h" + +#include + +#include "storage/index/snii/format/phrase_bigram.h" +#include "storage/index/snii/format/sampled_term_index.h" + +namespace doris::snii::compaction { + +namespace { + +Status add_entry_memory(uint64_t bytes, uint64_t* total) { + if (bytes > std::numeric_limits::max() - *total) { + return Status::Error( + "term_cursor: decoded dictionary entries exceed uint64 memory accounting"); + } + *total += bytes; + return Status::OK(); +} + +Status decoded_entries_memory_bytes(const std::vector& entries, uint64_t* out) { + if (entries.capacity() > std::numeric_limits::max() / sizeof(format::DictEntry)) { + return Status::Error( + "term_cursor: decoded dictionary entry slots exceed uint64 memory accounting"); + } + uint64_t bytes = entries.capacity() * sizeof(format::DictEntry); + for (const format::DictEntry& entry : entries) { + RETURN_IF_ERROR(add_entry_memory(format::std_string_heap_bytes(entry.term), &bytes)); + RETURN_IF_ERROR(add_entry_memory(entry.frq_bytes.capacity(), &bytes)); + RETURN_IF_ERROR(add_entry_memory(entry.prx_bytes.capacity(), &bytes)); + } + *out = bytes; + return Status::OK(); +} + +} // namespace + +uint64_t big_endian_term_prefix(std::string_view term) { + uint64_t prefix = 0; + for (size_t i = 0; i < term.size() && i < sizeof(prefix); ++i) { + prefix |= static_cast(static_cast(term[i])) << (56 - i * 8); + } + return prefix; +} + +Status SniiSegmentTermCursor::next(bool* has_term) { + if (has_term == nullptr) { + return Status::Error("term_cursor: null has_term"); + } + *has_term = false; + if (!failed_.ok()) { + return failed_; + } + if (index_ == nullptr) { + return Status::Error("term_cursor: null index"); + } + if (exhausted_) { + return Status::OK(); + } + + if (started_) { + ++pos_; + } else { + started_ = true; + } + // Cross a block boundary (or start): materialize the next DICT block. The + // loop form also tolerates a (format-legal but writer-never-produced) + // empty block. + while (pos_ >= entries_.size()) { + if (next_block_ >= index_->n_dict_blocks()) { + exhausted_ = true; + entries_.clear(); + return Status::OK(); + } + + writer::MemoryReporter::Reservation decode_reservation = + memory_reporter_ == nullptr ? writer::MemoryReporter::Reservation() + : memory_reporter_->make_reservation(); + if (memory_reporter_ != nullptr) { + reader::DictBlockScanMemory memory; + Status st = index_->dict_block_scan_memory(next_block_, &memory); + if (!st.ok()) { + failed_ = st; + return failed_; + } + st = decode_reservation.set_bytes(memory.decode_bytes); + if (!st.ok()) { + failed_ = st; + return failed_; + } + + previous_entries_reservation_.reset(); + previous_entries_reservation_ = std::move(entries_reservation_); + entries_reservation_ = memory_reporter_->make_reservation(); + st = entries_reservation_.set_bytes(memory.entries_bytes); + if (!st.ok()) { + failed_ = st; + return failed_; + } + } + const Status st = index_->decode_dict_block(next_block_, &entries_, &frq_base_, &prx_base_); + if (!st.ok()) { + failed_ = st; + return failed_; + } + if (memory_reporter_ != nullptr) { + uint64_t actual_entries_bytes = 0; + Status memory_status = decoded_entries_memory_bytes(entries_, &actual_entries_bytes); + if (memory_status.ok()) { + memory_status = entries_reservation_.set_bytes(actual_entries_bytes); + } + if (!memory_status.ok()) { + std::vector().swap(entries_); + entries_reservation_.reset(); + failed_ = memory_status; + return failed_; + } + } + ++next_block_; + pos_ = 0; + } + + const format::DictEntry& e = entries_[pos_]; + // Hidden bigram / sentinel gate: classify by the FULL marker so a user term + // that merely begins with a raw 0x1F byte passes through (design ruling on + // the 0x1F-prefix corner). Any full-marker term aborts this column's merge + // -- the caller must fall back to rebuild (v1 excludes legacy bigrams). + if (format::is_phrase_bigram_term(e.term)) { + failed_ = Status::Error( + "term_cursor: source dictionary contains a legacy phrase-bigram/sentinel term; " + "column must fall back to index rebuild (src_ord={})", + source_ordinal_); + return failed_; + } + if (has_prev_ && e.term <= prev_term_) { + failed_ = Status::Error( + "term_cursor: dictionary term order violated (src_ord={})", source_ordinal_); + return failed_; + } + prev_term_ = e.term; + term_prefix_ = big_endian_term_prefix(e.term); + has_prev_ = true; + *has_term = true; + return Status::OK(); +} + +} // namespace doris::snii::compaction diff --git a/be/src/storage/index/snii/compaction/term_cursor.h b/be/src/storage/index/snii/compaction/term_cursor.h new file mode 100644 index 00000000000000..2c73e8442c7f17 --- /dev/null +++ b/be/src/storage/index/snii/compaction/term_cursor.h @@ -0,0 +1,124 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/format/dict_entry.h" +#include "storage/index/snii/reader/logical_index_reader.h" +#include "storage/index/snii/writer/memory_reporter.h" + +// SniiSegmentTermCursor -- pull-model full-dictionary scan over ONE source +// segment's logical index (T2.3, compaction index-merge fast path). +// +// The cursor walks the source's DICT blocks in ordinal order and yields every +// DictEntry in lexicographic term order, one block resident at a time (never +// the whole vocabulary). Entries are passed through UNINTERPRETED: the locator +// (inline bytes / slim pod_ref / windowed pod_ref) and the per-block +// kNoTermStats flag (DictEntry::term_stats_present) reach the downstream +// decoder exactly as the reader produced them -- the merge pump (T2.4) decides +// how to decode, and kNoTermStats inputs must have their ttf/max_freq recomputed +// from the actual freq stream (correctness invariant 2), so this layer must not +// synthesize stats. +// +// Hidden-term gate (base-drift addendum ruling): the v1 merge fast path does +// NOT merge legacy phrase-bigram postings. Any dictionary term carrying the +// FULL 0x1F bigram marker (hidden bigram pair or the bare-marker sentinel, +// classified by format::is_phrase_bigram_term -- NOT by a raw leading 0x1F +// byte, which a legitimate user term may begin with) makes next() return +// INVERTED_INDEX_NOT_SUPPORTED so the caller aborts THIS column's merge and +// falls back to a full rebuild. The error is deliberately raised from next() +// (not swallowed by skipping) because silently dropping hidden postings would +// change phrase semantics on the merged output. +namespace doris::snii::compaction { + +uint64_t big_endian_term_prefix(std::string_view term); + +class SniiSegmentTermCursor { +public: + // `index` is borrowed and must outlive the cursor. `source_ordinal` is the + // caller's stable id for this source segment (frontier tie-break + docid + // remapping key downstream). + SniiSegmentTermCursor(const reader::LogicalIndexReader* index, uint32_t source_ordinal, + writer::MemoryReporter* memory_reporter = nullptr) + : index_(index), + source_ordinal_(source_ordinal), + memory_reporter_(memory_reporter), + previous_entries_reservation_(memory_reporter == nullptr + ? writer::MemoryReporter::Reservation() + : memory_reporter->make_reservation()), + entries_reservation_(memory_reporter == nullptr + ? writer::MemoryReporter::Reservation() + : memory_reporter->make_reservation()) {} + + // Advances to the next dictionary term. *has_term=false once the + // dictionary is exhausted (an empty index yields it on the first call). + // Errors: + // INVERTED_INDEX_NOT_SUPPORTED -- hidden bigram/sentinel term (see above); + // Corruption -- the dictionary violated strict lexicographic order. + // After an error the cursor is poisoned and keeps returning the error. + Status next(bool* has_term); + + // Accessors for the CURRENT term; valid only after next() returned + // *has_term=true and, for term()/entry(), before take_entry(). + const std::string& term() const { return entries_[pos_].term; } + uint64_t term_prefix() const { return term_prefix_; } + const format::DictEntry& entry() const { return entries_[pos_]; } + // Moves the current entry out (term + locator + any inline posting bytes). + // The cursor stays positioned, but term()/entry() must not be used again + // until the next next() call. + format::DictEntry take_entry() { return std::move(entries_[pos_]); } + + // frq/prx bases of the DICT block owning the current entry -- required to + // resolve pod_ref locators against the source's posting region. + uint64_t frq_base() const { return frq_base_; } + uint64_t prx_base() const { return prx_base_; } + uint32_t source_ordinal() const { return source_ordinal_; } + +private: + const reader::LogicalIndexReader* index_ = nullptr; + uint32_t source_ordinal_ = 0; + writer::MemoryReporter* memory_reporter_ = nullptr; + // Keep the preceding block's charge for one additional block transition. + // The frontier advances a source before the current term's posting cursor + // has released the DictEntry moved out of that block. + writer::MemoryReporter::Reservation previous_entries_reservation_; + writer::MemoryReporter::Reservation entries_reservation_; + + uint32_t next_block_ = 0; // next DICT block ordinal to decode + std::vector entries_; // current block, materialized + size_t pos_ = 0; // current entry within entries_ + uint64_t frq_base_ = 0; + uint64_t prx_base_ = 0; + uint64_t term_prefix_ = 0; + + bool started_ = false; // first next() must not pre-increment pos_ + bool exhausted_ = false; + Status failed_ = Status::OK(); // sticky error (poisoned cursor) + // Strict-order guard across block boundaries: DICT blocks and the merge + // frontier both assume a strictly increasing term sequence; a violation + // means a corrupt dictionary and must fail the merge, not scramble output. + std::string prev_term_; + bool has_prev_ = false; +}; + +} // namespace doris::snii::compaction diff --git a/be/src/storage/index/snii/compaction/term_merge_frontier.cpp b/be/src/storage/index/snii/compaction/term_merge_frontier.cpp new file mode 100644 index 00000000000000..8c5bd6ebc8c539 --- /dev/null +++ b/be/src/storage/index/snii/compaction/term_merge_frontier.cpp @@ -0,0 +1,103 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/compaction/term_merge_frontier.h" + +#include + +#include "common/check.h" + +namespace doris::snii::compaction { + +bool TermMergeFrontier::Before::operator()(size_t lhs, size_t rhs) const { + const uint64_t lhs_prefix = (*cursors)[lhs]->term_prefix(); + const uint64_t rhs_prefix = (*cursors)[rhs]->term_prefix(); + if (lhs_prefix != rhs_prefix) { + return lhs_prefix < rhs_prefix; + } + const std::string& lhs_term = (*cursors)[lhs]->term(); + const std::string& rhs_term = (*cursors)[rhs]->term(); + if (lhs_term != rhs_term) { + return lhs_term < rhs_term; + } + return (*cursors)[lhs]->source_ordinal() < (*cursors)[rhs]->source_ordinal(); +} + +Status TermMergeFrontier::init(std::vector cursors) { + if (initialized_) { + return Status::Error( + "term_merge_frontier: init called twice"); + } + initialized_ = true; + cursors_ = std::move(cursors); + std::vector live(cursors_.size(), 0); + for (size_t source = 0; source < cursors_.size(); ++source) { + SniiSegmentTermCursor* cursor = cursors_[source]; + if (cursor == nullptr) { + failed_ = Status::Error( + "term_merge_frontier: null cursor"); + return failed_; + } + bool has_term = false; + const Status status = cursor->next(&has_term); + if (!status.ok()) { + failed_ = status; + return failed_; + } + live[source] = has_term; + } + frontier_.build(cursors_.size(), [&live](size_t source) { return live[source] != 0; }); + return Status::OK(); +} + +bool TermMergeFrontier::empty() const { + DCHECK(initialized_); + DCHECK(failed_.ok()); + return frontier_.empty(); +} + +SniiSegmentTermCursor* TermMergeFrontier::front() const { + DCHECK(initialized_); + DCHECK(failed_.ok()); + return cursors_[frontier_.winner()]; +} + +Status TermMergeFrontier::advance_front() { + if (!initialized_) { + return Status::Error( + "term_merge_frontier: advance before init"); + } + if (!failed_.ok()) { + return failed_; + } + if (frontier_.empty()) { + return Status::Error( + "term_merge_frontier: advance on empty frontier"); + } + + const size_t source = frontier_.winner(); + bool has_term = false; + const Status status = cursors_[source]->next(&has_term); + if (!status.ok()) { + failed_ = status; + return failed_; + } + frontier_.update(source, has_term); + return Status::OK(); +} + +} // namespace doris::snii::compaction diff --git a/be/src/storage/index/snii/compaction/term_merge_frontier.h b/be/src/storage/index/snii/compaction/term_merge_frontier.h new file mode 100644 index 00000000000000..8bb36a3a849dca --- /dev/null +++ b/be/src/storage/index/snii/compaction/term_merge_frontier.h @@ -0,0 +1,56 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include "common/status.h" +#include "storage/index/snii/compaction/indexed_winner_tree.h" +#include "storage/index/snii/compaction/term_cursor.h" + +namespace doris::snii::compaction { + +// K-way term frontier. Each source cursor is a dense winner-tree leaf. +// The caller consumes front()->entry() before advance_front(); advancing a +// source updates one leaf-to-root path and never materializes an intermediate +// merged-term group. +class TermMergeFrontier { + struct Before { + const std::vector* cursors = nullptr; + + bool operator()(size_t lhs, size_t rhs) const; + }; + +public: + TermMergeFrontier() : frontier_(Before {.cursors = &cursors_}) {} + + Status init(std::vector cursors); + + bool empty() const; + SniiSegmentTermCursor* front() const; + Status advance_front(); + +private: + std::vector cursors_; + IndexedWinnerTree frontier_; + bool initialized_ = false; + Status failed_ = Status::OK(); +}; + +} // namespace doris::snii::compaction diff --git a/be/src/storage/index/snii/encoding/byte_sink.cpp b/be/src/storage/index/snii/encoding/byte_sink.cpp new file mode 100644 index 00000000000000..7d91a6552f6658 --- /dev/null +++ b/be/src/storage/index/snii/encoding/byte_sink.cpp @@ -0,0 +1,60 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/encoding/byte_sink.h" + +#include "storage/index/snii/encoding/varint.h" + +namespace doris::snii { + +void ByteSink::put_fixed16(uint16_t v) { + for (int i = 0; i < 2; ++i) buf_.push_back(static_cast(v >> (8 * i))); +} + +void ByteSink::put_fixed32(uint32_t v) { + for (int i = 0; i < 4; ++i) buf_.push_back(static_cast(v >> (8 * i))); +} + +void ByteSink::put_fixed64(uint64_t v) { + for (int i = 0; i < 8; ++i) buf_.push_back(static_cast(v >> (8 * i))); +} + +void ByteSink::put_varint32(uint32_t v) { + while (v >= 0x80) { + buf_.push_back(static_cast(v) | 0x80); + v >>= 7; + } + buf_.push_back(static_cast(v)); +} + +void ByteSink::put_varint64(uint64_t v) { + while (v >= 0x80) { + buf_.push_back(static_cast(v) | 0x80); + v >>= 7; + } + buf_.push_back(static_cast(v)); +} + +void ByteSink::put_zigzag(int64_t v) { + put_varint64(zigzag_encode(v)); +} + +void ByteSink::put_bytes(Slice s) { + buf_.insert(buf_.end(), s.data(), s.data() + s.size()); +} + +} // namespace doris::snii diff --git a/be/src/storage/index/snii/encoding/byte_sink.h b/be/src/storage/index/snii/encoding/byte_sink.h new file mode 100644 index 00000000000000..e96ae112e20a31 --- /dev/null +++ b/be/src/storage/index/snii/encoding/byte_sink.h @@ -0,0 +1,71 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include "storage/index/snii/common/slice.h" + +namespace doris::snii { + +// append-only write cursor: all section serialization goes through this; manual byte assembly is forbidden. +// All multi-byte fixed-width fields are little-endian. +class ByteSink { +public: + void put_u8(uint8_t v) { buf_.push_back(v); } + void put_fixed16(uint16_t v); + void put_fixed32(uint32_t v); + void put_fixed64(uint64_t v); + void put_varint32(uint32_t v); + void put_varint64(uint64_t v); + void put_zigzag(int64_t v); + void put_bytes(Slice s); + + size_t size() const { return buf_.size(); } + const std::vector& buffer() const { return buf_; } + Slice view() const { return Slice(buf_); } + + // Reserves capacity for `additional` MORE bytes on top of the current size(), + // so a caller about to append a known-length run pays at most one reallocation + // instead of the geometric-growth reallocs a byte-at-a-time put_u8 loop would + // trigger. The argument is RELATIVE (absolute target = size() + additional): a + // sink REUSED across encodes (e.g. the shared PFOR-run `out`) keeps accumulating + // correctly rather than no-op'ing on a repeated per-run reserve of the same + // value. Affects only the backing buffer's capacity -- the emitted bytes are + // unchanged. + void reserve(size_t additional) { buf_.reserve(buf_.size() + additional); } + + // Resets the cursor to empty while RETAINING the backing capacity, so a sink can + // be reused across many small encodes (e.g. per-window region/prx scratch in the + // windowed posting builder) without re-allocating each time -- this avoids the + // cumulative small-allocation churn that fragments the heap arena and inflates + // peak RSS during the merge of a high-df term split into thousands of windows. + void clear() { buf_.clear(); } + + // Moves the backing buffer OUT to the caller (the sink is left empty), so an encoded + // section can be handed off without the copy (+ copy-induced capacity slack) that + // reading buffer() and copy-assigning would incur. Use only when the sink is not + // reused afterward (a stack-local about to die, or one that is clear()'d next). + std::vector take() { return std::move(buf_); } + +private: + std::vector buf_; +}; + +} // namespace doris::snii diff --git a/be/src/storage/index/snii/encoding/byte_source.cpp b/be/src/storage/index/snii/encoding/byte_source.cpp new file mode 100644 index 00000000000000..c393c71a7e99a5 --- /dev/null +++ b/be/src/storage/index/snii/encoding/byte_source.cpp @@ -0,0 +1,227 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/encoding/byte_source.h" + +#include +#include +#include + +#include "storage/index/snii/encoding/varint.h" + +namespace doris::snii { + +namespace { + +Status decode_delta_value(const uint8_t** cursor, const uint8_t* end, uint32_t* previous, + bool* first_position, uint32_t* value) { + const uint8_t* p = *cursor; + if (p >= end) { + return Status::Error( + "byte_source: delta run past end"); + } + uint32_t byte = *p++; + uint32_t delta = byte & 0x7FU; + if (byte >= 0x80) { + uint32_t shift = 7; + for (;;) { + if (p >= end) { + return Status::Error( + "byte_source: delta run past end"); + } + byte = *p++; + if (shift == 28 && (byte & 0xF0U) != 0) { + return Status::Error( + "byte_source: delta varint32 overflow"); + } + delta |= (byte & 0x7FU) << shift; + if ((byte & 0x80) == 0) { + break; + } + if (shift == 28) { + return Status::Error( + "byte_source: delta varint32 overflow"); + } + shift += 7; + } + } + if (!*first_position && delta > std::numeric_limits::max() - *previous) { + return Status::Error( + "byte_source: delta prefix sum overflow"); + } + *value = *first_position ? delta : *previous + delta; + *previous = *value; + *first_position = false; + *cursor = p; + return Status::OK(); +} + +} // namespace + +Status ByteSource::get_u8(uint8_t* v) { + if (remaining() < 1) + return Status::Error("get_u8 overrun"); + *v = s_[pos_++]; + return Status::OK(); +} + +Status ByteSource::get_fixed16(uint16_t* v) { + if (remaining() < 2) + return Status::Error( + "get_fixed16 overrun"); + uint16_t r = 0; + for (int i = 0; i < 2; ++i) r |= static_cast(s_[pos_ + i]) << (8 * i); + pos_ += 2; + *v = r; + return Status::OK(); +} + +Status ByteSource::get_fixed32(uint32_t* v) { + if (remaining() < 4) + return Status::Error( + "get_fixed32 overrun"); + uint32_t r = 0; + for (int i = 0; i < 4; ++i) r |= static_cast(s_[pos_ + i]) << (8 * i); + pos_ += 4; + *v = r; + return Status::OK(); +} + +Status ByteSource::get_fixed64(uint64_t* v) { + if (remaining() < 8) + return Status::Error( + "get_fixed64 overrun"); + uint64_t r = 0; + for (int i = 0; i < 8; ++i) r |= static_cast(s_[pos_ + i]) << (8 * i); + pos_ += 8; + *v = r; + return Status::OK(); +} + +Status ByteSource::get_varint64(uint64_t* v) { + const uint8_t* p = s_.data() + pos_; + const uint8_t* next = nullptr; + RETURN_IF_ERROR(decode_varint64(p, s_.data() + s_.size(), v, &next)); + pos_ = static_cast(next - s_.data()); + return Status::OK(); +} + +Status ByteSource::get_varint32(uint32_t* v) { + uint64_t tmp; + RETURN_IF_ERROR(get_varint64(&tmp)); + if (tmp > 0xFFFFFFFFu) + return Status::Error("varint32 overflow"); + *v = static_cast(tmp); + return Status::OK(); +} + +// NOLINTNEXTLINE(readability-non-const-parameter): out is the decoded position output buffer. +Status ByteSource::decode_delta_run(size_t count, std::vector* out) { + if (out == nullptr) { + return Status::Error( + "byte_source: null delta run output"); + } + if (count > std::numeric_limits::max() - out->size()) { + return Status::Error( + "byte_source: delta run size overflow"); + } + const uint8_t* const begin = s_.data(); + const uint8_t* const end = begin + s_.size(); + const uint8_t* p = begin + pos_; + const size_t original_size = out->size(); + out->reserve(out->size() + count); + uint32_t previous = 0; + bool first_position = true; + for (size_t i = 0; i < count; ++i) { + uint32_t value = 0; + const Status status = decode_delta_value(&p, end, &previous, &first_position, &value); + if (!status.ok()) { + out->resize(original_size); + return status; + } + out->push_back(value); + } + pos_ = static_cast(p - begin); + return Status::OK(); +} + +Status ByteSource::decode_delta_batch(std::span out, uint32_t* previous, + bool* first_position) { + constexpr size_t kBatchCapacity = 16; + if (out.size() > kBatchCapacity) { + return Status::Error( + "byte_source: delta batch exceeds fixed capacity"); + } + std::array scratch {}; + const uint8_t* const begin = s_.data(); + const uint8_t* const end = begin + s_.size(); + const uint8_t* p = begin + pos_; + uint32_t local_previous = *previous; + bool local_first_position = *first_position; + for (size_t i = 0; i < out.size(); ++i) { + RETURN_IF_ERROR( + decode_delta_value(&p, end, &local_previous, &local_first_position, &scratch[i])); + } + std::copy_n(scratch.begin(), out.size(), out.begin()); + pos_ = static_cast(p - begin); + *previous = local_previous; + *first_position = local_first_position; + return Status::OK(); +} + +Status ByteSource::skip_varints(size_t count) { + const uint8_t* const begin = s_.data(); + const uint8_t* const end = begin + s_.size(); + const uint8_t* p = begin + pos_; + // Each varint ends at the first byte whose continuation bit (0x80) is clear. + // Scanning for `count` such terminators skips the values with one branch per + // byte -- no shift/accumulate/store and no per-value bounds Status. (A SIMD + // bulk terminator-count was tried and reverted: the skipped position runs + // between selected docs are almost always 1-3 varints -- far below a 16-byte + // block -- so the vector path never amortized, and the larger body stopped + // this function from inlining into the CSR reader, a net CPU regression on + // the httplogs/agentlogs phrase-prefix profiles.) + for (size_t k = 0; k < count; ++k) { + while (p < end && (*p & 0x80) != 0) { + ++p; + } + if (p >= end) { + return Status::Error( + "byte_source: varint skip past end"); + } + ++p; // consume the terminator byte + } + pos_ = static_cast(p - begin); + return Status::OK(); +} + +Status ByteSource::get_zigzag(int64_t* v) { + uint64_t tmp; + RETURN_IF_ERROR(get_varint64(&tmp)); + *v = zigzag_decode(tmp); + return Status::OK(); +} + +Status ByteSource::get_bytes(size_t n, Slice* out) { + if (remaining() < n) + return Status::Error("get_bytes overrun"); + *out = s_.subslice(pos_, n); + pos_ += n; + return Status::OK(); +} + +} // namespace doris::snii diff --git a/be/src/storage/index/snii/encoding/byte_source.h b/be/src/storage/index/snii/encoding/byte_source.h new file mode 100644 index 00000000000000..07249a826f79a7 --- /dev/null +++ b/be/src/storage/index/snii/encoding/byte_source.h @@ -0,0 +1,93 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/common/slice.h" + +namespace doris::snii { + +// Slice read cursor: all section deserialization goes through this; any overrun returns Corruption. +class ByteSource { +public: + explicit ByteSource(Slice s) : s_(s) {} + + Status get_u8(uint8_t* v); + Status get_fixed16(uint16_t* v); + Status get_fixed32(uint32_t* v); + Status get_fixed64(uint64_t* v); + Status get_varint32(uint32_t* v); + Status get_varint64(uint64_t* v); + + // Single-byte fast-path varint32. The CSR position reader decodes one count + // header (`pos_count`) per doc in a window -- for every doc, selected or + // skipped -- and that per-doc read, routed through the out-of-line + // get_varint32 -> get_varint64 -> decode_varint64 chain, dominates the loop + // once positions themselves are skipped/inlined. Almost all counts are < 128 + // (one byte), so inline that case here and fall back to the full decoder for + // multi-byte values and bounds handling. + Status get_varint32_fast(uint32_t* v) { + const size_t p = pos_; + if (p < s_.size()) { + const uint8_t b0 = s_[p]; + if (b0 < 0x80) { + *v = b0; + pos_ = p + 1; + return Status::OK(); + } + } + return get_varint32(v); + } + // Advances past `count` LEB128 varints WITHOUT decoding their values -- just + // scans continuation bytes. Cheaper than get_varint* per value when the + // decoded value is unused (e.g. skipping a non-selected doc's position + // deltas in a CSR window, where the vast majority of docs in a window are + // not in the candidate set). Returns Corruption on truncation. + Status skip_varints(size_t count); + + // Decodes `count` LEB128 varints, treats them as ASCENDING deltas + // (running prefix sum starting at 0), and APPENDS the running values to + // `out`. A tight inline decoder -- no per-value get_varint32/get_varint64/ + // decode_varint64 call chain, no per-value Status, and a single-byte fast + // path (position deltas are almost always < 128). This is the hot loop of + // the CSR position reader for candidate (selected) docs. Returns Corruption + // on truncation, a >32-bit value, or a uint32 prefix-sum overflow. Failure + // leaves the cursor and output vector unchanged. + Status decode_delta_run(size_t count, std::vector* out); + Status decode_delta_batch(std::span out, uint32_t* previous, bool* first_position); + Status get_zigzag(int64_t* v); + Status get_bytes(size_t n, Slice* out); + + size_t remaining() const { return s_.size() - pos_; } + size_t position() const { return pos_; } + bool eof() const { return pos_ == s_.size(); } + + // Returns a sub-view starting at absolute offset start with length len (used by framer etc. to rewind over the CRC coverage region). + Slice slice_from(size_t start, size_t len) const { return s_.subslice(start, len); } + +private: + Slice s_; + size_t pos_ = 0; +}; + +} // namespace doris::snii diff --git a/be/src/storage/index/snii/encoding/crc32c.cpp b/be/src/storage/index/snii/encoding/crc32c.cpp new file mode 100644 index 00000000000000..39d7c6f58fe487 --- /dev/null +++ b/be/src/storage/index/snii/encoding/crc32c.cpp @@ -0,0 +1,278 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/encoding/crc32c.h" + +// T21 test-seam implementation. Production crc32c()/crc32c_extend() (in the header) +// delegate to the bundled Google crc32c thirdparty, which already runs a +// runtime-dispatched, hardware-accelerated and interleaved CRC32C. The reference +// sub-paths defined here -- portable slice-by-8, serial SSE4.2 hardware, and the +// 3-way interleaved SSE4.2 hardware algorithm T21 specifies -- exist ONLY so that +// unit tests can prove, byte-for-byte across all sizes and alignments, that the +// production path equals the canonical CRC32C (same bit-reflected Castagnoli +// polynomial) and that the hardware path is engaged. All of them are compiled out +// of release builds by the BE_TEST gate, so production pays nothing: no extra code, +// no static slice-by-8 table, no startup CPUID probe. Every function here is a pure +// function with no shared mutable state, so it is trivially thread-safe. +#ifdef BE_TEST + +#include +#include +#include + +#if defined(__x86_64__) || defined(_M_X64) +#define SNII_CRC32C_X86 1 +#include // __get_cpuid, bit_SSE4_2 +#include // _mm_crc32_u8/u32/u64 (SSE4.2) +#endif + +namespace doris::snii { +namespace { + +// Bit-reflected Castagnoli polynomial (CRC32C / iSCSI). Identical to the constant +// the removed in-tree implementation used, so every reference path below yields +// the same on-disk checksum value as the production library. +constexpr uint32_t kPoly = 0x82F63B78U; + +// Below this length the 3-way path's lane setup and GF(2) shift-combine outweigh +// the throughput win, so crc32c_hw3 falls back to the serial hardware path. Small +// buffers (inline prx windows, small pod_ref regions) therefore never pay the +// combine cost. Exposed to tests via crc32c_interleave_threshold(). +constexpr size_t kInterleaveThreshold = 1024; + +// Builds the slice-by-8 lookup tables. Column 0 is the classic byte table; each +// successive column folds in one more byte of look-ahead, letting the inner loop +// consume 8 bytes per iteration with 8 table reads + XORs instead of 8 dependent +// shift/lookup steps. The checksum value is identical to the byte-at-a-time loop. +std::array, 8> make_slice8_table() { + std::array, 8> t {}; + for (uint32_t i = 0; i < 256; ++i) { + uint32_t c = i; + for (int k = 0; k < 8; ++k) { + c = (c & 1) ? (kPoly ^ (c >> 1)) : (c >> 1); + } + t[0][i] = c; + } + for (uint32_t i = 0; i < 256; ++i) { + uint32_t c = t[0][i]; + for (int s = 1; s < 8; ++s) { + c = t[0][c & 0xFF] ^ (c >> 8); + t[s][i] = c; + } + } + return t; +} + +const std::array, 8> kSlice8 = make_slice8_table(); + +inline uint32_t load_le32(const uint8_t* p) { + return static_cast(p[0]) | (static_cast(p[1]) << 8) | + (static_cast(p[2]) << 16) | (static_cast(p[3]) << 24); +} + +// Pure software slice-by-8 (used as the portable path and the hardware fallback). +// Operates on the raw (pre/post-inversion applied by the callers) CRC register. +uint32_t crc32c_slice8(uint32_t crc, const uint8_t* p, size_t n) { + while (n >= 8) { + crc ^= load_le32(p); + const uint32_t hi = load_le32(p + 4); + crc = kSlice8[7][crc & 0xFF] ^ kSlice8[6][(crc >> 8) & 0xFF] ^ + kSlice8[5][(crc >> 16) & 0xFF] ^ kSlice8[4][crc >> 24] ^ kSlice8[3][hi & 0xFF] ^ + kSlice8[2][(hi >> 8) & 0xFF] ^ kSlice8[1][(hi >> 16) & 0xFF] ^ kSlice8[0][hi >> 24]; + p += 8; + n -= 8; + } + while (n--) { + crc = kSlice8[0][(crc ^ *p++) & 0xFF] ^ (crc >> 8); + } + return crc; +} + +#if SNII_CRC32C_X86 +// Serial hardware CRC32C via the SSE4.2 crc32 instruction. The intrinsics operate +// on the same bit-reflected Castagnoli polynomial as the tables, so the result is +// byte-identical. This TU is compiled without -msse4.2, so gate the intrinsics +// behind a function-level target attribute and a runtime CPUID check. This is the +// authoritative serial hardware path that crc32c_hw3 reuses for each lane and tail. +__attribute__((target("sse4.2"))) uint32_t crc32c_hw_serial(uint32_t crc, const uint8_t* p, + size_t n) { + while (n >= 8) { + uint64_t v; + std::memcpy(&v, p, sizeof(v)); // unaligned-safe; x86 folds to a plain load + crc = static_cast(_mm_crc32_u64(crc, v)); + p += 8; + n -= 8; + } + if (n >= 4) { + crc = _mm_crc32_u32(crc, load_le32(p)); + p += 4; + n -= 4; + } + while (n--) { + crc = _mm_crc32_u8(crc, *p++); + } + return crc; +} + +// GF(2) 32x32 bit-matrix helpers (zlib crc32_combine style). A matrix column mat[i] +// is the image of the i-th unit CRC register; gf2_matrix_times sums (XORs) the +// columns selected by the set bits of vec. +uint32_t gf2_matrix_times(const uint32_t* mat, uint32_t vec) { + uint32_t sum = 0; + while (vec != 0) { + if (vec & 1) { + sum ^= *mat; + } + vec >>= 1; + ++mat; + } + return sum; +} + +void gf2_matrix_square(uint32_t* square, const uint32_t* mat) { + for (int n = 0; n < 32; ++n) { + square[n] = gf2_matrix_times(mat, mat[n]); + } +} + +// crc32c_shift(crc, bytes): advance the raw CRC32C register as if `bytes` zero +// bytes were appended, i.e. crc . x^(8*bytes) mod P in the bit-reflected domain. +// This is the linear operator the 3-way combine applies to a lane's partial CRC so +// it lines up with the following lanes -- a pure table/matrix computation with no +// PCLMULQDQ, hence no extra CPUID gate. bytes == 0 returns crc unchanged. +uint32_t crc32c_shift(uint32_t crc, size_t bytes) { + uint32_t even[32]; // operator for 2^k zero bits, doubled each round + uint32_t odd[32]; // operator for 2^(k-1) zero bits + + // odd = operator for a single zero bit: column 0 is the polynomial, columns + // 1..31 shift the register right by one (bit i maps to bit i-1). + odd[0] = kPoly; + uint32_t row = 1; + for (int n = 1; n < 32; ++n) { + odd[n] = row; + row <<= 1; + } + gf2_matrix_square(even, odd); // even = two zero bits + gf2_matrix_square(odd, even); // odd = four zero bits + + size_t len = bytes; + do { + gf2_matrix_square(even, odd); // first pass: even = one zero byte (8 bits) + if (len & 1) { + crc = gf2_matrix_times(even, crc); + } + len >>= 1; + if (len == 0) { + break; + } + gf2_matrix_square(odd, even); + if (len & 1) { + crc = gf2_matrix_times(odd, crc); + } + len >>= 1; + } while (len != 0); + return crc; +} + +// 3-way interleaved hardware CRC32C (rocksdb/folly/Intel style). Splits the buffer +// into three equal, 8-byte-aligned lanes processed by independent _mm_crc32_u64 +// accumulators (breaking the ~3-cycle loop-carried dependency of the serial path), +// then stitches them with the GF(2) shift-combine and finishes the remainder +// serially. Byte-identical to crc32c_hw_serial / crc32c_slice8. Below the +// threshold it defers to the serial path so small buffers skip the combine cost. +uint32_t crc32c_hw3(uint32_t crc, const uint8_t* p, size_t n) { + if (n < kInterleaveThreshold) { + return crc32c_hw_serial(crc, p, n); + } + // 8-byte-aligned lane length keeps every lane on the u64 fast path. n >= 1024 + // guarantees L >= 336 > 0, so 3L <= n and the tail (n - 3L) is well defined. + const size_t lane = (n / 3) & ~static_cast(7); + const uint32_t crc_a = crc32c_hw_serial(crc, p, lane); // seeded lane + const uint32_t crc_b = crc32c_hw_serial(0, p + lane, lane); // raw lane + const uint32_t crc_c = crc32c_hw_serial(0, p + 2 * lane, lane); // raw lane + // crc(seed, A||B) == shift(crc(seed, A), |B|) ^ crc(0, B), applied twice. + uint32_t comb = crc32c_shift(crc_a, lane) ^ crc_b; + comb = crc32c_shift(comb, lane) ^ crc_c; + return crc32c_hw_serial(comb, p + 3 * lane, n - 3 * lane); // serial tail +} + +bool detect_sse42() { + unsigned int eax = 0, ebx = 0, ecx = 0, edx = 0; + if (!__get_cpuid(1, &eax, &ebx, &ecx, &edx)) { + return false; + } + return (ecx & bit_SSE4_2) != 0; +} + +const bool kHasSse42 = detect_sse42(); +#endif // SNII_CRC32C_X86 + +} // namespace + +namespace detail { + +// Portable software path. Always available; the canonical scalar reference for +// every other path and for the on-disk checksum value. +uint32_t crc32c_slice8_extend(uint32_t crc, Slice data) { + crc = ~crc; + crc = crc32c_slice8(crc, data.data(), data.size()); + return ~crc; +} + +// Serial hardware path (falls back to slice8 without SSE4.2). +uint32_t crc32c_hw_serial_extend(uint32_t crc, Slice data) { + crc = ~crc; +#if SNII_CRC32C_X86 + if (kHasSse42) { + crc = crc32c_hw_serial(crc, data.data(), data.size()); + return ~crc; + } +#endif + crc = crc32c_slice8(crc, data.data(), data.size()); + return ~crc; +} + +// 3-way interleaved hardware path (falls back to slice8 without SSE4.2; internally +// falls back to the serial path below the interleave threshold). +uint32_t crc32c_hw3_extend(uint32_t crc, Slice data) { + crc = ~crc; +#if SNII_CRC32C_X86 + if (kHasSse42) { + crc = crc32c_hw3(crc, data.data(), data.size()); + return ~crc; + } +#endif + crc = crc32c_slice8(crc, data.data(), data.size()); + return ~crc; +} + +size_t crc32c_interleave_threshold() { + return kInterleaveThreshold; +} + +bool crc32c_has_hw() { +#if SNII_CRC32C_X86 + return kHasSse42; +#else + return false; +#endif +} + +} // namespace detail +} // namespace doris::snii + +#endif // BE_TEST diff --git a/be/src/storage/index/snii/encoding/crc32c.h b/be/src/storage/index/snii/encoding/crc32c.h new file mode 100644 index 00000000000000..775a781c3d39f0 --- /dev/null +++ b/be/src/storage/index/snii/encoding/crc32c.h @@ -0,0 +1,72 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include +#include + +#include "storage/index/snii/common/slice.h" + +namespace doris::snii { + +// CRC32C (Castagnoli, polynomial 0x1EDC6F41). Used to checksum the tail of each +// format block. Thin inline adapter over Doris's bundled Google crc32c thirdparty +// (crc32c::Extend / crc32c::Crc32c). That library computes the same canonical +// CRC32C (same reflected polynomial, same standard pre/post inversion), so every +// on-disk checksum stays byte-identical to the previous in-tree slice-by-8 / +// SSE4.2 implementation -- this is an implementation swap, not a format change. +// The leading :: keeps the crc32c namespace distinct from crc32c() below. +inline uint32_t crc32c_extend(uint32_t crc, Slice data) { + return ::crc32c::Extend(crc, data.data(), data.size()); +} + +inline uint32_t crc32c(Slice data) { + return ::crc32c::Crc32c(data.data(), data.size()); +} + +#ifdef BE_TEST +// T21 test seam. The production crc32c()/crc32c_extend() above delegate to the +// bundled Google crc32c thirdparty (see commit d0416bb4129), which already runs a +// runtime-dispatched, hardware-accelerated and interleaved CRC32C -- so T21's +// "hardware interleaved CRC" goal is already met (and exceeded: that library adds +// a PCLMULQDQ fold a hand-rolled 3-way _mm_crc32_u64 lacks). Rather than regress +// that reuse, the reference sub-paths below let unit tests prove, byte-for-byte +// across all sizes/alignments, that the production path equals the canonical +// CRC32C and that the hardware path is engaged: +// * crc32c_slice8_extend -- portable software slice-by-8 (always available); +// * crc32c_hw_serial_extend -- serial SSE4.2 _mm_crc32 hardware path; +// * crc32c_hw3_extend -- 3-way interleaved SSE4.2 hardware path with a +// GF(2) shift-combine and a 1024-byte fall-back to +// the serial path (the algorithm T21 specifies). +// hw_serial/hw3 fall back to slice8 when SSE4.2 is absent. Each *_extend applies +// the standard ~crc pre/post inversion, so *_extend(0, d) == crc32c(d). The whole +// seam plus its static slice-by-8 table and startup CPUID probe are compiled out +// of release builds by this BE_TEST gate, so production carries no extra code. +// Pure functions with no shared mutable state (CONCURRENCY: N/A). +namespace detail { +uint32_t crc32c_slice8_extend(uint32_t crc, Slice data); +uint32_t crc32c_hw_serial_extend(uint32_t crc, Slice data); +uint32_t crc32c_hw3_extend(uint32_t crc, Slice data); +size_t crc32c_interleave_threshold(); +bool crc32c_has_hw(); +} // namespace detail +#endif // BE_TEST + +} // namespace doris::snii diff --git a/be/src/storage/index/snii/encoding/pfor.cpp b/be/src/storage/index/snii/encoding/pfor.cpp new file mode 100644 index 00000000000000..58caae4305b5e3 --- /dev/null +++ b/be/src/storage/index/snii/encoding/pfor.cpp @@ -0,0 +1,460 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/encoding/pfor.h" + +#include +#include +#include +#include +#include + +#include "storage/index/snii/common/slice.h" + +namespace doris::snii { +namespace { + +// Unaligned little-endian 64-bit load from a raw byte pointer (single +// instruction on x86; memcpy is the portable, UB-free spelling the compiler +// folds to a mov). +inline uint64_t load_u64_le(const uint8_t* p) { + uint64_t v; + std::memcpy(&v, p, sizeof(v)); +#if defined(__BIG_ENDIAN__) || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) + v = __builtin_bswap64(v); +#endif + return v; +} + +// TEST-ONLY seam backing doris::snii::testing::pfor_width_evals(). value_width() +// is the SINGLE per-value bit-width evaluation point on the encode path, so this +// counter equals the number of values processed per run -- the deterministic +// signal that the histogram path scans each value exactly once (vs the former +// O(maxw*n) re-scan). Compiled out entirely in non-test builds so production pays +// nothing; a relaxed atomic under BE_TEST keeps it data-race-free under TSAN. +#ifdef BE_TEST +std::atomic g_width_evals {0}; +#endif + +// Number of significant bits of v (its minimal bit_width); value_width(0) == 0 is +// preserved (matching the former bits_for). clz(0) is undefined behaviour, so +// v == 0 is mapped explicitly here -- never via clz(v | 1), which would mis-score +// 0 as width 1 and corrupt the histogram. +inline uint8_t value_width(uint32_t v) { +#ifdef BE_TEST + g_width_evals.fetch_add(1, std::memory_order_relaxed); +#endif + return v ? static_cast(32 - __builtin_clz(v)) : 0; +} + +// Choose the bit_width that minimizes total bytes (packed + exceptions), with +// exception cost estimated at ~6 bytes each. A single O(n) pass builds a bit-width +// histogram (recording each value's width into widths[] for the encoder to reuse), +// then an O(maxw) suffix-sum gives the exception count per candidate width -- +// replacing the former O(maxw*n) re-scan. The cost formula and the ascending-w +// strict-'<' tie-break are kept identical, so the chosen width (and hence every +// encoded byte) is unchanged. +uint8_t choose_width(const uint32_t* v, size_t n, uint8_t* widths) { + // hist[b] = #values whose bit-width == b. n is capped at kFrqBaseUnit (256) on + // the production path; uint32_t buckets keep a direct caller with a larger run + // correct without affecting the chosen width. + uint32_t hist[33] = {0}; + uint8_t maxw = 0; + for (size_t i = 0; i < n; ++i) { + const uint8_t b = value_width(v[i]); + widths[i] = b; + ++hist[b]; + maxw = std::max(b, maxw); + } + // suffix[k] = #values whose bit-width >= k, so the exceptions for candidate + // width w (values needing more than w bits) are exactly suffix[w + 1]. + size_t suffix[34] = {0}; + for (int k = 32; k >= 0; --k) { + suffix[k] = suffix[k + 1] + hist[k]; + } + uint8_t best = maxw; + size_t best_cost = SIZE_MAX; + for (uint8_t w = 0; w <= maxw; ++w) { + const size_t exc = suffix[w + 1]; + const size_t cost = (static_cast(w) * n + 7) / 8 + exc * 6; + if (cost < best_cost) { + best_cost = cost; + best = w; + } + } + return best; +} + +uint32_t low_mask(uint8_t w) { + return (w >= 32) ? 0xFFFFFFFFU : ((1U << w) - 1U); +} + +// Bit-pack the low w bits of each value, writing 0 at exception positions +// (widths[i] > w) instead of the value's low bits. This is byte-identical to +// packing a copy in which those slots were pre-zeroed (the former `low[i] = 0` +// placeholder), so the encoder no longer materializes that copy. +void bitpack_masked(const uint32_t* v, const uint8_t* widths, size_t n, uint8_t w, ByteSink* out) { + if (w == 0) { + return; + } + // Pre-size for the exact packed byte count (ceil(w*n/8)) so the per-byte put_u8 + // loop below never reallocates mid-pack. ByteSink::reserve is RELATIVE to out's + // current size, and `out` is reused across the PFOR runs of one region, so this + // accumulates run-to-run instead of no-op'ing. Capacity-only: the packed bytes + // and their values are byte-identical to the un-reserved path. + const size_t packed = (static_cast(w) * n + 7) / 8; + out->reserve(packed); + const uint32_t mask = low_mask(w); + uint64_t acc = 0; + int filled = 0; + for (size_t i = 0; i < n; ++i) { + const uint32_t lo = (widths[i] > w) ? 0U : (v[i] & mask); + acc |= static_cast(lo) << filled; + filled += w; + while (filled >= 8) { + out->put_u8(static_cast(acc)); + acc >>= 8; + filled -= 8; + } + } + if (filled > 0) { + out->put_u8(static_cast(acc)); + } +} + +void bitunpack_tail(const uint8_t* base, size_t packed, size_t n, uint8_t w, size_t i, + uint64_t mask, uint32_t* out) { + for (; i < n; ++i) { + const size_t bit_off = static_cast(w) * i; + const size_t byte_off = bit_off >> 3; + uint64_t word = 0; + for (size_t b = byte_off; b < packed && b < byte_off + 8; ++b) { + word |= static_cast(base[b]) << ((b - byte_off) * 8); + } + out[i] = static_cast((word >> (bit_off & 7)) & mask); + } +} + +void bitunpack_w1(const uint8_t* base, size_t n, uint32_t* out) { + size_t i = 0; + size_t byte = 0; + for (; i + 8 <= n; i += 8, ++byte) { + const uint8_t v = base[byte]; + out[i] = v & 1U; + out[i + 1] = (v >> 1) & 1U; + out[i + 2] = (v >> 2) & 1U; + out[i + 3] = (v >> 3) & 1U; + out[i + 4] = (v >> 4) & 1U; + out[i + 5] = (v >> 5) & 1U; + out[i + 6] = (v >> 6) & 1U; + out[i + 7] = (v >> 7) & 1U; + } + if (i < n) { + const uint8_t v = base[byte]; + for (uint8_t bit = 0; i < n; ++i, ++bit) { + out[i] = (v >> bit) & 1U; + } + } +} + +void bitunpack_w2(const uint8_t* base, size_t n, uint32_t* out) { + size_t i = 0; + size_t byte = 0; + for (; i + 4 <= n; i += 4, ++byte) { + const uint8_t v = base[byte]; + out[i] = v & 3U; + out[i + 1] = (v >> 2) & 3U; + out[i + 2] = (v >> 4) & 3U; + out[i + 3] = (v >> 6) & 3U; + } + if (i < n) { + const uint8_t v = base[byte]; + for (uint8_t shift = 0; i < n; ++i, shift += 2) { + out[i] = (v >> shift) & 3U; + } + } +} + +void bitunpack_w3(const uint8_t* base, size_t packed, size_t n, uint32_t* out) { + size_t i = 0; + size_t byte = 0; + for (; i + 8 <= n; i += 8, byte += 3) { + const uint32_t b0 = base[byte]; + const uint32_t b1 = base[byte + 1]; + const uint32_t b2 = base[byte + 2]; + out[i] = b0 & 7U; + out[i + 1] = (b0 >> 3) & 7U; + out[i + 2] = ((b0 >> 6) | (b1 << 2)) & 7U; + out[i + 3] = (b1 >> 1) & 7U; + out[i + 4] = (b1 >> 4) & 7U; + out[i + 5] = ((b1 >> 7) | (b2 << 1)) & 7U; + out[i + 6] = (b2 >> 2) & 7U; + out[i + 7] = (b2 >> 5) & 7U; + } + bitunpack_tail(base, packed, n, 3, i, 7U, out); +} + +void bitunpack_w4(const uint8_t* base, size_t n, uint32_t* out) { + size_t i = 0; + size_t byte = 0; + for (; i + 2 <= n; i += 2, ++byte) { + const uint8_t v = base[byte]; + out[i] = v & 15U; + out[i + 1] = (v >> 4) & 15U; + } + if (i < n) { + out[i] = base[byte] & 15U; + } +} + +void bitunpack_w5(const uint8_t* base, size_t packed, size_t n, uint32_t* out) { + size_t i = 0; + size_t byte = 0; + for (; i + 8 <= n; i += 8, byte += 5) { + const uint32_t b0 = base[byte]; + const uint32_t b1 = base[byte + 1]; + const uint32_t b2 = base[byte + 2]; + const uint32_t b3 = base[byte + 3]; + const uint32_t b4 = base[byte + 4]; + out[i] = b0 & 31U; + out[i + 1] = ((b0 >> 5) | (b1 << 3)) & 31U; + out[i + 2] = (b1 >> 2) & 31U; + out[i + 3] = ((b1 >> 7) | (b2 << 1)) & 31U; + out[i + 4] = ((b2 >> 4) | (b3 << 4)) & 31U; + out[i + 5] = (b3 >> 1) & 31U; + out[i + 6] = ((b3 >> 6) | (b4 << 2)) & 31U; + out[i + 7] = (b4 >> 3) & 31U; + } + bitunpack_tail(base, packed, n, 5, i, 31U, out); +} + +void bitunpack_w6(const uint8_t* base, size_t packed, size_t n, uint32_t* out) { + size_t i = 0; + size_t byte = 0; + for (; i + 4 <= n; i += 4, byte += 3) { + const uint32_t b0 = base[byte]; + const uint32_t b1 = base[byte + 1]; + const uint32_t b2 = base[byte + 2]; + out[i] = b0 & 63U; + out[i + 1] = ((b0 >> 6) | (b1 << 2)) & 63U; + out[i + 2] = ((b1 >> 4) | (b2 << 4)) & 63U; + out[i + 3] = (b2 >> 2) & 63U; + } + bitunpack_tail(base, packed, n, 6, i, 63U, out); +} + +void bitunpack_w7(const uint8_t* base, size_t packed, size_t n, uint32_t* out) { + size_t i = 0; + size_t byte = 0; + for (; i + 8 <= n; i += 8, byte += 7) { + const uint32_t b0 = base[byte]; + const uint32_t b1 = base[byte + 1]; + const uint32_t b2 = base[byte + 2]; + const uint32_t b3 = base[byte + 3]; + const uint32_t b4 = base[byte + 4]; + const uint32_t b5 = base[byte + 5]; + const uint32_t b6 = base[byte + 6]; + out[i] = b0 & 127U; + out[i + 1] = ((b0 >> 7) | (b1 << 1)) & 127U; + out[i + 2] = ((b1 >> 6) | (b2 << 2)) & 127U; + out[i + 3] = ((b2 >> 5) | (b3 << 3)) & 127U; + out[i + 4] = ((b3 >> 4) | (b4 << 4)) & 127U; + out[i + 5] = ((b4 >> 3) | (b5 << 5)) & 127U; + out[i + 6] = ((b5 >> 2) | (b6 << 6)) & 127U; + out[i + 7] = (b6 >> 1) & 127U; + } + bitunpack_tail(base, packed, n, 7, i, 127U, out); +} + +void bitunpack_w8(const uint8_t* base, size_t n, uint32_t* out) { + for (size_t i = 0; i < n; ++i) { + out[i] = base[i]; + } +} + +void bitunpack_generic(const uint8_t* base, size_t packed, size_t n, uint8_t w, uint32_t* out) { + const uint64_t mask = low_mask(w); + size_t i = 0; + if (packed >= 8) { + const size_t last_safe_byte = packed - 8; + for (; i < n; ++i) { + const size_t bit_off = static_cast(w) * i; + const size_t byte_off = bit_off >> 3; + if (byte_off > last_safe_byte) { + break; + } + out[i] = static_cast((load_u64_le(base + byte_off) >> (bit_off & 7)) & mask); + } + } + bitunpack_tail(base, packed, n, w, i, mask, out); +} + +Status bitunpack(ByteSource* src, size_t n, uint8_t w, uint32_t* out) { + if (w == 0) { + std::memset(out, 0, n * sizeof(uint32_t)); + return Status::OK(); + } + // Pull the packed run once and unpack from the contiguous slice; this keeps + // the hot decode path free of per-byte ByteSource calls. + const size_t packed = (static_cast(w) * n + 7) / 8; + Slice buf; + RETURN_IF_ERROR(src->get_bytes(packed, &buf)); + const uint8_t* base = buf.data(); + + switch (w) { + case 1: + bitunpack_w1(base, n, out); + break; + case 2: + bitunpack_w2(base, n, out); + break; + case 3: + bitunpack_w3(base, packed, n, out); + break; + case 4: + bitunpack_w4(base, n, out); + break; + case 5: + bitunpack_w5(base, packed, n, out); + break; + case 6: + bitunpack_w6(base, packed, n, out); + break; + case 7: + bitunpack_w7(base, packed, n, out); + break; + case 8: + bitunpack_w8(base, n, out); + break; + default: + bitunpack_generic(base, packed, n, w, out); + break; + } + return Status::OK(); +} + +} // namespace + +namespace testing { +// Test-only op-count seam; see pfor.h. Reports/resets the per-value bit-width +// evaluation counter, and is a no-op in non-test builds where the counter is +// compiled out. +uint64_t pfor_width_evals() { +#ifdef BE_TEST + return g_width_evals.load(std::memory_order_relaxed); +#else + return 0; +#endif +} +void reset_pfor_width_evals() { +#ifdef BE_TEST + g_width_evals.store(0, std::memory_order_relaxed); +#endif +} +} // namespace testing + +void pfor_encode(const uint32_t* values, size_t n, ByteSink* out) { + // n is hard-capped at kFrqBaseUnit (256) by encode_pfor_runs, so the stack + // buffer is used on every production call; the heap fallback only guards a + // direct caller passing a larger run. + uint8_t widths_stack[256]; + std::vector widths_heap; + uint8_t* widths = widths_stack; + if (n > sizeof(widths_stack)) { + widths_heap.resize(n); + widths = widths_heap.data(); + } + + const uint8_t w = choose_width(values, n, widths); + out->put_u8(w); + + // Count exceptions for the varint header by reusing the cached per-value + // widths -- no further bit-width evaluation. + uint32_t n_exc = 0; + for (size_t i = 0; i < n; ++i) { + n_exc += (widths[i] > w); + } + out->put_varint32(n_exc); + + // Pack the low w bits, writing 0 at exception slots (byte-identical to the + // former zeroed-copy approach) so no separate `low` buffer is materialized. + bitpack_masked(values, widths, n, w, out); + + // Exception table: (index_delta, full_value) in ascending index order. + uint32_t prev = 0; + for (size_t i = 0; i < n; ++i) { + if (widths[i] > w) { + out->put_varint32(static_cast(i) - prev); + out->put_varint32(values[i]); + prev = static_cast(i); + } + } +} + +Status pfor_decode(ByteSource* src, size_t n, uint32_t* out) { + uint8_t w; + RETURN_IF_ERROR(src->get_u8(&w)); + // choose_width never returns above 32. A larger width is damage, and while + // the unpackers stay in bounds for it, they would read up to 32x the honest + // byte count and hand back silently truncated values -- a wrong answer where + // a corruption Status belongs. + if (w > 32) { + return Status::Error( + "pfor bit width {} exceeds 32", w); + } + uint32_t n_exc; + RETURN_IF_ERROR(src->get_varint32(&n_exc)); + RETURN_IF_ERROR(bitunpack(src, n, w, out)); + uint32_t idx = 0; + for (uint32_t i = 0; i < n_exc; ++i) { + uint32_t d, val; + RETURN_IF_ERROR(src->get_varint32(&d)); + RETURN_IF_ERROR(src->get_varint32(&val)); + idx += d; + if (idx >= n) { + return Status::Error( + "pfor exception index out of range"); + } + out[idx] = val; + } + return Status::OK(); +} + +Status pfor_skip(ByteSource* src, size_t n) { + uint8_t w = 0; + RETURN_IF_ERROR(src->get_u8(&w)); + uint32_t n_exc = 0; + RETURN_IF_ERROR(src->get_varint32(&n_exc)); + const size_t packed = (static_cast(w) * n + 7) / 8; + Slice unused; + RETURN_IF_ERROR(src->get_bytes(packed, &unused)); + uint32_t idx = 0; + for (uint32_t i = 0; i < n_exc; ++i) { + uint32_t d = 0; + uint32_t val = 0; + RETURN_IF_ERROR(src->get_varint32(&d)); + RETURN_IF_ERROR(src->get_varint32(&val)); + idx += d; + if (idx >= n) { + return Status::Error( + "pfor exception index out of range"); + } + } + return Status::OK(); +} + +} // namespace doris::snii diff --git a/be/src/storage/index/snii/encoding/pfor.h b/be/src/storage/index/snii/encoding/pfor.h new file mode 100644 index 00000000000000..709d35748249f1 --- /dev/null +++ b/be/src/storage/index/snii/encoding/pfor.h @@ -0,0 +1,53 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include "common/status.h" +#include "storage/index/snii/encoding/byte_sink.h" +#include "storage/index/snii/encoding/byte_source.h" + +namespace doris::snii { + +// PFOR integer block encoder/decoder (unsigned uint32 array). +// Encoded layout: [u8 bit_width][varint n_exceptions][bit-packed low +// bits][exception table]. Selects the bit_width that minimizes total byte size; +// values exceeding it go into the exception table (index_delta, full_value). +// delta/zigzag is handled by the upper layer (.frq window); PFOR only processes +// unsigned integer arrays. +void pfor_encode(const uint32_t* values, size_t n, ByteSink* out); +Status pfor_decode(ByteSource* src, size_t n, uint32_t* out); +Status pfor_skip(ByteSource* src, size_t n); + +} // namespace doris::snii + +// Test-only instrumentation seam (mirrors the dict-block decode-counter pattern). +// pfor_width_evals() returns a process-global count of per-value bit-width +// evaluations performed by pfor_encode since the last reset -- one per +// value_width() call, the single evaluation point on the encode path. Deterministic +// perf tests assert it equals the number of encoded values per run, proving the +// histogram path scans each value exactly once (vs the former O(maxw*n) re-scan). +// Compiled to a no-op in non-test builds; reset between tests. +namespace doris::snii::testing { + +uint64_t pfor_width_evals(); +void reset_pfor_width_evals(); + +} // namespace doris::snii::testing diff --git a/be/src/storage/index/snii/encoding/section_framer.cpp b/be/src/storage/index/snii/encoding/section_framer.cpp new file mode 100644 index 00000000000000..e67b618758dae8 --- /dev/null +++ b/be/src/storage/index/snii/encoding/section_framer.cpp @@ -0,0 +1,59 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/encoding/section_framer.h" + +#include "storage/index/snii/encoding/crc32c.h" + +namespace doris::snii { + +void SectionFramer::write(ByteSink& sink, uint8_t section_type, Slice payload) { + // Single-copy framing: write [type][varint64 len][payload] straight into the + // target sink, then crc exactly those bytes. view() is taken AFTER the payload + // and BEFORE the crc, so subslice([start, framed_len)) is over a settled, + // contiguous buffer with no pending realloc/aliasing. Byte-identical to the + // former temp-ByteSink assembly, minus one heap alloc + one payload copy. + const size_t start = sink.size(); + sink.put_u8(section_type); + sink.put_varint64(payload.size()); + sink.put_bytes(payload); + const size_t framed_len = sink.size() - start; + const uint32_t crc = crc32c(sink.view().subslice(start, framed_len)); + sink.put_fixed32(crc); +} + +Status SectionFramer::read(ByteSource& src, FramedSection* out) { + size_t start = src.position(); + uint8_t type; + RETURN_IF_ERROR(src.get_u8(&type)); + uint64_t len; + RETURN_IF_ERROR(src.get_varint64(&len)); + Slice payload; + RETURN_IF_ERROR(src.get_bytes(static_cast(len), &payload)); + size_t framed_len = src.position() - start; + uint32_t stored; + RETURN_IF_ERROR(src.get_fixed32(&stored)); + if (crc32c(src.slice_from(start, framed_len)) != stored) { + return Status::Error( + "section crc mismatch"); + } + out->type = type; + out->payload = payload; + return Status::OK(); +} + +} // namespace doris::snii diff --git a/be/src/storage/index/snii/encoding/section_framer.h b/be/src/storage/index/snii/encoding/section_framer.h new file mode 100644 index 00000000000000..9a248381ea97b6 --- /dev/null +++ b/be/src/storage/index/snii/encoding/section_framer.h @@ -0,0 +1,44 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include "common/status.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/encoding/byte_sink.h" +#include "storage/index/snii/encoding/byte_source.h" + +namespace doris::snii { + +// A framed section: type + payload view. +struct FramedSection { + uint8_t type = 0; + Slice payload; +}; + +// Unified section framing: [u8 type][varint64 len][payload][fixed32 crc32c(type+len+payload)]. +// All full-format sections reuse this encode/checksum path to avoid ad-hoc hand-assembly. +// Unknown optional sections are dispatched by the caller based on type; read still verifies the CRC and skips the payload. +class SectionFramer { +public: + static void write(ByteSink& sink, uint8_t section_type, Slice payload); + static Status read(ByteSource& src, FramedSection* out); +}; + +} // namespace doris::snii diff --git a/be/src/storage/index/snii/encoding/varint.cpp b/be/src/storage/index/snii/encoding/varint.cpp new file mode 100644 index 00000000000000..a53a08de6b8d3b --- /dev/null +++ b/be/src/storage/index/snii/encoding/varint.cpp @@ -0,0 +1,73 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/encoding/varint.h" + +namespace doris::snii { + +size_t varint_len(uint64_t v) { + size_t n = 1; + while (v >= 0x80) { + v >>= 7; + ++n; + } + return n; +} + +size_t encode_varint64(uint64_t v, uint8_t* out) { + size_t i = 0; + while (v >= 0x80) { + out[i++] = static_cast(v) | 0x80; + v >>= 7; + } + out[i++] = static_cast(v); + return i; +} + +size_t encode_varint32(uint32_t v, uint8_t* out) { + return encode_varint64(v, out); +} + +Status decode_varint64(const uint8_t* p, const uint8_t* end, uint64_t* v, const uint8_t** next) { + uint64_t result = 0; + int shift = 0; + while (p < end) { + uint8_t b = *p++; + result |= static_cast(b & 0x7F) << shift; + if ((b & 0x80) == 0) { + *v = result; + *next = p; + return Status::OK(); + } + shift += 7; + if (shift >= 64) + return Status::Error( + "varint64 overflow"); + } + return Status::Error("varint truncated"); +} + +Status decode_varint32(const uint8_t* p, const uint8_t* end, uint32_t* v, const uint8_t** next) { + uint64_t tmp; + RETURN_IF_ERROR(decode_varint64(p, end, &tmp, next)); + if (tmp > 0xFFFFFFFFu) + return Status::Error("varint32 overflow"); + *v = static_cast(tmp); + return Status::OK(); +} + +} // namespace doris::snii diff --git a/be/src/storage/index/snii/encoding/varint.h b/be/src/storage/index/snii/encoding/varint.h new file mode 100644 index 00000000000000..978812ea6e66f6 --- /dev/null +++ b/be/src/storage/index/snii/encoding/varint.h @@ -0,0 +1,43 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include "common/status.h" + +namespace doris::snii { + +// LEB128 variable-length integer encoding + zigzag. out buffer must be >=10 bytes; returns number of bytes written. +size_t varint_len(uint64_t v); +size_t encode_varint32(uint32_t v, uint8_t* out); +size_t encode_varint64(uint64_t v, uint8_t* out); + +// Decode a varint from the range [p, end); on success *next points to the next byte after the consumed input. +Status decode_varint32(const uint8_t* p, const uint8_t* end, uint32_t* v, const uint8_t** next); +Status decode_varint64(const uint8_t* p, const uint8_t* end, uint64_t* v, const uint8_t** next); + +inline uint64_t zigzag_encode(int64_t v) { + return (static_cast(v) << 1) ^ static_cast(v >> 63); +} +inline int64_t zigzag_decode(uint64_t v) { + return static_cast(v >> 1) ^ -static_cast(v & 1); +} + +} // namespace doris::snii diff --git a/be/src/storage/index/snii/encoding/zstd_codec.cpp b/be/src/storage/index/snii/encoding/zstd_codec.cpp new file mode 100644 index 00000000000000..8ebc67c0a590e3 --- /dev/null +++ b/be/src/storage/index/snii/encoding/zstd_codec.cpp @@ -0,0 +1,52 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/encoding/zstd_codec.h" + +#include + +#include + +namespace doris::snii { + +Status zstd_compress(Slice input, int level, std::vector* out) { + size_t bound = ZSTD_compressBound(input.size()); + out->resize(bound); + size_t n = ZSTD_compress(out->data(), bound, input.data(), input.size(), level); + if (ZSTD_isError(n)) { + return Status::Error(std::string("zstd compress: ") + + ZSTD_getErrorName(n)); + } + out->resize(n); + return Status::OK(); +} + +Status zstd_decompress(Slice input, size_t expected_uncomp_len, std::vector* out) { + out->resize(expected_uncomp_len); + size_t n = ZSTD_decompress(out->data(), expected_uncomp_len, input.data(), input.size()); + if (ZSTD_isError(n)) { + return Status::Error( + std::string("zstd decompress: ") + ZSTD_getErrorName(n)); + } + if (n != expected_uncomp_len) { + return Status::Error( + "zstd decompressed length mismatch"); + } + return Status::OK(); +} + +} // namespace doris::snii diff --git a/be/src/storage/index/snii/encoding/zstd_codec.h b/be/src/storage/index/snii/encoding/zstd_codec.h new file mode 100644 index 00000000000000..c8291ce024fca7 --- /dev/null +++ b/be/src/storage/index/snii/encoding/zstd_codec.h @@ -0,0 +1,33 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/common/slice.h" + +namespace doris::snii { + +// Thin ZSTD wrapper. Used for compressing large payloads such as .prx windows. Decompression requires the caller to supply the original uncompressed length (from the block header). +Status zstd_compress(Slice input, int level, std::vector* out); +Status zstd_decompress(Slice input, size_t expected_uncomp_len, std::vector* out); + +} // namespace doris::snii diff --git a/be/src/storage/index/snii/format/bootstrap_header.cpp b/be/src/storage/index/snii/format/bootstrap_header.cpp new file mode 100644 index 00000000000000..6cdc1869385bec --- /dev/null +++ b/be/src/storage/index/snii/format/bootstrap_header.cpp @@ -0,0 +1,113 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/format/bootstrap_header.h" + +#include "storage/index/snii/encoding/byte_source.h" +#include "storage/index/snii/encoding/crc32c.h" + +namespace doris::snii::format { + +namespace { + +// Number of bytes covered by header_checksum: everything except the trailing +// crc32c. +constexpr size_t kChecksumCoverage = kBootstrapHeaderSize - 4; + +// Writes all fixed fields except the trailing checksum. Field order is the +// on-disk contract; reuse ByteSink fixed-width primitives, never hand-assemble +// bytes. +void encode_fields(const BootstrapHeader& header, ByteSink* sink) { + sink->put_fixed32(header.magic); + sink->put_fixed32((static_cast(header.min_reader_version) << 16) | + header.format_version); + sink->put_fixed32(header.flags); + sink->put_fixed32(kBootstrapHeaderSize); // header_length is always derived + sink->put_u8(header.tail_pointer_size); +} + +} // namespace + +Status encode_bootstrap_header(const BootstrapHeader& header, ByteSink* sink) { + if (sink == nullptr) { + return Status::Error("bootstrap_header: null sink"); + } + ByteSink fields; + encode_fields(header, &fields); + const uint32_t checksum = crc32c(fields.view()); + sink->put_bytes(fields.view()); + sink->put_fixed32(checksum); + return Status::OK(); +} + +Status decode_bootstrap_header(Slice data, BootstrapHeader* out) { + if (out == nullptr) { + return Status::Error("bootstrap_header: null out"); + } + // Reject any size other than the exact fixed header: short input is + // truncation, longer input means stray trailing bytes the parser would + // otherwise ignore. + if (data.size() != kBootstrapHeaderSize) { + return Status::Error( + "bootstrap_header: wrong header size"); + } + + ByteSource src(data); + uint32_t magic = 0; + uint32_t version_pair = 0; + uint32_t flags = 0; + uint32_t header_length = 0; + uint8_t tail_pointer_size = 0; + uint32_t stored_checksum = 0; + RETURN_IF_ERROR(src.get_fixed32(&magic)); + RETURN_IF_ERROR(src.get_fixed32(&version_pair)); + RETURN_IF_ERROR(src.get_fixed32(&flags)); + RETURN_IF_ERROR(src.get_fixed32(&header_length)); + RETURN_IF_ERROR(src.get_u8(&tail_pointer_size)); + RETURN_IF_ERROR(src.get_fixed32(&stored_checksum)); + + if (magic != kContainerMagic) { + return Status::Error( + "bootstrap_header: bad container magic"); + } + const uint32_t computed = crc32c(data.subslice(0, kChecksumCoverage)); + if (computed != stored_checksum) { + return Status::Error( + "bootstrap_header: checksum mismatch"); + } + + const auto min_reader_version = static_cast((version_pair >> 16) & 0xFFFFu); + const auto format_version = static_cast(version_pair & 0xFFFFu); + if (format_version != kFormatVersion) { + return Status::Error( + "bootstrap_header: unsupported container format_version"); + } + if (min_reader_version > kFormatVersion) { + return Status::Error( + "bootstrap_header: container requires a newer reader version"); + } + + out->magic = magic; + out->format_version = format_version; + out->min_reader_version = min_reader_version; + out->flags = flags; + out->header_length = header_length; + out->tail_pointer_size = tail_pointer_size; + return Status::OK(); +} + +} // namespace doris::snii::format diff --git a/be/src/storage/index/snii/format/bootstrap_header.h b/be/src/storage/index/snii/format/bootstrap_header.h new file mode 100644 index 00000000000000..6a8c0e8083d3ac --- /dev/null +++ b/be/src/storage/index/snii/format/bootstrap_header.h @@ -0,0 +1,71 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include "common/status.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/encoding/byte_sink.h" +#include "storage/index/snii/format/format_constants.h" + +namespace doris::snii::format { + +// Fixed container header at the very start of a {rowset_id}_{seg_id}.idx file. +// Identifies the SNII container and carries basic compatibility info so a +// reader can fail fast before touching any streamed section or the tail meta +// region. +// +// On-disk layout (all multi-byte fields little-endian, fixed width; NOT framed +// by SectionFramer because it must be parseable without prior knowledge of the +// file): +// u32 magic == kContainerMagic +// u16 format_version == kFormatVersion +// u16 min_reader_version readers with kFormatVersion < this MUST refuse to +// read u32 flags container-level feature flags u32 +// header_length total bytes of this header including the checksum u8 +// tail_pointer_size size of the fixed tail pointer at EOF (hint for the +// reader) u32 header_checksum crc32c over all preceding header bytes +struct BootstrapHeader { + uint32_t magic = kContainerMagic; + uint16_t format_version = kFormatVersion; + uint16_t min_reader_version = kMinReaderVersion; + uint32_t flags = 0; + uint32_t header_length = 0; + uint8_t tail_pointer_size = 0; +}; + +// Total fixed on-disk size of the header, including the trailing crc32c. +inline constexpr uint32_t kBootstrapHeaderSize = + 4 /*magic*/ + 2 /*format_version*/ + 2 /*min_reader_version*/ + 4 /*flags*/ + + 4 /*header_length*/ + 1 /*tail_pointer_size*/ + 4 /*header_checksum*/; + +// Serializes the header to sink: writes header_length = kBootstrapHeaderSize +// and appends a crc32c over all preceding bytes. The caller's header_length +// field is ignored on input (it is always derived). Returns OK. +Status encode_bootstrap_header(const BootstrapHeader& header, ByteSink* sink); + +// Parses and validates a bootstrap header from the front of data. +// - too short / trailing bytes beyond the fixed header -> kCorruption +// - magic != kContainerMagic -> kCorruption +// - checksum mismatch -> kCorruption +// - format_version != kFormatVersion -> kUnsupported +// - min_reader_version > kFormatVersion -> kUnsupported +Status decode_bootstrap_header(Slice data, BootstrapHeader* out); + +} // namespace doris::snii::format diff --git a/be/src/storage/index/snii/format/bsbf.cpp b/be/src/storage/index/snii/format/bsbf.cpp new file mode 100644 index 00000000000000..c249cad1e90229 --- /dev/null +++ b/be/src/storage/index/snii/format/bsbf.cpp @@ -0,0 +1,255 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/format/bsbf.h" + +#include + +#include "storage/index/snii/encoding/crc32c.h" + +#if defined(__x86_64__) || defined(_M_X64) +#include +#define SNII_BSBF_X86 1 +#endif + +#define XXH_INLINE_ALL +#include "xxhash.h" + +namespace doris::snii::format { + +const uint32_t kBsbfSalt[kBsbfBitsSetPerBlock] = {0x47b6137bU, 0x44974d91U, 0x8824ad5bU, + 0xa2b7289dU, 0x705495c7U, 0x2df1424bU, + 0x9efc4947U, 0x5c6bfb31U}; + +namespace { + +void store_le32(uint8_t* p, uint32_t v) { + p[0] = static_cast(v); + p[1] = static_cast(v >> 8); + p[2] = static_cast(v >> 16); + p[3] = static_cast(v >> 24); +} +uint32_t load_le32(const uint8_t* p) { + return static_cast(p[0]) | (static_cast(p[1]) << 8) | + (static_cast(p[2]) << 16) | (static_cast(p[3]) << 24); +} + +#if defined(SNII_BSBF_X86) +bool cpu_has_avx2() { + static const bool v = __builtin_cpu_supports("avx2"); + return v; +} +#endif + +// --- scalar kernels --- +inline void masks_scalar(uint32_t key, uint32_t m[8]) { + for (int i = 0; i < 8; ++i) m[i] = 1u << ((key * kBsbfSalt[i]) >> 27); +} +bool block_contains_scalar(uint64_t hash, const uint8_t* block) { + const uint32_t* w = reinterpret_cast(block); // LE + uint32_t m[8]; + masks_scalar(static_cast(hash), m); + for (int i = 0; i < 8; ++i) + if ((load_le32(reinterpret_cast(w + i)) & m[i]) != m[i]) return false; + return true; +} +void insert_scalar(uint32_t* words, uint32_t block, uint32_t key) { + uint32_t m[8]; + masks_scalar(key, m); + for (int i = 0; i < 8; ++i) words[block * 8 + i] |= m[i]; +} +bool find_scalar(const uint32_t* words, uint32_t block, uint32_t key) { + uint32_t m[8]; + masks_scalar(key, m); + for (int i = 0; i < 8; ++i) + if ((words[block * 8 + i] & m[i]) != m[i]) return false; + return true; +} + +#if defined(SNII_BSBF_X86) +// --- AVX2 kernels: a 256-bit block is one YMM register --- +__attribute__((target("avx2"))) __m256i mask_avx2(uint32_t key) { + const __m256i salt = + _mm256_setr_epi32(static_cast(kBsbfSalt[0]), static_cast(kBsbfSalt[1]), + static_cast(kBsbfSalt[2]), static_cast(kBsbfSalt[3]), + static_cast(kBsbfSalt[4]), static_cast(kBsbfSalt[5]), + static_cast(kBsbfSalt[6]), static_cast(kBsbfSalt[7])); + const __m256i prod = _mm256_mullo_epi32(_mm256_set1_epi32(static_cast(key)), salt); + const __m256i shifts = _mm256_srli_epi32(prod, 27); // top 5 bits -> 0..31 + return _mm256_sllv_epi32(_mm256_set1_epi32(1), shifts); +} +__attribute__((target("avx2"))) bool block_contains_avx2(uint64_t hash, const uint8_t* block) { + const __m256i m = mask_avx2(static_cast(hash)); + const __m256i b = _mm256_loadu_si256(reinterpret_cast(block)); + return _mm256_testc_si256(b, m) != 0; // (~b & m) == 0 -> b contains m +} +__attribute__((target("avx2"))) void insert_avx2(uint32_t* words, uint32_t block, uint32_t key) { + __m256i* p = reinterpret_cast<__m256i*>(words + block * 8); + _mm256_storeu_si256(p, _mm256_or_si256(_mm256_loadu_si256(p), mask_avx2(key))); +} +__attribute__((target("avx2"))) bool find_avx2(const uint32_t* words, uint32_t block, + uint32_t key) { + const __m256i m = mask_avx2(key); + const __m256i b = _mm256_loadu_si256(reinterpret_cast(words + block * 8)); + return _mm256_testc_si256(b, m) != 0; +} +#endif + +} // namespace + +uint64_t bsbf_hash(std::string_view term) { + return XXH64(term.data(), term.size(), /*seed=*/0); +} + +uint32_t bsbf_optimal_num_bytes(uint32_t ndv, double fpp) { + // Parquet OptimalNumOfBits, then >>3 for bytes. + const double m = -8.0 * ndv / std::log(1 - std::pow(fpp, 1.0 / 8)); + uint32_t num_bits; + if (m < 0 || m > static_cast(kBsbfMaxBytes) * 8) { + num_bits = kBsbfMaxBytes << 3; + } else { + num_bits = static_cast(m); + } + if (num_bits < (kBsbfMinBytes << 3)) num_bits = kBsbfMinBytes << 3; + // G16-g: round up to the 32-byte block only, NOT to the next power of 2. + // The block index is fastrange ((h >> 32) * num_blocks >> 32, bsbf.h), + // which is uniform for ANY block count -- the old power-of-2 rounding was + // a sizing convention, not an addressing requirement, and cost up to ~2x + // bitset bytes (e.g. a 31M-term segment: 40.3 MB optimal -> 64 MB rounded) + // while silently over-delivering on fpp. Exact sizing keeps the fpp at + // the kBsbfFpp design point. + const uint32_t block_bits = kBsbfBytesPerBlock << 3; + num_bits = (num_bits + block_bits - 1) / block_bits * block_bits; + if (num_bits > (kBsbfMaxBytes << 3)) num_bits = kBsbfMaxBytes << 3; + return num_bits >> 3; +} + +bool bsbf_block_contains(uint64_t hash, const uint8_t block[kBsbfBytesPerBlock]) { +#if defined(SNII_BSBF_X86) + if (cpu_has_avx2()) return block_contains_avx2(hash, block); +#endif + return block_contains_scalar(hash, block); +} + +Status BsbfBuilder::create(uint32_t ndv, double fpp, BsbfBuilder* out) { + if (out == nullptr) return Status::Error("bsbf: null out"); + if (!(fpp > 0.0 && fpp < 1.0)) + return Status::Error("bsbf: fpp out of (0,1)"); + if (ndv == 0) ndv = 1; + out->num_bytes_ = bsbf_optimal_num_bytes(ndv, fpp); + out->num_blocks_ = out->num_bytes_ / kBsbfBytesPerBlock; + out->ndv_ = ndv; + out->words_.assign(out->num_bytes_ / 4, 0u); + return Status::OK(); +} + +void BsbfBuilder::insert(uint64_t hash) { + const uint32_t block = bsbf_block_index(hash, num_blocks_); + const uint32_t key = static_cast(hash); +#if defined(SNII_BSBF_X86) + if (cpu_has_avx2()) { + insert_avx2(words_.data(), block, key); + return; + } +#endif + insert_scalar(words_.data(), block, key); +} + +bool BsbfBuilder::maybe_contains(uint64_t hash) const { + const uint32_t block = bsbf_block_index(hash, num_blocks_); + const uint32_t key = static_cast(hash); +#if defined(SNII_BSBF_X86) + if (cpu_has_avx2()) return find_avx2(words_.data(), block, key); +#endif + return find_scalar(words_.data(), block, key); +} + +Status BsbfBuilder::serialize(ByteSink* sink) const { + if (sink == nullptr) + return Status::Error("bsbf: null sink"); + if (num_bytes_ == 0) + return Status::Error("bsbf: not built"); + uint8_t hdr[kBsbfHeaderSize] = {0}; + hdr[0] = 'B'; + hdr[1] = 'S'; + hdr[2] = 'B'; + hdr[3] = 'F'; + hdr[4] = 1; // version + hdr[5] = 0; // hash strategy: XXH64 seed 0 + hdr[6] = 0; // index strategy: fastrange + hdr[7] = 0; // pad + store_le32(hdr + 8, num_bytes_); + store_le32(hdr + 12, num_blocks_); + store_le32(hdr + 16, ndv_); + store_le32(hdr + 20, crc32c(Slice(hdr, 20))); // header crc over [0,20) + const uint8_t* bits = reinterpret_cast(words_.data()); + store_le32(hdr + 24, crc32c(Slice(bits, num_bytes_))); // bitset crc + sink->put_bytes(Slice(hdr, kBsbfHeaderSize)); + sink->put_bytes(Slice(bits, num_bytes_)); // contiguous, uncompressed, LE + return Status::OK(); +} + +Status BsbfHeader::parse(Slice h, uint64_t section_base, BsbfHeader* out) { + if (out == nullptr) return Status::Error("bsbf: null out"); + if (h.size() < kBsbfHeaderSize) + return Status::Error("bsbf: short header"); + const uint8_t* p = h.data(); + if (p[0] != 'B' || p[1] != 'S' || p[2] != 'B' || p[3] != 'F') + return Status::Error("bsbf: bad magic"); + if (p[4] != 1) + return Status::Error("bsbf: bad version"); + if (p[5] != 0) + return Status::Error( + "bsbf: unsupported hash strategy"); + if (p[6] != 0) + return Status::Error( + "bsbf: unsupported index strategy"); + if (crc32c(Slice(p, 20)) != load_le32(p + 20)) + return Status::Error( + "bsbf: header crc mismatch"); + const uint32_t nb = load_le32(p + 8); + const uint32_t nblk = load_le32(p + 12); + // G16-g: num_bytes only needs 32-byte block alignment -- the fastrange + // block index is uniform for any block count. Power-of-2 sizes (all + // pre-G16-g filters) remain valid as a subset. + if (nb < kBsbfMinBytes || nb > kBsbfMaxBytes || nb % kBsbfBytesPerBlock != 0) + return Status::Error( + "bsbf: num_bytes out of range or not block-aligned"); + if (nblk != nb / kBsbfBytesPerBlock) + return Status::Error( + "bsbf: num_blocks mismatch"); + out->num_bytes = nb; + out->num_blocks = nblk; + out->bitset_crc = load_le32(p + 24); + out->bitset_base = section_base + kBsbfHeaderSize; + return Status::OK(); +} + +Status bsbf_probe(io::FileReader* reader, const BsbfHeader& header, uint64_t hash, + bool* maybe_present) { + if (reader == nullptr || maybe_present == nullptr) + return Status::Error("bsbf: null arg"); + std::vector blk; + RETURN_IF_ERROR(reader->read_at(header.block_offset(hash), kBsbfBytesPerBlock, &blk)); + if (blk.size() < kBsbfBytesPerBlock) + return Status::Error( + "bsbf: short block read"); + *maybe_present = bsbf_block_contains(hash, blk.data()); + return Status::OK(); +} + +} // namespace doris::snii::format diff --git a/be/src/storage/index/snii/format/bsbf.h b/be/src/storage/index/snii/format/bsbf.h new file mode 100644 index 00000000000000..7877073fedd10b --- /dev/null +++ b/be/src/storage/index/snii/format/bsbf.h @@ -0,0 +1,135 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/encoding/byte_sink.h" +#include "storage/index/snii/io/file_reader.h" + +// Block-split bloom filter (BSBF) -- Apache Parquet split-block spec, with an +// S3-native on-demand single-block probe that none of the reference implementations +// (Apache Parquet, Doris storage, Doris format/parquet) ship. +// +// BIT FORMAT IS PARQUET-CANONICAL (interoperable with Apache Parquet / Doris +// format/parquet for the bitset bytes): +// - 256-bit (32-byte) blocks, 8 bits set per block. +// - key = XXH64(term, seed=0); high 32 bits select the block via FASTRANGE +// `block = ((hash>>32) * num_blocks) >> 32` (no power-of-2 requirement); low 32 +// bits select 8 in-block positions `1 << ((key * SALT[i]) >> 27)`. +// - num_bytes via Parquet OptimalNumOfBytes: power of 2 in [32, 128 MiB]. +// +// SNII WRAPPER (NOT Parquet's variable thrift header): a FIXED 28-byte header, then +// the contiguous, uncompressed, little-endian bitset. Because the header size is a +// constant, the bitset start is a constant offset (`section_base + 28`) and block i +// is at `section_base + 28 + i*32` -- so a single 32-byte block can be range-read on +// demand WITHOUT parsing a variable-length header and WITHOUT loading the whole blob. +namespace doris::snii::format { + +constexpr uint32_t kBsbfBytesPerBlock = 32; // 256-bit block +constexpr uint32_t kBsbfBitsSetPerBlock = 8; // 8 uint32 words / block +constexpr uint32_t kBsbfMinBytes = 32; +constexpr uint32_t kBsbfMaxBytes = 128u * 1024 * 1024; // Parquet kMaximumBloomFilterBytes +constexpr uint32_t kBsbfHeaderSize = 28; // FIXED (constant bitset offset) +// L0/L1 tiering threshold (the "fast-reject absent terms" design): a bsbf section whose total +// size is <= this is loaded WHOLE into the resident reader at open (L0 -> free +// in-memory probe, no per-lookup round); larger filters stay L1 (header-only, probed +// one 32-byte block on demand). 256 KiB fits in a single cloud FileCache block. +constexpr uint32_t kBsbfResidentMaxBytes = 256u * 1024; + +// Canonical Parquet/Doris split-block SALT (8 odd 32-bit constants). +extern const uint32_t kBsbfSalt[kBsbfBitsSetPerBlock]; + +// XXH64(term, seed=0) -- the Parquet-canonical key (NOT XXH3, NOT Doris murmur). +uint64_t bsbf_hash(std::string_view term); + +// Parquet OptimalNumOfBytes(ndv, fpp): power of 2 in [32, 128 MiB]. +uint32_t bsbf_optimal_num_bytes(uint32_t ndv, double fpp); + +// Fastrange block index from a 64-bit hash and the block count. +inline uint32_t bsbf_block_index(uint64_t hash, uint32_t num_blocks) { + return static_cast(((hash >> 32) * num_blocks) >> 32); +} + +// Pure 32-byte-block kernel: does `block` contain the key's 8 bits? SIMD (AVX2) +// accelerated at runtime when available, scalar otherwise. Returns true => the term +// MAY be present (could be a false positive); false => DEFINITELY ABSENT. +bool bsbf_block_contains(uint64_t hash, const uint8_t block[kBsbfBytesPerBlock]); + +// In-memory builder + serializer. +class BsbfBuilder { +public: + BsbfBuilder() = default; + + // Sizes the filter for `ndv` distinct keys at target `fpp`. fpp in (0,1). + static Status create(uint32_t ndv, double fpp, BsbfBuilder* out); + + // Insert a key / term. SIMD-accelerated. + void insert(uint64_t hash); + void insert_term(std::string_view term) { insert(bsbf_hash(term)); } + + // In-memory probe over the resident bitset (build/warm path). SIMD-accelerated. + bool maybe_contains(uint64_t hash) const; + bool maybe_contains_term(std::string_view term) const { + return maybe_contains(bsbf_hash(term)); + } + + // Serialize [28-byte header][contiguous LE bitset] into `sink`. The header carries + // magic/version/hash+index strategy/num_bytes/num_blocks/ndv + header & bitset + // crc32c. The bitset is Parquet-canonical bytes. + Status serialize(ByteSink* sink) const; + + uint32_t num_bytes() const { return num_bytes_; } + uint32_t num_blocks() const { return num_blocks_; } + size_t resident_capacity_bytes() const { return words_.capacity() * sizeof(uint32_t); } + +private: + std::vector words_; // num_bytes_/4, blocks of 8 words + uint32_t num_bytes_ = 0; + uint32_t num_blocks_ = 0; + uint32_t ndv_ = 0; +}; + +// Resident header (28 bytes), parsed once at open. Validates magic/version/crc/bounds. +struct BsbfHeader { + uint32_t num_bytes = 0; + uint32_t num_blocks = 0; + uint32_t bitset_crc = 0; // stored crc32c of the bitset body (for L0 verification) + uint64_t bitset_base = 0; // absolute file offset of block 0 = section_base + 28 + + // Parse a 28-byte header located at `section_base` in the file. The bitset_base + // is set to section_base + kBsbfHeaderSize. + static Status parse(Slice header28, uint64_t section_base, BsbfHeader* out); + + // Absolute file offset of the 32-byte block this hash maps to. + uint64_t block_offset(uint64_t hash) const { + return bitset_base + + static_cast(bsbf_block_index(hash, num_blocks)) * kBsbfBytesPerBlock; + } +}; + +// On-demand probe: read EXACTLY ONE 32-byte block via `reader`, then test. No whole +// blob load, no deep copy. *maybe_present=false means DEFINITELY ABSENT. +Status bsbf_probe(io::FileReader* reader, const BsbfHeader& header, uint64_t hash, + bool* maybe_present); + +} // namespace doris::snii::format diff --git a/be/src/storage/index/snii/format/core_metadata.cpp b/be/src/storage/index/snii/format/core_metadata.cpp new file mode 100644 index 00000000000000..f9007cc96768b3 --- /dev/null +++ b/be/src/storage/index/snii/format/core_metadata.cpp @@ -0,0 +1,285 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/format/core_metadata.h" + +#include +#include +#include +#include + +#include "gen_cpp/snii.pb.h" +#include "storage/index/snii/encoding/byte_source.h" +#include "storage/index/snii/encoding/section_framer.h" + +namespace doris::snii::format { +namespace { + +using segment_v2::inverted_index::CommonGramsCoverage; +using segment_v2::inverted_index::CommonGramsSegmentMetadata; +using segment_v2::inverted_index::PlainTermKeyVersion; +using segment_v2::inverted_index::ScoringCoverage; +using segment_v2::inverted_index::validate_common_grams_segment_metadata; +using segment_v2::inverted_index::validate_snii_scoring_metadata; + +Status corrupted(std::string_view message) { + return Status::Error(message); +} + +Status unsupported(std::string_view message) { + return Status::Error(message); +} + +Status validate_index_config(uint32_t value, IndexConfig* out) { + switch (value) { + case static_cast(IndexConfig::kDocsOnly): + case static_cast(IndexConfig::kDocsPositions): + case static_cast(IndexConfig::kDocsPositionsScoring): + *out = static_cast(value); + return Status::OK(); + default: + return unsupported("core metadata: unsupported index config"); + } +} + +Status validate_posting_policy(uint32_t value, CommonGramsPostingPolicy* out) { + switch (value) { + case 0: + *out = CommonGramsPostingPolicy::kNone; + return Status::OK(); + case 1: + *out = CommonGramsPostingPolicy::kHybridV1; + return Status::OK(); + default: + return unsupported("core metadata: unsupported CommonGrams posting policy"); + } +} + +Status validate_plain_term_key_version(uint32_t value) { + switch (value) { + case static_cast(PlainTermKeyVersion::kLegacyRaw): + case static_cast(PlainTermKeyVersion::kEscapedV1): + case static_cast(PlainTermKeyVersion::kRawNoInternal): + return Status::OK(); + default: + return unsupported("core metadata: unsupported plain-term key version"); + } +} + +Status validate_common_grams_coverage(uint32_t value) { + switch (value) { + case static_cast(CommonGramsCoverage::kNone): + case static_cast(CommonGramsCoverage::kComplete): + case static_cast(CommonGramsCoverage::kMixed): + return Status::OK(); + default: + return unsupported("core metadata: unsupported CommonGrams coverage"); + } +} + +Status validate_scoring_coverage(uint32_t value) { + switch (value) { + case static_cast(ScoringCoverage::kNone): + case static_cast(ScoringCoverage::kComplete): + return Status::OK(); + default: + return unsupported("core metadata: unsupported scoring coverage"); + } +} + +void encode_region_ref(const RegionRef& ref, doris::snii::SniiRegionRefPB* out) { + out->set_offset(ref.offset); + out->set_length(ref.length); +} + +void encode_common_grams(const CommonGramsSegmentMetadata& metadata, + doris::snii::SniiCommonGramsMetadataPB* out) { + out->set_plain_term_key_version(static_cast(metadata.plain_term_key_version)); + out->set_common_grams_coverage(static_cast(metadata.common_grams_coverage)); + out->set_common_grams_semantics_version(metadata.common_grams_semantics_version); + out->set_common_grams_key_version(metadata.common_grams_key_version); + out->set_common_grams_dictionary_identity(metadata.common_grams_dictionary_identity); + out->set_base_analyzer_fingerprint(metadata.base_analyzer_fingerprint); + out->set_common_grams_fingerprint(metadata.common_grams_fingerprint); + out->set_scoring_coverage(static_cast(metadata.scoring_coverage)); + out->set_scoring_stats_version(metadata.scoring_stats_version); + out->set_norm_semantics_version(metadata.norm_semantics_version); + out->set_scoring_doc_count(metadata.scoring_doc_count); + out->set_scoring_token_count(metadata.scoring_token_count); +} + +Status decode_region_ref(const doris::snii::SniiRegionRefPB& input, RegionRef* out) { + if (!input.has_offset() || !input.has_length()) { + return corrupted("core metadata: missing region reference field"); + } + *out = {.offset = input.offset(), .length = input.length()}; + return Status::OK(); +} + +Status decode_common_grams(const doris::snii::SniiCommonGramsMetadataPB& input, + CommonGramsSegmentMetadata* out) { + if (!input.has_plain_term_key_version() || !input.has_common_grams_coverage() || + !input.has_common_grams_semantics_version() || !input.has_common_grams_key_version() || + !input.has_common_grams_dictionary_identity() || !input.has_base_analyzer_fingerprint() || + !input.has_common_grams_fingerprint() || !input.has_scoring_coverage() || + !input.has_scoring_stats_version() || !input.has_norm_semantics_version() || + !input.has_scoring_doc_count() || !input.has_scoring_token_count()) { + return corrupted("core metadata: missing CommonGrams metadata field"); + } + RETURN_IF_ERROR(validate_plain_term_key_version(input.plain_term_key_version())); + RETURN_IF_ERROR(validate_common_grams_coverage(input.common_grams_coverage())); + RETURN_IF_ERROR(validate_scoring_coverage(input.scoring_coverage())); + *out = {.plain_term_key_version = + static_cast(input.plain_term_key_version()), + .common_grams_coverage = + static_cast(input.common_grams_coverage()), + .common_grams_semantics_version = input.common_grams_semantics_version(), + .common_grams_key_version = input.common_grams_key_version(), + .common_grams_dictionary_identity = input.common_grams_dictionary_identity(), + .base_analyzer_fingerprint = input.base_analyzer_fingerprint(), + .common_grams_fingerprint = input.common_grams_fingerprint(), + .scoring_coverage = static_cast(input.scoring_coverage()), + .scoring_stats_version = input.scoring_stats_version(), + .norm_semantics_version = input.norm_semantics_version(), + .scoring_doc_count = input.scoring_doc_count(), + .scoring_token_count = input.scoring_token_count()}; + return validate_common_grams_segment_metadata(*out); +} + +Status decode_core_pb(const doris::snii::SniiCoreMetadataPB& input, CoreMetadata* out) { + if (!input.has_index_config() || !input.has_stats() || !input.has_section_refs()) { + return corrupted("core metadata: missing required field"); + } + RETURN_IF_ERROR(validate_index_config(input.index_config(), &out->index_config)); + + const auto& stats = input.stats(); + if (!stats.has_doc_count() || !stats.has_indexed_doc_count() || !stats.has_term_count() || + !stats.has_sum_total_term_freq() || !stats.has_null_count()) { + return corrupted("core metadata: missing statistics field"); + } + out->stats = {.doc_count = stats.doc_count(), + .indexed_doc_count = stats.indexed_doc_count(), + .term_count = stats.term_count(), + .sum_total_term_freq = stats.sum_total_term_freq(), + .null_count = stats.null_count()}; + + const auto& refs = input.section_refs(); + if (!refs.has_dict_region() || !refs.has_posting_region() || !refs.has_norms() || + !refs.has_null_bitmap() || !refs.has_bsbf()) { + return corrupted("core metadata: missing section reference"); + } + RETURN_IF_ERROR(decode_region_ref(refs.dict_region(), &out->section_refs.dict_region)); + RETURN_IF_ERROR(decode_region_ref(refs.posting_region(), &out->section_refs.posting_region)); + RETURN_IF_ERROR(decode_region_ref(refs.norms(), &out->section_refs.norms)); + RETURN_IF_ERROR(decode_region_ref(refs.null_bitmap(), &out->section_refs.null_bitmap)); + RETURN_IF_ERROR(decode_region_ref(refs.bsbf(), &out->section_refs.bsbf)); + + if (input.has_common_grams()) { + CommonGramsSegmentMetadata common_grams; + RETURN_IF_ERROR(decode_common_grams(input.common_grams(), &common_grams)); + out->common_grams_metadata = std::move(common_grams); + } + + RETURN_IF_ERROR(validate_posting_policy(input.common_grams_posting_policy(), + &out->common_grams_posting_policy)); + if (out->common_grams_posting_policy == CommonGramsPostingPolicy::kHybridV1 && + (!out->common_grams_metadata.has_value() || + out->common_grams_metadata->common_grams_coverage != CommonGramsCoverage::kMixed)) { + return corrupted("core metadata: hybrid policy requires mixed CommonGrams metadata"); + } + const bool has_scoring_tier = out->index_config == IndexConfig::kDocsPositionsScoring; + if (has_scoring_tier) { + if (out->section_refs.norms.length == 0) { + return corrupted("core metadata: scoring index requires a norms region"); + } + } + if (has_scoring_tier || + (out->common_grams_metadata.has_value() && + out->common_grams_metadata->scoring_coverage == ScoringCoverage::kComplete)) { + RETURN_IF_ERROR(validate_snii_scoring_metadata( + out->common_grams_metadata ? &*out->common_grams_metadata : nullptr, + out->stats.doc_count, out->stats.sum_total_term_freq, has_scoring_tier, + has_positions(out->index_config), out->section_refs.norms.length != 0)); + } + return Status::OK(); +} + +} // namespace + +Status encode_core_metadata(const CoreMetadata& metadata, ByteSink* out) { + if (out == nullptr) { + return Status::Error("core metadata: null output"); + } + + doris::snii::SniiCoreMetadataPB core; + core.set_index_config(static_cast(metadata.index_config)); + auto* stats = core.mutable_stats(); + stats->set_doc_count(metadata.stats.doc_count); + stats->set_indexed_doc_count(metadata.stats.indexed_doc_count); + stats->set_term_count(metadata.stats.term_count); + stats->set_sum_total_term_freq(metadata.stats.sum_total_term_freq); + stats->set_null_count(metadata.stats.null_count); + auto* refs = core.mutable_section_refs(); + encode_region_ref(metadata.section_refs.dict_region, refs->mutable_dict_region()); + encode_region_ref(metadata.section_refs.posting_region, refs->mutable_posting_region()); + encode_region_ref(metadata.section_refs.norms, refs->mutable_norms()); + encode_region_ref(metadata.section_refs.null_bitmap, refs->mutable_null_bitmap()); + encode_region_ref(metadata.section_refs.bsbf, refs->mutable_bsbf()); + if (metadata.common_grams_metadata.has_value()) { + encode_common_grams(*metadata.common_grams_metadata, core.mutable_common_grams()); + } + if (metadata.common_grams_posting_policy != CommonGramsPostingPolicy::kNone) { + core.set_common_grams_posting_policy( + static_cast(metadata.common_grams_posting_policy)); + } + + CoreMetadata validated; + RETURN_IF_ERROR(decode_core_pb(core, &validated)); + const size_t size = core.ByteSizeLong(); + if (size > static_cast(std::numeric_limits::max())) { + return corrupted("core metadata: protobuf payload exceeds INT_MAX"); + } + std::string payload(size, '\0'); + if (!core.SerializeToArray(payload.data(), static_cast(size))) { + return corrupted("core metadata: protobuf serialization failed"); + } + SectionFramer::write(*out, static_cast(SectionType::kCoreMetadataPB), Slice(payload)); + return Status::OK(); +} + +Status decode_core_metadata(Slice framed_bytes, CoreMetadata* out) { + if (out == nullptr) { + return Status::Error("core metadata: null output"); + } + *out = {}; + ByteSource source(framed_bytes); + FramedSection section; + RETURN_IF_ERROR(SectionFramer::read(source, §ion)); + if (!source.eof() || section.type != static_cast(SectionType::kCoreMetadataPB)) { + return corrupted("core metadata: invalid frame"); + } + if (section.payload.size() > static_cast(std::numeric_limits::max())) { + return corrupted("core metadata: protobuf payload exceeds INT_MAX"); + } + doris::snii::SniiCoreMetadataPB core; + if (!core.ParseFromArray(section.payload.data(), static_cast(section.payload.size()))) { + return corrupted("core metadata: protobuf parsing failed"); + } + return decode_core_pb(core, out); +} + +} // namespace doris::snii::format diff --git a/be/src/storage/index/snii/format/core_metadata.h b/be/src/storage/index/snii/format/core_metadata.h new file mode 100644 index 00000000000000..ce761901a145ec --- /dev/null +++ b/be/src/storage/index/snii/format/core_metadata.h @@ -0,0 +1,62 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include "common/status.h" +#include "storage/index/inverted/common_grams/common_grams_segment_metadata.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/encoding/byte_sink.h" +#include "storage/index/snii/format/format_constants.h" +#include "storage/index/snii/format/stats_block.h" + +namespace doris::snii::format { + +struct RegionRef { + uint64_t offset = 0; + uint64_t length = 0; +}; + +struct SectionRefs { + RegionRef dict_region; + RegionRef posting_region; + RegionRef norms; + RegionRef null_bitmap; + RegionRef bsbf; +}; + +enum class CommonGramsPostingPolicy : uint8_t { + kNone = 0, + kDocsOnlyV1 = 1, + kHybridV1 = kDocsOnlyV1, +}; + +struct CoreMetadata { + IndexConfig index_config = IndexConfig::kDocsOnly; + StatsBlock stats; + SectionRefs section_refs; + std::optional common_grams_metadata; + CommonGramsPostingPolicy common_grams_posting_policy = CommonGramsPostingPolicy::kNone; +}; + +Status encode_core_metadata(const CoreMetadata& metadata, ByteSink* out); +Status decode_core_metadata(Slice framed_bytes, CoreMetadata* out); + +} // namespace doris::snii::format diff --git a/be/src/storage/index/snii/format/dict_block.cpp b/be/src/storage/index/snii/format/dict_block.cpp new file mode 100644 index 00000000000000..56416924f8abd0 --- /dev/null +++ b/be/src/storage/index/snii/format/dict_block.cpp @@ -0,0 +1,604 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/format/dict_block.h" + +#include +#include + +#include "storage/index/snii/encoding/byte_source.h" +#include "storage/index/snii/encoding/crc32c.h" +#include "storage/index/snii/encoding/varint.h" +#include "storage/index/snii/format/sampled_term_index.h" // std_string_heap_bytes + +namespace doris::snii::format { + +namespace { + +constexpr size_t kFooterBytes = sizeof(uint32_t); // trailing crc32c +constexpr size_t kNAnchorsBytes = sizeof(uint32_t); // n_anchors u32 +constexpr size_t kAnchorOffBytes = sizeof(uint32_t); // per-anchor offset u32 + +size_t estimate_statless_entry_upper_bound(const DictEntry& e, IndexTier tier) { + size_t body = 0; + body += varint_len(static_cast(e.term.size())); + body += varint_len(static_cast(e.term.size())); + body += e.term.size(); + body += 1; // flags + body += varint_len(e.df); + + const bool tier_has_stats = tier >= IndexTier::kT2; + if (e.kind == DictEntryKind::kInline) { + body += varint_len(static_cast(e.frq_bytes.size())) + e.frq_bytes.size(); + body += varint_len(e.inline_dd_disk_len); + body += 1 + varint_len(e.dd_meta.uncomp_len); // win_mode + DD metadata + if (tier_has_stats) { + body += varint_len(e.freq_meta.uncomp_len); + body += varint_len(static_cast(e.prx_bytes.size())) + e.prx_bytes.size(); + } + } else { + body += varint_len(e.frq_off_delta) + varint_len(e.frq_len); + if (e.enc == DictEntryEnc::kWindowed) { + body += varint_len(e.prelude_len) + varint_len(e.frq_docs_len); + } else { + body += varint_len(e.frq_docs_len); + body += 1 + varint_len(e.dd_meta.uncomp_len) + sizeof(uint32_t); + if (tier_has_stats) { + body += varint_len(e.freq_meta.uncomp_len) + sizeof(uint32_t); + } + } + if (tier_has_stats) { + body += varint_len(e.prx_off_delta) + varint_len(e.prx_len); + } + } + return varint_len(static_cast(body)) + body; +} + +// Estimate the encoded upper-bound byte size of one entry (no actual encoding; used by +// estimated_bytes). Take the maximum varint width of each variable-length field plus payload bytes +// to guarantee an upper bound. +size_t estimate_entry_bytes(const DictEntry& e, IndexTier tier, bool term_stats) { + size_t body = 0; + body += varint_len(static_cast(e.term.size())); // prefix_len upper bound + body += varint_len(static_cast(e.term.size())); // suffix_len upper bound + body += e.term.size(); // suffix bytes upper bound + body += 1; // flags + body += 10; // df upper bound + if (term_stats) { + body += 10; // ttf_delta + body += 10; // max_freq + } + if (e.kind == DictEntryKind::kInline) { + body += 10 + e.frq_bytes.size(); + body += 10 + e.prx_bytes.size(); + } else { + body += 10 * 5; // frq_off/frq_len/prelude/prx_off/prx_len upper bound + } + const size_t legacy_estimate = varint_len(static_cast(body)) + body; + if (term_stats) { + return legacy_estimate; + } + return std::max(legacy_estimate, estimate_statless_entry_upper_bound(e, tier)); +} + +} // namespace + +// ---- DictBlockBuilder ---- + +DictBlockBuilder::DictBlockBuilder(IndexTier tier, bool has_positions, uint64_t frq_base, + uint64_t prx_base, uint32_t anchor_interval, bool term_stats) + : tier_(tier), + has_positions_(has_positions), + term_stats_(term_stats), + frq_base_(frq_base), + prx_base_(prx_base), + anchor_interval_(anchor_interval == 0 ? 1 : anchor_interval) {} + +void DictBlockBuilder::add_entry(const DictEntry& entry) { + if (is_anchor(n_entries_)) { + ++n_anchors_; + } + entries_est_ += estimate_entry_bytes(entry, tier_, term_stats_); + entries_.push_back(entry); + ++n_entries_; +} + +void DictBlockBuilder::add_entry(DictEntry&& entry) { + if (is_anchor(n_entries_)) { + ++n_anchors_; + } + // estimate_entry_bytes reads `entry`, so it MUST run before the move below: + // sizing a moved-from (empty) entry would undercount entries_est_ and split + // blocks incorrectly. finish() output is unaffected either way -- it depends + // only on the entries actually queued, not on how they were appended. + entries_est_ += estimate_entry_bytes(entry, tier_, term_stats_); + entries_.push_back(std::move(entry)); + ++n_entries_; +} + +size_t DictBlockBuilder::estimated_bytes() const { + size_t header = varint_len(static_cast(n_entries_)) + 2; // +ver +flags + header += varint_len(frq_base_); + if (has_positions_) { + header += varint_len(prx_base_); + } + const size_t anchors = n_anchors_ * kAnchorOffBytes + kNAnchorsBytes; + return header + entries_est_ + anchors + kFooterBytes; +} + +void DictBlockBuilder::encode_covered(ByteSink* sink) const { + // header. + sink->put_varint64(static_cast(n_entries_)); + sink->put_u8(kDictBlockFormatVer); + sink->put_u8(static_cast((has_positions_ ? dict_block_flags::kHasPositions : 0U) | + (term_stats_ ? 0U : dict_block_flags::kNoTermStats))); + sink->put_varint64(frq_base_); + if (has_positions_) { + sink->put_varint64(prx_base_); + } + + // entries: anchor entries use prev_term="" and record their byte offset within the block. + std::vector anchor_offsets; + anchor_offsets.reserve(n_anchors_); + ByteSink entry_body_scratch; + std::string_view prev; + for (uint32_t i = 0; i < n_entries_; ++i) { + const bool anchor = is_anchor(i); + if (anchor) { + anchor_offsets.push_back(static_cast(sink->size())); + } + const std::string_view prev_term = anchor ? std::string_view {} : prev; + // finish() is void and entry encoding into an in-memory ByteSink cannot fail; + // explicitly discard the (now [[nodiscard]] Status) return. + static_cast(encode_dict_entry(entries_[i], prev_term, tier_, sink, term_stats_, + &entry_body_scratch)); + prev = entries_[i].term; + } + + // anchor_offsets[] + n_anchors. + for (uint32_t off : anchor_offsets) { + sink->put_fixed32(off); + } + sink->put_fixed32(static_cast(anchor_offsets.size())); +} + +void DictBlockBuilder::finish(ByteSink* sink) const { + ByteSink body; // header + entries + anchor_offsets + n_anchors (crc covered region) + encode_covered(&body); + + // Write the entire block (including crc footer) to sink. + sink->put_bytes(body.view()); + sink->put_fixed32(crc32c(body.view())); +} + +std::vector DictBlockBuilder::finish_owned() const { + ByteSink block; + encode_covered(&block); + const uint32_t checksum = crc32c(block.view()); + block.reserve(kFooterBytes); + block.put_fixed32(checksum); + return block.take(); +} + +// ---- DictBlockReader ---- + +namespace { + +// Verify the block length is sufficient and validate the trailing crc; return a Slice of the covered region (excluding crc footer). +Status verify_crc(Slice block, Slice* covered) { + if (block.size() < kFooterBytes + kNAnchorsBytes) { + return Status::Error( + "dict_block: block too short to contain footer"); + } + const size_t covered_len = block.size() - kFooterBytes; + *covered = block.subslice(0, covered_len); + + ByteSource crc_src(block.subslice(covered_len, kFooterBytes)); + uint32_t stored = 0; + RETURN_IF_ERROR(crc_src.get_fixed32(&stored)); + if (crc32c(*covered) != stored) { + return Status::Error( + "dict_block: crc32c checksum mismatch"); + } + return Status::OK(); +} + +// Read and verify that block_flags is consistent with has_positions. +Status check_flags(uint8_t flags, bool has_positions) { + const bool flag_pos = (flags & dict_block_flags::kHasPositions) != 0; + if (flag_pos != has_positions) { + return Status::Error( + "dict_block: has_positions inconsistent with block_flags"); + } + return Status::OK(); +} + +} // namespace + +Status DictBlockReader::open(Slice block, IndexTier tier, bool has_positions, + DictBlockReader* out) { + if (out == nullptr) { + return Status::Error("dict_block: out is null"); + } + *out = DictBlockReader {}; + + // Decode instrumentation seam: one increment per block materialization (the + // CRC verify + anchor parse below, preceded by a zstd decompress for a + // compressed block). A dict-block cache eliminates repeats of exactly this. + testing::note_dict_block_decode(); + + Slice covered; + RETURN_IF_ERROR(verify_crc(block, &covered)); + out->block_ = covered; + out->tier_ = tier; + out->has_positions_ = has_positions; + + // header. + ByteSource src(covered); + uint64_t n_entries = 0; + RETURN_IF_ERROR(src.get_varint64(&n_entries)); + uint8_t ver = 0; + uint8_t flags = 0; + RETURN_IF_ERROR(src.get_u8(&ver)); + RETURN_IF_ERROR(src.get_u8(&flags)); + if (ver != kDictBlockFormatVer) { + return Status::Error( + "dict_block: unsupported entry_format_ver"); + } + RETURN_IF_ERROR(check_flags(flags, has_positions)); + out->term_stats_ = (flags & dict_block_flags::kNoTermStats) == 0; + RETURN_IF_ERROR(src.get_varint64(&out->frq_base_)); + if (has_positions) { + RETURN_IF_ERROR(src.get_varint64(&out->prx_base_)); + } + + out->n_entries_ = static_cast(n_entries); + out->entries_begin_ = src.position(); + + // The anchor table is at the tail of covered: [... anchor_offsets[n] n_anchors(u32)]. + if (covered.size() < kNAnchorsBytes) { + return Status::Error( + "dict_block: missing n_anchors"); + } + ByteSource na_src(covered.subslice(covered.size() - kNAnchorsBytes, kNAnchorsBytes)); + uint32_t n_anchors = 0; + RETURN_IF_ERROR(na_src.get_fixed32(&n_anchors)); + + const size_t anchor_table_bytes = static_cast(n_anchors) * kAnchorOffBytes; + if (covered.size() < kNAnchorsBytes + anchor_table_bytes || + out->entries_begin_ + anchor_table_bytes + kNAnchorsBytes > covered.size()) { + return Status::Error( + "dict_block: anchor table out of range"); + } + const size_t anchor_table_begin = covered.size() - kNAnchorsBytes - anchor_table_bytes; + + ByteSource at_src(covered.subslice(anchor_table_begin, anchor_table_bytes)); + out->anchor_offsets_.resize(n_anchors); + out->anchor_terms_.resize(n_anchors); + for (uint32_t i = 0; i < n_anchors; ++i) { + uint32_t off = 0; + RETURN_IF_ERROR(at_src.get_fixed32(&off)); + if (off >= anchor_table_begin) { + return Status::Error( + "dict_block: anchor offset out of range"); + } + // Anchor offsets must be strictly monotonically increasing, and the first anchor must be exactly the start of the entries region (entry 0 is always an anchor). + // Otherwise scan_from_anchor's segment-length computation seg_end-seg_begin would underflow as size_t and cause an out-of-range read, + // guarding against non-monotonic offset tables with a re-stamped crc (remote on-demand read / cache misalignment scenarios). + if (i == 0) { + if (off != out->entries_begin_) { + return Status::Error( + "dict_block: first anchor offset is not the start of entries"); + } + } else if (off <= out->anchor_offsets_[i - 1]) { + return Status::Error( + "dict_block: anchor offsets are not strictly increasing"); + } + out->anchor_offsets_[i] = off; + // Anchor entries are encoded with prev_term="" and can be decoded independently to retrieve their term. + ByteSource e_src(covered.subslice(off, anchor_table_begin - off)); + DictEntry probe; + RETURN_IF_ERROR(decode_dict_entry(&e_src, std::string_view {}, tier, &probe, + (flags & dict_block_flags::kNoTermStats) == 0)); + out->anchor_terms_[i] = std::move(probe.term); + } + return Status::OK(); +} + +size_t DictBlockReader::heap_bytes() const { + size_t bytes = anchor_offsets_.capacity() * sizeof(uint32_t) + + anchor_terms_.capacity() * sizeof(std::string); + for (const auto& term : anchor_terms_) { + bytes += std_string_heap_bytes(term); + } + return bytes; +} + +bool DictBlockReader::locate_anchor(std::string_view target, size_t* anchor_idx) const { + if (anchor_terms_.empty()) { + return false; + } + if (target < std::string_view(anchor_terms_.front())) { + return false; + } + // The last anchor_term <= target. + size_t lo = 0; + size_t hi = anchor_terms_.size(); // open interval + while (lo + 1 < hi) { + const size_t mid = lo + (hi - lo) / 2; + if (std::string_view(anchor_terms_[mid]) <= target) { + lo = mid; + } else { + hi = mid; + } + } + *anchor_idx = lo; + return true; +} + +Status DictBlockReader::decode_all(std::vector* out) const { + if (out == nullptr) { + return Status::Error("dict_block: out is null"); + } + out->clear(); + out->reserve(n_entries_); + for (size_t a = 0; a < anchor_offsets_.size(); ++a) { + const size_t seg_begin = anchor_offsets_[a]; + const bool is_last = a + 1 == anchor_offsets_.size(); + const size_t seg_end = is_last ? (block_.size() - kNAnchorsBytes - + anchor_offsets_.size() * kAnchorOffBytes) + : anchor_offsets_[a + 1]; + if (seg_end < seg_begin || seg_end > block_.size()) { + return Status::Error( + "dict_block: anchor segment range invalid"); + } + ByteSource src(block_.subslice(seg_begin, seg_end - seg_begin)); + std::string prev; // first entry of a segment is an anchor (prev_term="") + while (!src.eof()) { + DictEntry e; + RETURN_IF_ERROR( + decode_dict_entry(&src, std::string_view(prev), tier_, &e, term_stats_)); + prev = e.term; + out->push_back(std::move(e)); + } + } + if (out->size() != n_entries_) { + return Status::Error( + "dict_block: decoded entry count mismatch"); + } + return Status::OK(); +} + +Status DictBlockReader::scan_from_anchor(size_t anchor_idx, std::string_view target, bool* found, + DictEntry* out) const { + // Byte range of this anchor segment: [anchor_offset, next anchor offset or anchor table start). + const size_t seg_begin = anchor_offsets_[anchor_idx]; + const bool is_last = anchor_idx + 1 == anchor_offsets_.size(); + const size_t seg_end = + is_last ? (block_.size() - kNAnchorsBytes - anchor_offsets_.size() * kAnchorOffBytes) + : anchor_offsets_[anchor_idx + 1]; + + // Fallback: open() has already verified anchor monotonicity; this additionally guards against seg_end block_.size()) { + return Status::Error( + "dict_block: anchor segment range invalid"); + } + ByteSource src(block_.subslice(seg_begin, seg_end - seg_begin)); + std::string prev; // the first entry in the segment is an anchor, prev_term="" + while (!src.eof()) { + // Key-first: decode only the (front-coded) term key, then decide whether + // the body is worth materializing. Non-matching entries skip their body + // entirely -- with anchor_interval=16 that turns ~16 body decodes per + // lookup into 1 (the matched entry) or 0 (a miss). + DictEntry e; + size_t body_start = 0; + uint64_t entry_total = 0; + RETURN_IF_ERROR( + decode_dict_entry_key(&src, std::string_view(prev), &e, &body_start, &entry_total)); + if (e.term == target) { + RETURN_IF_ERROR( + decode_dict_entry_rest(&src, tier_, body_start, entry_total, &e, term_stats_)); + *found = true; + *out = std::move(e); + return Status::OK(); + } + if (std::string_view(e.term) > target) { + *found = false; // already past target; entries are sorted so it does not exist + return Status::OK(); + } + // Before target: skip the body but keep the key as the front-coding base. + RETURN_IF_ERROR(skip_dict_entry_body(&src, body_start, entry_total)); + prev = std::move(e.term); + } + *found = false; + return Status::OK(); +} + +Status DictBlockReader::find_term(std::string_view target, bool* found, DictEntry* out) const { + if (found == nullptr || out == nullptr) { + return Status::Error("dict_block: found / out is null"); + } + *found = false; + size_t anchor_idx = 0; + if (!locate_anchor(target, &anchor_idx)) { + return Status::OK(); + } + return scan_from_anchor(anchor_idx, target, found, out); +} + +Status DictBlockReader::visit_prefix_range(std::string_view prefix, + const std::function& accept_key, + const std::function& on_hit, + bool* prefix_exhausted) const { + if (!on_hit || prefix_exhausted == nullptr) { + return Status::Error( + "dict_block: null visit_prefix_range args"); + } + *prefix_exhausted = false; + if (anchor_offsets_.empty()) { + return Status::OK(); // empty block: nothing to enumerate + } + + // Anchor-jump: start at the anchor segment that may contain prefix. Earlier + // segments hold only terms < prefix (every term is < the next anchor term + // <= prefix), so they are skipped without any decode. An empty prefix or one + // sorting before the first anchor starts at anchor 0. + size_t anchor_idx = 0; + if (!prefix.empty()) { + locate_anchor(prefix, &anchor_idx); // false leaves anchor_idx at 0 + } + + // Scan from the chosen anchor to the end of the entries region: the prefix + // range may span several anchor segments. Anchor entries are encoded with + // prefix_len=0, so a single running `prev` reconstructs every term correctly + // even as the scan crosses segment boundaries. + const size_t seg_begin = anchor_offsets_[anchor_idx]; + const size_t entries_end = + block_.size() - kNAnchorsBytes - anchor_offsets_.size() * kAnchorOffBytes; + if (entries_end < seg_begin || entries_end > block_.size()) { + return Status::Error( + "dict_block: entries region range invalid"); + } + + ByteSource src(block_.subslice(seg_begin, entries_end - seg_begin)); + std::string prev; // the chosen anchor segment starts at an anchor (prev="") + while (!src.eof()) { + DictEntry e; + size_t body_start = 0; + uint64_t entry_total = 0; + RETURN_IF_ERROR( + decode_dict_entry_key(&src, std::string_view(prev), &e, &body_start, &entry_total)); + const std::string_view t(e.term); + if (t < prefix) { + // Still before the range (only reachable inside the anchor segment + // that straddles prefix): skip the body, keep the front-coding base. + RETURN_IF_ERROR(skip_dict_entry_body(&src, body_start, entry_total)); + prev = std::move(e.term); + continue; + } + const bool has_prefix = t.size() >= prefix.size() && t.starts_with(prefix); + if (!has_prefix) { + *prefix_exhausted = true; // sorted: no further matches here or later + return Status::OK(); + } + if (accept_key && !accept_key(t)) { + // Key-only rejection: never pay for this entry's body. + RETURN_IF_ERROR(skip_dict_entry_body(&src, body_start, entry_total)); + prev = std::move(e.term); + continue; + } + // Accepted: materialize the body and hand the entry to the visitor. + RETURN_IF_ERROR( + decode_dict_entry_rest(&src, tier_, body_start, entry_total, &e, term_stats_)); + prev = e.term; // copy the key before the entry is moved into on_hit + bool stop = false; + RETURN_IF_ERROR(on_hit(std::move(e), &stop)); + if (stop) { + return Status::OK(); + } + } + return Status::OK(); +} + +Status DictBlockReader::visit_term_range(std::string_view lower_inclusive, + std::optional upper_exclusive, + const std::function& accept_key, + const std::function& on_hit, + bool* range_exhausted) const { + if (!on_hit || range_exhausted == nullptr) { + return Status::Error( + "dict_block: null visit_term_range args"); + } + *range_exhausted = false; + if (upper_exclusive.has_value() && *upper_exclusive <= lower_inclusive) { + *range_exhausted = true; + return Status::OK(); + } + if (anchor_offsets_.empty()) { + return Status::OK(); + } + + size_t anchor_idx = 0; + if (!lower_inclusive.empty()) { + locate_anchor(lower_inclusive, &anchor_idx); + } + const size_t seg_begin = anchor_offsets_[anchor_idx]; + const size_t entries_end = + block_.size() - kNAnchorsBytes - anchor_offsets_.size() * kAnchorOffBytes; + if (entries_end < seg_begin || entries_end > block_.size()) { + return Status::Error( + "dict_block: entries region range invalid"); + } + + ByteSource src(block_.subslice(seg_begin, entries_end - seg_begin)); + std::string prev; + while (!src.eof()) { + DictEntry entry; + size_t body_start = 0; + uint64_t entry_total = 0; + RETURN_IF_ERROR(decode_dict_entry_key(&src, std::string_view(prev), &entry, &body_start, + &entry_total)); + const std::string_view term(entry.term); + if (term < lower_inclusive) { + RETURN_IF_ERROR(skip_dict_entry_body(&src, body_start, entry_total)); + prev = std::move(entry.term); + continue; + } + if (upper_exclusive.has_value() && term >= *upper_exclusive) { + *range_exhausted = true; + return Status::OK(); + } + if (accept_key && !accept_key(term)) { + RETURN_IF_ERROR(skip_dict_entry_body(&src, body_start, entry_total)); + prev = std::move(entry.term); + continue; + } + RETURN_IF_ERROR( + decode_dict_entry_rest(&src, tier_, body_start, entry_total, &entry, term_stats_)); + prev = entry.term; + bool stop = false; + RETURN_IF_ERROR(on_hit(std::move(entry), &stop)); + if (stop) { + return Status::OK(); + } + } + return Status::OK(); +} + +} // namespace doris::snii::format + +namespace doris::snii::testing { +namespace { +std::atomic& dict_decode_atomic() { + static std::atomic counter {0}; + return counter; +} +} // namespace + +uint64_t dict_decode_counter() { + return dict_decode_atomic().load(std::memory_order_relaxed); +} + +void reset_dict_decode_counter() { + dict_decode_atomic().store(0, std::memory_order_relaxed); +} + +void note_dict_block_decode() { + dict_decode_atomic().fetch_add(1, std::memory_order_relaxed); +} + +} // namespace doris::snii::testing diff --git a/be/src/storage/index/snii/format/dict_block.h b/be/src/storage/index/snii/format/dict_block.h new file mode 100644 index 00000000000000..7a299a1f162f8e --- /dev/null +++ b/be/src/storage/index/snii/format/dict_block.h @@ -0,0 +1,232 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/encoding/byte_sink.h" +#include "storage/index/snii/format/dict_entry.h" +#include "storage/index/snii/format/format_constants.h" + +// DICT block —— a positioning unit mapping term → postings read plan, and also +// the unit for remote on-demand fetching, caching, and CRC checksum +// verification (see docs/design/SNII-design-spec.source.md "DICT block" and +// "dict lookup flow summary" sections). +// +// Byte layout (strictly implemented; multi-byte fixed-width fields are +// little-endian, variable-length integers use LEB128): +// header: +// n_entries varint +// entry_format_ver u8 # = kDictBlockFormatVer +// block_flags u8 # bit0 = has_positions (consistency check +// against the value passed to reader) frq_base varint64 prx_base +// varint64 # present only when has_positions is set +// entries[n_entries] # variable-length DictEntry, front-coded in +// lexicographic order anchor_offsets[n_anchors] # u32 * n_anchors, byte +// offset of each anchor entry within the block n_anchors u32 crc32c +// u32 # covers [header .. n_anchors], detects corruption (sole CRC +// layer) +// +// Anchor rule: every anchor_interval entries, one "term anchor" is forced — +// that entry is encoded with prev_term="" (prefix_len=0, storing the full +// term), and its byte offset is recorded in anchor_offsets; non-anchor entries +// use the preceding entry's term as prev_term for front coding. The reader can +// start from any anchor and scan independently without needing earlier terms, +// enabling anchor binary search + local scan for exact term lookup. +namespace doris::snii::format { + +// DICT block entry_format_ver: self-describing version of the DictEntry +// encoding. Reader rejects a mismatch so a query-only run cannot silently read +// an older dict-entry layout as the current one. +inline constexpr uint8_t kDictBlockFormatVer = 2; + +// block_flags bit definitions. +namespace dict_block_flags { +inline constexpr uint8_t kHasPositions = 1U << 0; // whether to write prx_base / .prx fields +// G16-f: entries omit the ttf_delta/max_freq varints (freq-dropped index -- +// the stats serve only BM25 scoring). Self-describing per block; absent on +// pre-G16-f blocks, whose entries always carry the stats on tier>=T2. +inline constexpr uint8_t kNoTermStats = 1U << 1; +// bit1-7 reserved +} // namespace dict_block_flags + +// DICT block writer: entries are added in lexicographic order via add_entry; +// internally determines anchors and accumulates size estimates, and on finish +// serializes header + entries + anchor table + CRC in one pass. The front-coding +// base is rebuilt from a local prev inside finish(), so no prev_term is retained +// as builder state. +class DictBlockBuilder { +public: + DictBlockBuilder(IndexTier tier, bool has_positions, uint64_t frq_base, uint64_t prx_base, + uint32_t anchor_interval = 16, bool term_stats = true); + + // Append one entry (caller must guarantee lexicographic term order). + // Internally decides whether it becomes an anchor. The copy overload is kept + // for callers that must retain their entry afterwards (materialized fallback, + // tests); the move overload avoids the per-term DictEntry copy -- which for an + // inline entry is two std::vector heap allocations plus the term + // copy -- on the SPIMI build path. + void add_entry(const DictEntry& entry); + void add_entry(DictEntry&& entry); + + // Upper-bound estimate of the serialized size of the current block (including + // header + entries + anchor table + CRC footer), used by the upper layer to + // decide when to cut a new block based on target_dict_block_bytes. + size_t estimated_bytes() const; + + // Number of entries. + uint32_t n_entries() const { return n_entries_; } + + // Serialize the entire block and append it to sink. + void finish(ByteSink* sink) const; + + // Serialize the entire block into an owned buffer. This avoids copying the + // CRC-covered bytes when the caller needs ownership of the complete block. + std::vector finish_owned() const; + +private: + bool is_anchor(uint32_t index) const { return index % anchor_interval_ == 0; } + void encode_covered(ByteSink* sink) const; + + IndexTier tier_; + bool has_positions_; + bool term_stats_ = true; // false: entries omit ttf/max_freq (kNoTermStats) + uint64_t frq_base_; + uint64_t prx_base_; + uint32_t anchor_interval_; + + uint32_t n_entries_ = 0; + std::vector entries_; + size_t entries_est_ = 0; // accumulated byte estimate for the entries section + size_t n_anchors_ = 0; // number of anchors +}; + +// DICT block reader: on open, verifies the CRC and parses the header / anchor +// table; find_term uses anchor binary search + local scan to locate a +// DictEntry. Holds a byte view of the block (non-owning); lifetime is managed +// by the caller. +class DictBlockReader { +public: + DictBlockReader() = default; + + // Parse and verify the entire block. CRC mismatch / truncation / invalid + // structure → Corruption; has_positions in the header inconsistent with the + // supplied argument → InvalidArgument. + static Status open(Slice block, IndexTier tier, bool has_positions, DictBlockReader* out); + + // Anchor binary search + local scan to locate target. Hit → *found=true and + // *out is filled; miss (including out-of-range, gap) → *found=false. + // Structural error → non-OK Status. + Status find_term(std::string_view target, bool* found, DictEntry* out) const; + + // Decodes EVERY entry in the block in lexicographic order into *out (each a + // self-contained DictEntry, owning its term). Used for ordered term + // enumeration (prefix / range scans). Resets the front-coding base at each + // anchor segment. Retained as the golden reference; the prefix path now + // streams via visit_prefix_range. + Status decode_all(std::vector* out) const; + + // Streams the entries of this block whose term lies in [prefix, prefix+) in + // lexicographic order, materializing only the bodies a caller keeps (T07): + // 1) anchor-jump to the anchor segment containing prefix (segments whose + // terms are all < prefix are skipped without any decode); an empty + // prefix or one before the first anchor starts at anchor 0; + // 2) within range, decode each entry's term key only; term < prefix skips + // the body and continues; a term that leaves the prefix range sets + // *prefix_exhausted=true and ends the scan (sorted order guarantees no + // further matches here or in later blocks); + // 3) accept_key(term)==false skips the body (lets callers push a key-only + // predicate down so a non-match never pays for its body); + // 4) only accepted entries have their body decoded and are handed to + // on_hit, which may request an early stop via *stop. + // accept_key may be empty (treated as accept-all). prefix_exhausted is set + // false on entry and true only when a term past the range is seen. + Status visit_prefix_range(std::string_view prefix, + const std::function& accept_key, + const std::function& on_hit, + bool* prefix_exhausted) const; + + // Streams entries in [lower_inclusive, upper_exclusive). A missing upper + // bound scans to the end of the block. Like visit_prefix_range, this starts + // from the nearest anchor and only decodes accepted entry bodies. + Status visit_term_range(std::string_view lower_inclusive, + std::optional upper_exclusive, + const std::function& accept_key, + const std::function& on_hit, + bool* range_exhausted) const; + + uint64_t frq_base() const { return frq_base_; } + uint64_t prx_base() const { return prx_base_; } + uint32_t n_entries() const { return n_entries_; } + + // Resident heap held beyond sizeof(*this): the anchor_offsets_ / anchor_terms_ + // vector buffers plus each non-SSO anchor term's heap allocation. block_ is a + // NON-owning view and is deliberately NOT counted here -- its owning buffer is + // charged by the caller (the resident block's `bytes` vector, or a + // request-scoped decoded block). Summed into + // LogicalIndexReader::memory_usage() per resident block. + size_t heap_bytes() const; + +private: + // Sequentially scan from anchor anchor_idx to the end of that anchor segment, + // searching for target. + Status scan_from_anchor(size_t anchor_idx, std::string_view target, bool* found, + DictEntry* out) const; + + // Find the last anchor index where first_term(anchor) <= target; return false + // if none exists. + bool locate_anchor(std::string_view target, size_t* anchor_idx) const; + + Slice block_; // [header .. crc) full block view + IndexTier tier_ = IndexTier::kT1; + bool has_positions_ = false; + bool term_stats_ = true; // from block flags (kNoTermStats absent => true) + uint64_t frq_base_ = 0; + uint64_t prx_base_ = 0; + uint32_t n_entries_ = 0; + + size_t entries_begin_ = 0; // absolute offset of the start of the entries section + std::vector anchor_offsets_; // byte offset within the block for each anchor entry + std::vector + anchor_terms_; // full term of each anchor entry (used for binary search) +}; + +} // namespace doris::snii::format + +// Test-only instrumentation seam. dict_decode_counter() returns a process-global +// count of DICT block decodes performed by DictBlockReader::open -- i.e. the +// optional zstd decompress + CRC verify + anchor parse that turns on-disk block +// bytes into a usable reader. This is precisely the unit a dict-block cache +// eliminates on repeat, so tests assert dict_decode_counter() == unique_blocks. +// In production DICT blocks are zstd-compressed, so this equals the zstd +// decompress count. Counters use relaxed atomics; reset between tests. +namespace doris::snii::testing { + +uint64_t dict_decode_counter(); +void reset_dict_decode_counter(); +void note_dict_block_decode(); + +} // namespace doris::snii::testing diff --git a/be/src/storage/index/snii/format/dict_block_directory.cpp b/be/src/storage/index/snii/format/dict_block_directory.cpp new file mode 100644 index 00000000000000..8801418f8a3c6e --- /dev/null +++ b/be/src/storage/index/snii/format/dict_block_directory.cpp @@ -0,0 +1,131 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/format/dict_block_directory.h" + +#include + +#include "storage/index/snii/encoding/byte_source.h" +#include "storage/index/snii/encoding/section_framer.h" +#include "storage/index/snii/format/format_constants.h" + +namespace doris::snii::format { + +namespace { + +// Each block_ref has a fixed field order; reuse ByteSink varint/fixed primitives — do not hand-craft bytes manually. +// uncomp_len trails only when the kZstd flag is set, so uncompressed-block +// directories keep their compact (v1-identical) per-ref byte layout. +void encode_ref(const BlockRef& ref, ByteSink* payload) { + payload->put_varint64(ref.offset); + payload->put_varint64(ref.length); + payload->put_varint32(ref.n_entries); + payload->put_u8(ref.flags); + payload->put_fixed32(ref.checksum); + if (ref.flags & block_ref_flags::kZstd) payload->put_varint64(ref.uncomp_len); +} + +Status decode_ref(ByteSource* ps, BlockRef* ref) { + RETURN_IF_ERROR(ps->get_varint64(&ref->offset)); + RETURN_IF_ERROR(ps->get_varint64(&ref->length)); + RETURN_IF_ERROR(ps->get_varint32(&ref->n_entries)); + RETURN_IF_ERROR(ps->get_u8(&ref->flags)); + RETURN_IF_ERROR(ps->get_fixed32(&ref->checksum)); + if (ref->flags & block_ref_flags::kZstd) { + RETURN_IF_ERROR(ps->get_varint64(&ref->uncomp_len)); + } + return Status::OK(); +} + +Status decode_payload(Slice payload, std::vector* refs) { + ByteSource ps(payload); + uint32_t n_blocks = 0; + RETURN_IF_ERROR(ps.get_varint32(&n_blocks)); + // Guard against a corrupted, inflated count from untrusted bytes: each BlockRef + // needs >= 8 bytes (flags u8 + checksum u32 + >= 1 byte for each of 3 varints), + // so cap before reserve to avoid a huge allocation. + constexpr size_t kMinRefBytes = 8; + if (n_blocks > ps.remaining() / kMinRefBytes) { + return Status::Error( + "dict_block_directory: n_blocks exceeds payload capacity"); + } + refs->clear(); + refs->reserve(n_blocks); + uint64_t previous_end = 0; + for (uint32_t i = 0; i < n_blocks; ++i) { + BlockRef ref {}; + RETURN_IF_ERROR(decode_ref(&ps, &ref)); + if (ref.length == 0) { + return Status::Error( + "dict_block_directory: zero-length block"); + } + if (ref.length > std::numeric_limits::max() - ref.offset) { + return Status::Error( + "dict_block_directory: block range end overflow"); + } + const uint64_t block_end = ref.offset + ref.length; + if (i != 0 && ref.offset < previous_end) { + return Status::Error( + "dict_block_directory: blocks overlap or are out of physical order"); + } + previous_end = block_end; + refs->push_back(ref); + } + if (!ps.eof()) { + return Status::Error( + "dict_block_directory: trailing bytes in payload"); + } + return Status::OK(); +} + +} // namespace + +void DictBlockDirectoryBuilder::finish(ByteSink* sink) const { + ByteSink payload; + payload.put_varint32(static_cast(refs_.size())); + for (const auto& ref : refs_) { + encode_ref(ref, &payload); + } + SectionFramer::write(*sink, static_cast(SectionType::kDictBlockDirectory), + payload.view()); +} + +Status DictBlockDirectoryReader::open(Slice section, DictBlockDirectoryReader* out) { + ByteSource src(section); + FramedSection sec; + RETURN_IF_ERROR(SectionFramer::read(src, &sec)); + if (!src.eof()) { + return Status::Error( + "dict_block_directory: trailing framed section bytes"); + } + if (sec.type != static_cast(SectionType::kDictBlockDirectory)) { + return Status::Error( + "dict_block_directory: unexpected section type"); + } + return decode_payload(sec.payload, &out->refs_); +} + +Status DictBlockDirectoryReader::get(uint32_t ordinal, BlockRef* out) const { + if (ordinal >= refs_.size()) { + return Status::Error( + "dict_block_directory: ordinal out of range"); + } + *out = refs_[ordinal]; + return Status::OK(); +} + +} // namespace doris::snii::format diff --git a/be/src/storage/index/snii/format/dict_block_directory.h b/be/src/storage/index/snii/format/dict_block_directory.h new file mode 100644 index 00000000000000..909b32f29d1374 --- /dev/null +++ b/be/src/storage/index/snii/format/dict_block_directory.h @@ -0,0 +1,95 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/encoding/byte_sink.h" + +namespace doris::snii::format { + +// BlockRef.flags bit definitions. +namespace block_ref_flags { +// bit0: the on-disk block bytes are zstd(uncompressed_block). When set, the +// directory also stores uncomp_len, and the reader zstd-decompresses the fetched +// [offset, offset+length) range to uncomp_len before parsing the dict block. The +// block-level crc32c (and BlockRef.checksum) cover the UNCOMPRESSED bytes, so a +// zstd block shrinks the bytes fetched from S3 while keeping the same integrity +// guarantees after decompression in RAM. +inline constexpr uint8_t kZstd = 1u << 0; +} // namespace block_ref_flags + +// Physical location and checksum info for a single DICT block. Aligned with SampledTermIndex by ordinal: +// SampledTermIndex[i]'s first_term corresponds to DictBlockDirectory[i] (see design spec +// "sampled dict index"). The read path issues a single range read over [offset, offset+length). +struct BlockRef { + uint64_t offset = 0; // absolute byte offset of the block within the container + uint64_t length = 0; // ON-DISK byte length of the block (compressed when kZstd) + uint32_t n_entries = 0; // number of DictEntry records within this block + uint8_t flags = 0; // block-level flags (block_ref_flags::*) + uint32_t checksum = 0; // crc32c of the block's UNCOMPRESSED content (verified after read) + uint64_t uncomp_len = 0; // uncompressed block byte length (stored only when kZstd set) +}; + +// DICT block directory: block ordinal → physical location mapping. +// +// on-disk layout (framed by SectionFramer with a unified type+len+crc32c wrapper): +// [u8 type=kDictBlockDirectory][varint64 payload_len][payload][fixed32 crc32c] +// payload = varint32 n_blocks +// then n_blocks × block_ref{ +// varint64 offset, varint64 length, varint32 n_entries, +// u8 flags, fixed32 checksum } +// Section-level crc detects truncation/corruption; block_ref.checksum is the per-block crc. +class DictBlockDirectoryBuilder { +public: + void add(const BlockRef& ref) { refs_.push_back(ref); } + + // Encodes as a kDictBlockDirectory framed section (with embedded crc32c) and appends to sink. + void finish(ByteSink* sink) const; + +private: + std::vector refs_; +}; + +// Reads and verifies a kDictBlockDirectory framed section; provides ordinal → BlockRef lookup. +// After parsing, all block_refs reside in the reader (entering the searcher cache along with meta). +class DictBlockDirectoryReader { +public: + // Verifies the section crc and deserializes all block_refs. + // crc mismatch / truncation / trailing bytes → kCorruption; wrong section type → kInvalidArgument. + static Status open(Slice section, DictBlockDirectoryReader* out); + + uint32_t n_blocks() const { return static_cast(refs_.size()); } + + // Resident heap held beyond sizeof(*this): the refs_ vector buffer. BlockRef + // is trivially copyable (no per-element heap), so the vector buffer is the + // whole charge. Summed into LogicalIndexReader::memory_usage(). + size_t heap_bytes() const { return refs_.capacity() * sizeof(BlockRef); } + + // Returns the ordinal-th block_ref; ordinal >= n_blocks → kNotFound. + Status get(uint32_t ordinal, BlockRef* out) const; + +private: + std::vector refs_; +}; + +} // namespace doris::snii::format diff --git a/be/src/storage/index/snii/format/dict_entry.cpp b/be/src/storage/index/snii/format/dict_entry.cpp new file mode 100644 index 00000000000000..e9ee72912f461b --- /dev/null +++ b/be/src/storage/index/snii/format/dict_entry.cpp @@ -0,0 +1,383 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/format/dict_entry.h" + +#include +#include + +#include "common/check.h" +#include "storage/index/snii/common/slice.h" + +namespace doris::snii::format { + +namespace { + +// Body-decode counter seam (T07). Incremented once per decode_dict_entry_rest +// call so tests can assert a key-first scan only materializes the bodies it +// actually produces. Relaxed atomic: read by tests only, never a sync point. +std::atomic& body_decode_atomic() { + static std::atomic counter {0}; + return counter; +} + +// Pure-function assembly / parsing of flags bits; avoids a long inline if-else +// chain. +uint8_t pack_flags(const DictEntry& e) { + uint8_t f = 0; + if (e.kind == DictEntryKind::kInline) f |= dict_flags::kKind; + if (e.enc == DictEntryEnc::kWindowed) f |= dict_flags::kEnc; + if (e.has_sb) f |= dict_flags::kHasSb; + // bit3 has_champion / bit4 offsets_ref are always 0 in v1. + return f; +} + +void apply_flags(uint8_t f, DictEntry* e) { + e->kind = (f & dict_flags::kKind) ? DictEntryKind::kInline : DictEntryKind::kPodRef; + e->enc = (f & dict_flags::kEnc) ? DictEntryEnc::kWindowed : DictEntryEnc::kSlim; + e->has_sb = (f & dict_flags::kHasSb) != 0; +} + +// Length of the longest common prefix between term and prev_term. +uint32_t common_prefix_len(std::string_view term, std::string_view prev) { + uint32_t n = 0; + const uint32_t lim = static_cast(std::min(term.size(), prev.size())); + while (n < lim && term[n] == prev[n]) ++n; + return n; +} + +bool tier_has_stats(IndexTier tier) { + return tier >= IndexTier::kT2; +} + +// ---- Encode entry body (excluding entry_len and trailing crc) ---- + +void write_term_key(const DictEntry& e, std::string_view prev, ByteSink* sink) { + const uint32_t prefix = common_prefix_len(e.term, prev); + const std::string_view suffix = std::string_view(e.term).substr(prefix); + sink->put_varint32(prefix); + sink->put_varint32(static_cast(suffix.size())); + sink->put_bytes(Slice(suffix)); +} + +void write_stats(const DictEntry& e, IndexTier tier, bool term_stats, ByteSink* sink) { + sink->put_varint32(e.df); + // G16-f: term_stats == false (freq-dropped index, block header flag) omits + // the ttf_delta/max_freq varints -- BM25-only fields, dead without freq. + if (!tier_has_stats(tier) || !term_stats) return; + sink->put_varint64(e.ttf_delta); + sink->put_varint64(e.max_freq); +} + +// Per-window codec mode byte shared by slim/inline single-window regions. +uint8_t pack_win_mode(const DictEntry& e) { + uint8_t mode = 0; + if (e.dd_meta.zstd) mode |= 1u << 0; // dd_zstd + if (e.freq_meta.zstd) mode |= 1u << 1; // freq_zstd + return mode; +} + +// Writes the slim/inline region codec metadata (dd always; freq when tier>=T2). +// store_crc=false (INLINE entries, format v2) omits the redundant per-region +// crc32c: the inline bytes already sit inside the dict block, whose own +// block-level crc32c covers them. POD-ref entries pass store_crc=true (their +// regions live in the separately-fetched .frq POD, uncovered by the block crc). +void write_region_meta(const DictEntry& e, IndexTier tier, bool store_crc, ByteSink* sink) { + sink->put_u8(pack_win_mode(e)); + sink->put_varint64(e.dd_meta.uncomp_len); + if (store_crc) sink->put_fixed32(e.dd_meta.crc); + if (!tier_has_stats(tier)) return; + sink->put_varint64(e.freq_meta.uncomp_len); + if (store_crc) sink->put_fixed32(e.freq_meta.crc); +} + +void write_pod_ref(const DictEntry& e, IndexTier tier, ByteSink* sink) { + sink->put_varint64(e.frq_off_delta); + sink->put_varint64(e.frq_len); + if (e.enc == DictEntryEnc::kWindowed) { + sink->put_varint64(e.prelude_len); + sink->put_varint64(e.frq_docs_len); + } else { + sink->put_varint64(e.frq_docs_len); // slim pod_ref: dd region on-disk length + // POD-ref regions live in the .frq POD (not covered by the block crc): keep + // crc. + write_region_meta(e, tier, /*store_crc=*/true, sink); + } + if (!tier_has_stats(tier)) return; + sink->put_varint64(e.prx_off_delta); + sink->put_varint64(e.prx_len); +} + +void write_inline(const DictEntry& e, IndexTier tier, ByteSink* sink) { + sink->put_varint64(static_cast(e.frq_bytes.size())); + sink->put_bytes(Slice(e.frq_bytes)); + sink->put_varint64(e.inline_dd_disk_len); + // INLINE bytes are covered by the dict block crc32c: omit the redundant + // per-region crc. + write_region_meta(e, tier, /*store_crc=*/false, sink); + if (!tier_has_stats(tier)) return; + sink->put_varint64(static_cast(e.prx_bytes.size())); + sink->put_bytes(Slice(e.prx_bytes)); +} + +void write_body(const DictEntry& e, std::string_view prev, IndexTier tier, bool term_stats, + ByteSink* sink) { + write_term_key(e, prev, sink); + sink->put_u8(pack_flags(e)); + write_stats(e, tier, term_stats, sink); + if (e.kind == DictEntryKind::kInline) { + write_inline(e, tier, sink); + } else { + write_pod_ref(e, tier, sink); + } +} + +// ---- Decode entry body ---- + +Status read_term_key(ByteSource* src, std::string_view prev, DictEntry* out) { + uint32_t prefix = 0; + uint32_t suffix_len = 0; + RETURN_IF_ERROR(src->get_varint32(&prefix)); + RETURN_IF_ERROR(src->get_varint32(&suffix_len)); + if (prefix > prev.size()) { + return Status::Error( + "dict_entry: prefix_len exceeds prev_term length"); + } + Slice suffix; + RETURN_IF_ERROR(src->get_bytes(suffix_len, &suffix)); + out->term.assign(prev.substr(0, prefix)); + out->term.append(reinterpret_cast(suffix.data()), suffix.size()); + return Status::OK(); +} + +Status read_stats(ByteSource* src, IndexTier tier, bool term_stats, DictEntry* out) { + RETURN_IF_ERROR(src->get_varint32(&out->df)); + out->term_stats_present = tier_has_stats(tier) && term_stats; + if (!out->term_stats_present) return Status::OK(); + RETURN_IF_ERROR(src->get_varint64(&out->ttf_delta)); + RETURN_IF_ERROR(src->get_varint64(&out->max_freq)); + return Status::OK(); +} + +// Reads the slim/inline region codec metadata (mode/uncomp/[crc]) and fills the +// dd/freq region disk_len from the supplied total/split lengths. has_crc=false +// (INLINE entries, format v2) means no per-region crc was stored: the on-disk +// crc field is absent and region decode must skip crc verification (verify_crc= +// false) since the dict block's own crc32c already covers the inline bytes. +Status read_region_meta(ByteSource* src, IndexTier tier, bool has_crc, uint64_t dd_disk_len, + uint64_t freq_disk_len, DictEntry* out) { + uint8_t mode = 0; + RETURN_IF_ERROR(src->get_u8(&mode)); + if ((mode & ~0x3u) != 0) { + return Status::Error( + "dict_entry: unknown win_mode bits"); + } + out->dd_meta.zstd = (mode & (1u << 0)) != 0; + out->dd_meta.disk_len = dd_disk_len; + out->dd_meta.verify_crc = has_crc; + RETURN_IF_ERROR(src->get_varint64(&out->dd_meta.uncomp_len)); + if (has_crc) RETURN_IF_ERROR(src->get_fixed32(&out->dd_meta.crc)); + if (!tier_has_stats(tier)) { + if (mode & (1u << 1)) { + return Status::Error( + "dict_entry: freq mode set without freq tier"); + } + return Status::OK(); + } + out->freq_meta.zstd = (mode & (1u << 1)) != 0; + out->freq_meta.disk_len = freq_disk_len; + out->freq_meta.verify_crc = has_crc; + RETURN_IF_ERROR(src->get_varint64(&out->freq_meta.uncomp_len)); + if (has_crc) RETURN_IF_ERROR(src->get_fixed32(&out->freq_meta.crc)); + return Status::OK(); +} + +Status read_pod_ref(ByteSource* src, IndexTier tier, DictEntry* out) { + RETURN_IF_ERROR(src->get_varint64(&out->frq_off_delta)); + RETURN_IF_ERROR(src->get_varint64(&out->frq_len)); + if (out->enc == DictEntryEnc::kWindowed) { + RETURN_IF_ERROR(src->get_varint64(&out->prelude_len)); + RETURN_IF_ERROR(src->get_varint64(&out->frq_docs_len)); + if (out->prelude_len == 0 || out->prelude_len > out->frq_docs_len || + out->frq_docs_len > out->frq_len) { + return Status::Error( + "dict_entry: invalid windowed docs prefix"); + } + } else { + RETURN_IF_ERROR(src->get_varint64(&out->frq_docs_len)); + if (out->frq_docs_len > out->frq_len) { + return Status::Error( + "dict_entry: frq_docs_len exceeds frq_len"); + } + RETURN_IF_ERROR(read_region_meta(src, tier, /*has_crc=*/true, out->frq_docs_len, + out->frq_len - out->frq_docs_len, out)); + } + if (!tier_has_stats(tier)) return Status::OK(); + RETURN_IF_ERROR(src->get_varint64(&out->prx_off_delta)); + RETURN_IF_ERROR(src->get_varint64(&out->prx_len)); + return Status::OK(); +} + +Status read_byte_blob(ByteSource* src, std::vector* out) { + uint64_t len = 0; + RETURN_IF_ERROR(src->get_varint64(&len)); + Slice bytes; + RETURN_IF_ERROR(src->get_bytes(static_cast(len), &bytes)); + out->assign(bytes.data(), bytes.data() + bytes.size()); + return Status::OK(); +} + +Status read_inline(ByteSource* src, IndexTier tier, DictEntry* out) { + RETURN_IF_ERROR(read_byte_blob(src, &out->frq_bytes)); + RETURN_IF_ERROR(src->get_varint64(&out->inline_dd_disk_len)); + if (out->inline_dd_disk_len > out->frq_bytes.size()) { + return Status::Error( + "dict_entry: inline_dd_disk_len exceeds frq_bytes"); + } + const uint64_t freq_disk_len = + static_cast(out->frq_bytes.size()) - out->inline_dd_disk_len; + // INLINE entries store no per-region crc (covered by the block crc): + // has_crc=false. + RETURN_IF_ERROR(read_region_meta(src, tier, /*has_crc=*/false, out->inline_dd_disk_len, + freq_disk_len, out)); + if (!tier_has_stats(tier)) return Status::OK(); + RETURN_IF_ERROR(read_byte_blob(src, &out->prx_bytes)); + return Status::OK(); +} + +Status read_locator(ByteSource* src, IndexTier tier, DictEntry* out) { + if (out->kind == DictEntryKind::kInline) return read_inline(src, tier, out); + return read_pod_ref(src, tier, out); +} + +// Read entry_len (= body length) and verify that src has enough remaining +// bytes. +Status read_entry_len(ByteSource* src, uint64_t* total) { + RETURN_IF_ERROR(src->get_varint64(total)); + if (*total > src->remaining()) { + return Status::Error( + "dict_entry: entry_len out of range"); + } + return Status::OK(); +} + +} // namespace + +Status encode_dict_entry(const DictEntry& entry, std::string_view prev_term, IndexTier tier, + ByteSink* sink, bool term_stats) { + ByteSink body_scratch; + return encode_dict_entry(entry, prev_term, tier, sink, term_stats, &body_scratch); +} + +Status encode_dict_entry(const DictEntry& entry, std::string_view prev_term, IndexTier tier, + ByteSink* sink, bool term_stats, ByteSink* body_scratch) { + if (sink == nullptr || body_scratch == nullptr || sink == body_scratch) { + return Status::Error( + "dict_entry: sink and body_scratch must be non-null and distinct"); + } + + // Serialize the body into a temporary buffer first to obtain the exact + // length, then write entry_len + body. CRC verification is done uniformly at + // the DICT block level (covering block header + all entries + anchor table); + // CRC is not repeated at the entry level, to keep slim/inline low-frequency + // terms maximally compact (spec §DICT block/§dict entry). + body_scratch->clear(); + write_body(entry, prev_term, tier, term_stats, body_scratch); + sink->put_varint64(static_cast(body_scratch->size())); + sink->put_bytes(body_scratch->view()); + return Status::OK(); +} + +Status decode_dict_entry_key(ByteSource* src, std::string_view prev_term, DictEntry* out, + size_t* body_start, uint64_t* entry_total) { + if (src == nullptr || out == nullptr) { + return Status::Error("dict_entry: src / out is null"); + } + *out = DictEntry {}; + + uint64_t total = 0; + RETURN_IF_ERROR(read_entry_len(src, &total)); + *body_start = src->position(); + *entry_total = total; + + return read_term_key(src, prev_term, out); +} + +Status decode_dict_entry_rest(ByteSource* src, IndexTier tier, size_t body_start, + uint64_t entry_total, DictEntry* out, bool term_stats) { + if (src == nullptr || out == nullptr) { + return Status::Error("dict_entry: src / out is null"); + } + // Body-decode seam: count exactly the entries whose body we materialize. + body_decode_atomic().fetch_add(1, std::memory_order_relaxed); + + uint8_t flags = 0; + RETURN_IF_ERROR(src->get_u8(&flags)); + apply_flags(flags, out); + RETURN_IF_ERROR(read_stats(src, tier, term_stats, out)); + RETURN_IF_ERROR(read_locator(src, tier, out)); + + // The body must consume exactly entry_len bytes; otherwise the structure is + // inconsistent with the tier. + const size_t consumed = src->position() - body_start; + if (consumed != static_cast(entry_total)) { + return Status::Error( + "dict_entry: body length does not match entry_len"); + } + return Status::OK(); +} + +Status skip_dict_entry_body(ByteSource* src, size_t body_start, uint64_t entry_total) { + if (src == nullptr) { + return Status::Error("dict_entry: src is null"); + } + const size_t consumed = src->position() - body_start; + if (consumed > static_cast(entry_total)) { + return Status::Error( + "dict_entry: term key overruns entry body"); + } + const size_t advance = static_cast(entry_total) - consumed; + Slice unused; + return src->get_bytes(advance, &unused); +} + +Status decode_dict_entry(ByteSource* src, std::string_view prev_term, IndexTier tier, + DictEntry* out, bool term_stats) { + size_t body_start = 0; + uint64_t entry_total = 0; + RETURN_IF_ERROR(decode_dict_entry_key(src, prev_term, out, &body_start, &entry_total)); + return decode_dict_entry_rest(src, tier, body_start, entry_total, out, term_stats); +} + +Status skip_dict_entry(ByteSource* src) { + if (src == nullptr) + return Status::Error("dict_entry: src is null"); + uint64_t total = 0; + RETURN_IF_ERROR(read_entry_len(src, &total)); + Slice unused; + return src->get_bytes(static_cast(total), &unused); +} + +uint64_t dict_entry_body_decode_count() { + return body_decode_atomic().load(std::memory_order_relaxed); +} + +void reset_dict_entry_counters() { + body_decode_atomic().store(0, std::memory_order_relaxed); +} + +} // namespace doris::snii::format diff --git a/be/src/storage/index/snii/format/dict_entry.h b/be/src/storage/index/snii/format/dict_entry.h new file mode 100644 index 00000000000000..df427080067f01 --- /dev/null +++ b/be/src/storage/index/snii/format/dict_entry.h @@ -0,0 +1,184 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/encoding/byte_sink.h" +#include "storage/index/snii/encoding/byte_source.h" +#include "storage/index/snii/format/format_constants.h" +#include "storage/index/snii/format/frq_pod.h" + +// DictEntry —— on-disk encoding/decoding of a dict entry. +// +// Byte layout (see docs/design/SNII-design-spec.source.md "dict entry" +// section): +// entry_len varint # byte length of entry body, allowing reader to skip +// unknown extensions or fast-skip entries +// --- entry body begins here, covered by entry_len --- +// prefix_len varint # length of shared prefix with prev_term +// suffix_len varint # number of suffix bytes +// suffix u8[] # suffix bytes that differ from prev_term +// flags u8 # bit0 kind / bit1 enc / bit2 has_sb / bit3 +// has_champion(=0) / bit4 offsets_ref(=0) df varint ttf_delta varint +// # only when tier>=T2 max_freq varint # only when tier>=T2 locator: +// pod_ref: frq_off_delta varint, frq_len varint, +// [prelude_len varint, frq_docs_len varint when enc=windowed] +// # docs-only prefix [prelude][dd-block]; windowed entries +// carry # per-window region metadata in the prelude. +// [frq_docs_len varint, slim region meta when enc=slim]: +// # frq_docs_len == dd region on-disk length; the docs-only +// prefix # [frq_off, frq_off+frq_docs_len) a docid-only reader +// fetches # without the freq region. win_mode u8 (bit0 +// dd_zstd, bit1 freq_zstd) dd_uncomp_len varint, crc_dd u32 +// [freq_uncomp_len varint, crc_freq u32 when tier>=T2] +// # The single slim window is [dd_region][freq_region]; +// dd_disk_len # = frq_docs_len, freq_disk_len = frq_len - +// frq_docs_len. +// [prx_off_delta varint, prx_len varint when tier>=T2] +// inline: frq_len varint, frq_bytes u8[], # frq_bytes = +// [dd_region][freq_region] +// slim region meta (as above, sans frq_docs_len which == dd disk +// len +// carried as inline_dd_disk_len varint), +// [prx_len varint, prx_bytes u8[] when tier>=T2] +// --- entry body ends --- +// +// CRC verification is performed at the DICT block level (covering block header +// + all entries + anchor offset table), no per-entry CRC to keep slim/inline +// low-frequency terms compact (spec §DICT block line 330/348). tier and +// positions capability are provided by Core metadata (not stored redundantly +// inside entries): when tier>=T2, ttf_delta / max_freq and .prx locator/bytes +// are written. +namespace doris::snii::format { + +// Dict entry: inline or pod-ref (two states), self-described length, supports +// intra-block front coding. +struct DictEntry { + // term key (front coding relative to prev_term is applied during + // encode/decode; full term stored here). + std::string term; + + // flags. + DictEntryKind kind = DictEntryKind::kPodRef; + DictEntryEnc enc = DictEntryEnc::kSlim; + bool has_sb = false; + + // term stats. + uint32_t df = 0; + uint64_t ttf_delta = 0; // only when tier>=T2 AND the block carries term stats + // G16-f: false when decoded from a kNoTermStats block (freq-dropped index): + // ttf_delta/max_freq above are then meaningless defaults, NOT real zeros. + // Consumers that need them (stats provider / BM25) must check this flag. + bool term_stats_present = true; + uint64_t max_freq = 0; // only when tier>=T2 + + // pod_ref locator. + uint64_t frq_off_delta = 0; + uint64_t frq_len = 0; + uint64_t prelude_len = 0; // only when enc=windowed + uint64_t frq_docs_len = 0; // pod_ref docs-only prefix length + uint64_t prx_off_delta = 0; // only when tier>=T2 + uint64_t prx_len = 0; // only when tier>=T2 + + // slim/inline single-window region codecs. The window is + // [dd_region][freq_region] (no self-describing header). dd_meta drives the + // docs-only decode; freq_meta the scoring decode (only when tier>=T2). For + // slim pod_ref dd_meta.disk_len == frq_docs_len; for inline it is stored as + // inline_dd_disk_len. + FrqRegionMeta dd_meta; + FrqRegionMeta freq_meta; // only when tier>=T2 + uint64_t inline_dd_disk_len = 0; // only for inline: dd region on-disk length + + // inline payload. + std::vector frq_bytes; // = [dd_region][freq_region] + std::vector prx_bytes; // only when tier>=T2 +}; + +// Encodes an entry into sink (appending) using the layout above, with front +// coding relative to prev_term. tier determines whether optional fields are +// written. term_stats == false (G16-f: freq-dropped indexes, declared by the +// block header's kNoTermStats flag) omits the ttf_delta/max_freq varints -- +// they serve only BM25 scoring, dead on an index that dropped freq; df stays. +// Region metadata (freq/prx locators) remains tier-conditioned. +Status encode_dict_entry(const DictEntry& entry, std::string_view prev_term, IndexTier tier, + ByteSink* sink, bool term_stats = true); + +// Same encoding with caller-owned body scratch. The scratch is cleared before +// use while retaining capacity, allowing block builders to avoid one allocation +// per dictionary entry. sink and body_scratch must be distinct objects. +Status encode_dict_entry(const DictEntry& entry, std::string_view prev_term, IndexTier tier, + ByteSink* sink, bool term_stats, ByteSink* body_scratch); + +// Decodes one entry from the current position of src; term is reconstructed +// from prev_term + suffix. Verifies the trailing CRC; out-of-range / CRC +// mismatch / invalid prefix_len all return Corruption. term_stats must match +// the writer's choice (the dict block header flag carries it). +Status decode_dict_entry(ByteSource* src, std::string_view prev_term, IndexTier tier, + DictEntry* out, bool term_stats = true); + +// Skips one entry using only entry_len (does not parse internal fields or +// verify CRC). +Status skip_dict_entry(ByteSource* src); + +// ---- Key-first decode primitives (T07) ---- +// +// decode_dict_entry is split into a "key" stage and a "rest" (body) stage so a +// caller scanning many entries can decide on the (front-coded) term key alone +// whether the entry's body is worth materializing. decode_dict_entry below is +// re-expressed as key + rest and stays byte-for-byte identical in output. + +// Reads only entry_len + the front-coded term key. On return src is positioned +// at the start of the entry body (the flags byte). out is reset to defaults and +// out->term is reconstructed from prev_term + suffix; no other field is touched. +// *body_start receives the absolute src position of the body (right after the +// entry_len varint) and *entry_total the body byte length (already bounds-checked +// against src->remaining()). Used to drive key-first scans (find_term, prefix +// streaming) that skip non-matching bodies. +Status decode_dict_entry_key(ByteSource* src, std::string_view prev_term, DictEntry* out, + size_t* body_start, uint64_t* entry_total); + +// Continues from the position decode_dict_entry_key left at: reads +// flags/stats/locator into *out and verifies the body consumed exactly +// entry_total bytes (body_start anchors the consumed count). Increments the +// body-decode counter seam (see dict_entry_body_decode_count) at its top so +// tests can assert how many entry bodies a scan actually materialized. +Status decode_dict_entry_rest(ByteSource* src, IndexTier tier, size_t body_start, + uint64_t entry_total, DictEntry* out, bool term_stats = true); + +// Skips the remaining body bytes after decode_dict_entry_key, advancing src to +// the next entry without parsing flags/stats/locator. advance = entry_total - +// (src.position() - body_start); the term key already consumed by the key stage +// is accounted for. +Status skip_dict_entry_body(ByteSource* src, size_t body_start, uint64_t entry_total); + +// Test-only instrumentation seam: dict_entry_body_decode_count() returns a +// process-global count of decode_dict_entry_rest calls -- i.e. how many entry +// bodies (flags/stats/locator + any inline byte copies) a scan materialized. +// Key-first find_term / prefix streaming drive this toward the number of +// produced hits instead of the number of entries walked. Counters use relaxed +// atomics; reset between tests. +uint64_t dict_entry_body_decode_count(); +void reset_dict_entry_counters(); + +} // namespace doris::snii::format diff --git a/be/src/storage/index/snii/format/format_constants.h b/be/src/storage/index/snii/format/format_constants.h new file mode 100644 index 00000000000000..5af14d755573dd --- /dev/null +++ b/be/src/storage/index/snii/format/format_constants.h @@ -0,0 +1,144 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +// SNII container and per-section on-disk contract constants. +// Once published, these values are format semantics; changes require bumping +// format_version and maintaining a compatibility policy. All multi-byte +// fixed-width fields are little-endian; variable-length integers use LEB128 +// (see snii/encoding/varint.h). +namespace doris::snii::format { + +// ---- Container-level magic / version ---- +// "SNII" reads as 0x49494E53 in little-endian. +inline constexpr uint32_t kContainerMagic = 0x49494E53u; // 'S''N''I''I' +inline constexpr uint32_t kTailMagic = 0x4C494154u; // 'T''A''I''L' +inline constexpr uint16_t kFormatVersion = 1; +inline constexpr uint16_t kMinReaderVersion = 1; + +// ---- Metadata directory required_features values ---- +// The directory carries opaque blob logical index entries (kind != INVERTED). +// Set iff at least one such entry exists; readers whitelist known values and +// reject unknown ones as Unsupported, so a pure-text directory stays +// byte-identical to the pre-blob format. +inline constexpr uint32_t kFeatureBlobLogicalIndex = 1; + +// ---- SectionFramer type ids for standalone metadata blobs ---- +enum class SectionType : uint8_t { + kSampledTermIndex = 2, + kDictBlockDirectory = 3, + kCoreMetadataPB = 6, + // G13: zstd-compressed carriers for the two large metadata blobs + // (they are highly compressible sorted string/offset tables and dominate the + // metadata group fetched serially at open). Payload = varint64 + // uncomp_len followed by zstd(original full frame), where "original full + // frame" is the byte-exact kSampledTermIndex / kDictBlockDirectory frame + // (type+len+payload+crc32c) used by the raw layout. Decompression + // therefore reproduces the raw frame verbatim and the sub-module readers + // (which re-verify the inner crc) stay unchanged. The writer emits these ONLY + // when the raw frame reaches kMetaSectionCompressMinBytes AND compression + // shrinks it; otherwise it emits the raw frame. + kSampledTermIndexZstd = 11, + kDictBlockDirectoryZstd = 12, + // Per-document one-byte BM25 norms. This must remain distinct from + // Core metadata so a corrupt section reference cannot reinterpret valid + // collection statistics as document norms. + kNormsPod = 14, +}; + +// ---- Logical index postings storage content configuration (fixed per logical +// index, not per-term) ---- Determines whether to write freq / positions / +// norms+stats. +enum class IndexConfig : uint8_t { + kDocsOnly = 0, // docid only: term/match filtering + kDocsPositions = 1, // docid+positions (+freq only when the caller keeps + // it -- SniiIndexInput::write_freq, G16-c): MATCH_PHRASE + kDocsPositionsScoring = 2, // + norms + stats: phrase + BM25 + kPositionsOffsets = 3, // reserved (highlight/RAG), not implemented in this release +}; + +// term stats / postings capability tiers: only tier>=kT2 writes +// ttf_delta/max_freq and .prx. +enum class IndexTier : uint8_t { + kT1 = 1, // docs-only + kT2 = 2, // docs-positions + kT3 = 3, // docs-positions-scoring +}; + +inline constexpr IndexTier tier_of(IndexConfig cfg) { + return cfg == IndexConfig::kDocsOnly ? IndexTier::kT1 + : cfg == IndexConfig::kDocsPositions ? IndexTier::kT2 + : IndexTier::kT3; // scoring / offsets +} +inline constexpr bool has_positions(IndexConfig cfg) { + return cfg != IndexConfig::kDocsOnly; +} +inline constexpr bool has_scoring(IndexConfig cfg) { + return cfg == IndexConfig::kDocsPositionsScoring; +} + +// ---- DictEntry flags bit definitions ---- +namespace dict_flags { +inline constexpr uint8_t kKind = 1u << 0; // 0=pod_ref / 1=inline +inline constexpr uint8_t kEnc = 1u << 1; // 0=slim / 1=windowed +inline constexpr uint8_t kHasSb = 1u << 2; // posting prelude includes sub-block directory +inline constexpr uint8_t kHasChampion = 1u << 3; // v1 always 0 +inline constexpr uint8_t kOffsetsRef = 1u << 4; // v1 always 0 +// bit5-7 reserved +} // namespace dict_flags + +enum class DictEntryKind : uint8_t { kPodRef = 0, kInline = 1 }; +enum class DictEntryEnc : uint8_t { kSlim = 0, kWindowed = 1 }; + +// ---- .prx window codec (codec byte bit0-5) ---- +// kRaw : plaintext varint payload (doc_count, per-doc pos_count + position +// deltas). kZstd : zstd-compressed plaintext payload (legacy reader still +// supported). kPfor : doc_count + per-doc pos_count (varint), then position +// deltas bit-packed +// as PFOR runs (kFrqBaseUnit each). No entropy coding -> far cheaper +// build CPU than zstd while staying competitive on size for ascending +// deltas. +enum class PrxCodec : uint8_t { + kRaw = 0, + kZstd = 1, + kPfor = 2 /* bit7 cont-reserved */ +}; + +// ---- Build-time parameters (not format semantics; may be tuned against real +// metrics) ---- +inline constexpr uint32_t kFrqBaseUnit = 256; // window base unit +inline constexpr uint32_t kSlimDfThreshold = 512; // df < this → slim +inline constexpr uint32_t kDefaultInlineThreshold = 256; // slim encoded bytes ≤ this → inline +// Adaptive window sizing (design #4): high-df windowed terms use larger windows +// to cut prelude rows + per-window header/crc overhead. Windows remain a whole +// multiple of kFrqBaseUnit so .prx alignment and win_base/last_docid semantics +// are preserved. A term whose df >= kAdaptiveWindowDfThreshold splits into +// kAdaptiveWindowDocs-sized windows instead of kFrqBaseUnit-sized ones. +inline constexpr uint32_t kAdaptiveWindowDfThreshold = 8192; // df >= this -> larger windows +inline constexpr uint32_t kAdaptiveWindowDocs = 1024; // larger window size (4 * base unit) +inline constexpr uint32_t kDefaultTargetDictBlockBytes = 64 * 1024; +// G13: SampledTermIndex / DictBlockDirectory metadata frames +// at or above this raw size are emitted zstd-compressed (kSampledTermIndexZstd / +// kDictBlockDirectoryZstd); smaller ones stay raw -- compression overhead is not +// worth it below a few KB. A build-time parameter, not format semantics: readers +// accept both layouts regardless of the value. +inline constexpr size_t kMetaSectionCompressMinBytes = 4 * 1024; + +} // namespace doris::snii::format diff --git a/be/src/storage/index/snii/format/frq_pod.cpp b/be/src/storage/index/snii/format/frq_pod.cpp new file mode 100644 index 00000000000000..5080ecaed89fc6 --- /dev/null +++ b/be/src/storage/index/snii/format/frq_pod.cpp @@ -0,0 +1,437 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/format/frq_pod.h" + +#include +#include +#include +#include +#include +#include +#include + +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/encoding/byte_source.h" +#include "storage/index/snii/encoding/crc32c.h" +#include "storage/index/snii/encoding/pfor.h" +#include "storage/index/snii/encoding/zstd_codec.h" +#include "storage/index/snii/format/format_constants.h" +#include "storage/index/snii/format/prx_decode_stats.h" + +namespace doris::snii::testing { +void note_frq_dd_validation_doc_visits(uint64_t count); +void note_frq_dd_materialized_values(uint64_t count); +void note_frq_raw_region_copy_bytes(uint64_t count); +} // namespace doris::snii::testing + +namespace doris::snii::format { + +namespace { + +// Auto-compression threshold: use raw when a region is smaller than this byte +// count (zstd gain is negligible and metadata overhead is relatively large). +inline constexpr size_t kAutoZstdMinBytes = 512; +// Default zstd level for auto mode. +inline constexpr int kDefaultZstdLevel = 3; +// Maximum decompressed byte size for a single region. Guards against a +// corrupted uncomp_len read from S3 that inflated to a huge value: sanity-check +// before allocating/decompressing to avoid GB-scale allocations. Windows are +// 256-doc aligned and normally far smaller than this. +inline constexpr uint32_t kMaxRegionUncompBytes = 256u * 1024 * 1024; +// Maximum doc count per .frq window (guards against a corrupted n). Window +// baseline is 256, practical combined cap is 2048, so this is a loose but +// astronomically-large-number-blocking upper bound. +inline constexpr uint32_t kMaxWindowDocs = 1u << 24; + +// Encode a uint32 array into multiple PFOR runs, each of 256 (kFrqBaseUnit) +// elements. n / run count is not written: the number of runs is derived from +// total length n and kFrqBaseUnit, and the decoder computes it the same way. +void encode_pfor_runs(std::span values, ByteSink* out) { + size_t n = values.size(); + for (size_t off = 0; off < n; off += kFrqBaseUnit) { + size_t run = (n - off < kFrqBaseUnit) ? (n - off) : kFrqBaseUnit; + pfor_encode(values.data() + off, run, out); + } +} + +// Decode n uint32 values from source (multiple PFOR runs of 256 each). +Status decode_pfor_runs(ByteSource* src, size_t n, std::vector* out) { + out->resize(n); + for (size_t off = 0; off < n; off += kFrqBaseUnit) { + size_t run = (n - off < kFrqBaseUnit) ? (n - off) : kFrqBaseUnit; + RETURN_IF_ERROR(pfor_decode(src, run, out->data() + off)); + } + return Status::OK(); +} + +// Verifies docids are ascending and the first entry is not below win_base. +Status validate_docs(std::span docs, uint64_t win_base) { + if (docs.empty()) return Status::OK(); +#ifdef BE_TEST + ::doris::snii::testing::note_frq_dd_validation_doc_visits(1); +#endif + if (static_cast(docs.front()) < win_base) { + return Status::Error("frq: first docid below win_base"); + } + for (size_t i = 1; i < docs.size(); ++i) { +#ifdef BE_TEST + ::doris::snii::testing::note_frq_dd_validation_doc_visits(1); +#endif + if (docs[i] < docs[i - 1]) { + return Status::Error( + "frq: docids must be ascending"); + } + } + return Status::OK(); +} + +// Decision: given level and plaintext length, determine whether to compress. +bool should_compress(int level, size_t plain_len) { + if (level == 0) return false; // force raw + if (level > 0) return true; // force zstd + return plain_len >= kAutoZstdMinBytes; // auto +} + +// Encodes one region's plaintext into raw or zstd, appends the on-disk bytes to +// out, and fills meta (mode/uncomp_len/disk_len/crc). The region carries no +// header. +Status emit_region(Slice plain, int level, ByteSink* out, FrqRegionMeta* meta) { + if (out == nullptr || meta == nullptr) { + return Status::Error("frq: null region out"); + } + meta->uncomp_len = plain.size(); + if (should_compress(level, plain.size())) { + // zstd needs its own buffer: the compressed bytes differ from `plain`. + std::vector disk; + meta->zstd = true; + RETURN_IF_ERROR(zstd_compress(plain, level > 0 ? level : kDefaultZstdLevel, &disk)); + meta->disk_len = static_cast(disk.size()); + meta->crc = crc32c(Slice(disk)); + out->put_bytes(Slice(disk)); + return Status::OK(); + } + // Raw: the on-disk bytes ARE `plain` (a view over the caller's contiguous + // ByteSink), so crc and emit straight from it -- no temp `disk` alloc/copy. + // disk_len MUST stay == plain.size(): open_region enforces uncomp_len == + // disk_len for raw regions. Byte-identical to the former disk.assign() path + // (disk == plain, so crc32c(disk) == crc32c(plain), put_bytes(disk) == same + // bytes). + meta->zstd = false; + meta->disk_len = static_cast(plain.size()); + meta->crc = crc32c(plain); +#ifdef BE_TEST + ::doris::snii::testing::note_frq_raw_region_copy_bytes(plain.size()); +#endif + out->put_bytes(plain); + return Status::OK(); +} + +void finish_raw_region(ByteSink* out, size_t begin, FrqRegionMeta* meta) { + const size_t length = out->size() - begin; + const Slice appended = length == 0 ? Slice() : Slice(out->buffer().data() + begin, length); + meta->zstd = false; + meta->uncomp_len = static_cast(length); + meta->disk_len = static_cast(length); + meta->crc = crc32c(appended); +} + +void rollback_raw_region(ByteSink* out, size_t begin) { + ByteSink restored; + if (begin != 0) { + restored.put_bytes(Slice(out->buffer().data(), begin)); + } + *out = std::move(restored); +} + +Status append_raw_dd_region(std::span docs, uint64_t win_base, ByteSink* out, + FrqRegionMeta* meta) { + if (!docs.empty() && static_cast(docs.front()) < win_base) { + return Status::Error("frq: first docid below win_base"); + } + + const size_t begin = out->size(); + out->put_varint32(static_cast(docs.size())); + std::array deltas; + uint64_t previous = win_base; + for (size_t offset = 0; offset < docs.size(); offset += kFrqBaseUnit) { + const size_t count = std::min(docs.size() - offset, static_cast(kFrqBaseUnit)); + for (size_t i = 0; i < count; ++i) { + const uint32_t doc = docs[offset + i]; + if (static_cast(doc) < previous) { + rollback_raw_region(out, begin); + return Status::Error( + "frq: docids must be ascending"); + } + deltas[i] = static_cast(static_cast(doc) - previous); + previous = doc; + } + pfor_encode(deltas.data(), count, out); + } + finish_raw_region(out, begin, meta); + return Status::OK(); +} + +// Materializes a region's plaintext (raw borrows the view; zstd decompresses) +// and verifies its crc + slice length against meta. +Status open_region(Slice disk, const FrqRegionMeta& meta, std::vector* holder, + PrxCsrAllocationGate* allocation_gate, Slice* plain) { + if (disk.size() != static_cast(meta.disk_len)) { + return Status::Error( + "frq: region slice length mismatch"); + } + if (meta.uncomp_len > kMaxRegionUncompBytes) { + return Status::Error( + "frq: region uncomp_len exceeds sane cap"); + } + // Inline entries (verify_crc=false) carry no per-region crc: their on-disk + // bytes are covered by the enclosing dict block's block-level crc32c, so the + // region crc would be redundant. POD-ref regions keep their own crc check. + if (meta.verify_crc && crc32c(disk) != meta.crc) { + return Status::Error( + "frq: region crc mismatch"); + } + if (!meta.zstd) { + if (meta.uncomp_len != meta.disk_len) { + return Status::Error( + "frq: raw region length inconsistent"); + } + *plain = disk; + return Status::OK(); + } + if (allocation_gate != nullptr) { + RETURN_IF_ERROR(allocation_gate->reserve_decompression(static_cast(meta.uncomp_len), + &holder)); + DCHECK(holder != nullptr); + } + RETURN_IF_ERROR(zstd_decompress(disk, static_cast(meta.uncomp_len), holder)); + *plain = Slice(*holder); + return Status::OK(); +} + +} // namespace + +Status build_dd_region(std::span docids_ascending, uint64_t win_base, + int zstd_level_or_neg_for_auto, ByteSink* out, FrqRegionMeta* meta) { + if (out == nullptr || meta == nullptr) { + return Status::Error("frq: null dd region out"); + } + if (zstd_level_or_neg_for_auto == 0) { + return append_raw_dd_region(docids_ascending, win_base, out, meta); + } + RETURN_IF_ERROR(validate_docs(docids_ascending, win_base)); + ByteSink plain; // VInt n ++ PFOR_runs(doc_delta) + std::vector dd(docids_ascending.size()); +#ifdef BE_TEST + ::doris::snii::testing::note_frq_dd_materialized_values(dd.size()); +#endif + uint64_t prev = win_base; + for (size_t i = 0; i < docids_ascending.size(); ++i) { + dd[i] = static_cast(static_cast(docids_ascending[i]) - prev); + prev = docids_ascending[i]; + } + plain.put_varint32(static_cast(docids_ascending.size())); + encode_pfor_runs(dd, &plain); + return emit_region(plain.view(), zstd_level_or_neg_for_auto, out, meta); +} + +Status build_dd_region_from_deltas(std::span doc_deltas, + int zstd_level_or_neg_for_auto, ByteSink* out, + FrqRegionMeta* meta) { + if (out == nullptr || meta == nullptr) { + return Status::Error("frq: null dd region out"); + } + if (zstd_level_or_neg_for_auto != 0) { + return Status::Error( + "frq: direct doc-delta encoding requires raw level 0"); + } + + const size_t begin = out->size(); + out->put_varint32(static_cast(doc_deltas.size())); + encode_pfor_runs(doc_deltas, out); + finish_raw_region(out, begin, meta); + return Status::OK(); +} + +} // namespace doris::snii::format + +namespace doris::snii::testing { +#ifdef BE_TEST +namespace { +std::atomic validation_visits {0}; +std::atomic materialized_values {0}; +std::atomic raw_copy_bytes {0}; +} // namespace +#endif + +void note_frq_dd_validation_doc_visits(uint64_t count) { +#ifdef BE_TEST + validation_visits.fetch_add(count, std::memory_order_relaxed); +#endif +} +void note_frq_dd_materialized_values(uint64_t count) { +#ifdef BE_TEST + materialized_values.fetch_add(count, std::memory_order_relaxed); +#endif +} +void note_frq_raw_region_copy_bytes(uint64_t count) { +#ifdef BE_TEST + raw_copy_bytes.fetch_add(count, std::memory_order_relaxed); +#endif +} +void reset_frq_raw_encode_work() { +#ifdef BE_TEST + validation_visits.store(0, std::memory_order_relaxed); + materialized_values.store(0, std::memory_order_relaxed); + raw_copy_bytes.store(0, std::memory_order_relaxed); +#endif +} +uint64_t frq_dd_validation_doc_visits() { +#ifdef BE_TEST + return validation_visits.load(std::memory_order_relaxed); +#else + return 0; +#endif +} +uint64_t frq_dd_materialized_values() { +#ifdef BE_TEST + return materialized_values.load(std::memory_order_relaxed); +#else + return 0; +#endif +} +uint64_t frq_raw_region_copy_bytes() { +#ifdef BE_TEST + return raw_copy_bytes.load(std::memory_order_relaxed); +#else + return 0; +#endif +} +} // namespace doris::snii::testing + +namespace doris::snii::format { + +Status build_freq_region(std::span freqs, int zstd_level_or_neg_for_auto, + ByteSink* out, FrqRegionMeta* meta) { + if (out == nullptr || meta == nullptr) { + return Status::Error("frq: null freq region out"); + } + if (zstd_level_or_neg_for_auto == 0) { + const size_t begin = out->size(); + encode_pfor_runs(freqs, out); + finish_raw_region(out, begin, meta); + return Status::OK(); + } + ByteSink plain; + encode_pfor_runs(freqs, &plain); + return emit_region(plain.view(), zstd_level_or_neg_for_auto, out, meta); +} + +namespace { + +Status decode_dd_region_impl(Slice dd_disk, const FrqRegionMeta& meta, uint64_t win_base, + const uint32_t* expected_doc_count, + PrxCsrAllocationGate* allocation_gate, std::vector* docids) { + if (docids == nullptr) + return Status::Error("frq: null docids out"); + std::vector holder; + Slice plain; + RETURN_IF_ERROR(open_region(dd_disk, meta, &holder, allocation_gate, &plain)); + ByteSource src(plain); + uint32_t n = 0; + RETURN_IF_ERROR(src.get_varint32(&n)); + if (n > kMaxWindowDocs) + return Status::Error( + "frq: doc count exceeds sane cap"); + if (expected_doc_count != nullptr && n != *expected_doc_count) { + return Status::Error( + "frq: encoded doc count differs from metadata"); + } + RETURN_IF_ERROR(decode_pfor_runs(&src, n, docids)); + if (!src.eof()) { + return Status::Error( + "frq: trailing bytes after dd region payload"); + } + uint64_t cur = win_base; + if (n != 0 && cur > std::numeric_limits::max()) { + return Status::Error( + "frq: window base exceeds uint32 docid range"); + } + for (uint32_t i = 0; i < n; ++i) { + const uint32_t delta = (*docids)[i]; + if (i != 0 && delta == 0) { + return Status::Error( + "frq: zero docid delta"); + } + if (delta > std::numeric_limits::max() - cur) { + return Status::Error( + "frq: docid accumulation overflow"); + } + cur += delta; + (*docids)[i] = static_cast(cur); + } + return Status::OK(); +} + +} // namespace + +Status decode_dd_region(Slice dd_disk, const FrqRegionMeta& meta, uint64_t win_base, + std::vector* docids) { + return decode_dd_region_impl(dd_disk, meta, win_base, nullptr, nullptr, docids); +} + +Status decode_dd_region(Slice dd_disk, const FrqRegionMeta& meta, uint64_t win_base, + uint32_t expected_doc_count, std::vector* docids) { + return decode_dd_region_impl(dd_disk, meta, win_base, &expected_doc_count, nullptr, docids); +} + +Status decode_dd_region(Slice dd_disk, const FrqRegionMeta& meta, uint64_t win_base, + uint32_t expected_doc_count, PrxCsrAllocationGate* allocation_gate, + std::vector* docids) { + if (allocation_gate == nullptr) { + return Status::Error( + "frq: allocation gate must be non-null"); + } + return decode_dd_region_impl(dd_disk, meta, win_base, &expected_doc_count, allocation_gate, + docids); +} + +Status decode_freq_region(Slice freq_disk, const FrqRegionMeta& meta, size_t doc_count, + std::vector* freqs) { + if (freqs == nullptr) + return Status::Error("frq: null freqs out"); + std::vector holder; + Slice plain; + RETURN_IF_ERROR(open_region(freq_disk, meta, &holder, nullptr, &plain)); + if (doc_count == 0) { + if (meta.uncomp_len != 0) { + return Status::Error( + "frq: empty freq region expected"); + } + freqs->clear(); + return Status::OK(); + } + ByteSource src(plain); + RETURN_IF_ERROR(decode_pfor_runs(&src, doc_count, freqs)); + if (!src.eof()) { + return Status::Error( + "frq: trailing bytes after freq region payload"); + } + return Status::OK(); +} + +} // namespace doris::snii::format diff --git a/be/src/storage/index/snii/format/frq_pod.h b/be/src/storage/index/snii/format/frq_pod.h new file mode 100644 index 00000000000000..4a3ca5d95b137c --- /dev/null +++ b/be/src/storage/index/snii/format/frq_pod.h @@ -0,0 +1,150 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/encoding/byte_sink.h" + +// .frq region codec (FrqPod): doc-delta (dd) and freq postings, columnar + PFOR +// (see docs/design SNII "frq design" and the read-byte-optimizations +// design 1.6). +// +// PHASE D (posting-level dd/freq grouping): windows are NO LONGER +// self-describing. A windowed .frq payload is laid out as +// [prelude][dd-block][freq-block] +// where the dd-block concatenates every window's dd_region and the freq-block +// concatenates every window's freq_region. Each region is independently encoded +// (raw or zstd, chosen by size) and the per-window codec metadata (mode, +// lengths, crc, offsets) is hoisted into the frq_prelude rows -- the region +// bytes carry NO header. This makes the docs-only prefix ([prelude][dd-block]) +// ONE contiguous run a docid-only / phrase reader can fetch in a single range, +// skipping the freq-block entirely. +// +// dd_region plaintext = VInt n ++ PFOR_runs(doc_delta) # n = doc count +// dd[0] = first_docid - win_base; dd[i] = docid[i] - docid[i-1]; win_base is +// the previous window's last docid (first window = 0). +// freq_region plaintext = PFOR_runs(freq) # present iff +// has_freq PFOR runs are segmented at 256 docs (kFrqBaseUnit); a partial +// segment writes the remainder. Variable-length integers reuse +// snii/encoding/varint; PFOR reuses snii/encoding/pfor; crc32c covers each +// region's ON-DISK bytes. +namespace doris::snii::format { + +class PrxCsrAllocationGate; + +// Codec metadata for ONE encoded region (dd or freq), hoisted into the prelude. +// The region's on-disk bytes are pure payload (no header); these fields drive +// the decode. crc covers the on-disk (disk_len) bytes. +struct FrqRegionMeta { + bool zstd = false; // true => disk bytes are zstd(plaintext); false => raw + uint64_t uncomp_len = 0; // plaintext byte length (== disk_len when raw) + uint64_t disk_len = 0; // on-disk byte length of this region + uint32_t crc = 0; // crc32c of the on-disk (disk_len) bytes + // When false, decode_*_region SKIPS the per-region crc check (and the writer + // omits the 4-byte crc from the dict entry). Set false for INLINE entries: + // their region bytes live inside the dict block, whose own block-level crc32c + // already covers them, so a per-region crc is fully redundant. POD-ref + // regions (slim/windowed) live in the separately-fetched .frq POD -- their + // crc stays. + bool verify_crc = true; +}; + +// Encodes a window's dd_region plaintext (VInt n ++ PFOR_runs(doc_delta)) into +// raw or zstd (per zstd_level_or_neg_for_auto), APPENDS the on-disk bytes to +// out, and fills meta (mode/uncomp_len/disk_len/crc). The region carries no +// header. docids_ascending: ascending docids in this window (single doc or +// empty allowed). win_base: previous window's last docid (first window = 0); +// requires docids[0] >= win_base. zstd_level_or_neg_for_auto: <0 auto (zstd +// when large enough, else raw); 0 force +// raw; >0 force zstd at that level. +// Non-ascending docids / first_docid < win_base / null out returns +// InvalidArgument. +Status build_dd_region(std::span docids_ascending, uint64_t win_base, + int zstd_level_or_neg_for_auto, ByteSink* out, FrqRegionMeta* meta); + +// Trusted writer fast path for a window whose doc deltas have already been +// produced and validated by the posting stream. Appends the same +// VInt n ++ PFOR_runs(doc_delta) bytes as build_dd_region(..., level=0), without +// reconstructing absolute docids or scanning the deltas. Only raw level 0 is +// supported; any other level returns InvalidArgument without modifying out or +// meta. +Status build_dd_region_from_deltas(std::span doc_deltas, + int zstd_level_or_neg_for_auto, ByteSink* out, + FrqRegionMeta* meta); + +// Vector convenience overload (forwards a span view; no copy of the elements). +inline Status build_dd_region(const std::vector& docids_ascending, uint64_t win_base, + int zstd_level_or_neg_for_auto, ByteSink* out, FrqRegionMeta* meta) { + return build_dd_region(std::span(docids_ascending), win_base, + zstd_level_or_neg_for_auto, out, meta); +} + +// Encodes a window's freq_region plaintext (PFOR_runs(freq)) into raw or zstd, +// APPENDS the on-disk bytes to out, and fills meta. Empty freqs yields a +// zero-length region. Null out returns InvalidArgument. +Status build_freq_region(std::span freqs, int zstd_level_or_neg_for_auto, + ByteSink* out, FrqRegionMeta* meta); + +// Vector convenience overload (forwards a span view; no copy of the elements). +inline Status build_freq_region(const std::vector& freqs, int zstd_level_or_neg_for_auto, + ByteSink* out, FrqRegionMeta* meta) { + return build_freq_region(std::span(freqs), zstd_level_or_neg_for_auto, out, + meta); +} + +// Decodes a dd_region from its on-disk slice (exactly disk_len bytes) + meta + +// win_base, reconstructing ascending docids. Verifies meta.crc against the +// slice. crc mismatch / wrong slice length / truncation / decompression / +// oversized count all return a non-OK Status. The freq region is irrelevant +// here (docs-only path). +Status decode_dd_region(Slice dd_disk, const FrqRegionMeta& meta, uint64_t win_base, + std::vector* docids); +// Compaction variant: validates the encoded count before resizing `docids`, so +// a corrupt frame cannot grow a pre-reserved destination buffer past its +// metadata-derived hard budget. +Status decode_dd_region(Slice dd_disk, const FrqRegionMeta& meta, uint64_t win_base, + uint32_t expected_doc_count, std::vector* docids); +// Compaction variant that charges a zstd decompression buffer before allocation +// and may reuse the caller's bounded decoder workspace. +Status decode_dd_region(Slice dd_disk, const FrqRegionMeta& meta, uint64_t win_base, + uint32_t expected_doc_count, PrxCsrAllocationGate* allocation_gate, + std::vector* docids); + +// Decodes a freq_region from its on-disk slice (exactly disk_len bytes) + meta, +// producing doc_count freqs. Verifies meta.crc. doc_count == 0 yields empty +// freqs (and requires a zero-length region). crc mismatch / wrong slice length +// / etc. return a non-OK Status. +Status decode_freq_region(Slice freq_disk, const FrqRegionMeta& meta, size_t doc_count, + std::vector* freqs); + +} // namespace doris::snii::format + +// Test-only work counters for the level-0 writer path. They expose redundant +// work that a direct raw encoder must eliminate without making it part of the +// production codec API. +namespace doris::snii::testing { +void reset_frq_raw_encode_work(); +uint64_t frq_dd_validation_doc_visits(); +uint64_t frq_dd_materialized_values(); +uint64_t frq_raw_region_copy_bytes(); +} // namespace doris::snii::testing diff --git a/be/src/storage/index/snii/format/frq_prelude.cpp b/be/src/storage/index/snii/format/frq_prelude.cpp new file mode 100644 index 00000000000000..9cec931244d7d4 --- /dev/null +++ b/be/src/storage/index/snii/format/frq_prelude.cpp @@ -0,0 +1,642 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/format/frq_prelude.h" + +#include +#include +#include + +#include "storage/index/snii/encoding/byte_source.h" +#include "storage/index/snii/encoding/crc32c.h" + +namespace doris::snii::format { + +namespace { + +// Anti-DoS: a segment holds at most ~15M docs (>=1 doc/window), so 1<<24 +// windows is a generous ceiling that still prevents multi-GB allocations from a +// crafted N. (crc32c is not a MAC and cannot defend a re-stamped inflated count.) +constexpr uint64_t kMaxWindows = 1ull << 24; + +uint64_t ceil_div(uint64_t a, uint64_t b) { + return (a + b - 1) / b; +} + +uint8_t make_flags(const FrqPreludeColumns& cols) { + uint8_t flags = 0; + if (cols.has_freq) flags |= frq_prelude_flags::kHasFreq; + if (cols.has_prx) flags |= frq_prelude_flags::kHasPrx; + return flags; +} + +uint8_t make_win_mode(const WindowMeta& m, bool has_freq) { + uint8_t mode = 0; + if (m.dd_zstd) mode |= frq_win_mode::kDdZstd; + if (has_freq && m.freq_zstd) mode |= frq_win_mode::kFreqZstd; + return mode; +} + +Status checked_add_u64(uint64_t lhs, uint64_t rhs, const char* message, uint64_t* out) { + if (rhs > std::numeric_limits::max() - lhs) { + return Status::Error(message); + } + *out = lhs + rhs; + return Status::OK(); +} + +Status checked_u32(uint64_t value, const char* message, uint32_t* out) { + if (value > std::numeric_limits::max()) { + return Status::Error(message); + } + *out = static_cast(value); + return Status::OK(); +} + +Status validate_window_doc_count(bool first_window, uint64_t win_base, uint64_t last_docid, + uint64_t doc_count) { + uint64_t first_docid = 0; + if (!first_window) { + RETURN_IF_ERROR(checked_add_u64(win_base, 1, "frq_prelude: window base exceeds docid range", + &first_docid)); + } + if (last_docid < first_docid) { + return Status::Error( + "frq_prelude: invalid window docid range"); + } + const uint64_t width = last_docid - first_docid + 1; + if (doc_count > width) { + return Status::Error( + "frq_prelude: doc_count exceeds window width"); + } + return Status::OK(); +} + +// Validates builder input: non-null sink, group_size>=1, sane count, and +// non-decreasing absolute last_docid across windows. +Status validate_input(const FrqPreludeColumns& cols, ByteSink* out) { + if (out == nullptr) + return Status::Error("frq_prelude: null sink"); + if (cols.group_size == 0) { + return Status::Error( + "frq_prelude: group_size must be >= 1"); + } + if (cols.windows.size() > kMaxWindows) { + return Status::Error( + "frq_prelude: window count exceeds cap"); + } + for (size_t w = 1; w < cols.windows.size(); ++w) { + if (cols.windows[w].last_docid < cols.windows[w - 1].last_docid) { + return Status::Error( + "frq_prelude: last_docid not monotonic"); + } + } + return Status::OK(); +} + +// Encodes one window row into a per-block sink. last_docid_delta is the row's +// absolute last_docid minus prev_last (the previous window's absolute last). +// +// dd_off/freq_off/prx_off are NOT serialized: each was a pure running prefix sum +// of the per-window disk/prx lengths, which the reader reconstructs (see +// decode_window_row / RunningOffsets). dd_uncomp_len/freq_uncomp_len are written +// ONLY when the region's zstd win_mode bit is set; a raw region's uncomp_len is +// defined to equal its disk_len (open_region enforces uncomp_len == disk_len on +// raw region bytes), so the reader derives it without a stored field. +void encode_window_row(const WindowMeta& m, bool has_freq, bool has_prx, uint64_t prev_last, + ByteSink* block) { + const uint8_t win_mode = make_win_mode(m, has_freq); + block->put_varint64(static_cast(m.last_docid) - prev_last); + block->put_varint64(m.doc_count); + block->put_u8(win_mode); + block->put_varint64(m.dd_disk_len); + if ((win_mode & frq_win_mode::kDdZstd) != 0) { + block->put_varint64(m.dd_uncomp_len); + } + block->put_fixed32(m.crc_dd); + if (has_freq) { + block->put_varint64(m.freq_disk_len); + if ((win_mode & frq_win_mode::kFreqZstd) != 0) { + block->put_varint64(m.freq_uncomp_len); + } + block->put_fixed32(m.crc_freq); + } + if (has_prx) { + block->put_varint64(m.prx_len); + } + block->put_varint64(m.max_freq); + block->put_u8(m.max_norm); +} + +// One super-block's serialized window block plus its directory fields. +struct SuperBlock { + ByteSink block; + uint64_t last_docid = 0; // absolute last docid of this super-block's last window +}; + +// Builds every super-block's window block (row-encoded) and records the running +// absolute last docid at each super-block boundary. +std::vector encode_super_blocks(const FrqPreludeColumns& cols) { + const uint32_t g = cols.group_size; + const size_t n = cols.windows.size(); + std::vector blocks; + blocks.reserve(static_cast(ceil_div(n, g))); + uint64_t prev_last = 0; // previous window's absolute last docid (chains across blocks) + for (size_t start = 0; start < n; start += g) { + const size_t end = std::min(n, start + g); + SuperBlock sb; + for (size_t w = start; w < end; ++w) { + encode_window_row(cols.windows[w], cols.has_freq, cols.has_prx, prev_last, &sb.block); + prev_last = cols.windows[w].last_docid; + } + sb.last_docid = prev_last; + blocks.push_back(std::move(sb)); + } + return blocks; +} + +// Serializes the super_block_dir (one row per super-block) into dir_sink, using +// each block's byte length to compute its offset within the window_dir region. +void encode_super_block_dir(const std::vector& blocks, ByteSink* dir_sink) { + uint64_t prev_last = 0; + uint64_t block_off = 0; + for (const SuperBlock& sb : blocks) { + dir_sink->put_varint64(sb.last_docid - prev_last); + dir_sink->put_varint64(block_off); + dir_sink->put_varint64(sb.block.size()); + prev_last = sb.last_docid; + block_off += sb.block.size(); + } +} + +} // namespace + +Status build_frq_prelude(const FrqPreludeColumns& cols, ByteSink* out) { + RETURN_IF_ERROR(validate_input(cols, out)); + + const std::vector blocks = encode_super_blocks(cols); + ByteSink dir_sink; + encode_super_block_dir(blocks, &dir_sink); + + // covered = header + super_block_dir (the crc covers exactly this region). + ByteSink covered; + covered.put_u8(make_flags(cols)); + covered.put_varint64(cols.windows.size()); + covered.put_varint64(cols.group_size); + covered.put_varint64(blocks.size()); + covered.put_varint64(dir_sink.size()); + covered.put_bytes(dir_sink.view()); + + out->put_bytes(covered.view()); + out->put_fixed32(crc32c(covered.view())); + for (const SuperBlock& sb : blocks) out->put_bytes(sb.block.view()); + return Status::OK(); +} + +namespace { + +// Decoded header fields shared between parse phases. +struct Header { + bool has_freq = false; + bool has_prx = false; + uint64_t n = 0; + uint64_t group_size = 0; + uint64_t n_super = 0; + uint64_t sbdir_len = 0; +}; + +// Verifies the trailing crc covers [start of buffer .. end of super_block_dir]. +// covered_len = header bytes (up to and including sbdir_len) + sbdir_len. +Status verify_covered_crc(Slice prelude, size_t header_end, uint64_t sbdir_len) { + const size_t covered = header_end + static_cast(sbdir_len); + if (covered + sizeof(uint32_t) > prelude.size()) { + return Status::Error( + "frq_prelude: buffer too short for crc region"); + } + uint32_t stored = 0; + ByteSource crc_src(prelude.subslice(covered, sizeof(uint32_t))); + RETURN_IF_ERROR(crc_src.get_fixed32(&stored)); + if (crc32c(prelude.subslice(0, covered)) != stored) { + return Status::Error( + "frq_prelude: crc32c mismatch"); + } + return Status::OK(); +} + +// Parses + validates the header (counts capped before any later reserve). +Status parse_header(ByteSource* src, Header* h) { + uint8_t flags = 0; + RETURN_IF_ERROR(src->get_u8(&flags)); + h->has_freq = (flags & frq_prelude_flags::kHasFreq) != 0; + h->has_prx = (flags & frq_prelude_flags::kHasPrx) != 0; + RETURN_IF_ERROR(src->get_varint64(&h->n)); + RETURN_IF_ERROR(src->get_varint64(&h->group_size)); + RETURN_IF_ERROR(src->get_varint64(&h->n_super)); + RETURN_IF_ERROR(src->get_varint64(&h->sbdir_len)); + if (h->n > kMaxWindows || h->n_super > kMaxWindows) { + return Status::Error( + "frq_prelude: window count exceeds sane cap"); + } + if (h->group_size == 0) { + return Status::Error( + "frq_prelude: group_size is zero"); + } + if (h->n_super != ceil_div(h->n, h->group_size)) { + return Status::Error( + "frq_prelude: n_super inconsistent with N/G"); + } + return Status::OK(); +} + +// One super-block directory row. +struct SbDirRow { + uint64_t last_docid = 0; + uint64_t block_off = 0; + uint64_t block_len = 0; +}; + +// Decodes the super_block_dir region into absolute-last-docid rows, validating +// monotonic last docids and contiguous, in-bounds block offsets. +Status decode_super_block_dir(Slice dir, const Header& h, std::vector* rows, + uint64_t* window_region_len) { + ByteSource src(dir); + rows->clear(); + rows->reserve(static_cast(h.n_super)); + uint64_t prev_last = 0; + uint64_t expect_off = 0; + for (uint64_t s = 0; s < h.n_super; ++s) { + SbDirRow r; + uint64_t ldd = 0; + RETURN_IF_ERROR(src.get_varint64(&ldd)); + RETURN_IF_ERROR(src.get_varint64(&r.block_off)); + RETURN_IF_ERROR(src.get_varint64(&r.block_len)); + RETURN_IF_ERROR(checked_add_u64( + prev_last, ldd, "frq_prelude: super-block last_docid overflow", &r.last_docid)); + uint32_t checked_last = 0; + RETURN_IF_ERROR(checked_u32(r.last_docid, "frq_prelude: super-block last_docid exceeds u32", + &checked_last)); + if (r.last_docid < prev_last || r.block_off != expect_off) { + return Status::Error( + "frq_prelude: super-block dir inconsistent"); + } + expect_off += r.block_len; + prev_last = r.last_docid; + rows->push_back(r); + } + if (!src.eof()) { + return Status::Error( + "frq_prelude: super-block dir has trailing bytes"); + } + *window_region_len = expect_off; + return Status::OK(); +} + +// Validates a per-window codec mode byte against the known bits. +Status check_win_mode(uint8_t mode, bool has_freq) { + if ((mode & ~frq_win_mode::kKnownBits) != 0) { + return Status::Error( + "frq_prelude: unknown win_mode bits"); + } + if (!has_freq && (mode & frq_win_mode::kFreqZstd) != 0) { + return Status::Error( + "frq_prelude: freq mode set without has_freq"); + } + return Status::OK(); +} + +// Running per-block byte offsets, chained across windows AND across super-blocks +// with the same lifetime as prev_last: dd/freq are prefix sums of the per-window +// on-disk region lengths, prx the prefix sum of prx lengths. They replace the +// three offset columns the row used to serialize (each was exactly this sum), so +// the reader reproduces dd_off/freq_off/prx_off bit-identically to the old +// explicit fields. +struct RunningOffsets { + uint64_t dd = 0; + uint64_t freq = 0; + uint64_t prx = 0; +}; + +// Decodes one window row, advancing prev_last to this window's absolute last and +// the running offsets past this window's dd/freq/prx regions. +Status decode_window_row(ByteSource* src, bool has_freq, bool has_prx, bool first_window, + uint64_t* prev_last, RunningOffsets* run, WindowMeta* m) { + uint64_t ldd = 0, doc_count = 0; + RETURN_IF_ERROR(src->get_varint64(&ldd)); + RETURN_IF_ERROR(src->get_varint64(&doc_count)); + uint8_t mode = 0; + RETURN_IF_ERROR(src->get_u8(&mode)); + RETURN_IF_ERROR(check_win_mode(mode, has_freq)); + m->dd_zstd = (mode & frq_win_mode::kDdZstd) != 0; + m->freq_zstd = has_freq && (mode & frq_win_mode::kFreqZstd) != 0; + + // dd region: read disk_len, derive dd_off from the running dd-block offset, + // then advance it (overflow-guarded). uncomp_len is stored only for a zstd + // region; a raw region's uncomp_len == disk_len by contract. + RETURN_IF_ERROR(src->get_varint64(&m->dd_disk_len)); + m->dd_off = run->dd; + RETURN_IF_ERROR(checked_add_u64(run->dd, m->dd_disk_len, + "frq_prelude: dd-block offset overflow", &run->dd)); + if (m->dd_zstd) { + RETURN_IF_ERROR(src->get_varint64(&m->dd_uncomp_len)); + } else { + m->dd_uncomp_len = m->dd_disk_len; + } + RETURN_IF_ERROR(src->get_fixed32(&m->crc_dd)); + if (has_freq) { + RETURN_IF_ERROR(src->get_varint64(&m->freq_disk_len)); + m->freq_off = run->freq; + RETURN_IF_ERROR(checked_add_u64(run->freq, m->freq_disk_len, + "frq_prelude: freq-block offset overflow", &run->freq)); + if (m->freq_zstd) { + RETURN_IF_ERROR(src->get_varint64(&m->freq_uncomp_len)); + } else { + m->freq_uncomp_len = m->freq_disk_len; + } + RETURN_IF_ERROR(src->get_fixed32(&m->crc_freq)); + } + if (has_prx) { + RETURN_IF_ERROR(src->get_varint64(&m->prx_len)); + m->prx_off = run->prx; + RETURN_IF_ERROR(checked_add_u64(run->prx, m->prx_len, "frq_prelude: prx offset overflow", + &run->prx)); + } + uint64_t max_freq = 0; + RETURN_IF_ERROR(src->get_varint64(&max_freq)); + RETURN_IF_ERROR(src->get_u8(&m->max_norm)); + uint64_t last_docid = 0; + RETURN_IF_ERROR(checked_add_u64(*prev_last, ldd, "frq_prelude: window last_docid overflow", + &last_docid)); + RETURN_IF_ERROR(validate_window_doc_count(first_window, *prev_last, last_docid, doc_count)); + m->win_base = *prev_last; + RETURN_IF_ERROR( + checked_u32(last_docid, "frq_prelude: window last_docid exceeds u32", &m->last_docid)); + RETURN_IF_ERROR( + checked_u32(doc_count, "frq_prelude: window doc_count exceeds u32", &m->doc_count)); + RETURN_IF_ERROR( + checked_u32(max_freq, "frq_prelude: window max_freq exceeds u32", &m->max_freq)); + *prev_last = last_docid; + return Status::OK(); +} + +// Decodes one super-block's window block (<=G rows) into the global window list, +// seeding win_base from prev_last and re-checking the recorded sb last docid. +Status decode_one_block(Slice block, const Header& h, uint64_t sb_last_docid, size_t row_count, + uint64_t* prev_last, RunningOffsets* run, + std::vector* windows) { + ByteSource src(block); + for (size_t i = 0; i < row_count; ++i) { + WindowMeta m; + RETURN_IF_ERROR(decode_window_row(&src, h.has_freq, h.has_prx, windows->empty(), prev_last, + run, &m)); + windows->push_back(m); + } + if (!src.eof()) { + return Status::Error( + "frq_prelude: window block has trailing bytes"); + } + if (*prev_last != sb_last_docid) { + return Status::Error( + "frq_prelude: window block last docid mismatch"); + } + return Status::OK(); +} + +// Decodes all window blocks pointed to by the super_block_dir. +Status decode_all_blocks(Slice window_region, const Header& h, const std::vector& dir, + std::vector* windows) { + windows->clear(); + windows->reserve(static_cast(h.n)); + uint64_t prev_last = 0; + // dd/freq/prx running offsets chain across ALL super-blocks (not reset per + // block), the same lifetime as prev_last, so the derived per-window offsets are + // continuous over the whole dd-block / freq-block / prx span. + RunningOffsets run; + for (size_t s = 0; s < dir.size(); ++s) { + const SbDirRow& r = dir[s]; + if (r.block_off + r.block_len > window_region.size() || + r.block_off + r.block_len < r.block_off) { + return Status::Error( + "frq_prelude: window block out of region"); + } + const uint64_t already = static_cast(windows->size()); + const uint64_t rows = std::min(h.group_size, h.n - already); + Slice block = window_region.subslice(static_cast(r.block_off), + static_cast(r.block_len)); + RETURN_IF_ERROR(decode_one_block(block, h, r.last_docid, static_cast(rows), + &prev_last, &run, windows)); + } + if (windows->size() != h.n) { + return Status::Error( + "frq_prelude: decoded window count mismatch"); + } + return Status::OK(); +} + +// Sums the per-window dd/freq on-disk lengths into the dd-block / freq-block +// lengths, guarding each running sum against u64 overflow. The dd_off/freq_off +// contiguity cross-checks the old prelude ran here are now tautological -- the +// reader DERIVES dd_off/freq_off as these very prefix sums (decode_window_row), +// so `m.dd_off == running-sum` holds by construction and is dropped. The +// length-overflow guards and the returned block lengths are retained: they still +// bound the dd-block/freq-block range the callers' in_bounds checks fetch against, +// and mirror the same checked_add_u64 the offset derivation uses. +Status validate_region_layout(const Header& h, const std::vector& windows, + uint64_t* dd_block_len, uint64_t* freq_block_len) { + uint64_t dd_expect = 0; + uint64_t freq_expect = 0; + for (const WindowMeta& m : windows) { + // Raw regions carry uncomp_len == disk_len (derived, not stored); this + // guard stays as defensive documentation of that invariant. + if (m.dd_disk_len > m.dd_uncomp_len && !m.dd_zstd) { + return Status::Error( + "frq_prelude: raw dd region length inconsistent"); + } + if (dd_expect + m.dd_disk_len < dd_expect) { + return Status::Error( + "frq_prelude: dd block length overflow"); + } + dd_expect += m.dd_disk_len; + if (h.has_freq) { + if (freq_expect + m.freq_disk_len < freq_expect) { + return Status::Error( + "frq_prelude: freq block length overflow"); + } + freq_expect += m.freq_disk_len; + } + } + *dd_block_len = dd_expect; + *freq_block_len = freq_expect; + return Status::OK(); +} + +} // namespace + +namespace { +// TEST-ONLY seam backing testing::window_probe_count(): one increment per window +// last_docid comparison, in select_covering_windows_cursor and in locate_window's +// level-2 scan. thread_local => race-free under the shared const reader and free of +// atomic cost in the production cursor's hot loop; tests reset/read on their own thread. +thread_local uint64_t g_window_probes = 0; +inline void note_window_probe() { + ++g_window_probes; +} +} // namespace + +Status FrqPreludeReader::open(Slice prelude, FrqPreludeReader* out) { + ByteSource src(prelude); + Header h; + RETURN_IF_ERROR(parse_header(&src, &h)); + const size_t header_end = src.position(); + RETURN_IF_ERROR(verify_covered_crc(prelude, header_end, h.sbdir_len)); + + if (header_end + static_cast(h.sbdir_len) > prelude.size()) { + return Status::Error( + "frq_prelude: sbdir_len past buffer"); + } + Slice dir = prelude.subslice(header_end, static_cast(h.sbdir_len)); + std::vector rows; + uint64_t window_region_len = 0; + RETURN_IF_ERROR(decode_super_block_dir(dir, h, &rows, &window_region_len)); + + const size_t region_start = header_end + static_cast(h.sbdir_len) + sizeof(uint32_t); + if (region_start + static_cast(window_region_len) > prelude.size()) { + return Status::Error( + "frq_prelude: window region past buffer"); + } + Slice window_region = prelude.subslice(region_start, static_cast(window_region_len)); + + out->has_freq_ = h.has_freq; + out->has_prx_ = h.has_prx; + out->group_size_ = static_cast(h.group_size); + out->n_super_ = static_cast(h.n_super); + out->sb_last_docid_.clear(); + out->sb_last_docid_.reserve(rows.size()); + for (const SbDirRow& r : rows) out->sb_last_docid_.push_back(r.last_docid); + RETURN_IF_ERROR(decode_all_blocks(window_region, h, rows, &out->windows_)); + // Packed last_docid catalogue for the covering-window cursor (in-memory only; + // byte-identical to each windows_[w].last_docid, never serialized). + out->win_last_docid_.clear(); + out->win_last_docid_.reserve(out->windows_.size()); + for (const WindowMeta& m : out->windows_) { + out->win_last_docid_.push_back(m.last_docid); + } + return validate_region_layout(h, out->windows_, &out->dd_block_len_, &out->freq_block_len_); +} + +Status FrqPreludeReader::window(uint32_t w, WindowMeta* out) const { + if (out == nullptr) + return Status::Error("frq_prelude: null window out"); + if (w >= windows_.size()) { + return Status::Error( + "frq_prelude: window index out of range"); + } + *out = windows_[w]; + return Status::OK(); +} + +Status FrqPreludeReader::locate_window(uint32_t docid, bool* found, uint32_t* w) const { + if (found == nullptr || w == nullptr) { + return Status::Error("frq_prelude: null locate out"); + } + *found = false; + if (windows_.empty()) return Status::OK(); + if (docid > windows_.back().last_docid) return Status::OK(); + + // Level 1: first super-block whose absolute last docid >= docid. + const auto sb_it = std::lower_bound(sb_last_docid_.begin(), sb_last_docid_.end(), + static_cast(docid)); + const size_t sb = static_cast(sb_it - sb_last_docid_.begin()); + // Level 2: window binary search within [sb*G, min((sb+1)*G, N)). + const size_t lo = sb * group_size_; + const size_t hi = std::min(lo + group_size_, windows_.size()); + for (size_t i = lo; i < hi; ++i) { + note_window_probe(); + if (docid <= windows_[i].last_docid) { + *found = true; + *w = static_cast(i); + return Status::OK(); + } + } + return Status::OK(); // unreachable when invariants hold; defensive miss. +} + +void select_covering_windows_cursor(const uint32_t* win_last_docid, uint32_t n_windows, + const uint64_t* sb_last_docid, uint32_t n_super, + uint32_t group_size, const std::vector& candidates, + std::vector* windows) { + windows->clear(); + if (n_windows == 0) { + return; // empty-windows guard (mirrors locate_window's windows_.empty() early-out). + } + uint32_t sb = 0; // monotonic super-block cursor + uint32_t w = 0; // monotonic window cursor + uint32_t last_emitted = UINT32_MAX; // last window pushed (run-collapse dedup) + for (uint32_t d : candidates) { + const uint64_t target = d; // widen once; keeps the comparisons template-bracket free + // Level 1: first super-block whose absolute last docid >= d. sb only advances + // forward, so across the whole call it steps at most n_super times. + while (sb < n_super && sb_last_docid[sb] < target) { + ++sb; + } + if (sb == n_super) { + break; // d past the term's last docid -> this and every later candidate miss. + } + // Boundary jump: never scan windows below the current super-block's first window. + if (w < sb * group_size) { + w = sb * group_size; + } + // Level 2: first window whose absolute last docid >= d. w only advances forward, + // so across the whole call the comparisons below total at most n_windows misses + // plus one hit per candidate => probe_count <= candidates + n_windows. + while (w < n_windows) { + note_window_probe(); + if (d <= win_last_docid[w]) { + break; + } + ++w; + } + if (w == n_windows) { + break; // defensive: invariants guarantee a hit once sb < n_super. + } + if (w != last_emitted) { + windows->push_back(w); + last_emitted = w; + } + } +} + +void FrqPreludeReader::select_covering_windows(const std::vector& candidates, + std::vector* windows) const { + select_covering_windows_cursor( + win_last_docid_.data(), static_cast(win_last_docid_.size()), + sb_last_docid_.data(), static_cast(sb_last_docid_.size()), group_size_, + candidates, windows); +} + +} // namespace doris::snii::format + +namespace doris::snii::format::testing { + +uint64_t window_probe_count() { + return g_window_probes; +} + +void reset_window_probe_count() { + g_window_probes = 0; +} + +} // namespace doris::snii::format::testing diff --git a/be/src/storage/index/snii/format/frq_prelude.h b/be/src/storage/index/snii/format/frq_prelude.h new file mode 100644 index 00000000000000..642f3aed96b71c --- /dev/null +++ b/be/src/storage/index/snii/format/frq_prelude.h @@ -0,0 +1,256 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include "common/status.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/encoding/byte_sink.h" + +// FrqPrelude: a TWO-LEVEL (super-block -> window) skippable directory that +// precedes a windowed .frq posting whose payload is laid out (PHASE D, design +// 1.6) with dd and freq regions GROUPED at posting level: +// windowed .frq payload = [prelude][dd-block][freq-block] +// dd-block = dd_region_0 ++ dd_region_1 ++ ... ++ dd_region_{N-1} +// freq-block = freq_region_0 ++ ... ++ freq_region_{N-1} (iff has_freq) +// Windows are NOT self-describing: each window's full codec metadata (region +// offsets, on-disk/uncompressed lengths, modes, crcs) lives in the prelude rows. +// The docs-only prefix [prelude][dd-block] is therefore ONE contiguous run a +// docid-only / phrase reader fetches in a single range, skipping the freq-block. +// +// DictEntry records prelude_len, frq_len (whole payload) and frq_docs_len +// (= prelude_len + dd_block_len) so a reader can range-fetch the prelude first, +// then fetch either the contiguous dd-block (docs-only) or both blocks (scoring). +// +// On-disk layout (strict; all multi-byte fixed fields little-endian, VInt = +// LEB128 via snii/encoding): +// header: +// u8 flags # bit0 has_freq, bit1 has_prx +// VInt N # number of .frq windows +// VInt G # windows per super-block (group_size; >=1) +// VInt n_super # = ceil(N / G); 0 when N==0 +// VInt sbdir_len # byte length of the super_block_dir region +// u32 crc32c # covers header + super_block_dir (NOT the window blocks) +// super_block_dir[n_super]: # small, resident: one row per super-block +// VInt sb_last_docid_delta # cumulative across super-blocks => absolute last +// # docid of the super-block's last window +// VInt sb_block_off # byte offset of this super-block's window block, +// # measured from the start of the window_dir region +// VInt sb_block_len # byte length of this super-block's window block +// window_dir: n_super self-contained blocks, each holding <=G window rows. +// per window row (T18 slim layout -- dd_off/freq_off/prx_off are NOT stored; +// the reader derives them as running prefix sums of the disk/prx lengths): +// VInt last_docid_delta # cumulative WITHIN the block => absolute last docid +// # (previous window's absolute last docid = win_base; +// # first window of first block: win_base = 0) +// VInt doc_count # number of docs in the window (frq_pod needs it) +// u8 win_mode # bit0 dd_zstd, bit1 freq_zstd +// VInt dd_disk_len # dd_region on-disk byte length +// [VInt dd_uncomp_len] # dd_region plaintext length; present ONLY when +// # win_mode & kDdZstd. A raw region's uncomp_len +// # == dd_disk_len (derived, not stored). +// u32 crc_dd # crc32c of the dd_region on-disk bytes +// VInt freq_disk_len # freq_region on-disk byte length (has_freq) +// [VInt freq_uncomp_len] # freq_region plaintext length; present ONLY when +// # has_freq && win_mode & kFreqZstd (raw: derived +// # == freq_disk_len). +// u32 crc_freq # crc32c of the freq_region on-disk bytes (has_freq) +// VInt prx_len # .prx payload byte length (present iff has_prx) +// VInt max_freq # window max term frequency (WAND block-max) +// u8 max_norm # window score-max norm (WAND); 0 acceptable +// +// The reader reconstructs each window's dd_off / freq_off (byte offset within the +// dd-block / freq-block) and prx_off (offset within the entry's .prx span) as the +// running prefix sums of dd_disk_len / freq_disk_len / prx_len over all windows, +// chained across super-blocks; WindowMeta still exposes those offsets, now derived. +// +// Reconstructing win_base / absolute last_docid (READER CONTRACT) is unchanged: +// the writer chains absolute last docids across windows; each row stores the delta +// of its absolute last docid from the previous window, and sb_last_docid seeds +// each block, so super-block binary search then in-block window binary search +// locate the window covering any docid without decoding the .frq blocks. +// +// The trailing crc32c covers only header + super_block_dir; every region carries +// its own crc (crc_dd / crc_freq) in the row. +namespace doris::snii::format { + +namespace frq_prelude_flags { +inline constexpr uint8_t kHasFreq = 1u << 0; +inline constexpr uint8_t kHasPrx = 1u << 1; +// Reserved extension point (T18): kSlimRows = 1u << 2 would gate the trimmed +// window-row layout (no stored dd_off/freq_off/prx_off, conditional uncomp_len) +// as a distinct on-disk path. It is NOT emitted today: the trim folds into the +// single pre-launch v1 encoding (writer/reader symmetric, no dual decode path). +// If a `lifecycle: launched` index appears before this lands, set this bit on the +// slim writer and branch the reader on it instead of unconditionally decoding slim. +} // namespace frq_prelude_flags + +// Per-window codec mode bits (win_mode byte). +namespace frq_win_mode { +inline constexpr uint8_t kDdZstd = 1u << 0; +inline constexpr uint8_t kFreqZstd = 1u << 1; +inline constexpr uint8_t kKnownBits = kDdZstd | kFreqZstd; +} // namespace frq_win_mode + +// Absolute, decoded metadata for one window (as the reader exposes it). The dd / +// freq region locators are offsets WITHIN the dd-block / freq-block respectively +// (both blocks follow the prelude). dd_off/freq_off/prx_off are DERIVED by the +// reader as running prefix sums of the disk/prx lengths (they are no longer stored +// per row; see the header layout note) -- these public members are unchanged and +// still populated, just by derivation. The reader derives the dd-block length from +// the last window's dd_off + dd_disk_len. +struct WindowMeta { + uint32_t last_docid = 0; // absolute last docid in the window + uint64_t win_base = 0; // absolute last docid of the previous window (0 for w==0) + uint32_t doc_count = 0; + + // dd_region locator (within the dd-block). + bool dd_zstd = false; + uint64_t dd_off = 0; // DERIVED: running sum of prior windows' dd_disk_len + uint64_t dd_disk_len = 0; + uint64_t dd_uncomp_len = 0; // DERIVED == dd_disk_len for raw; stored only when dd_zstd + uint32_t crc_dd = 0; + + // freq_region locator (within the freq-block); valid only when has_freq. + bool freq_zstd = false; + uint64_t freq_off = 0; // DERIVED: running sum of prior windows' freq_disk_len + uint64_t freq_disk_len = 0; + uint64_t freq_uncomp_len = 0; // DERIVED == freq_disk_len for raw; stored only when freq_zstd + uint32_t crc_freq = 0; + + uint64_t prx_off = 0; // valid only when has_prx; DERIVED: running sum of prior prx_len + uint64_t prx_len = 0; // valid only when has_prx + uint32_t max_freq = 0; + uint8_t max_norm = 0; + + // In-memory only (NOT serialized in the prelude row). When false, the dd/freq + // region decode skips crc verification -- used when these region bytes are + // covered by an enclosing crc (e.g. an INLINE entry inside its dict block). + // Windowed/slim POD-ref rows leave this true (their regions carry a crc). + bool verify_crc = true; +}; + +// Builder input: one fully-computed WindowMeta per window, in term order, plus the +// super-block grouping factor. The writer fills last_docid (absolute), doc_count, +// the region locators/crcs, prx locator, max_freq and max_norm; win_base is derived +// during build (so callers may leave it 0). group_size must be >= 1. +struct FrqPreludeColumns { + bool has_freq = true; + bool has_prx = false; + uint32_t group_size = 64; // windows per super-block (G) + std::vector windows; +}; + +// Builds the prelude bytes and appends them to out. +// Returns InvalidArgument when out is null, group_size is 0, or the windows are +// not in non-decreasing last_docid order (a window's absolute last docid must be +// >= the previous window's). +Status build_frq_prelude(const FrqPreludeColumns& cols, ByteSink* out); + +// Reads and verifies a prelude buffer, exposing two-level skip access. The reader +// parses the header + super_block_dir on open (verifying the trailing crc) and +// eagerly decodes every window block into owned WindowMeta rows (the prelude is +// small relative to the postings). It does not retain the input. +class FrqPreludeReader { +public: + // Parses + verifies the prelude. crc mismatch / truncation / inconsistent + // offsets-or-lengths / oversized counts => kCorruption. + static Status open(Slice prelude, FrqPreludeReader* out); + + uint32_t n_windows() const { return static_cast(windows_.size()); } + uint32_t n_super_blocks() const { return n_super_; } + bool has_freq() const { return has_freq_; } + bool has_prx() const { return has_prx_; } + + // Total on-disk byte length of the dd-block (== sum of dd_disk_len; the docs-only + // prefix after the prelude). 0 when there are no windows. + uint64_t dd_block_len() const { return dd_block_len_; } + // Total on-disk byte length of the freq-block (== sum of freq_disk_len). 0 when + // !has_freq or no windows. + uint64_t freq_block_len() const { return freq_block_len_; } + + // Returns the absolute WindowMeta for window w. Out-of-range => InvalidArgument. + Status window(uint32_t w, WindowMeta* out) const; + + // Locates the window covering docid via super-block binary search then window + // binary search. *found=false (with OK) when docid is past the term's last + // docid; otherwise *w is the index of the covering window (the first window + // whose absolute last_docid >= docid). + Status locate_window(uint32_t docid, bool* found, uint32_t* w) const; + + // Selects, as a monotonic two-pointer cursor, the ascending de-duplicated set of + // windows covering the ascending `candidates` (each window covering its + // (win_base, last_docid] span). Writes them to *windows (cleared first). The + // result is element-for-element identical to calling locate_window per candidate + // and collapsing equal runs, but uses O(C + N) window last_docid comparisons + // (C = candidates, N = windows) instead of O(C * group_size). Pure in-memory over + // the decoded directory; never fails. + void select_covering_windows(const std::vector& candidates, + std::vector* windows) const; + + // Packed absolute last_docid of window w (byte-identical to window(w).last_docid), + // exposed for the covering-window cursor's contiguous scan and equivalence tests. + uint32_t window_last_docid(uint32_t w) const { + DCHECK_LT(w, win_last_docid_.size()); + return win_last_docid_[w]; + } + +private: + bool has_freq_ = false; + bool has_prx_ = false; + uint32_t group_size_ = 1; + uint32_t n_super_ = 0; + uint64_t dd_block_len_ = 0; + uint64_t freq_block_len_ = 0; + // Absolute last docid at each super-block boundary (size n_super_). + std::vector sb_last_docid_; + // All windows decoded with absolute fields, in term order (size N). + std::vector windows_; + // Packed copy of each window's absolute last_docid (size N; win_last_docid_[w] == + // windows_[w].last_docid). Built in open() so the covering-window cursor scans a + // contiguous 4B/window array rather than the ~104B WindowMeta rows. In-memory only: + // never serialized; immutable after open() (same lifetime as windows_). + std::vector win_last_docid_; +}; + +// Pure cursor core (no FrqPreludeReader / IO): selects into *windows the ascending, +// de-duplicated indices of the windows covering the ascending `candidates`, given the +// packed window last_docid array (size n_windows), the super-block last_docid directory +// (size n_super) and group_size. A super-block cursor does boundary jumps while a window +// cursor advances forward only => O(C + N) window comparisons, element-for-element equal +// to per-candidate locate_window + run collapse. *windows is cleared first; n_windows == 0 +// yields an empty result. Exposed for isolated equivalence / complexity tests. +void select_covering_windows_cursor(const uint32_t* win_last_docid, uint32_t n_windows, + const uint64_t* sb_last_docid, uint32_t n_super, + uint32_t group_size, const std::vector& candidates, + std::vector* windows); + +// TEST-ONLY observability seam (mirrors the format dict-block decode counter). Counts the +// window last_docid comparisons performed by select_covering_windows_cursor and by +// locate_window's level-2 scan, so tests can assert the cursor stays O(C + N) and bounded +// by C + N regardless of group_size, while the legacy per-candidate scan grows with G. The +// counter is thread-local: race-free under the shared const reader and free of atomic cost +// in the production cursor loop; reset and read on the thread that ran the cursor. +namespace testing { +uint64_t window_probe_count(); +void reset_window_probe_count(); +} // namespace testing + +} // namespace doris::snii::format diff --git a/be/src/storage/index/snii/format/metadata_blob.cpp b/be/src/storage/index/snii/format/metadata_blob.cpp new file mode 100644 index 00000000000000..0e2483bd1cf076 --- /dev/null +++ b/be/src/storage/index/snii/format/metadata_blob.cpp @@ -0,0 +1,107 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/format/metadata_blob.h" + +#include "storage/index/snii/encoding/byte_source.h" +#include "storage/index/snii/encoding/section_framer.h" +#include "storage/index/snii/encoding/zstd_codec.h" + +namespace doris::snii::format { + +namespace { + +constexpr int kMetaSectionZstdLevel = 3; +constexpr uint64_t kMaxMetaSectionUncompBytes = 256ULL * 1024 * 1024; + +} // namespace + +Status encode_metadata_blob(Slice raw_frame, SectionType raw_type, SectionType compressed_type, + ByteSink* out) { + if (out == nullptr) { + return Status::Error("metadata_blob: null sink"); + } + + ByteSource source(raw_frame); + FramedSection raw_section; + RETURN_IF_ERROR(SectionFramer::read(source, &raw_section)); + if (!source.eof() || raw_section.type != static_cast(raw_type)) { + return Status::Error( + "metadata_blob: raw input is not exactly one frame of the expected type"); + } + + if (raw_frame.size() >= kMetaSectionCompressMinBytes) { + std::vector compressed; + if (zstd_compress(raw_frame, kMetaSectionZstdLevel, &compressed).ok()) { + ByteSink payload; + payload.put_varint64(raw_frame.size()); + payload.put_bytes(Slice(compressed)); + if (payload.size() + 16 < raw_frame.size()) { + SectionFramer::write(*out, static_cast(compressed_type), payload.view()); + return Status::OK(); + } + } + } + out->put_bytes(raw_frame); + return Status::OK(); +} + +Status materialize_metadata_blob(Slice stored_frame, SectionType raw_type, + SectionType compressed_type, std::vector* scratch, + Slice* raw_frame) { + if (scratch == nullptr || raw_frame == nullptr) { + return Status::Error("metadata_blob: null frame out"); + } + + ByteSource source(stored_frame); + FramedSection stored_section; + RETURN_IF_ERROR(SectionFramer::read(source, &stored_section)); + if (!source.eof()) { + return Status::Error( + "metadata_blob: trailing stored frame bytes"); + } + if (stored_section.type == static_cast(raw_type)) { + *raw_frame = stored_frame; + return Status::OK(); + } + if (stored_section.type != static_cast(compressed_type)) { + return Status::Error( + "metadata_blob: unexpected stored frame type"); + } + + ByteSource payload(stored_section.payload); + uint64_t uncomp_len = 0; + RETURN_IF_ERROR(payload.get_varint64(&uncomp_len)); + if (uncomp_len == 0 || uncomp_len > kMaxMetaSectionUncompBytes) { + return Status::Error( + "metadata_blob: zstd uncomp_len out of range"); + } + Slice compressed; + RETURN_IF_ERROR(payload.get_bytes(payload.remaining(), &compressed)); + RETURN_IF_ERROR(zstd_decompress(compressed, static_cast(uncomp_len), scratch)); + ByteSource raw_source {Slice(*scratch)}; + FramedSection raw_section; + RETURN_IF_ERROR(SectionFramer::read(raw_source, &raw_section)); + if (!raw_source.eof() || raw_section.type != static_cast(raw_type)) { + return Status::Error( + "metadata_blob: decompressed bytes are not exactly one expected raw frame"); + } + *raw_frame = Slice(*scratch); + return Status::OK(); +} + +} // namespace doris::snii::format diff --git a/be/src/storage/index/snii/format/metadata_blob.h b/be/src/storage/index/snii/format/metadata_blob.h new file mode 100644 index 00000000000000..5e098a797e08d2 --- /dev/null +++ b/be/src/storage/index/snii/format/metadata_blob.h @@ -0,0 +1,40 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include "common/status.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/encoding/byte_sink.h" +#include "storage/index/snii/format/format_constants.h" + +namespace doris::snii::format { + +// Appends a raw metadata frame or a zstd carrier for it. The carrier payload is +// varint64 raw-frame length followed by zstd(raw frame). +Status encode_metadata_blob(Slice raw_frame, SectionType raw_type, SectionType compressed_type, + ByteSink* out); + +// Returns a raw metadata frame from a raw frame or zstd carrier. A raw frame +// remains a view into stored_frame; a materialized carrier is a view into scratch. +Status materialize_metadata_blob(Slice stored_frame, SectionType raw_type, + SectionType compressed_type, std::vector* scratch, + Slice* raw_frame); + +} // namespace doris::snii::format diff --git a/be/src/storage/index/snii/format/metadata_directory.cpp b/be/src/storage/index/snii/format/metadata_directory.cpp new file mode 100644 index 00000000000000..01fe5dd6221cd3 --- /dev/null +++ b/be/src/storage/index/snii/format/metadata_directory.cpp @@ -0,0 +1,283 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/format/metadata_directory.h" + +#include +#include +#include +#include +#include +#include + +#include "gen_cpp/snii.pb.h" +#include "storage/index/snii/format/format_constants.h" + +namespace doris::snii::format { +namespace { + +Status metadata_directory_corrupted(std::string_view message) { + return Status::Error(message); +} + +Status metadata_directory_unsupported(std::string_view message) { + return Status::Error(message); +} + +Status decode_blob_ref(const doris::snii::SniiBlobRefPB& input, MetadataBlobRef* out) { + if (!input.has_offset() || !input.has_length()) { + return metadata_directory_corrupted("metadata directory: missing blob reference field"); + } + if (input.length() == 0) { + return metadata_directory_corrupted("metadata directory: empty mandatory blob reference"); + } + *out = {.offset = input.offset(), .length = input.length()}; + return Status::OK(); +} + +// Decodes one opaque blob sub-file. Unlike decode_blob_ref, length == 0 is +// LEGAL here: an empty BKD segment stores 0-byte `bkd` / `bkd_index` files. +Status decode_named_blob(const doris::snii::SniiNamedBlobPB& input, NamedBlobFileRef* out) { + if (!input.has_name() || !input.has_offset() || !input.has_length() || !input.has_crc32c()) { + return metadata_directory_corrupted("metadata directory: missing named blob field"); + } + if (input.name().empty()) { + return metadata_directory_corrupted("metadata directory: empty blob file name"); + } + if (input.length() > std::numeric_limits::max() - input.offset()) { + return metadata_directory_corrupted("metadata directory: blob file range overflows"); + } + out->name = input.name(); + out->offset = input.offset(); + out->length = input.length(); + out->crc32c = input.crc32c(); + return Status::OK(); +} + +// Rows 1/2 of the decode matrix: the original three-blob contract, unchanged. +Status decode_inverted_entry(const doris::snii::SniiLogicalIndexMetadataPB& index, + LogicalIndexMetadataRef* entry) { + if (!index.has_core_metadata() || !index.has_sampled_term_index() || + !index.has_dict_block_directory()) { + return metadata_directory_corrupted("metadata directory: missing required logical field"); + } + if (index.files_size() != 0) { + return metadata_directory_corrupted( + "metadata directory: inverted entry carries blob files"); + } + entry->kind = LogicalIndexKind::kInverted; + RETURN_IF_ERROR(decode_blob_ref(index.core_metadata(), &entry->core_metadata)); + RETURN_IF_ERROR(decode_blob_ref(index.sampled_term_index(), &entry->sampled_term_index)); + RETURN_IF_ERROR(decode_blob_ref(index.dict_block_directory(), &entry->dict_block_directory)); + return Status::OK(); +} + +// Rows 3/4 of the decode matrix: an opaque named-file table and nothing else. +Status decode_blob_entry(const doris::snii::SniiLogicalIndexMetadataPB& index, uint32_t kind_value, + LogicalIndexMetadataRef* entry) { + if (index.has_core_metadata() || index.has_sampled_term_index() || + index.has_dict_block_directory()) { + return metadata_directory_corrupted( + "metadata directory: blob entry carries inverted metadata"); + } + if (index.files_size() == 0) { + return metadata_directory_corrupted("metadata directory: blob entry has no files"); + } + entry->kind = static_cast(kind_value); + entry->files.reserve(index.files_size()); + for (const auto& file : index.files()) { + NamedBlobFileRef decoded; + RETURN_IF_ERROR(decode_named_blob(file, &decoded)); + for (const auto& existing : entry->files) { + if (existing.name == decoded.name) { + return metadata_directory_corrupted("metadata directory: duplicate blob file name"); + } + } + entry->files.push_back(std::move(decoded)); + } + return Status::OK(); +} + +// The decode validation matrix (design 2026-07-28 §3.1). Shared by the reader +// and by the encoder's self-check, so it constrains both sides at once. +Status decode_directory_pb(const doris::snii::SniiMetadataDirectoryPB& input, + std::vector* out) { + // Row 0a/0b: whitelist of known required features; anything else is a + // format this binary does not understand. + bool feature_blob = false; + for (const uint32_t feature : input.required_features()) { + if (feature == kFeatureBlobLogicalIndex) { + feature_blob = true; + continue; + } + return metadata_directory_unsupported( + "metadata directory: required feature is not supported"); + } + + bool has_blob_entry = false; + std::vector entries; + entries.reserve(input.indexes_size()); + for (const auto& index : input.indexes()) { + if (!index.has_index_id() || !index.has_index_suffix()) { + return metadata_directory_corrupted( + "metadata directory: missing required logical field"); + } + + LogicalIndexMetadataRef entry; + entry.index_id = index.index_id(); + entry.index_suffix = index.index_suffix(); + + const uint32_t kind_value = index.has_kind() + ? index.kind() + : static_cast(LogicalIndexKind::kInverted); + switch (kind_value) { + case static_cast(LogicalIndexKind::kInverted): + RETURN_IF_ERROR(decode_inverted_entry(index, &entry)); + break; + case static_cast(LogicalIndexKind::kBkd): + case static_cast(LogicalIndexKind::kAnn): + has_blob_entry = true; + RETURN_IF_ERROR(decode_blob_entry(index, kind_value, &entry)); + break; + default: + // Row 5: a kind this binary does not know how to open. + return metadata_directory_unsupported("metadata directory: unknown logical index kind"); + } + + // Row 6: keys are unique across kinds. + for (const auto& existing : entries) { + if (existing.index_id == entry.index_id && + existing.index_suffix == entry.index_suffix) { + return metadata_directory_corrupted( + "metadata directory: duplicate logical index key"); + } + } + entries.push_back(std::move(entry)); + } + + // The feature flag and the entries must agree in BOTH directions: a blob + // entry without the flag would make old binaries misreport Corruption + // instead of Unsupported; a flag without blob entries would make every + // pre-blob binary reject a file it could actually read. + if (has_blob_entry != feature_blob) { + return metadata_directory_corrupted( + "metadata directory: blob feature flag disagrees with entries"); + } + *out = std::move(entries); + return Status::OK(); +} + +void encode_blob_ref(const MetadataBlobRef& input, doris::snii::SniiBlobRefPB* out) { + out->set_offset(input.offset); + out->set_length(input.length); +} + +} // namespace + +Status MetadataDirectory::decode(Slice bytes, MetadataDirectory* out) { + if (out == nullptr) { + return Status::Error("metadata directory: null output"); + } + out->entries_.clear(); + if (bytes.size() > static_cast(std::numeric_limits::max())) { + return metadata_directory_corrupted("metadata directory: protobuf payload exceeds INT_MAX"); + } + + doris::snii::SniiMetadataDirectoryPB directory; + if (!directory.ParseFromArray(bytes.data(), static_cast(bytes.size()))) { + return metadata_directory_corrupted("metadata directory: protobuf parsing failed"); + } + + std::vector entries; + RETURN_IF_ERROR(decode_directory_pb(directory, &entries)); + out->entries_ = std::move(entries); + return Status::OK(); +} + +const LogicalIndexMetadataRef* MetadataDirectory::find(uint64_t index_id, + std::string_view suffix) const { + for (const auto& entry : entries_) { + if (entry.index_id == index_id && entry.index_suffix == suffix) { + return &entry; + } + } + return nullptr; +} + +Status encode_metadata_directory(const std::vector& entries, + ByteSink* out) { + if (out == nullptr) { + return Status::Error("metadata directory: null output"); + } + + doris::snii::SniiMetadataDirectoryPB directory; + bool any_blob = false; + for (const auto& entry : entries) { + auto* index = directory.add_indexes(); + index->set_index_id(entry.index_id); + index->set_index_suffix(entry.index_suffix); + if (entry.kind == LogicalIndexKind::kInverted) { + // BYTE GATE: never touch field 6/7 here -- proto2 presence + // semantics would serialize even set_kind(0) and change the bytes + // of every pure-text directory (golden digests included). + if (!entry.files.empty()) { + return metadata_directory_corrupted( + "metadata directory: inverted entry carries blob files"); + } + encode_blob_ref(entry.core_metadata, index->mutable_core_metadata()); + encode_blob_ref(entry.sampled_term_index, index->mutable_sampled_term_index()); + encode_blob_ref(entry.dict_block_directory, index->mutable_dict_block_directory()); + } else { + // The three inverted refs do not serialize for blob entries, so + // the shared decode self-check below could not catch a caller that + // filled them; reject that in-memory shape here. + if (entry.core_metadata.offset != 0 || entry.core_metadata.length != 0 || + entry.sampled_term_index.offset != 0 || entry.sampled_term_index.length != 0 || + entry.dict_block_directory.offset != 0 || entry.dict_block_directory.length != 0) { + return metadata_directory_corrupted( + "metadata directory: blob entry carries inverted metadata"); + } + any_blob = true; + index->set_kind(static_cast(entry.kind)); + for (const auto& file : entry.files) { + auto* named = index->add_files(); + named->set_name(file.name); + named->set_offset(file.offset); + named->set_length(file.length); + named->set_crc32c(file.crc32c); + } + } + } + if (any_blob) { + directory.add_required_features(kFeatureBlobLogicalIndex); + } + + std::vector validated; + RETURN_IF_ERROR(decode_directory_pb(directory, &validated)); + const size_t size = directory.ByteSizeLong(); + if (size > static_cast(std::numeric_limits::max())) { + return metadata_directory_corrupted("metadata directory: protobuf payload exceeds INT_MAX"); + } + std::string payload(size, '\0'); + if (!directory.SerializeToArray(payload.data(), static_cast(size))) { + return metadata_directory_corrupted("metadata directory: protobuf serialization failed"); + } + out->put_bytes(Slice(payload)); + return Status::OK(); +} + +} // namespace doris::snii::format diff --git a/be/src/storage/index/snii/format/metadata_directory.h b/be/src/storage/index/snii/format/metadata_directory.h new file mode 100644 index 00000000000000..6bbd4d7d57d91b --- /dev/null +++ b/be/src/storage/index/snii/format/metadata_directory.h @@ -0,0 +1,85 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/encoding/byte_sink.h" + +namespace doris::snii::format { + +struct MetadataBlobRef { + uint64_t offset = 0; + uint64_t length = 0; +}; + +// Directory entry kinds. kInverted keeps the original three-blob contract +// (Core/STI/DBD); blob kinds carry an opaque named-file table instead, so a +// non-text index (BKD / ANN) can live in the container without SNII knowing +// its internal encoding. +enum class LogicalIndexKind : uint32_t { + kInverted = 0, + kBkd = 1, + kAnn = 2, +}; + +// One opaque sub-file of a blob logical index. `offset` is an absolute +// container offset. length == 0 is LEGAL here (an empty BKD segment writes +// 0-byte `bkd` / `bkd_index` files) -- deliberately unlike MetadataBlobRef, +// whose zero length is Corruption. +struct NamedBlobFileRef { + std::string name; + uint64_t offset = 0; + uint64_t length = 0; + uint32_t crc32c = 0; +}; + +struct LogicalIndexMetadataRef { + uint64_t index_id = 0; + std::string index_suffix; + MetadataBlobRef core_metadata; // kInverted only + MetadataBlobRef sampled_term_index; // kInverted only + MetadataBlobRef dict_block_directory; // kInverted only + // Fields below extend the entry for blob kinds. They sit AFTER the + // original members so existing designated-initializer sites stay valid. + LogicalIndexKind kind = LogicalIndexKind::kInverted; + std::vector files; // blob kinds only +}; + +class MetadataDirectory { +public: + static Status decode(Slice bytes, MetadataDirectory* out); + + const LogicalIndexMetadataRef* find(uint64_t index_id, std::string_view suffix) const; + size_t size() const { return entries_.size(); } + const std::vector& entries() const { return entries_; } + +private: + std::vector entries_; +}; + +Status encode_metadata_directory(const std::vector& entries, + ByteSink* out); + +} // namespace doris::snii::format diff --git a/be/src/storage/index/snii/format/norms_pod.cpp b/be/src/storage/index/snii/format/norms_pod.cpp new file mode 100644 index 00000000000000..b3da050c17726f --- /dev/null +++ b/be/src/storage/index/snii/format/norms_pod.cpp @@ -0,0 +1,85 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/format/norms_pod.h" + +#include + +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/encoding/byte_source.h" +#include "storage/index/snii/encoding/section_framer.h" +#include "storage/index/snii/encoding/varint.h" +#include "storage/index/snii/format/format_constants.h" + +namespace doris::snii::format { + +void NormsPodWriter::finish(ByteSink* sink) const { + finish(norms_, sink); +} + +void NormsPodWriter::finish(std::span norms, ByteSink* sink) { + // Build inner payload: [varint64 doc_count][raw norm bytes]. + ByteSink payload; + const size_t payload_size = varint_len(norms.size()) + norms.size(); + payload.reserve(payload_size); + payload.put_varint64(norms.size()); + payload.put_bytes(Slice(norms.data(), norms.size())); + // Delegate outer framing to SectionFramer to append type+len+crc32c, avoiding manual checksum assembly. + sink->reserve(1 + varint_len(payload_size) + payload_size + sizeof(uint32_t)); + SectionFramer::write(*sink, static_cast(SectionType::kNormsPod), payload.view()); +} + +Status NormsPodReader::open(Slice framed, NormsPodReader* out) { + // framer handles CRC verify, truncation detection, and payload slicing. + ByteSource src(framed); + FramedSection sec; + RETURN_IF_ERROR(SectionFramer::read(src, &sec)); + if (sec.type != static_cast(SectionType::kNormsPod)) { + return Status::Error( + "norms POD section type mismatch"); + } + if (!src.eof()) { + return Status::Error( + "norms POD trailing framed bytes"); + } + + // Parse inner payload: [varint64 doc_count][bytes]. + ByteSource payload(sec.payload); + uint64_t doc_count = 0; + RETURN_IF_ERROR(payload.get_varint64(&doc_count)); + if (payload.position() != varint_len(doc_count)) { + return Status::Error( + "norms POD non-canonical doc_count"); + } + if (doc_count > std::numeric_limits::max()) { + return Status::Error( + "norms POD doc_count overflows uint32"); + } + // doc_count must exactly equal the remaining byte count (1 byte per doc). + if (payload.remaining() != doc_count) { + return Status::Error( + "norms POD length mismatch"); + } + + Slice bytes; + RETURN_IF_ERROR(payload.get_bytes(static_cast(doc_count), &bytes)); + out->doc_count_ = static_cast(doc_count); + out->norms_ = bytes.data(); + return Status::OK(); +} + +} // namespace doris::snii::format diff --git a/be/src/storage/index/snii/format/norms_pod.h b/be/src/storage/index/snii/format/norms_pod.h new file mode 100644 index 00000000000000..11791240615a65 --- /dev/null +++ b/be/src/storage/index/snii/format/norms_pod.h @@ -0,0 +1,89 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/encoding/byte_sink.h" + +namespace doris::snii::format { + +// norms POD: per logical index / field stores 1-byte encoded doc length per doc, +// used by BM25 length normalization (SniiStatsProvider::encoded_norm) for per-docid lookup. +// +// On-disk layout (the whole section is framed by SectionFramer, which adds a type+len+crc32c envelope): +// framer payload = [varint64 doc_count][bytes encoded_norm[doc_count]] +// framer envelope = [u8 type][varint64 payload_len][payload][fixed32 crc32c] +// The encoding of encoded_norm (length -> 1B) is out of scope for this module; here we only handle raw byte storage and retrieval. +class NormsPodWriter { +public: + // Appends the encoded_norm for the next docid (docid is implicit, assigned in append order starting from 0). + void add(uint8_t encoded_norm) { norms_.push_back(encoded_norm); } + + // Number of docs accumulated so far (i.e., the next docid to be assigned). + size_t count() const { return norms_.size(); } + + // Writes [doc_count][bytes] framed by SectionFramer into sink (appends; does not clear sink). + void finish(ByteSink* sink) const; + // Zero-copy source overload used by streamed compaction. + static void finish(std::span norms, ByteSink* sink); + +private: + std::vector norms_; +}; + +// Read-only view: on open, verifies the framer CRC and checks that doc_count/payload length are consistent, +// afterwards encoded_norm(docid) is O(1) direct indexing (zero-copy, borrows the underlying buffer). +class NormsPodReader { +public: + NormsPodReader() = default; + + // Parses the entire section (including the framer envelope). Returns Corruption on CRC mismatch, truncation, or length inconsistency. + // On success, *out borrows the memory pointed to by framer_payload; the caller must ensure its lifetime. + static Status open(Slice framed, NormsPodReader* out); + + uint32_t doc_count() const { return doc_count_; } + + // Precondition (hard contract): docid < doc_count(). Semantics match std::vector::operator[]: + // the caller is responsible for guaranteeing this (docid comes from trusted postings decoded internally by SNII). Asserts in debug builds; + // no check in Release (NDEBUG). Use try_encoded_norm when the docid is untrusted and needs validation. + uint8_t encoded_norm(uint32_t docid) const { + assert(docid < doc_count_); + return norms_[docid]; + } + + // Checked access: returns InvalidArgument if docid is out of range; never reads out-of-range memory. + Status try_encoded_norm(uint32_t docid, uint8_t* out) const { + if (docid >= doc_count_) + return Status::Error("norms: docid out of range"); + *out = norms_[docid]; + return Status::OK(); + } + +private: + const uint8_t* norms_ = nullptr; + uint32_t doc_count_ = 0; +}; + +} // namespace doris::snii::format diff --git a/be/src/storage/index/snii/format/null_bitmap.cpp b/be/src/storage/index/snii/format/null_bitmap.cpp new file mode 100644 index 00000000000000..9e7d0d298f99f8 --- /dev/null +++ b/be/src/storage/index/snii/format/null_bitmap.cpp @@ -0,0 +1,295 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/format/null_bitmap.h" + +#include +#include +#include +#include + +#include "common/check.h" +// clang-format off +// CRoaring's public header defines ROARING_CONTAINER_T; its internal headers undefine it. +#include "roaring/roaring.hh" +#include "roaring/containers/array.h" +#include "roaring/containers/bitset.h" +#include "roaring/containers/run.h" +// clang-format on +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/encoding/byte_source.h" +#include "storage/index/snii/encoding/section_framer.h" +#include "storage/index/snii/encoding/varint.h" + +namespace doris::snii::format { + +namespace { + +constexpr uint32_t kPortableCookieNoRun = 12346; +constexpr uint16_t kPortableCookieRun = 12347; +constexpr uint32_t kMaxPortableContainers = uint32_t {1} << 16; + +struct ParsedNullBitmap { + uint32_t doc_count = 0; + Slice roaring_bytes; + uint32_t container_count = 0; +}; + +Status parse_null_bitmap(Slice framed, ParsedNullBitmap* out) { + ByteSource src(framed); + FramedSection sec; + RETURN_IF_ERROR(SectionFramer::read(src, &sec)); + if (sec.type != kNullBitmapSectionType || src.remaining() != 0) { + return Status::Error( + "null bitmap: invalid framed section"); + } + + ByteSource payload(sec.payload); + uint64_t doc_count = 0; + RETURN_IF_ERROR(payload.get_varint64(&doc_count)); + if (doc_count > std::numeric_limits::max()) { + return Status::Error( + "null bitmap doc_count overflows uint32"); + } + + uint64_t roaring_size = 0; + RETURN_IF_ERROR(payload.get_varint64(&roaring_size)); + if (roaring_size != payload.remaining()) { + return Status::Error( + "null bitmap roaring_size differs from payload"); + } + RETURN_IF_ERROR(payload.get_bytes(static_cast(roaring_size), &out->roaring_bytes)); + + ByteSource portable(out->roaring_bytes); + uint32_t cookie = 0; + RETURN_IF_ERROR(portable.get_fixed32(&cookie)); + if (static_cast(cookie) == kPortableCookieRun) { + out->container_count = (cookie >> 16) + 1; + } else if (cookie == kPortableCookieNoRun) { + RETURN_IF_ERROR(portable.get_fixed32(&out->container_count)); + if (out->container_count > kMaxPortableContainers) { + return Status::Error( + "null bitmap: portable container count out of range"); + } + } else { + return Status::Error( + "null bitmap: invalid portable cookie"); + } + + const char* data = reinterpret_cast(out->roaring_bytes.data()); + const size_t size = out->roaring_bytes.size(); + const size_t probed = roaring::api::roaring_bitmap_portable_deserialize_size(data, size); + if (probed == 0 || probed != size) { + return Status::Error( + "null bitmap: malformed roaring container"); + } + out->doc_count = static_cast(doc_count); + return Status::OK(); +} + +} // namespace + +NullBitmapWriter:: + NullBitmapWriter() // NOLINT(modernize-use-equals-default): roaring type is incomplete in the header. + : bitmap_(std::make_unique()) {} + +NullBitmapWriter::~NullBitmapWriter() = default; + +void NullBitmapWriter::add_null(uint32_t docid) { + bitmap_->add(docid); +} + +void NullBitmapWriter::add_many(std::span docids) { + bitmap_->addMany(docids.size(), docids.data()); +} + +uint32_t NullBitmapWriter::null_count() const { + return static_cast(bitmap_->cardinality()); +} + +uint64_t NullBitmapWriter::build_memory_upper_bound(std::span sorted_docids) { + if (sorted_docids.empty()) { + return 0; + } + + uint64_t container_count = 0; + uint64_t sparse_container_count = 0; + uint64_t sparse_value_count = 0; + uint64_t dense_container_count = 0; + size_t begin = 0; + while (begin < sorted_docids.size()) { + const uint32_t key = sorted_docids[begin] >> 16; + size_t end = begin + 1; + while (end < sorted_docids.size() && sorted_docids[end] >> 16 == key) { + DCHECK_GT(sorted_docids[end], sorted_docids[end - 1]); + ++end; + } + const uint64_t cardinality = end - begin; + ++container_count; + if (cardinality <= roaring::internal::DEFAULT_MAX_SIZE) { + ++sparse_container_count; + sparse_value_count += cardinality; + } else { + ++dense_container_count; + } + begin = end; + } + + // roaring_array_t uses three parallel arrays. Account a minimum allocation + // of four slots and old+replacement overlap at geometric growth. + constexpr uint64_t kTopEntryBytes = sizeof(void*) + sizeof(uint16_t) + sizeof(uint8_t); + const uint64_t top_capacity = std::max(container_count, 4); + const uint64_t top_array_peak = 3 * top_capacity * kTopEntryBytes; + + // Sparse array growth can retain the old uint16 array while allocating its + // replacement. Eight bytes per live value covers both capacity slack and + // replacement overlap. A dense container additionally covers the largest + // geometric array capacity immediately before conversion and the new 8 KiB + // bitset while both allocations are live. + constexpr uint64_t kSparseValuePeakBytes = 8; + constexpr uint64_t kBitsetBytes = + roaring::internal::BITSET_CONTAINER_SIZE_IN_WORDS * sizeof(uint64_t); + constexpr uint64_t kDenseArrayConversionCapacity = roaring::internal::DEFAULT_MAX_SIZE * 5 / 4; + constexpr uint64_t kDenseArrayConversionBytes = + kDenseArrayConversionCapacity * sizeof(uint16_t); + const uint64_t sparse_peak = + sparse_value_count * kSparseValuePeakBytes + + sparse_container_count * sizeof(roaring::internal::array_container_t); + const uint64_t dense_peak = + dense_container_count * (kDenseArrayConversionBytes + kBitsetBytes + + sizeof(roaring::internal::array_container_t) + + sizeof(roaring::internal::bitset_container_t)); + return sizeof(roaring::Roaring) + top_array_peak + sparse_peak + dense_peak; +} + +Status NullBitmapWriter::serialization_sizes(uint32_t doc_count, + NullBitmapSerializationSizes* out) const { + if (out == nullptr) { + return Status::Error( + "null bitmap: null serialization size output"); + } + const size_t roaring_bytes = bitmap_->getSizeInBytes(); + const size_t prefix_bytes = varint_len(doc_count) + varint_len(roaring_bytes); + if (roaring_bytes > std::numeric_limits::max() - prefix_bytes) { + return Status::Error( + "null bitmap: payload size overflows"); + } + const size_t payload_bytes = prefix_bytes + roaring_bytes; + const size_t envelope_bytes = 1 + varint_len(payload_bytes) + sizeof(uint32_t); + if (payload_bytes > std::numeric_limits::max() - envelope_bytes) { + return Status::Error( + "null bitmap: framed size overflows"); + } + *out = {.roaring_bytes = roaring_bytes, + .payload_bytes = payload_bytes, + .framed_bytes = envelope_bytes + payload_bytes}; + return Status::OK(); +} + +Status NullBitmapWriter::finish(uint32_t doc_count, ByteSink* sink) const { + if (sink == nullptr) { + return Status::Error("null bitmap: null output sink"); + } + NullBitmapSerializationSizes sizes; + RETURN_IF_ERROR(serialization_sizes(doc_count, &sizes)); + + // Serialize the Roaring bitmap to its portable on-disk form. + std::vector roaring_buf(sizes.roaring_bytes); + bitmap_->write(roaring_buf.data()); + + // Build inner payload: [varint64 doc_count][varint64 roaring_size][bytes]. + ByteSink payload; + payload.reserve(sizes.payload_bytes); + payload.put_varint64(doc_count); + payload.put_varint64(sizes.roaring_bytes); + payload.put_bytes( + Slice(reinterpret_cast(roaring_buf.data()), sizes.roaring_bytes)); + DORIS_CHECK_EQ(payload.size(), sizes.payload_bytes); + + // Delegate the type + len + crc32c envelope to SectionFramer. + const size_t start = sink->size(); + sink->reserve(sizes.framed_bytes); + SectionFramer::write(*sink, kNullBitmapSectionType, payload.view()); + DORIS_CHECK_EQ(sink->size() - start, sizes.framed_bytes); + return Status::OK(); +} + +NullBitmapReader:: + NullBitmapReader() // NOLINT(modernize-use-equals-default): roaring type is incomplete in the header. + : bitmap_(std::make_unique()) {} + +NullBitmapReader::~NullBitmapReader() = default; + +NullBitmapReader::NullBitmapReader(NullBitmapReader&&) noexcept = default; +NullBitmapReader& NullBitmapReader::operator=(NullBitmapReader&&) noexcept = default; + +Status NullBitmapReader::open(Slice framed, NullBitmapReader* out) { + ParsedNullBitmap parsed; + RETURN_IF_ERROR(parse_null_bitmap(framed, &parsed)); + *out->bitmap_ = + roaring::Roaring::readSafe(reinterpret_cast(parsed.roaring_bytes.data()), + parsed.roaring_bytes.size()); + out->doc_count_ = parsed.doc_count; + return Status::OK(); +} + +Status NullBitmapReader::decoded_memory_bytes(Slice framed, uint64_t* out) { + if (out == nullptr) { + return Status::Error( + "null bitmap: null decoded memory output"); + } + ParsedNullBitmap parsed; + RETURN_IF_ERROR(parse_null_bitmap(framed, &parsed)); + + constexpr uint64_t kContainerObjectBytes = + std::max({sizeof(roaring::internal::array_container_t), + sizeof(roaring::internal::bitset_container_t), + sizeof(roaring::internal::run_container_t)}); + constexpr uint64_t kContainerMetadataBytes = + sizeof(void*) + sizeof(uint16_t) + sizeof(uint8_t) + kContainerObjectBytes; + constexpr uint64_t kFixedBytes = + sizeof(roaring::Roaring) + sizeof(roaring::api::roaring_bitmap_t); + const uint64_t container_bytes = + static_cast(parsed.container_count) * kContainerMetadataBytes; + if (parsed.roaring_bytes.size() > + std::numeric_limits::max() - container_bytes - kFixedBytes) { + return Status::Error( + "null bitmap: decoded memory size overflows"); + } + *out = parsed.roaring_bytes.size() + container_bytes + kFixedBytes; + return Status::OK(); +} + +bool NullBitmapReader::is_null(uint32_t docid) const { + return bitmap_->contains(docid); +} + +uint32_t NullBitmapReader::null_count() const { + return static_cast(bitmap_->cardinality()); +} + +void NullBitmapReader::copy_to(roaring::Roaring* out) const { + *out = *bitmap_; +} + +void NullBitmapReader::append_docids(std::vector& out) const { + for (uint32_t docid : *bitmap_) { + out.push_back(docid); + } +} + +} // namespace doris::snii::format diff --git a/be/src/storage/index/snii/format/null_bitmap.h b/be/src/storage/index/snii/format/null_bitmap.h new file mode 100644 index 00000000000000..a975a76d399d5d --- /dev/null +++ b/be/src/storage/index/snii/format/null_bitmap.h @@ -0,0 +1,131 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/encoding/byte_sink.h" + +// Forward-declare the CRoaring C++ bitmap so this header stays free of the +// (large) roaring include; the concrete type is only needed in the .cpp. +namespace roaring { +class Roaring; +} // namespace roaring + +namespace doris::snii::format { + +// SectionFramer type byte for the null-bitmap POD. There is no dedicated +// SectionType enum value yet, so we use a documented literal (0x20) outside the +// currently allocated enum range (1..9) to avoid colliding with existing types. +inline constexpr uint8_t kNullBitmapSectionType = 0x20; + +struct NullBitmapSerializationSizes { + size_t roaring_bytes = 0; + size_t payload_bytes = 0; + size_t framed_bytes = 0; +}; + +// NullBitmap POD: per logical index, a Roaring bitmap of null docids (docs whose +// value is NULL / not indexed). It decouples per-doc NULL information from the +// per-term dictionary / postings so NULL handling can pull only this side POD. +// +// On-disk layout (the whole section is framed by SectionFramer, which adds a +// type + varint64 len + payload + fixed32 crc32c envelope): +// framer payload = [varint64 doc_count][varint64 roaring_size][roaring_bytes] +// roaring_bytes is the portable CRoaring serialization (Roaring::write). +class NullBitmapWriter { +public: + NullBitmapWriter(); + ~NullBitmapWriter(); + + NullBitmapWriter(const NullBitmapWriter&) = delete; + NullBitmapWriter& operator=(const NullBitmapWriter&) = delete; + + // Marks docid as NULL (adding the same docid twice is idempotent). + void add_null(uint32_t docid); + void add_many(std::span docids); + + // Number of distinct null docids accumulated so far. + uint32_t null_count() const; + + // Conservative pre-allocation charge for constructing CRoaring from sorted + // docids. It includes top-level array growth and array-to-bitset conversion + // overlap, so the caller must retain this charge until the bitmap is destroyed. + static uint64_t build_memory_upper_bound(std::span sorted_docids); + + Status serialization_sizes(uint32_t doc_count, NullBitmapSerializationSizes* out) const; + + // Serializes [doc_count][roaring_size][roaring_bytes] framed by SectionFramer + // and appends it to sink (does not clear sink). doc_count is the total number + // of docs in the logical index (recorded so the reader can round-trip it). + Status finish(uint32_t doc_count, ByteSink* sink) const; + +private: + std::unique_ptr bitmap_; +}; + +// Read-only view: on open, SectionFramer verifies the CRC and truncation; this +// class then guards roaring_size against the remaining payload bytes before +// deserializing the Roaring bitmap (anti-DoS), so a corrupt size cannot trigger +// an oversized allocation/read. is_null() is then an O(1) membership test. +class NullBitmapReader { +public: + NullBitmapReader(); + ~NullBitmapReader(); + + NullBitmapReader(const NullBitmapReader&) = delete; + NullBitmapReader& operator=(const NullBitmapReader&) = delete; + NullBitmapReader(NullBitmapReader&&) noexcept; + NullBitmapReader& operator=(NullBitmapReader&&) noexcept; + + // Parses the entire section (framer envelope + payload). Returns Corruption on + // CRC mismatch, truncation, doc_count overflow, or an oversized roaring_size. + static Status open(Slice framed, NullBitmapReader* out); + + // Exact pre-allocation charge for CRoaring::readSafe, excluding the caller's + // framed input bytes and decoded docid output vector. + static Status decoded_memory_bytes(Slice framed, uint64_t* out); + + // True iff docid was marked NULL. docids outside the null set (including those + // >= doc_count) return false. + bool is_null(uint32_t docid) const; + + // Number of distinct null docids in the bitmap. + uint32_t null_count() const; + + // Copies the decoded bitmap into the caller-owned Roaring object. + void copy_to(roaring::Roaring* out) const; + + // Appends decoded docids in ascending order without cloning the bitmap. + void append_docids(std::vector& out) const; + + // Total doc count of the logical index, as recorded by the writer. + uint32_t doc_count() const { return doc_count_; } + +private: + std::unique_ptr bitmap_; + uint32_t doc_count_ = 0; +}; + +} // namespace doris::snii::format diff --git a/be/src/storage/index/snii/format/phrase_bigram.h b/be/src/storage/index/snii/format/phrase_bigram.h new file mode 100644 index 00000000000000..3d88f897e9bd6b --- /dev/null +++ b/be/src/storage/index/snii/format/phrase_bigram.h @@ -0,0 +1,33 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +namespace doris::snii::format { + +inline constexpr std::string_view kPhraseBigramTermMarker = + "\x1F" + "SNII_PHRASE_BIGRAM" + "\x1F"; + +inline bool is_phrase_bigram_term(std::string_view term) { + return term.starts_with(kPhraseBigramTermMarker); +} + +} // namespace doris::snii::format diff --git a/be/src/storage/index/snii/format/prx_decode_stats.h b/be/src/storage/index/snii/format/prx_decode_stats.h new file mode 100644 index 00000000000000..a77e0157f1e687 --- /dev/null +++ b/be/src/storage/index/snii/format/prx_decode_stats.h @@ -0,0 +1,107 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "common/status.h" + +namespace doris::snii::format { + +// Optional allocation seam for full CSR decode callers that must gate every +// retained/output allocation before touching the physical buffers. Query +// decoders leave this null; native compaction supplies its shared-budget +// workspace. Implementations must leave both CSR buffers unchanged when a +// reservation fails. +class PrxCsrAllocationGate { +public: + virtual ~PrxCsrAllocationGate() = default; + + virtual Status reserve_csr(std::vector* pos_flat, size_t position_count, + std::vector* pos_off, size_t offset_count) = 0; + virtual Status reserve_decompression(size_t bytes, std::vector** buffer) = 0; +}; + +struct PrxDecodeStats { + uint64_t raw_frames = 0; + uint64_t zstd_frames = 0; + uint64_t pfor_frames = 0; + uint64_t plaintext_bytes = 0; + uint64_t total_docs = 0; + uint64_t selected_docs = 0; + uint64_t total_positions = 0; + uint64_t selected_positions = 0; + uint64_t fetch_ns = 0; + // Inclusive successful-frame time: header/CRC validation, optional + // decompression, and payload decode. + uint64_t decode_ns = 0; + // Phrase verification excluding only the inclusive decode_ns delta. + uint64_t phrase_verify_ns = 0; + + void merge(const PrxDecodeStats& other); + [[nodiscard]] uint64_t frame_count() const { return raw_frames + zstd_frames + pfor_frames; } + [[nodiscard]] bool is_valid() const { + return selected_docs <= total_docs && selected_positions <= total_positions; + } + bool operator==(const PrxDecodeStats&) const = default; +}; + +struct PrxDecodedShape { + uint32_t total_docs = 0; + uint64_t total_positions = 0; + uint32_t max_frequency = 0; + bool has_zero_frequency = false; +}; + +// Query-plan and matcher calibration inputs are deliberately separate from PrxDecodeStats: the +// latter's 11 production counters remain a stable decode contract. +struct PhraseQueryExecutionStats { + uint64_t exact_candidate_docs = 0; + uint64_t exact_candidate_visits = 0; + uint64_t prx_streaming_frames = 0; + uint64_t prefix_leading_candidate_docs = 0; + uint64_t prefix_tail_candidate_visits = 0; + uint64_t common_grams_candidate_queries = 0; + uint64_t common_grams_plain_plans = 0; + uint64_t common_grams_gram_plans = 0; + uint64_t common_grams_fallback_no_gram = 0; + uint64_t common_grams_fallback_incompatible = 0; + uint64_t common_grams_fallback_kill_switch = 0; + uint64_t common_grams_fallback_cost = 0; + uint64_t common_grams_fallback_base_analyzer_mismatch = 0; + uint64_t common_grams_fallback_prefix_tail_empty = 0; + uint64_t common_grams_authoritative_empty = 0; + uint64_t common_grams_plain_posting_bytes = 0; + uint64_t common_grams_gram_posting_bytes = 0; + uint64_t common_grams_plain_estimated_candidate_df = 0; + uint64_t common_grams_gram_estimated_candidate_df = 0; + uint64_t common_grams_plain_estimated_cost = 0; + uint64_t common_grams_gram_estimated_cost = 0; + uint64_t common_grams_planning_ns = 0; +}; + +struct PrxDecodeContext { + PrxDecodeStats* stats = nullptr; + PrxDecodedShape* shape = nullptr; + PhraseQueryExecutionStats* query_stats = nullptr; + PrxCsrAllocationGate* allocation_gate = nullptr; +}; + +} // namespace doris::snii::format diff --git a/be/src/storage/index/snii/format/prx_frame.cpp b/be/src/storage/index/snii/format/prx_frame.cpp new file mode 100644 index 00000000000000..7b7e94f14278fe --- /dev/null +++ b/be/src/storage/index/snii/format/prx_frame.cpp @@ -0,0 +1,58 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/format/prx_frame.h" + +#include "storage/index/snii/encoding/byte_source.h" +#include "storage/index/snii/encoding/crc32c.h" +#include "storage/index/snii/format/prx_pod.h" + +namespace doris::snii::format { + +Status read_prx_frame(ByteSource* source, PrxFrameView* frame) { + const size_t start = source->position(); + uint8_t codec = 0; + RETURN_IF_ERROR(source->get_u8(&codec)); + if (codec != static_cast(PrxCodec::kRaw) && + codec != static_cast(PrxCodec::kZstd) && + codec != static_cast(PrxCodec::kPfor)) { + return Status::Error("prx: unknown codec"); + } + frame->codec = static_cast(codec); + RETURN_IF_ERROR(source->get_varint32(&frame->uncompressed_length)); + if (frame->uncompressed_length > kReaderPrxWindowLimits.max_uncomp_bytes) { + return Status::Error( + "prx: uncomp_len exceeds sane window cap"); + } + size_t payload_length = frame->uncompressed_length; + if (frame->codec == PrxCodec::kZstd) { + uint32_t compressed_length = 0; + RETURN_IF_ERROR(source->get_varint32(&compressed_length)); + payload_length = compressed_length; + } + RETURN_IF_ERROR(source->get_bytes(payload_length, &frame->payload)); + const size_t framed_length = source->position() - start; + uint32_t stored_crc = 0; + RETURN_IF_ERROR(source->get_fixed32(&stored_crc)); + if (crc32c(source->slice_from(start, framed_length)) != stored_crc) { + return Status::Error( + "prx: window crc mismatch"); + } + return Status::OK(); +} + +} // namespace doris::snii::format diff --git a/be/src/storage/index/snii/format/prx_frame.h b/be/src/storage/index/snii/format/prx_frame.h new file mode 100644 index 00000000000000..038f8175f2a7b2 --- /dev/null +++ b/be/src/storage/index/snii/format/prx_frame.h @@ -0,0 +1,41 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include "common/status.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/format/format_constants.h" + +namespace doris::snii { + +class ByteSource; + +namespace format { + +struct PrxFrameView { + PrxCodec codec = PrxCodec::kRaw; + uint32_t uncompressed_length = 0; + Slice payload; +}; + +Status read_prx_frame(ByteSource* source, PrxFrameView* frame); + +} // namespace format +} // namespace doris::snii diff --git a/be/src/storage/index/snii/format/prx_pod.cpp b/be/src/storage/index/snii/format/prx_pod.cpp new file mode 100644 index 00000000000000..99cfe498486dcb --- /dev/null +++ b/be/src/storage/index/snii/format/prx_pod.cpp @@ -0,0 +1,1417 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/format/prx_pod.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/encoding/byte_source.h" +#include "storage/index/snii/encoding/crc32c.h" +#include "storage/index/snii/encoding/pfor.h" +#include "storage/index/snii/encoding/zstd_codec.h" +#include "storage/index/snii/format/format_constants.h" +#include "storage/index/snii/format/prx_frame.h" + +namespace doris::snii::format { + +void PrxDecodeStats::merge(const PrxDecodeStats& other) { + raw_frames += other.raw_frames; + zstd_frames += other.zstd_frames; + pfor_frames += other.pfor_frames; + plaintext_bytes += other.plaintext_bytes; + total_docs += other.total_docs; + selected_docs += other.selected_docs; + total_positions += other.total_positions; + selected_positions += other.selected_positions; + fetch_ns += other.fetch_ns; + decode_ns += other.decode_ns; + phrase_verify_ns += other.phrase_verify_ns; +} + +Status validate_prx_window_limits(const PrxWindowLimits& limits) { + if (limits.max_docs == 0 || limits.max_positions == 0 || limits.max_uncomp_bytes == 0) { + return Status::Error( + "prx: window limits must be non-zero"); + } + if (limits.max_docs > kReaderPrxWindowLimits.max_docs || + limits.max_positions > kReaderPrxWindowLimits.max_positions || + limits.max_uncomp_bytes > kReaderPrxWindowLimits.max_uncomp_bytes) { + return Status::Error( + "prx: writer window limits exceed reader limits"); + } + return Status::OK(); +} + +namespace { + +using PrxClock = std::chrono::steady_clock; + +PrxClock::time_point prx_clock_now() { +#ifdef BE_TEST + testing::note_prx_clock_read(); +#endif + return PrxClock::now(); +} + +uint64_t elapsed_ns(PrxClock::time_point start) { + const auto elapsed = + std::chrono::duration_cast(prx_clock_now() - start).count(); + return std::max(1, static_cast(elapsed)); +} + +// Auto-compression threshold: use raw when payload is smaller than this (zstd +// gain is negligible and metadata overhead is relatively large). +inline constexpr size_t kPrxPodAutoZstdMinBytes = 512; +// Default zstd level in auto mode. +inline constexpr int kPrxPodDefaultZstdLevel = 3; +// Anti-DoS cap on position count decoded from a single window before +// allocation. +inline constexpr uint32_t kMaxWindowPositions = + kReaderPrxWindowLimits.max_positions; // 64M positions/window +// Anti-DoS cap on doc count decoded from a single window before allocation. A +// corrupt doc_count is otherwise fed straight to assign()/reserve() -> +// bad_alloc. +inline constexpr uint32_t kPrxPodMaxWindowDocs = kReaderPrxWindowLimits.max_docs; // 16M docs/window + +// Writer-side precondition for the FLAT builders: the per-doc partition `freqs` +// must address exactly the positions present in `flat`. If sum(freqs) overruns +// flat.size() a (positions_flat, freqs) mismatch would index flat[off+i] past +// the span end -- an out-of-bounds read on caller-supplied data. Reject it as +// InvalidArgument BEFORE any indexing so the bug surfaces as a clean Status, +// never UB. (sum < size leaves trailing positions unused, which is also a +// writer bug, so we require exact equality.) Uint64 accumulation cannot +// overflow for uint32 freqs. +Status check_flat_partition(std::span flat, std::span freqs) { + size_t sum = 0; + for (uint32_t fc : freqs) { + if (fc > flat.size() - sum) { + return Status::Error( + "prx: sum(freqs) exceeds positions_flat size"); + } + sum += fc; + } + if (sum != flat.size()) { + return Status::Error( + "prx: sum(freqs) does not match positions_flat size"); + } + return Status::OK(); +} + +Status validate_flat_positions(std::span flat, std::span freqs) { + size_t off = 0; + for (uint32_t fc : freqs) { + uint32_t previous = 0; + for (uint32_t i = 0; i < fc; ++i) { + const uint32_t position = flat[off + i]; + if (i != 0 && position < previous) { + return Status::Error( + "prx: positions within a doc must be ascending"); + } + previous = position; + } + off += fc; + } + return Status::OK(); +} + +Status validate_per_doc_window(std::span> per_doc, + const PrxWindowLimits& limits, size_t* total_positions) { + RETURN_IF_ERROR(validate_prx_window_limits(limits)); + if (per_doc.size() > limits.max_docs) { + return Status::Error( + "prx: doc count exceeds writer window limit"); + } + uint64_t total = 0; + for (const auto& positions : per_doc) { + total += positions.size(); + if (total > limits.max_positions) { + return Status::Error( + "prx: position count exceeds writer window limit"); + } + } + *total_positions = static_cast(total); + return Status::OK(); +} + +// Encode per-doc position lists into a self-describing plain payload (doc_count +// + per-doc delta stream). +Status encode_payload(std::span> per_doc, ByteSink* out) { + out->put_varint32(static_cast(per_doc.size())); + for (const auto& doc : per_doc) { + out->put_varint32(static_cast(doc.size())); + uint32_t prev = 0; + for (size_t i = 0; i < doc.size(); ++i) { + uint32_t pos = doc[i]; + if (i > 0 && pos < prev) { + return Status::Error( + "prx: positions within a doc must be ascending"); + } + out->put_varint32(i == 0 ? pos : pos - prev); + prev = pos; + } + } + return Status::OK(); +} + +// FLAT-positions encoder: identical wire output to encode_payload above, but +// reads positions from a single flat span partitioned per-doc by `freqs` (doc d +// owns the next freqs[d] entries). The public entry point has already validated +// that sum(freqs) == flat.size(). This avoids materializing a vector-of-vectors +// for the window. +Status encode_payload_flat(std::span flat, std::span freqs, + ByteSink* out) { + out->put_varint32(static_cast(freqs.size())); + size_t off = 0; + for (uint32_t fc : freqs) { + out->put_varint32(fc); + uint32_t prev = 0; + for (uint32_t i = 0; i < fc; ++i) { + const uint32_t pos = flat[off + i]; + if (i > 0 && pos < prev) { + return Status::Error( + "prx: positions within a doc must be ascending"); + } + out->put_varint32(i == 0 ? pos : pos - prev); + prev = pos; + } + off += fc; + } + return Status::OK(); +} + +// Encode a uint32 array into PFOR runs of kFrqBaseUnit (256) elements each. The +// run count is derived by the decoder from the total length, so it is not +// stored. +void prx_pod_encode_pfor_runs(std::span values, ByteSink* out) { + const size_t n = values.size(); + for (size_t off = 0; off < n; off += kFrqBaseUnit) { + const size_t run = (n - off < kFrqBaseUnit) ? (n - off) : kFrqBaseUnit; + pfor_encode(values.data() + off, run, out); + } +} + +// Decode n uint32 values (multiple PFOR runs of kFrqBaseUnit each) into out. +Status prx_pod_decode_pfor_runs(ByteSource* src, size_t n, std::vector* out) { + out->resize(n); + for (size_t off = 0; off < n; off += kFrqBaseUnit) { + const size_t run = (n - off < kFrqBaseUnit) ? (n - off) : kFrqBaseUnit; + RETURN_IF_ERROR(pfor_decode(src, run, out->data() + off)); + } + return Status::OK(); +} + +size_t varint32_size(uint32_t value) { + size_t bytes = 1; + while (value >= 128) { + value >>= 7; + ++bytes; + } + return bytes; +} + +// Derive the per-doc position deltas ONCE into `deltas` (flat, in doc order: the +// first position of each doc is absolute, the rest are deltas within the doc), +// enforcing the ascending-position precondition after the public entry point +// validated the exact (flat, freqs) partition. The loop is identical to the delta +// derivation the old encode_pfor_payload_flat ran inline, lifted out so the auto +// path can feed BOTH the PFOR payload and (only when needed) the raw plaintext +// payload from one buffer instead of walking `flat` twice. Accumulate the exact +// raw payload size in the same pass so codec selection does not rescan deltas. +Status compute_flat_deltas(std::span flat, std::span freqs, + std::vector* const deltas, size_t* const plain_size) { +#ifdef BE_TEST + testing::note_prx_delta_materialization(); +#endif + deltas->clear(); + deltas->reserve(flat.size()); + *plain_size = varint32_size(static_cast(freqs.size())); + size_t off = 0; + for (uint32_t fc : freqs) { + *plain_size += varint32_size(fc); + uint32_t prev = 0; + for (uint32_t i = 0; i < fc; ++i) { + const uint32_t pos = flat[off + i]; + if (i > 0 && pos < prev) { + return Status::Error( + "prx: positions within a doc must be ascending"); + } + const uint32_t delta = i == 0 ? pos : pos - prev; + deltas->push_back(delta); + *plain_size += varint32_size(delta); + prev = pos; + } + off += fc; + } + return Status::OK(); +} + +// PFOR window payload (self-describing; no entropy coding): +// VInt doc_count +// VInt total_pos # sum of all pos_counts +// PFOR_runs(pos_counts) # doc_count values (bit-packed; mostly 1 -> ~1 +// bit) PFOR_runs(position_deltas) # total_pos deltas, flat across docs (first +// per +// # doc absolute, rest delta-within-doc) +// Bit-packing the per-doc pos_counts (vs one varint each) is the size win: in a +// uniform corpus most docs have freq 1, so the count column packs to ~1 bit/doc. +// Emits byte-for-byte the same payload the old encode_pfor_payload_flat produced +// (doc_count == freqs.size(), total_pos == deltas.size() == sum(freqs)), but +// reads the already-derived `deltas` instead of re-walking the positions. +void encode_pfor_payload_from_deltas(std::span freqs, + std::span deltas, ByteSink* out) { + out->put_varint32(static_cast(freqs.size())); + out->put_varint32(static_cast(deltas.size())); + prx_pod_encode_pfor_runs(freqs, out); + prx_pod_encode_pfor_runs(deltas, out); +} + +// Raw plaintext payload (self-describing per-doc boundaries): +// VInt doc_count +// per doc: VInt pos_count, then pos_count position deltas (VInt) +// Emits byte-for-byte the same payload the old encode_payload_flat produced, but +// reads the already-derived `deltas` instead of re-walking the positions and +// re-running the partition/ascending checks. +void encode_payload_from_deltas(std::span freqs, std::span deltas, + ByteSink* out) { + out->put_varint32(static_cast(freqs.size())); + size_t off = 0; + for (uint32_t fc : freqs) { + out->put_varint32(fc); + for (uint32_t i = 0; i < fc; ++i) { + out->put_varint32(deltas[off + i]); + } + off += fc; + } +} + +// Decode per-doc position lists from a PFOR payload. +Status decode_pfor_payload(Slice plain, std::vector>* out) { + ByteSource src(plain); + uint32_t doc_count = 0, total_pos = 0; + RETURN_IF_ERROR(src.get_varint32(&doc_count)); + RETURN_IF_ERROR(src.get_varint32(&total_pos)); + if (total_pos > kMaxWindowPositions) { + return Status::Error( + "prx: position count exceeds sane cap"); + } + if (doc_count > kPrxPodMaxWindowDocs) { + return Status::Error( + "prx: doc count exceeds sane cap"); + } + std::vector pos_counts; + RETURN_IF_ERROR(prx_pod_decode_pfor_runs(&src, doc_count, &pos_counts)); + uint64_t sum = 0; + for (uint32_t d = 0; d < doc_count; ++d) sum += pos_counts[d]; + if (sum != total_pos) { + return Status::Error( + "prx: pos_count sum mismatch"); + } + std::vector deltas; + RETURN_IF_ERROR(prx_pod_decode_pfor_runs(&src, total_pos, &deltas)); + out->clear(); + out->reserve(doc_count); + size_t off = 0; + for (uint32_t d = 0; d < doc_count; ++d) { + std::vector doc; + doc.reserve(pos_counts[d]); + uint32_t prev = 0; + for (uint32_t i = 0; i < pos_counts[d]; ++i) { + prev = (i == 0) ? deltas[off + i] : prev + deltas[off + i]; + doc.push_back(prev); + } + off += pos_counts[d]; + out->push_back(std::move(doc)); + } + if (!src.eof()) + return Status::Error( + "prx: trailing bytes after pfor payload"); + return Status::OK(); +} + +// Writes a PFOR window: codec=pfor, payload, crc(header+payload). +void write_pfor(Slice payload, ByteSink* sink) { + // Single-copy framing: write [codec][varint len][payload] straight into the + // caller's sink, then crc exactly those bytes. view() is taken AFTER the + // payload and BEFORE the crc, so subslice([start, framed_len)) is over a + // settled, contiguous buffer with no pending realloc/aliasing. Byte-identical + // to the former temp-ByteSink assembly, minus one heap alloc + one payload copy. + const size_t start = sink->size(); + sink->put_u8(static_cast(PrxCodec::kPfor)); + sink->put_varint32(static_cast(payload.size())); + sink->put_bytes(payload); + const size_t framed_len = sink->size() - start; + const uint32_t crc = crc32c(sink->view().subslice(start, framed_len)); + sink->put_fixed32(crc); +} + +void write_raw(Slice plain, ByteSink* sink); + +// Emit a RAW frame directly from the already-derived deltas. This is used by +// the singleton fast path so choosing RAW does not allocate a temporary plain +// payload or encode/copy a losing PFOR payload first. +void write_raw_from_deltas(std::span freqs, std::span deltas, + size_t plain_size, ByteSink* sink) { + const size_t start = sink->size(); + sink->put_u8(static_cast(PrxCodec::kRaw)); + sink->put_varint32(static_cast(plain_size)); + const size_t payload_start = sink->size(); + encode_payload_from_deltas(freqs, deltas, sink); + DCHECK_EQ(sink->size() - payload_start, plain_size); + const size_t framed_len = sink->size() - start; + const uint32_t crc = crc32c(sink->view().subslice(start, framed_len)); + sink->put_fixed32(crc); +} + +Status validate_single_doc_byte_limits(std::span positions, + std::span freqs, bool auto_codec, + uint32_t max_uncomp_bytes) { + size_t offset = 0; + std::vector one_doc_deltas; + for (size_t doc = 0; doc < freqs.size(); ++doc) { + const uint32_t frequency = freqs[doc]; + const auto one_freq = freqs.subspan(doc, 1); + const auto one_doc_positions = positions.subspan(offset, frequency); + size_t plain_size = varint32_size(1) + varint32_size(frequency); + uint32_t previous = 0; + for (size_t i = 0; i < one_doc_positions.size(); ++i) { + const uint32_t position = one_doc_positions[i]; + plain_size += varint32_size(i == 0 ? position : position - previous); + previous = position; + } + if (plain_size > max_uncomp_bytes) { + if (!auto_codec) { + return Status::Error( + "prx: one document exceeds the writer window byte limit"); + } + if (one_doc_deltas.capacity() < frequency) { + std::vector().swap(one_doc_deltas); + one_doc_deltas.reserve(frequency); + } else { + one_doc_deltas.clear(); + } + previous = 0; + for (size_t i = 0; i < one_doc_positions.size(); ++i) { + const uint32_t position = one_doc_positions[i]; + one_doc_deltas.push_back(i == 0 ? position : position - previous); + previous = position; + } + ByteSink pfor_payload; + encode_pfor_payload_from_deltas(one_freq, one_doc_deltas, &pfor_payload); + if (pfor_payload.size() > max_uncomp_bytes) { + return Status::Error( + "prx: one document exceeds the writer window byte limit"); + } + } + offset += frequency; + } + return Status::OK(); +} + +size_t pfor_frame_size(size_t payload_size) { + return 1 + varint32_size(static_cast(payload_size)) + payload_size + sizeof(uint32_t); +} + +size_t raw_frame_size(size_t plain_size) { + return 1 + varint32_size(static_cast(plain_size)) + plain_size + sizeof(uint32_t); +} + +size_t zstd_frame_size(size_t plain_size, size_t compressed_size) { + return 1 + varint32_size(static_cast(plain_size)) + + varint32_size(static_cast(compressed_size)) + compressed_size + + sizeof(uint32_t); +} + +struct AutoPrxCodecChoice { + PrxCodec codec = PrxCodec::kPfor; + bool readable = false; +}; + +// Select the smallest complete reader-safe frame among the candidates already +// materialized by the zstd/fallback path. Preserve the existing codec on equal +// sizes by considering PFOR, then ZSTD, then RAW and replacing the winner only +// on a strict size reduction. Sub-threshold singleton RAW selection happens +// before PFOR materialization in build_prx_window_auto_from_flat. +AutoPrxCodecChoice select_auto_prx_codec(size_t pfor_payload_size, size_t plain_payload_size, + size_t compressed_payload_size, bool has_compressed, + uint32_t max_uncomp_bytes) { + const bool pfor_readable = pfor_payload_size <= max_uncomp_bytes; + const bool plain_readable = plain_payload_size <= max_uncomp_bytes; + AutoPrxCodecChoice choice; + size_t selected_frame_size = 0; + if (pfor_readable) { + choice = {.codec = PrxCodec::kPfor, .readable = true}; + selected_frame_size = pfor_frame_size(pfor_payload_size); + } + if (has_compressed && plain_readable) { + const size_t frame_size = zstd_frame_size(plain_payload_size, compressed_payload_size); + if (!choice.readable || frame_size < selected_frame_size) { + choice = {.codec = PrxCodec::kZstd, .readable = true}; + selected_frame_size = frame_size; + } + } + if (plain_readable) { + const size_t frame_size = raw_frame_size(plain_payload_size); + if (!choice.readable || frame_size < selected_frame_size) { + choice = {.codec = PrxCodec::kRaw, .readable = true}; + } + } + return choice; +} + +void write_zstd_compressed(Slice plain, Slice compressed, ByteSink* sink) { + // Single-copy framing (see write_pfor): assemble [codec][uncomp_len][comp_len] + // [compressed] in the caller's sink and crc that span before appending the crc. + const size_t start = sink->size(); + sink->put_u8(static_cast(PrxCodec::kZstd)); + sink->put_varint32(static_cast(plain.size())); + sink->put_varint32(static_cast(compressed.size())); + sink->put_bytes(compressed); + const size_t framed_len = sink->size() - start; + const uint32_t crc = crc32c(sink->view().subslice(start, framed_len)); + sink->put_fixed32(crc); +} + +// Shared auto-mode path for BOTH .prx builders. A single-doc/freq=1 window has a +// provably smaller RAW frame: RAW stores three payload varints, while PFOR adds +// two run headers and is always 4-5 bytes larger. Emit that RAW frame directly +// before any PFOR work. Other sub-threshold windows retain the existing PFOR +// policy so an unmeasured second payload encode cannot regress import CPU. At +// and above the zstd threshold the raw plaintext is already required for +// compression, so all materialized candidates participate in the exact complete +// frame-size comparison at no additional encoding cost. +Status build_prx_window_auto_from_flat(std::span positions_flat, + std::span freqs, int zstd_level, + const PrxWindowLimits& limits, ByteSink* sink, + PrxWindowBuildOutcome* outcome) { + if (freqs.size() == 1 && freqs.front() == 1) { + const size_t plain_size = + varint32_size(1) + varint32_size(1) + varint32_size(positions_flat.front()); + if (plain_size > limits.max_uncomp_bytes) { + return Status::Error( + "prx: one document exceeds the writer window byte limit"); + } + write_raw_from_deltas(freqs, positions_flat, plain_size, sink); + *outcome = PrxWindowBuildOutcome::kBuilt; + return Status::OK(); + } + + std::vector deltas; + size_t plain_size = 0; + RETURN_IF_ERROR(compute_flat_deltas(positions_flat, freqs, &deltas, &plain_size)); + const bool plain_readable = plain_size <= limits.max_uncomp_bytes; + + ByteSink payload; + encode_pfor_payload_from_deltas(freqs, deltas, &payload); + const bool pfor_readable = payload.size() <= limits.max_uncomp_bytes; + if (!pfor_readable && !plain_readable) { + if (freqs.size() <= 1) { + return Status::Error( + "prx: one document exceeds the writer window byte limit"); + } + RETURN_IF_ERROR(validate_single_doc_byte_limits(positions_flat, freqs, true, + limits.max_uncomp_bytes)); + *outcome = PrxWindowBuildOutcome::kNeedsSplit; + return Status::OK(); + } + if (plain_readable && (plain_size >= kPrxPodAutoZstdMinBytes || !pfor_readable)) { + ByteSink plain; + encode_payload_from_deltas(freqs, deltas, &plain); + DCHECK_EQ(plain.size(), plain_size); + std::vector compressed; + const bool has_compressed = plain_size >= kPrxPodAutoZstdMinBytes; + if (plain_size >= kPrxPodAutoZstdMinBytes) { + testing::note_prx_raw_build(); + RETURN_IF_ERROR(zstd_compress(plain.view(), zstd_level, &compressed)); + } + const AutoPrxCodecChoice choice = + select_auto_prx_codec(payload.size(), plain.size(), compressed.size(), + has_compressed, limits.max_uncomp_bytes); + DCHECK(choice.readable); + if (choice.codec == PrxCodec::kZstd) { + write_zstd_compressed(plain.view(), Slice(compressed), sink); + } else if (choice.codec == PrxCodec::kRaw) { + write_raw(plain.view(), sink); + } else { + write_pfor(payload.view(), sink); + } + *outcome = PrxWindowBuildOutcome::kBuilt; + return Status::OK(); + } + DCHECK(pfor_readable); + write_pfor(payload.view(), sink); + *outcome = PrxWindowBuildOutcome::kBuilt; + return Status::OK(); +} + +// Decode per-doc position lists from a plain payload. +Status decode_payload(Slice plain, std::vector>* out) { + ByteSource src(plain); + uint32_t doc_count = 0; + RETURN_IF_ERROR(src.get_varint32(&doc_count)); + if (doc_count > kPrxPodMaxWindowDocs) { + return Status::Error( + "prx: doc count exceeds sane cap"); + } + out->clear(); + out->reserve(doc_count); + for (uint32_t d = 0; d < doc_count; ++d) { + uint32_t pos_count = 0; + RETURN_IF_ERROR(src.get_varint32_fast(&pos_count)); + std::vector doc; + doc.reserve(pos_count); + uint32_t prev = 0; + for (uint32_t i = 0; i < pos_count; ++i) { + uint32_t delta = 0; + RETURN_IF_ERROR(src.get_varint32(&delta)); + if (i != 0 && delta > std::numeric_limits::max() - prev) { + return Status::Error( + "prx: position accumulation overflow"); + } + prev = (i == 0) ? delta : prev + delta; + doc.push_back(prev); + } + out->push_back(std::move(doc)); + } + if (!src.eof()) + return Status::Error( + "prx: trailing bytes after payload"); + return Status::OK(); +} + +// CSR decode of a PFOR payload: all docs' positions into one flat buffer + +// per-doc offsets, with NO per-doc std::vector allocation. `pos_off` has +// doc_count+1 entries (pos_off[0]==0); doc d's positions are +// pos_flat[pos_off[d] .. pos_off[d+1]). +Status decode_pfor_payload_csr(Slice plain, std::vector* pos_flat, + std::vector* pos_off, + PrxCsrAllocationGate* allocation_gate, uint32_t* max_frequency, + bool* has_zero_frequency) { + ByteSource src(plain); + uint32_t doc_count = 0, total_pos = 0; + RETURN_IF_ERROR(src.get_varint32(&doc_count)); + RETURN_IF_ERROR(src.get_varint32(&total_pos)); + if (total_pos > kMaxWindowPositions) { + return Status::Error( + "prx: position count exceeds sane cap"); + } + if (doc_count > kPrxPodMaxWindowDocs) { + return Status::Error( + "prx: doc count exceeds sane cap"); + } + if (allocation_gate != nullptr) { + RETURN_IF_ERROR(allocation_gate->reserve_csr(pos_flat, total_pos, pos_off, + static_cast(doc_count) + 1)); + } + pos_off->clear(); + pos_off->reserve(static_cast(doc_count) + 1); + RETURN_IF_ERROR(prx_pod_decode_pfor_runs(&src, doc_count, pos_off)); + uint64_t sum = 0; + uint32_t decoded_max_frequency = 0; + bool decoded_zero_frequency = false; + for (uint32_t d = 0; d < doc_count; ++d) { + sum += (*pos_off)[d]; + decoded_max_frequency = std::max(decoded_max_frequency, (*pos_off)[d]); + decoded_zero_frequency |= (*pos_off)[d] == 0; + } + if (sum != total_pos) + return Status::Error( + "prx: pos_count sum mismatch"); + // prx_pod_decode_pfor_runs sizes pos_flat to total_pos, so a separate reserve is redundant. pos_off + // keeps its reserve for the push_back loop below. + RETURN_IF_ERROR(prx_pod_decode_pfor_runs(&src, total_pos, pos_flat)); + size_t off = 0; + uint32_t next_off = 0; + for (uint32_t d = 0; d < doc_count; ++d) { + const uint32_t pos_count = (*pos_off)[d]; + (*pos_off)[d] = next_off; + uint32_t prev = 0; + for (uint32_t i = 0; i < pos_count; ++i) { + uint32_t& value = (*pos_flat)[off + i]; + if (i != 0 && value > std::numeric_limits::max() - prev) { + return Status::Error( + "prx: position accumulation overflow"); + } + prev = (i == 0) ? value : prev + value; + value = prev; + } + off += pos_count; + next_off += pos_count; + } + pos_off->push_back(next_off); + if (!src.eof()) + return Status::Error( + "prx: trailing bytes after pfor payload"); + *max_frequency = decoded_max_frequency; + *has_zero_frequency = decoded_zero_frequency; + return Status::OK(); +} + +Status validate_doc_ordinals(std::span doc_ordinals, uint32_t doc_count) { + uint32_t prev = 0; + for (size_t i = 0; i < doc_ordinals.size(); ++i) { + const uint32_t doc = doc_ordinals[i]; + if (doc >= doc_count) { + return Status::Error( + "prx: selected doc ordinal out of range"); + } + if (i != 0 && doc <= prev) { + return Status::Error( + "prx: selected doc ordinals must be strictly ascending"); + } + prev = doc; + } + return Status::OK(); +} + +struct SelectedRange { + SelectedRange(uint32_t begin_, uint32_t end_, uint32_t out_begin_) + : begin(begin_), end(end_), out_begin(out_begin_) {} + + uint32_t begin; + uint32_t end; + uint32_t out_begin; +}; + +uint32_t count_covered_pfor_runs(std::span selected, uint32_t total_pos) { + if (selected.empty() || total_pos == 0) { + return 0; + } + uint32_t runs = 0; + uint32_t next_run = 0; + for (const SelectedRange& range : selected) { + if (range.begin == range.end) { + continue; + } + const uint32_t first_run = range.begin / kFrqBaseUnit; + const uint32_t last_run = (range.end - 1) / kFrqBaseUnit; + const uint32_t counted_first = std::max(first_run, next_run); + if (counted_first <= last_run) { + runs += last_run - counted_first + 1; + next_run = last_run + 1; + } + } + return runs; +} + +bool should_decode_full_prx_positions(std::span selected, + uint32_t selected_pos_count, uint32_t total_pos) { + if (selected.empty() || total_pos == 0) { + return false; + } + if (selected_pos_count * 2 >= total_pos) { + return true; + } + const uint32_t total_runs = (total_pos + kFrqBaseUnit - 1) / kFrqBaseUnit; + const uint32_t covered_runs = count_covered_pfor_runs(selected, total_pos); + return covered_runs * 4 >= total_runs * 3; +} + +Status decode_selected_pfor_count_ranges(ByteSource* src, uint32_t doc_count, + std::span doc_ordinals, + std::vector& selected, + std::vector& pos_off, uint64_t* total_pos_count, + uint32_t* selected_pos_count, uint32_t* max_frequency, + bool* has_zero_frequency) { + selected.clear(); + selected.reserve(doc_ordinals.size()); + pos_off.clear(); + pos_off.reserve(doc_ordinals.size() + 1); + pos_off.push_back(0); + + *selected_pos_count = 0; + uint32_t delta_begin = 0; + size_t next_doc = 0; + *total_pos_count = 0; + *max_frequency = 0; + *has_zero_frequency = false; + std::array run_buf {}; + for (uint32_t run_begin = 0; run_begin < doc_count; run_begin += kFrqBaseUnit) { + const uint32_t run_len = std::min(kFrqBaseUnit, doc_count - run_begin); + RETURN_IF_ERROR(pfor_decode(src, run_len, run_buf.data())); + for (uint32_t i = 0; i < run_len; ++i) { + const uint32_t d = run_begin + i; + const uint32_t count = run_buf[i]; + *max_frequency = std::max(*max_frequency, count); + *has_zero_frequency |= count == 0; + *total_pos_count += count; + if (*total_pos_count > kMaxWindowPositions) { + return Status::Error( + "prx: pos_count sum exceeds sane cap"); + } + if (next_doc < doc_ordinals.size() && doc_ordinals[next_doc] == d) { + selected.emplace_back(delta_begin, delta_begin + count, *selected_pos_count); + *selected_pos_count += count; + pos_off.push_back(*selected_pos_count); + ++next_doc; + } + delta_begin += count; + } + } + if (next_doc != doc_ordinals.size()) { + return Status::Error( + "prx: selected doc ordinal was not decoded"); + } + return Status::OK(); +} + +Status decode_selected_pfor_positions(ByteSource* src, uint32_t total_pos, + std::span selected, bool decode_all_runs, + std::span pos_flat) { + std::array run_buf {}; + size_t range_idx = 0; + uint32_t prev = 0; + for (uint32_t run_begin = 0; run_begin < total_pos; run_begin += kFrqBaseUnit) { + const uint32_t run_len = std::min(kFrqBaseUnit, total_pos - run_begin); + const uint32_t run_end = run_begin + run_len; + while (range_idx < selected.size() && selected[range_idx].end <= run_begin) { + ++range_idx; + prev = 0; + } + if (!decode_all_runs && + (range_idx == selected.size() || selected[range_idx].begin >= run_end)) { + RETURN_IF_ERROR(pfor_skip(src, run_len)); + continue; + } + + RETURN_IF_ERROR(pfor_decode(src, run_len, run_buf.data())); + while (range_idx < selected.size() && selected[range_idx].begin < run_end) { + const SelectedRange& range = selected[range_idx]; + const uint32_t copy_begin = std::max(range.begin, run_begin); + const uint32_t copy_end = std::min(range.end, run_end); + if (copy_begin == range.begin) { + prev = 0; + } + uint32_t dst = range.out_begin + copy_begin - range.begin; + for (uint32_t off = copy_begin; off < copy_end; ++off) { + const uint32_t delta = run_buf[off - run_begin]; + if (off != range.begin && delta > std::numeric_limits::max() - prev) { + return Status::Error( + "prx: position accumulation overflow"); + } + prev = (off == range.begin) ? delta : prev + delta; + pos_flat[dst++] = prev; + } + if (copy_end < range.end) { + break; + } + ++range_idx; + prev = 0; + } + } + return Status::OK(); +} + +Status decode_pfor_payload_csr_selective(Slice plain, std::span doc_ordinals, + std::vector* pos_flat, + std::vector* pos_off, + uint32_t* decoded_doc_count, + uint64_t* decoded_total_positions, uint32_t* max_frequency, + bool* has_zero_frequency) { + ByteSource src(plain); + uint32_t doc_count = 0, total_pos = 0; + RETURN_IF_ERROR(src.get_varint32(&doc_count)); + RETURN_IF_ERROR(src.get_varint32(&total_pos)); + if (total_pos > kMaxWindowPositions) { + return Status::Error( + "prx: position count exceeds sane cap"); + } + if (doc_count > kPrxPodMaxWindowDocs) { + return Status::Error( + "prx: doc count exceeds sane cap"); + } + RETURN_IF_ERROR(validate_doc_ordinals(doc_ordinals, doc_count)); + + pos_flat->clear(); + + std::vector selected; + uint64_t sum = 0; + uint32_t selected_pos_count = 0; + RETURN_IF_ERROR(decode_selected_pfor_count_ranges(&src, doc_count, doc_ordinals, selected, + *pos_off, &sum, &selected_pos_count, + max_frequency, has_zero_frequency)); + if (sum != total_pos) { + return Status::Error( + "prx: pos_count sum mismatch"); + } + + const bool decode_all_runs = + should_decode_full_prx_positions(selected, selected_pos_count, total_pos); + pos_flat->resize(selected_pos_count); + RETURN_IF_ERROR(decode_selected_pfor_positions( + &src, total_pos, selected, decode_all_runs, + std::span(pos_flat->data(), pos_flat->size()))); + if (!src.eof()) { + return Status::Error( + "prx: trailing bytes after pfor payload"); + } + *decoded_doc_count = doc_count; + *decoded_total_positions = total_pos; + return Status::OK(); +} + +// CSR decode of a plain (raw) payload. See decode_pfor_payload_csr. +Status scan_payload_csr_shape(Slice plain, uint32_t* doc_count, uint32_t* total_positions) { + ByteSource src(plain); + RETURN_IF_ERROR(src.get_varint32(doc_count)); + if (*doc_count > kPrxPodMaxWindowDocs) { + return Status::Error( + "prx: doc count exceeds sane cap"); + } + uint64_t total_pos = 0; + for (uint32_t d = 0; d < *doc_count; ++d) { + uint32_t pos_count = 0; + RETURN_IF_ERROR(src.get_varint32_fast(&pos_count)); + total_pos += pos_count; + if (total_pos > kMaxWindowPositions) { + return Status::Error( + "prx: position count exceeds sane cap"); + } + RETURN_IF_ERROR(src.skip_varints(pos_count)); + } + if (!src.eof()) { + return Status::Error( + "prx: trailing bytes after payload"); + } + *total_positions = static_cast(total_pos); + return Status::OK(); +} + +Status decode_payload_csr(Slice plain, std::vector* pos_flat, + std::vector* pos_off, PrxCsrAllocationGate* allocation_gate, + uint32_t* max_frequency, bool* has_zero_frequency) { + if (allocation_gate != nullptr) { + uint32_t preflight_doc_count = 0; + uint32_t preflight_total_positions = 0; + RETURN_IF_ERROR( + scan_payload_csr_shape(plain, &preflight_doc_count, &preflight_total_positions)); + RETURN_IF_ERROR(allocation_gate->reserve_csr(pos_flat, preflight_total_positions, pos_off, + static_cast(preflight_doc_count) + 1)); + } + ByteSource src(plain); + uint32_t doc_count = 0; + RETURN_IF_ERROR(src.get_varint32(&doc_count)); + if (doc_count > kPrxPodMaxWindowDocs) { + return Status::Error( + "prx: doc count exceeds sane cap"); + } + pos_flat->clear(); + pos_off->clear(); + pos_off->reserve(static_cast(doc_count) + 1); + pos_off->push_back(0); + uint64_t total_pos = 0; + uint32_t decoded_max_frequency = 0; + bool decoded_zero_frequency = false; + for (uint32_t d = 0; d < doc_count; ++d) { + uint32_t pos_count = 0; + RETURN_IF_ERROR(src.get_varint32_fast(&pos_count)); + decoded_max_frequency = std::max(decoded_max_frequency, pos_count); + decoded_zero_frequency |= pos_count == 0; + total_pos += pos_count; + if (total_pos > kMaxWindowPositions) { + return Status::Error( + "prx: position count exceeds sane cap"); + } + // Tight inline prefix-sum decode (single-byte fast path) -- see + // decode_delta_run. Shared with the selective reader below. + RETURN_IF_ERROR(src.decode_delta_run(pos_count, pos_flat)); + pos_off->push_back(static_cast(pos_flat->size())); + } + if (!src.eof()) + return Status::Error( + "prx: trailing bytes after payload"); + *max_frequency = decoded_max_frequency; + *has_zero_frequency = decoded_zero_frequency; + return Status::OK(); +} + +Status decode_payload_csr_selective(Slice plain, std::span doc_ordinals, + std::vector* pos_flat, std::vector* pos_off, + uint32_t* decoded_doc_count, uint64_t* decoded_total_positions, + uint32_t* max_frequency, bool* has_zero_frequency) { + ByteSource src(plain); + uint32_t doc_count = 0; + RETURN_IF_ERROR(src.get_varint32(&doc_count)); + if (doc_count > kPrxPodMaxWindowDocs) { + return Status::Error( + "prx: doc count exceeds sane cap"); + } + RETURN_IF_ERROR(validate_doc_ordinals(doc_ordinals, doc_count)); + pos_flat->clear(); + pos_off->clear(); + pos_off->reserve(doc_ordinals.size() + 1); + pos_off->push_back(0); + size_t next_doc = 0; + uint64_t total_pos = 0; + uint32_t decoded_max_frequency = 0; + bool decoded_zero_frequency = false; + for (uint32_t d = 0; d < doc_count; ++d) { + uint32_t pos_count = 0; + RETURN_IF_ERROR(src.get_varint32_fast(&pos_count)); + decoded_max_frequency = std::max(decoded_max_frequency, pos_count); + decoded_zero_frequency |= pos_count == 0; + total_pos += pos_count; + if (total_pos > kMaxWindowPositions) { + return Status::Error( + "prx: position count exceeds sane cap"); + } + const bool selected = next_doc < doc_ordinals.size() && doc_ordinals[next_doc] == d; + if (!selected) { + // Skip this doc's position deltas without decoding them -- the CSR + // layout is sequential so we must advance past them, but only the + // candidate (selected) docs' positions are ever used. With a sparse + // candidate set (the common phrase / phrase-prefix case after docid + // narrowing) most docs in a window are skipped, so this avoids the + // dominant varint-decode cost. + RETURN_IF_ERROR(src.skip_varints(pos_count)); + continue; + } + // Selected doc: decode its `pos_count` ascending position deltas with a + // tight inline prefix-sum decoder (single-byte fast path, no per-value + // get_varint32/Status call chain). This is the CPU hotspot for narrowed + // phrase/phrase-prefix candidate sets, where each selected doc's varint + // run dominates after non-selected docs are skipped. + RETURN_IF_ERROR(src.decode_delta_run(pos_count, pos_flat)); + pos_off->push_back(static_cast(pos_flat->size())); + ++next_doc; + } + if (!src.eof()) + return Status::Error( + "prx: trailing bytes after payload"); + *decoded_doc_count = doc_count; + *decoded_total_positions = total_pos; + *max_frequency = decoded_max_frequency; + *has_zero_frequency = decoded_zero_frequency; + return Status::OK(); +} + +// Decision: given level and plain length, determine whether to compress. +bool prx_pod_should_compress(int level, size_t plain_len) { + if (level == 0) return false; // force raw + if (level > 0) return true; // force zstd + return plain_len >= kPrxPodAutoZstdMinBytes; // auto +} + +// Write a raw window: codec=raw, uncomp_len, crc(header+payload), payload. +void write_raw(Slice plain, ByteSink* sink) { + // Single-copy framing (see write_pfor): assemble [codec][uncomp_len][payload] + // in the caller's sink and crc that span before appending the crc. + const size_t start = sink->size(); + sink->put_u8(static_cast(PrxCodec::kRaw)); + sink->put_varint32(static_cast(plain.size())); + sink->put_bytes(plain); + const size_t framed_len = sink->size() - start; + const uint32_t crc = crc32c(sink->view().subslice(start, framed_len)); + sink->put_fixed32(crc); +} + +// Write a zstd window: codec=zstd, uncomp_len, comp_len, crc(header+payload), +// payload. +Status write_zstd(Slice plain, int level, ByteSink* sink) { + std::vector comp; + RETURN_IF_ERROR(zstd_compress(plain, level > 0 ? level : kPrxPodDefaultZstdLevel, &comp)); + write_zstd_compressed(plain, Slice(comp), sink); + return Status::OK(); +} + +void initialize_frame_stats(const PrxFrameView& encoded, PrxDecodeStats* stats) { + if (encoded.codec == PrxCodec::kRaw) { + stats->raw_frames = 1; + stats->plaintext_bytes = encoded.payload.size(); + } else if (encoded.codec == PrxCodec::kZstd) { + stats->zstd_frames = 1; + stats->plaintext_bytes = encoded.uncompressed_length; + } else { + stats->pfor_frames = 1; + stats->plaintext_bytes = encoded.uncompressed_length; + } +} + +// NOLINTNEXTLINE(readability-function-cognitive-complexity): codec dispatch shares one stats commit. +Status decode_csr_frame(const PrxFrameView& encoded, std::span doc_ordinals, + bool decode_all_docs, bool all_docs_selected, + std::vector* pos_flat, std::vector* pos_off, + PrxDecodeStats* stats, PrxDecodedShape* shape, + PrxCsrAllocationGate* allocation_gate) { + if (!decode_all_docs && allocation_gate != nullptr) { + return Status::Error( + "prx: selective decode cannot use an allocation gate"); + } + if (stats != nullptr) { + initialize_frame_stats(encoded, stats); + } + uint32_t total_docs = 0; + uint64_t total_positions = 0; + uint32_t max_frequency = 0; + bool has_zero_frequency = false; + + std::vector local_decompressed; + Slice plain = encoded.payload; + if (encoded.codec == PrxCodec::kZstd) { + std::vector* decompressed = &local_decompressed; + if (allocation_gate != nullptr) { + RETURN_IF_ERROR(allocation_gate->reserve_decompression(encoded.uncompressed_length, + &decompressed)); + DCHECK(decompressed != nullptr); + DCHECK_GE(decompressed->capacity(), encoded.uncompressed_length); + } + RETURN_IF_ERROR( + zstd_decompress(encoded.payload, encoded.uncompressed_length, decompressed)); + plain = Slice(*decompressed); + } + + if (decode_all_docs) { + if (encoded.codec == PrxCodec::kPfor) { + RETURN_IF_ERROR(decode_pfor_payload_csr(plain, pos_flat, pos_off, allocation_gate, + &max_frequency, &has_zero_frequency)); + } else { + RETURN_IF_ERROR(decode_payload_csr(plain, pos_flat, pos_off, allocation_gate, + &max_frequency, &has_zero_frequency)); + } + total_docs = static_cast(pos_off->size() - 1); + total_positions = pos_flat->size(); + if (!all_docs_selected) { + RETURN_IF_ERROR(validate_doc_ordinals(doc_ordinals, total_docs)); + } + } else if (encoded.codec == PrxCodec::kPfor) { + RETURN_IF_ERROR(decode_pfor_payload_csr_selective(plain, doc_ordinals, pos_flat, pos_off, + &total_docs, &total_positions, + &max_frequency, &has_zero_frequency)); + } else { + RETURN_IF_ERROR(decode_payload_csr_selective(plain, doc_ordinals, pos_flat, pos_off, + &total_docs, &total_positions, &max_frequency, + &has_zero_frequency)); + } + if (shape != nullptr) { + shape->total_docs = total_docs; + shape->total_positions = total_positions; + shape->max_frequency = max_frequency; + shape->has_zero_frequency = has_zero_frequency; + } + if (stats != nullptr) { + stats->total_docs = total_docs; + stats->total_positions = total_positions; + if (all_docs_selected) { + stats->selected_docs = total_docs; + stats->selected_positions = total_positions; + } else if (!decode_all_docs) { + stats->selected_docs = doc_ordinals.size(); + stats->selected_positions = pos_flat->size(); + } + } + return Status::OK(); +} + +Status read_prx_window_csr_impl(ByteSource* source, std::span doc_ordinals, + bool decode_all_docs, bool all_docs_selected, + std::vector* pos_flat, std::vector* pos_off, + PrxDecodeContext* context) { + if (source == nullptr || pos_flat == nullptr || pos_off == nullptr) { + return Status::Error("prx: null arg"); + } + const bool collect_stats = context != nullptr && context->stats != nullptr; + PrxDecodeStats frame_stats; + PrxDecodeStats* stats = collect_stats ? &frame_stats : nullptr; + + PrxClock::time_point decode_start; + if (collect_stats) { + decode_start = prx_clock_now(); + } + PrxFrameView encoded; + RETURN_IF_ERROR(read_prx_frame(source, &encoded)); + RETURN_IF_ERROR(decode_csr_frame(encoded, doc_ordinals, decode_all_docs, all_docs_selected, + pos_flat, pos_off, stats, + context == nullptr ? nullptr : context->shape, + context == nullptr ? nullptr : context->allocation_gate)); + if (collect_stats) { + // Stop inclusive decode timing before the logical-selection scan. Phrase + // execution wraps this call in PhraseVerifyTimer, so that scan remains + // part of verification rather than format decode. + frame_stats.decode_ns = elapsed_ns(decode_start); + if (decode_all_docs && !all_docs_selected) { + uint64_t selected_positions = 0; + for (uint32_t ordinal : doc_ordinals) { + DCHECK_LT(static_cast(ordinal) + 1, pos_off->size()); + selected_positions += (*pos_off)[ordinal + 1] - (*pos_off)[ordinal]; + } + frame_stats.selected_docs = doc_ordinals.size(); + frame_stats.selected_positions = selected_positions; + } + // Format decode is complete. Caller-level CSR invariants are validated + // afterwards, so a later phrase error intentionally retains this work. + context->stats->merge(frame_stats); + } + return Status::OK(); +} + +} // namespace + +Status build_prx_window(std::span> per_doc_positions, + int zstd_level_or_negative_for_auto, ByteSink* sink) { + return build_prx_window(per_doc_positions, zstd_level_or_negative_for_auto, + kReaderPrxWindowLimits, sink); +} + +Status build_prx_window(std::span> per_doc_positions, + int zstd_level_or_negative_for_auto, const PrxWindowLimits& limits, + ByteSink* sink) { + if (sink == nullptr) return Status::Error("prx: null sink"); + size_t total_positions = 0; + RETURN_IF_ERROR(validate_per_doc_window(per_doc_positions, limits, &total_positions)); + // Forced legacy codecs (level 0 = raw varint, level > 0 = zstd) are kept so + // the test/legacy paths still exercise them; the auto path (< 0) now emits + // PFOR bit-packed deltas -- no entropy coding, far cheaper build CPU than + // zstd-3. + if (zstd_level_or_negative_for_auto >= 0) { + ByteSink plain; + RETURN_IF_ERROR(encode_payload(per_doc_positions, &plain)); + if (plain.size() > limits.max_uncomp_bytes) { + return Status::Error( + "prx: encoded payload exceeds writer window byte limit"); + } + Slice plain_view = plain.view(); + if (!prx_pod_should_compress(zstd_level_or_negative_for_auto, plain_view.size())) { + write_raw(plain_view, sink); + return Status::OK(); + } + return write_zstd(plain_view, zstd_level_or_negative_for_auto, sink); + } + // Auto mode: flatten the per-doc lists into (positions_flat, freqs) exactly as + // the former encode_pfor_payload did, then run the shared single-encode path so + // this builder stays byte-identical to build_prx_window_flat. + std::vector flat, freqs; + freqs.reserve(per_doc_positions.size()); + flat.reserve(total_positions); + for (const auto& doc : per_doc_positions) { + freqs.push_back(static_cast(doc.size())); + flat.insert(flat.end(), doc.begin(), doc.end()); + } + // G16-h: level < -1 is auto mode at zstd level |level| (-1 stays the default). + const int auto_level = zstd_level_or_negative_for_auto == -1 ? kPrxPodDefaultZstdLevel + : -zstd_level_or_negative_for_auto; + PrxWindowBuildOutcome outcome = PrxWindowBuildOutcome::kBuilt; + RETURN_IF_ERROR( + build_prx_window_auto_from_flat(flat, freqs, auto_level, limits, sink, &outcome)); + if (outcome == PrxWindowBuildOutcome::kNeedsSplit) { + return Status::Error( + "prx: encoded payload exceeds writer window byte limit"); + } + return Status::OK(); +} + +Status build_prx_window_flat(std::span positions_flat, + std::span freqs, int zstd_level_or_negative_for_auto, + ByteSink* sink) { + return build_prx_window_flat(positions_flat, freqs, zstd_level_or_negative_for_auto, + kReaderPrxWindowLimits, sink); +} + +Status build_prx_window_flat(std::span positions_flat, + std::span freqs, int zstd_level_or_negative_for_auto, + const PrxWindowLimits& limits, ByteSink* sink) { + PrxWindowBuildOutcome outcome = PrxWindowBuildOutcome::kBuilt; + RETURN_IF_ERROR(try_build_prx_window_flat( + positions_flat, freqs, zstd_level_or_negative_for_auto, limits, sink, &outcome)); + if (outcome == PrxWindowBuildOutcome::kNeedsSplit) { + return Status::Error( + "prx: window exceeds writer limits"); + } + return Status::OK(); +} + +Status try_build_prx_window_flat(std::span positions_flat, + std::span freqs, + int zstd_level_or_negative_for_auto, const PrxWindowLimits& limits, + ByteSink* sink, PrxWindowBuildOutcome* outcome) { + if (sink == nullptr || outcome == nullptr) { + return Status::Error("prx: null arg"); + } + RETURN_IF_ERROR(validate_prx_window_limits(limits)); + RETURN_IF_ERROR(check_flat_partition(positions_flat, freqs)); + if (freqs.size() > limits.max_docs || positions_flat.size() > limits.max_positions) { + RETURN_IF_ERROR(validate_flat_positions(positions_flat, freqs)); + for (uint32_t frequency : freqs) { + if (frequency > limits.max_positions) { + return Status::Error( + "prx: one document exceeds the writer window position limit"); + } + } + if (freqs.size() <= 1) { + return Status::Error( + "prx: one document exceeds the writer window shape limit"); + } + RETURN_IF_ERROR(validate_single_doc_byte_limits(positions_flat, freqs, + zstd_level_or_negative_for_auto < 0, + limits.max_uncomp_bytes)); + *outcome = PrxWindowBuildOutcome::kNeedsSplit; + return Status::OK(); + } + if (zstd_level_or_negative_for_auto >= 0) { + ByteSink plain; + RETURN_IF_ERROR(encode_payload_flat(positions_flat, freqs, &plain)); + if (plain.size() > limits.max_uncomp_bytes) { + if (freqs.size() <= 1) { + return Status::Error( + "prx: one document exceeds the writer window byte limit"); + } + RETURN_IF_ERROR(validate_single_doc_byte_limits(positions_flat, freqs, false, + limits.max_uncomp_bytes)); + *outcome = PrxWindowBuildOutcome::kNeedsSplit; + return Status::OK(); + } + Slice plain_view = plain.view(); + if (!prx_pod_should_compress(zstd_level_or_negative_for_auto, plain_view.size())) { + write_raw(plain_view, sink); + *outcome = PrxWindowBuildOutcome::kBuilt; + return Status::OK(); + } + RETURN_IF_ERROR(write_zstd(plain_view, zstd_level_or_negative_for_auto, sink)); + *outcome = PrxWindowBuildOutcome::kBuilt; + return Status::OK(); + } + // Auto mode: shared path with a direct singleton RAW fast path, then PFOR, + // with raw plaintext materialized only for zstd or a tightened-limit fallback. + // G16-h: level < -1 is auto mode at zstd level |level| (-1 stays the default). + const int auto_level = zstd_level_or_negative_for_auto == -1 ? kPrxPodDefaultZstdLevel + : -zstd_level_or_negative_for_auto; + return build_prx_window_auto_from_flat(positions_flat, freqs, auto_level, limits, sink, + outcome); +} + +Status read_prx_window(ByteSource* source, std::vector>* per_doc_positions) { + if (source == nullptr || per_doc_positions == nullptr) { + return Status::Error("prx: null arg"); + } + PrxFrameView frame; + RETURN_IF_ERROR(read_prx_frame(source, &frame)); + if (frame.codec == PrxCodec::kPfor) { + return decode_pfor_payload(frame.payload, per_doc_positions); + } + if (frame.codec == PrxCodec::kRaw) { + return decode_payload(frame.payload, per_doc_positions); + } + std::vector plain; + RETURN_IF_ERROR(zstd_decompress(frame.payload, frame.uncompressed_length, &plain)); + return decode_payload(Slice(plain), per_doc_positions); +} + +Status read_prx_window_csr(ByteSource* source, std::vector* pos_flat, + std::vector* pos_off) { + return read_prx_window_csr_impl(source, {}, true, true, pos_flat, pos_off, nullptr); +} + +Status read_prx_window_csr(ByteSource* source, std::vector* pos_flat, + std::vector* pos_off, PrxDecodeContext* context) { + return read_prx_window_csr_impl(source, {}, true, true, pos_flat, pos_off, context); +} + +Status read_prx_window_csr_for_selection(ByteSource* source, std::span doc_ordinals, + std::vector* pos_flat, + std::vector* pos_off, + PrxDecodeContext* context) { + return read_prx_window_csr_impl(source, doc_ordinals, true, false, pos_flat, pos_off, context); +} + +Status read_prx_window_csr_selective(ByteSource* source, std::span doc_ordinals, + std::vector* pos_flat, + std::vector* pos_off) { + return read_prx_window_csr_impl(source, doc_ordinals, false, false, pos_flat, pos_off, nullptr); +} + +Status read_prx_window_csr_selective(ByteSource* source, std::span doc_ordinals, + std::vector* pos_flat, + std::vector* pos_off, PrxDecodeContext* context) { + return read_prx_window_csr_impl(source, doc_ordinals, false, false, pos_flat, pos_off, context); +} + +} // namespace doris::snii::format + +namespace doris::snii::format::testing { +namespace { +std::atomic& prx_raw_build_atomic() { + static std::atomic counter {0}; + return counter; +} + +#ifdef BE_TEST +std::atomic& prx_clock_read_atomic() { + static std::atomic counter {0}; + return counter; +} + +std::atomic& prx_delta_materialization_atomic() { + static std::atomic counter {0}; + return counter; +} +#endif +} // namespace + +uint64_t prx_raw_build_count() { + return prx_raw_build_atomic().load(std::memory_order_relaxed); +} + +void reset_prx_raw_build_count() { + prx_raw_build_atomic().store(0, std::memory_order_relaxed); +} + +void note_prx_raw_build() { + prx_raw_build_atomic().fetch_add(1, std::memory_order_relaxed); +} + +#ifdef BE_TEST +uint64_t prx_clock_read_count() { + return prx_clock_read_atomic().load(std::memory_order_relaxed); +} + +void reset_prx_clock_read_count() { + prx_clock_read_atomic().store(0, std::memory_order_relaxed); +} + +void note_prx_clock_read() { + prx_clock_read_atomic().fetch_add(1, std::memory_order_relaxed); +} + +uint64_t prx_delta_materialization_count() { + return prx_delta_materialization_atomic().load(std::memory_order_relaxed); +} + +void reset_prx_delta_materialization_count() { + prx_delta_materialization_atomic().store(0, std::memory_order_relaxed); +} + +void note_prx_delta_materialization() { + prx_delta_materialization_atomic().fetch_add(1, std::memory_order_relaxed); +} + +uint8_t select_auto_prx_codec_for_test(size_t pfor_payload_size, size_t plain_payload_size, + size_t compressed_payload_size, uint32_t max_uncomp_bytes) { + const AutoPrxCodecChoice choice = + select_auto_prx_codec(pfor_payload_size, plain_payload_size, compressed_payload_size, + plain_payload_size >= kPrxPodAutoZstdMinBytes, max_uncomp_bytes); + DCHECK(choice.readable); + return static_cast(choice.codec); +} +#endif + +} // namespace doris::snii::format::testing diff --git a/be/src/storage/index/snii/format/prx_pod.h b/be/src/storage/index/snii/format/prx_pod.h new file mode 100644 index 00000000000000..fd96136d3ffe9f --- /dev/null +++ b/be/src/storage/index/snii/format/prx_pod.h @@ -0,0 +1,189 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/encoding/byte_sink.h" +#include "storage/index/snii/encoding/byte_source.h" +#include "storage/index/snii/format/prx_decode_stats.h" + +// .prx position window (PrxPod): stores term position information for several +// docs within one window. +// +// Single-window on-disk byte layout (see docs/design SNII "prx design"): +// u8 codec # PrxCodec: 0=raw / 1=zstd / 2=pfor (bit7 cont-reserved) +// VInt uncomp_len # payload length (raw/pfor: on-disk payload bytes; zstd: +// plaintext) VInt comp_len # present only when codec==zstd u32 crc32c # +// covers header (codec..comp_len) + payload bytes payload # raw: varint +// plaintext; zstd: compressed; pfor: bit-packed +// +// raw/zstd plaintext payload (self-describing per-doc boundaries): +// VInt doc_count +// per doc: VInt pos_count, followed by pos_count position deltas (VInt) +// positions within a doc are ascending, stored as deltas (first absolute). +// +// pfor payload (one auto-build candidate; no entropy coding): +// VInt doc_count +// VInt total_pos # sum of all pos_counts +// PFOR_runs(pos_counts) # doc_count values +// PFOR_runs(position_deltas) # total_pos deltas, kFrqBaseUnit per run, +// # flat doc order (first per doc +// absolute) +// +// Multi-byte fixed-length fields are little-endian; variable-length integers +// reuse snii/encoding/varint. crc32c checksum at window tail detects +// corruption. +namespace doris::snii::format { + +struct PrxWindowLimits { + uint32_t max_docs; + uint32_t max_positions; + uint32_t max_uncomp_bytes; +}; + +enum class PrxWindowBuildOutcome : uint8_t { + kBuilt = 0, + kNeedsSplit = 1, +}; + +inline constexpr PrxWindowLimits kReaderPrxWindowLimits { + .max_docs = 1U << 24, + .max_positions = 1U << 26, + .max_uncomp_bytes = 256U * 1024 * 1024, +}; + +// Writer policies may tighten, but never exceed, the reader's allocation +// guards. Keeping this validation in the format layer makes it impossible for +// a caller to configure a writer that emits frames the current reader rejects. +Status validate_prx_window_limits(const PrxWindowLimits& limits); + +// Build a .prx window and append it to sink. +// per_doc_positions[d] is the position list for the d-th doc within this +// window; must be ascending (duplicates allowed). +// zstd_level_or_negative_for_auto: +// <0 → auto: use direct RAW for a single-doc/freq=1 window; otherwise use +// PFOR below the compression threshold and choose the smallest +// PFOR/ZSTD/RAW frame once raw plaintext is already required (default +// zstd level). +// 0 → force raw varint payload. +// >0 → force ZSTD with the given level. +// Non-ascending positions within a doc return InvalidArgument. +Status build_prx_window(std::span> per_doc_positions, + int zstd_level_or_negative_for_auto, ByteSink* sink); +Status build_prx_window(std::span> per_doc_positions, + int zstd_level_or_negative_for_auto, const PrxWindowLimits& limits, + ByteSink* sink); + +// Vector convenience overload (forwards a span view over the window's per-doc +// lists; the writer can pass a slice of its flat positions WITHOUT deep-copying +// the inner vectors into a fresh std::vector> per +// window). +inline Status build_prx_window(const std::vector>& per_doc_positions, + int zstd_level_or_negative_for_auto, ByteSink* sink) { + return build_prx_window(std::span>(per_doc_positions), + zstd_level_or_negative_for_auto, sink); +} + +// FLAT-positions builder: byte-identical output to build_prx_window above, but +// reads the window's positions from a single flat span partitioned per-doc by +// `freqs` (doc d owns the next freqs[d] entries; freqs.size() == doc count and +// sum(freqs) == positions_flat.size()). Lets the writer pass a subspan of the +// term's flat positions/freqs with NO vector-of-vectors materialization. +Status build_prx_window_flat(std::span positions_flat, + std::span freqs, int zstd_level_or_negative_for_auto, + ByteSink* sink); +Status build_prx_window_flat(std::span positions_flat, + std::span freqs, int zstd_level_or_negative_for_auto, + const PrxWindowLimits& limits, ByteSink* sink); + +// Writer-facing exact attempt. kNeedsSplit is returned only when the current +// multi-document window exceeds a shape or encoded-byte limit and every +// standalone document is representable, so retrying at document boundaries is +// guaranteed to resolve the limit. An unsplittable document, malformed input +// and codec failures remain errors. The sink is unchanged unless the outcome is +// kBuilt. +Status try_build_prx_window_flat(std::span positions_flat, + std::span freqs, + int zstd_level_or_negative_for_auto, const PrxWindowLimits& limits, + ByteSink* sink, PrxWindowBuildOutcome* outcome); + +// Read and verify a .prx window from source, reconstructing the per-doc +// position list. CRC mismatch / invalid codec / truncation / decompression +// failure all return a non-OK Status. +Status read_prx_window(ByteSource* source, std::vector>* per_doc_positions); + +// CSR variant of read_prx_window: decodes ALL docs' positions into one flat +// buffer `pos_flat` with per-doc offsets `pos_off` (size doc_count+1, +// pos_off[0]==0), so doc d's positions are pos_flat[pos_off[d] .. +// pos_off[d+1]). Avoids the per-doc std::vector allocation of read_prx_window +// -- both output vectors are flat uint32 buffers whose capacity a caller can +// retain (clear()) across windows/queries. +Status read_prx_window_csr(ByteSource* source, std::vector* pos_flat, + std::vector* pos_off); +Status read_prx_window_csr(ByteSource* source, std::vector* pos_flat, + std::vector* pos_off, PrxDecodeContext* context); + +// Decode every physical document while reporting the supplied ordinals as the +// logical selection. Used by the density fallback after docid narrowing. +Status read_prx_window_csr_for_selection(ByteSource* source, std::span doc_ordinals, + std::vector* pos_flat, + std::vector* pos_off, PrxDecodeContext* context); + +// Selective CSR variant: decodes positions only for the requested local doc +// ordinals within this PRX window. `doc_ordinals` must be strictly ascending. +// The output uses the same CSR shape, but has doc_ordinals.size()+1 offsets. +Status read_prx_window_csr_selective(ByteSource* source, std::span doc_ordinals, + std::vector* pos_flat, + std::vector* pos_off); +Status read_prx_window_csr_selective(ByteSource* source, std::span doc_ordinals, + std::vector* pos_flat, + std::vector* pos_off, PrxDecodeContext* context); + +} // namespace doris::snii::format + +// Test-only instrumentation seam. prx_raw_build_count() returns a process-global +// count of zstd-candidate plaintext payloads materialized by the auto-mode (.prx) +// window builders. Sub-threshold RAW fallback after an unreadable PFOR is not a +// zstd candidate and is intentionally not counted. In ordinary auto mode the +// small windows that dominate a Zipfian corpus skip the throwaway plaintext and +// second delta walk entirely, so tests assert the count is 0 for sub-threshold +// PFOR windows and exactly 1 per zstd candidate. +// Counters use relaxed atomics and are reset between tests; the writer's segment +// build is single-threaded, so the atomic adds introduce no data race. +namespace doris::snii::format::testing { + +uint64_t prx_raw_build_count(); +void reset_prx_raw_build_count(); +void note_prx_raw_build(); +#ifdef BE_TEST +uint64_t prx_clock_read_count(); +void reset_prx_clock_read_count(); +void note_prx_clock_read(); +uint64_t prx_delta_materialization_count(); +void reset_prx_delta_materialization_count(); +void note_prx_delta_materialization(); +uint8_t select_auto_prx_codec_for_test(size_t pfor_payload_size, size_t plain_payload_size, + size_t compressed_payload_size, uint32_t max_uncomp_bytes); +#endif + +} // namespace doris::snii::format::testing diff --git a/be/src/storage/index/snii/format/prx_position_iterator.cpp b/be/src/storage/index/snii/format/prx_position_iterator.cpp new file mode 100644 index 00000000000000..fe66e6f3c118ec --- /dev/null +++ b/be/src/storage/index/snii/format/prx_position_iterator.cpp @@ -0,0 +1,394 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/format/prx_position_iterator.h" + +#include +#include + +#include "storage/index/snii/encoding/pfor.h" +#include "storage/index/snii/encoding/zstd_codec.h" +#include "storage/index/snii/format/format_constants.h" +#include "storage/index/snii/format/prx_frame.h" +#include "storage/index/snii/format/prx_pod.h" + +namespace doris::snii::format { +namespace { + +Status invalid_iterator_state(const char* message) { + return Status::Error(message); +} + +Status corrupted_iterator_payload(const char* message) { + return Status::Error(message); +} + +Status validate_selected_ordinals(std::span selected_doc_ordinals, + uint32_t doc_count) { + uint32_t previous_ordinal = 0; + bool first_ordinal = true; + for (uint32_t ordinal : selected_doc_ordinals) { + if (ordinal >= doc_count || (!first_ordinal && ordinal <= previous_ordinal)) { + return invalid_iterator_state( + "prx iterator: selected doc ordinals must be strictly increasing and valid"); + } + previous_ordinal = ordinal; + first_ordinal = false; + } + return Status::OK(); +} + +} // namespace + +Status PrxPositionIterator::fail(Status status) { + failed_ = true; + return status; +} + +void PrxPositionIterator::reset_state(PrxDecodeContext* context) { + context_ = context; + frame_stats_ = {}; + payload_source_.reset(); + decompressed_.clear(); + pfor_counts_.clear(); + pfor_offsets_.clear(); + pfor_run_begin_ = 0; + pfor_run_length_ = 0; + pfor_run_index_ = 0; + pfor_stream_index_ = 0; + codec_ = PrxCodec::kRaw; + doc_count_ = 0; + next_doc_ordinal_ = 0; + frequency_ = 0; + decoded_from_doc_ = 0; + scratch_position_ = 0; + scratch_size_ = 0; + previous_position_ = 0; + first_position_ = true; + active_doc_ = false; + failed_ = false; + finished_ = false; +} + +Status PrxPositionIterator::initialize_frame(Slice framed_window, uint32_t expected_doc_count, + std::span selected_doc_ordinals) { + ByteSource frame_source(framed_window); + PrxFrameView frame; + Status status = read_prx_frame(&frame_source, &frame); + if (!status.ok()) { + return fail(std::move(status)); + } + if (!frame_source.eof()) { + return fail(corrupted_iterator_payload("prx iterator: trailing bytes after frame")); + } + codec_ = frame.codec; + + Slice plaintext = frame.payload; + if (frame.codec == PrxCodec::kZstd) { + status = zstd_decompress(frame.payload, frame.uncompressed_length, &decompressed_); + if (!status.ok()) { + return fail(std::move(status)); + } + plaintext = Slice(decompressed_); + frame_stats_.zstd_frames = 1; + } else if (frame.codec == PrxCodec::kPfor) { + frame_stats_.pfor_frames = 1; + } else { + frame_stats_.raw_frames = 1; + } + frame_stats_.plaintext_bytes = frame.uncompressed_length; + payload_source_.emplace(plaintext); + status = payload_source_->get_varint32(&doc_count_); + if (!status.ok()) { + return fail(std::move(status)); + } + if (doc_count_ > kReaderPrxWindowLimits.max_docs) { + return fail(corrupted_iterator_payload("prx iterator: doc count exceeds sane cap")); + } + if (doc_count_ != expected_doc_count) { + return fail(corrupted_iterator_payload( + "prx iterator: doc count differs from posting metadata")); + } + frame_stats_.total_docs = doc_count_; + + status = validate_selected_ordinals(selected_doc_ordinals, doc_count_); + if (!status.ok()) { + return fail(std::move(status)); + } + if (codec_ == PrxCodec::kPfor) { + uint32_t declared_total_positions = 0; + status = payload_source_->get_varint32(&declared_total_positions); + if (!status.ok()) { + return fail(std::move(status)); + } + if (declared_total_positions > kReaderPrxWindowLimits.max_positions) { + return fail( + corrupted_iterator_payload("prx iterator: position count exceeds sane cap")); + } + RETURN_IF_ERROR(decode_pfor_counts(declared_total_positions)); + } + return Status::OK(); +} + +Status PrxPositionIterator::reset(Slice framed_window, uint32_t expected_doc_count, + std::span selected_doc_ordinals, + PrxDecodeContext* context) { + reset_state(context); + return initialize_frame(framed_window, expected_doc_count, selected_doc_ordinals); +} + +Status PrxPositionIterator::decode_pfor_counts(uint32_t declared_total_positions) { + pfor_counts_.resize(doc_count_); + pfor_offsets_.resize(static_cast(doc_count_) + 1); + + for (uint32_t offset = 0; offset < doc_count_; offset += kFrqBaseUnit) { + const uint32_t run_length = std::min(kFrqBaseUnit, doc_count_ - offset); + Status status = pfor_decode(&*payload_source_, run_length, pfor_counts_.data() + offset); + if (!status.ok()) { + return fail(std::move(status)); + } + } + uint64_t count_sum = 0; + pfor_offsets_[0] = 0; + for (uint32_t doc = 0; doc < doc_count_; ++doc) { + count_sum += pfor_counts_[doc]; + if (count_sum > kReaderPrxWindowLimits.max_positions) { + return fail( + corrupted_iterator_payload("prx iterator: position count exceeds sane cap")); + } + pfor_offsets_[doc + 1] = static_cast(count_sum); + } + if (count_sum != declared_total_positions) { + return fail(corrupted_iterator_payload( + "prx iterator: position count sum differs from declared total")); + } + frame_stats_.total_positions = declared_total_positions; + return Status::OK(); +} + +Status PrxPositionIterator::decode_pfor_run(uint32_t run_begin, uint32_t run_length) { + DCHECK_EQ(run_begin, pfor_stream_index_); + Status status = pfor_decode(&*payload_source_, run_length, pfor_run_.data()); + if (!status.ok()) { + return fail(std::move(status)); + } + pfor_run_begin_ = run_begin; + pfor_run_length_ = run_length; + pfor_run_index_ = 0; + pfor_stream_index_ += run_length; + return Status::OK(); +} + +Status PrxPositionIterator::skip_pfor_run(uint32_t run_length) { + Status status = pfor_skip(&*payload_source_, run_length); + if (!status.ok()) { + return fail(std::move(status)); + } + pfor_stream_index_ += run_length; + return Status::OK(); +} + +Status PrxPositionIterator::advance_pfor_cursor(uint32_t target, bool decode_partial_run, + bool require_position) { + const uint32_t total_positions = pfor_offsets_.back(); + DCHECK_LE(target, total_positions); + + if (pfor_run_length_ != 0) { + const uint32_t run_end = pfor_run_begin_ + pfor_run_length_; + if (target >= pfor_run_begin_ && + (target < run_end || (!require_position && target == run_end))) { + pfor_run_index_ = target - pfor_run_begin_; + return Status::OK(); + } + DCHECK_GE(target, run_end); + pfor_run_length_ = 0; + pfor_run_index_ = 0; + } + + DCHECK_LE(pfor_stream_index_, target); + while (pfor_stream_index_ < target) { + const uint32_t run_begin = pfor_stream_index_; + const uint32_t run_length = std::min(kFrqBaseUnit, total_positions - run_begin); + if (run_begin + run_length <= target) { + RETURN_IF_ERROR(skip_pfor_run(run_length)); + continue; + } + if (!decode_partial_run) { + return Status::OK(); + } + RETURN_IF_ERROR(decode_pfor_run(run_begin, run_length)); + pfor_run_index_ = target - run_begin; + return Status::OK(); + } + + if (require_position) { + DCHECK_LT(target, total_positions); + const uint32_t run_length = + std::min(kFrqBaseUnit, total_positions - pfor_stream_index_); + RETURN_IF_ERROR(decode_pfor_run(pfor_stream_index_, run_length)); + } + return Status::OK(); +} + +// NOLINTNEXTLINE(readability-non-const-parameter): frequency is populated from the payload cursor. +Status PrxPositionIterator::read_frequency(uint32_t* frequency) { + DCHECK(codec_ != PrxCodec::kPfor); + Status status = payload_source_->get_varint32_fast(frequency); + if (!status.ok()) { + return fail(std::move(status)); + } + frame_stats_.total_positions += *frequency; + if (frame_stats_.total_positions > kReaderPrxWindowLimits.max_positions) { + return fail(corrupted_iterator_payload("prx iterator: position count exceeds sane cap")); + } + return Status::OK(); +} + +Status PrxPositionIterator::skip_positions(uint32_t count) { + DCHECK(codec_ != PrxCodec::kPfor); + Status status = payload_source_->skip_varints(count); + if (!status.ok()) { + return fail(std::move(status)); + } + return Status::OK(); +} + +Status PrxPositionIterator::seek(uint32_t doc_ordinal) { + if (failed_ || finished_ || active_doc_) { + return fail(invalid_iterator_state("prx iterator: seek in invalid state")); + } + if (doc_ordinal < next_doc_ordinal_ || doc_ordinal >= doc_count_) { + return fail(invalid_iterator_state("prx iterator: seek ordinal is not increasing")); + } + if (codec_ == PrxCodec::kPfor) { + next_doc_ordinal_ = doc_ordinal; + frequency_ = pfor_counts_[doc_ordinal]; + RETURN_IF_ERROR(advance_pfor_cursor(pfor_offsets_[doc_ordinal], false, false)); + } else { + while (next_doc_ordinal_ < doc_ordinal) { + uint32_t skipped_frequency = 0; + RETURN_IF_ERROR(read_frequency(&skipped_frequency)); + RETURN_IF_ERROR(skip_positions(skipped_frequency)); + ++next_doc_ordinal_; + } + RETURN_IF_ERROR(read_frequency(&frequency_)); + } + ++frame_stats_.selected_docs; + frame_stats_.selected_positions += frequency_; + decoded_from_doc_ = 0; + scratch_position_ = 0; + scratch_size_ = 0; + previous_position_ = 0; + first_position_ = true; + active_doc_ = true; + return Status::OK(); +} + +Status PrxPositionIterator::next_position(uint32_t* position, bool* available) { + if (failed_ || finished_ || !active_doc_) { + return fail(invalid_iterator_state("prx iterator: next_position in invalid state")); + } + if (codec_ == PrxCodec::kPfor) { + if (decoded_from_doc_ == frequency_) { + *available = false; + return Status::OK(); + } + const uint32_t stream_position = pfor_offsets_[next_doc_ordinal_] + decoded_from_doc_; + RETURN_IF_ERROR(advance_pfor_cursor(stream_position, true, true)); + const uint32_t delta = pfor_run_[pfor_run_index_++]; + if (!first_position_ && delta > std::numeric_limits::max() - previous_position_) { + return fail(corrupted_iterator_payload("prx iterator: position accumulation overflow")); + } + previous_position_ = first_position_ ? delta : previous_position_ + delta; + first_position_ = false; + ++decoded_from_doc_; + *position = previous_position_; + *available = true; + return Status::OK(); + } + if (scratch_position_ == scratch_size_) { + if (decoded_from_doc_ == frequency_) { + *available = false; + return Status::OK(); + } + const uint32_t batch_size = std::min(static_cast(scratch_.size()), + frequency_ - decoded_from_doc_); + Status status = + payload_source_->decode_delta_batch(std::span(scratch_).first(batch_size), + &previous_position_, &first_position_); + if (!status.ok()) { + return fail(std::move(status)); + } + decoded_from_doc_ += batch_size; + scratch_position_ = 0; + scratch_size_ = batch_size; + } + *position = scratch_[scratch_position_++]; + *available = true; + return Status::OK(); +} + +Status PrxPositionIterator::finish_doc() { + if (failed_ || finished_ || !active_doc_) { + return fail(invalid_iterator_state("prx iterator: finish_doc in invalid state")); + } + if (codec_ == PrxCodec::kPfor) { + RETURN_IF_ERROR( + advance_pfor_cursor(pfor_offsets_[next_doc_ordinal_ + 1], frequency_ != 0, false)); + decoded_from_doc_ = frequency_; + } else { + RETURN_IF_ERROR(skip_positions(frequency_ - decoded_from_doc_)); + } + scratch_position_ = 0; + scratch_size_ = 0; + active_doc_ = false; + ++next_doc_ordinal_; + return Status::OK(); +} + +Status PrxPositionIterator::finish_frame() { + if (failed_ || finished_) { + return fail(invalid_iterator_state("prx iterator: finish_frame in invalid state")); + } + if (active_doc_) { + RETURN_IF_ERROR(finish_doc()); + } + if (codec_ == PrxCodec::kPfor) { + RETURN_IF_ERROR(advance_pfor_cursor(pfor_offsets_.back(), false, false)); + next_doc_ordinal_ = doc_count_; + } else { + while (next_doc_ordinal_ < doc_count_) { + uint32_t skipped_frequency = 0; + RETURN_IF_ERROR(read_frequency(&skipped_frequency)); + RETURN_IF_ERROR(skip_positions(skipped_frequency)); + ++next_doc_ordinal_; + } + } + if (!payload_source_->eof()) { + return fail(corrupted_iterator_payload("prx iterator: trailing bytes after payload")); + } + if (context_ != nullptr && context_->stats != nullptr) { + context_->stats->merge(frame_stats_); + } + if (context_ != nullptr && context_->query_stats != nullptr) { + ++context_->query_stats->prx_streaming_frames; + } + finished_ = true; + return Status::OK(); +} + +} // namespace doris::snii::format diff --git a/be/src/storage/index/snii/format/prx_position_iterator.h b/be/src/storage/index/snii/format/prx_position_iterator.h new file mode 100644 index 00000000000000..1f70d75d731112 --- /dev/null +++ b/be/src/storage/index/snii/format/prx_position_iterator.h @@ -0,0 +1,81 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/encoding/byte_source.h" +#include "storage/index/snii/format/format_constants.h" +#include "storage/index/snii/format/prx_decode_stats.h" + +namespace doris::snii::format { + +class PrxPositionIterator { +public: + Status reset(Slice framed_window, uint32_t expected_doc_count, + std::span selected_doc_ordinals, PrxDecodeContext* context); + Status seek(uint32_t doc_ordinal); + [[nodiscard]] uint32_t freq() const { return frequency_; } + Status next_position(uint32_t* position, bool* available); + Status finish_doc(); + Status finish_frame(); + +private: + void reset_state(PrxDecodeContext* context); + Status initialize_frame(Slice framed_window, uint32_t expected_doc_count, + std::span selected_doc_ordinals); + Status read_frequency(uint32_t* frequency); + Status skip_positions(uint32_t count); + Status decode_pfor_counts(uint32_t declared_total_positions); + Status advance_pfor_cursor(uint32_t target, bool decode_partial_run, bool require_position); + Status decode_pfor_run(uint32_t run_begin, uint32_t run_length); + Status skip_pfor_run(uint32_t run_length); + Status fail(Status status); + + std::vector decompressed_; + std::optional payload_source_; + PrxDecodeContext* context_ = nullptr; + PrxDecodeStats frame_stats_; + std::vector pfor_counts_; + std::vector pfor_offsets_; + std::array pfor_run_ {}; + uint32_t pfor_run_begin_ = 0; + uint32_t pfor_run_length_ = 0; + uint32_t pfor_run_index_ = 0; + uint32_t pfor_stream_index_ = 0; + alignas(64) std::array scratch_ {}; + PrxCodec codec_ = PrxCodec::kRaw; + uint32_t doc_count_ = 0; + uint32_t next_doc_ordinal_ = 0; + uint32_t frequency_ = 0; + uint32_t decoded_from_doc_ = 0; + uint32_t scratch_position_ = 0; + uint32_t scratch_size_ = 0; + uint32_t previous_position_ = 0; + bool first_position_ = true; + bool active_doc_ = false; + bool failed_ = false; + bool finished_ = false; +}; + +} // namespace doris::snii::format diff --git a/be/src/storage/index/snii/format/sampled_term_index.cpp b/be/src/storage/index/snii/format/sampled_term_index.cpp new file mode 100644 index 00000000000000..38bd5500077c08 --- /dev/null +++ b/be/src/storage/index/snii/format/sampled_term_index.cpp @@ -0,0 +1,198 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/format/sampled_term_index.h" + +#include + +#include "storage/index/snii/encoding/byte_source.h" +#include "storage/index/snii/encoding/section_framer.h" + +namespace doris::snii::format { + +namespace { + +// Longest common prefix length of term and prev (front coding primitive, consistent with dict_entry). +uint32_t sampled_term_index_common_prefix_len(std::string_view term, std::string_view prev) { + uint32_t n = 0; + const uint32_t lim = static_cast(std::min(term.size(), prev.size())); + while (n < lim && term[n] == prev[n]) ++n; + return n; +} + +// Write a front-coded term key (prefix_len + suffix_len + suffix). +void write_term_key(std::string_view term, std::string_view prev, ByteSink* sink) { + const uint32_t prefix = sampled_term_index_common_prefix_len(term, prev); + const std::string_view suffix = term.substr(prefix); + sink->put_varint32(prefix); + sink->put_varint32(static_cast(suffix.size())); + sink->put_bytes(Slice(suffix)); +} + +// Read a front-coded term key and reconstruct it into out from prev + suffix. +Status read_term_key(ByteSource* src, std::string_view prev, std::string* out) { + uint32_t prefix = 0; + uint32_t suffix_len = 0; + RETURN_IF_ERROR(src->get_varint32(&prefix)); + RETURN_IF_ERROR(src->get_varint32(&suffix_len)); + if (prefix > prev.size()) { + return Status::Error( + "sampled_term_index: prefix_len exceeds prev_term length"); + } + Slice suffix; + RETURN_IF_ERROR(src->get_bytes(suffix_len, &suffix)); + out->assign(prev.substr(0, prefix)); + out->append(reinterpret_cast(suffix.data()), suffix.size()); + return Status::OK(); +} + +} // namespace + +void SampledTermIndexBuilder::add_block_first_term(std::string_view first_term) { + first_terms_.emplace_back(first_term); +} + +void SampledTermIndexBuilder::finish(ByteSink* sink) { + ByteSink payload; + payload.put_varint32(static_cast(first_terms_.size())); + // min_term / max_term are written only when non-empty (== first/last sample_term). + if (!first_terms_.empty()) { + write_term_key(first_terms_.front(), std::string_view {}, &payload); + write_term_key(first_terms_.back(), std::string_view {}, &payload); + std::string_view prev {}; + for (const auto& t : first_terms_) { + write_term_key(t, prev, &payload); + prev = t; + } + } + SectionFramer::write(*sink, static_cast(SectionType::kSampledTermIndex), + payload.view()); +} + +namespace { + +// Parse n_blocks, min/max (not used directly; consumed for checksum alignment), and all sample_terms from payload. +Status parse_payload(Slice payload, std::vector* terms) { + ByteSource src(payload); + uint32_t n_blocks = 0; + RETURN_IF_ERROR(src.get_varint32(&n_blocks)); + if (n_blocks == 0) { + if (!src.eof()) { + return Status::Error( + "sampled_term_index: empty index contains trailing bytes"); + } + terms->clear(); + return Status::OK(); + } + + // min_term / max_term (do not drive binary search directly; must be consumed to verify structural alignment). + std::string min_term; + std::string max_term; + RETURN_IF_ERROR(read_term_key(&src, std::string_view {}, &min_term)); + RETURN_IF_ERROR(read_term_key(&src, std::string_view {}, &max_term)); + + // Guard against a corrupted, inflated count from untrusted bytes: each term key + // needs >= 2 bytes (prefix varint + suffix_len varint, each >= 1 byte; the + // suffix itself may be empty), so cap before reserve to avoid a huge allocation. + constexpr size_t kMinTermKeyBytes = 2; + if (n_blocks > src.remaining() / kMinTermKeyBytes) { + return Status::Error( + "sampled_term_index: n_blocks exceeds payload capacity"); + } + + std::vector out; + out.reserve(n_blocks); + std::string prev; + for (uint32_t i = 0; i < n_blocks; ++i) { + std::string term; + RETURN_IF_ERROR(read_term_key(&src, prev, &term)); + prev = term; + out.push_back(std::move(term)); + } + if (!src.eof()) { + return Status::Error( + "sampled_term_index: payload contains trailing bytes"); + } + if (out.front() != min_term || out.back() != max_term) { + return Status::Error( + "sampled_term_index: min/max inconsistent with sample_terms"); + } + *terms = std::move(out); + return Status::OK(); +} + +} // namespace + +Status SampledTermIndexReader::open(Slice section, SampledTermIndexReader* out) { + if (out == nullptr) { + return Status::Error("sampled_term_index: out is null"); + } + ByteSource src(section); + FramedSection sec; + RETURN_IF_ERROR(SectionFramer::read(src, &sec)); + if (!src.eof()) { + return Status::Error( + "sampled_term_index: trailing framed section bytes"); + } + if (sec.type != static_cast(SectionType::kSampledTermIndex)) { + return Status::Error( + "sampled_term_index: not a kSampledTermIndex section"); + } + *out = SampledTermIndexReader {}; + return parse_payload(sec.payload, &out->sample_terms_); +} + +size_t SampledTermIndexReader::heap_bytes() const { + size_t bytes = sample_terms_.capacity() * sizeof(std::string); + for (const auto& term : sample_terms_) { + bytes += std_string_heap_bytes(term); + } + return bytes; +} + +Status SampledTermIndexReader::locate(std::string_view target, bool* maybe_present, + uint32_t* block_ordinal) const { + if (maybe_present == nullptr || block_ordinal == nullptr) { + return Status::Error( + "sampled_term_index: output pointer is null"); + } + *maybe_present = false; + *block_ordinal = 0; + if (sample_terms_.empty()) { + return Status::OK(); // empty index: always out of range. + } + // target < min_term (first block's first term) -> before the first block, so it + // cannot exist in any block. NOTE: a target GREATER than the last sample term is + // NOT out of range -- sample_terms_ holds each block's FIRST term, so the LAST + // block can contain terms greater than its first term. Such a target routes to + // the last block (upper_bound -> end()), where find_term confirms presence. + if (target < std::string_view(sample_terms_.front())) { + return Status::OK(); + } + // Last sample_term <= target: step back one position after upper_bound. For a + // target past every sample term, upper_bound returns end() and idx = n-1 (the + // last block), which is correct. + auto it = std::upper_bound( + sample_terms_.begin(), sample_terms_.end(), target, + [](std::string_view t, const std::string& s) { return t < std::string_view(s); }); + const auto idx = (it - sample_terms_.begin()) - 1; // it > begin (< min excluded). + *maybe_present = true; + *block_ordinal = static_cast(idx); + return Status::OK(); +} + +} // namespace doris::snii::format diff --git a/be/src/storage/index/snii/format/sampled_term_index.h b/be/src/storage/index/snii/format/sampled_term_index.h new file mode 100644 index 00000000000000..f2d63af88334e8 --- /dev/null +++ b/be/src/storage/index/snii/format/sampled_term_index.h @@ -0,0 +1,103 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/encoding/byte_sink.h" +#include "storage/index/snii/format/format_constants.h" + +// SampledTermIndex -- resident metadata for locating a query term to a candidate DICT block. +// +// Sampling granularity is per DICT block (not a fixed term count): each time the writer produces a DICT block, +// it writes the block's first_term into this index. Size grows proportionally to block count. At read time it is +// loaded into the searcher cache together with SniiLogicalIndexReader. See design spec "Sampled Term Index". +// +// On-disk layout (framed by SectionFramer, uniform type+len+crc32c): +// [u8 type=kSampledTermIndex][varint64 payload_len][payload][fixed32 crc32c] +// payload = +// n_blocks varint32 +// min_term len(varint32) + bytes # == sample_terms[0], omitted when n_blocks=0 +// max_term len(varint32) + bytes # == sample_terms[n-1], omitted when n_blocks=0 +// sample_terms[n_blocks]: # first_term of each block, in ascending order +// prefix_len varint32 # shared prefix length with the previous sample_term +// suffix_len varint32 +// suffix u8[suffix_len] +// +// Term bytes are compared as unsigned byte order (UTF-8 friendly, binary-safe). Front coding reuses +// the same prefix/suffix primitives as DictEntry; do not reimplement. +namespace doris::snii::format { + +// SSO-aware heap-byte accounting for a std::string. libstdc++ keeps up to 15 +// chars inline (SSO), so only capacity() > 15 implies a separate heap buffer of +// capacity()+1 bytes (the +1 is the NUL terminator); an SSO string owns no heap +// and contributes 0. Shared by the resident format readers' heap_bytes() charge +// helpers, which back LogicalIndexReader::memory_usage() (the searcher-cache +// charge). NOTE: the threshold 15 is libstdc++-specific; a different standard +// library needs a different SSO bound here. +inline size_t std_string_heap_bytes(const std::string& s) { + return s.capacity() > 15 ? s.capacity() + 1 : 0; +} + +// Builder: appends the first_term of each DICT block in block ordinal order (must be strictly ascending), +// and serializes the entire set into a single kSampledTermIndex framed section on finish. +class SampledTermIndexBuilder { +public: + // Appends the first_term of the next DICT block. Call order determines block ordinal order. + void add_block_first_term(std::string_view first_term); + + // Serializes and appends to sink. An empty collection (no blocks) is valid; n_blocks=0. + void finish(ByteSink* sink); + +private: + std::vector first_terms_; +}; + +// Reader: verifies the checksum and materializes all sample_terms on open; subsequent locate calls are pure in-memory binary search. +class SampledTermIndexReader { +public: + SampledTermIndexReader() = default; + + // Parses a kSampledTermIndex framed section. + // CRC mismatch / truncation / field overrun → kCorruption; type != kSampledTermIndex → kInvalidArgument. + static Status open(Slice section, SampledTermIndexReader* out); + + // Binary-search locate: returns the block ordinal of the last sample_term <= target. + // target < min_term or target > max_term (including empty index) → *maybe_present=false (out of range, term is definitely absent). + // Otherwise *maybe_present=true and *block_ordinal is the ordinal of the matching block. + Status locate(std::string_view target, bool* maybe_present, uint32_t* block_ordinal) const; + + uint32_t n_blocks() const { return static_cast(sample_terms_.size()); } + + // Resident heap held beyond sizeof(*this): the sample_terms_ vector buffer + // plus each non-SSO term's heap allocation. Summed into + // LogicalIndexReader::memory_usage() so the searcher-cache charge reflects the + // decoded sampled index (previously omitted -> under-charge -> over-commit). + size_t heap_bytes() const; + +private: + std::vector sample_terms_; +}; + +} // namespace doris::snii::format diff --git a/be/src/storage/index/snii/format/stats_block.h b/be/src/storage/index/snii/format/stats_block.h new file mode 100644 index 00000000000000..d1b07283ae437b --- /dev/null +++ b/be/src/storage/index/snii/format/stats_block.h @@ -0,0 +1,34 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +namespace doris::snii::format { + +// Runtime counting statistics used for query planning and BM25. Core protobuf +// metadata owns their on-disk representation. +struct StatsBlock { + uint64_t doc_count = 0; // total doc count at segment level (including unindexed/NULL) + uint64_t indexed_doc_count = 0; // number of docs actually indexed (denominator for avgdl) + uint64_t term_count = 0; // number of unique terms in this index + uint64_t sum_total_term_freq = 0; // total token count across all indexed docs + uint64_t null_count = 0; // number of NULL / not-indexed docs +}; + +} // namespace doris::snii::format diff --git a/be/src/storage/index/snii/format/tail_pointer.cpp b/be/src/storage/index/snii/format/tail_pointer.cpp new file mode 100644 index 00000000000000..4dacfb0bf7767a --- /dev/null +++ b/be/src/storage/index/snii/format/tail_pointer.cpp @@ -0,0 +1,122 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/format/tail_pointer.h" + +#include "storage/index/snii/encoding/byte_source.h" +#include "storage/index/snii/encoding/crc32c.h" +#include "storage/index/snii/format/format_constants.h" + +namespace doris::snii::format { + +namespace { + +// Byte widths of every fixed field, used to derive the constant on-disk size: +// u32 magic + u16 version + 2*u64 + u32 directory crc + u8 size + u32 tail crc. +constexpr size_t kMagicBytes = 4; +constexpr size_t kVersionBytes = 2; +constexpr size_t kU64Bytes = 8; +constexpr size_t kU32Bytes = 4; +constexpr size_t kSizeByteBytes = 1; + +constexpr size_t kFixedSize = + kMagicBytes + kVersionBytes + 2 * kU64Bytes + kU32Bytes + kSizeByteBytes + kU32Bytes; +// tail_checksum is the trailing u32 and covers every byte before it. +constexpr size_t kTailPointerChecksumCoverage = kFixedSize - kU32Bytes; + +// Serializes the checksum-covered region in fixed field order into covered. +void serialize_covered(const TailPointer& tp, ByteSink* covered) { + covered->put_fixed32(kTailMagic); + covered->put_fixed16(kFormatVersion); + covered->put_fixed64(tp.directory_offset); + covered->put_fixed64(tp.directory_length); + covered->put_fixed32(tp.directory_crc32c); + covered->put_u8(static_cast(kFixedSize)); +} + +} // namespace + +size_t tail_pointer_size() { + return kFixedSize; +} + +Status encode_tail_pointer(const TailPointer& tp, ByteSink* sink) { + if (sink == nullptr) { + return Status::Error("tail_pointer: null sink"); + } + ByteSink covered; + serialize_covered(tp, &covered); + DORIS_CHECK_EQ(covered.size(), kTailPointerChecksumCoverage); + const uint32_t tail_checksum = crc32c(covered.view()); + sink->put_bytes(covered.view()); + sink->put_fixed32(tail_checksum); + return Status::OK(); +} + +Status decode_tail_pointer(Slice last_bytes, TailPointer* out) { + if (out == nullptr) { + return Status::Error("tail_pointer: null output"); + } + // Anti-DoS / framing: the tail pointer is a fixed-size footer, so reject any + // input that is not exactly the fixed size before touching its contents. + if (last_bytes.size() != kFixedSize) { + return Status::Error( + "tail_pointer: input is not the fixed size"); + } + const Slice covered = last_bytes.subslice(0, kTailPointerChecksumCoverage); + DORIS_CHECK_EQ(covered.size(), kTailPointerChecksumCoverage); + ByteSource checksum_source(last_bytes.subslice(kTailPointerChecksumCoverage, kU32Bytes)); + uint32_t tail_checksum = 0; + RETURN_IF_ERROR(checksum_source.get_fixed32(&tail_checksum)); + DORIS_CHECK(checksum_source.eof()); + if (tail_checksum != crc32c(covered)) { + return Status::Error( + "tail_pointer: tail_checksum mismatch"); + } + + // Only interpret fields after authenticating the complete covered region. + ByteSource src(covered); + + uint32_t magic = 0; + RETURN_IF_ERROR(src.get_fixed32(&magic)); + if (magic != kTailMagic) { + return Status::Error( + "tail_pointer: bad magic"); + } + + uint16_t tail_format_version = 0; + RETURN_IF_ERROR(src.get_fixed16(&tail_format_version)); + if (tail_format_version != kFormatVersion) { + return Status::Error( + "tail_pointer: unsupported container format_version"); + } + RETURN_IF_ERROR(src.get_fixed64(&out->directory_offset)); + RETURN_IF_ERROR(src.get_fixed64(&out->directory_length)); + RETURN_IF_ERROR(src.get_fixed32(&out->directory_crc32c)); + + uint8_t on_disk_size = 0; + RETURN_IF_ERROR(src.get_u8(&on_disk_size)); + if (on_disk_size != kFixedSize) { + return Status::Error( + "tail_pointer: embedded size mismatch"); + } + + DORIS_CHECK(src.eof()); + return Status::OK(); +} + +} // namespace doris::snii::format diff --git a/be/src/storage/index/snii/format/tail_pointer.h b/be/src/storage/index/snii/format/tail_pointer.h new file mode 100644 index 00000000000000..fe2eb6e0bd19fc --- /dev/null +++ b/be/src/storage/index/snii/format/tail_pointer.h @@ -0,0 +1,67 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include "common/status.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/encoding/byte_sink.h" + +namespace doris::snii::format { + +// Fixed-size entry written at the very end of a segment's .idx file. It lets a +// reader locate the raw metadata directory with a single read of the trailing +// tail_pointer_size() bytes (see design spec "fixed tail pointer"). +// +// On-disk layout (all multi-byte fields little-endian, FIXED total size so the +// reader can read exactly the last tail_pointer_size() bytes): +// [u32 magic = kTailMagic] +// [u16 format_version = kFormatVersion] +// [u64 directory_offset] +// [u64 directory_length] +// [u32 directory_crc32c] +// [u8 tail_pointer_size] (== tail_pointer_size()) +// [u32 tail_checksum] (crc32c over all preceding tail-pointer bytes) +// +// The fixed layout deliberately does NOT use the SectionFramer (which is +// variable-length): a footer needs a constant trailing size the reader knows up +// front. +struct TailPointer { + uint64_t directory_offset = 0; + uint64_t directory_length = 0; + uint32_t directory_crc32c = 0; +}; + +// Constant on-disk size of the tail pointer, so the reader knows how many +// trailing bytes to read. +size_t tail_pointer_size(); + +// Appends the fixed-layout tail-pointer bytes (magic / version / fields / size / +// tail_checksum) to sink. Returns Internal if the encoded size would not fit the +// fixed-size contract (a programming error, never expected at runtime). +Status encode_tail_pointer(const TailPointer& tp, ByteSink* sink); + +// Parses the trailing tail-pointer bytes. last_bytes must be exactly +// tail_pointer_size() bytes long. Verifies magic and tail_checksum, then fills +// out with the parsed fields. Wrong magic / checksum mismatch / wrong length -> +// Corruption. +Status decode_tail_pointer(Slice last_bytes, TailPointer* out); + +} // namespace doris::snii::format diff --git a/be/src/storage/index/snii/io/batch_range_fetcher.cpp b/be/src/storage/index/snii/io/batch_range_fetcher.cpp new file mode 100644 index 00000000000000..762c01d1c78024 --- /dev/null +++ b/be/src/storage/index/snii/io/batch_range_fetcher.cpp @@ -0,0 +1,102 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/io/batch_range_fetcher.h" + +#include +#include + +namespace doris::snii::io { +namespace { + +Status checked_end(uint64_t offset, uint64_t len, uint64_t* out) { + if (len > std::numeric_limits::max() - offset) { + return Status::Error( + "batch_range_fetcher: range end overflow"); + } + *out = offset + len; + return Status::OK(); +} + +Status checked_size(uint64_t len, size_t* out) { + if (len > static_cast(std::numeric_limits::max())) { + return Status::Error( + "batch_range_fetcher: physical range too large"); + } + *out = static_cast(len); + return Status::OK(); +} + +} // namespace + +BatchRangeFetcher::BatchRangeFetcher(FileReader* reader, uint64_t coalesce_gap) + : reader_(reader), coalesce_gap_(coalesce_gap) {} + +size_t BatchRangeFetcher::add(uint64_t offset, uint64_t len) { + reqs_.push_back(Req {offset, len}); + return reqs_.size() - 1; +} + +void BatchRangeFetcher::clear() { + reqs_.clear(); + phys_.clear(); +} + +Status BatchRangeFetcher::fetch() { + if (reader_ == nullptr) + return Status::Error( + "batch_range_fetcher: null reader"); + phys_.clear(); + if (reqs_.empty()) return Status::OK(); + + std::vector order(reqs_.size()); + for (size_t i = 0; i < order.size(); ++i) order[i] = i; + std::sort(order.begin(), order.end(), + [&](size_t a, size_t b) { return reqs_[a].offset < reqs_[b].offset; }); + + // Sweep in offset order, merging requests into physical segments. + std::vector segs; + uint64_t cur_start = 0; + uint64_t cur_end = 0; + for (size_t k = 0; k < order.size(); ++k) { + Req& r = reqs_[order[k]]; + uint64_t r_end = 0; + RETURN_IF_ERROR(checked_end(r.offset, r.len, &r_end)); + RETURN_IF_ERROR(checked_size(r.len, &r.len_size)); + const bool disjoint = r.offset > cur_end && r.offset - cur_end > coalesce_gap_; + if (segs.empty() || disjoint) { + segs.push_back(Range {r.offset, 0}); // length finalized below + cur_start = r.offset; + cur_end = r_end; + } else { + cur_end = std::max(cur_end, r_end); + } + r.phys_idx = segs.size() - 1; + RETURN_IF_ERROR(checked_size(r.offset - cur_start, &r.sub_offset)); + RETURN_IF_ERROR(checked_size(cur_end - cur_start, &segs.back().len)); + } + + return reader_->read_batch(segs, &phys_); +} + +Slice BatchRangeFetcher::get(size_t h) const { + const Req& r = reqs_[h]; + const std::vector& buf = phys_[r.phys_idx]; + return Slice(buf.data() + r.sub_offset, r.len_size); +} + +} // namespace doris::snii::io diff --git a/be/src/storage/index/snii/io/batch_range_fetcher.h b/be/src/storage/index/snii/io/batch_range_fetcher.h new file mode 100644 index 00000000000000..1ef41c2fdc75d0 --- /dev/null +++ b/be/src/storage/index/snii/io/batch_range_fetcher.h @@ -0,0 +1,70 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/io/file_reader.h" + +namespace doris::snii::io { + +// Collects the byte ranges a query plan needs, coalesces overlapping/adjacent +// ranges into physical reads, and fetches them in a single batch (one serial +// I/O round as the test-side MeteredFileReader counts it). Callers retrieve each requested range by +// the handle returned from add(). This is the SNII read path's batching layer: +// it front-loads range planning so reads are issued concurrently rather than +// cursor-by-cursor. +class BatchRangeFetcher { +public: + // coalesce_gap: requests separated by a gap <= this many bytes are merged into + // one physical read (reads a few extra bytes to save a request). 0 merges only + // overlapping/adjacent ranges. + explicit BatchRangeFetcher(FileReader* reader, uint64_t coalesce_gap = 0); + + // Registers a desired range; returns a handle usable with get() after fetch(). + size_t add(uint64_t offset, uint64_t len); + + // Coalesces and issues one batched read; fills internal buffers. + Status fetch(); + + // Bytes for handle h (valid only after a successful fetch(), until clear()). + Slice get(size_t h) const; + + size_t pending() const { return reqs_.size(); } + void clear(); + +private: + struct Req { + uint64_t offset; + uint64_t len; + size_t len_size = 0; // validated size_t length after successful fetch() + size_t phys_idx = 0; // index into phys_ after fetch + size_t sub_offset = 0; // byte offset of this req within its physical read + }; + + FileReader* reader_; + uint64_t coalesce_gap_; + std::vector reqs_; + std::vector> phys_; // physical read buffers after fetch +}; + +} // namespace doris::snii::io diff --git a/be/src/storage/index/snii/io/file_reader.h b/be/src/storage/index/snii/io/file_reader.h new file mode 100644 index 00000000000000..533db3f22e0f22 --- /dev/null +++ b/be/src/storage/index/snii/io/file_reader.h @@ -0,0 +1,94 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include + +#include "common/check.h" +#include "common/status.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/io/io_metrics.h" + +namespace doris::snii::io { + +// One logical read request (offset, length). +struct Range { + uint64_t offset = 0; + size_t len = 0; +}; + +// The single physical-read primitive (a BE-internal read_at). All higher layers +// route reads through this so I/O can be accounted and backed by local files or +// object storage interchangeably. +class FileReader { +public: + virtual ~FileReader() = default; + + // Reads exactly len bytes starting at offset into *out (which is resized to + // len). Reading past EOF is an error (Corruption/IoError). + virtual Status read_at(uint64_t offset, size_t len, std::vector* out) = 0; + + // Reads exactly out_len bytes starting at offset into the CALLER-OWNED + // buffer out. Same EOF semantics as read_at. The default delegates to the + // vector overload for readers that predate this entry point; concrete + // readers override it to fill `out` directly -- the blob read shim depends + // on that to avoid a per-refill allocation and a second GiB-scale buffer + // on whole-blob loads (BufferedIndexInput hands large reads straight + // through to the caller buffer). + virtual Status read_into(uint64_t offset, uint8_t* out, size_t out_len) { + if (out_len == 0) { + return Status::OK(); + } + if (out == nullptr) { + return Status::Error( + "read_into: null output buffer"); + } + std::vector scratch; + RETURN_IF_ERROR(read_at(offset, out_len, &scratch)); + // read_at's "reads exactly len bytes" is the contract every implementation + // owes; assert it rather than memcpy past the end of what it produced. + // inherit() guards the identical postcondition the same way. + DORIS_CHECK_EQ(scratch.size(), out_len); + std::memcpy(out, scratch.data(), out_len); + return Status::OK(); + } + + // Reads a batch of ranges that may be served concurrently. The default is a + // sequential loop; backends that model concurrency (the test-side + // MeteredFileReader) or perform real parallel fetches (object storage) + // override this. + virtual Status read_batch(const std::vector& ranges, + std::vector>* outs) { + outs->resize(ranges.size()); + for (size_t i = 0; i < ranges.size(); ++i) { + RETURN_IF_ERROR(read_at(ranges[i].offset, ranges[i].len, &(*outs)[i])); + } + return Status::OK(); + } + + // Total size of the underlying object in bytes. + virtual uint64_t size() const = 0; + + // Optional live metrics. Readers that do not account I/O return nullptr. + virtual const IoMetrics* io_metrics() const { return nullptr; } +}; + +} // namespace doris::snii::io diff --git a/be/src/storage/index/snii/io/file_writer.h b/be/src/storage/index/snii/io/file_writer.h new file mode 100644 index 00000000000000..c14c61d2beaefd --- /dev/null +++ b/be/src/storage/index/snii/io/file_writer.h @@ -0,0 +1,40 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include "common/status.h" +#include "storage/index/snii/common/slice.h" + +namespace doris::snii::io { + +// Append-only writer (no seek-back), so the format can be produced in a single +// streaming pass compatible with S3FileWriter / StreamSinkFileWriter / packed +// writer. All container bytes are written front-to-back; back-references are +// resolved by writing metadata last. +class FileWriter { +public: + virtual ~FileWriter() = default; + + virtual Status append(Slice data) = 0; + virtual Status finalize() = 0; + virtual uint64_t bytes_written() const = 0; +}; + +} // namespace doris::snii::io diff --git a/be/src/storage/index/snii/io/io_metrics.h b/be/src/storage/index/snii/io/io_metrics.h new file mode 100644 index 00000000000000..0e1c628ede0137 --- /dev/null +++ b/be/src/storage/index/snii/io/io_metrics.h @@ -0,0 +1,43 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +namespace doris::snii::io { + +// Object-storage access metrics collected at FileReader boundaries. +struct IoMetrics { + uint64_t read_at_calls = 0; // BE-internal logical read requests issued + uint64_t serial_rounds = 0; // dependent serial I/O rounds + uint64_t range_gets = 0; // remote range GETs after cache coalescing + uint64_t remote_bytes = 0; // bytes fetched from remote + uint64_t total_request_bytes = 0; // sum of requested lengths before cache +}; + +inline IoMetrics delta(const IoMetrics& after, const IoMetrics& before) { + IoMetrics out; + out.read_at_calls = after.read_at_calls - before.read_at_calls; + out.serial_rounds = after.serial_rounds - before.serial_rounds; + out.range_gets = after.range_gets - before.range_gets; + out.remote_bytes = after.remote_bytes - before.remote_bytes; + out.total_request_bytes = after.total_request_bytes - before.total_request_bytes; + return out; +} + +} // namespace doris::snii::io diff --git a/be/src/storage/index/snii/query/bm25_scorer.cpp b/be/src/storage/index/snii/query/bm25_scorer.cpp new file mode 100644 index 00000000000000..dc3e82ad42706d --- /dev/null +++ b/be/src/storage/index/snii/query/bm25_scorer.cpp @@ -0,0 +1,64 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/query/bm25_scorer.h" + +#include +#include + +namespace doris::snii::query { + +double decode_norm(uint8_t encoded) { + return encoded == 0 ? 1.0 : static_cast(encoded); +} + +uint8_t encode_norm(uint64_t doc_length) { + const uint64_t clamped = std::clamp(doc_length, 1, 255); + return static_cast(clamped); +} + +ScorerContext ScorerContext::make(uint64_t n, uint64_t df) { + ScorerContext ctx; + ctx.df_ = df; + const double nn = static_cast(n); + const double dff = static_cast(df); + // idf = log(1 + (N - df + 0.5) / (df + 0.5)); always positive for df <= N. + ctx.idf_ = std::log(1.0 + (nn - dff + 0.5) / (dff + 0.5)); + return ctx; +} + +ScorerContext ScorerContext::from_idf(double idf) { + ScorerContext ctx; + ctx.idf_ = idf; + return ctx; +} + +double ScorerContext::score(double tf, uint8_t encoded_norm, double avgdl, + const Bm25Params& params) const { + const double dl = decode_norm(encoded_norm); + const double denom = tf + params.k1 * (1.0 - params.b + params.b * dl / avgdl); + return idf_ * (tf * (params.k1 + 1.0)) / denom; +} + +double ScorerContext::max_score(uint32_t max_freq, uint8_t min_norm, double avgdl, + const Bm25Params& params) const { + // The score grows monotonically with tf and shrinks with dl, so the per-window + // upper bound uses the window's largest tf and smallest dl (min encoded norm). + return score(max_freq, min_norm, avgdl, params); +} + +} // namespace doris::snii::query diff --git a/be/src/storage/index/snii/query/bm25_scorer.h b/be/src/storage/index/snii/query/bm25_scorer.h new file mode 100644 index 00000000000000..841edb171f07f5 --- /dev/null +++ b/be/src/storage/index/snii/query/bm25_scorer.h @@ -0,0 +1,85 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +// Bm25Scorer -- classic Okapi BM25 relevance scoring over SNII native stats. +// +// Per query term, idf is precomputed once from the collection statistics: +// idf = log(1 + (N - df + 0.5) / (df + 0.5)) +// where N = indexed doc count and df = the term's document frequency. The +// per-document contribution of a term then is: +// score = idf * (tf * (k1 + 1)) / (tf + k1 * (1 - b + b * dl / avgdl)) +// where tf is the in-doc term frequency, dl the document length decoded from the +// 1-byte encoded norm, and avgdl the average document length. +// +// Norm encode/decode (DOCUMENTED CONTRACT): the writer stores doc length as a +// byte-quantized value floor-clamped to [1, 255]; decode is the identity map +// back to a double length. encode_norm(len) = clamp(len, 1, 255); +// decode_norm(b) = (b == 0 ? 1.0 : (double)b). This keeps short docs (len <= 255) +// exact and saturates longer docs at 255, matching the reference oracle. +namespace doris::snii::query { + +// BM25 free parameters. Defaults are the classic Lucene/Elasticsearch values. +struct Bm25Params { + double k1 = 1.2; + double b = 0.75; +}; + +// Decodes a 1-byte encoded norm into a document length. byte 0 maps to 1.0 to +// avoid a zero-length divisor; otherwise it is the byte value itself. +double decode_norm(uint8_t encoded); + +// Encodes a document length into a 1-byte norm (clamped to [1, 255]). Provided +// so writers and test oracles share one quantization. +uint8_t encode_norm(uint64_t doc_length); + +// Per-term scoring context: the precomputed idf and the term's df. Built once per +// query term, then reused for every candidate document of that term. +class ScorerContext { +public: + // Builds the context from collection size n (indexed doc count) and the term's + // document frequency df. avgdl and params are supplied per score call. + static ScorerContext make(uint64_t n, uint64_t df); + + // Builds a context from a collection-scoped IDF that was computed outside + // the segment reader. This keeps segment-local TF/norm decoding separate + // from scanner-collection N/DF aggregation. + static ScorerContext from_idf(double idf); + + double idf() const { return idf_; } + uint64_t df() const { return df_; } + + // Scores one document occurrence: tf is the in-doc term frequency, encoded_norm + // the doc's 1-byte length norm, avgdl the collection average length. + double score(double tf, uint8_t encoded_norm, double avgdl, const Bm25Params& params) const; + + // Upper bound on score() over any document, given a window's maximum tf and the + // shortest doc length in the window (smallest dl maximizes the score). Used by + // the WAND-style block-max pruner. max_freq is the window's max tf; min_norm is + // the smallest encoded norm (=> smallest dl => largest score). + double max_score(uint32_t max_freq, uint8_t min_norm, double avgdl, + const Bm25Params& params) const; + +private: + double idf_ = 0.0; + uint64_t df_ = 0; +}; + +} // namespace doris::snii::query diff --git a/be/src/storage/index/snii/query/boolean_query.cpp b/be/src/storage/index/snii/query/boolean_query.cpp new file mode 100644 index 00000000000000..ab26dee1005efb --- /dev/null +++ b/be/src/storage/index/snii/query/boolean_query.cpp @@ -0,0 +1,124 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/query/boolean_query.h" + +#include +#include +#include +#include + +#include "storage/index/snii/format/dict_entry.h" +#include "storage/index/snii/query/docid_sink.h" +#include "storage/index/snii/query/internal/docid_conjunction.h" +#include "storage/index/snii/query/internal/docid_posting_reader.h" +#include "storage/index/snii/query/internal/docid_union.h" +#include "storage/index/snii/reader/dict_block_cache.h" + +namespace doris::snii::query { + +namespace { + +std::vector unique_terms(const std::vector& terms) { + std::vector out; + out.reserve(terms.size()); + for (const std::string& term : terms) out.emplace_back(term); + std::sort(out.begin(), out.end()); + out.erase(std::unique(out.begin(), out.end()), out.end()); + return out; +} + +Status resolve_or_postings(const reader::LogicalIndexReader& idx, + const std::vector& terms, + std::vector* postings) { + postings->clear(); + // Request-scoped (stack-local, single-threaded) cache: OR terms that fall in the + // same on-demand DICT block read + zstd-decode + CRC-verify that block once + // instead of once per term. The resolved DictEntry is copied out, so the cache + // (and any pin it holds) can die when this returns. The shared reader stays const + // and lock-free -- no lock is ever held across the decode/IO. + reader::DictBlockCache dict_cache; + for (std::string_view term : unique_terms(terms)) { + bool found = false; + format::DictEntry entry; + uint64_t frq_base = 0; + uint64_t prx_base = 0; + RETURN_IF_ERROR(idx.lookup(term, &found, &entry, &frq_base, &prx_base, &dict_cache)); + if (!found) continue; + + postings->push_back({std::move(entry), frq_base, prx_base}); + } + return Status::OK(); +} + +} // namespace + +Status boolean_or(const reader::LogicalIndexReader& idx, const std::vector& terms, + std::vector* docids) { + if (docids == nullptr) + return Status::Error("boolean_or: null out"); + docids->clear(); + if (terms.empty()) return Status::OK(); + + std::vector postings; + RETURN_IF_ERROR(resolve_or_postings(idx, terms, &postings)); + return internal::build_docid_union(idx, postings, docids); +} + +Status boolean_or(const reader::LogicalIndexReader& idx, const std::vector& terms, + std::vector* docids, QueryProfile* profile) { + QueryProfileScope profile_scope(idx.reader(), profile); + return boolean_or(idx, terms, docids); +} + +Status boolean_or(const reader::LogicalIndexReader& idx, const std::vector& terms, + DocIdSink* sink) { + if (sink == nullptr) + return Status::Error("boolean_or: null sink"); + if (terms.empty()) return Status::OK(); + + std::vector postings; + RETURN_IF_ERROR(resolve_or_postings(idx, terms, &postings)); + return internal::emit_docid_union(idx, postings, sink); +} + +Status boolean_and(const reader::LogicalIndexReader& idx, const std::vector& terms, + std::vector* docids) { + if (docids == nullptr) + return Status::Error("boolean_and: null out"); + docids->clear(); + if (terms.empty()) return Status::OK(); + + io::BatchRangeFetcher round1(idx.reader()); + std::vector plans; + bool all_present = false; + RETURN_IF_ERROR(internal::plan_terms(idx, terms, &round1, &plans, &all_present, + /*need_positions=*/false)); + if (!all_present) return Status::OK(); + if (round1.pending() > 0) RETURN_IF_ERROR(round1.fetch()); + RETURN_IF_ERROR(internal::open_preludes(round1, &plans, + /*need_positions=*/false)); + return internal::build_docid_only_conjunction(idx, round1, plans, docids); +} + +Status boolean_and(const reader::LogicalIndexReader& idx, const std::vector& terms, + std::vector* docids, QueryProfile* profile) { + QueryProfileScope profile_scope(idx.reader(), profile); + return boolean_and(idx, terms, docids); +} + +} // namespace doris::snii::query diff --git a/be/src/storage/index/snii/query/boolean_query.h b/be/src/storage/index/snii/query/boolean_query.h new file mode 100644 index 00000000000000..40cbb25644fdc1 --- /dev/null +++ b/be/src/storage/index/snii/query/boolean_query.h @@ -0,0 +1,50 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/query/docid_sink.h" +#include "storage/index/snii/query/query_profile.h" +#include "storage/index/snii/reader/logical_index_reader.h" + +// boolean_or -- MATCH_ANY semantics: return the sorted docid set containing at +// least one query term. Empty terms or all-absent terms produce an empty +// result. Duplicate input terms are ignored semantically and do not duplicate +// output docids. +namespace doris::snii::query { + +Status boolean_or(const reader::LogicalIndexReader& idx, const std::vector& terms, + std::vector* docids); +Status boolean_or(const reader::LogicalIndexReader& idx, const std::vector& terms, + std::vector* docids, QueryProfile* profile); +Status boolean_or(const reader::LogicalIndexReader& idx, const std::vector& terms, + DocIdSink* sink); + +// boolean_and (MATCH all-terms): sorted docid set of docs containing EVERY +// term, no positional constraint. Valid on docs-only indexes. Empty terms or +// any absent term -> empty result. +Status boolean_and(const reader::LogicalIndexReader& idx, const std::vector& terms, + std::vector* docids); +Status boolean_and(const reader::LogicalIndexReader& idx, const std::vector& terms, + std::vector* docids, QueryProfile* profile); + +} // namespace doris::snii::query diff --git a/be/src/storage/index/snii/query/count_query.cpp b/be/src/storage/index/snii/query/count_query.cpp new file mode 100644 index 00000000000000..3cc77ff9605e88 --- /dev/null +++ b/be/src/storage/index/snii/query/count_query.cpp @@ -0,0 +1,80 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/query/count_query.h" + +#include +#include + +#include "roaring/roaring.hh" +#include "storage/index/snii/format/dict_entry.h" +#include "storage/index/snii/query/internal/query_test_counters.h" + +namespace doris::snii::query { + +using format::DictEntry; +using reader::LogicalIndexReader; + +Status count_only_term_df(const LogicalIndexReader& idx, std::string_view term, uint64_t* count) { + if (count == nullptr) { + return Status::Error("count_only_term_df: null out"); + } + bool found = false; + DictEntry entry; + uint64_t frq_base = 0; + uint64_t prx_base = 0; + RETURN_IF_ERROR(idx.lookup(term, &found, &entry, &frq_base, &prx_base)); + *count = found ? entry.df : 0; + SNII_QUERY_COUNT(count_fastpath_hits); + return Status::OK(); +} + +Status fabricate_null_disjoint_count_bitmap(uint64_t count, const roaring::Roaring& nulls, + roaring::Roaring* out) { + if (out == nullptr) { + return Status::Error( + "fabricate_null_disjoint_count_bitmap: null out"); + } + roaring::Roaring result; + if (count > 0) { + // [0, count + |nulls|) holds at least `count` non-null ids: at most + // |nulls| of its members are null. count counts only non-null docs, so + // the window end never exceeds the segment doc count (row space). + const uint64_t window_end = count + nulls.cardinality(); + if (window_end > uint64_t(std::numeric_limits::max()) + 1) { + return Status::Error( + "fabricate_null_disjoint_count_bitmap: count {} + null count {} exceeds the " + "uint32 docid domain (corrupt df or null bitmap)", + count, nulls.cardinality()); + } + result.addRange(0, window_end); + result -= nulls; + uint32_t last_kept = 0; + // Keep exactly the first `count` survivors (select ranks are 0-based). + if (!result.select(static_cast(count - 1), &last_kept)) { + return Status::Error( + "fabricate_null_disjoint_count_bitmap: window [0, {}) holds fewer than {} " + "non-null ids (corrupt df or null bitmap)", + window_end, count); + } + result.removeRange(uint64_t(last_kept) + 1, window_end); + } + *out = std::move(result); + return Status::OK(); +} + +} // namespace doris::snii::query diff --git a/be/src/storage/index/snii/query/count_query.h b/be/src/storage/index/snii/query/count_query.h new file mode 100644 index 00000000000000..bae76b4d91eed8 --- /dev/null +++ b/be/src/storage/index/snii/query/count_query.h @@ -0,0 +1,69 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include "common/status.h" +#include "storage/index/snii/reader/logical_index_reader.h" + +// Forward-declare the CRoaring C++ bitmap so this header stays free of the +// (large) roaring include; the concrete type is only needed in the .cpp. +namespace roaring { +class Roaring; +} // namespace roaring + +// count_query -- G02 single-term count-only fast path primitives. They answer +// "how many docs match" from a dict entry alone, without reading .frq bytes. +// Multi-term queries, prefix/regexp/wildcard expansion, and phrases execute the +// normal query path. Deletes and extra predicates are a caller responsibility; +// see SniiIndexReader::_try_count_only_fastpath and the SegmentIterator guards +// in count_on_index_fastpath.h. +namespace doris::snii::query { + +// df of `term` in this segment without decoding postings. An absent term is a +// deterministic answer too: *count = 0 (mirrors term_query's empty result). +// Increments the count_fastpath_hits test seam. +Status count_only_term_df(const reader::LogicalIndexReader& idx, std::string_view term, + uint64_t* count); + +// Builds the fabricated count bitmap for a segment WITH a null bitmap: exactly +// `count` row ids DISJOINT from `nulls` (the first `count` non-null row ids, +// all < count + |nulls|). Why disjoint: the MATCH machinery unconditionally +// subtracts the segment null bitmap from every index result +// (FunctionMatchBase -> InvertedIndexResultBitmap::mask_out_null). Real +// postings never contain null docs -- the writer adds NO tokens for a null doc +// (scalar add_nulls) and a NULL array row is stored as an empty range (zero +// tokens) -- so that subtraction is a no-op on true results and df already IS +// the exact match count regardless of nulls. A naive [0, df) range however MAY +// collide with null row ids and be shrunk by mask_out_null; picking the ids +// from the non-null space makes the subtraction provably a no-op, preserving +// cardinality == df end to end. +// +// The window bound is doc-count-free: count counts only non-null docs, so +// count + |nulls| <= segment doc count and [0, count + |nulls|) always holds +// >= count non-null ids; every fabricated id therefore stays inside the +// segment's [0, num_rows) row space. Errors (id space would exceed the uint32 +// docid domain, or the window unexpectedly holds fewer than `count` survivors) +// only occur on a corrupt index; callers treat them as "fall through to the +// decode path", never as a fabricated answer. +Status fabricate_null_disjoint_count_bitmap(uint64_t count, const roaring::Roaring& nulls, + roaring::Roaring* out); + +} // namespace doris::snii::query diff --git a/be/src/storage/index/snii/query/docid_conjunction.cpp b/be/src/storage/index/snii/query/docid_conjunction.cpp new file mode 100644 index 00000000000000..1e3b892b4abfac --- /dev/null +++ b/be/src/storage/index/snii/query/docid_conjunction.cpp @@ -0,0 +1,929 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/query/internal/docid_conjunction.h" + +#include +#include +#include +#include +#include + +#include "common/check.h" +#include "storage/index/snii/format/frq_pod.h" +#include "storage/index/snii/query/internal/docid_set_ops.h" +#include "storage/index/snii/query/internal/query_test_counters.h" +#include "storage/index/snii/reader/windowed_posting.h" + +namespace doris::snii::query::internal { + +using format::DictEntry; +using format::DictEntryEnc; +using format::DictEntryKind; +using format::FrqPreludeReader; +using format::WindowMeta; +using reader::LogicalIndexReader; + +namespace { + +using CandidateIt = std::vector::const_iterator; + +constexpr uint32_t kBoundedSpanBitsetDocs = 16 * 1024; +constexpr size_t kBoundedSpanBitsetWords = kBoundedSpanBitsetDocs / 64; +constexpr size_t kBoundedSpanBitsetMinInput = 32; + +struct CandidateRange { + size_t begin = 0; + size_t end = 0; +}; + +Status slim_frq_docs_len(const DictEntry& entry, uint64_t win_len, uint64_t* out) { + if (entry.frq_docs_len > win_len) { + return Status::Error( + "docid_conjunction: slim frq_docs_len exceeds frq window"); + } + *out = entry.frq_docs_len > 0 ? entry.frq_docs_len : win_len; + return Status::OK(); +} + +Status add_u64(uint64_t lhs, uint64_t rhs, const char* message, uint64_t* out) { + if (rhs > std::numeric_limits::max() - lhs) { + return Status::Error(message); + } + *out = lhs + rhs; + return Status::OK(); +} + +Status posting_abs_offset(const LogicalIndexReader& idx, uint64_t base, uint64_t delta, + const char* message, uint64_t* out) { + uint64_t with_base = 0; + RETURN_IF_ERROR(add_u64(idx.section_refs().posting_region.offset, base, message, &with_base)); + return add_u64(with_base, delta, message, out); +} + +Status configure_term_plan(const LogicalIndexReader& idx, bool need_positions, + io::BatchRangeFetcher* fetcher, TermPlan* p) { + p->df = p->entry.df; + p->pod_ref = (p->entry.kind == DictEntryKind::kPodRef); + p->windowed = p->pod_ref && p->entry.enc == DictEntryEnc::kWindowed; + if (p->windowed) { + uint64_t prelude_abs = 0; + RETURN_IF_ERROR(posting_abs_offset(idx, p->frq_base, p->entry.frq_off_delta, + "docid_conjunction: prelude offset overflow", + &prelude_abs)); + p->prelude_handle = fetcher->add(prelude_abs, p->entry.prelude_len); + } else if (p->pod_ref) { + uint64_t foff = 0; + uint64_t flen = 0; + uint64_t poff = 0; + uint64_t plen = 0; + RETURN_IF_ERROR(idx.resolve_frq_window(p->entry, p->frq_base, &foff, &flen)); + uint64_t frq_fetch = flen; + RETURN_IF_ERROR(slim_frq_docs_len(p->entry, flen, &frq_fetch)); + p->frq_handle = fetcher->add(foff, frq_fetch); + if (need_positions) { + RETURN_IF_ERROR(idx.resolve_prx_window(p->entry, p->prx_base, &poff, &plen)); + p->prx_handle = fetcher->add(poff, plen); + } + } + return Status::OK(); +} + +std::vector all_windows(const FrqPreludeReader& prelude) { + std::vector ws(prelude.n_windows()); + for (uint32_t i = 0; i < prelude.n_windows(); ++i) ws[i] = i; + return ws; +} + +std::vector ascending_df_order(const std::vector& plans) { + std::vector order(plans.size()); + for (size_t i = 0; i < plans.size(); ++i) order[i] = i; + std::sort(order.begin(), order.end(), + [&](size_t a, size_t b) { return plans[a].df < plans[b].df; }); + return order; +} + +Status first_docid_in_window(const WindowMeta& meta, uint32_t window_ordinal, uint32_t* first) { + if (window_ordinal == 0) { + *first = 0; + return Status::OK(); + } + if (meta.win_base >= std::numeric_limits::max()) { + return Status::Error( + "docid_conjunction: window base exceeds docid range"); + } + *first = static_cast(meta.win_base + 1); + if (*first > meta.last_docid) { + return Status::Error( + "docid_conjunction: invalid window docid range"); + } + return Status::OK(); +} + +Status is_dense_full_window(const WindowMeta& meta, uint32_t window_ordinal, bool* full) { + uint32_t first = 0; + RETURN_IF_ERROR(first_docid_in_window(meta, window_ordinal, &first)); + const uint64_t width = static_cast(meta.last_docid) - first + 1; + *full = meta.doc_count == width; + return Status::OK(); +} + +Status append_docid_range(uint32_t first, uint32_t last, std::vector* out) { + if (last < first) { + return Status::Error( + "docid_conjunction: invalid dense docid range"); + } + const uint64_t count64 = static_cast(last) - first + 1; + if (count64 > static_cast(std::numeric_limits::max() - out->size())) { + return Status::Error( + "docid_conjunction: dense docid range too large"); + } + out->reserve(out->size() + static_cast(count64)); + uint32_t docid = first; + while (true) { + out->push_back(docid); + if (docid == last) break; + ++docid; + } + return Status::OK(); +} + +CandidateRange find_candidate_range(const std::vector& candidates, size_t* search_begin, + uint32_t first, uint32_t last) { + const auto from = candidates.begin() + *search_begin; + const auto begin = std::lower_bound(from, candidates.end(), first); + const auto end = std::upper_bound(begin, candidates.end(), last); + *search_begin = static_cast(end - candidates.begin()); + return {.begin = static_cast(begin - candidates.begin()), + .end = static_cast(end - candidates.begin())}; +} + +void append_candidate_range(CandidateIt begin, CandidateIt end, std::vector* out) { + out->insert(out->end(), begin, end); +} + +void append_new_chunk_docids_to_out(const DocidChunk& chunk, size_t chunk_docids_begin, + std::vector* out) { + out->insert(out->end(), chunk.docids.begin() + chunk_docids_begin, chunk.docids.end()); +} + +void clear_ordinals_if_all_term_docs_selected(const std::vector& term_docids, + DocidChunk* chunk) { + if (chunk->docids.size() == term_docids.size() && !chunk->docids.empty() && + chunk->docids.front() == term_docids.front() && + chunk->docids.back() == term_docids.back()) { + chunk->prx_doc_ordinals.clear(); + } +} + +bool append_term_docs_if_candidates_cover_span(CandidateIt begin, CandidateIt end, + const std::vector& term_docids, + std::vector* out, DocidChunk* chunk) { + const uint32_t first = term_docids.front(); + const uint32_t last = term_docids.back(); + const uint64_t width = static_cast(last) - first + 1; + const size_t candidate_count = static_cast(end - begin); + if (width > candidate_count) { + return false; + } + + const auto span_begin = *begin == first ? begin : std::lower_bound(begin, end, first); + if (span_begin == end || *span_begin != first) { + return false; + } + if (static_cast(end - span_begin) < width) { + return false; + } + + const auto span_last = span_begin + static_cast(width) - 1; + if (*span_last != last) { + return false; + } + + const size_t chunk_docids_begin = chunk->docids.size(); + chunk->docids.insert(chunk->docids.end(), term_docids.begin(), term_docids.end()); + append_new_chunk_docids_to_out(*chunk, chunk_docids_begin, out); + return true; +} + +Status append_candidate_range_with_ordinals(CandidateIt begin, CandidateIt end, uint32_t first, + uint32_t last, std::vector* out, + DocidChunk* chunk) { + const size_t candidate_count = static_cast(end - begin); + chunk->docids.reserve(candidate_count); + const uint64_t width = static_cast(last) - first + 1; + if (width > std::numeric_limits::max()) { + return Status::Error( + "docid_conjunction: dense window exceeds doc count range"); + } + chunk->prx_doc_count = static_cast(width); + const bool full_dense_range = + candidate_count == width && begin != end && *begin == first && *(end - 1) == last; + const size_t chunk_docids_begin = chunk->docids.size(); + if (full_dense_range) { + chunk->docids.insert(chunk->docids.end(), begin, end); + append_new_chunk_docids_to_out(*chunk, chunk_docids_begin, out); + return Status::OK(); + } + chunk->prx_doc_ordinals.reserve(chunk->prx_doc_ordinals.size() + candidate_count); + for (auto it = begin; it != end; ++it) { + chunk->docids.push_back(*it); + chunk->prx_doc_ordinals.push_back(*it - first); + } + append_new_chunk_docids_to_out(*chunk, chunk_docids_begin, out); + return Status::OK(); +} + +bool intersect_dense_term_span_with_ordinals(CandidateIt begin, CandidateIt end, + const std::vector& term_docids, + size_t candidate_count, std::vector* out, + DocidChunk* chunk) { + const uint32_t first = term_docids.front(); + const uint32_t last = term_docids.back(); + const uint64_t width = static_cast(last) - first + 1; + if (term_docids.size() > width) { + return false; + } + const uint64_t missing_count = width - term_docids.size(); + if (missing_count != 0 && + (missing_count * 8 > width || missing_count >= candidate_count || + missing_count > static_cast(std::numeric_limits::max()))) { + return false; + } + + const size_t chunk_docids_begin = chunk->docids.size(); + if (missing_count == 0) { + for (auto it = begin; it != end; ++it) { + if (*it < first) { + continue; + } + if (*it > last) { + break; + } + chunk->docids.push_back(*it); + chunk->prx_doc_ordinals.push_back(*it - first); + } + append_new_chunk_docids_to_out(*chunk, chunk_docids_begin, out); + clear_ordinals_if_all_term_docs_selected(term_docids, chunk); + return true; + } + + std::vector missing; + missing.reserve(static_cast(missing_count)); + uint32_t expect = first; + for (uint32_t docid : term_docids) { + while (expect < docid) { + missing.push_back(expect); + ++expect; + } + if (docid < std::numeric_limits::max()) { + expect = docid + 1; + } + } + while (expect <= last) { + missing.push_back(expect); + if (expect == std::numeric_limits::max()) { + break; + } + ++expect; + } + + size_t miss = 0; + for (auto it = begin; it != end; ++it) { + if (*it < first) { + continue; + } + if (*it > last) { + break; + } + while (miss < missing.size() && missing[miss] < *it) { + ++miss; + } + if (miss < missing.size() && missing[miss] == *it) { + continue; + } + chunk->docids.push_back(*it); + chunk->prx_doc_ordinals.push_back(static_cast(*it - first - miss)); + } + append_new_chunk_docids_to_out(*chunk, chunk_docids_begin, out); + clear_ordinals_if_all_term_docs_selected(term_docids, chunk); + return true; +} + +bool intersect_bounded_span_with_ordinals(CandidateIt begin, CandidateIt end, + const std::vector& term_docids, + size_t candidate_count, std::vector* out, + DocidChunk* chunk) { + if (candidate_count < kBoundedSpanBitsetMinInput || + term_docids.size() < kBoundedSpanBitsetMinInput) { + return false; + } + + const uint32_t first = std::min(*begin, term_docids.front()); + const uint32_t last = std::max(*(end - 1), term_docids.back()); + const uint64_t width = static_cast(last) - first + 1; + if (width > kBoundedSpanBitsetDocs || term_docids.size() > width) { + return false; + } + + const auto word_count = static_cast((width + 63) >> 6); + std::array bits; + std::fill_n(bits.begin(), word_count, 0); + for (uint32_t docid : term_docids) { + const uint32_t off = docid - first; + bits[off >> 6] |= 1ULL << (off & 63); + } + + std::array ordinal_base; + uint32_t ordinal = 0; + for (size_t word = 0; word < word_count; ++word) { + ordinal_base[word] = ordinal; + ordinal += static_cast(__builtin_popcountll(bits[word])); + } + + const size_t chunk_docids_begin = chunk->docids.size(); + for (auto it = begin; it != end; ++it) { + const uint32_t off = *it - first; + const size_t word = off >> 6; + const uint64_t mask = 1ULL << (off & 63); + if ((bits[word] & mask) == 0) { + continue; + } + chunk->docids.push_back(*it); + chunk->prx_doc_ordinals.push_back( + ordinal_base[word] + + static_cast(__builtin_popcountll(bits[word] & (mask - 1)))); + } + append_new_chunk_docids_to_out(*chunk, chunk_docids_begin, out); + clear_ordinals_if_all_term_docs_selected(term_docids, chunk); + return true; +} + +size_t log2_ceil(size_t n) { + if (n <= 1) return 1; + --n; + size_t bits = 0; + while (n != 0) { + ++bits; + n >>= 1; + } + return bits; +} + +void intersect_window_candidate_range(CandidateIt begin, CandidateIt end, + const std::vector& term_docids, uint32_t first, + uint32_t last, std::vector* out) { + const size_t candidate_count = static_cast(end - begin); + if (candidate_count == 0 || term_docids.empty()) return; + + const uint64_t width = static_cast(last) - first + 1; + const uint64_t missing_count = term_docids.size() <= width ? width - term_docids.size() : width; + if (term_docids.size() <= width && missing_count != 0 && missing_count * 8 <= width && + missing_count < candidate_count) { + std::vector missing; + missing.reserve(static_cast(missing_count)); + uint32_t expect = first; + for (uint32_t docid : term_docids) { + while (expect < docid) { + missing.push_back(expect); + ++expect; + } + if (docid < std::numeric_limits::max()) expect = docid + 1; + } + while (expect <= last) { + missing.push_back(expect); + if (expect == std::numeric_limits::max()) break; + ++expect; + } + size_t miss = 0; + for (auto it = begin; it != end; ++it) { + while (miss < missing.size() && missing[miss] < *it) ++miss; + if (miss == missing.size() || missing[miss] != *it) out->push_back(*it); + } + return; + } + + const size_t probes_per_candidate = log2_ceil(term_docids.size()) + 1; + if (candidate_count < term_docids.size() / probes_per_candidate) { + for (auto it = begin; it != end; ++it) { + if (std::binary_search(term_docids.begin(), term_docids.end(), *it)) { + out->push_back(*it); + } + } + return; + } + std::set_intersection(begin, end, term_docids.begin(), term_docids.end(), + std::back_inserter(*out)); +} + +Status intersect_window_candidate_range_with_ordinals(CandidateIt begin, CandidateIt end, + const std::vector& term_docids, + std::vector* out, + DocidChunk* chunk) { + if (term_docids.size() > std::numeric_limits::max()) { + return Status::Error( + "docid_conjunction: prx doc count exceeds u32"); + } + chunk->prx_doc_count = static_cast(term_docids.size()); + if (begin == end || term_docids.empty()) return Status::OK(); + + const size_t candidate_count = static_cast(end - begin); + const size_t max_matches = std::min(candidate_count, term_docids.size()); + out->reserve(out->size() + max_matches); + chunk->docids.reserve(chunk->docids.size() + max_matches); + if (candidate_count == term_docids.size() && *begin == term_docids.front() && + *(end - 1) == term_docids.back() && std::equal(begin, end, term_docids.begin())) { + const size_t chunk_docids_begin = chunk->docids.size(); + chunk->docids.insert(chunk->docids.end(), begin, end); + append_new_chunk_docids_to_out(*chunk, chunk_docids_begin, out); + return Status::OK(); + } + if (append_term_docs_if_candidates_cover_span(begin, end, term_docids, out, chunk)) { + return Status::OK(); + } + + chunk->prx_doc_ordinals.reserve(chunk->prx_doc_ordinals.size() + max_matches); + if (intersect_dense_term_span_with_ordinals(begin, end, term_docids, candidate_count, out, + chunk)) { + return Status::OK(); + } + if (intersect_bounded_span_with_ordinals(begin, end, term_docids, candidate_count, out, + chunk)) { + return Status::OK(); + } + + const size_t probes_per_candidate = log2_ceil(term_docids.size()) + 1; + if (candidate_count < term_docids.size() / probes_per_candidate) { + const size_t chunk_docids_begin = chunk->docids.size(); + size_t doc_index = 0; + for (auto it = begin; it != end; ++it) { + const auto found = + std::lower_bound(term_docids.begin() + doc_index, term_docids.end(), *it); + if (found == term_docids.end()) break; + doc_index = static_cast(found - term_docids.begin()); + if (*found != *it) continue; + chunk->docids.push_back(*it); + chunk->prx_doc_ordinals.push_back(static_cast(doc_index)); + ++doc_index; + } + append_new_chunk_docids_to_out(*chunk, chunk_docids_begin, out); + clear_ordinals_if_all_term_docs_selected(term_docids, chunk); + return Status::OK(); + } + + const size_t probes_per_term_doc = log2_ceil(candidate_count) + 1; + if (term_docids.size() < candidate_count / probes_per_term_doc) { + const size_t chunk_docids_begin = chunk->docids.size(); + auto candidate_it = begin; + for (size_t doc_index = 0; doc_index < term_docids.size(); ++doc_index) { + const uint32_t docid = term_docids[doc_index]; + candidate_it = std::lower_bound(candidate_it, end, docid); + if (candidate_it == end) break; + if (*candidate_it != docid) continue; + chunk->docids.push_back(docid); + chunk->prx_doc_ordinals.push_back(static_cast(doc_index)); + ++candidate_it; + } + append_new_chunk_docids_to_out(*chunk, chunk_docids_begin, out); + clear_ordinals_if_all_term_docs_selected(term_docids, chunk); + return Status::OK(); + } + + const size_t chunk_docids_begin = chunk->docids.size(); + size_t doc_index = 0; + for (auto it = begin; it != end; ++it) { + while (doc_index < term_docids.size() && term_docids[doc_index] < *it) { + ++doc_index; + } + if (doc_index == term_docids.size()) break; + if (term_docids[doc_index] != *it) continue; + chunk->docids.push_back(*it); + chunk->prx_doc_ordinals.push_back(static_cast(doc_index)); + ++doc_index; + } + append_new_chunk_docids_to_out(*chunk, chunk_docids_begin, out); + clear_ordinals_if_all_term_docs_selected(term_docids, chunk); + return Status::OK(); +} + +bool should_scan_all_windows(const LogicalIndexReader& idx, const TermPlan& p, + size_t candidate_count) { + const size_t window_count = p.prelude.n_windows(); + if (candidate_count > window_count * 64) return true; + + const uint64_t doc_count = idx.stats().doc_count; + const bool near_full = doc_count != 0 && static_cast(p.df) * 10 >= doc_count * 9; + return near_full && candidate_count > window_count * 4; +} + +Status decode_flat_docids_only(const io::BatchRangeFetcher& round1, const TermPlan& p, + std::vector* docids) { + Slice dd; + if (p.pod_ref) { + dd = round1.get(p.frq_handle); + } else { + RETURN_IF_ERROR(inline_dd_region(p.entry, &dd)); + } + return format::decode_dd_region(dd, p.entry.dd_meta, /*win_base=*/0, docids); +} + +struct WindowWork { + uint32_t ordinal = 0; + WindowMeta meta; + CandidateRange candidates; + size_t handle = 0; + bool dense_full = false; +}; + +Status emit_dense_full_window_docids(const WindowWork& f, const std::vector* candidates, + std::vector& out, DocidSource* source) { + uint32_t first = 0; + RETURN_IF_ERROR(first_docid_in_window(f.meta, f.ordinal, &first)); + if (source != nullptr) { + DocidChunk chunk; + chunk.windowed = true; + chunk.window = f.ordinal; + chunk.prx_doc_count = f.meta.doc_count; + if (candidates == nullptr) { + RETURN_IF_ERROR(append_docid_range(first, f.meta.last_docid, &chunk.docids)); + } else { + const auto begin = candidates->begin() + f.candidates.begin; + const auto end = candidates->begin() + f.candidates.end; + RETURN_IF_ERROR(append_candidate_range_with_ordinals(begin, end, first, + f.meta.last_docid, &out, &chunk)); + } + source->chunks.push_back(std::move(chunk)); + } + if (candidates == nullptr) { + RETURN_IF_ERROR(append_docid_range(first, f.meta.last_docid, &out)); + } else if (source == nullptr) { + append_candidate_range(candidates->begin() + f.candidates.begin, + candidates->begin() + f.candidates.end, &out); + } + return Status::OK(); +} + +Status emit_decoded_window_docids(const WindowWork& f, const io::BatchRangeFetcher& fetcher, + const std::vector* candidates, + std::vector& out, DocidSource* source, + std::vector& docs, std::vector& freqs, + std::vector>& positions) { + docs.clear(); + freqs.clear(); + positions.clear(); + RETURN_IF_ERROR(reader::decode_window_slices(f.meta, fetcher.get(f.handle), Slice(), Slice(), + /*want_positions=*/false, /*want_freq=*/false, + &docs, &freqs, &positions)); + if (source != nullptr) { + DocidChunk chunk; + chunk.windowed = true; + chunk.window = f.ordinal; + if (candidates == nullptr) { + chunk.docids = docs; + if (docs.size() > std::numeric_limits::max()) { + return Status::Error( + "docid_conjunction: prx doc count exceeds u32"); + } + chunk.prx_doc_count = static_cast(docs.size()); + source->chunks.push_back(std::move(chunk)); + } else { + const auto begin = candidates->begin() + f.candidates.begin; + const auto end = candidates->begin() + f.candidates.end; + RETURN_IF_ERROR( + intersect_window_candidate_range_with_ordinals(begin, end, docs, &out, &chunk)); + if (!chunk.docids.empty()) { + source->chunks.push_back(std::move(chunk)); + } + } + } + if (candidates == nullptr) { + out.insert(out.end(), docs.begin(), docs.end()); + return Status::OK(); + } + if (source != nullptr) { + return Status::OK(); + } + uint32_t first = 0; + RETURN_IF_ERROR(first_docid_in_window(f.meta, f.ordinal, &first)); + intersect_window_candidate_range(candidates->begin() + f.candidates.begin, + candidates->begin() + f.candidates.end, docs, first, + f.meta.last_docid, &out); + return Status::OK(); +} + +Status collect_windowed_docids_only(const LogicalIndexReader& idx, const TermPlan& p, + const std::vector& windows, + const std::vector* candidates, + std::vector* out, DocidSource* source) { + io::BatchRangeFetcher fetcher(idx.reader(), reader::kSameTermCoalesceGap); + std::vector work; + work.reserve(windows.size()); + out->reserve(candidates == nullptr ? p.entry.df : candidates->size()); + size_t candidate_search_begin = 0; + for (uint32_t w : windows) { + WindowMeta meta; + RETURN_IF_ERROR(p.prelude.window(w, &meta)); + uint32_t first = 0; + RETURN_IF_ERROR(first_docid_in_window(meta, w, &first)); + CandidateRange candidate_range; + if (candidates != nullptr) { + candidate_range = find_candidate_range(*candidates, &candidate_search_begin, first, + meta.last_docid); + if (candidate_range.begin == candidate_range.end) { + continue; + } + } + bool dense_full = false; + RETURN_IF_ERROR(is_dense_full_window(meta, w, &dense_full)); + if (dense_full) { + work.push_back(WindowWork { + .ordinal = w, .meta = meta, .candidates = candidate_range, .dense_full = true}); + continue; + } + + reader::WindowAbsRange range; + RETURN_IF_ERROR(reader::windowed_window_range( + idx, p.entry, p.frq_base, p.prx_base, p.prelude, w, + /*want_positions=*/false, /*want_freq=*/false, &range)); + WindowWork f; + f.ordinal = w; + f.meta = meta; + f.candidates = candidate_range; + f.handle = fetcher.add(range.dd_off, range.dd_len); + work.push_back(f); + } + if (fetcher.pending() > 0) { + RETURN_IF_ERROR(fetcher.fetch()); + } + + std::vector docs; + std::vector freqs; + std::vector> positions; + for (const WindowWork& f : work) { + if (f.dense_full) { + RETURN_IF_ERROR(emit_dense_full_window_docids(f, candidates, *out, source)); + continue; + } + RETURN_IF_ERROR(emit_decoded_window_docids(f, fetcher, candidates, *out, source, docs, + freqs, positions)); + } + return Status::OK(); +} + +Status collect_docids_only(const LogicalIndexReader& idx, const io::BatchRangeFetcher& round1, + const TermPlan& p, const std::vector* candidates, + std::vector* out, DocidSource* source) { + if (p.windowed) { + std::vector windows; + if (candidates == nullptr) { + windows = all_windows(p.prelude); + } else if (should_scan_all_windows(idx, p, candidates->size())) { + // Dense candidate sets cover most windows; for near-full terms this also + // avoids a thousands-to-millions probe covering-window cursor pass with no + // byte win. + windows = all_windows(p.prelude); + } else { + p.prelude.select_covering_windows(*candidates, &windows); + } + return collect_windowed_docids_only(idx, p, windows, candidates, out, source); + } + + std::vector term_docids; + RETURN_IF_ERROR(decode_flat_docids_only(round1, p, &term_docids)); + if (source != nullptr) { + DocidChunk chunk; + if (term_docids.size() > std::numeric_limits::max()) { + return Status::Error( + "docid_conjunction: prx doc count exceeds u32"); + } + chunk.prx_doc_count = static_cast(term_docids.size()); + if (candidates == nullptr) { + chunk.docids = term_docids; + } else if (!term_docids.empty()) { + const auto begin = std::ranges::lower_bound(*candidates, term_docids.front()); + const auto end = std::upper_bound(begin, candidates->end(), term_docids.back()); + RETURN_IF_ERROR(intersect_window_candidate_range_with_ordinals(begin, end, term_docids, + out, &chunk)); + } + if (candidates == nullptr || !chunk.docids.empty()) { + source->chunks.push_back(std::move(chunk)); + } + } + if (candidates == nullptr) { + *out = std::move(term_docids); + return Status::OK(); + } + if (source != nullptr) { + return Status::OK(); + } + *out = intersect_sorted(*candidates, term_docids); + return Status::OK(); +} + +Status run_docid_only_conjunction_impl(const LogicalIndexReader& idx, + const io::BatchRangeFetcher& round1, + const std::vector& plans, + const std::vector* initial_candidates, + std::vector* candidates, + std::vector* sources) { + if (sources != nullptr) { + sources->assign(plans.size(), DocidSource {}); + } + candidates->clear(); + if (plans.empty()) { + // No terms: the result is the initial candidate set verbatim (or empty). + if (initial_candidates != nullptr) { + *candidates = *initial_candidates; + } + return Status::OK(); + } + if (initial_candidates != nullptr && initial_candidates->empty()) { + return Status::OK(); + } + const std::vector order = ascending_df_order(plans); + for (size_t k = 0; k < order.size(); ++k) { + const size_t ti = order[k]; + std::vector next; + DocidSource* source = sources == nullptr ? nullptr : &(*sources)[ti]; + // k == 0 intersects against the (const) initial_candidates DIRECTLY, so + // the whole set is never copied once per plan -- the previous code seeded + // *candidates = *initial_candidates before the loop, which for a single + // plan (e.g. one phrase-prefix tail verified against the leading-term + // expected docids) was an O(|initial|) copy per call with no benefit. + // k > 0 chains on the previous term's already-whittled result. + const std::vector* input_candidates = k == 0 ? initial_candidates : candidates; + RETURN_IF_ERROR( + collect_docids_only(idx, round1, plans[ti], input_candidates, &next, source)); + if (source != nullptr && k + 1 == order.size()) { + source->docids_are_final_candidates = true; + } + *candidates = std::move(next); + if (candidates->empty()) { + return Status::OK(); + } + } + return Status::OK(); +} + +} // namespace + +Status resolve_query_term(const LogicalIndexReader& idx, std::string_view term, + ResolvedQueryTerm* resolved, bool* found) { + *found = false; + RETURN_IF_ERROR( + idx.lookup(term, found, &resolved->entry, &resolved->frq_base, &resolved->prx_base)); + return Status::OK(); +} + +Status resolve_query_terms_batch(const LogicalIndexReader& idx, + const std::vector& terms, + std::vector* resolved, + std::vector* found) { + DCHECK(std::ranges::is_sorted(terms)); + DCHECK(std::adjacent_find(terms.begin(), terms.end()) == terms.end()); + resolved->assign(terms.size(), ResolvedQueryTerm {}); + found->assign(terms.size(), 0); + std::vector lookup_results; + RETURN_IF_ERROR(idx.lookup_batch(terms, &lookup_results)); + for (size_t i = 0; i < terms.size(); ++i) { + (*found)[i] = lookup_results[i].found; + if (lookup_results[i].found) { + (*resolved)[i].entry = std::move(lookup_results[i].entry); + (*resolved)[i].frq_base = lookup_results[i].frq_base; + (*resolved)[i].prx_base = lookup_results[i].prx_base; + } + } + return Status::OK(); +} + +Status plan_terms(const LogicalIndexReader& idx, const std::vector& terms, + io::BatchRangeFetcher* fetcher, std::vector* plans, bool* all_present, + bool need_positions) { + *all_present = true; + plans->resize(terms.size()); + for (size_t i = 0; i < terms.size(); ++i) { + ResolvedQueryTerm resolved; + bool found = false; + RETURN_IF_ERROR(resolve_query_term(idx, terms[i], &resolved, &found)); + if (!found) { + *all_present = false; + return Status::OK(); + } + TermPlan& p = (*plans)[i]; + p.order = i; + p.entry = std::move(resolved.entry); + p.frq_base = resolved.frq_base; + p.prx_base = resolved.prx_base; + RETURN_IF_ERROR(configure_term_plan(idx, need_positions, fetcher, &p)); + } + return Status::OK(); +} + +Status plan_resolved_terms(const LogicalIndexReader& idx, + const std::vector& terms, + io::BatchRangeFetcher* fetcher, std::vector* plans, + bool need_positions) { + plans->resize(terms.size()); + for (size_t i = 0; i < terms.size(); ++i) { + TermPlan& p = (*plans)[i]; + p.order = i; + p.entry = terms[i].entry; + SNII_QUERY_COUNT(resolved_term_entry_copies); + p.frq_base = terms[i].frq_base; + p.prx_base = terms[i].prx_base; + RETURN_IF_ERROR(configure_term_plan(idx, need_positions, fetcher, &p)); + } + return Status::OK(); +} + +Status plan_resolved_terms(const LogicalIndexReader& idx, std::vector&& terms, + io::BatchRangeFetcher* fetcher, std::vector* plans, + bool need_positions) { + plans->resize(terms.size()); + for (size_t i = 0; i < terms.size(); ++i) { + TermPlan& p = (*plans)[i]; + p.order = i; +#ifdef BE_TEST + const bool has_frq_payload = !terms[i].entry.frq_bytes.empty(); + const bool has_prx_payload = !terms[i].entry.prx_bytes.empty(); + const uint8_t* const frq_payload = terms[i].entry.frq_bytes.data(); + const uint8_t* const prx_payload = terms[i].entry.prx_bytes.data(); +#endif + p.entry = std::move(terms[i].entry); + SNII_QUERY_COUNT(resolved_term_entry_moves); +#ifdef BE_TEST + SNII_QUERY_ADD( + resolved_term_payload_pointer_reuses, + static_cast(has_frq_payload && p.entry.frq_bytes.data() == frq_payload) + + static_cast(has_prx_payload && + p.entry.prx_bytes.data() == prx_payload)); +#endif + p.frq_base = terms[i].frq_base; + p.prx_base = terms[i].prx_base; + RETURN_IF_ERROR(configure_term_plan(idx, need_positions, fetcher, &p)); + } + return Status::OK(); +} + +Status open_preludes(const io::BatchRangeFetcher& fetcher, std::vector* plans, + bool need_positions) { + for (TermPlan& p : *plans) { + if (!p.windowed) continue; + RETURN_IF_ERROR(FrqPreludeReader::open(fetcher.get(p.prelude_handle), &p.prelude)); + if (need_positions && !p.prelude.has_prx()) { + return Status::Error( + "docid_conjunction: windowed prelude has no positions"); + } + } + return Status::OK(); +} + +Status inline_dd_region(const DictEntry& entry, Slice* out) { + if (entry.dd_meta.disk_len > entry.frq_bytes.size()) { + return Status::Error( + "docid_conjunction: inline dd region exceeds frq bytes"); + } + *out = Slice(entry.frq_bytes.data(), static_cast(entry.dd_meta.disk_len)); + return Status::OK(); +} + +Status build_docid_only_conjunction(const LogicalIndexReader& idx, + const io::BatchRangeFetcher& round1, + const std::vector& plans, + std::vector* candidates) { + return run_docid_only_conjunction_impl(idx, round1, plans, nullptr, candidates, nullptr); +} + +Status build_docid_only_conjunction(const LogicalIndexReader& idx, + const io::BatchRangeFetcher& round1, + const std::vector& plans, + std::vector* candidates, + std::vector* sources) { + return run_docid_only_conjunction_impl(idx, round1, plans, nullptr, candidates, sources); +} + +Status filter_docids_by_conjunction(const LogicalIndexReader& idx, + const io::BatchRangeFetcher& round1, + const std::vector& plans, + const std::vector& initial_candidates, + std::vector* candidates, + std::vector* sources) { + return run_docid_only_conjunction_impl(idx, round1, plans, &initial_candidates, candidates, + sources); +} + +} // namespace doris::snii::query::internal diff --git a/be/src/storage/index/snii/query/docid_posting_reader.cpp b/be/src/storage/index/snii/query/docid_posting_reader.cpp new file mode 100644 index 00000000000000..4446167652f203 --- /dev/null +++ b/be/src/storage/index/snii/query/docid_posting_reader.cpp @@ -0,0 +1,383 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/query/internal/docid_posting_reader.h" + +#include +#include + +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/format/dict_entry.h" +#include "storage/index/snii/format/frq_pod.h" +#include "storage/index/snii/format/frq_prelude.h" +#include "storage/index/snii/io/batch_range_fetcher.h" +#include "storage/index/snii/reader/windowed_posting.h" + +namespace doris::snii::query::internal { + +using format::DictEntry; +using format::DictEntryEnc; +using format::DictEntryKind; +using format::FrqPreludeReader; +using format::WindowMeta; +using reader::LogicalIndexReader; + +namespace { + +Status decode_flat_docs(const DictEntry& entry, Slice dd_region, std::vector* docids) { + return format::decode_dd_region(dd_region, entry.dd_meta, + /*win_base=*/0, docids); +} + +Status decode_inline_docs(const DictEntry& entry, std::vector* docids) { + if (entry.dd_meta.disk_len > entry.frq_bytes.size()) { + return Status::Error( + "docid_posting_reader: inline dd region exceeds frq bytes"); + } + return decode_flat_docs( + entry, Slice(entry.frq_bytes.data(), static_cast(entry.dd_meta.disk_len)), + docids); +} + +Status slim_docs_fetch_len(const DictEntry& entry, uint64_t win_len, uint64_t* out) { + if (entry.frq_docs_len > win_len) { + return Status::Error( + "docid_posting_reader: slim frq_docs_len exceeds frq window"); + } + *out = entry.frq_docs_len > 0 ? entry.frq_docs_len : win_len; + return Status::OK(); +} + +Status posting_reader_add_u64(uint64_t lhs, uint64_t rhs, const char* message, uint64_t* out) { + if (rhs > std::numeric_limits::max() - lhs) { + return Status::Error(message); + } + *out = lhs + rhs; + return Status::OK(); +} + +Status prelude_abs(const LogicalIndexReader& idx, const DictEntry& entry, uint64_t frq_base, + uint64_t* out) { + uint64_t with_base = 0; + RETURN_IF_ERROR(posting_reader_add_u64(idx.section_refs().posting_region.offset, frq_base, + "docid_posting_reader: prelude offset overflow", + &with_base)); + return posting_reader_add_u64(with_base, entry.frq_off_delta, + "docid_posting_reader: prelude offset overflow", out); +} + +Status validate_windowed_docs_prefix(const DictEntry& entry) { + if (entry.prelude_len == 0) { + return Status::Error( + "docid_posting_reader: windowed entry has no prelude"); + } + if (entry.prelude_len > entry.frq_docs_len) { + return Status::Error( + "docid_posting_reader: prelude_len exceeds docs prefix"); + } + if (entry.frq_docs_len > entry.frq_len) { + return Status::Error( + "docid_posting_reader: docs prefix exceeds frq_len"); + } + return Status::OK(); +} + +struct FlatPlan { + size_t out_index = 0; + const DictEntry* entry = nullptr; + size_t handle = 0; +}; + +struct WindowPlan { + size_t out_index = 0; + const ResolvedDocidPosting* posting = nullptr; + size_t prefix_handle = 0; +}; + +Status plan_flat_docs(const LogicalIndexReader& idx, const ResolvedDocidPosting& posting, + io::BatchRangeFetcher* fetcher, FlatPlan* plan) { + uint64_t win_abs = 0; + uint64_t win_len = 0; + RETURN_IF_ERROR(idx.resolve_frq_window(posting.entry, posting.frq_base, &win_abs, &win_len)); + uint64_t docs_len = 0; + RETURN_IF_ERROR(slim_docs_fetch_len(posting.entry, win_len, &docs_len)); + plan->handle = fetcher->add(win_abs, docs_len); + return Status::OK(); +} + +Status plan_window_prefix(const LogicalIndexReader& idx, WindowPlan* plan, + io::BatchRangeFetcher* fetcher) { + const ResolvedDocidPosting& posting = *plan->posting; + RETURN_IF_ERROR(validate_windowed_docs_prefix(posting.entry)); + uint64_t abs = 0; + RETURN_IF_ERROR(prelude_abs(idx, posting.entry, posting.frq_base, &abs)); + plan->prefix_handle = fetcher->add(abs, posting.entry.frq_docs_len); + return Status::OK(); +} + +// Records a non-inline (flat or windowed) posting into the per-encoding plan lists, +// adding the windowed prefix range to the shared fetcher. Flat ranges are added in a +// later pass (after all preludes are registered). Shared by the batched and streamed +// docid readers so both fetch the whole OR in one round. `posting` must outlive the +// plans (its address is captured); callers pass an element of a stable vector. +Status plan_noninline_posting(const LogicalIndexReader& idx, const ResolvedDocidPosting& posting, + size_t out_index, io::BatchRangeFetcher* fetcher, + std::vector* flat_plans, + std::vector* window_plans) { + if (posting.entry.enc == DictEntryEnc::kWindowed) { + WindowPlan plan; + plan.out_index = out_index; + plan.posting = &posting; + RETURN_IF_ERROR(plan_window_prefix(idx, &plan, fetcher)); + window_plans->push_back(std::move(plan)); + return Status::OK(); + } + FlatPlan plan; + plan.out_index = out_index; + plan.entry = &posting.entry; + flat_plans->push_back(plan); + return Status::OK(); +} + +Status window_dd_slice(Slice dd_block, const WindowMeta& meta, Slice* out) { + if (meta.dd_off > dd_block.size() || meta.dd_disk_len > dd_block.size() - meta.dd_off) { + return Status::Error( + "docid_posting_reader: window dd range out of prefix"); + } + *out = dd_block.subslice(static_cast(meta.dd_off), + static_cast(meta.dd_disk_len)); + return Status::OK(); +} + +Status posting_reader_first_docid_in_window(const WindowMeta& meta, uint32_t window_ordinal, + uint32_t* first) { + if (window_ordinal == 0) { + *first = 0; + return Status::OK(); + } + if (meta.win_base >= std::numeric_limits::max()) { + return Status::Error( + "docid_posting_reader: window base exceeds docid range"); + } + *first = static_cast(meta.win_base + 1); + if (*first > meta.last_docid) { + return Status::Error( + "docid_posting_reader: invalid window docid range"); + } + return Status::OK(); +} + +Status posting_reader_is_dense_full_window(const WindowMeta& meta, uint32_t window_ordinal, + bool* full) { + uint32_t first = 0; + RETURN_IF_ERROR(posting_reader_first_docid_in_window(meta, window_ordinal, &first)); + const uint64_t width = static_cast(meta.last_docid) - first + 1; + *full = meta.doc_count == width; + return Status::OK(); +} + +Status decode_flat_plan(const io::BatchRangeFetcher& fetcher, const FlatPlan& plan, + std::vector* out) { + return decode_flat_docs(*plan.entry, fetcher.get(plan.handle), out); +} + +Status decode_window_prefix_plan(const io::BatchRangeFetcher& fetcher, const WindowPlan& plan, + DocIdSink* sink); + +Status decode_window_prefix_plan(const io::BatchRangeFetcher& fetcher, const WindowPlan& plan, + std::vector* out) { + VectorDocIdSink sink(*out); + return decode_window_prefix_plan(fetcher, plan, &sink); +} + +Status decode_window_prefix_plan(const io::BatchRangeFetcher& fetcher, const WindowPlan& plan, + DocIdSink* sink) { + const DictEntry& entry = plan.posting->entry; + const Slice prefix = fetcher.get(plan.prefix_handle); + if (entry.prelude_len > prefix.size()) { + return Status::Error( + "docid_posting_reader: short docs prefix"); + } + const size_t prelude_len = static_cast(entry.prelude_len); + FrqPreludeReader prelude; + RETURN_IF_ERROR(FrqPreludeReader::open(prefix.subslice(0, prelude_len), &prelude)); + const uint64_t dd_block_len = prelude.dd_block_len(); + if (dd_block_len > static_cast(std::numeric_limits::max()) - prelude_len) { + return Status::Error( + "docid_posting_reader: docs prefix length overflow"); + } + const size_t expected_prefix_len = prelude_len + static_cast(dd_block_len); + if (prefix.size() != expected_prefix_len) { + return Status::Error( + "docid_posting_reader: docs prefix length mismatch"); + } + const Slice dd_block = prefix.subslice(prelude_len, prefix.size() - prelude_len); + std::vector docs; + std::vector freqs; + std::vector> positions; + for (uint32_t w = 0; w < prelude.n_windows(); ++w) { + WindowMeta meta; + Slice dd_region; + RETURN_IF_ERROR(prelude.window(w, &meta)); + RETURN_IF_ERROR(window_dd_slice(dd_block, meta, &dd_region)); + bool dense_full = false; + RETURN_IF_ERROR(posting_reader_is_dense_full_window(meta, w, &dense_full)); + if (dense_full) { + uint32_t first = 0; + RETURN_IF_ERROR(posting_reader_first_docid_in_window(meta, w, &first)); + RETURN_IF_ERROR(sink->append_range(first, static_cast(meta.last_docid) + 1)); + continue; + } + docs.clear(); + freqs.clear(); + positions.clear(); + RETURN_IF_ERROR(reader::decode_window_slices( + meta, dd_region, Slice(), Slice(), /*want_positions=*/false, + /*want_freq=*/false, &docs, &freqs, &positions)); + RETURN_IF_ERROR(sink->append_sorted(docs)); + } + return Status::OK(); +} + +} // namespace + +Status read_docid_posting(const LogicalIndexReader& idx, const DictEntry& entry, uint64_t frq_base, + uint64_t prx_base, std::vector* docids) { + if (docids == nullptr) { + return Status::Error("docid_posting_reader: null out"); + } + docids->clear(); + VectorDocIdSink sink(*docids); + return read_docid_posting(idx, entry, frq_base, prx_base, &sink); +} + +Status read_docid_posting(const LogicalIndexReader& idx, const DictEntry& entry, uint64_t frq_base, + uint64_t prx_base, DocIdSink* sink) { + if (sink == nullptr) { + return Status::Error("docid_posting_reader: null sink"); + } + ResolvedDocidPosting posting {entry, frq_base, prx_base}; + if (posting.entry.kind == DictEntryKind::kInline) { + std::vector docs; + RETURN_IF_ERROR(decode_inline_docs(posting.entry, &docs)); + return sink->append_sorted(docs); + } + + io::BatchRangeFetcher docs_fetcher(idx.reader()); + if (posting.entry.enc == DictEntryEnc::kWindowed) { + WindowPlan plan; + plan.out_index = 0; + plan.posting = &posting; + RETURN_IF_ERROR(plan_window_prefix(idx, &plan, &docs_fetcher)); + if (docs_fetcher.pending() > 0) RETURN_IF_ERROR(docs_fetcher.fetch()); + return decode_window_prefix_plan(docs_fetcher, plan, sink); + } + + FlatPlan plan; + plan.out_index = 0; + plan.entry = &posting.entry; + RETURN_IF_ERROR(plan_flat_docs(idx, posting, &docs_fetcher, &plan)); + if (docs_fetcher.pending() > 0) RETURN_IF_ERROR(docs_fetcher.fetch()); + std::vector docs; + RETURN_IF_ERROR(decode_flat_plan(docs_fetcher, plan, &docs)); + return sink->append_sorted(docs); +} + +Status read_docid_postings_batched(const LogicalIndexReader& idx, + const std::vector& postings, + std::vector>* docids) { + if (docids == nullptr) { + return Status::Error( + "docid_posting_reader: null batched out"); + } + docids->clear(); + docids->resize(postings.size()); + + std::vector flat_plans; + std::vector window_plans; + io::BatchRangeFetcher docs_fetcher(idx.reader()); + + for (size_t i = 0; i < postings.size(); ++i) { + const ResolvedDocidPosting& posting = postings[i]; + if (posting.entry.kind == DictEntryKind::kInline) { + RETURN_IF_ERROR(decode_inline_docs(posting.entry, &(*docids)[i])); + continue; + } + RETURN_IF_ERROR( + plan_noninline_posting(idx, posting, i, &docs_fetcher, &flat_plans, &window_plans)); + } + + for (FlatPlan& plan : flat_plans) { + const ResolvedDocidPosting& posting = postings[plan.out_index]; + RETURN_IF_ERROR(plan_flat_docs(idx, posting, &docs_fetcher, &plan)); + } + if (docs_fetcher.pending() > 0) RETURN_IF_ERROR(docs_fetcher.fetch()); + + for (const FlatPlan& plan : flat_plans) { + RETURN_IF_ERROR(decode_flat_plan(docs_fetcher, plan, &(*docids)[plan.out_index])); + } + for (const WindowPlan& plan : window_plans) { + RETURN_IF_ERROR(decode_window_prefix_plan(docs_fetcher, plan, &(*docids)[plan.out_index])); + } + return Status::OK(); +} + +Status emit_docid_postings_streamed(const LogicalIndexReader& idx, + const std::vector& postings, + DocIdSink* sink) { + if (sink == nullptr) { + return Status::Error( + "docid_posting_reader: null streamed sink"); + } + + std::vector flat_plans; + std::vector window_plans; + io::BatchRangeFetcher docs_fetcher(idx.reader()); + // One scratch buffer reused across flat/inline postings: clear() keeps capacity, + // so at most one growth total instead of a fresh vector per posting. + std::vector scratch; + + for (size_t i = 0; i < postings.size(); ++i) { + const ResolvedDocidPosting& posting = postings[i]; + if (posting.entry.kind == DictEntryKind::kInline) { + scratch.clear(); + RETURN_IF_ERROR(decode_inline_docs(posting.entry, &scratch)); + RETURN_IF_ERROR(sink->append_sorted(scratch)); + continue; + } + RETURN_IF_ERROR( + plan_noninline_posting(idx, posting, i, &docs_fetcher, &flat_plans, &window_plans)); + } + + for (FlatPlan& plan : flat_plans) { + const ResolvedDocidPosting& posting = postings[plan.out_index]; + RETURN_IF_ERROR(plan_flat_docs(idx, posting, &docs_fetcher, &plan)); + } + if (docs_fetcher.pending() > 0) RETURN_IF_ERROR(docs_fetcher.fetch()); + + for (const FlatPlan& plan : flat_plans) { + scratch.clear(); + RETURN_IF_ERROR(decode_flat_plan(docs_fetcher, plan, &scratch)); + RETURN_IF_ERROR(sink->append_sorted(scratch)); + } + for (const WindowPlan& plan : window_plans) { + RETURN_IF_ERROR(decode_window_prefix_plan(docs_fetcher, plan, sink)); + } + return Status::OK(); +} + +} // namespace doris::snii::query::internal diff --git a/be/src/storage/index/snii/query/docid_set_ops.cpp b/be/src/storage/index/snii/query/docid_set_ops.cpp new file mode 100644 index 00000000000000..5f8931f4146e16 --- /dev/null +++ b/be/src/storage/index/snii/query/docid_set_ops.cpp @@ -0,0 +1,127 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/query/internal/docid_set_ops.h" + +#include +#include +#include +#include + +namespace doris::snii::query::internal { + +std::vector intersect_sorted(const std::vector& a, + const std::vector& b) { + std::vector out; + out.reserve(std::min(a.size(), b.size())); + std::set_intersection(a.begin(), a.end(), b.begin(), b.end(), std::back_inserter(out)); + return out; +} + +void union_sorted_into(std::vector* acc, const std::vector& next) { + std::vector merged; + merged.reserve(acc->size() + next.size()); + std::set_union(acc->begin(), acc->end(), next.begin(), next.end(), std::back_inserter(merged)); + *acc = std::move(merged); +} + +std::vector union_sorted_many(const std::vector>& lists, + size_t reserve_cap) { + constexpr size_t kLinearFanInMax = 8; + struct Cursor { + uint32_t docid = 0; + size_t list = 0; + size_t offset = 0; + }; + struct GreaterDocId { + bool operator()(const Cursor& a, const Cursor& b) const { return a.docid > b.docid; } + }; + + size_t non_empty = 0; + size_t total = 0; + std::priority_queue, GreaterDocId> heap; + for (size_t i = 0; i < lists.size(); ++i) { + if (lists[i].empty()) continue; + ++non_empty; + total += lists[i].size(); + heap.push(Cursor {lists[i][0], i, 0}); + } + // The union is at most `total` (exactly that for disjoint inputs); reserve to it + // so the output grows in a single allocation rather than O(log) geometric + // reallocations. Cap guards against over-reserving for heavily-overlapping inputs. + const size_t reserve_hint = std::min(total, reserve_cap); + if (non_empty == 0) return {}; + if (non_empty == 1) { + for (const std::vector& docs : lists) { + if (!docs.empty()) return docs; + } + } + + if (non_empty <= kLinearFanInMax) { + std::vector offsets(lists.size(), 0); + std::vector out; + out.reserve(reserve_hint); + bool has_last = false; + uint32_t last = 0; + for (;;) { + bool found = false; + uint32_t next = 0; + for (size_t i = 0; i < lists.size(); ++i) { + if (offsets[i] >= lists[i].size()) continue; + const uint32_t docid = lists[i][offsets[i]]; + if (!found || docid < next) { + found = true; + next = docid; + } + } + if (!found) break; + if (!has_last || next != last) { + out.push_back(next); + last = next; + has_last = true; + } + for (size_t i = 0; i < lists.size(); ++i) { + while (offsets[i] < lists[i].size() && lists[i][offsets[i]] == next) { + ++offsets[i]; + } + } + } + return out; + } + + std::vector out; + out.reserve(reserve_hint); + bool has_last = false; + uint32_t last = 0; + while (!heap.empty()) { + const Cursor cur = heap.top(); + heap.pop(); + if (!has_last || cur.docid != last) { + out.push_back(cur.docid); + last = cur.docid; + has_last = true; + } + const size_t next_offset = cur.offset + 1; + const std::vector& docs = lists[cur.list]; + if (next_offset < docs.size()) { + heap.push(Cursor {docs[next_offset], cur.list, next_offset}); + } + } + return out; +} + +} // namespace doris::snii::query::internal diff --git a/be/src/storage/index/snii/query/docid_sink.h b/be/src/storage/index/snii/query/docid_sink.h new file mode 100644 index 00000000000000..604d3dde8bdbb8 --- /dev/null +++ b/be/src/storage/index/snii/query/docid_sink.h @@ -0,0 +1,90 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include + +#include "common/status.h" + +namespace doris::snii::query { + +// Bulk docid handoff for query operators. Each span is sorted ascending; callers +// that need a single vector can use VectorDocIdSink. +class DocIdSink { +public: + virtual ~DocIdSink() = default; + virtual Status append_sorted(std::span docids) = 0; + virtual Status append_range(uint32_t first, uint64_t last_exclusive) = 0; + + // True iff the sink deduplicates and globally orders on its own (e.g. a Roaring + // bitmap via addMany/addRange). For such sinks a multi-term OR can stream each + // posting straight in -- skipping the per-term vector materialization plus the + // K-way merge accumulator. Sinks that hand back a single globally-sorted, + // deduplicated vector (VectorDocIdSink) keep the default false, so callers + // materialize + merge before appending. The gate must stay conservative: + // streaming several postings into a non-dedup sink would break that contract. + virtual bool dedups() const { return false; } +}; + +class VectorDocIdSink final : public DocIdSink { +public: + explicit VectorDocIdSink(std::vector& docids) : docids_(docids) {} + + Status append_sorted(std::span docids) override { + docids_.insert(docids_.end(), docids.begin(), docids.end()); + return Status::OK(); + } + + Status append_range(uint32_t first, uint64_t last_exclusive) override { + if (last_exclusive <= first) { + return Status::OK(); + } + if (last_exclusive > static_cast(std::numeric_limits::max()) + 1) { + return Status::Error( + "docid_sink: range exceeds uint32 docid space"); + } + const uint64_t count = last_exclusive - first; + if (count > static_cast(docids_.max_size() - docids_.size())) { + return Status::Error("docid_sink: range too large"); + } + // GEOMETRIC BULK reserve -- never an exact one: append_range can be + // called once per docid run for a query, and an exact + // reserve(size()+count) caps capacity at "just enough" so the next + // append reallocates + memcpys the whole accumulated vector -- + // quadratic total memcpy across runs (same anti-pattern as the writer's + // add_nulls). Doubling on overflow keeps the O(count) amortization AND + // makes one large range pay at most one reallocation. + const size_t need = docids_.size() + static_cast(count); + if (need > docids_.capacity()) { + docids_.reserve(std::max(need, docids_.capacity() * 2)); + } + for (uint64_t docid = first; docid < last_exclusive; ++docid) { + docids_.push_back(static_cast(docid)); + } + return Status::OK(); + } + +private: + std::vector& docids_; +}; + +} // namespace doris::snii::query diff --git a/be/src/storage/index/snii/query/docid_union.cpp b/be/src/storage/index/snii/query/docid_union.cpp new file mode 100644 index 00000000000000..f1296b18cd685e --- /dev/null +++ b/be/src/storage/index/snii/query/docid_union.cpp @@ -0,0 +1,58 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/query/internal/docid_union.h" + +#include + +#include "storage/index/snii/query/internal/docid_set_ops.h" + +namespace doris::snii::query::internal { + +Status build_docid_union(const reader::LogicalIndexReader& idx, + const std::vector& postings, + std::vector* out) { + if (out == nullptr) + return Status::Error("docid_union: null out"); + out->clear(); + if (postings.empty()) return Status::OK(); + + std::vector> docs_by_posting; + RETURN_IF_ERROR(read_docid_postings_batched(idx, postings, &docs_by_posting)); + *out = union_sorted_many(docs_by_posting); + return Status::OK(); +} + +Status emit_docid_union(const reader::LogicalIndexReader& idx, + const std::vector& postings, DocIdSink* sink) { + if (sink == nullptr) + return Status::Error("docid_union: null sink"); + if (postings.empty()) return Status::OK(); + // A dedup-capable sink (Roaring) orders + dedups across postings itself, so stream + // each posting straight in over a single shared fetch round -- no per-term vector + // or K-way merge accumulator. A plain (non-dedup) sink keeps the materialize+merge + // path so its single-span contract (globally sorted, deduplicated) holds. + if (sink->dedups()) { + return emit_docid_postings_streamed(idx, postings, sink); + } + std::vector acc; + RETURN_IF_ERROR(build_docid_union(idx, postings, &acc)); + if (acc.empty()) return Status::OK(); + return sink->append_sorted(acc); +} + +} // namespace doris::snii::query::internal diff --git a/be/src/storage/index/snii/query/internal/docid_conjunction.h b/be/src/storage/index/snii/query/internal/docid_conjunction.h new file mode 100644 index 00000000000000..ea63d9446a0071 --- /dev/null +++ b/be/src/storage/index/snii/query/internal/docid_conjunction.h @@ -0,0 +1,113 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/format/dict_entry.h" +#include "storage/index/snii/format/frq_prelude.h" +#include "storage/index/snii/io/batch_range_fetcher.h" +#include "storage/index/snii/reader/logical_index_reader.h" + +namespace doris::snii::query::internal { + +struct ResolvedQueryTerm { + format::DictEntry entry; + uint64_t frq_base = 0; + uint64_t prx_base = 0; +}; + +struct TermPlan { + format::DictEntry entry; + uint64_t frq_base = 0; + uint64_t prx_base = 0; + uint32_t df = 0; + size_t order = 0; + size_t frq_handle = 0; + size_t prx_handle = 0; + size_t prelude_handle = 0; + bool pod_ref = false; + bool windowed = false; + format::FrqPreludeReader prelude; +}; + +struct DocidChunk { + std::vector docids; + std::vector prx_doc_ordinals; + uint32_t prx_doc_count = 0; + bool windowed = false; + uint32_t window = 0; +}; + +struct DocidSource { + std::vector chunks; + bool docids_are_final_candidates = false; +}; + +Status resolve_query_term(const reader::LogicalIndexReader& idx, std::string_view term, + ResolvedQueryTerm* resolved, bool* found); + +// Resolves one sorted, duplicate-free term batch through bounded physical DICT +// reads. Results stay aligned with `terms`; absent terms have found[i]=0. +Status resolve_query_terms_batch(const reader::LogicalIndexReader& idx, + const std::vector& terms, + std::vector* resolved, + std::vector* found); + +Status plan_terms(const reader::LogicalIndexReader& idx, const std::vector& terms, + io::BatchRangeFetcher* fetcher, std::vector* plans, bool* all_present, + bool need_positions); + +Status plan_resolved_terms(const reader::LogicalIndexReader& idx, + const std::vector& terms, + io::BatchRangeFetcher* fetcher, std::vector* plans, + bool need_positions); + +Status plan_resolved_terms(const reader::LogicalIndexReader& idx, + std::vector&& terms, io::BatchRangeFetcher* fetcher, + std::vector* plans, bool need_positions); + +Status open_preludes(const io::BatchRangeFetcher& fetcher, std::vector* plans, + bool need_positions); + +Status inline_dd_region(const format::DictEntry& entry, Slice* out); + +Status build_docid_only_conjunction(const reader::LogicalIndexReader& idx, + const io::BatchRangeFetcher& round1, + const std::vector& plans, + std::vector* candidates); + +Status build_docid_only_conjunction(const reader::LogicalIndexReader& idx, + const io::BatchRangeFetcher& round1, + const std::vector& plans, + std::vector* candidates, + std::vector* sources); + +Status filter_docids_by_conjunction(const reader::LogicalIndexReader& idx, + const io::BatchRangeFetcher& round1, + const std::vector& plans, + const std::vector& initial_candidates, + std::vector* candidates, + std::vector* sources); + +} // namespace doris::snii::query::internal diff --git a/be/src/storage/index/snii/query/internal/docid_posting_reader.h b/be/src/storage/index/snii/query/internal/docid_posting_reader.h new file mode 100644 index 00000000000000..0230a1f526039a --- /dev/null +++ b/be/src/storage/index/snii/query/internal/docid_posting_reader.h @@ -0,0 +1,62 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include "common/status.h" +#include "storage/index/snii/format/dict_entry.h" +#include "storage/index/snii/query/docid_sink.h" +#include "storage/index/snii/reader/logical_index_reader.h" + +namespace doris::snii::query::internal { + +struct ResolvedDocidPosting { + format::DictEntry entry; + uint64_t frq_base = 0; + uint64_t prx_base = 0; +}; + +// Decodes the docid-only posting for a resolved term. The caller owns term +// lookup and can batch/plan lookups independently; this module owns only the +// three posting encodings (inline, slim pod_ref, windowed pod_ref). +Status read_docid_posting(const reader::LogicalIndexReader& idx, const format::DictEntry& entry, + uint64_t frq_base, uint64_t prx_base, std::vector* docids); + +Status read_docid_posting(const reader::LogicalIndexReader& idx, const format::DictEntry& entry, + uint64_t frq_base, uint64_t prx_base, query::DocIdSink* sink); + +// Batch counterpart for multi-term docid-only operators. Windowed terms share one +// prelude fetch round and one docid fetch round, so OR-style operators pay by +// stage rather than by term. +Status read_docid_postings_batched(const reader::LogicalIndexReader& idx, + const std::vector& postings, + std::vector>* docids); + +// Streaming counterpart of read_docid_postings_batched for a dedup-capable sink +// (DocIdSink::dedups()==true, e.g. a Roaring bitmap). Shares the exact same single +// docid fetch round, but decodes each posting straight into the sink -- dense-full +// windows via append_range (run-preserving), the rest via append_sorted from one +// reused scratch buffer -- so no per-term vector or K-way merge accumulator is +// materialized. The sink dedups/orders across postings. One I/O round is preserved. +Status emit_docid_postings_streamed(const reader::LogicalIndexReader& idx, + const std::vector& postings, + query::DocIdSink* sink); + +} // namespace doris::snii::query::internal diff --git a/be/src/storage/index/snii/query/internal/docid_set_ops.h b/be/src/storage/index/snii/query/internal/docid_set_ops.h new file mode 100644 index 00000000000000..f651e930a1cdb3 --- /dev/null +++ b/be/src/storage/index/snii/query/internal/docid_set_ops.h @@ -0,0 +1,39 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include + +namespace doris::snii::query::internal { + +std::vector intersect_sorted(const std::vector& a, + const std::vector& b); + +void union_sorted_into(std::vector* acc, const std::vector& next); + +// Sorted-deduplicated union of many sorted lists. The output is reserved by the +// summed input size (the union is at most the total of all inputs; for disjoint +// inputs it is exactly that), capped by `reserve_cap` so heavily-overlapping +// inputs (union << total) do not over-reserve. Default cap = no cap. +std::vector union_sorted_many(const std::vector>& lists, + size_t reserve_cap = std::numeric_limits::max()); + +} // namespace doris::snii::query::internal diff --git a/be/src/storage/index/snii/query/internal/docid_union.h b/be/src/storage/index/snii/query/internal/docid_union.h new file mode 100644 index 00000000000000..3243c082bbeec2 --- /dev/null +++ b/be/src/storage/index/snii/query/internal/docid_union.h @@ -0,0 +1,38 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include "common/status.h" +#include "storage/index/snii/query/docid_sink.h" +#include "storage/index/snii/query/internal/docid_posting_reader.h" +#include "storage/index/snii/reader/logical_index_reader.h" + +namespace doris::snii::query::internal { + +// Reads already-resolved docid postings in planned batches, merges them as a +// sorted deduplicated union, then emits one bulk span to the sink. +Status build_docid_union(const reader::LogicalIndexReader& idx, + const std::vector& postings, + std::vector* out); + +Status emit_docid_union(const reader::LogicalIndexReader& idx, + const std::vector& postings, DocIdSink* sink); + +} // namespace doris::snii::query::internal diff --git a/be/src/storage/index/snii/query/internal/exact_phrase_stream_matcher.h b/be/src/storage/index/snii/query/internal/exact_phrase_stream_matcher.h new file mode 100644 index 00000000000000..23367557441d56 --- /dev/null +++ b/be/src/storage/index/snii/query/internal/exact_phrase_stream_matcher.h @@ -0,0 +1,150 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "common/check.h" +#include "common/status.h" +#include "storage/index/snii/query/internal/position_math.h" + +namespace doris::snii::query::internal { +namespace exact_phrase_stream_matcher_detail { + +template +Status finish_document(std::span cursors, std::span phrase_plan_index) { + Status first_error; + for (size_t cursor_index : phrase_plan_index) { + const Status status = cursors[cursor_index].finish_doc(); + if (!status.ok() && first_error.ok()) { + first_error = status; + } + } + return first_error; +} + +template +Status seek_document(std::span cursors, std::span phrase_plan_index, + uint32_t docid) { + for (size_t cursor_index : phrase_plan_index) { + RETURN_IF_ERROR(cursors[cursor_index].seek(docid)); + } + return Status::OK(); +} + +template +Status advance_to(Cursor* cursor, uint32_t target, uint32_t* position, bool* available) { + do { + RETURN_IF_ERROR(cursor->next_position(position, available)); + } while (*available && *position < target); + return Status::OK(); +} + +} // namespace exact_phrase_stream_matcher_detail + +template +void validate_exact_phrase_stream_inputs(std::span cursors, + std::span phrase_plan_index, + std::span position_offsets) { + DORIS_CHECK_GT(phrase_plan_index.size(), 1); + DORIS_CHECK_EQ(phrase_plan_index.size(), position_offsets.size()); + for (size_t clause = 0; clause < phrase_plan_index.size(); ++clause) { + DORIS_CHECK_LT(phrase_plan_index[clause], cursors.size()); + if (clause != 0) { + DORIS_CHECK_LT(position_offsets[clause - 1], position_offsets[clause]); + } + for (size_t preceding = 0; preceding < clause; ++preceding) { + DORIS_CHECK_NE(phrase_plan_index[preceding], phrase_plan_index[clause]); + } + } +} + +template +Status match_exact_phrase_document(std::span cursors, + std::span phrase_plan_index, + std::span position_offsets, uint32_t docid, + bool* matched) { + DORIS_CHECK(matched != nullptr); + + *matched = false; + RETURN_IF_ERROR( + exact_phrase_stream_matcher_detail::seek_document(cursors, phrase_plan_index, docid)); + + Cursor& lead = cursors[phrase_plan_index.front()]; + uint32_t lead_position = 0; + bool available = false; + RETURN_IF_ERROR( + exact_phrase_stream_matcher_detail::advance_to(&lead, 0, &lead_position, &available)); + if (!available) { + return exact_phrase_stream_matcher_detail::finish_document(cursors, phrase_plan_index); + } + + const size_t no_retained_clause = phrase_plan_index.size(); + size_t retained_clause = no_retained_clause; + uint32_t retained_position = 0; + while (true) { + bool restart = false; + for (size_t clause = 1; clause < phrase_plan_index.size(); ++clause) { + const uint32_t offset = position_offsets[clause] - position_offsets.front(); + uint32_t expected_position = 0; + if (!add_position_offset(lead_position, offset, &expected_position)) { + return exact_phrase_stream_matcher_detail::finish_document(cursors, + phrase_plan_index); + } + + uint32_t clause_position = 0; + if (retained_clause == clause) { + clause_position = retained_position; + retained_clause = no_retained_clause; + } else { + RETURN_IF_ERROR(exact_phrase_stream_matcher_detail::advance_to( + &cursors[phrase_plan_index[clause]], expected_position, &clause_position, + &available)); + if (!available) { + return exact_phrase_stream_matcher_detail::finish_document(cursors, + phrase_plan_index); + } + } + if (clause_position == expected_position) { + continue; + } + + const uint32_t lead_target = clause_position - offset; + RETURN_IF_ERROR(exact_phrase_stream_matcher_detail::advance_to( + &lead, lead_target, &lead_position, &available)); + if (!available) { + return exact_phrase_stream_matcher_detail::finish_document(cursors, + phrase_plan_index); + } + retained_clause = lead_position == lead_target ? clause : no_retained_clause; + retained_position = clause_position; + restart = true; + break; + } + if (restart) { + continue; + } + + *matched = true; + return exact_phrase_stream_matcher_detail::finish_document(cursors, phrase_plan_index); + } +} + +} // namespace doris::snii::query::internal diff --git a/be/src/storage/index/snii/query/internal/phrase_query_split.h b/be/src/storage/index/snii/query/internal/phrase_query_split.h new file mode 100644 index 00000000000000..521930d57e2bd4 --- /dev/null +++ b/be/src/storage/index/snii/query/internal/phrase_query_split.h @@ -0,0 +1,620 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/check.h" +#include "storage/index/inverted/common_grams/common_grams_key_codec.h" +#include "storage/index/inverted/common_grams/common_grams_query_cost.h" +#include "storage/index/inverted/common_grams/common_grams_segment_metadata.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/encoding/byte_source.h" +#include "storage/index/snii/format/dict_entry.h" +#include "storage/index/snii/format/frq_pod.h" +#include "storage/index/snii/format/frq_prelude.h" +#include "storage/index/snii/format/prx_pod.h" +#include "storage/index/snii/io/batch_range_fetcher.h" +#include "storage/index/snii/query/internal/docid_conjunction.h" +#include "storage/index/snii/query/internal/docid_posting_reader.h" +#include "storage/index/snii/query/internal/docid_set_ops.h" +#include "storage/index/snii/query/internal/docid_union.h" +#include "storage/index/snii/query/internal/plain_term_routing.h" +#include "storage/index/snii/query/internal/position_math.h" +#include "storage/index/snii/query/internal/query_test_counters.h" +#include "storage/index/snii/query/internal/resolved_phrase_plan.h" +#include "storage/index/snii/query/internal/term_expansion.h" +#include "storage/index/snii/query/phrase_prx_validation.h" +#include "storage/index/snii/query/phrase_query.h" +#include "storage/index/snii/query/phrase_verify_timer.h" +#include "storage/index/snii/query/prefix_query.h" +#include "storage/index/snii/query/term_query.h" +#include "storage/index/snii/reader/windowed_posting.h" +#include "util/debug_points.h" + +// phrase_query implements MATCH_PHRASE with WINDOW (sub-block) SKIPPING for +// high-df windowed terms (design spec section 6.2): +// 1. Resolve every term; reject if any is absent. +// 2. Batch-read each windowed term's prelude + each slim/inline term's full +// docid posting in one round; open the two-level prelude readers. +// 3. Pick the DRIVER = smallest-df term; materialize it fully -> the initial +// candidate docid set. +// 4. For every other term in ascending-df order, narrow the candidate set: +// - slim/inline: intersect with its (already decoded) full posting. +// - windowed: locate_window() the CURRENT candidates -> the SET of +// windows covering them; batch-fetch ONLY those windows' +// .frq docid regions; keep candidates present in some +// covering window. A high-df term thus reads +// O(candidates) windows instead of its whole O(df) +// posting. +// 5. Fetch PRX only for retained chunks and run the positional phrase check +// (term[0]@p, term[1]@p+1, ...) on the survivors. +// The result is identical to a full-read intersection; only the bytes read for +// high-df windowed terms shrink. +// +// Internal to the phrase-query implementation, which spans phrase_plan.cpp, +// phrase_position_source.cpp, phrase_emit.cpp, phrase_prefix_exec.cpp, +// phrase_planned_query.cpp and phrase_query.cpp. This header carries the types and +// functions those translation units share; nothing outside query/ may include it. +namespace doris::snii::query::phrase_impl { + +struct PosSource; + +using query::internal::DocidChunk; +using query::internal::DocidSource; +using query::internal::ResolvedQueryTerm; +using query::internal::TermPlan; +using reader::LogicalIndexReader; + +bool apply_common_grams_plan_debug_override(bool cost_prefers_gram, + CommonGramsPlanDebugOverride debug_override); + +bool should_use_streaming_exact_phrase(const std::vector& plans, + const std::vector& sources, + std::span phrase_plan_index, + size_t candidate_count, bool needs_frequency, + const PhraseQueryOptions& options, + internal::ExactPhrasePositionAccess position_access); + +class CommonGramsPlanningTimer { +public: + explicit CommonGramsPlanningTimer(format::PhraseQueryExecutionStats* stats) : stats_(stats) { + if (stats_ != nullptr) { + start_ = std::chrono::steady_clock::now(); + } + } + + ~CommonGramsPlanningTimer() { finish(); } + + CommonGramsPlanningTimer(const CommonGramsPlanningTimer&) = delete; + CommonGramsPlanningTimer& operator=(const CommonGramsPlanningTimer&) = delete; + + void finish() { + if (finished_) { + return; + } + finished_ = true; + if (stats_ == nullptr) { + return; + } + const auto elapsed = std::chrono::duration_cast( + std::chrono::steady_clock::now() - start_) + .count(); + stats_->common_grams_planning_ns += static_cast(std::max(1, elapsed)); + } + +private: + format::PhraseQueryExecutionStats* stats_ = nullptr; + std::chrono::steady_clock::time_point start_; + bool finished_ = false; +}; + +size_t position_span_size(std::pair span); + +bool should_use_monotonic_position_scan(std::pair anchor_span, + size_t checked_span_size, uint32_t anchor_offset, + uint32_t checked_offset); + +struct ExpectedTailPositions { + uint32_t docid = 0; + uint32_t phrase_frequency = 0; + size_t positions_begin = 0; + size_t positions_end = 0; +}; + +static_assert(sizeof(ExpectedTailPositions) == 3 * sizeof(uint64_t)); + +struct ExpectedTailPositionSet { + std::vector docs; + std::vector positions; + std::vector position_matched; + size_t matched_count = 0; + + void clear() { + docs.clear(); + positions.clear(); + position_matched.clear(); + matched_count = 0; + } + + void reserve_docs(size_t count) { + docs.reserve(count); + positions.reserve(count); + } +}; + +// One decoded chunk of a term's posting: a windowed term's covering window, or +// a slim/inline term's single posting. `docids` is decoded in the conjunction +// phase (and reused by the streaming cursor -- the dd region is decoded exactly +// once); `prx` is the on-disk positions bytes, decoded lazily by the cursor +// (once per chunk) during phrase verification. + +struct PosChunk { + std::vector docids; // ascending, absolute + // Empty means the chunk keeps every PRX doc in on-disk order. Non-empty means + // `docids[i]` corresponds to on-disk local document ordinal + // `prx_doc_ordinals[i]`, allowing PRX decode to skip positions for docs that + // were removed by the docid-only conjunction. + std::vector prx_doc_ordinals; + uint32_t prx_doc_count = 0; + Slice prx; // .prx window bytes (reference fetcher/round1/entry) + bool windowed = false; + uint32_t window = 0; +}; + +// A term's retained posting as an ordered list of chunks (windowed: covering +// windows in docid order; slim/inline: one). The referenced prx bytes live in +// `round1` / the per-term fetchers kept alive in phrase_query::owners for the +// whole query, so the cursor can decode positions during verification. + +struct PosSource { + std::vector chunks; + format::PrxDecodeContext* observer_context = nullptr; + uint64_t logical_position_work = 0; + uint64_t logical_position_docs = 0; +}; + +struct PhraseExecutionState { + std::vector srcs; + std::vector> owners; + std::vector candidates; +}; + +struct PhraseTermMapping { + std::vector unique_terms; + std::vector phrase_plan_index; +}; + +struct PhysicalPhrasePlan { + std::vector unique_terms; + std::vector phrase_plan_index; + std::vector position_offsets; + std::vector common_gram_clauses; +}; + +bool has_common_grams_capability( + const LogicalIndexReader& idx, + const segment_v2::inverted_index::CommonGramsQueryIdentity* query_identity); + +bool entry_has_positions(const format::DictEntry& entry); + +Status build_physical_phrase_plan_prefix(const LogicalIndexReader& idx, + const segment_v2::InvertedIndexQueryInfo& query_info, + size_t clause_count, bool allow_common_grams, + PhysicalPhrasePlan* plan, bool* all_representable); + +Status build_physical_phrase_plan(const LogicalIndexReader& idx, + const segment_v2::InvertedIndexQueryInfo& query_info, + bool allow_common_grams, PhysicalPhrasePlan* plan, + bool* all_representable); + +size_t resolved_batch_index(const std::vector& batch_terms, std::string_view term); + +bool all_plan_terms_present(const PhysicalPhrasePlan& plan, + const std::vector& batch_terms, + const std::vector& found); + +uint64_t plan_visible_posting_bytes(const format::DictEntry& entry, bool need_positions); + +segment_v2::inverted_index::CommonGramsPlanRawCost phrase_plan_raw_cost( + const PhysicalPhrasePlan& plan, const std::vector& batch_terms, + const std::vector& resolved, const std::vector& found, + bool need_positions); + +segment_v2::inverted_index::CommonGramsPlanRawCost alternative_clause_raw_cost( + const std::vector& terms, bool need_positions); + +void append_alternative_clause_cost( + const segment_v2::inverted_index::CommonGramsPlanRawCost& clause, + segment_v2::inverted_index::CommonGramsPlanRawCost* plan); + +segment_v2::inverted_index::CommonGramsPlanRawCost hybrid_verification_raw_cost( + const segment_v2::inverted_index::CommonGramsPlanRawCost& prefilter_cost, + const segment_v2::inverted_index::CommonGramsPlanRawCost& verification_cost); + +internal::ResolvedPhrasePlan materialize_resolved_phrase_plan( + const PhysicalPhrasePlan& plan, const std::vector& batch_terms, + std::vector* resolved); + +internal::ResolvedPhrasePlan copy_resolved_phrase_plan( + const PhysicalPhrasePlan& plan, const std::vector& batch_terms, + const std::vector& resolved); + +bool physical_phrase_plan_has_docs_only_term(const PhysicalPhrasePlan& plan, + const std::vector& batch_terms, + const std::vector& resolved); + +void append_physical_phrase_clause(const PhysicalPhrasePlan& source, size_t clause, + uint32_t position_offset, PhysicalPhrasePlan* target); + +struct HybridPositionedCover { + PhysicalPhrasePlan candidate_prefilter; + PhysicalPhrasePlan verification; +}; + +struct HybridExactPlanArtifact { + std::optional positioned_cover; +}; + +HybridExactPlanArtifact build_hybrid_exact_plan_artifact( + const PhysicalPhrasePlan& plain_plan, const PhysicalPhrasePlan& gram_plan, + const std::vector& batch_terms, + const std::vector& resolved); + +struct ResolvedMappedTail { + size_t batch_index = 0; + uint32_t expansion_ordinal = 0; +}; + +struct HybridPrefixMappedTails { + std::vector positioned_indices; + std::vector docs_only_indices; + std::vector docs_only_ordinals; +}; + +struct HybridPrefixPlanArtifact { + HybridPositionedCover plain_tail_cover; + HybridPrefixMappedTails mapped_tail_split; + std::optional positioned_tail_verification; + uint32_t plain_tail_position_offset = 0; + bool maps_tail_to_gram = false; +}; + +std::optional try_build_hybrid_prefix_plan_artifact( + const PhysicalPhrasePlan& plain_leading, const PhysicalPhrasePlan& gram_leading, + const std::vector& batch_terms, const std::vector& resolved, + const std::vector& mapped_tails, bool maps_tail_to_gram); + +struct HybridPrefixCandidateSet { + bool active = false; + std::vector docs; +}; + +Status build_hybrid_leading_candidates(const LogicalIndexReader& idx, + const PhysicalPhrasePlan& candidate_prefilter, + const std::vector& batch_terms, + const std::vector& resolved, + HybridPrefixCandidateSet* candidates); + +Status build_hybrid_docs_only_tail_candidates(const LogicalIndexReader& idx, + const std::vector& resolved, + const std::vector& gram_tail_indices, + const HybridPrefixCandidateSet& leading_candidates, + std::vector* candidates); + +Status execute_hybrid_exact_phrase_plan( + const LogicalIndexReader& idx, const PhysicalPhrasePlan& gram_plan, + const std::vector& batch_terms, const HybridExactPlanArtifact& artifact, + std::vector* resolved, std::vector* docids, + format::PrxDecodeContext* decode_context, bool* candidate_intersection_empty = nullptr); + +void append_resolved_phrase_clause(ResolvedQueryTerm term, uint32_t position_offset, + internal::ResolvedPhrasePlan* plan); + +internal::ResolvedPhrasePlan build_resolved_phrase_plan( + std::vector resolved_terms); + +Status resolve_and_execute_physical_phrase_plan(const LogicalIndexReader& idx, + const PhysicalPhrasePlan& plan, + std::vector* docids, + format::PrxDecodeContext* decode_context, + CommonGramsPlanningTimer& planning_timer); + +Status planned_exact_phrase_query_impl( + const LogicalIndexReader& idx, const segment_v2::InvertedIndexQueryInfo& plain_query_info, + const segment_v2::InvertedIndexQueryInfo& gram_query_info, + const segment_v2::inverted_index::CommonGramsQueryIdentity* common_grams_identity, + std::vector* docids, format::PrxDecodeContext* decode_context, + ExactPhrasePlanKind* selected_plan, + segment_v2::inverted_index::CommonGramsPlanCostModel cost_model, + CommonGramsPlanDebugOverride debug_override); + +PhraseTermMapping build_phrase_term_mapping(const std::vector& terms); + +Status build_position_sources_for_candidates( + const LogicalIndexReader& idx, const io::BatchRangeFetcher& round1, + const std::vector& plans, std::vector* doc_sources, + const std::vector& candidates, + std::vector>* owners, std::vector* srcs, + format::PrxDecodeContext* observer_context); + +class PosChunkDecoder { +public: + explicit PosChunkDecoder(format::PrxDecodeContext* observer_context = nullptr) + : observer_context_(observer_context) {} + + void set_decode_state(format::PrxDecodeContext* observer_context) { + observer_context_ = observer_context; + } + + void reset() { + chunk_ = nullptr; + offsets_by_prx_ordinal_ = false; + } + + Status decode(const PosChunk& chunk) { + chunk_ = &chunk; + ByteSource ps(chunk.prx); + const bool selected_all = chunk.prx_doc_ordinals.empty(); + const bool decode_full = selected_all || should_decode_full_prx_window(chunk); + offsets_by_prx_ordinal_ = decode_full && !selected_all; + return internal::decode_and_validate_prx_frame( + &ps, chunk.prx_doc_ordinals, decode_full, selected_all, chunk.prx_doc_count, + chunk.docids.size(), &pflat_, &poff_, observer_context_); + } + + Status positions(size_t doc_index, std::pair* out) const { + if (chunk_ == nullptr || doc_index >= chunk_->docids.size()) { + return Status::Error( + "phrase_query: decoded chunk doc index out of range"); + } + const size_t pos_index = + offsets_by_prx_ordinal_ ? chunk_->prx_doc_ordinals[doc_index] : doc_index; + if (pos_index + 1 >= poff_.size()) { + return Status::Error( + "phrase_query: prx ordinal offset out of range"); + } + const uint32_t begin = poff_[pos_index]; + const uint32_t end = poff_[pos_index + 1]; + if (begin == end) { + *out = {nullptr, nullptr}; + return Status::OK(); + } + if (end > pflat_.size()) { + return Status::Error( + "phrase_query: prx offset out of range"); + } + *out = {pflat_.data() + begin, pflat_.data() + end}; + return Status::OK(); + } + + inline __attribute__((always_inline)) std::pair + positions_unchecked(size_t doc_index) const { + const size_t pos_index = + offsets_by_prx_ordinal_ ? chunk_->prx_doc_ordinals[doc_index] : doc_index; + const uint32_t begin = poff_[pos_index]; + const uint32_t end = poff_[pos_index + 1]; + if (begin == end) { + return {nullptr, nullptr}; + } + return {pflat_.data() + begin, pflat_.data() + end}; + } + +private: + static bool should_decode_full_prx_window(const PosChunk& chunk) { + return chunk.prx_doc_count != 0 && + static_cast(chunk.prx_doc_ordinals.size()) * 2 >= chunk.prx_doc_count; + } + + const PosChunk* chunk_ = nullptr; + bool offsets_by_prx_ordinal_ = false; + std::vector pflat_; + std::vector poff_; + format::PrxDecodeContext* observer_context_ = nullptr; +}; + +// Streaming position cursor over one term's retained chunks. It advances ONLY +// forward (callers seek ascending candidate docids), decodes each chunk's +// docids once (reused from the conjunction phase) and each chunk's positions at +// most once (lazily, into a flat CSR whose capacity is retained across chunks). +// No per-doc allocation, no per-candidate docid binary search: positions are +// addressed by the doc's local index within its chunk. This is the read-side +// dual of the windowed posting layout -- the S3-native batch fetch already +// pulled every needed chunk into memory; the cursor is pure in-memory column +// iteration. + +class PostingCursor { +public: + void init(const PosSource* src) { + src_ = src; + ci_ = 0; + li_ = 0; + decoded_pos_chunk_ = kNoChunk; + decoder_.set_decode_state(src->observer_context); + decoder_.reset(); + } + + // Positions the cursor at `target` (guaranteed present: candidates are the + // intersection of exactly these chunks' docids). Monotonic forward advance. + Status seek(uint32_t target) { + while (ci_ < src_->chunks.size() && + (src_->chunks[ci_].docids.empty() || src_->chunks[ci_].docids.back() < target)) { + ++ci_; + li_ = 0; + } + if (ci_ >= src_->chunks.size()) { + return Status::Error( + "phrase_query: cursor exhausted before target docid"); + } + const std::vector& d = src_->chunks[ci_].docids; + while (li_ < d.size() && d[li_] < target) { + ++li_; + } + if (li_ >= d.size() || d[li_] != target) { + return Status::Error( + "phrase_query: candidate missing from posting chunk"); + } + return Status::OK(); + } + + // [begin,end) of the current doc's positions, decoding the current chunk's + // .prx exactly once (cached). Must follow a seek that landed on a real doc. + Status positions(std::pair* out) { + if (ci_ >= src_->chunks.size() || li_ >= src_->chunks[ci_].docids.size()) { + return Status::Error( + "phrase_query: cursor positions out of range"); + } + if (decoded_pos_chunk_ != ci_) { + RETURN_IF_ERROR(decoder_.decode(src_->chunks[ci_])); + decoded_pos_chunk_ = ci_; + } + return decoder_.positions(li_, out); + } + + Status next(uint32_t* docid, std::pair* out) { + while (ci_ < src_->chunks.size() && + (src_->chunks[ci_].docids.empty() || li_ >= src_->chunks[ci_].docids.size())) { + ++ci_; + li_ = 0; + } + if (ci_ >= src_->chunks.size()) { + return Status::Error( + "phrase_query: cursor exhausted before next docid"); + } + *docid = src_->chunks[ci_].docids[li_]; + RETURN_IF_ERROR(positions(out)); + ++li_; + return Status::OK(); + } + +private: + static constexpr size_t kNoChunk = static_cast(-1); + + const PosSource* src_ = nullptr; + size_t ci_ = 0; // current chunk + size_t li_ = 0; // current local doc index within the chunk + size_t decoded_pos_chunk_ = kNoChunk; // which chunk decoder_ currently holds + PosChunkDecoder decoder_; +}; + +enum class PhraseCandidateMetric : uint8_t { + kExact, + kPrefixLeading, +}; + +Status build_phrase_execution_state(const LogicalIndexReader& idx, io::BatchRangeFetcher* round1, + std::vector* plans, PhraseExecutionState* state, + const std::vector* candidate_prefilter, + format::PrxDecodeContext* observer_context, + PhraseCandidateMetric candidate_metric); + +Status execute_phrase_plans(const LogicalIndexReader& idx, io::BatchRangeFetcher* round1, + std::vector* plans, + const std::vector& phrase_plan_index, + std::vector* docids, + format::PrxDecodeContext* observer_context, + std::vector* matches, const PhraseQueryOptions& options); + +Status execute_resolved_phrase_prefix_terms( + const LogicalIndexReader& idx, internal::ResolvedPhrasePlan exact_plan, + std::vector tail_terms, uint32_t tail_position_offset, + std::vector* docids, format::PrxDecodeContext* decode_context, + std::vector* matches = nullptr, + const std::vector* candidate_prefilter = nullptr); + +Status execute_hybrid_phrase_prefix_plan( + const LogicalIndexReader& idx, const HybridPrefixPlanArtifact& artifact, + const std::vector& batch_terms, const std::vector& resolved, + const std::vector& plain_tail_terms, std::vector* docids, + format::PrxDecodeContext* decode_context, CommonGramsPlanningTimer& planning_timer, + bool* candidate_intersection_empty); + +struct HybridPrefixCostEstimate { + segment_v2::inverted_index::CommonGramsPlanRawCost raw_cost; + uint64_t estimated_cost = 0; +}; + +HybridPrefixCostEstimate estimate_hybrid_prefix_plan_cost( + const HybridPrefixPlanArtifact& artifact, const std::vector& batch_terms, + const std::vector& resolved, const std::vector& found, + const std::vector& plain_tail_terms, uint32_t position_verify_factor); + +Status phrase_query_impl(const LogicalIndexReader& idx, const std::vector& terms, + std::vector* const docids, + format::PrxDecodeContext* decode_context, + std::vector* matches, const PhraseQueryOptions& options); + +Status phrase_prefix_query_impl(const LogicalIndexReader& idx, + const std::vector& terms, + std::vector* const docids, int32_t max_expansions, + format::PrxDecodeContext* decode_context, + CommonGramsPlanningTimer* planning_timer, + std::vector* matches = nullptr); + +Status planned_phrase_prefix_query_impl( + const LogicalIndexReader& idx, const segment_v2::InvertedIndexQueryInfo& plain_query_info, + const segment_v2::InvertedIndexQueryInfo& gram_query_info, + const segment_v2::inverted_index::CommonGramsQueryIdentity* common_grams_identity, + std::vector* docids, int32_t max_expansions, + format::PrxDecodeContext* decode_context, PhrasePrefixPlanKind* selected_plan, + segment_v2::inverted_index::CommonGramsPlanCostModel cost_model, + CommonGramsPlanDebugOverride debug_override); + +template +segment_v2::inverted_index::CommonGramsPlanRawCost alternative_clause_raw_cost( + const std::vector& terms, const std::vector& indices, + bool need_positions) { + segment_v2::inverted_index::CommonGramsPlanRawCost cost; + unsigned __int128 posting_bytes = 0; + unsigned __int128 candidate_df = 0; + for (size_t index : indices) { + DORIS_CHECK_LT(index, terms.size()); + posting_bytes += plan_visible_posting_bytes(terms[index].entry, need_positions); + candidate_df += terms[index].entry.df; + } + cost.posting_bytes_or_df_sum = posting_bytes > std::numeric_limits::max() + ? std::numeric_limits::max() + : static_cast(posting_bytes); + cost.estimated_candidate_df = candidate_df > std::numeric_limits::max() + ? std::numeric_limits::max() + : static_cast(candidate_df); + cost.clause_count = 1; + return cost; +} + +} // namespace doris::snii::query::phrase_impl + +#ifdef BE_TEST +namespace doris::snii::query::internal::testing { + +uint64_t streaming_exact_phrase_execution_count(); +void reset_streaming_exact_phrase_execution_count(); +void note_streaming_exact_phrase_execution(); + +} // namespace doris::snii::query::internal::testing +#endif diff --git a/be/src/storage/index/snii/query/internal/plain_term_routing.h b/be/src/storage/index/snii/query/internal/plain_term_routing.h new file mode 100644 index 00000000000000..787d237ddaf933 --- /dev/null +++ b/be/src/storage/index/snii/query/internal/plain_term_routing.h @@ -0,0 +1,194 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/inverted/common_grams/common_grams_key_codec.h" +#include "storage/index/inverted/query/query_info.h" +#include "storage/index/snii/reader/logical_index_reader.h" + +namespace doris::snii::query::internal { + +inline segment_v2::inverted_index::PlainTermKeyVersion plain_term_key_version( + const reader::LogicalIndexReader& idx) { + const auto* metadata = idx.common_grams_metadata(); + return metadata == nullptr ? segment_v2::inverted_index::PlainTermKeyVersion::kLegacyRaw + : metadata->plain_term_key_version; +} + +inline Status route_plain_query_term_view(const reader::LogicalIndexReader& idx, + std::string_view logical_term, std::string* scratch, + std::string_view* physical_term, bool* representable) { + DORIS_CHECK(scratch != nullptr); + DORIS_CHECK(physical_term != nullptr); + DORIS_CHECK(representable != nullptr); + scratch->clear(); + *physical_term = {}; + *representable = false; + + const auto version = plain_term_key_version(idx); + if (version == segment_v2::inverted_index::PlainTermKeyVersion::kLegacyRaw && + segment_v2::inverted_index::legacy_raw_exact_requires_bypass(logical_term)) { + return Status::Error( + "SNII legacy raw term overlaps an internal term namespace"); + } + auto encoded = + segment_v2::inverted_index::try_encode_plain_term_view(logical_term, version, scratch); + if (!encoded.has_value()) { + return std::move(encoded.error()); + } + if (encoded->has_value()) { + *physical_term = **encoded; + *representable = true; + } + return Status::OK(); +} + +inline Status route_plain_query_term(const reader::LogicalIndexReader& idx, + std::string_view logical_term, std::string* physical_term, + bool* representable) { + DORIS_CHECK(physical_term != nullptr); + std::string scratch; + std::string_view physical_term_view; + RETURN_IF_ERROR(route_plain_query_term_view(idx, logical_term, &scratch, &physical_term_view, + representable)); + physical_term->assign(physical_term_view); + return Status::OK(); +} + +inline Status route_query_term_view(const reader::LogicalIndexReader& idx, + const segment_v2::TermInfo& term_info, std::string* scratch, + std::string_view* physical_term, bool* representable) { + DORIS_CHECK(term_info.is_single_term()); + if (term_info.key_kind == segment_v2::TermKeyKind::kCommonGram) { + return Status::Error( + "CommonGrams query plan requires segment capability validation"); + } + return route_plain_query_term_view(idx, term_info.get_single_term(), scratch, physical_term, + representable); +} + +inline Status route_query_term(const reader::LogicalIndexReader& idx, + const segment_v2::TermInfo& term_info, std::string* physical_term, + bool* representable) { + DORIS_CHECK(term_info.is_single_term()); + if (term_info.key_kind == segment_v2::TermKeyKind::kCommonGram) { + return Status::Error( + "CommonGrams query plan requires segment capability validation"); + } + return route_plain_query_term(idx, term_info.get_single_term(), physical_term, representable); +} + +inline Status route_query_terms(const reader::LogicalIndexReader& idx, + const segment_v2::InvertedIndexQueryInfo& query_info, + std::vector* routed_terms, bool* all_representable) { + DORIS_CHECK(routed_terms != nullptr); + DORIS_CHECK(all_representable != nullptr); + DORIS_CHECK(routed_terms->size() == query_info.term_infos.size()); + *all_representable = true; + const auto version = plain_term_key_version(idx); + size_t output_index = 0; + std::string scratch; + for (size_t i = 0; i < query_info.term_infos.size(); ++i) { + const auto& term_info = query_info.term_infos[i]; + DORIS_CHECK(term_info.is_single_term()); + if (term_info.key_kind == segment_v2::TermKeyKind::kCommonGram) { + return Status::Error( + "CommonGrams query plan requires segment capability validation"); + } + const auto logical_term = std::string_view((*routed_terms)[i]); + if (version == segment_v2::inverted_index::PlainTermKeyVersion::kLegacyRaw && + segment_v2::inverted_index::legacy_raw_exact_requires_bypass(logical_term)) { + return Status::Error( + "SNII legacy raw term overlaps an internal term namespace"); + } + auto physical_term = segment_v2::inverted_index::try_encode_plain_term_view( + logical_term, version, &scratch); + if (!physical_term.has_value()) { + return std::move(physical_term.error()); + } + if (!physical_term->has_value()) { + *all_representable = false; + continue; + } + if (output_index != i) { + if (scratch.empty()) { + (*routed_terms)[output_index] = std::move((*routed_terms)[i]); + } else { + (*routed_terms)[output_index] = std::move(scratch); + } + } else if (!scratch.empty()) { + (*routed_terms)[i] = std::move(scratch); + } + ++output_index; + } + routed_terms->resize(output_index); + return Status::OK(); +} + +inline Status route_plain_query_terms(const reader::LogicalIndexReader& idx, + const std::vector& logical_terms, + std::vector* physical_terms, + bool* all_representable) { + DORIS_CHECK(physical_terms != nullptr); + DORIS_CHECK(all_representable != nullptr); + physical_terms->clear(); + physical_terms->reserve(logical_terms.size()); + *all_representable = true; + for (const std::string& logical_term : logical_terms) { + std::string physical_term; + bool representable = false; + RETURN_IF_ERROR(route_plain_query_term(idx, logical_term, &physical_term, &representable)); + if (representable) { + physical_terms->push_back(std::move(physical_term)); + } else { + *all_representable = false; + } + } + return Status::OK(); +} + +inline Status route_plain_enumeration_prefix(const reader::LogicalIndexReader& idx, + std::string_view logical_prefix, + std::string* physical_prefix, bool* representable) { + DORIS_CHECK(physical_prefix != nullptr); + DORIS_CHECK(representable != nullptr); + physical_prefix->clear(); + *representable = false; + + const auto version = plain_term_key_version(idx); + if (version == segment_v2::inverted_index::PlainTermKeyVersion::kLegacyRaw && + segment_v2::inverted_index::legacy_raw_prefix_requires_bypass(logical_prefix)) { + return Status::Error( + "SNII legacy raw expansion overlaps an internal term namespace"); + } + auto encoded = segment_v2::inverted_index::try_encode_plain_term(logical_prefix, version, + physical_prefix); + if (!encoded.has_value()) { + return std::move(encoded.error()); + } + *representable = encoded.value(); + return Status::OK(); +} + +} // namespace doris::snii::query::internal diff --git a/be/src/storage/index/snii/query/internal/position_math.h b/be/src/storage/index/snii/query/internal/position_math.h new file mode 100644 index 00000000000000..6db2a0ee4b599b --- /dev/null +++ b/be/src/storage/index/snii/query/internal/position_math.h @@ -0,0 +1,47 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include + +namespace doris::snii::query::internal { + +inline bool build_position_offsets(size_t count, std::vector* out) { + if (count >= std::numeric_limits::max()) { + return false; + } + out->clear(); + out->reserve(count); + uint32_t offset = 0; + while (out->size() < count) { + out->push_back(offset); + ++offset; + } + return true; +} + +inline bool add_position_offset(uint32_t start, uint32_t offset, uint32_t* out) { + if (start > std::numeric_limits::max() - offset) return false; + *out = start + offset; + return true; +} + +} // namespace doris::snii::query::internal diff --git a/be/src/storage/index/snii/query/internal/query_test_counters.h b/be/src/storage/index/snii/query/internal/query_test_counters.h new file mode 100644 index 00000000000000..5e6b8acfd84e5d --- /dev/null +++ b/be/src/storage/index/snii/query/internal/query_test_counters.h @@ -0,0 +1,110 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +// Deterministic op-count seam for the phrase-query hot path (T24/G01). The +// counters let the phrase UTs assert routing/complexity directly: +// - expected_docids_build : how many times the multi-tail phrase-prefix branch +// materializes the expected-docid vector for a query. +// Hoisted out of the per-tail loop, so == 1 per query +// (was == tail_hits when rebuilt inside the loop). +// - anchor_iterations : total sparsest-anchor outer-enumeration size summed +// over candidate docs (== docs x min_span). Anchoring +// on the shortest per-doc span instead of the hardcoded +// phrase-position-0 span shrinks this when the leading +// exact term is high-frequency. +// - monotonic_position_scans +// : number of non-anchor spans whose phrase-position +// lookup uses a forward-only scan instead of one +// binary search per anchor position. +// - prefix_expected_doc_visits +// : total expected-doc iterations performed by the +// multi-tail phrase-prefix verification and result +// materialization path. +// - count_fastpath_hits : count-only (G02) answers produced from dict-entry +// df alone for a single term, with NO posting decode +// (count_query.cpp). +// - resolved_term_entry_copies / moves +// : DictEntry ownership transfers performed by the two +// plan_resolved_terms overloads. +// - resolved_term_payload_pointer_reuses +// : non-empty inline FRQ/PRX vectors whose data pointer +// survives an entry move into TermPlan. +// - phrase_position_epoch_cache_hits / misses +// : same-document PhrasePositionLoader lookups served +// from the plan span cache vs loaded from its cursor. +// +// The seam is active only under SNII_QUERY_TEST_COUNTERS, which is auto-enabled by +// the library-wide BE_TEST define (be/CMakeLists.txt `if (MAKE_TEST)`) used to +// build doris_be_test. Because BE_TEST is applied to the whole BE tree, both the +// phrase_query.cpp increments AND the test translation unit that reads them observe +// the SAME process-wide singleton (the inline function below has one instance +// across every including TU). In a release build BE_TEST is undefined, the struct +// and singleton do not exist, and SNII_QUERY_COUNT/SNII_QUERY_ADD expand to +// ((void)0): zero overhead and NO global mutable state on the production query +// path. +// +// CONCURRENCY: the singleton is intentionally unsynchronized. It is a +// single-threaded, test-only seam -- one phrase query at a time -- and is never +// touched on the production path. Do NOT read or write it from concurrent tests. +// Reset it between test cases with `query_test_counters() = {}`. +#if defined(BE_TEST) && !defined(SNII_QUERY_TEST_COUNTERS) +#define SNII_QUERY_TEST_COUNTERS +#endif + +#ifdef SNII_QUERY_TEST_COUNTERS + +namespace doris::snii::query::internal { + +struct QueryTestCounters { + uint64_t expected_docids_build = 0; + uint64_t anchor_iterations = 0; + uint64_t monotonic_position_scans = 0; + uint64_t prefix_expected_doc_visits = 0; + uint64_t count_fastpath_hits = 0; + uint64_t resolved_term_entry_copies = 0; + uint64_t resolved_term_entry_moves = 0; + uint64_t resolved_term_payload_pointer_reuses = 0; + uint64_t phrase_position_epoch_cache_hits = 0; + uint64_t phrase_position_epoch_cache_misses = 0; +}; + +// `inline` gives a single shared instance across all TUs that include this header +// (phrase_query.cpp and the test), so counter increments made in the library are +// visible to the test that reads them. +inline QueryTestCounters& query_test_counters() { + static QueryTestCounters counters; + return counters; +} + +} // namespace doris::snii::query::internal + +// NOLINTBEGIN(clang-diagnostic-unused-macros): expanded by phrase_query.cpp, not by this header's TU +#define SNII_QUERY_COUNT(field) (++::doris::snii::query::internal::query_test_counters().field) +#define SNII_QUERY_ADD(field, n) \ + (::doris::snii::query::internal::query_test_counters().field += (n)) +// NOLINTEND(clang-diagnostic-unused-macros) + +#else + +#define SNII_QUERY_COUNT(field) ((void)0) +#define SNII_QUERY_ADD(field, n) ((void)0) + +#endif // SNII_QUERY_TEST_COUNTERS diff --git a/be/src/storage/index/snii/query/internal/regex_prefix.h b/be/src/storage/index/snii/query/internal/regex_prefix.h new file mode 100644 index 00000000000000..304ab6161b2ddc --- /dev/null +++ b/be/src/storage/index/snii/query/internal/regex_prefix.h @@ -0,0 +1,37 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include +#include + +namespace doris::snii::query::internal { + +// Computes the dictionary-enumeration prefix used to narrow regexp term +// expansion. For left-anchored ("^") patterns it derives a tight common prefix +// from RE2::PossibleMatchRange (e.g. "^(order)" -> "order", where a naive literal +// scan stops at '(' and yields ""); if RE2 cannot compile the pattern, it falls +// back to a conservative literal-prefix scan. Unanchored patterns return no +// prefix because Hyperscan may match after the beginning of a dictionary term. +// The returned prefix only bounds how many terms visit_prefix_terms enumerates; +// final term acceptance is always decided by Hyperscan. +std::string regex_enum_prefix(std::string_view pattern, const re2::RE2& re); + +} // namespace doris::snii::query::internal diff --git a/be/src/storage/index/snii/query/internal/resolved_phrase_plan.h b/be/src/storage/index/snii/query/internal/resolved_phrase_plan.h new file mode 100644 index 00000000000000..0938a4a01447f2 --- /dev/null +++ b/be/src/storage/index/snii/query/internal/resolved_phrase_plan.h @@ -0,0 +1,82 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/format/prx_decode_stats.h" +#include "storage/index/snii/query/internal/docid_conjunction.h" +#include "storage/index/snii/reader/logical_index_reader.h" + +namespace doris::snii::query { +struct PhraseMatch; +} + +namespace doris::snii::query::internal { + +enum class ExactPhrasePositionAccess : uint8_t { + kAuto = 0, + kMaterializedOnly = 1, +}; + +struct ResolvedPhrasePlan { + std::vector unique_terms; + std::vector phrase_plan_index; + std::vector position_offsets; + + [[nodiscard]] bool is_valid() const { + if (phrase_plan_index.size() != position_offsets.size() || + unique_terms.empty() != phrase_plan_index.empty()) { + return false; + } + if (phrase_plan_index.empty()) { + return true; + } + + std::vector referenced(unique_terms.size(), 0); + for (size_t i = 0; i < phrase_plan_index.size(); ++i) { + if ((i == 0 && position_offsets[i] != 0) || + (i != 0 && position_offsets[i - 1] >= position_offsets[i])) { + return false; + } + const size_t plan_index = phrase_plan_index[i]; + if (plan_index >= unique_terms.size()) { + return false; + } + referenced[plan_index] = 1; + } + return std::ranges::all_of(referenced, [](uint8_t used) { return used != 0; }); + } +}; + +// Executes an already-resolved exact phrase plan. Planning policy and term-key +// semantics stay above this boundary; the executor treats terms as opaque and +// consumes the resolved entries so inline posting payloads can move into the +// execution plans without another allocation/copy. +Status execute_resolved_phrase_plan( + const reader::LogicalIndexReader& idx, ResolvedPhrasePlan&& plan, + std::vector* docids, format::PrxDecodeContext* observer_context = nullptr, + std::vector* matches = nullptr, + const std::vector* candidate_prefilter = nullptr, + ExactPhrasePositionAccess position_access = ExactPhrasePositionAccess::kAuto); + +} // namespace doris::snii::query::internal diff --git a/be/src/storage/index/snii/query/internal/sloppy_phrase_matcher.h b/be/src/storage/index/snii/query/internal/sloppy_phrase_matcher.h new file mode 100644 index 00000000000000..b96f3ae00eb43d --- /dev/null +++ b/be/src/storage/index/snii/query/internal/sloppy_phrase_matcher.h @@ -0,0 +1,76 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include + +namespace doris::snii::query::internal { + +using PhrasePositionSpan = std::pair; + +// Matches one candidate document at a time against already-decoded position +// spans. Query-shape storage and all scratch buffers are allocated once and +// reused across documents. +class SloppyPhraseMatcher { +public: + SloppyPhraseMatcher(std::span phrase_plan_index, + std::span position_offsets, uint32_t slop, bool ordered); + + // Returns 1 for the first match when frequencies are not requested. For + // scoring, returns the V3 sloppy frequency: sum(1 / (1 + match_width)). + float match(std::span positions, bool collect_frequency); + +private: + struct Clause { + PhrasePositionSpan positions; + const uint32_t* next = nullptr; + uint32_t raw_position = 0; + int64_t adjusted_position = 0; + bool has_position = false; + }; + + bool initialize_unordered(std::span positions); + bool advance_clause(size_t clause, bool update_end); + bool advance_repeat_collisions(size_t clause); + size_t collision(size_t clause) const; + bool clause_less(size_t left, size_t right) const; + bool clause_greater(size_t left, size_t right) const; + void rebuild_heap(); + size_t pop_heap(); + void push_heap(size_t clause); + bool next_unordered_match(uint64_t* match_width); + float match_unordered(std::span positions, bool collect_frequency); + float match_ordered(std::span positions, bool collect_frequency); + bool advance_ordered_to(size_t clause, int64_t target); + + std::vector phrase_plan_index_; + std::vector position_offsets_; + uint32_t slop_ = 0; + bool ordered_ = false; + bool has_repeats_ = false; + bool positioned_ = false; + int64_t end_ = 0; + std::vector clauses_; + std::vector heap_; +}; + +} // namespace doris::snii::query::internal diff --git a/be/src/storage/index/snii/query/internal/term_expansion.h b/be/src/storage/index/snii/query/internal/term_expansion.h new file mode 100644 index 00000000000000..6da0b4c3f8ed3f --- /dev/null +++ b/be/src/storage/index/snii/query/internal/term_expansion.h @@ -0,0 +1,47 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/query/docid_sink.h" +#include "storage/index/snii/reader/logical_index_reader.h" + +namespace doris::snii::query::internal { + +using TermMatcher = std::function; + +// Enumerates logical plain terms while retaining each matching physical +// DictEntry for direct posting resolution. PrefixHit::term is decoded logical +// text; PrefixHit::entry.term remains the physical dictionary key. +Status visit_expanded_plain_terms(const reader::LogicalIndexReader& idx, + std::string_view enum_prefix, const TermMatcher& matches, + const reader::LogicalIndexReader::PrefixHitVisitor& visitor, + int32_t max_expansions = 0); + +// Enumerates dictionary terms from `enum_prefix`, filters them with `matches`, +// and emits the sorted docid union for matching entries. PrefixHit carries the +// DictEntry and block bases, so callers avoid a second lookup per expanded term. +Status emit_expanded_docid_union(const reader::LogicalIndexReader& idx, + std::string_view enum_prefix, const TermMatcher& matches, + DocIdSink* const sink, int32_t max_expansions = 0); + +} // namespace doris::snii::query::internal diff --git a/be/src/storage/index/snii/query/internal/wildcard_matcher.h b/be/src/storage/index/snii/query/internal/wildcard_matcher.h new file mode 100644 index 00000000000000..78eec25097a997 --- /dev/null +++ b/be/src/storage/index/snii/query/internal/wildcard_matcher.h @@ -0,0 +1,81 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace doris::snii::query::internal { + +// Glob matcher with reusable scratch. '*' matches >=0 bytes, '?' matches exactly +// one byte, every other byte is literal; matching is anchored at both ends (full +// match). The matching result is bit-for-bit identical to the former per-call DP +// in wildcard_query.cpp: the only change is that the two DP rows are constructed +// once and reused (assign(), never reallocated once capacity is large enough) +// across every term in a single expansion. A whole-dictionary scan therefore +// performs O(1) heap allocations for scratch instead of O(2N) -- two small +// std::vector constructions per visited term. +// +// The allocator is templated only so deterministic allocation-counting tests can +// inject a CountingAllocator; production constructs WildcardMatcher<> (default +// std::allocator). The matcher is request-scoped (a stack local of the calling +// wildcard_query frame), holds no shared mutable state, and is not thread-safe by +// design: each query owns its own instance. +template > +class WildcardMatcher { +public: + explicit WildcardMatcher(std::string_view pattern) : pattern_(pattern) {} + + bool operator()(std::string_view text) { + const size_t n = text.size() + 1; + prev_.assign(n, 0); // reuses the buffer; no realloc once capacity >= n + curr_.assign(n, 0); + prev_[0] = 1; + for (char p : pattern_) { + std::fill(curr_.begin(), curr_.end(), 0); + if (p == '*') { + curr_[0] = prev_[0]; + for (size_t i = 1; i < n; ++i) { + curr_[i] = prev_[i] || curr_[i - 1]; + } + } else { + for (size_t i = 1; i < n; ++i) { + curr_[i] = prev_[i - 1] && (p == '?' || p == text[i - 1]); + } + } + prev_.swap(curr_); + } + return prev_[text.size()] != 0; + } + + // Test-only debug accessor: the production path never depends on it. Reports + // the larger of the two scratch-row capacities so perf tests can assert the + // buffer stops reallocating after warmup. + size_t scratch_capacity() const { return std::max(prev_.capacity(), curr_.capacity()); } + +private: + std::string_view pattern_; + std::vector prev_; + std::vector curr_; +}; + +} // namespace doris::snii::query::internal diff --git a/be/src/storage/index/snii/query/phrase_cost.cpp b/be/src/storage/index/snii/query/phrase_cost.cpp new file mode 100644 index 00000000000000..fec24172cc5b2e --- /dev/null +++ b/be/src/storage/index/snii/query/phrase_cost.cpp @@ -0,0 +1,276 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/check.h" +#include "storage/index/inverted/common_grams/common_grams_key_codec.h" +#include "storage/index/inverted/common_grams/common_grams_query_cost.h" +#include "storage/index/inverted/common_grams/common_grams_segment_metadata.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/encoding/byte_source.h" +#include "storage/index/snii/format/dict_entry.h" +#include "storage/index/snii/format/frq_pod.h" +#include "storage/index/snii/format/frq_prelude.h" +#include "storage/index/snii/format/prx_pod.h" +#include "storage/index/snii/io/batch_range_fetcher.h" +#include "storage/index/snii/query/internal/docid_conjunction.h" +#include "storage/index/snii/query/internal/docid_posting_reader.h" +#include "storage/index/snii/query/internal/docid_set_ops.h" +#include "storage/index/snii/query/internal/docid_union.h" +#include "storage/index/snii/query/internal/phrase_query_split.h" +#include "storage/index/snii/query/internal/plain_term_routing.h" +#include "storage/index/snii/query/internal/position_math.h" +#include "storage/index/snii/query/internal/query_test_counters.h" +#include "storage/index/snii/query/internal/resolved_phrase_plan.h" +#include "storage/index/snii/query/internal/term_expansion.h" +#include "storage/index/snii/query/phrase_prx_validation.h" +#include "storage/index/snii/query/phrase_query.h" +#include "storage/index/snii/query/phrase_verify_timer.h" +#include "storage/index/snii/query/prefix_query.h" +#include "storage/index/snii/query/term_query.h" +#include "storage/index/snii/reader/windowed_posting.h" +#include "util/debug_points.h" + +namespace doris::snii::query::phrase_impl { + +using query::internal::DocidChunk; +using query::internal::DocidSource; +using query::internal::ResolvedQueryTerm; +using query::internal::TermPlan; +using reader::LogicalIndexReader; +using internal::PhraseVerifyTimer; + +uint64_t plan_visible_posting_bytes(const format::DictEntry& entry, bool need_positions) { + unsigned __int128 bytes = entry.kind == format::DictEntryKind::kInline + ? entry.inline_dd_disk_len + : entry.frq_docs_len; + if (need_positions) { + bytes += entry.kind == format::DictEntryKind::kInline ? entry.prx_bytes.size() + : entry.prx_len; + } + return bytes > std::numeric_limits::max() ? std::numeric_limits::max() + : static_cast(bytes); +} + +segment_v2::inverted_index::CommonGramsPlanRawCost phrase_plan_raw_cost( + const PhysicalPhrasePlan& plan, const std::vector& batch_terms, + const std::vector& resolved, const std::vector& found, + bool need_positions) { + segment_v2::inverted_index::CommonGramsPlanRawCost cost; + unsigned __int128 posting_bytes = 0; + for (const std::string& term : plan.unique_terms) { + const size_t batch_index = resolved_batch_index(batch_terms, term); + if (found[batch_index] != 0) { + posting_bytes += + plan_visible_posting_bytes(resolved[batch_index].entry, need_positions); + } + } + cost.posting_bytes_or_df_sum = posting_bytes > std::numeric_limits::max() + ? std::numeric_limits::max() + : static_cast(posting_bytes); + cost.estimated_candidate_df = std::numeric_limits::max(); + for (size_t plan_index : plan.phrase_plan_index) { + const size_t batch_index = resolved_batch_index(batch_terms, plan.unique_terms[plan_index]); + if (found[batch_index] != 0) { + cost.estimated_candidate_df = + std::min(cost.estimated_candidate_df, resolved[batch_index].entry.df); + } + } + cost.clause_count = static_cast(plan.phrase_plan_index.size()); + if (cost.clause_count == 0) { + cost.estimated_candidate_df = 0; + } + return cost; +} + +segment_v2::inverted_index::CommonGramsPlanRawCost alternative_clause_raw_cost( + const std::vector& terms, bool need_positions) { + segment_v2::inverted_index::CommonGramsPlanRawCost cost; + unsigned __int128 posting_bytes = 0; + unsigned __int128 candidate_df = 0; + for (const auto& term : terms) { + posting_bytes += plan_visible_posting_bytes(term.entry, need_positions); + candidate_df += term.entry.df; + } + cost.posting_bytes_or_df_sum = posting_bytes > std::numeric_limits::max() + ? std::numeric_limits::max() + : static_cast(posting_bytes); + cost.estimated_candidate_df = candidate_df > std::numeric_limits::max() + ? std::numeric_limits::max() + : static_cast(candidate_df); + cost.clause_count = 1; + return cost; +} + +void append_alternative_clause_cost( + const segment_v2::inverted_index::CommonGramsPlanRawCost& clause, + segment_v2::inverted_index::CommonGramsPlanRawCost* plan) { + const unsigned __int128 posting_bytes = + static_cast(plan->posting_bytes_or_df_sum) + + clause.posting_bytes_or_df_sum; + plan->posting_bytes_or_df_sum = posting_bytes > std::numeric_limits::max() + ? std::numeric_limits::max() + : static_cast(posting_bytes); + plan->estimated_candidate_df = + plan->clause_count == 0 + ? clause.estimated_candidate_df + : std::min(plan->estimated_candidate_df, clause.estimated_candidate_df); + ++plan->clause_count; +} + +segment_v2::inverted_index::CommonGramsPlanRawCost hybrid_verification_raw_cost( + const segment_v2::inverted_index::CommonGramsPlanRawCost& prefilter_cost, + const segment_v2::inverted_index::CommonGramsPlanRawCost& verification_cost) { + auto cost = prefilter_cost; + const unsigned __int128 posting_bytes = + static_cast(cost.posting_bytes_or_df_sum) + + verification_cost.posting_bytes_or_df_sum; + cost.posting_bytes_or_df_sum = posting_bytes > std::numeric_limits::max() + ? std::numeric_limits::max() + : static_cast(posting_bytes); + cost.estimated_candidate_df = + std::min(cost.estimated_candidate_df, verification_cost.estimated_candidate_df); + cost.clause_count = verification_cost.clause_count; + return cost; +} + +bool physical_phrase_plan_has_docs_only_term(const PhysicalPhrasePlan& plan, + const std::vector& batch_terms, + const std::vector& resolved) { + for (const std::string& term : plan.unique_terms) { + if (!entry_has_positions(resolved[resolved_batch_index(batch_terms, term)].entry)) { + return true; + } + } + return false; +} + +void append_physical_phrase_clause(const PhysicalPhrasePlan& source, size_t clause, + uint32_t position_offset, PhysicalPhrasePlan* target) { + DORIS_CHECK_LT(clause, source.phrase_plan_index.size()); + DORIS_CHECK_EQ(source.phrase_plan_index.size(), source.position_offsets.size()); + DORIS_CHECK_EQ(source.phrase_plan_index.size(), source.common_gram_clauses.size()); + const size_t source_term = source.phrase_plan_index[clause]; + DORIS_CHECK_LT(source_term, source.unique_terms.size()); + const std::string& physical_term = source.unique_terms[source_term]; + + const auto unique = std::ranges::find(target->unique_terms, physical_term); + if (unique == target->unique_terms.end()) { + target->phrase_plan_index.push_back(target->unique_terms.size()); + target->unique_terms.push_back(physical_term); + } else { + target->phrase_plan_index.push_back( + static_cast(unique - target->unique_terms.begin())); + } + target->position_offsets.push_back(position_offset); + target->common_gram_clauses.push_back(source.common_gram_clauses[clause]); +} + +HybridPrefixCostEstimate estimate_hybrid_prefix_plan_cost( + const HybridPrefixPlanArtifact& artifact, const std::vector& batch_terms, + const std::vector& resolved, const std::vector& found, + const std::vector& plain_tail_terms, uint32_t position_verify_factor) { + const HybridPositionedCover& plain_tail_cover = artifact.plain_tail_cover; + const bool has_leading_prefilter = + !plain_tail_cover.candidate_prefilter.phrase_plan_index.empty(); + const auto leading_prefilter_cost = + phrase_plan_raw_cost(plain_tail_cover.candidate_prefilter, batch_terms, resolved, found, + /*need_positions=*/false); + unsigned __int128 posting_bytes = leading_prefilter_cost.posting_bytes_or_df_sum; + unsigned __int128 candidate_df_sum = 0; + unsigned __int128 position_verify_work = 0; + uint32_t max_clause_count = 0; + + const auto append_branch = [&](const PhysicalPhrasePlan& verification, + const auto& tail_verification_cost, + const auto* candidate_filter_cost) { + auto verification_cost = phrase_plan_raw_cost(verification, batch_terms, resolved, found, + /*need_positions=*/true); + append_alternative_clause_cost(tail_verification_cost, &verification_cost); + posting_bytes += verification_cost.posting_bytes_or_df_sum; + uint64_t candidate_df = verification_cost.estimated_candidate_df; + if (has_leading_prefilter) { + candidate_df = std::min(candidate_df, leading_prefilter_cost.estimated_candidate_df); + } + if (candidate_filter_cost != nullptr) { + posting_bytes += candidate_filter_cost->posting_bytes_or_df_sum; + candidate_df = std::min(candidate_df, candidate_filter_cost->estimated_candidate_df); + } + candidate_df_sum += candidate_df; + position_verify_work += + static_cast(candidate_df) * verification_cost.clause_count; + max_clause_count = std::max(max_clause_count, verification_cost.clause_count); + }; + + if (!artifact.maps_tail_to_gram) { + DORIS_CHECK(has_leading_prefilter); + DORIS_CHECK(artifact.mapped_tail_split.positioned_indices.empty()); + DORIS_CHECK(artifact.mapped_tail_split.docs_only_indices.empty()); + append_branch( + plain_tail_cover.verification, + alternative_clause_raw_cost(plain_tail_terms, /*need_positions=*/true), + static_cast(nullptr)); + } else { + const HybridPrefixMappedTails& split = artifact.mapped_tail_split; + DORIS_CHECK(!split.positioned_indices.empty() || !split.docs_only_indices.empty()); + DORIS_CHECK(has_leading_prefilter || !split.docs_only_indices.empty()); + if (!split.positioned_indices.empty()) { + DORIS_CHECK(artifact.positioned_tail_verification.has_value()); + append_branch(*artifact.positioned_tail_verification, + alternative_clause_raw_cost(resolved, split.positioned_indices, + /*need_positions=*/true), + static_cast( + nullptr)); + } + if (!split.docs_only_indices.empty()) { + const auto docs_only_filter_cost = + alternative_clause_raw_cost(resolved, split.docs_only_indices, + /*need_positions=*/false); + append_branch(plain_tail_cover.verification, + alternative_clause_raw_cost(plain_tail_terms, split.docs_only_ordinals, + /*need_positions=*/true), + &docs_only_filter_cost); + } + } + + HybridPrefixCostEstimate result; + result.raw_cost.posting_bytes_or_df_sum = posting_bytes > std::numeric_limits::max() + ? std::numeric_limits::max() + : static_cast(posting_bytes); + result.raw_cost.estimated_candidate_df = candidate_df_sum > std::numeric_limits::max() + ? std::numeric_limits::max() + : static_cast(candidate_df_sum); + result.raw_cost.clause_count = max_clause_count; + const unsigned __int128 estimated_cost = + posting_bytes + position_verify_work * position_verify_factor; + result.estimated_cost = estimated_cost > std::numeric_limits::max() + ? std::numeric_limits::max() + : static_cast(estimated_cost); + return result; +} + +} // namespace doris::snii::query::phrase_impl diff --git a/be/src/storage/index/snii/query/phrase_emit.cpp b/be/src/storage/index/snii/query/phrase_emit.cpp new file mode 100644 index 00000000000000..68c279eed94b41 --- /dev/null +++ b/be/src/storage/index/snii/query/phrase_emit.cpp @@ -0,0 +1,964 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/check.h" +#include "storage/index/inverted/common_grams/common_grams_key_codec.h" +#include "storage/index/inverted/common_grams/common_grams_query_cost.h" +#include "storage/index/inverted/common_grams/common_grams_segment_metadata.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/encoding/byte_source.h" +#include "storage/index/snii/format/dict_entry.h" +#include "storage/index/snii/format/frq_pod.h" +#include "storage/index/snii/format/frq_prelude.h" +#include "storage/index/snii/format/prx_pod.h" +#include "storage/index/snii/format/prx_position_iterator.h" +#include "storage/index/snii/io/batch_range_fetcher.h" +#include "storage/index/snii/query/internal/docid_conjunction.h" +#include "storage/index/snii/query/internal/docid_posting_reader.h" +#include "storage/index/snii/query/internal/docid_set_ops.h" +#include "storage/index/snii/query/internal/docid_union.h" +#include "storage/index/snii/query/internal/exact_phrase_stream_matcher.h" +#include "storage/index/snii/query/internal/phrase_query_split.h" +#include "storage/index/snii/query/internal/plain_term_routing.h" +#include "storage/index/snii/query/internal/position_math.h" +#include "storage/index/snii/query/internal/query_test_counters.h" +#include "storage/index/snii/query/internal/resolved_phrase_plan.h" +#include "storage/index/snii/query/internal/sloppy_phrase_matcher.h" +#include "storage/index/snii/query/internal/term_expansion.h" +#include "storage/index/snii/query/phrase_prx_validation.h" +#include "storage/index/snii/query/phrase_query.h" +#include "storage/index/snii/query/phrase_verify_timer.h" +#include "storage/index/snii/query/prefix_query.h" +#include "storage/index/snii/query/term_query.h" +#include "storage/index/snii/reader/windowed_posting.h" +#include "util/debug_points.h" + +namespace doris::snii::query::phrase_impl { + +using query::internal::DocidChunk; +using query::internal::DocidSource; +using query::internal::ResolvedQueryTerm; +using query::internal::TermPlan; +using reader::LogicalIndexReader; +using internal::PhraseVerifyTimer; + +bool should_use_streaming_exact_phrase(const std::vector& plans, + const std::vector& sources, + std::span phrase_plan_index, + size_t candidate_count, bool needs_frequency, + const PhraseQueryOptions& options, + internal::ExactPhrasePositionAccess position_access) { + constexpr uint64_t kMinMaximumPositionWork = 8; + constexpr uint64_t kMinEstimatedPositionWork = 512; + if (position_access == internal::ExactPhrasePositionAccess::kMaterializedOnly || + options.slop != 0 || needs_frequency) { + return false; + } + DORIS_CHECK_EQ(plans.size(), sources.size()); + + unsigned __int128 sum_position_work = 0; + uint64_t max_position_work = 0; + for (size_t clause = 0; clause < phrase_plan_index.size(); ++clause) { + const size_t plan_index = phrase_plan_index[clause]; + DORIS_CHECK_LT(plan_index, plans.size()); + for (size_t preceding = 0; preceding < clause; ++preceding) { + if (phrase_plan_index[preceding] == plan_index) { + return false; + } + } + const TermPlan& plan = plans[plan_index]; + DORIS_CHECK_NE(plan.df, 0); + DORIS_CHECK(plan.entry.term_stats_present || + sources[plan_index].logical_position_docs != 0); + const uint64_t position_work = plan.entry.term_stats_present + ? plan.entry.ttf_delta / plan.df + : sources[plan_index].logical_position_work / + sources[plan_index].logical_position_docs; + sum_position_work += position_work; + max_position_work = std::max(max_position_work, position_work); + } + if (max_position_work < kMinMaximumPositionWork) { + return false; + } + + constexpr unsigned __int128 kMaxU128 = ~static_cast(0); + const auto candidates = static_cast(candidate_count); + const unsigned __int128 raw_estimate = + candidates > kMaxU128 / sum_position_work ? kMaxU128 : candidates * sum_position_work; + const uint64_t estimated_position_work = raw_estimate > std::numeric_limits::max() + ? std::numeric_limits::max() + : static_cast(raw_estimate); + return estimated_position_work >= kMinEstimatedPositionWork; +} + +namespace { +class StreamingPostingCursor { +public: + void init(const PosSource* source) { + DORIS_CHECK(source != nullptr); + source_ = source; + chunk_index_ = 0; + local_doc_index_ = 0; + active_frame_ = kNoChunk; + local_query_stats_ = {}; + if (source_->observer_context != nullptr) { + iterator_context_ = *source_->observer_context; + iterator_context_.query_stats = source_->observer_context->query_stats == nullptr + ? nullptr + : &local_query_stats_; + } + } + + Status seek(uint32_t docid) { + while (chunk_index_ < source_->chunks.size() && + (source_->chunks[chunk_index_].docids.empty() || + source_->chunks[chunk_index_].docids.back() < docid)) { + RETURN_IF_ERROR(finish_active_frame()); + ++chunk_index_; + local_doc_index_ = 0; + } + if (chunk_index_ >= source_->chunks.size()) { + return Status::Error( + "phrase_query: streaming cursor exhausted before target docid"); + } + + const PosChunk& chunk = source_->chunks[chunk_index_]; + while (local_doc_index_ < chunk.docids.size() && chunk.docids[local_doc_index_] < docid) { + ++local_doc_index_; + } + if (local_doc_index_ >= chunk.docids.size() || chunk.docids[local_doc_index_] != docid) { + return Status::Error( + "phrase_query: candidate missing from streaming posting chunk"); + } + + if (active_frame_ != chunk_index_) { + RETURN_IF_ERROR(finish_active_frame()); + RETURN_IF_ERROR(positions_.reset( + chunk.prx, chunk.prx_doc_count, chunk.prx_doc_ordinals, + source_->observer_context == nullptr ? nullptr : &iterator_context_)); + active_frame_ = chunk_index_; + } + DORIS_CHECK(chunk.prx_doc_ordinals.empty() || + chunk.prx_doc_ordinals.size() == chunk.docids.size()); + DORIS_CHECK(!chunk.prx_doc_ordinals.empty() || + local_doc_index_ <= std::numeric_limits::max()); + const uint32_t prx_doc_ordinal = chunk.prx_doc_ordinals.empty() + ? static_cast(local_doc_index_) + : chunk.prx_doc_ordinals[local_doc_index_]; + return positions_.seek(prx_doc_ordinal); + } + + // `available` is an output parameter required by the exact matcher cursor contract. + Status next_position(uint32_t* position, + bool* available) { // NOLINT(readability-non-const-parameter) + return positions_.next_position(position, available); + } + + Status finish_doc() { + RETURN_IF_ERROR(positions_.finish_doc()); + ++local_doc_index_; + return Status::OK(); + } + + Status finish() { + RETURN_IF_ERROR(finish_active_frame()); + while (chunk_index_ < source_->chunks.size() && + local_doc_index_ == source_->chunks[chunk_index_].docids.size()) { + ++chunk_index_; + local_doc_index_ = 0; + } + if (chunk_index_ != source_->chunks.size()) { + return Status::Error( + "phrase_query: streaming cursor has unconsumed candidate docs"); + } + return Status::OK(); + } + + void add_query_stats(format::PhraseQueryExecutionStats* stats) const { + stats->prx_streaming_frames += local_query_stats_.prx_streaming_frames; + } + +private: + Status finish_active_frame() { + if (active_frame_ == kNoChunk) { + return Status::OK(); + } + RETURN_IF_ERROR(positions_.finish_frame()); + active_frame_ = kNoChunk; + return Status::OK(); + } + + static constexpr size_t kNoChunk = static_cast(-1); + + format::PrxPositionIterator positions_; + format::PrxDecodeContext iterator_context_; + format::PhraseQueryExecutionStats local_query_stats_; + const PosSource* source_ = nullptr; + size_t chunk_index_ = 0; + size_t local_doc_index_ = 0; + size_t active_frame_ = kNoChunk; +}; + +class PhrasePositionLoader { +public: + PhrasePositionLoader(size_t plan_count, std::vector& srcs) + : cursors_(plan_count), plan_spans_(plan_count), loaded_epoch_(plan_count, 0) { + for (size_t i = 0; i < plan_count; ++i) { + cursors_[i].init(&srcs[i]); + } + } + + void begin_doc(uint32_t docid) { + docid_ = docid; + ++epoch_; + if (epoch_ == 0) { + std::ranges::fill(loaded_epoch_, 0); + epoch_ = 1; + } + } + + Status positions_for_phrase_pos(const std::vector& phrase_plan_index, size_t phrase_pos, + std::pair* out) { + const size_t plan_index = phrase_plan_index[phrase_pos]; + if (loaded_epoch_[plan_index] != epoch_) { + RETURN_IF_ERROR(cursors_[plan_index].seek(docid_)); + RETURN_IF_ERROR(cursors_[plan_index].positions(&plan_spans_[plan_index])); + loaded_epoch_[plan_index] = epoch_; + SNII_QUERY_COUNT(phrase_position_epoch_cache_misses); + } else { + SNII_QUERY_COUNT(phrase_position_epoch_cache_hits); + } + *out = plan_spans_[plan_index]; + return Status::OK(); + } + +private: + std::vector cursors_; + std::vector> plan_spans_; + std::vector loaded_epoch_; + uint32_t docid_ = 0; + uint32_t epoch_ = 0; +}; + +class PhraseMatchCollector { +public: + PhraseMatchCollector(std::vector* docids, std::vector* matches) + : docids_(docids), matches_(matches) { + DCHECK(docids_ != nullptr || matches_ != nullptr); + } + + bool needs_frequency() const { return matches_ != nullptr; } + + void emit(uint32_t docid, uint32_t frequency) { + DCHECK_GT(frequency, 0); + if (docids_ != nullptr) { + docids_->push_back(docid); + } + if (matches_ != nullptr) { + matches_->push_back({.docid = docid, .frequency = static_cast(frequency)}); + } + } + + void emit_sloppy(uint32_t docid, float frequency) { + DCHECK_GT(frequency, 0.0F); + if (docids_ != nullptr) { + docids_->push_back(docid); + } + if (matches_ != nullptr) { + matches_->push_back({.docid = docid, .frequency = frequency}); + } + } + +private: + std::vector* docids_; + std::vector* matches_; +}; + +bool contains_two_term_phrase(std::pair left_span, + std::pair right_span, + uint32_t right_delta) { + const uint32_t* left = left_span.first; + const uint32_t* right = right_span.first; + if (left == left_span.second || right == right_span.second) { + return false; + } + const uint32_t max_start = std::numeric_limits::max() - right_delta; + if (left + 1 == left_span.second && right + 1 == right_span.second) { + return *left <= max_start && *right == *left + right_delta; + } + while (left != left_span.second && right != right_span.second) { + if (*left > max_start) { + return false; + } + const uint32_t want = *left + right_delta; + while (right != right_span.second && *right < want) { + ++right; + } + if (right == right_span.second) { + return false; + } + if (*right == want) { + return true; + } + ++left; + } + return false; +} + +size_t select_phrase_verification_pair(const std::vector& plans, + const std::vector& phrase_plan_index) { + size_t best_left = 0; + uint64_t best_score = std::numeric_limits::max(); + for (size_t left = 0; left + 1 < phrase_plan_index.size(); ++left) { + const uint64_t score = static_cast(plans[phrase_plan_index[left]].df) + + plans[phrase_plan_index[left + 1]].df; + if (score < best_score) { + best_score = score; + best_left = left; + } + } + return best_left; +} + +class TwoTermPhraseStartCursor { +public: + TwoTermPhraseStartCursor(std::pair left_span, + std::pair right_span, + uint32_t right_delta, uint32_t left_offset) + : left_(left_span.first), + left_end_(left_span.second), + right_(right_span.first), + right_end_(right_span.second), + right_delta_(right_delta), + left_offset_(left_offset), + max_left_(std::numeric_limits::max() - right_delta) {} + + bool next(uint32_t* start) { + DCHECK(start != nullptr); + while (left_ != left_end_ && right_ != right_end_) { + if (*left_ > max_left_) { + return false; + } + const uint32_t want = *left_ + right_delta_; + while (right_ != right_end_ && *right_ < want) { + ++right_; + } + if (right_ == right_end_) { + return false; + } + const uint32_t left_position = *left_++; + if (*right_ == want && left_position >= left_offset_) { + *start = left_position - left_offset_; + return true; + } + } + return false; + } + +private: + const uint32_t* left_; + const uint32_t* left_end_; + const uint32_t* right_; + const uint32_t* right_end_; + uint32_t right_delta_; + uint32_t left_offset_; + uint32_t max_left_; +}; + +uint32_t count_two_term_phrase(std::pair left_span, + std::pair right_span, + uint32_t right_delta) { + TwoTermPhraseStartCursor starts(left_span, right_span, right_delta, /*left_offset=*/0); + uint32_t frequency = 0; + uint32_t start = 0; + while (starts.next(&start)) { + DCHECK_NE(frequency, std::numeric_limits::max()); + ++frequency; + } + return frequency; +} + +Status emit_two_term_phrase_streaming(const std::vector& phrase_plan_index, + const std::vector& position_offsets, + std::vector& srcs, + const std::vector& candidates, + PhraseMatchCollector* collector) { + const size_t left_plan = phrase_plan_index[0]; + const size_t right_plan = phrase_plan_index[1]; + const uint32_t right_delta = position_offsets[1] - position_offsets[0]; + + if (left_plan == right_plan) { + PostingCursor cursor; + cursor.init(&srcs[left_plan]); + for (uint32_t expected_docid : candidates) { + uint32_t docid = 0; + std::pair span; + RETURN_IF_ERROR(cursor.next(&docid, &span)); + if (docid != expected_docid) { + return Status::Error( + "phrase_query: repeated-term cursor/docid mismatch"); + } + const uint32_t frequency = collector->needs_frequency() + ? count_two_term_phrase(span, span, right_delta) + : contains_two_term_phrase(span, span, right_delta); + if (frequency != 0) { + collector->emit(docid, frequency); + } + } + return Status::OK(); + } + + PostingCursor left_cursor; + PostingCursor right_cursor; + left_cursor.init(&srcs[left_plan]); + right_cursor.init(&srcs[right_plan]); + for (uint32_t expected_docid : candidates) { + uint32_t left_docid = 0; + uint32_t right_docid = 0; + std::pair left_span; + std::pair right_span; + RETURN_IF_ERROR(left_cursor.next(&left_docid, &left_span)); + RETURN_IF_ERROR(right_cursor.next(&right_docid, &right_span)); + if (left_docid != expected_docid || right_docid != expected_docid) { + return Status::Error( + "phrase_query: two-term cursor/docid mismatch"); + } + const uint32_t frequency = + collector->needs_frequency() + ? count_two_term_phrase(left_span, right_span, right_delta) + : contains_two_term_phrase(left_span, right_span, right_delta); + if (frequency != 0) { + collector->emit(expected_docid, frequency); + } + } + return Status::OK(); +} + +Status emit_sloppy_phrase_streaming(const std::vector& phrase_plan_index, + const std::vector& position_offsets, + std::vector& srcs, + const std::vector& candidates, + const PhraseQueryOptions& options, + PhraseMatchCollector* collector) { + PhrasePositionLoader loader(srcs.size(), srcs); + std::vector spans(phrase_plan_index.size()); + internal::SloppyPhraseMatcher matcher(phrase_plan_index, position_offsets, options.slop, + options.ordered); + for (uint32_t docid : candidates) { + loader.begin_doc(docid); + for (size_t i = 0; i < phrase_plan_index.size(); ++i) { + RETURN_IF_ERROR(loader.positions_for_phrase_pos(phrase_plan_index, i, &spans[i])); + } + const float frequency = matcher.match(spans, collector->needs_frequency()); + if (frequency > 0.0F) { + collector->emit_sloppy(docid, frequency); + } + } + return Status::OK(); +} + +void emit_two_term_phrase_chunk_pair(const PosChunk& left, const PosChunk& right, + const PosChunkDecoder& left_decoder, + const PosChunkDecoder& right_decoder, uint32_t right_delta, + PhraseMatchCollector* collector) { + size_t li = static_cast(std::ranges::lower_bound(left.docids, right.docids.front()) - + left.docids.begin()); + size_t ri = static_cast(std::ranges::lower_bound(right.docids, left.docids.front()) - + right.docids.begin()); + while (li < left.docids.size() && ri < right.docids.size()) { + const uint32_t left_docid = left.docids[li]; + const uint32_t right_docid = right.docids[ri]; + if (left_docid < right_docid) { + ++li; + continue; + } + if (right_docid < left_docid) { + ++ri; + continue; + } + + const std::pair left_span = + left_decoder.positions_unchecked(li); + const std::pair right_span = + right_decoder.positions_unchecked(ri); + const uint32_t frequency = + collector->needs_frequency() + ? count_two_term_phrase(left_span, right_span, right_delta) + : contains_two_term_phrase(left_span, right_span, right_delta); + if (frequency != 0) { + collector->emit(left_docid, frequency); + } + ++li; + ++ri; + } +} + +Status emit_two_term_phrase_chunk_merge(const std::vector& phrase_plan_index, + const std::vector& position_offsets, + std::vector& srcs, + PhraseMatchCollector* collector) { + const size_t left_plan = phrase_plan_index[0]; + const size_t right_plan = phrase_plan_index[1]; + const uint32_t right_delta = position_offsets[1] - position_offsets[0]; + const PosSource& left_src = srcs[left_plan]; + const PosSource& right_src = srcs[right_plan]; + + PosChunkDecoder left_decoder(left_src.observer_context); + PosChunkDecoder right_decoder(right_src.observer_context); + auto decoded_left_chunk = static_cast(-1); + auto decoded_right_chunk = static_cast(-1); + size_t left_chunk = 0; + size_t right_chunk = 0; + while (left_chunk < left_src.chunks.size() && right_chunk < right_src.chunks.size()) { + const PosChunk& left = left_src.chunks[left_chunk]; + const PosChunk& right = right_src.chunks[right_chunk]; + if (left.docids.empty()) { + ++left_chunk; + continue; + } + if (right.docids.empty()) { + ++right_chunk; + continue; + } + if (left.docids.back() < right.docids.front()) { + ++left_chunk; + continue; + } + if (right.docids.back() < left.docids.front()) { + ++right_chunk; + continue; + } + + if (decoded_left_chunk != left_chunk) { + RETURN_IF_ERROR(left_decoder.decode(left)); + decoded_left_chunk = left_chunk; + } + if (decoded_right_chunk != right_chunk) { + RETURN_IF_ERROR(right_decoder.decode(right)); + decoded_right_chunk = right_chunk; + } + + emit_two_term_phrase_chunk_pair(left, right, left_decoder, right_decoder, right_delta, + collector); + + const uint32_t left_last = left.docids.back(); + const uint32_t right_last = right.docids.back(); + if (left_last <= right_last) { + ++left_chunk; + } + if (right_last <= left_last) { + ++right_chunk; + } + } + return Status::OK(); +} + +bool phrase_start_matches_all_terms( + uint32_t start, size_t phrase_len, size_t pair_left, size_t pair_right, + const std::vector& position_offsets, + const std::vector>& span) { + for (size_t t = 0; t < phrase_len; ++t) { + if (t == pair_left || t == pair_right) { + continue; + } + uint32_t want = 0; + if (!internal::add_position_offset(start, position_offsets[t], &want)) { + return false; + } + if (!std::binary_search(span[t].first, span[t].second, want)) { + return false; + } + } + return true; +} + +Status emit_single_term_phrase_streaming(const std::vector& phrase_plan_index, + std::vector& srcs, + const std::vector& candidates, + PhraseMatchCollector* collector) { + PhrasePositionLoader loader(srcs.size(), srcs); + for (uint32_t d : candidates) { + loader.begin_doc(d); + std::pair single_span; + RETURN_IF_ERROR(loader.positions_for_phrase_pos(phrase_plan_index, 0, &single_span)); + if (single_span.first != single_span.second) { + const auto span_size = static_cast(single_span.second - single_span.first); + DCHECK_LE(span_size, std::numeric_limits::max()); + collector->emit(d, collector->needs_frequency() ? static_cast(span_size) : 1); + } + } + return Status::OK(); +} + +Status emit_multi_term_phrase_streaming(const std::vector& plans, + const std::vector& phrase_plan_index, + const std::vector& position_offsets, + std::vector& srcs, + const std::vector& candidates, + PhraseMatchCollector* collector) { + const size_t phrase_len = phrase_plan_index.size(); + PhrasePositionLoader loader(plans.size(), srcs); + std::vector> span(phrase_len); + const size_t pair_left = select_phrase_verification_pair(plans, phrase_plan_index); + const size_t pair_right = pair_left + 1; + for (uint32_t d : candidates) { + loader.begin_doc(d); + std::pair left_span; + std::pair right_span; + RETURN_IF_ERROR(loader.positions_for_phrase_pos(phrase_plan_index, pair_left, &left_span)); + RETURN_IF_ERROR( + loader.positions_for_phrase_pos(phrase_plan_index, pair_right, &right_span)); + + // `starts` retains raw pointers into the selected pair while the remaining + // clause spans are loaded below. Every unique plan owns an independent + // PostingCursor/PosChunkDecoder in PhrasePositionLoader; repeated phrase + // positions map back to one plan and reuse that plan's epoch-cached span. + TwoTermPhraseStartCursor starts(left_span, right_span, + position_offsets[pair_right] - position_offsets[pair_left], + position_offsets[pair_left]); + uint32_t start = 0; + if (!starts.next(&start)) { + continue; + } + + span[pair_left] = left_span; + span[pair_right] = right_span; + for (size_t pp = 0; pp < phrase_len; ++pp) { + if (pp == pair_left || pp == pair_right) { + continue; + } + RETURN_IF_ERROR(loader.positions_for_phrase_pos(phrase_plan_index, pp, &span[pp])); + } + + uint32_t frequency = 0; + bool has_previous_start = false; + uint32_t previous_start = 0; + const uint32_t* first_clause_position = span[0].first; + while (true) { + if (!collector->needs_frequency()) { + if (phrase_start_matches_all_terms(start, phrase_len, pair_left, pair_right, + position_offsets, span)) { + collector->emit(d, 1); + break; + } + } else if (!has_previous_start || start != previous_start) { + has_previous_start = true; + previous_start = start; + if (phrase_start_matches_all_terms(start, phrase_len, pair_left, pair_right, + position_offsets, span)) { + uint32_t first_clause_want = 0; + const bool representable = internal::add_position_offset( + start, position_offsets[0], &first_clause_want); + DCHECK(representable); + while (first_clause_position != span[0].second && + *first_clause_position < first_clause_want) { + ++first_clause_position; + } + const uint32_t* run_end = first_clause_position; + while (run_end != span[0].second && *run_end == first_clause_want) { + ++run_end; + } + const auto multiplicity = + static_cast(run_end - first_clause_position); + DCHECK_NE(multiplicity, 0); + DCHECK_LE(frequency, std::numeric_limits::max() - multiplicity); + frequency += multiplicity; + first_clause_position = run_end; + } + } + if (!starts.next(&start)) { + break; + } + } + if (frequency != 0) { + collector->emit(d, frequency); + } + } + return Status::OK(); +} + +// Single streaming pass over the candidates: for each (ascending) candidate, +// gather positions lazily, and test the consecutive-phrase predicate +// (term[0]@p, term[1]@p+1, ...). Multi-term phrases first test the cheapest +// adjacent pair by df before decoding the remaining terms for that document. +// Cursors decode each retained chunk at most once and address positions by +// local index -- no per-candidate docid binary search, no full-candidate +// position materialization. Candidates are ascending so the emitted docids are +// already sorted. + +Status emit_phrase_streaming(const std::vector& plans, + const std::vector& phrase_plan_index, + const std::vector& position_offsets, + std::vector& srcs, const std::vector& candidates, + PhraseMatchCollector* collector, const PhraseQueryOptions& options) { + const size_t phrase_len = phrase_plan_index.size(); + if (options.slop != 0) { + return emit_sloppy_phrase_streaming(phrase_plan_index, position_offsets, srcs, candidates, + options, collector); + } + if (phrase_len == 1) { + return emit_single_term_phrase_streaming(phrase_plan_index, srcs, candidates, collector); + } + if (phrase_len == 2) { + if (phrase_plan_index[0] != phrase_plan_index[1]) { + return emit_two_term_phrase_chunk_merge(phrase_plan_index, position_offsets, srcs, + collector); + } + return emit_two_term_phrase_streaming(phrase_plan_index, position_offsets, srcs, candidates, + collector); + } + return emit_multi_term_phrase_streaming(plans, phrase_plan_index, position_offsets, srcs, + candidates, collector); +} + +Status emit_exact_phrase_streaming_positions(const std::vector& phrase_plan_index, + const std::vector& position_offsets, + std::vector& srcs, + const std::vector& candidates, + PhraseMatchCollector* collector, + format::PhraseQueryExecutionStats* query_stats) { +#ifdef BE_TEST + internal::testing::note_streaming_exact_phrase_execution(); +#endif + std::vector cursors(srcs.size()); + internal::validate_exact_phrase_stream_inputs(std::span(cursors), std::span(phrase_plan_index), + std::span(position_offsets)); + for (size_t plan_index : phrase_plan_index) { + cursors[plan_index].init(&srcs[plan_index]); + } + for (uint32_t docid : candidates) { + bool matched = false; + RETURN_IF_ERROR(internal::match_exact_phrase_document( + std::span(cursors), std::span(phrase_plan_index), std::span(position_offsets), + docid, &matched)); + if (matched) { + collector->emit(docid, 1); + } + } + + Status first_error; + for (size_t plan_index : phrase_plan_index) { + const Status status = cursors[plan_index].finish(); + if (!status.ok() && first_error.ok()) { + first_error = status; + } + } + if (!first_error.ok()) { + return first_error; + } + for (size_t plan_index : phrase_plan_index) { + cursors[plan_index].add_query_stats(query_stats); + } + return Status::OK(); +} + +// candidate_prefilter (optional): an ascending docid set the phrase must ALSO +// lie in. When provided, the leading-term conjunction is intersected with it so +// only docs in the prefilter get their positions read. Docs outside the +// prefilter cannot contribute (the caller guarantees the final answer is a +// subset), so this is result-preserving while cutting the position decode -- +// used by phrase-prefix to restrict the huge leading-phrase candidate set to +// the docs that also carry some tail expansion. + +} // namespace +Status build_phrase_execution_state(const LogicalIndexReader& idx, io::BatchRangeFetcher* round1, + std::vector* plans, PhraseExecutionState* state, + const std::vector* candidate_prefilter, + format::PrxDecodeContext* observer_context, + PhraseCandidateMetric candidate_metric) { + if (round1->pending() > 0) { + RETURN_IF_ERROR(round1->fetch()); + } + RETURN_IF_ERROR(internal::open_preludes(*round1, plans, + /*need_positions=*/true)); + + state->owners.clear(); + state->candidates.clear(); + std::vector doc_sources; + if (candidate_prefilter != nullptr) { + if (candidate_prefilter->empty()) { + return Status::OK(); + } + RETURN_IF_ERROR(internal::filter_docids_by_conjunction( + idx, *round1, *plans, *candidate_prefilter, &state->candidates, &doc_sources)); + } else { + RETURN_IF_ERROR(internal::build_docid_only_conjunction(idx, *round1, *plans, + &state->candidates, &doc_sources)); + } + if (observer_context != nullptr && observer_context->query_stats != nullptr) { + if (candidate_metric == PhraseCandidateMetric::kExact) { + observer_context->query_stats->exact_candidate_docs += state->candidates.size(); + observer_context->query_stats->exact_candidate_visits += state->candidates.size(); + } else { + observer_context->query_stats->prefix_leading_candidate_docs += + state->candidates.size(); + } + } + if (state->candidates.empty()) { + return Status::OK(); + } + RETURN_IF_ERROR(build_position_sources_for_candidates(idx, *round1, *plans, &doc_sources, + state->candidates, &state->owners, + &state->srcs, observer_context)); + return Status::OK(); +} + +namespace { +Status execute_phrase_plans_at_offsets( + const LogicalIndexReader& idx, io::BatchRangeFetcher* round1, std::vector* plans, + const std::vector& phrase_plan_index, const std::vector& position_offsets, + std::vector* docids, format::PrxDecodeContext* observer_context, + std::vector* matches, const PhraseQueryOptions& options, + const std::vector* candidate_prefilter, + internal::ExactPhrasePositionAccess position_access) { + DCHECK_EQ(phrase_plan_index.size(), position_offsets.size()); + PhraseExecutionState state; + RETURN_IF_ERROR(build_phrase_execution_state(idx, round1, plans, &state, candidate_prefilter, + observer_context, PhraseCandidateMetric::kExact)); + if (state.candidates.empty()) { + return Status::OK(); + } + + const bool use_streaming = should_use_streaming_exact_phrase( + *plans, state.srcs, phrase_plan_index, state.candidates.size(), matches != nullptr, + options, position_access); + PhraseVerifyTimer verify_timer(observer_context); + format::PhraseQueryExecutionStats streaming_stats; + if (use_streaming) { + DCHECK(docids != nullptr); + std::vector staged_docids = std::move(*docids); + docids->clear(); + PhraseMatchCollector collector(&staged_docids, nullptr); + RETURN_IF_ERROR(emit_exact_phrase_streaming_positions(phrase_plan_index, position_offsets, + state.srcs, state.candidates, + &collector, &streaming_stats)); + *docids = std::move(staged_docids); + } else { + PhraseMatchCollector collector(docids, matches); + RETURN_IF_ERROR(emit_phrase_streaming(*plans, phrase_plan_index, position_offsets, + state.srcs, state.candidates, &collector, options)); + } + verify_timer.commit_success(); + if (observer_context != nullptr && observer_context->query_stats != nullptr) { + observer_context->query_stats->prx_streaming_frames += streaming_stats.prx_streaming_frames; + } + return Status::OK(); +} + +} // namespace +Status execute_phrase_plans(const LogicalIndexReader& idx, io::BatchRangeFetcher* round1, + std::vector* plans, + const std::vector& phrase_plan_index, + std::vector* docids, + format::PrxDecodeContext* observer_context, + std::vector* matches, const PhraseQueryOptions& options) { + std::vector position_offsets; + if (!internal::build_position_offsets(phrase_plan_index.size(), &position_offsets)) { + return Status::Error( + "phrase_query: phrase length exceeds doc position range"); + } + return execute_phrase_plans_at_offsets(idx, round1, plans, phrase_plan_index, position_offsets, + docids, observer_context, matches, options, nullptr, + internal::ExactPhrasePositionAccess::kAuto); +} + +} // namespace doris::snii::query::phrase_impl + +namespace doris::snii::query { + +using namespace phrase_impl; // NOLINT(google-build-using-namespace): module-internal impl namespace + +Status internal::execute_resolved_phrase_plan(const LogicalIndexReader& idx, + internal::ResolvedPhrasePlan&& plan, + std::vector* docids, + format::PrxDecodeContext* observer_context, + std::vector* matches, + const std::vector* candidate_prefilter, + internal::ExactPhrasePositionAccess position_access) { + if (docids == nullptr && matches == nullptr) { + return Status::Error("resolved_phrase_plan: null out"); + } + if (docids != nullptr) { + docids->clear(); + } + if (matches != nullptr) { + matches->clear(); + } + DORIS_CHECK(plan.is_valid()); + if (plan.phrase_plan_index.empty()) { + return Status::OK(); + } + + if (plan.phrase_plan_index.size() == 1) { + DORIS_CHECK(matches == nullptr); + const internal::ResolvedQueryTerm& term = plan.unique_terms[plan.phrase_plan_index.front()]; + RETURN_IF_ERROR(internal::read_docid_posting(idx, term.entry, term.frq_base, term.prx_base, + docids)); + if (candidate_prefilter != nullptr) { + *docids = internal::intersect_sorted(*docids, *candidate_prefilter); + } + return Status::OK(); + } + + io::BatchRangeFetcher round1(idx.reader()); + std::vector plans; + RETURN_IF_ERROR(internal::plan_resolved_terms(idx, std::move(plan.unique_terms), &round1, + &plans, + /*need_positions=*/false)); + return execute_phrase_plans_at_offsets(idx, &round1, &plans, plan.phrase_plan_index, + plan.position_offsets, docids, observer_context, matches, + {}, candidate_prefilter, position_access); +} + +} // namespace doris::snii::query + +#ifdef BE_TEST +namespace doris::snii::query::internal::testing { +namespace { + +std::atomic& streaming_exact_phrase_execution_atomic() { + static std::atomic counter {0}; + return counter; +} + +} // namespace + +uint64_t streaming_exact_phrase_execution_count() { + return streaming_exact_phrase_execution_atomic().load(std::memory_order_relaxed); +} + +void reset_streaming_exact_phrase_execution_count() { + streaming_exact_phrase_execution_atomic().store(0, std::memory_order_relaxed); +} + +void note_streaming_exact_phrase_execution() { + streaming_exact_phrase_execution_atomic().fetch_add(1, std::memory_order_relaxed); +} + +} // namespace doris::snii::query::internal::testing +#endif diff --git a/be/src/storage/index/snii/query/phrase_plan.cpp b/be/src/storage/index/snii/query/phrase_plan.cpp new file mode 100644 index 00000000000000..d9facfddcf9b8d --- /dev/null +++ b/be/src/storage/index/snii/query/phrase_plan.cpp @@ -0,0 +1,600 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/check.h" +#include "storage/index/inverted/common_grams/common_grams_key_codec.h" +#include "storage/index/inverted/common_grams/common_grams_query_cost.h" +#include "storage/index/inverted/common_grams/common_grams_segment_metadata.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/encoding/byte_source.h" +#include "storage/index/snii/format/dict_entry.h" +#include "storage/index/snii/format/frq_pod.h" +#include "storage/index/snii/format/frq_prelude.h" +#include "storage/index/snii/format/prx_pod.h" +#include "storage/index/snii/io/batch_range_fetcher.h" +#include "storage/index/snii/query/internal/docid_conjunction.h" +#include "storage/index/snii/query/internal/docid_posting_reader.h" +#include "storage/index/snii/query/internal/docid_set_ops.h" +#include "storage/index/snii/query/internal/docid_union.h" +#include "storage/index/snii/query/internal/phrase_query_split.h" +#include "storage/index/snii/query/internal/plain_term_routing.h" +#include "storage/index/snii/query/internal/position_math.h" +#include "storage/index/snii/query/internal/query_test_counters.h" +#include "storage/index/snii/query/internal/resolved_phrase_plan.h" +#include "storage/index/snii/query/internal/term_expansion.h" +#include "storage/index/snii/query/phrase_prx_validation.h" +#include "storage/index/snii/query/phrase_query.h" +#include "storage/index/snii/query/phrase_verify_timer.h" +#include "storage/index/snii/query/prefix_query.h" +#include "storage/index/snii/query/term_query.h" +#include "storage/index/snii/reader/windowed_posting.h" +#include "util/debug_points.h" + +namespace doris::snii::query::phrase_impl { + +using query::internal::DocidChunk; +using query::internal::DocidSource; +using query::internal::ResolvedQueryTerm; +using query::internal::TermPlan; +using reader::LogicalIndexReader; +using internal::PhraseVerifyTimer; + +bool apply_common_grams_plan_debug_override(bool cost_prefers_gram, + CommonGramsPlanDebugOverride debug_override) { + switch (debug_override) { + case CommonGramsPlanDebugOverride::kNone: + return cost_prefers_gram; + case CommonGramsPlanDebugOverride::kForcePlain: + return false; + case CommonGramsPlanDebugOverride::kForceCommonGrams: + return true; + } + DORIS_CHECK(false); + return cost_prefers_gram; +} + +size_t position_span_size(std::pair span) { + if (span.first == span.second) { + return 0; + } + DCHECK(span.first != nullptr); + DCHECK(span.second != nullptr); + return static_cast(span.second - span.first); +} + +bool should_use_monotonic_position_scan(std::pair anchor_span, + size_t checked_span_size, uint32_t anchor_offset, + uint32_t checked_offset) { + const uint64_t anchor_count = position_span_size(anchor_span); + const uint64_t binary_search_upper_bound = + anchor_count * (static_cast(std::bit_width(checked_span_size)) + 1); + const uint64_t monotonic_scan_upper_bound = checked_span_size + 2 * anchor_count + 2; + + // Require a 2x comparison margin before paying even the O(1) validity + // checks and adding the scan-path branches. This keeps low-TF spans on the + // simpler binary-search path while retaining the high-TF dense case. + if (2 * monotonic_scan_upper_bound > binary_search_upper_bound) { + return false; + } + + // Scanning is considered only when every anchor yields a representable + // phrase start and checked-term position. Endpoint checks are sufficient + // because anchor positions are sorted; invalid boundary shapes stay on the + // existing per-anchor path without extra binary searches in this gate. + if (*anchor_span.first < anchor_offset) { + return false; + } + if (checked_offset <= anchor_offset) { + return true; + } + const uint32_t offset_delta = checked_offset - anchor_offset; + return anchor_span.second[-1] <= std::numeric_limits::max() - offset_delta; +} + +bool has_common_grams_capability( + const LogicalIndexReader& idx, + const segment_v2::inverted_index::CommonGramsQueryIdentity* query_identity) { + const auto* metadata = idx.common_grams_metadata(); + if (metadata == nullptr || query_identity == nullptr) { + return false; + } + if (idx.common_grams_posting_policy() == format::CommonGramsPostingPolicy::kHybridV1) { + return segment_v2::inverted_index::is_common_grams_query_compatible( + *metadata, *query_identity, + segment_v2::inverted_index::CommonGramsCoverage::kMixed); + } + return segment_v2::inverted_index::is_common_grams_query_compatible(*metadata, *query_identity); +} + +bool entry_has_positions(const format::DictEntry& entry) { + return entry.kind == format::DictEntryKind::kInline ? !entry.prx_bytes.empty() + : entry.prx_len != 0; +} + +Status build_physical_phrase_plan_prefix(const LogicalIndexReader& idx, + const segment_v2::InvertedIndexQueryInfo& query_info, + size_t clause_count, bool allow_common_grams, + PhysicalPhrasePlan* plan, bool* all_representable) { + plan->unique_terms.clear(); + plan->phrase_plan_index.clear(); + plan->position_offsets.clear(); + plan->common_gram_clauses.clear(); + *all_representable = true; + DORIS_CHECK_LE(clause_count, query_info.term_infos.size()); + if (clause_count == 0) { + return Status::OK(); + } + + const int32_t first_position = query_info.term_infos.front().position; + DORIS_CHECK_LE(clause_count, static_cast(std::numeric_limits::max())); + plan->phrase_plan_index.reserve(clause_count); + plan->position_offsets.reserve(clause_count); + plan->common_gram_clauses.reserve(clause_count); + for (size_t i = 0; i < clause_count; ++i) { + const segment_v2::TermInfo& term_info = query_info.term_infos[i]; + DORIS_CHECK(term_info.is_single_term()); + DORIS_CHECK_EQ(static_cast(term_info.position), + static_cast(first_position) + static_cast(i)); + + std::string physical_term; + if (term_info.key_kind == segment_v2::TermKeyKind::kCommonGram) { + DORIS_CHECK(allow_common_grams); + DORIS_CHECK(std::string_view(term_info.get_single_term()) + .starts_with(segment_v2::inverted_index::CG_V1_MARKER)); + physical_term = term_info.get_single_term(); + } else { + bool representable = false; + RETURN_IF_ERROR(internal::route_plain_query_term(idx, term_info.get_single_term(), + &physical_term, &representable)); + if (!representable) { + *all_representable = false; + return Status::OK(); + } + } + + auto unique = std::ranges::find(plan->unique_terms, physical_term); + if (unique == plan->unique_terms.end()) { + plan->phrase_plan_index.push_back(plan->unique_terms.size()); + plan->unique_terms.push_back(std::move(physical_term)); + } else { + plan->phrase_plan_index.push_back( + static_cast(unique - plan->unique_terms.begin())); + } + plan->position_offsets.push_back(static_cast(i)); + plan->common_gram_clauses.push_back( + static_cast(term_info.key_kind == segment_v2::TermKeyKind::kCommonGram)); + } + return Status::OK(); +} + +Status build_physical_phrase_plan(const LogicalIndexReader& idx, + const segment_v2::InvertedIndexQueryInfo& query_info, + bool allow_common_grams, PhysicalPhrasePlan* plan, + bool* all_representable) { + return build_physical_phrase_plan_prefix(idx, query_info, query_info.term_infos.size(), + allow_common_grams, plan, all_representable); +} + +size_t resolved_batch_index(const std::vector& batch_terms, std::string_view term) { + const auto it = std::ranges::lower_bound(batch_terms, term); + DORIS_CHECK(it != batch_terms.end()); + DORIS_CHECK_EQ(*it, term); + return static_cast(it - batch_terms.begin()); +} + +bool all_plan_terms_present(const PhysicalPhrasePlan& plan, + const std::vector& batch_terms, + const std::vector& found) { + for (const std::string& term : plan.unique_terms) { + if (found[resolved_batch_index(batch_terms, term)] == 0) { + return false; + } + } + return true; +} + +internal::ResolvedPhrasePlan materialize_resolved_phrase_plan( + const PhysicalPhrasePlan& plan, const std::vector& batch_terms, + std::vector* resolved) { + internal::ResolvedPhrasePlan result; + result.phrase_plan_index = plan.phrase_plan_index; + result.position_offsets = plan.position_offsets; + result.unique_terms.reserve(plan.unique_terms.size()); + for (const std::string& term : plan.unique_terms) { + result.unique_terms.push_back( + std::move((*resolved)[resolved_batch_index(batch_terms, term)])); + } + return result; +} + +internal::ResolvedPhrasePlan copy_resolved_phrase_plan( + const PhysicalPhrasePlan& plan, const std::vector& batch_terms, + const std::vector& resolved) { + internal::ResolvedPhrasePlan result; + result.phrase_plan_index = plan.phrase_plan_index; + result.position_offsets = plan.position_offsets; + result.unique_terms.reserve(plan.unique_terms.size()); + for (const std::string& term : plan.unique_terms) { + result.unique_terms.push_back(resolved[resolved_batch_index(batch_terms, term)]); + } + return result; +} + +namespace { +PhysicalPhrasePlan build_hybrid_positioned_verification( + const PhysicalPhrasePlan& plain_plan, const PhysicalPhrasePlan& gram_plan, + const std::vector& batch_terms, const std::vector& resolved, + bool tail_covers_last_plain_clause, PhysicalPhrasePlan* candidate_prefilter) { + const size_t original_clause_count = plain_plan.phrase_plan_index.size(); + DORIS_CHECK_GT(original_clause_count, 0); + DORIS_CHECK_EQ(plain_plan.position_offsets.size(), original_clause_count); + DORIS_CHECK_EQ(plain_plan.common_gram_clauses.size(), original_clause_count); + DORIS_CHECK_EQ(gram_plan.position_offsets.size(), gram_plan.phrase_plan_index.size()); + DORIS_CHECK_EQ(gram_plan.common_gram_clauses.size(), gram_plan.phrase_plan_index.size()); + + PhysicalPhrasePlan verification; + std::vector positioned_gram_at(original_clause_count, + gram_plan.phrase_plan_index.size()); + for (size_t clause = 0; clause < gram_plan.phrase_plan_index.size(); ++clause) { + if (gram_plan.common_gram_clauses[clause] == 0) { + continue; + } + const size_t gram_term = gram_plan.phrase_plan_index[clause]; + DORIS_CHECK_LT(gram_term, gram_plan.unique_terms.size()); + const size_t batch_index = + resolved_batch_index(batch_terms, gram_plan.unique_terms[gram_term]); + if (!entry_has_positions(resolved[batch_index].entry)) { + if (candidate_prefilter != nullptr) { + append_physical_phrase_clause(gram_plan, clause, gram_plan.position_offsets[clause], + candidate_prefilter); + } + continue; + } + + const size_t original_offset = gram_plan.position_offsets[clause]; + DORIS_CHECK_LT(original_offset + 1, original_clause_count); + DORIS_CHECK_EQ(positioned_gram_at[original_offset], gram_plan.phrase_plan_index.size()); + positioned_gram_at[original_offset] = clause; + } + // Start from every positioned gram edge, then remove an edge only when both + // endpoint tokens remain covered by another positioned edge. The left-to-right + // pass produces a minimum-clause edge cover while retaining grams in preference + // to adding their two plain components during the pass below. + std::vector positioned_coverage(original_clause_count, 0); + if (tail_covers_last_plain_clause) { + ++positioned_coverage.back(); + } + for (size_t original_offset = 0; original_offset < original_clause_count; ++original_offset) { + if (positioned_gram_at[original_offset] == gram_plan.phrase_plan_index.size()) { + continue; + } + ++positioned_coverage[original_offset]; + ++positioned_coverage[original_offset + 1]; + } + for (size_t original_offset = 0; original_offset < original_clause_count; ++original_offset) { + if (positioned_gram_at[original_offset] == gram_plan.phrase_plan_index.size() || + positioned_coverage[original_offset] <= 1 || + positioned_coverage[original_offset + 1] <= 1) { + continue; + } + positioned_gram_at[original_offset] = gram_plan.phrase_plan_index.size(); + --positioned_coverage[original_offset]; + --positioned_coverage[original_offset + 1]; + } + + for (size_t original_offset = 0; original_offset < original_clause_count; ++original_offset) { + const size_t gram_clause = positioned_gram_at[original_offset]; + if (gram_clause != gram_plan.phrase_plan_index.size()) { + append_physical_phrase_clause(gram_plan, gram_clause, + static_cast(original_offset), &verification); + continue; + } + if (positioned_coverage[original_offset] == 0) { + DORIS_CHECK_EQ(plain_plan.position_offsets[original_offset], original_offset); + append_physical_phrase_clause(plain_plan, original_offset, + static_cast(original_offset), &verification); + } + } + DORIS_CHECK(!verification.phrase_plan_index.empty() || + (tail_covers_last_plain_clause && original_clause_count == 1)); + return verification; +} + +HybridPositionedCover build_hybrid_positioned_cover(const PhysicalPhrasePlan& plain_plan, + const PhysicalPhrasePlan& gram_plan, + const std::vector& batch_terms, + const std::vector& resolved, + bool tail_covers_last_plain_clause) { + HybridPositionedCover result; + result.verification = build_hybrid_positioned_verification( + plain_plan, gram_plan, batch_terms, resolved, tail_covers_last_plain_clause, + &result.candidate_prefilter); + return result; +} + +} // namespace +HybridExactPlanArtifact build_hybrid_exact_plan_artifact( + const PhysicalPhrasePlan& plain_plan, const PhysicalPhrasePlan& gram_plan, + const std::vector& batch_terms, + const std::vector& resolved) { + HybridExactPlanArtifact artifact; + if (gram_plan.phrase_plan_index.size() > 1 && + physical_phrase_plan_has_docs_only_term(gram_plan, batch_terms, resolved)) { + artifact.positioned_cover.emplace( + build_hybrid_positioned_cover(plain_plan, gram_plan, batch_terms, resolved, + /*tail_covers_last_plain_clause=*/false)); + DORIS_CHECK(!artifact.positioned_cover->candidate_prefilter.phrase_plan_index.empty()); + } + return artifact; +} + +namespace { +Status build_physical_phrase_plan_candidates(const LogicalIndexReader& idx, + const PhysicalPhrasePlan& plan, + const std::vector& batch_terms, + const std::vector& resolved, + std::vector* candidates) { + std::vector candidate_terms; + candidate_terms.reserve(plan.unique_terms.size()); + for (const std::string& term : plan.unique_terms) { + candidate_terms.push_back(resolved[resolved_batch_index(batch_terms, term)]); + } + + io::BatchRangeFetcher round1(idx.reader()); + std::vector plans; + RETURN_IF_ERROR(internal::plan_resolved_terms(idx, std::move(candidate_terms), &round1, &plans, + /*need_positions=*/false)); + if (round1.pending() > 0) { + RETURN_IF_ERROR(round1.fetch()); + } + RETURN_IF_ERROR(internal::open_preludes(round1, &plans, /*need_positions=*/false)); + return internal::build_docid_only_conjunction(idx, round1, plans, candidates); +} + +HybridPrefixMappedTails split_hybrid_prefix_mapped_tails( + const std::vector& resolved, + const std::vector& mapped_tails) { + HybridPrefixMappedTails result; + for (const ResolvedMappedTail& tail : mapped_tails) { + DORIS_CHECK_LT(tail.batch_index, resolved.size()); + if (entry_has_positions(resolved[tail.batch_index].entry)) { + result.positioned_indices.push_back(tail.batch_index); + } else { + result.docs_only_indices.push_back(tail.batch_index); + result.docs_only_ordinals.push_back(tail.expansion_ordinal); + } + } + return result; +} + +} // namespace +std::optional try_build_hybrid_prefix_plan_artifact( + const PhysicalPhrasePlan& plain_leading, const PhysicalPhrasePlan& gram_leading, + const std::vector& batch_terms, const std::vector& resolved, + const std::vector& mapped_tails, bool maps_tail_to_gram) { + const bool requires_plain_verification = + physical_phrase_plan_has_docs_only_term(gram_leading, batch_terms, resolved) || + std::ranges::any_of(mapped_tails, [&](const ResolvedMappedTail& tail) { + DORIS_CHECK_LT(tail.batch_index, resolved.size()); + return !entry_has_positions(resolved[tail.batch_index].entry); + }); + if (!requires_plain_verification) { + return std::nullopt; + } + + DORIS_CHECK(!plain_leading.phrase_plan_index.empty()); + DORIS_CHECK_LE(plain_leading.phrase_plan_index.size(), + static_cast(std::numeric_limits::max())); + HybridPrefixPlanArtifact artifact; + artifact.plain_tail_cover = + build_hybrid_positioned_cover(plain_leading, gram_leading, batch_terms, resolved, + /*tail_covers_last_plain_clause=*/false); + artifact.plain_tail_position_offset = + static_cast(plain_leading.phrase_plan_index.size()); + artifact.maps_tail_to_gram = maps_tail_to_gram; + if (!maps_tail_to_gram) { + DORIS_CHECK(mapped_tails.empty()); + return artifact; + } + + DORIS_CHECK(!mapped_tails.empty()); + artifact.mapped_tail_split = split_hybrid_prefix_mapped_tails(resolved, mapped_tails); + if (!artifact.mapped_tail_split.positioned_indices.empty()) { + artifact.positioned_tail_verification.emplace(build_hybrid_positioned_verification( + plain_leading, gram_leading, batch_terms, resolved, + /*tail_covers_last_plain_clause=*/true, + /*candidate_prefilter=*/nullptr)); + } + return artifact; +} + +Status build_hybrid_leading_candidates(const LogicalIndexReader& idx, + const PhysicalPhrasePlan& candidate_prefilter, + const std::vector& batch_terms, + const std::vector& resolved, + HybridPrefixCandidateSet* candidates) { + candidates->active = !candidate_prefilter.phrase_plan_index.empty(); + candidates->docs.clear(); + if (!candidates->active) { + return Status::OK(); + } + return build_physical_phrase_plan_candidates(idx, candidate_prefilter, batch_terms, resolved, + &candidates->docs); +} + +namespace { +Status build_tail_candidates_within_leading(const LogicalIndexReader& idx, + const std::vector& resolved, + const std::vector& tail_indices, + const std::vector& leading_candidates, + std::vector* candidates) { + std::vector tail_terms; + tail_terms.reserve(tail_indices.size()); + for (size_t index : tail_indices) { + DORIS_CHECK_LT(index, resolved.size()); + tail_terms.push_back(resolved[index]); + } + + io::BatchRangeFetcher round1(idx.reader()); + std::vector tail_plans; + RETURN_IF_ERROR(internal::plan_resolved_terms(idx, std::move(tail_terms), &round1, &tail_plans, + /*need_positions=*/false)); + if (round1.pending() > 0) { + RETURN_IF_ERROR(round1.fetch()); + } + RETURN_IF_ERROR(internal::open_preludes(round1, &tail_plans, /*need_positions=*/false)); + + candidates->clear(); + std::vector one_tail_plan; + one_tail_plan.reserve(1); + for (auto& tail_plan : tail_plans) { + one_tail_plan.clear(); + one_tail_plan.push_back(std::move(tail_plan)); + std::vector tail_matches; + RETURN_IF_ERROR(internal::filter_docids_by_conjunction( + idx, round1, one_tail_plan, leading_candidates, &tail_matches, nullptr)); + internal::union_sorted_into(candidates, tail_matches); + if (candidates->size() == leading_candidates.size()) { + break; + } + } + return Status::OK(); +} + +} // namespace +Status build_hybrid_docs_only_tail_candidates(const LogicalIndexReader& idx, + const std::vector& resolved, + const std::vector& gram_tail_indices, + const HybridPrefixCandidateSet& leading_candidates, + std::vector* candidates) { + DORIS_CHECK(!gram_tail_indices.empty()); + unsigned __int128 tail_df_sum = 0; + for (size_t index : gram_tail_indices) { + DORIS_CHECK_LT(index, resolved.size()); + tail_df_sum += resolved[index].entry.df; + } + if (leading_candidates.active && + static_cast(leading_candidates.docs.size()) <= tail_df_sum) { + return build_tail_candidates_within_leading(idx, resolved, gram_tail_indices, + leading_candidates.docs, candidates); + } + + std::vector tail_postings; + tail_postings.reserve(gram_tail_indices.size()); + for (size_t index : gram_tail_indices) { + const auto& tail = resolved[index]; + tail_postings.push_back({tail.entry, tail.frq_base, tail.prx_base}); + } + std::vector tail_candidates; + RETURN_IF_ERROR(internal::build_docid_union(idx, tail_postings, &tail_candidates)); + *candidates = leading_candidates.active + ? internal::intersect_sorted(leading_candidates.docs, tail_candidates) + : std::move(tail_candidates); + return Status::OK(); +} + +Status execute_hybrid_exact_phrase_plan( + const LogicalIndexReader& idx, const PhysicalPhrasePlan& gram_plan, + const std::vector& batch_terms, const HybridExactPlanArtifact& artifact, + std::vector* resolved, std::vector* docids, + format::PrxDecodeContext* decode_context, bool* candidate_intersection_empty) { + DORIS_CHECK(idx.common_grams_posting_policy() == format::CommonGramsPostingPolicy::kHybridV1); + if (candidate_intersection_empty != nullptr) { + *candidate_intersection_empty = false; + } + if (!artifact.positioned_cover.has_value()) { + auto resolved_gram = materialize_resolved_phrase_plan(gram_plan, batch_terms, resolved); + return internal::execute_resolved_phrase_plan(idx, std::move(resolved_gram), docids, + decode_context); + } + + const HybridPositionedCover& hybrid_plan = *artifact.positioned_cover; + std::vector gram_candidates; + RETURN_IF_ERROR(build_physical_phrase_plan_candidates( + idx, hybrid_plan.candidate_prefilter, batch_terms, *resolved, &gram_candidates)); + if (gram_candidates.empty()) { + if (candidate_intersection_empty != nullptr) { + *candidate_intersection_empty = true; + } + return Status::OK(); + } + auto resolved_verification = + materialize_resolved_phrase_plan(hybrid_plan.verification, batch_terms, resolved); + return internal::execute_resolved_phrase_plan(idx, std::move(resolved_verification), docids, + decode_context, nullptr, &gram_candidates); +} + +void append_resolved_phrase_clause(ResolvedQueryTerm term, uint32_t position_offset, + internal::ResolvedPhrasePlan* plan) { + const auto unique = std::ranges::find(plan->unique_terms, term.entry.term, + [](const ResolvedQueryTerm& candidate) { + return std::string_view(candidate.entry.term); + }); + if (unique == plan->unique_terms.end()) { + plan->phrase_plan_index.push_back(plan->unique_terms.size()); + plan->unique_terms.push_back(std::move(term)); + } else { + plan->phrase_plan_index.push_back(static_cast(unique - plan->unique_terms.begin())); + } + plan->position_offsets.push_back(position_offset); +} + +internal::ResolvedPhrasePlan build_resolved_phrase_plan( + std::vector resolved_terms) { + internal::ResolvedPhrasePlan plan; + plan.unique_terms.reserve(resolved_terms.size()); + plan.phrase_plan_index.reserve(resolved_terms.size()); + plan.position_offsets.reserve(resolved_terms.size()); + for (size_t i = 0; i < resolved_terms.size(); ++i) { + DORIS_CHECK_LE(i, static_cast(std::numeric_limits::max())); + append_resolved_phrase_clause(std::move(resolved_terms[i]), static_cast(i), + &plan); + } + return plan; +} + +Status resolve_and_execute_physical_phrase_plan(const LogicalIndexReader& idx, + const PhysicalPhrasePlan& plan, + std::vector* docids, + format::PrxDecodeContext* decode_context, + CommonGramsPlanningTimer& planning_timer) { + std::vector batch_terms = plan.unique_terms; + std::ranges::sort(batch_terms); + std::vector resolved; + std::vector found; + RETURN_IF_ERROR(internal::resolve_query_terms_batch(idx, batch_terms, &resolved, &found)); + if (!all_plan_terms_present(plan, batch_terms, found)) { + return Status::OK(); + } + auto resolved_plan = materialize_resolved_phrase_plan(plan, batch_terms, &resolved); + planning_timer.finish(); + return internal::execute_resolved_phrase_plan(idx, std::move(resolved_plan), docids, + decode_context); +} + +} // namespace doris::snii::query::phrase_impl diff --git a/be/src/storage/index/snii/query/phrase_planned_query.cpp b/be/src/storage/index/snii/query/phrase_planned_query.cpp new file mode 100644 index 00000000000000..487978d82626d6 --- /dev/null +++ b/be/src/storage/index/snii/query/phrase_planned_query.cpp @@ -0,0 +1,745 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/check.h" +#include "storage/index/inverted/common_grams/common_grams_key_codec.h" +#include "storage/index/inverted/common_grams/common_grams_query_cost.h" +#include "storage/index/inverted/common_grams/common_grams_segment_metadata.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/encoding/byte_source.h" +#include "storage/index/snii/format/dict_entry.h" +#include "storage/index/snii/format/frq_pod.h" +#include "storage/index/snii/format/frq_prelude.h" +#include "storage/index/snii/format/prx_pod.h" +#include "storage/index/snii/io/batch_range_fetcher.h" +#include "storage/index/snii/query/internal/docid_conjunction.h" +#include "storage/index/snii/query/internal/docid_posting_reader.h" +#include "storage/index/snii/query/internal/docid_set_ops.h" +#include "storage/index/snii/query/internal/docid_union.h" +#include "storage/index/snii/query/internal/phrase_query_split.h" +#include "storage/index/snii/query/internal/plain_term_routing.h" +#include "storage/index/snii/query/internal/position_math.h" +#include "storage/index/snii/query/internal/query_test_counters.h" +#include "storage/index/snii/query/internal/resolved_phrase_plan.h" +#include "storage/index/snii/query/internal/term_expansion.h" +#include "storage/index/snii/query/phrase_prx_validation.h" +#include "storage/index/snii/query/phrase_query.h" +#include "storage/index/snii/query/phrase_verify_timer.h" +#include "storage/index/snii/query/prefix_query.h" +#include "storage/index/snii/query/term_query.h" +#include "storage/index/snii/reader/windowed_posting.h" +#include "util/debug_points.h" + +namespace doris::snii::query::phrase_impl { + +using query::internal::DocidChunk; +using query::internal::DocidSource; +using query::internal::ResolvedQueryTerm; +using query::internal::TermPlan; +using reader::LogicalIndexReader; +using internal::PhraseVerifyTimer; + +Status planned_exact_phrase_query_impl( + const LogicalIndexReader& idx, const segment_v2::InvertedIndexQueryInfo& plain_query_info, + const segment_v2::InvertedIndexQueryInfo& gram_query_info, + const segment_v2::inverted_index::CommonGramsQueryIdentity* common_grams_identity, + std::vector* docids, format::PrxDecodeContext* decode_context, + ExactPhrasePlanKind* selected_plan, + segment_v2::inverted_index::CommonGramsPlanCostModel cost_model, + CommonGramsPlanDebugOverride debug_override) { + if (docids == nullptr) { + return Status::Error( + "planned_exact_phrase_query: null out"); + } + docids->clear(); + DORIS_CHECK(!plain_query_info.has_common_gram()); + auto* query_stats = decode_context == nullptr ? nullptr : decode_context->query_stats; + CommonGramsPlanningTimer planning_timer(query_stats); + if (query_stats != nullptr) { + ++query_stats->common_grams_candidate_queries; + } + + PhysicalPhrasePlan plain_plan; + bool plain_representable = false; + RETURN_IF_ERROR(build_physical_phrase_plan(idx, plain_query_info, + /*allow_common_grams=*/false, &plain_plan, + &plain_representable)); + if (!plain_representable || plain_plan.phrase_plan_index.empty()) { + if (query_stats != nullptr) { + ++query_stats->common_grams_plain_plans; + } + if (selected_plan != nullptr) { + *selected_plan = ExactPhrasePlanKind::kPlain; + } + return Status::OK(); + } + + const bool index_has_common_grams = has_common_grams_capability(idx, common_grams_identity); + const bool gram_capable = gram_query_info.has_common_gram() && index_has_common_grams; + if (!gram_capable) { + if (query_stats != nullptr) { + ++query_stats->common_grams_plain_plans; + if (gram_query_info.has_common_gram()) { + ++query_stats->common_grams_fallback_incompatible; + } else { + ++query_stats->common_grams_fallback_no_gram; + } + } + if (selected_plan != nullptr) { + *selected_plan = ExactPhrasePlanKind::kPlain; + } + return resolve_and_execute_physical_phrase_plan(idx, plain_plan, docids, decode_context, + planning_timer); + } + + PhysicalPhrasePlan gram_plan; + bool gram_representable = false; + RETURN_IF_ERROR(build_physical_phrase_plan(idx, gram_query_info, + /*allow_common_grams=*/true, &gram_plan, + &gram_representable)); + DORIS_CHECK(gram_representable); + DORIS_CHECK(!gram_plan.phrase_plan_index.empty()); + + std::vector batch_terms = plain_plan.unique_terms; + batch_terms.insert(batch_terms.end(), gram_plan.unique_terms.begin(), + gram_plan.unique_terms.end()); + std::ranges::sort(batch_terms); + batch_terms.erase(std::unique(batch_terms.begin(), batch_terms.end()), batch_terms.end()); + + std::vector resolved; + std::vector found; + RETURN_IF_ERROR(internal::resolve_query_terms_batch(idx, batch_terms, &resolved, &found)); + const bool plain_present = all_plan_terms_present(plain_plan, batch_terms, found); + const bool gram_present = all_plan_terms_present(gram_plan, batch_terms, found); + if (!plain_present || !gram_present) { + if (query_stats != nullptr) { + if (plain_present) { + ++query_stats->common_grams_gram_plans; + ++query_stats->common_grams_authoritative_empty; + } else { + ++query_stats->common_grams_plain_plans; + ++query_stats->common_grams_authoritative_empty; + } + } + if (selected_plan != nullptr) { + *selected_plan = + plain_present ? ExactPhrasePlanKind::kCommonGrams : ExactPhrasePlanKind::kPlain; + } + return Status::OK(); + } + + std::optional hybrid_artifact; + if (idx.common_grams_posting_policy() == format::CommonGramsPostingPolicy::kHybridV1) { + hybrid_artifact.emplace( + build_hybrid_exact_plan_artifact(plain_plan, gram_plan, batch_terms, resolved)); + } + const bool plain_needs_positions = plain_plan.phrase_plan_index.size() > 1; + const bool hybrid_gram_verification = + hybrid_artifact.has_value() && hybrid_artifact->positioned_cover.has_value(); + const bool gram_needs_positions = + gram_plan.phrase_plan_index.size() > 1 && !hybrid_gram_verification; + const auto plain_raw_cost = + phrase_plan_raw_cost(plain_plan, batch_terms, resolved, found, plain_needs_positions); + segment_v2::inverted_index::CommonGramsPlanRawCost gram_raw_cost; + if (hybrid_gram_verification) { + const HybridPositionedCover& hybrid_plan = *hybrid_artifact->positioned_cover; + const auto gram_prefilter_raw_cost = + phrase_plan_raw_cost(hybrid_plan.candidate_prefilter, batch_terms, resolved, found, + /*need_positions=*/false); + const auto gram_verification_raw_cost = + phrase_plan_raw_cost(hybrid_plan.verification, batch_terms, resolved, found, + /*need_positions=*/true); + gram_raw_cost = + hybrid_verification_raw_cost(gram_prefilter_raw_cost, gram_verification_raw_cost); + } else { + gram_raw_cost = + phrase_plan_raw_cost(gram_plan, batch_terms, resolved, found, gram_needs_positions); + } + const uint64_t plain_cost = segment_v2::inverted_index::estimate_common_grams_plan_cost( + plain_raw_cost, plain_needs_positions ? cost_model.position_verify_factor : 0); + const uint64_t gram_cost = segment_v2::inverted_index::estimate_common_grams_plan_cost( + gram_raw_cost, (gram_needs_positions || hybrid_gram_verification) + ? cost_model.position_verify_factor + : 0); + const bool cost_prefers_gram = segment_v2::inverted_index::common_grams_plan_cost_wins( + plain_cost, gram_cost, cost_model.common_grams_cost_ratio_percent); + const bool use_gram = apply_common_grams_plan_debug_override(cost_prefers_gram, debug_override); + if (query_stats != nullptr) { + query_stats->common_grams_plain_posting_bytes += plain_raw_cost.posting_bytes_or_df_sum; + query_stats->common_grams_gram_posting_bytes += gram_raw_cost.posting_bytes_or_df_sum; + query_stats->common_grams_plain_estimated_candidate_df += + plain_raw_cost.estimated_candidate_df; + query_stats->common_grams_gram_estimated_candidate_df += + gram_raw_cost.estimated_candidate_df; + query_stats->common_grams_plain_estimated_cost += plain_cost; + query_stats->common_grams_gram_estimated_cost += gram_cost; + } + const ExactPhrasePlanKind chosen_kind = + use_gram ? ExactPhrasePlanKind::kCommonGrams : ExactPhrasePlanKind::kPlain; + if (query_stats != nullptr) { + if (use_gram) { + ++query_stats->common_grams_gram_plans; + } else { + ++query_stats->common_grams_plain_plans; + if (debug_override == CommonGramsPlanDebugOverride::kNone) { + ++query_stats->common_grams_fallback_cost; + } + } + } + if (selected_plan != nullptr) { + *selected_plan = chosen_kind; + } + planning_timer.finish(); + if (use_gram && + idx.common_grams_posting_policy() == format::CommonGramsPostingPolicy::kHybridV1) { + DORIS_CHECK(hybrid_artifact.has_value()); + bool candidate_intersection_empty = false; + RETURN_IF_ERROR(execute_hybrid_exact_phrase_plan( + idx, gram_plan, batch_terms, *hybrid_artifact, &resolved, docids, decode_context, + &candidate_intersection_empty)); + if (candidate_intersection_empty) { + if (query_stats != nullptr) { + ++query_stats->common_grams_authoritative_empty; + } + } + return Status::OK(); + } + auto resolved_plan = materialize_resolved_phrase_plan(use_gram ? gram_plan : plain_plan, + batch_terms, &resolved); + return internal::execute_resolved_phrase_plan(idx, std::move(resolved_plan), docids, + decode_context, nullptr, nullptr, + internal::ExactPhrasePositionAccess::kAuto); +} + +Status phrase_query_impl(const LogicalIndexReader& idx, const std::vector& terms, + std::vector* const docids, + format::PrxDecodeContext* decode_context, + std::vector* matches, const PhraseQueryOptions& options) { + if (docids == nullptr && matches == nullptr) { + return Status::Error("phrase_query: null out"); + } + if (docids != nullptr) { + docids->clear(); + } + if (matches != nullptr) { + matches->clear(); + } + if (terms.empty()) { + return Status::OK(); + } + if (terms.size() == 1) { + DORIS_CHECK(matches == nullptr); + return term_query(idx, terms.front(), docids); + } + if (!idx.has_positions()) { + return Status::Error( + "phrase_query: index has no positions"); + } + io::BatchRangeFetcher round1(idx.reader()); + const PhraseTermMapping mapping = build_phrase_term_mapping(terms); + std::vector plans; + bool all_present = false; + RETURN_IF_ERROR(internal::plan_terms(idx, mapping.unique_terms, &round1, &plans, &all_present, + /*need_positions=*/false)); + if (!all_present) { + return Status::OK(); + } + return execute_phrase_plans(idx, &round1, &plans, mapping.phrase_plan_index, docids, + decode_context, matches, options); +} + +Status phrase_prefix_query_impl(const LogicalIndexReader& idx, + const std::vector& terms, + std::vector* const docids, int32_t max_expansions, + format::PrxDecodeContext* decode_context, + CommonGramsPlanningTimer* planning_timer, + std::vector* matches) { + if (docids == nullptr && matches == nullptr) { + return Status::Error("phrase_prefix_query: null out"); + } + if (docids != nullptr) { + docids->clear(); + } + if (matches != nullptr) { + matches->clear(); + } + if (terms.empty()) { + return Status::OK(); + } + if (terms.size() == 1) { + DORIS_CHECK(matches == nullptr); + if (planning_timer != nullptr) { + planning_timer->finish(); + } + return prefix_query(idx, terms.front(), docids, max_expansions); + } + std::vector exact_terms; + exact_terms.reserve(terms.size() - 1); + std::string physical_term_scratch; + for (size_t i = 0; i + 1 < terms.size(); ++i) { + std::string_view physical_term; + bool representable = false; + RETURN_IF_ERROR(internal::route_plain_query_term_view(idx, terms[i], &physical_term_scratch, + &physical_term, &representable)); + if (!representable) { + return Status::OK(); + } + ResolvedQueryTerm resolved; + bool found = false; + RETURN_IF_ERROR(internal::resolve_query_term(idx, physical_term, &resolved, &found)); + if (!found) { + return Status::OK(); + } + exact_terms.push_back(std::move(resolved)); + } + + // Expand the tail in the logical plain namespace. The visitor range-seeks + // past typed internal namespaces before counting max_expansions and decodes + // escaped physical keys before applying the logical prefix. + std::vector tail_hits; + RETURN_IF_ERROR(internal::visit_expanded_plain_terms( + idx, terms.back(), [](std::string_view) { return true; }, + [&](LogicalIndexReader::PrefixHit&& hit, bool*) { + tail_hits.push_back(std::move(hit)); + return Status::OK(); + }, + max_expansions)); + if (tail_hits.empty()) { + return Status::OK(); + } + std::vector tail_terms; + tail_terms.reserve(tail_hits.size()); + for (auto& hit : tail_hits) { + tail_terms.push_back(ResolvedQueryTerm { + .entry = std::move(hit.entry), .frq_base = hit.frq_base, .prx_base = hit.prx_base}); + } + auto exact_plan = build_resolved_phrase_plan(std::move(exact_terms)); + if (planning_timer != nullptr) { + planning_timer->finish(); + } + DORIS_CHECK_LE(terms.size() - 1, static_cast(std::numeric_limits::max())); + return execute_resolved_phrase_prefix_terms(idx, std::move(exact_plan), std::move(tail_terms), + static_cast(terms.size() - 1), docids, + decode_context, matches); +} + +Status planned_phrase_prefix_query_impl( + const LogicalIndexReader& idx, const segment_v2::InvertedIndexQueryInfo& plain_query_info, + const segment_v2::InvertedIndexQueryInfo& gram_query_info, + const segment_v2::inverted_index::CommonGramsQueryIdentity* common_grams_identity, + std::vector* docids, int32_t max_expansions, + format::PrxDecodeContext* decode_context, PhrasePrefixPlanKind* selected_plan, + segment_v2::inverted_index::CommonGramsPlanCostModel cost_model, + CommonGramsPlanDebugOverride debug_override) { + if (docids == nullptr) { + return Status::Error( + "planned_phrase_prefix_query: null out"); + } + docids->clear(); + if (selected_plan != nullptr) { + *selected_plan = PhrasePrefixPlanKind::kPlain; + } + DORIS_CHECK(!plain_query_info.has_common_gram()); + auto* query_stats = decode_context == nullptr ? nullptr : decode_context->query_stats; + CommonGramsPlanningTimer planning_timer(query_stats); + if (query_stats != nullptr) { + ++query_stats->common_grams_candidate_queries; + } + if (plain_query_info.term_infos.empty()) { + if (query_stats != nullptr) { + ++query_stats->common_grams_plain_plans; + } + return Status::OK(); + } + for (const auto& term_info : plain_query_info.term_infos) { + DORIS_CHECK(term_info.is_single_term()); + } + + PhysicalPhrasePlan plain_leading; + bool plain_representable = false; + RETURN_IF_ERROR(build_physical_phrase_plan_prefix( + idx, plain_query_info, plain_query_info.term_infos.size() - 1, + /*allow_common_grams=*/false, &plain_leading, &plain_representable)); + if (!plain_representable) { + if (query_stats != nullptr) { + ++query_stats->common_grams_plain_plans; + } + return Status::OK(); + } + + const auto execute_plain = [&]() { + std::vector terms; + terms.reserve(plain_query_info.term_infos.size()); + for (const auto& term_info : plain_query_info.term_infos) { + terms.push_back(term_info.get_single_term()); + } + return phrase_prefix_query_impl(idx, terms, docids, max_expansions, decode_context, + &planning_timer); + }; + + const bool index_has_common_grams = has_common_grams_capability(idx, common_grams_identity); + + const bool can_plan_common_grams = gram_query_info.has_common_gram() && index_has_common_grams; + if (!can_plan_common_grams) { + if (query_stats != nullptr) { + ++query_stats->common_grams_plain_plans; + if (gram_query_info.has_common_gram()) { + ++query_stats->common_grams_fallback_incompatible; + } else { + ++query_stats->common_grams_fallback_no_gram; + } + } + return execute_plain(); + } + DORIS_CHECK(!gram_query_info.term_infos.empty()); + DORIS_CHECK(gram_query_info.term_infos.back().is_single_term()); + + PhysicalPhrasePlan gram_leading; + bool gram_representable = false; + RETURN_IF_ERROR(build_physical_phrase_plan_prefix( + idx, gram_query_info, gram_query_info.term_infos.size() - 1, + /*allow_common_grams=*/true, &gram_leading, &gram_representable)); + if (!gram_representable) { + if (query_stats != nullptr) { + ++query_stats->common_grams_plain_plans; + ++query_stats->common_grams_fallback_incompatible; + } + return execute_plain(); + } + + internal::ResolvedPhrasePlan selected_leading; + std::vector selected_tail_terms; + DORIS_CHECK_LE(plain_leading.phrase_plan_index.size(), + static_cast(std::numeric_limits::max())); + const uint32_t plain_tail_position_offset = + static_cast(plain_leading.phrase_plan_index.size()); + uint32_t selected_tail_position_offset = plain_tail_position_offset; + bool authoritative_empty = false; + bool execute_as_exact_phrase = false; + bool hybrid_executed = false; + const Status planning_status = [&]() -> Status { + const bool maps_tail_to_gram = + gram_query_info.term_infos.back().key_kind == segment_v2::TermKeyKind::kCommonGram; + std::vector logical_tail_terms; + std::vector plain_tail_terms; + std::vector tail_hits; + RETURN_IF_ERROR(internal::visit_expanded_plain_terms( + idx, plain_query_info.term_infos.back().get_single_term(), + [](std::string_view) { return true; }, + [&](LogicalIndexReader::PrefixHit&& hit, bool*) { + tail_hits.push_back(std::move(hit)); + return Status::OK(); + }, + max_expansions)); + if (tail_hits.empty()) { + if (query_stats != nullptr) { + ++query_stats->common_grams_plain_plans; + ++query_stats->common_grams_fallback_prefix_tail_empty; + ++query_stats->common_grams_authoritative_empty; + } + authoritative_empty = true; + return Status::OK(); + } + if (maps_tail_to_gram) { + logical_tail_terms.reserve(tail_hits.size()); + } else { + DORIS_CHECK(gram_query_info.term_infos.back().key_kind == + segment_v2::TermKeyKind::kPlain); + } + plain_tail_terms.reserve(tail_hits.size()); + for (auto& hit : tail_hits) { + if (maps_tail_to_gram) { + logical_tail_terms.push_back(std::move(hit.term)); + } + plain_tail_terms.push_back(ResolvedQueryTerm {.entry = std::move(hit.entry), + .frq_base = hit.frq_base, + .prx_base = hit.prx_base}); + } + + const auto select_plain_plan = [&]() -> Status { + if (query_stats != nullptr) { + ++query_stats->common_grams_plain_plans; + ++query_stats->common_grams_fallback_incompatible; + } + std::vector batch_terms = plain_leading.unique_terms; + std::ranges::sort(batch_terms); + std::vector resolved; + std::vector found; + RETURN_IF_ERROR( + internal::resolve_query_terms_batch(idx, batch_terms, &resolved, &found)); + if (!all_plan_terms_present(plain_leading, batch_terms, found)) { + if (query_stats != nullptr) { + ++query_stats->common_grams_authoritative_empty; + } + authoritative_empty = true; + return Status::OK(); + } + selected_leading = + materialize_resolved_phrase_plan(plain_leading, batch_terms, &resolved); + selected_tail_terms = std::move(plain_tail_terms); + selected_tail_position_offset = plain_tail_position_offset; + return Status::OK(); + }; + + std::vector mapped_tail_terms; + if (maps_tail_to_gram) { + DORIS_CHECK_GE(plain_query_info.term_infos.size(), 2U); + const std::string& left = + plain_query_info.term_infos[plain_query_info.term_infos.size() - 2] + .get_single_term(); + mapped_tail_terms.reserve(logical_tail_terms.size()); + for (const std::string& tail : logical_tail_terms) { + std::string gram; + auto encoded = + segment_v2::inverted_index::try_encode_common_gram(left, tail, &gram); + if (!encoded.has_value()) { + return std::move(encoded.error()); + } + if (!*encoded) { + return select_plain_plan(); + } + mapped_tail_terms.push_back(std::move(gram)); + } + } + + std::vector batch_terms = plain_leading.unique_terms; + batch_terms.insert(batch_terms.end(), gram_leading.unique_terms.begin(), + gram_leading.unique_terms.end()); + batch_terms.insert(batch_terms.end(), mapped_tail_terms.begin(), mapped_tail_terms.end()); + std::ranges::sort(batch_terms); + batch_terms.erase(std::unique(batch_terms.begin(), batch_terms.end()), batch_terms.end()); + + std::vector resolved; + std::vector found; + RETURN_IF_ERROR(internal::resolve_query_terms_batch(idx, batch_terms, &resolved, &found)); + const bool plain_present = all_plan_terms_present(plain_leading, batch_terms, found); + const bool gram_present = all_plan_terms_present(gram_leading, batch_terms, found); + if (!plain_present || !gram_present) { + if (query_stats != nullptr) { + if (plain_present) { + ++query_stats->common_grams_gram_plans; + ++query_stats->common_grams_authoritative_empty; + } else { + ++query_stats->common_grams_plain_plans; + ++query_stats->common_grams_authoritative_empty; + } + } + if (selected_plan != nullptr && plain_present) { + *selected_plan = PhrasePrefixPlanKind::kCommonGrams; + } + authoritative_empty = true; + return Status::OK(); + } + + std::vector present_mapped_tail_indices; + std::vector present_mapped_tails; + std::vector present_gram_tail_ordinals; + if (maps_tail_to_gram) { + present_mapped_tail_indices.reserve(mapped_tail_terms.size()); + present_mapped_tails.reserve(mapped_tail_terms.size()); + present_gram_tail_ordinals.reserve(mapped_tail_terms.size()); + for (size_t ordinal = 0; ordinal < mapped_tail_terms.size(); ++ordinal) { + const std::string& term = mapped_tail_terms[ordinal]; + const size_t batch_index = resolved_batch_index(batch_terms, term); + if (found[batch_index] != 0) { + present_mapped_tail_indices.push_back(batch_index); + DORIS_CHECK_LE(ordinal, + static_cast(std::numeric_limits::max())); + const uint32_t expansion_ordinal = static_cast(ordinal); + present_mapped_tails.push_back(ResolvedMappedTail { + .batch_index = batch_index, .expansion_ordinal = expansion_ordinal}); + present_gram_tail_ordinals.push_back(expansion_ordinal); + } + } + if (present_mapped_tail_indices.empty()) { + if (query_stats != nullptr) { + ++query_stats->common_grams_gram_plans; + ++query_stats->common_grams_authoritative_empty; + } + if (selected_plan != nullptr) { + *selected_plan = PhrasePrefixPlanKind::kCommonGrams; + } + authoritative_empty = true; + return Status::OK(); + } + } + + std::optional hybrid_artifact; + if (idx.common_grams_posting_policy() == format::CommonGramsPostingPolicy::kHybridV1) { + hybrid_artifact = try_build_hybrid_prefix_plan_artifact( + plain_leading, gram_leading, batch_terms, resolved, present_mapped_tails, + maps_tail_to_gram); + } + const bool hybrid_plan_requires_plain_verification = hybrid_artifact.has_value(); + const bool plain_needs_positions = !plain_leading.phrase_plan_index.empty(); + const bool gram_needs_positions = + !gram_leading.phrase_plan_index.empty() && !hybrid_plan_requires_plain_verification; + auto plain_raw_cost = phrase_plan_raw_cost(plain_leading, batch_terms, resolved, found, + plain_needs_positions); + const auto plain_tail_raw_cost = + alternative_clause_raw_cost(plain_tail_terms, plain_needs_positions); + append_alternative_clause_cost(plain_tail_raw_cost, &plain_raw_cost); + const uint64_t plain_cost = segment_v2::inverted_index::estimate_common_grams_plan_cost( + plain_raw_cost, plain_needs_positions ? cost_model.position_verify_factor : 0); + segment_v2::inverted_index::CommonGramsPlanRawCost gram_raw_cost; + uint64_t gram_cost = 0; + if (hybrid_plan_requires_plain_verification) { + const HybridPrefixCostEstimate hybrid_cost = estimate_hybrid_prefix_plan_cost( + *hybrid_artifact, batch_terms, resolved, found, plain_tail_terms, + cost_model.position_verify_factor); + gram_raw_cost = hybrid_cost.raw_cost; + gram_cost = hybrid_cost.estimated_cost; + } else { + gram_raw_cost = phrase_plan_raw_cost(gram_leading, batch_terms, resolved, found, + gram_needs_positions); + if (maps_tail_to_gram) { + append_alternative_clause_cost( + alternative_clause_raw_cost(resolved, present_mapped_tail_indices, + gram_needs_positions), + &gram_raw_cost); + } else { + append_alternative_clause_cost(plain_tail_raw_cost, &gram_raw_cost); + } + gram_cost = segment_v2::inverted_index::estimate_common_grams_plan_cost( + gram_raw_cost, gram_needs_positions ? cost_model.position_verify_factor : 0); + } + const bool cost_prefers_gram = segment_v2::inverted_index::common_grams_plan_cost_wins( + plain_cost, gram_cost, cost_model.common_grams_cost_ratio_percent); + const bool use_gram = + apply_common_grams_plan_debug_override(cost_prefers_gram, debug_override); + if (query_stats != nullptr) { + query_stats->common_grams_plain_posting_bytes += plain_raw_cost.posting_bytes_or_df_sum; + query_stats->common_grams_gram_posting_bytes += gram_raw_cost.posting_bytes_or_df_sum; + query_stats->common_grams_plain_estimated_candidate_df += + plain_raw_cost.estimated_candidate_df; + query_stats->common_grams_gram_estimated_candidate_df += + gram_raw_cost.estimated_candidate_df; + query_stats->common_grams_plain_estimated_cost += plain_cost; + query_stats->common_grams_gram_estimated_cost += gram_cost; + } + if (query_stats != nullptr) { + if (use_gram) { + ++query_stats->common_grams_gram_plans; + } else { + ++query_stats->common_grams_plain_plans; + if (debug_override == CommonGramsPlanDebugOverride::kNone) { + ++query_stats->common_grams_fallback_cost; + } + } + } + if (selected_plan != nullptr) { + *selected_plan = + use_gram ? PhrasePrefixPlanKind::kCommonGrams : PhrasePrefixPlanKind::kPlain; + } + + const bool hybrid_needs_plain_verification = + use_gram && hybrid_plan_requires_plain_verification; + if (hybrid_needs_plain_verification) { + DORIS_CHECK(hybrid_artifact.has_value()); + bool candidate_intersection_empty = false; + RETURN_IF_ERROR(execute_hybrid_phrase_prefix_plan( + idx, *hybrid_artifact, batch_terms, resolved, plain_tail_terms, docids, + decode_context, planning_timer, &candidate_intersection_empty)); + hybrid_executed = true; + if (candidate_intersection_empty) { + if (query_stats != nullptr) { + ++query_stats->common_grams_authoritative_empty; + } + authoritative_empty = true; + return Status::OK(); + } + return Status::OK(); + } + + const PhysicalPhrasePlan& selected_physical_leading = + use_gram ? gram_leading : plain_leading; + selected_leading = + materialize_resolved_phrase_plan(selected_physical_leading, batch_terms, &resolved); + if (!use_gram || !maps_tail_to_gram) { + selected_tail_terms = std::move(plain_tail_terms); + selected_tail_position_offset = plain_tail_position_offset; + return Status::OK(); + } + + if (present_mapped_tail_indices.size() == 1) { + DORIS_CHECK_GT(plain_tail_position_offset, 0U); + const uint32_t mapped_tail_position = plain_tail_position_offset - 1; + const size_t batch_index = present_mapped_tail_indices.front(); + const auto existing = + std::ranges::find(selected_leading.unique_terms, batch_terms[batch_index], + [](const ResolvedQueryTerm& term) { + return std::string_view(term.entry.term); + }); + if (existing == selected_leading.unique_terms.end()) { + append_resolved_phrase_clause(std::move(resolved[batch_index]), + mapped_tail_position, &selected_leading); + } else { + selected_leading.phrase_plan_index.push_back( + static_cast(existing - selected_leading.unique_terms.begin())); + selected_leading.position_offsets.push_back(mapped_tail_position); + } + execute_as_exact_phrase = true; + return Status::OK(); + } + + selected_tail_terms.reserve(present_mapped_tail_indices.size()); + for (size_t batch_index : present_mapped_tail_indices) { + const auto existing = + std::ranges::find(selected_leading.unique_terms, batch_terms[batch_index], + [](const ResolvedQueryTerm& term) { + return std::string_view(term.entry.term); + }); + if (existing == selected_leading.unique_terms.end()) { + selected_tail_terms.push_back(std::move(resolved[batch_index])); + } else { + selected_tail_terms.push_back(*existing); + } + } + DORIS_CHECK_GT(plain_tail_position_offset, 0U); + selected_tail_position_offset = plain_tail_position_offset - 1; + return Status::OK(); + }(); + RETURN_IF_ERROR(planning_status); + planning_timer.finish(); + if (authoritative_empty) { + return Status::OK(); + } + if (hybrid_executed) { + return Status::OK(); + } + if (execute_as_exact_phrase) { + return internal::execute_resolved_phrase_plan( + idx, std::move(selected_leading), docids, decode_context, nullptr, nullptr, + internal::ExactPhrasePositionAccess::kMaterializedOnly); + } + return execute_resolved_phrase_prefix_terms( + idx, std::move(selected_leading), std::move(selected_tail_terms), + selected_tail_position_offset, docids, decode_context); +} + +} // namespace doris::snii::query::phrase_impl diff --git a/be/src/storage/index/snii/query/phrase_position_source.cpp b/be/src/storage/index/snii/query/phrase_position_source.cpp new file mode 100644 index 00000000000000..1f3bbb2ffcc667 --- /dev/null +++ b/be/src/storage/index/snii/query/phrase_position_source.cpp @@ -0,0 +1,437 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/check.h" +#include "storage/index/inverted/common_grams/common_grams_key_codec.h" +#include "storage/index/inverted/common_grams/common_grams_query_cost.h" +#include "storage/index/inverted/common_grams/common_grams_segment_metadata.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/encoding/byte_source.h" +#include "storage/index/snii/format/dict_entry.h" +#include "storage/index/snii/format/frq_pod.h" +#include "storage/index/snii/format/frq_prelude.h" +#include "storage/index/snii/format/prx_frame.h" +#include "storage/index/snii/format/prx_pod.h" +#include "storage/index/snii/io/batch_range_fetcher.h" +#include "storage/index/snii/query/internal/docid_conjunction.h" +#include "storage/index/snii/query/internal/docid_posting_reader.h" +#include "storage/index/snii/query/internal/docid_set_ops.h" +#include "storage/index/snii/query/internal/docid_union.h" +#include "storage/index/snii/query/internal/phrase_query_split.h" +#include "storage/index/snii/query/internal/plain_term_routing.h" +#include "storage/index/snii/query/internal/position_math.h" +#include "storage/index/snii/query/internal/query_test_counters.h" +#include "storage/index/snii/query/internal/resolved_phrase_plan.h" +#include "storage/index/snii/query/internal/term_expansion.h" +#include "storage/index/snii/query/phrase_prx_validation.h" +#include "storage/index/snii/query/phrase_query.h" +#include "storage/index/snii/query/phrase_verify_timer.h" +#include "storage/index/snii/query/prefix_query.h" +#include "storage/index/snii/query/term_query.h" +#include "storage/index/snii/reader/windowed_posting.h" +#include "util/debug_points.h" + +namespace doris::snii::query::phrase_impl { + +using query::internal::DocidChunk; +using query::internal::DocidSource; +using query::internal::ResolvedQueryTerm; +using query::internal::TermPlan; +using reader::LogicalIndexReader; +using internal::PhraseVerifyTimer; + +PhraseTermMapping build_phrase_term_mapping(const std::vector& terms) { + PhraseTermMapping mapping; + mapping.phrase_plan_index.reserve(terms.size()); + for (const std::string& term : terms) { + auto it = std::ranges::find(mapping.unique_terms, term); + if (it == mapping.unique_terms.end()) { + mapping.phrase_plan_index.push_back(mapping.unique_terms.size()); + mapping.unique_terms.push_back(term); + continue; + } + mapping.phrase_plan_index.push_back(static_cast(it - mapping.unique_terms.begin())); + } + return mapping; +} + +namespace { +Status accumulate_frame_position_work(Slice frames, uint64_t* work) { + ByteSource source(frames); + while (!source.eof()) { + format::PrxFrameView frame; + RETURN_IF_ERROR(format::read_prx_frame(&source, &frame)); + uint64_t frame_work = frame.uncompressed_length; + if (frame.codec == format::PrxCodec::kPfor) { + ByteSource payload(frame.payload); + uint32_t doc_count = 0; + uint32_t total_positions = 0; + RETURN_IF_ERROR(payload.get_varint32(&doc_count)); + RETURN_IF_ERROR(payload.get_varint32(&total_positions)); + if (doc_count > format::kReaderPrxWindowLimits.max_docs || + total_positions > format::kReaderPrxWindowLimits.max_positions) { + return Status::Error( + "phrase_query: PFOR routing metadata exceeds sane cap"); + } + frame_work = total_positions; + } + *work += frame_work; + } + return Status::OK(); +} + +Status populate_logical_position_work(const std::vector& plans, + std::vector* sources) { + DORIS_CHECK_EQ(plans.size(), sources->size()); + for (size_t plan_index = 0; plan_index < plans.size(); ++plan_index) { + if (plans[plan_index].entry.term_stats_present) { + continue; + } + for (const PosChunk& chunk : (*sources)[plan_index].chunks) { + (*sources)[plan_index].logical_position_docs += chunk.prx_doc_count; + RETURN_IF_ERROR(accumulate_frame_position_work( + chunk.prx, &(*sources)[plan_index].logical_position_work)); + } + } + return Status::OK(); +} + +Status append_prx_doc_ordinal(size_t ordinal, std::vector* out) { + if (ordinal > std::numeric_limits::max()) { + return Status::Error( + "phrase_query: prx doc ordinal exceeds u32"); + } + out->push_back(static_cast(ordinal)); + return Status::OK(); +} + +Status append_selected_ordinal(size_t doc_index, const std::vector& prx_doc_ordinals, + std::vector* selected_ordinals) { + if (!prx_doc_ordinals.empty()) { + selected_ordinals->push_back(prx_doc_ordinals[doc_index]); + return Status::OK(); + } + return append_prx_doc_ordinal(doc_index, selected_ordinals); +} + +Status append_selected_doc(size_t doc_index, uint32_t docid, + const std::vector& prx_doc_ordinals, + std::vector* selected_docids, + std::vector* selected_ordinals) { + selected_docids->push_back(docid); + return append_selected_ordinal(doc_index, prx_doc_ordinals, selected_ordinals); +} + +Status materialize_selected_prefix(size_t count, size_t capacity, + const std::vector& docids, + const std::vector& prx_doc_ordinals, + std::vector* selected_docids, + std::vector* selected_ordinals) { + selected_docids->reserve(capacity); + selected_ordinals->reserve(capacity); + selected_docids->insert(selected_docids->end(), docids.begin(), docids.begin() + count); + for (size_t i = 0; i < count; ++i) { + RETURN_IF_ERROR(append_selected_ordinal(i, prx_doc_ordinals, selected_ordinals)); + } + return Status::OK(); +} + +Status materialize_selected_prefix_if_needed(bool* selected_all, size_t count, size_t capacity, + const std::vector& docids, + const std::vector& prx_doc_ordinals, + std::vector* selected_docids, + std::vector* selected_ordinals) { + if (!*selected_all) { + return Status::OK(); + } + *selected_all = false; + return materialize_selected_prefix(count, capacity, docids, prx_doc_ordinals, selected_docids, + selected_ordinals); +} + +Status select_candidate_docs_for_prx(std::vector* docids, + std::vector* prx_doc_ordinals, + uint32_t prx_doc_count, + const std::vector& candidates, PosChunk* chunk) { + chunk->docids.clear(); + chunk->prx_doc_ordinals.clear(); + if (prx_doc_count == 0 && docids->size() > std::numeric_limits::max()) { + return Status::Error( + "phrase_query: prx doc count exceeds u32"); + } + chunk->prx_doc_count = + prx_doc_count == 0 ? static_cast(docids->size()) : prx_doc_count; + if (docids->empty() || candidates.empty()) { + return Status::OK(); + } + if (!prx_doc_ordinals->empty() && prx_doc_ordinals->size() != docids->size()) { + return Status::Error( + "phrase_query: prx ordinal/docid count mismatch"); + } + + std::vector selected_docids; + std::vector selected_ordinals; + bool selected_all = true; + const size_t selected_capacity = std::min(docids->size(), candidates.size()); + + auto candidate_it = std::ranges::lower_bound(candidates, docids->front()); + size_t candidate_index = static_cast(candidate_it - candidates.begin()); + for (size_t doc_index = 0; doc_index < docids->size(); ++doc_index) { + const uint32_t docid = (*docids)[doc_index]; + while (candidate_index < candidates.size() && candidates[candidate_index] < docid) { + ++candidate_index; + } + if (candidate_index == candidates.size()) { + RETURN_IF_ERROR(materialize_selected_prefix_if_needed( + &selected_all, doc_index, selected_capacity, *docids, *prx_doc_ordinals, + &selected_docids, &selected_ordinals)); + break; + } + if (candidates[candidate_index] != docid) { + RETURN_IF_ERROR(materialize_selected_prefix_if_needed( + &selected_all, doc_index, selected_capacity, *docids, *prx_doc_ordinals, + &selected_docids, &selected_ordinals)); + continue; + } + + if (!selected_all) { + RETURN_IF_ERROR(append_selected_doc(doc_index, docid, *prx_doc_ordinals, + &selected_docids, &selected_ordinals)); + } + ++candidate_index; + } + + if (selected_all) { + chunk->docids = std::move(*docids); + chunk->prx_doc_ordinals = std::move(*prx_doc_ordinals); + docids->clear(); + prx_doc_ordinals->clear(); + return Status::OK(); + } + if (selected_docids.empty()) { + return Status::OK(); + } + chunk->docids = std::move(selected_docids); + chunk->prx_doc_ordinals = std::move(selected_ordinals); + return Status::OK(); +} + +// PRX byte ranges for every candidate-bearing chunk across all phrase terms are +// added to one shared BatchRangeFetcher and fetched in a single batched round +// (T02). Pass 1 records, for each chunk that needs on-disk PRX bytes, where to +// write the fetched slice back: which plan's PosSource, which chunk within it, +// and the fetcher handle. + +struct PrxRangeAssignment { + size_t plan_index; + size_t chunk_index; + size_t handle; +}; + +void record_prx_assignment(std::vector* assignments, size_t plan_index, + size_t chunk_index, size_t handle) { + assignments->push_back(PrxRangeAssignment { + .plan_index = plan_index, .chunk_index = chunk_index, .handle = handle}); +} + +Status build_flat_position_source(const LogicalIndexReader& idx, + const io::BatchRangeFetcher& round1, DocidSource* doc_source, + const TermPlan& p, const std::vector& candidates, + size_t plan_index, io::BatchRangeFetcher* prx_fetcher, + std::vector* assignments, PosSource* src) { + PosChunk chunk; + std::vector docids; + std::vector prx_doc_ordinals; + const bool docids_are_final_candidates = + doc_source->docids_are_final_candidates && !doc_source->chunks.empty(); + if (!doc_source->chunks.empty()) { + DocidChunk& doc_chunk = doc_source->chunks.front(); + docids = std::move(doc_chunk.docids); + prx_doc_ordinals = std::move(doc_chunk.prx_doc_ordinals); + chunk.prx_doc_count = doc_chunk.prx_doc_count; + } + // pod_ref PRX bytes are read from the shared fetcher (one batched round for the + // whole phrase); inline PRX bytes already live in the dict entry. The pod_ref + // range is added unconditionally to keep the bytes read identical to the prior + // per-term fetch(); the handle is only recorded as an assignment when the chunk + // is kept (an empty chunk reads the same bytes but needs no backfill). + bool has_prx_handle = false; + size_t prx_handle = 0; + if (p.pod_ref) { + uint64_t poff = 0; + uint64_t plen = 0; + RETURN_IF_ERROR(idx.resolve_prx_window(p.entry, p.prx_base, &poff, &plen)); + prx_handle = prx_fetcher->add(poff, plen); + has_prx_handle = true; + } else { + chunk.prx = Slice(p.entry.prx_bytes); + } + if (docids.empty()) { + Slice dd; + if (p.pod_ref) { + dd = round1.get(p.frq_handle); + } else { + RETURN_IF_ERROR(internal::inline_dd_region(p.entry, &dd)); + } + RETURN_IF_ERROR(format::decode_dd_region(dd, p.entry.dd_meta, + /*win_base=*/0, &docids)); + if (docids.size() > std::numeric_limits::max()) { + return Status::Error( + "phrase_query: prx doc count exceeds u32"); + } + chunk.prx_doc_count = static_cast(docids.size()); + } + if (docids_are_final_candidates) { + chunk.docids = std::move(docids); + chunk.prx_doc_ordinals = std::move(prx_doc_ordinals); + if (!chunk.docids.empty()) { + if (has_prx_handle) { + record_prx_assignment(assignments, plan_index, src->chunks.size(), prx_handle); + } + src->chunks.push_back(std::move(chunk)); + } + return Status::OK(); + } + RETURN_IF_ERROR(select_candidate_docs_for_prx(&docids, &prx_doc_ordinals, chunk.prx_doc_count, + candidates, &chunk)); + if (!chunk.docids.empty()) { + if (has_prx_handle) { + record_prx_assignment(assignments, plan_index, src->chunks.size(), prx_handle); + } + src->chunks.push_back(std::move(chunk)); + } + return Status::OK(); +} + +bool chunk_may_contain_candidate(const DocidChunk& chunk, const std::vector& candidates) { + if (chunk.docids.empty() || candidates.empty()) { + return false; + } + const auto it = std::ranges::lower_bound(candidates, chunk.docids.front()); + return it != candidates.end() && *it <= chunk.docids.back(); +} + +Status decode_windowed_position_source(const LogicalIndexReader& idx, const TermPlan& p, + DocidSource* doc_source, + const std::vector& candidates, size_t plan_index, + io::BatchRangeFetcher* prx_fetcher, + std::vector* assignments, + PosSource* src) { + for (size_t i = 0; i < doc_source->chunks.size(); ++i) { + DocidChunk& doc_chunk = doc_source->chunks[i]; + if (!doc_source->docids_are_final_candidates && + !chunk_may_contain_candidate(doc_chunk, candidates)) { + continue; + } + if (!doc_chunk.windowed) { + return Status::Error( + "phrase_query: expected windowed doc chunk"); + } + PosChunk chunk; + if (doc_source->docids_are_final_candidates) { + chunk.docids = std::move(doc_chunk.docids); + chunk.prx_doc_ordinals = std::move(doc_chunk.prx_doc_ordinals); + chunk.prx_doc_count = doc_chunk.prx_doc_count; + } else { + RETURN_IF_ERROR( + select_candidate_docs_for_prx(&doc_chunk.docids, &doc_chunk.prx_doc_ordinals, + doc_chunk.prx_doc_count, candidates, &chunk)); + } + if (chunk.docids.empty()) { + continue; + } + + reader::WindowAbsRange range; + RETURN_IF_ERROR(reader::windowed_window_range( + idx, p.entry, p.frq_base, p.prx_base, p.prelude, doc_chunk.window, + /*want_positions=*/true, /*want_freq=*/false, &range)); + chunk.windowed = true; + chunk.window = doc_chunk.window; + const size_t prx_handle = prx_fetcher->add(range.prx_off, range.prx_len); + record_prx_assignment(assignments, plan_index, src->chunks.size(), prx_handle); + src->chunks.push_back(std::move(chunk)); + } + return Status::OK(); +} + +} // namespace +Status build_position_sources_for_candidates( + const LogicalIndexReader& idx, const io::BatchRangeFetcher& round1, + const std::vector& plans, std::vector* doc_sources, + const std::vector& candidates, + std::vector>* owners, std::vector* srcs, + format::PrxDecodeContext* observer_context) { + srcs->assign(plans.size(), PosSource {}); + for (PosSource& source : *srcs) { + source.observer_context = observer_context; + } + // All phrase terms share one PRX fetcher: pass 1 adds every candidate-bearing + // chunk's PRX range and records a backfill assignment; a single fetch() then + // issues one batched read (one serial round on a remote reader); pass 2 fills + // in each chunk's PRX slice. This collapses the prior per-term fetch() -- O(n) + // serial remote rounds for an n-term phrase -- into one. + auto prx_fetcher = + std::make_unique(idx.reader(), reader::kSameTermCoalesceGap); + std::vector assignments; + for (size_t i = 0; i < plans.size(); ++i) { + const TermPlan& p = plans[i]; + if (p.windowed) { + RETURN_IF_ERROR(decode_windowed_position_source(idx, p, &(*doc_sources)[i], candidates, + i, prx_fetcher.get(), &assignments, + &(*srcs)[i])); + continue; + } + RETURN_IF_ERROR(build_flat_position_source(idx, round1, &(*doc_sources)[i], p, candidates, + i, prx_fetcher.get(), &assignments, + &(*srcs)[i])); + } + if (prx_fetcher->pending() > 0) { + if (observer_context == nullptr || observer_context->stats == nullptr) { + RETURN_IF_ERROR(prx_fetcher->fetch()); + } else { + const auto fetch_start = std::chrono::steady_clock::now(); + RETURN_IF_ERROR(prx_fetcher->fetch()); + const auto elapsed = std::chrono::duration_cast( + std::chrono::steady_clock::now() - fetch_start) + .count(); + observer_context->stats->fetch_ns += + std::max(1, static_cast(elapsed)); + } + } + for (const PrxRangeAssignment& a : assignments) { + (*srcs)[a.plan_index].chunks[a.chunk_index].prx = prx_fetcher->get(a.handle); + } + RETURN_IF_ERROR(populate_logical_position_work(plans, srcs)); + // Keep the fetcher alive only when some chunk slice references its buffers. + if (!assignments.empty()) { + owners->push_back(std::move(prx_fetcher)); + } + return Status::OK(); +} + +} // namespace doris::snii::query::phrase_impl diff --git a/be/src/storage/index/snii/query/phrase_prefix_exec.cpp b/be/src/storage/index/snii/query/phrase_prefix_exec.cpp new file mode 100644 index 00000000000000..774f1fdc3c7a24 --- /dev/null +++ b/be/src/storage/index/snii/query/phrase_prefix_exec.cpp @@ -0,0 +1,720 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/check.h" +#include "storage/index/inverted/common_grams/common_grams_key_codec.h" +#include "storage/index/inverted/common_grams/common_grams_query_cost.h" +#include "storage/index/inverted/common_grams/common_grams_segment_metadata.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/encoding/byte_source.h" +#include "storage/index/snii/format/dict_entry.h" +#include "storage/index/snii/format/frq_pod.h" +#include "storage/index/snii/format/frq_prelude.h" +#include "storage/index/snii/format/prx_pod.h" +#include "storage/index/snii/io/batch_range_fetcher.h" +#include "storage/index/snii/query/internal/docid_conjunction.h" +#include "storage/index/snii/query/internal/docid_posting_reader.h" +#include "storage/index/snii/query/internal/docid_set_ops.h" +#include "storage/index/snii/query/internal/docid_union.h" +#include "storage/index/snii/query/internal/phrase_query_split.h" +#include "storage/index/snii/query/internal/plain_term_routing.h" +#include "storage/index/snii/query/internal/position_math.h" +#include "storage/index/snii/query/internal/query_test_counters.h" +#include "storage/index/snii/query/internal/resolved_phrase_plan.h" +#include "storage/index/snii/query/internal/term_expansion.h" +#include "storage/index/snii/query/phrase_prx_validation.h" +#include "storage/index/snii/query/phrase_query.h" +#include "storage/index/snii/query/phrase_verify_timer.h" +#include "storage/index/snii/query/prefix_query.h" +#include "storage/index/snii/query/term_query.h" +#include "storage/index/snii/reader/windowed_posting.h" +#include "util/debug_points.h" + +namespace doris::snii::query::phrase_impl { + +using query::internal::DocidChunk; +using query::internal::DocidSource; +using query::internal::ResolvedQueryTerm; +using query::internal::TermPlan; +using reader::LogicalIndexReader; +using internal::PhraseVerifyTimer; + +namespace { +Status collect_expected_tail_positions(const std::vector& plans, + const std::vector& phrase_plan_index, + const std::vector& position_offsets, + std::vector& srcs, + const std::vector& candidates, + ExpectedTailPositionSet* out, + bool preserve_first_clause_multiplicity) { + const size_t n = phrase_plan_index.size(); + DCHECK(n > 1); + DCHECK_EQ(plans.size(), srcs.size()); + std::vector cur(plans.size()); + for (size_t i = 0; i < plans.size(); ++i) { + cur[i].init(&srcs[i]); + } + + std::vector> unique_span(plans.size()); + std::vector> span(n); + for (uint32_t d : candidates) { + for (size_t i = 0; i < plans.size(); ++i) { + RETURN_IF_ERROR(cur[i].seek(d)); + RETURN_IF_ERROR(cur[i].positions(&unique_span[plans[i].order])); + } + for (size_t i = 0; i < n; ++i) { + DCHECK_LT(phrase_plan_index[i], unique_span.size()); + span[i] = unique_span[phrase_plan_index[i]]; + } + + // Anchor the outer enumeration on the SPARSEST exact term (smallest + // per-doc position span), not the hardcoded phrase-position-0 term. The + // set of valid phrase starts is anchor-independent -- each valid start + // maps 1:1 to exactly one anchor position (anchor_pos = start + + // offset[anchor]) -- so enumerating the shortest span and binary-searching + // the others yields the identical result set with the fewest outer + // iterations. A leading high-frequency exact term no longer forces + // O(|span[0]|) work per candidate doc. + size_t anchor = 0; + auto best = position_span_size(span[0]); + if (!preserve_first_clause_multiplicity) { + for (size_t t = 1; t < n; ++t) { + const auto sz = position_span_size(span[t]); + if (sz < best) { + best = sz; + anchor = t; + } + } + } + const uint32_t anchor_off = position_offsets[anchor]; + SNII_QUERY_ADD(anchor_iterations, best); + + // Only the first non-anchor term is probed for every viable anchor. + // Later terms can be skipped after an earlier mismatch, so using the + // total anchor count to choose a forward scan for them could turn one + // binary lookup into a full-span walk. The first term uses a forward + // scan only when all anchors are valid and its conservative comparison + // upper bound is at most half that of repeated binary search. + const size_t first_checked_term = anchor == 0 ? 1 : 0; + size_t monotonic_position_scan_term = n; + if (should_use_monotonic_position_scan(span[anchor], + position_span_size(span[first_checked_term]), + anchor_off, position_offsets[first_checked_term])) { + monotonic_position_scan_term = first_checked_term; + SNII_QUERY_COUNT(monotonic_position_scans); + } + + const size_t expected_begin = out->positions.size(); + for (const uint32_t* p = span[anchor].first; p != span[anchor].second; ++p) { + const uint32_t anchor_pos = *p; + // Underflow guard: a general anchor (offset > 0) can sit at a position + // smaller than its offset, which would wrap `start`. Such a position + // admits no valid phrase start and is skipped. (The old span[0] anchor + // had offset 0 and could never underflow.) + if (anchor_pos < anchor_off) { + continue; + } + const uint32_t start = anchor_pos - anchor_off; + bool ok = true; + for (size_t t = 0; t < n; ++t) { + if (t == anchor) { + continue; // the anchor term's position is satisfied by construction + } + uint32_t want = 0; + if (!internal::add_position_offset(start, position_offsets[t], &want)) { + ok = false; + break; + } + if (t == monotonic_position_scan_term) { + while (span[t].first != span[t].second && *span[t].first < want) { + ++span[t].first; + } + if (span[t].first == span[t].second || *span[t].first != want) { + ok = false; + break; + } + } else if (!std::binary_search(span[t].first, span[t].second, want)) { + ok = false; + break; + } + } + uint32_t tail_pos = 0; + if (ok && internal::add_position_offset(start, position_offsets[n], &tail_pos)) { + out->positions.push_back(tail_pos); + } + } + const size_t expected_end = out->positions.size(); + if (expected_end != expected_begin) { + out->docs.push_back(ExpectedTailPositions { + .docid = d, .positions_begin = expected_begin, .positions_end = expected_end}); + } + } + return Status::OK(); +} + +Status collect_single_term_expected_tail_positions(std::vector& srcs, + const std::vector& candidates, + uint32_t tail_offset, + ExpectedTailPositionSet* out) { + PostingCursor cursor; + cursor.init(srcs.data()); + out->reserve_docs(out->docs.size() + candidates.size()); + + for (uint32_t d : candidates) { + RETURN_IF_ERROR(cursor.seek(d)); + std::pair span; + RETURN_IF_ERROR(cursor.positions(&span)); + + const size_t expected_begin = out->positions.size(); + for (const uint32_t* p = span.first; p != span.second; ++p) { + uint32_t tail_pos = 0; + if (internal::add_position_offset(*p, tail_offset, &tail_pos)) { + out->positions.push_back(tail_pos); + } + } + const size_t expected_end = out->positions.size(); + if (expected_end != expected_begin) { + out->docs.push_back(ExpectedTailPositions { + .docid = d, .positions_begin = expected_begin, .positions_end = expected_end}); + } + } + return Status::OK(); +} + +Status collect_expected_tail_positions(const LogicalIndexReader& idx, + internal::ResolvedPhrasePlan exact_plan, + uint32_t tail_position_offset, ExpectedTailPositionSet* out, + const std::vector* candidate_prefilter, + format::PrxDecodeContext* observer_context, + bool preserve_first_clause_multiplicity) { + out->clear(); + io::BatchRangeFetcher round1(idx.reader()); + std::vector plans; + RETURN_IF_ERROR(internal::plan_resolved_terms(idx, std::move(exact_plan.unique_terms), &round1, + &plans, + /*need_positions=*/false)); + + PhraseExecutionState state; + RETURN_IF_ERROR(build_phrase_execution_state(idx, &round1, &plans, &state, candidate_prefilter, + observer_context, + PhraseCandidateMetric::kPrefixLeading)); + if (state.candidates.empty()) { + return Status::OK(); + } + out->reserve_docs(state.candidates.size()); + DORIS_CHECK(!exact_plan.position_offsets.empty()); + DORIS_CHECK_GT(tail_position_offset, exact_plan.position_offsets.back()); + std::vector position_offsets = std::move(exact_plan.position_offsets); + position_offsets.push_back(tail_position_offset); + PhraseVerifyTimer verify_timer(observer_context); + if (exact_plan.phrase_plan_index.size() == 1) { + DORIS_CHECK_LT(position_offsets[0], position_offsets[1]); + RETURN_IF_ERROR(collect_single_term_expected_tail_positions( + state.srcs, state.candidates, position_offsets[1] - position_offsets[0], out)); + } else { + RETURN_IF_ERROR(collect_expected_tail_positions( + plans, exact_plan.phrase_plan_index, position_offsets, state.srcs, state.candidates, + out, preserve_first_clause_multiplicity)); + } + verify_timer.commit_success(); + return Status::OK(); +} + +bool contains_any_position(const ExpectedTailPositionSet& expected, + const ExpectedTailPositions& wanted, + std::pair actual) { + for (size_t i = wanted.positions_begin; i < wanted.positions_end; ++i) { + if (std::binary_search(actual.first, actual.second, expected.positions[i])) { + return true; + } + } + return false; +} + +uint32_t mark_matching_positions(ExpectedTailPositionSet* expected, + const ExpectedTailPositions& wanted, + std::pair actual) { + DCHECK_EQ(expected->position_matched.size(), expected->positions.size()); + size_t expected_index = wanted.positions_begin; + const uint32_t* actual_position = actual.first; + uint32_t added = 0; + while (expected_index < wanted.positions_end && actual_position != actual.second) { + const uint32_t expected_position = expected->positions[expected_index]; + if (expected_position < *actual_position) { + ++expected_index; + continue; + } + if (*actual_position < expected_position) { + ++actual_position; + continue; + } + const size_t expected_run_end = static_cast( + std::upper_bound(expected->positions.begin() + expected_index, + expected->positions.begin() + wanted.positions_end, + expected_position) - + expected->positions.begin()); + while (expected_index < expected_run_end) { + if (expected->position_matched[expected_index] == 0) { + expected->position_matched[expected_index] = 1; + DCHECK_NE(added, std::numeric_limits::max()); + ++added; + } + ++expected_index; + } + actual_position = std::upper_bound(actual_position, actual.second, expected_position); + } + return added; +} + +// Upper bound on prefix expansions whose position cursors are held resident at +// once. The old per-tail loop verified a single expansion at a time (one +// PosSource + one PRX buffer live); the merged sweep below holds up to this many +// tail PosSources + cursors + PRX buffers simultaneously so it can read every +// tail's docid/prx bytes in ONE batched round and verify them in a single +// forward pass. `max_expansions` may be unbounded (<= 0), so this hard cap keeps +// resident memory bounded independent of the query: expansions beyond the cap +// are processed as additional capped groups (each a fresh single fetch) whose +// matched-doc flags are accumulated. The cap is tightened because each cursor +// here also holds decoded PRX rather than plain docids. +constexpr size_t kMaxTailMergeBatch = 32; + +// Phrase-prefix only reads the residual tails' docid union to prefilter the +// leading-phrase candidate set when the smallest leading term's df reaches this +// -- i.e. when the leading candidate set is large enough that decoding all its +// positions dwarfs an extra docid-only union read. Scale the threshold with the +// segment so a fixed absolute gate does not disable the prefilter after rowset +// segmentation, while retaining the original 1<<16 cap for large segments. +constexpr uint32_t kMinPrefixLeadingPrefilterMinDf = 256; +constexpr uint32_t kMaxPrefixLeadingPrefilterMinDf = 1u << 16; +constexpr uint32_t kPrefixLeadingPrefilterDocFraction = 8; +// The tail union is decoded once for filtering and again for verification. +// Require enough leading PRX work to cover both reads and union overhead. +constexpr uint32_t kPrefixLeadingToTailDfRatio = 8; + +uint32_t prefix_leading_prefilter_min_df(const LogicalIndexReader& idx, + bool allow_segment_relative_gate) { + if (!allow_segment_relative_gate) { + return kMaxPrefixLeadingPrefilterMinDf; + } + const uint64_t segment_relative = + idx.stats().indexed_doc_count / kPrefixLeadingPrefilterDocFraction; + return static_cast(std::clamp( + segment_relative, kMinPrefixLeadingPrefilterMinDf, kMaxPrefixLeadingPrefilterMinDf)); +} + +// Merged multi-tail verification for ONE resident-capped group of prefix +// expansions (`tails`, already truncated by max_expansions upstream). This +// replaces the per-tail verify-then-union loop: instead of re-planning + TWO +// remote rounds (docid, then prx) + a separate doc-walk PER tail and unioning N +// result lists, it plans every tail into ONE shared round1 fetch, intersects +// each tail with `expected_docids` in memory (no I/O), builds every surviving +// tail's position source in ONE batched PRX round, then sweeps the group's tail +// cursors over the ascending `expected` docs a SINGLE time -- marking a doc as +// soon as ANY tail has a position adjacent to a leading match. +// +// The marked set is byte-identical to the per-tail path's +// UNION_{tail in group} { d : d in tail INTERSECT expected AND +// contains_any_position(expected, doc_d, pos_tail(d)) } +// because each tail's PosSource is still built from its OWN final-candidate +// docids (the shared-candidate argument is ignored for final-candidate sources), +// so pos_tail(d) and the per-doc position test are unchanged. Only the I/O rounds +// (2N -> 2) and the N separate unions (-> in-place flags) collapse. Bigram +// postings are NEVER consulted: every tail is verified against its unigram +// positions here. + +Status collect_merged_tail_matches(const LogicalIndexReader& idx, + std::vector tails, + ExpectedTailPositionSet* expected, + const std::vector& expected_docids, bool final_group, + std::vector* final_matches, + format::PrxDecodeContext* observer_context, + std::vector* frequency_matches) { + DCHECK(expected != nullptr); + DCHECK(final_matches != nullptr || frequency_matches != nullptr); + const size_t n = tails.size(); + if (n == 0 || expected->docs.empty()) { + return Status::OK(); + } + + // Plan every tail into ONE fetcher so their docid postings + windowed + // preludes are read in a single batched round (the per-tail path issued one + // round per tail). Each tail keeps its own single-term plan vector so the + // conjunction filter below consumes it directly, without slicing a shared + // plan vector (whose prelude readers own decoded directory buffers). + io::BatchRangeFetcher round1(idx.reader()); + std::vector> tail_plans(n); + for (size_t i = 0; i < n; ++i) { + std::vector one; + one.push_back(std::move(tails[i])); + RETURN_IF_ERROR(internal::plan_resolved_terms(idx, std::move(one), &round1, &tail_plans[i], + /*need_positions=*/false)); + } + if (round1.pending() > 0) { + RETURN_IF_ERROR(round1.fetch()); + } + for (size_t i = 0; i < n; ++i) { + RETURN_IF_ERROR(internal::open_preludes(round1, &tail_plans[i], + /*need_positions=*/true)); + } + + // Per-tail candidate docids (tail posting INTERSECT expected) and the aligned + // final-candidate doc sources feeding the batched position builder. The + // conjunction reads only already-fetched round1 bytes; a single-plan filter + // marks its one source docids_are_final_candidates, so the position builder + // materializes each tail's PosSource directly over its own candidate docs. + // Tails whose intersection is empty are dropped here (exactly as the old + // per-tail early return did), so no dead tail decodes its full posting. + std::vector> tail_candidates(n); + std::vector active_plans; + std::vector active_sources; + std::vector active_index; // active slot -> tail index (into tail_candidates) + for (size_t i = 0; i < n; ++i) { + std::vector tail_source; + RETURN_IF_ERROR(internal::filter_docids_by_conjunction( + idx, round1, tail_plans[i], expected_docids, &tail_candidates[i], &tail_source)); + if (tail_candidates[i].empty()) { + continue; // this expansion has no doc in the expected set: nothing to verify + } + active_plans.push_back(std::move(tail_plans[i].front())); + active_sources.push_back(tail_source.empty() ? DocidSource {} + : std::move(tail_source.front())); + active_index.push_back(i); + } + if (active_plans.empty() && (!final_group || expected->matched_count == 0)) { + return Status::OK(); + } + // An empty final group must still sweep expected->docs and emit matches that + // earlier resident groups recorded in-place. + + // ONE batched PRX round for every retained chunk across all surviving tails + // (vs one round per tail before). `candidates` is intentionally empty: every + // source is a final-candidate source, so the builder addresses positions by + // the source's own docids and never consults the shared candidate list. + std::vector> owners; + std::vector srcs; + const std::vector no_shared_candidates; + if (!active_plans.empty()) { + RETURN_IF_ERROR(build_position_sources_for_candidates(idx, round1, active_plans, + &active_sources, no_shared_candidates, + &owners, &srcs, observer_context)); + } + + // Single forward sweep over the ascending expected docs. For each doc probe + // only the tails that actually posted it (per-tail ascending cursor over + // tail_candidates), decode positions once, and emit the doc the instant one + // tail's positions land adjacent to a leading match. Cursors advance strictly + // forward because expected.docs is strictly ascending and each cursor is + // sought at most once per doc. + std::vector cursors(active_plans.size()); + for (size_t a = 0; a < active_plans.size(); ++a) { + cursors[a].init(&srcs[a]); + } + std::vector tail_pos(active_plans.size(), 0); + PhraseVerifyTimer verify_timer(observer_context); + DCHECK_EQ(expected->docs.size(), expected_docids.size()); + for (size_t expected_index = 0; expected_index < expected->docs.size(); ++expected_index) { + ExpectedTailPositions& doc = expected->docs[expected_index]; + DCHECK_EQ(doc.docid, expected_docids[expected_index]); + SNII_QUERY_COUNT(prefix_expected_doc_visits); + if (observer_context != nullptr && observer_context->query_stats != nullptr) { + ++observer_context->query_stats->prefix_tail_candidate_visits; + } + const uint32_t d = doc.docid; + if (frequency_matches != nullptr || doc.phrase_frequency == 0) { + bool matched = false; + uint32_t added_frequency = 0; + for (size_t a = 0; + a < active_plans.size() && (frequency_matches != nullptr || !matched); ++a) { + std::vector& cand = tail_candidates[active_index[a]]; + size_t& ti = tail_pos[a]; + while (ti < cand.size() && cand[ti] < d) { + ++ti; + } + if (ti >= cand.size() || cand[ti] != d) { + continue; // this expansion has no posting at d + } + RETURN_IF_ERROR(cursors[a].seek(d)); + std::pair actual; + RETURN_IF_ERROR(cursors[a].positions(&actual)); + if (frequency_matches == nullptr && contains_any_position(*expected, doc, actual)) { + matched = true; + } else if (frequency_matches != nullptr) { + const uint32_t added = mark_matching_positions(expected, doc, actual); + DCHECK_LE(added_frequency, std::numeric_limits::max() - added); + added_frequency += added; + } + } + if (matched || added_frequency != 0) { + if (doc.phrase_frequency == 0) { + ++expected->matched_count; + } + if (frequency_matches == nullptr) { + doc.phrase_frequency = 1; + } else { + DCHECK_LE(doc.phrase_frequency, + std::numeric_limits::max() - added_frequency); + doc.phrase_frequency += added_frequency; + } + } + } + if (final_group && doc.phrase_frequency != 0) { + if (final_matches != nullptr) { + final_matches->push_back(d); + } + if (frequency_matches != nullptr) { + frequency_matches->push_back(PhraseMatch { + .docid = d, .frequency = static_cast(doc.phrase_frequency)}); + } + } + } + verify_timer.commit_success(); + return Status::OK(); +} + +} // namespace +Status execute_resolved_phrase_prefix_terms( + const LogicalIndexReader& idx, internal::ResolvedPhrasePlan exact_plan, + std::vector tail_terms, uint32_t tail_position_offset, + std::vector* docids, format::PrxDecodeContext* decode_context, + std::vector* matches, const std::vector* candidate_prefilter) { + DORIS_CHECK(docids != nullptr || matches != nullptr); + if (tail_terms.empty()) { + return Status::OK(); + } + if (exact_plan.phrase_plan_index.empty()) { + DORIS_CHECK(matches == nullptr); + DORIS_CHECK_EQ(tail_position_offset, 0U); + if (tail_terms.size() == 1) { + const auto& tail = tail_terms.front(); + RETURN_IF_ERROR(internal::read_docid_posting(idx, tail.entry, tail.frq_base, + tail.prx_base, docids)); + if (candidate_prefilter != nullptr) { + *docids = internal::intersect_sorted(*docids, *candidate_prefilter); + } + return Status::OK(); + } + std::vector tail_postings; + tail_postings.reserve(tail_terms.size()); + for (const auto& tail : tail_terms) { + tail_postings.push_back({tail.entry, tail.frq_base, tail.prx_base}); + } + RETURN_IF_ERROR(internal::build_docid_union(idx, tail_postings, docids)); + if (candidate_prefilter != nullptr) { + *docids = internal::intersect_sorted(*docids, *candidate_prefilter); + } + return Status::OK(); + } + if (!idx.has_positions()) { + return Status::Error( + "phrase_prefix_query: index has no positions"); + } + DORIS_CHECK(!exact_plan.position_offsets.empty()); + DORIS_CHECK_GT(tail_position_offset, exact_plan.position_offsets.back()); + if (tail_terms.size() == 1) { + append_resolved_phrase_clause(std::move(tail_terms.front()), tail_position_offset, + &exact_plan); + return internal::execute_resolved_phrase_plan( + idx, std::move(exact_plan), docids, decode_context, matches, candidate_prefilter, + internal::ExactPhrasePositionAccess::kMaterializedOnly); + } + + uint32_t min_lead_df = std::numeric_limits::max(); + for (const ResolvedQueryTerm& term : exact_plan.unique_terms) { + min_lead_df = std::min(min_lead_df, term.entry.df); + } + uint64_t tail_df_sum = 0; + for (const ResolvedQueryTerm& tail : tail_terms) { + tail_df_sum += tail.entry.df; + } + const std::vector* prefilter = candidate_prefilter; + std::vector tail_union; + std::vector combined_prefilter; + const bool allow_segment_relative_prefilter = + candidate_prefilter == nullptr && exact_plan.phrase_plan_index.size() == 1; + const uint32_t leading_prefilter_min_df = + prefix_leading_prefilter_min_df(idx, allow_segment_relative_prefilter); + if (min_lead_df >= leading_prefilter_min_df && + tail_df_sum <= static_cast(min_lead_df) / kPrefixLeadingToTailDfRatio) { + std::vector tail_postings; + tail_postings.reserve(tail_terms.size()); + for (const ResolvedQueryTerm& tail : tail_terms) { + tail_postings.push_back({tail.entry, tail.frq_base, tail.prx_base}); + } + RETURN_IF_ERROR(internal::build_docid_union(idx, tail_postings, &tail_union)); + if (tail_union.empty()) { + return Status::OK(); + } + if (candidate_prefilter == nullptr) { + prefilter = &tail_union; + } else { + combined_prefilter = internal::intersect_sorted(*candidate_prefilter, tail_union); + if (combined_prefilter.empty()) { + return Status::OK(); + } + prefilter = &combined_prefilter; + } + } + + ExpectedTailPositionSet expected; + RETURN_IF_ERROR(collect_expected_tail_positions(idx, std::move(exact_plan), + tail_position_offset, &expected, prefilter, + decode_context, matches != nullptr)); + if (expected.docs.empty()) { + return Status::OK(); + } + if (matches != nullptr) { + expected.position_matched.assign(expected.positions.size(), 0); + } + + std::vector expected_docids; + expected_docids.reserve(expected.docs.size()); + for (const ExpectedTailPositions& doc : expected.docs) { + expected_docids.push_back(doc.docid); + } + SNII_QUERY_COUNT(expected_docids_build); + + std::vector final_matches; + // Keep the expected-doc set stable across groups. Compacting matched docs + // speculates that later tail groups intersect it; an empty intersection + // skips the sweep entirely, so the compaction work cannot be repaid safely. + for (size_t start = 0; start < tail_terms.size(); start += kMaxTailMergeBatch) { + const size_t end = std::min(start + kMaxTailMergeBatch, tail_terms.size()); + const bool final_group = end == tail_terms.size(); + std::vector group; + group.reserve(end - start); + for (size_t i = start; i < end; ++i) { + group.push_back(std::move(tail_terms[i])); + } + RETURN_IF_ERROR(collect_merged_tail_matches( + idx, std::move(group), &expected, expected_docids, final_group, + docids == nullptr ? nullptr : &final_matches, decode_context, matches)); + if (matches == nullptr && !final_group && expected.matched_count == expected.docs.size()) { + final_matches.reserve(expected.docs.size()); + for (const ExpectedTailPositions& doc : expected.docs) { + DCHECK_NE(doc.phrase_frequency, 0); + final_matches.push_back(doc.docid); + } + break; + } + } + if (docids != nullptr) { + *docids = std::move(final_matches); + } + return Status::OK(); +} + +namespace { +template +std::vector copy_resolved_terms(const std::vector& resolved, + const std::vector& indices) { + std::vector result; + result.reserve(indices.size()); + for (size_t index : indices) { + DORIS_CHECK_LT(index, resolved.size()); + result.push_back(resolved[index]); + } + return result; +} + +} // namespace +Status execute_hybrid_phrase_prefix_plan( + const LogicalIndexReader& idx, const HybridPrefixPlanArtifact& artifact, + const std::vector& batch_terms, const std::vector& resolved, + const std::vector& plain_tail_terms, std::vector* docids, + format::PrxDecodeContext* decode_context, CommonGramsPlanningTimer& planning_timer, + bool* candidate_intersection_empty) { + DORIS_CHECK(idx.common_grams_posting_policy() == format::CommonGramsPostingPolicy::kHybridV1); + DORIS_CHECK(docids != nullptr); + DORIS_CHECK(candidate_intersection_empty != nullptr); + docids->clear(); + *candidate_intersection_empty = false; + + const HybridPositionedCover& plain_tail_cover = artifact.plain_tail_cover; + const uint32_t plain_tail_position_offset = artifact.plain_tail_position_offset; + HybridPrefixCandidateSet leading_candidates; + RETURN_IF_ERROR(build_hybrid_leading_candidates(idx, plain_tail_cover.candidate_prefilter, + batch_terms, resolved, &leading_candidates)); + if (leading_candidates.active && leading_candidates.docs.empty()) { + planning_timer.finish(); + *candidate_intersection_empty = true; + return Status::OK(); + } + + if (!artifact.maps_tail_to_gram) { + DORIS_CHECK(leading_candidates.active); + DORIS_CHECK(artifact.mapped_tail_split.positioned_indices.empty()); + DORIS_CHECK(artifact.mapped_tail_split.docs_only_indices.empty()); + planning_timer.finish(); + RETURN_IF_ERROR(execute_resolved_phrase_prefix_terms( + idx, + copy_resolved_phrase_plan(plain_tail_cover.verification, batch_terms, resolved), + plain_tail_terms, plain_tail_position_offset, docids, decode_context, nullptr, + &leading_candidates.docs)); + *candidate_intersection_empty = docids->empty(); + return Status::OK(); + } + + const HybridPrefixMappedTails& split = artifact.mapped_tail_split; + DORIS_CHECK(!split.positioned_indices.empty() || !split.docs_only_indices.empty()); + DORIS_CHECK(leading_candidates.active || !split.docs_only_indices.empty()); + std::vector docs_only_tail_candidates; + if (!split.docs_only_indices.empty()) { + RETURN_IF_ERROR(build_hybrid_docs_only_tail_candidates( + idx, resolved, split.docs_only_indices, leading_candidates, + &docs_only_tail_candidates)); + } + planning_timer.finish(); + + if (!split.positioned_indices.empty()) { + DORIS_CHECK(artifact.positioned_tail_verification.has_value()); + std::vector positioned_docs; + RETURN_IF_ERROR(execute_resolved_phrase_prefix_terms( + idx, + copy_resolved_phrase_plan(*artifact.positioned_tail_verification, batch_terms, + resolved), + copy_resolved_terms(resolved, split.positioned_indices), + plain_tail_position_offset - 1, &positioned_docs, decode_context, nullptr, + leading_candidates.active ? &leading_candidates.docs : nullptr)); + internal::union_sorted_into(docids, positioned_docs); + } + + if (!split.docs_only_indices.empty() && !docs_only_tail_candidates.empty()) { + std::vector docs_only_docs; + RETURN_IF_ERROR(execute_resolved_phrase_prefix_terms( + idx, + copy_resolved_phrase_plan(plain_tail_cover.verification, batch_terms, resolved), + copy_resolved_terms(plain_tail_terms, split.docs_only_ordinals), + plain_tail_position_offset, &docs_only_docs, decode_context, nullptr, + &docs_only_tail_candidates)); + internal::union_sorted_into(docids, docs_only_docs); + } + *candidate_intersection_empty = docids->empty(); + return Status::OK(); +} + +} // namespace doris::snii::query::phrase_impl diff --git a/be/src/storage/index/snii/query/phrase_prx_validation.h b/be/src/storage/index/snii/query/phrase_prx_validation.h new file mode 100644 index 00000000000000..a4082360affba7 --- /dev/null +++ b/be/src/storage/index/snii/query/phrase_prx_validation.h @@ -0,0 +1,55 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include + +#include "common/status.h" + +namespace doris::snii { +class ByteSource; +namespace format { +struct PrxDecodeContext; +} +} // namespace doris::snii + +namespace doris::snii::query::internal { + +// The production seam shared by PosChunkDecoder and focused shape-validation +// tests. A successful format decode commits its stats before caller-level CSR +// validation runs. +Status decode_and_validate_prx_frame(ByteSource* source, + std::span selected_doc_ordinals, + bool decode_full, bool all_docs_selected, + uint32_t expected_total_docs, size_t expected_selected_docs, + std::vector* pos_flat, + std::vector* pos_offsets, + format::PrxDecodeContext* decode_context); + +// Validates the CSR shape expected by phrase execution. Format-level decode +// statistics have already been committed when this function runs. +Status validate_prx_frame(std::span pos_flat, std::span pos_offsets, + uint32_t actual_total_docs, uint32_t expected_total_docs, + size_t expected_selected_docs, + std::span selected_doc_ordinals, + bool offsets_by_prx_ordinal, bool all_docs_selected); + +} // namespace doris::snii::query::internal diff --git a/be/src/storage/index/snii/query/phrase_query.cpp b/be/src/storage/index/snii/query/phrase_query.cpp new file mode 100644 index 00000000000000..e5fa8a6204ce48 --- /dev/null +++ b/be/src/storage/index/snii/query/phrase_query.cpp @@ -0,0 +1,326 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/query/phrase_query.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/check.h" +#include "storage/index/inverted/common_grams/common_grams_key_codec.h" +#include "storage/index/inverted/common_grams/common_grams_query_cost.h" +#include "storage/index/inverted/common_grams/common_grams_segment_metadata.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/encoding/byte_source.h" +#include "storage/index/snii/format/dict_entry.h" +#include "storage/index/snii/format/frq_pod.h" +#include "storage/index/snii/format/frq_prelude.h" +#include "storage/index/snii/format/prx_pod.h" +#include "storage/index/snii/io/batch_range_fetcher.h" +#include "storage/index/snii/query/internal/docid_conjunction.h" +#include "storage/index/snii/query/internal/docid_posting_reader.h" +#include "storage/index/snii/query/internal/docid_set_ops.h" +#include "storage/index/snii/query/internal/docid_union.h" +#include "storage/index/snii/query/internal/phrase_query_split.h" +#include "storage/index/snii/query/internal/plain_term_routing.h" +#include "storage/index/snii/query/internal/position_math.h" +#include "storage/index/snii/query/internal/query_test_counters.h" +#include "storage/index/snii/query/internal/resolved_phrase_plan.h" +#include "storage/index/snii/query/internal/term_expansion.h" +#include "storage/index/snii/query/phrase_prx_validation.h" +#include "storage/index/snii/query/phrase_verify_timer.h" +#include "storage/index/snii/query/prefix_query.h" +#include "storage/index/snii/query/term_query.h" +#include "storage/index/snii/reader/windowed_posting.h" +#include "util/debug_points.h" + +namespace doris::snii::query { + +using query::internal::DocidChunk; +using query::internal::DocidSource; +using query::internal::ResolvedQueryTerm; +using query::internal::TermPlan; +using reader::LogicalIndexReader; + +CommonGramsPlanDebugOverride common_grams_plan_debug_override() { + CommonGramsPlanDebugOverride result = CommonGramsPlanDebugOverride::kNone; + DBUG_EXECUTE_IF(COMMON_GRAMS_FORCE_PLAIN_PLAN_DEBUG_POINT, + { result = CommonGramsPlanDebugOverride::kForcePlain; }); + DBUG_EXECUTE_IF(COMMON_GRAMS_FORCE_GRAM_PLAN_DEBUG_POINT, { + DORIS_CHECK(result != CommonGramsPlanDebugOverride::kForcePlain); + result = CommonGramsPlanDebugOverride::kForceCommonGrams; + }); + return result; +} + +using namespace phrase_impl; // NOLINT(google-build-using-namespace): module-internal impl namespace + +namespace internal { + +Status validate_prx_frame(std::span pos_flat, std::span pos_offsets, + uint32_t actual_total_docs, uint32_t expected_total_docs, + size_t expected_selected_docs, + std::span selected_doc_ordinals, + bool offsets_by_prx_ordinal, bool all_docs_selected) { + if (!all_docs_selected && expected_selected_docs != selected_doc_ordinals.size()) { + return Status::Error( + "phrase_query: selected prx ordinal-count mismatch"); + } + if (actual_total_docs != expected_total_docs) { + return Status::Error( + "phrase_query: prx total doc-count mismatch"); + } + const size_t expected_offsets = offsets_by_prx_ordinal + ? static_cast(expected_total_docs) + 1 + : expected_selected_docs + 1; + if (pos_offsets.size() != expected_offsets) { + return Status::Error( + offsets_by_prx_ordinal ? "phrase_query: full prx doc-count mismatch" + : "phrase_query: selected prx/doc-count mismatch"); + } + if (pos_offsets.back() != pos_flat.size()) { + return Status::Error( + "phrase_query: prx final offset mismatch"); + } + if (offsets_by_prx_ordinal && !selected_doc_ordinals.empty() && + static_cast(selected_doc_ordinals.back()) + 1 >= pos_offsets.size()) { + return Status::Error( + "phrase_query: prx ordinal offset out of range"); + } + return Status::OK(); +} + +Status decode_and_validate_prx_frame(ByteSource* source, + std::span selected_doc_ordinals, + bool decode_full, bool all_docs_selected, + uint32_t expected_total_docs, size_t expected_selected_docs, + std::vector* pos_flat, + std::vector* pos_offsets, + format::PrxDecodeContext* decode_context) { + DCHECK(decode_full || !all_docs_selected); + format::PrxDecodedShape decoded_shape; + format::PrxDecodeContext frame_context { + .stats = decode_context == nullptr ? nullptr : decode_context->stats, + .shape = &decoded_shape}; + if (decode_full) { + if (all_docs_selected) { + RETURN_IF_ERROR( + format::read_prx_window_csr(source, pos_flat, pos_offsets, &frame_context)); + } else { + RETURN_IF_ERROR(format::read_prx_window_csr_for_selection( + source, selected_doc_ordinals, pos_flat, pos_offsets, &frame_context)); + } + } else { + RETURN_IF_ERROR(format::read_prx_window_csr_selective( + source, selected_doc_ordinals, pos_flat, pos_offsets, &frame_context)); + } + return validate_prx_frame(*pos_flat, *pos_offsets, decoded_shape.total_docs, + expected_total_docs, expected_selected_docs, selected_doc_ordinals, + decode_full && !all_docs_selected, all_docs_selected); +} + +namespace { + +using PhraseVerifyClock = std::chrono::steady_clock; + +PhraseVerifyClock::time_point phrase_verify_clock_now() { +#ifdef BE_TEST + testing::note_phrase_verify_clock_read(); +#endif + return PhraseVerifyClock::now(); +} + +} // namespace + +uint64_t exclusive_phrase_verify_ns(uint64_t elapsed_ns, uint64_t decode_ns_before, + uint64_t decode_ns_after) { + DCHECK_GE(decode_ns_after, decode_ns_before); + const uint64_t decode_delta = decode_ns_after - decode_ns_before; + return elapsed_ns > decode_delta ? elapsed_ns - decode_delta : 0; +} + +PhraseVerifyTimer::PhraseVerifyTimer(format::PrxDecodeContext* decode_context) + : stats_(decode_context == nullptr ? nullptr : decode_context->stats) { + if (stats_ != nullptr) { + decode_ns_before_ = stats_->decode_ns; + start_ = phrase_verify_clock_now(); + } +} + +void PhraseVerifyTimer::commit_success() { + if (stats_ == nullptr) { + return; + } + const auto elapsed = + std::chrono::duration_cast(phrase_verify_clock_now() - start_) + .count(); + stats_->phrase_verify_ns += exclusive_phrase_verify_ns(static_cast(elapsed), + decode_ns_before_, stats_->decode_ns); +} + +} // namespace internal + +Status phrase_query(const LogicalIndexReader& idx, const std::vector& terms, + std::vector* const docids) { + return phrase_query_impl(idx, terms, docids, nullptr, nullptr, {}); +} + +Status phrase_query(const LogicalIndexReader& idx, const std::vector& terms, + std::vector* const docids, QueryProfile* profile) { + return phrase_query(idx, terms, docids, profile, {}); +} + +Status phrase_query(const LogicalIndexReader& idx, const std::vector& terms, + std::vector* const docids, QueryProfile* profile, + const PhraseQueryOptions& options) { + QueryProfileScope profile_scope(idx.reader(), profile); + format::PrxDecodeContext decode_context { + .stats = profile == nullptr ? nullptr : &profile->prx_decode_stats, + .query_stats = profile == nullptr ? nullptr : &profile->phrase_query_stats}; + return phrase_query_impl(idx, terms, docids, profile == nullptr ? nullptr : &decode_context, + nullptr, options); +} + +Status phrase_query_with_frequencies(const LogicalIndexReader& idx, + const std::vector& terms, + std::vector* matches, QueryProfile* profile, + const PhraseQueryOptions& options) { + if (matches == nullptr) { + return Status::Error( + "phrase_query_with_frequencies: null out"); + } + matches->clear(); + if (terms.size() < 2) { + return Status::Error( + "phrase_query_with_frequencies: at least two terms are required"); + } + QueryProfileScope profile_scope(idx.reader(), profile); + format::PrxDecodeContext decode_context { + .stats = profile == nullptr ? nullptr : &profile->prx_decode_stats, + .query_stats = profile == nullptr ? nullptr : &profile->phrase_query_stats}; + return phrase_query_impl(idx, terms, nullptr, profile == nullptr ? nullptr : &decode_context, + matches, options); +} + +Status planned_exact_phrase_query( + const LogicalIndexReader& idx, const segment_v2::InvertedIndexQueryInfo& plain_query_info, + const segment_v2::InvertedIndexQueryInfo& gram_query_info, + const segment_v2::inverted_index::CommonGramsQueryIdentity* common_grams_identity, + std::vector* docids, QueryProfile* profile, ExactPhrasePlanKind* selected_plan, + segment_v2::inverted_index::CommonGramsPlanCostModel cost_model, + std::optional debug_override) { + QueryProfileScope profile_scope(idx.reader(), profile); + format::PrxDecodeContext decode_context { + .stats = profile == nullptr ? nullptr : &profile->prx_decode_stats, + .query_stats = profile == nullptr ? nullptr : &profile->phrase_query_stats}; + return planned_exact_phrase_query_impl( + idx, plain_query_info, gram_query_info, common_grams_identity, docids, + profile == nullptr ? nullptr : &decode_context, selected_plan, cost_model, + debug_override.has_value() ? *debug_override : common_grams_plan_debug_override()); +} + +Status planned_phrase_prefix_query( + const LogicalIndexReader& idx, const segment_v2::InvertedIndexQueryInfo& plain_query_info, + const segment_v2::InvertedIndexQueryInfo& gram_query_info, + const segment_v2::inverted_index::CommonGramsQueryIdentity* common_grams_identity, + std::vector* docids, QueryProfile* profile, int32_t max_expansions, + PhrasePrefixPlanKind* selected_plan, + segment_v2::inverted_index::CommonGramsPlanCostModel cost_model, + std::optional debug_override) { + QueryProfileScope profile_scope(idx.reader(), profile); + format::PrxDecodeContext decode_context { + .stats = profile == nullptr ? nullptr : &profile->prx_decode_stats, + .query_stats = profile == nullptr ? nullptr : &profile->phrase_query_stats}; + return planned_phrase_prefix_query_impl( + idx, plain_query_info, gram_query_info, common_grams_identity, docids, max_expansions, + profile == nullptr ? nullptr : &decode_context, selected_plan, cost_model, + debug_override.has_value() ? *debug_override : common_grams_plan_debug_override()); +} + +Status phrase_prefix_query(const LogicalIndexReader& idx, const std::vector& terms, + std::vector* const docids, int32_t max_expansions) { + return phrase_prefix_query_impl(idx, terms, docids, max_expansions, nullptr, nullptr); +} + +Status phrase_prefix_query(const LogicalIndexReader& idx, const std::vector& terms, + std::vector* const docids, QueryProfile* profile, + int32_t max_expansions) { + QueryProfileScope profile_scope(idx.reader(), profile); + format::PrxDecodeContext decode_context { + .stats = profile == nullptr ? nullptr : &profile->prx_decode_stats, + .query_stats = profile == nullptr ? nullptr : &profile->phrase_query_stats}; + return phrase_prefix_query_impl(idx, terms, docids, max_expansions, + profile == nullptr ? nullptr : &decode_context, nullptr); +} + +Status phrase_prefix_query_with_frequencies(const LogicalIndexReader& idx, + const std::vector& terms, + std::vector* matches, + QueryProfile* profile, int32_t max_expansions) { + if (matches == nullptr) { + return Status::Error( + "phrase_prefix_query_with_frequencies: null out"); + } + matches->clear(); + if (terms.size() < 2) { + return Status::Error( + "phrase_prefix_query_with_frequencies: at least two terms are required"); + } + QueryProfileScope profile_scope(idx.reader(), profile); + format::PrxDecodeContext decode_context { + .stats = profile == nullptr ? nullptr : &profile->prx_decode_stats, + .query_stats = profile == nullptr ? nullptr : &profile->phrase_query_stats}; + return phrase_prefix_query_impl(idx, terms, nullptr, max_expansions, + profile == nullptr ? nullptr : &decode_context, nullptr, + matches); +} + +} // namespace doris::snii::query + +#ifdef BE_TEST +namespace doris::snii::query::internal::testing { +namespace { + +std::atomic& phrase_verify_clock_read_atomic() { + static std::atomic counter {0}; + return counter; +} + +} // namespace + +uint64_t phrase_verify_clock_read_count() { + return phrase_verify_clock_read_atomic().load(std::memory_order_relaxed); +} + +void reset_phrase_verify_clock_read_count() { + phrase_verify_clock_read_atomic().store(0, std::memory_order_relaxed); +} + +void note_phrase_verify_clock_read() { + phrase_verify_clock_read_atomic().fetch_add(1, std::memory_order_relaxed); +} + +} // namespace doris::snii::query::internal::testing +#endif diff --git a/be/src/storage/index/snii/query/phrase_query.h b/be/src/storage/index/snii/query/phrase_query.h new file mode 100644 index 00000000000000..2db4285e1c2bb8 --- /dev/null +++ b/be/src/storage/index/snii/query/phrase_query.h @@ -0,0 +1,141 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/inverted/common_grams/common_grams_query_cost.h" +#include "storage/index/inverted/common_grams/common_grams_segment_metadata.h" +#include "storage/index/inverted/query/query_info.h" +#include "storage/index/snii/format/prx_decode_stats.h" +#include "storage/index/snii/query/query_profile.h" +#include "storage/index/snii/reader/logical_index_reader.h" + +// phrase_query -- MATCH_PHRASE: return the sorted docid set in which the terms +// occur consecutively (for some i, every term k appears at position pos+k in +// the same doc). It first builds the docid conjunction with docs-only posting +// reads, then fetches PRX only for chunks that can contain final candidates: +// 1. read preludes / docs-only posting ranges and intersect per-term docids; +// 2. fetch retained PRX chunks and stream positions for survivors; +// 3. for each surviving doc, check that some position p exists with +// term[0]@p, term[1]@p+1, ... term[n-1]@p+(n-1). +// An empty term list -> empty result. Any term absent -> empty result. +namespace doris::snii::query { + +enum class ExactPhrasePlanKind : uint8_t { + kPlain = 0, + kCommonGrams = 1, +}; + +enum class PhrasePrefixPlanKind : uint8_t { + kPlain = 0, + kCommonGrams = 1, +}; + +// Benchmark-only plan override. The debug points are inert unless the process-wide +// enable_debug_points switch is on, and planners apply them only after both complete plans resolve. +enum class CommonGramsPlanDebugOverride : uint8_t { + kNone = 0, + kForcePlain = 1, + kForceCommonGrams = 2, +}; + +inline constexpr char COMMON_GRAMS_FORCE_PLAIN_PLAN_DEBUG_POINT[] = + "snii.common_grams.force_plain_plan"; +inline constexpr char COMMON_GRAMS_FORCE_GRAM_PLAN_DEBUG_POINT[] = + "snii.common_grams.force_gram_plan"; + +CommonGramsPlanDebugOverride common_grams_plan_debug_override(); + +struct PhraseMatch { + uint32_t docid = 0; + float frequency = 0.0F; + + bool operator==(const PhraseMatch&) const = default; +}; + +struct PhraseQueryOptions { + uint32_t slop = 0; + bool ordered = false; +}; + +Status phrase_query(const reader::LogicalIndexReader& idx, const std::vector& terms, + std::vector* docids); +Status phrase_query(const reader::LogicalIndexReader& idx, const std::vector& terms, + std::vector* docids, QueryProfile* profile); +Status phrase_query(const reader::LogicalIndexReader& idx, const std::vector& terms, + std::vector* docids, QueryProfile* profile, + const PhraseQueryOptions& options); + +// Scoring-only multi-term entry point. It runs the same opaque-term matcher as +// phrase_query() and returns exact occurrence counts or V3-compatible, +// distance-weighted sloppy-phrase frequencies for each matching doc. +Status phrase_query_with_frequencies(const reader::LogicalIndexReader& idx, + const std::vector& terms, + std::vector* matches, + QueryProfile* profile = nullptr, + const PhraseQueryOptions& options = {}); + +// Selects between equivalent plain and CommonGrams exact-phrase plans above +// the existing opaque-term matcher. A missing or mismatched query identity +// forces the plain plan before any gram DICT lookup. +Status planned_exact_phrase_query( + const reader::LogicalIndexReader& idx, + const segment_v2::InvertedIndexQueryInfo& plain_query_info, + const segment_v2::InvertedIndexQueryInfo& gram_query_info, + const segment_v2::inverted_index::CommonGramsQueryIdentity* common_grams_identity, + std::vector* docids, QueryProfile* profile = nullptr, + ExactPhrasePlanKind* selected_plan = nullptr, + segment_v2::inverted_index::CommonGramsPlanCostModel cost_model = {}, + std::optional debug_override = std::nullopt); + +Status planned_phrase_prefix_query( + const reader::LogicalIndexReader& idx, + const segment_v2::InvertedIndexQueryInfo& plain_query_info, + const segment_v2::InvertedIndexQueryInfo& gram_query_info, + const segment_v2::inverted_index::CommonGramsQueryIdentity* common_grams_identity, + std::vector* docids, QueryProfile* profile = nullptr, int32_t max_expansions = 0, + PhrasePrefixPlanKind* selected_plan = nullptr, + segment_v2::inverted_index::CommonGramsPlanCostModel cost_model = {}, + std::optional debug_override = std::nullopt); + +// phrase_prefix_query -- MATCH_PHRASE_PREFIX: the last item in `terms` is a +// term prefix and preceding items are exact terms. For example {"quick", "bro"} +// matches "quick brown" and "quick bronze". Empty terms -> empty result. +Status phrase_prefix_query(const reader::LogicalIndexReader& idx, + const std::vector& terms, + std::vector* const docids, int32_t max_expansions = 0); +Status phrase_prefix_query(const reader::LogicalIndexReader& idx, + const std::vector& terms, + std::vector* const docids, QueryProfile* profile, + int32_t max_expansions = 0); + +// Scoring-only multi-term entry point. Tail expansions are one logical phrase +// clause, so each phrase start contributes at most once even when several +// expanded terms share that position. +Status phrase_prefix_query_with_frequencies(const reader::LogicalIndexReader& idx, + const std::vector& terms, + std::vector* matches, + QueryProfile* profile = nullptr, + int32_t max_expansions = 0); + +} // namespace doris::snii::query diff --git a/be/src/storage/index/snii/query/phrase_verify_timer.h b/be/src/storage/index/snii/query/phrase_verify_timer.h new file mode 100644 index 00000000000000..bb04f33a64abed --- /dev/null +++ b/be/src/storage/index/snii/query/phrase_verify_timer.h @@ -0,0 +1,51 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include "storage/index/snii/format/prx_decode_stats.h" + +namespace doris::snii::query::internal { + +uint64_t exclusive_phrase_verify_ns(uint64_t elapsed_ns, uint64_t decode_ns_before, + uint64_t decode_ns_after); + +class PhraseVerifyTimer { +public: + explicit PhraseVerifyTimer(format::PrxDecodeContext* decode_context); + + void commit_success(); + +private: + format::PrxDecodeStats* stats_ = nullptr; + std::chrono::steady_clock::time_point start_; + uint64_t decode_ns_before_ = 0; +}; + +#ifdef BE_TEST +namespace testing { + +uint64_t phrase_verify_clock_read_count(); +void reset_phrase_verify_clock_read_count(); +void note_phrase_verify_clock_read(); + +} // namespace testing +#endif +} // namespace doris::snii::query::internal diff --git a/be/src/storage/index/snii/query/prefix_query.cpp b/be/src/storage/index/snii/query/prefix_query.cpp new file mode 100644 index 00000000000000..632c80d8b099f8 --- /dev/null +++ b/be/src/storage/index/snii/query/prefix_query.cpp @@ -0,0 +1,56 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/query/prefix_query.h" + +#include +#include + +#include "storage/index/snii/query/internal/term_expansion.h" + +namespace doris::snii::query { + +using reader::LogicalIndexReader; + +Status prefix_query(const LogicalIndexReader& idx, std::string_view prefix, + std::vector* const docids, int32_t max_expansions) { + if (docids == nullptr) { + return Status::Error("prefix_query: null out"); + } + docids->clear(); + VectorDocIdSink sink(*docids); + return prefix_query(idx, prefix, &sink, max_expansions); +} + +Status prefix_query(const LogicalIndexReader& idx, std::string_view prefix, + std::vector* const docids, QueryProfile* profile, + int32_t max_expansions) { + QueryProfileScope profile_scope(idx.reader(), profile); + return prefix_query(idx, prefix, docids, max_expansions); +} + +Status prefix_query(const LogicalIndexReader& idx, std::string_view prefix, DocIdSink* const sink, + int32_t max_expansions) { + if (sink == nullptr) { + return Status::Error("prefix_query: null sink"); + } + + return internal::emit_expanded_docid_union( + idx, prefix, [](std::string_view) { return true; }, sink, max_expansions); +} + +} // namespace doris::snii::query diff --git a/be/src/storage/index/snii/query/prefix_query.h b/be/src/storage/index/snii/query/prefix_query.h new file mode 100644 index 00000000000000..6d17aa7b48b1c6 --- /dev/null +++ b/be/src/storage/index/snii/query/prefix_query.h @@ -0,0 +1,42 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/query/docid_sink.h" +#include "storage/index/snii/query/query_profile.h" +#include "storage/index/snii/reader/logical_index_reader.h" + +// prefix_query -- MATCH_PREFIX semantics: enumerate dictionary terms with the +// requested prefix, then return the sorted docid set containing any enumerated +// term. Empty prefix enumerates all terms. No matching terms -> empty result. +namespace doris::snii::query { + +Status prefix_query(const reader::LogicalIndexReader& idx, std::string_view prefix, + std::vector* const docids, int32_t max_expansions = 0); +Status prefix_query(const reader::LogicalIndexReader& idx, std::string_view prefix, + std::vector* const docids, QueryProfile* profile, + int32_t max_expansions = 0); +Status prefix_query(const reader::LogicalIndexReader& idx, std::string_view prefix, + DocIdSink* const sink, int32_t max_expansions = 0); + +} // namespace doris::snii::query diff --git a/be/src/storage/index/snii/query/query_profile.cpp b/be/src/storage/index/snii/query/query_profile.cpp new file mode 100644 index 00000000000000..025272c421eadd --- /dev/null +++ b/be/src/storage/index/snii/query/query_profile.cpp @@ -0,0 +1,94 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/query/query_profile.h" + +#include +#include +#include + +#include "storage/index/snii/io/file_reader.h" + +namespace doris::snii::query { + +namespace { + +#ifdef BE_TEST +std::atomic query_profile_clock_reads {0}; +#endif + +std::chrono::steady_clock::time_point query_profile_clock_now() { +#ifdef BE_TEST + query_profile_clock_reads.fetch_add(1, std::memory_order_relaxed); +#endif + return std::chrono::steady_clock::now(); +} + +} // namespace + +QueryProfileScope::QueryProfileScope(io::FileReader* reader, QueryProfile* profile) + : reader_(reader), profile_(profile) { + if (profile_ == nullptr) return; + + start_ = query_profile_clock_now(); + *profile_ = QueryProfile {}; + if (reader_ == nullptr) return; + + const io::IoMetrics* metrics = reader_->io_metrics(); + if (metrics == nullptr) return; + + profile_->has_io_metrics = true; + profile_->io_before = *metrics; +} + +QueryProfileScope::~QueryProfileScope() { + finish(); +} + +void QueryProfileScope::finish() { + if (profile_ == nullptr || finished_) return; + finished_ = true; + + const auto end = query_profile_clock_now(); + const auto elapsed = std::chrono::duration_cast(end - start_).count(); + profile_->elapsed_ns = std::max(1, static_cast(elapsed)); + + if (!profile_->has_io_metrics || reader_ == nullptr) return; + const io::IoMetrics* metrics = reader_->io_metrics(); + if (metrics == nullptr) { + profile_->has_io_metrics = false; + return; + } + profile_->io_after = *metrics; + profile_->io_delta = io::delta(profile_->io_after, profile_->io_before); +} + +#ifdef BE_TEST +namespace testing { + +uint64_t query_profile_clock_read_count() { + return query_profile_clock_reads.load(std::memory_order_relaxed); +} + +void reset_query_profile_clock_read_count() { + query_profile_clock_reads.store(0, std::memory_order_relaxed); +} + +} // namespace testing +#endif + +} // namespace doris::snii::query diff --git a/be/src/storage/index/snii/query/query_profile.h b/be/src/storage/index/snii/query/query_profile.h new file mode 100644 index 00000000000000..2598deeef24f85 --- /dev/null +++ b/be/src/storage/index/snii/query/query_profile.h @@ -0,0 +1,67 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include "storage/index/snii/format/prx_decode_stats.h" +#include "storage/index/snii/io/io_metrics.h" + +namespace doris::snii::io { +class FileReader; +} + +namespace doris::snii::query { + +#ifdef BE_TEST +namespace testing { + +uint64_t query_profile_clock_read_count(); +void reset_query_profile_clock_read_count(); + +} // namespace testing +#endif + +struct QueryProfile { + uint64_t elapsed_ns = 0; + bool has_io_metrics = false; + io::IoMetrics io_before; + io::IoMetrics io_after; + io::IoMetrics io_delta; + format::PrxDecodeStats prx_decode_stats; + format::PhraseQueryExecutionStats phrase_query_stats; +}; + +class QueryProfileScope { +public: + QueryProfileScope(io::FileReader* reader, QueryProfile* profile); + ~QueryProfileScope(); + QueryProfileScope(const QueryProfileScope&) = delete; + QueryProfileScope& operator=(const QueryProfileScope&) = delete; + + void finish(); + +private: + io::FileReader* reader_ = nullptr; + QueryProfile* profile_ = nullptr; + std::chrono::steady_clock::time_point start_; + bool finished_ = false; +}; + +} // namespace doris::snii::query diff --git a/be/src/storage/index/snii/query/regexp_query.cpp b/be/src/storage/index/snii/query/regexp_query.cpp new file mode 100644 index 00000000000000..af26fa5878fbdf --- /dev/null +++ b/be/src/storage/index/snii/query/regexp_query.cpp @@ -0,0 +1,192 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/query/regexp_query.h" + +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "common/logging.h" +#include "storage/index/snii/query/internal/regex_prefix.h" +#include "storage/index/snii/query/internal/term_expansion.h" + +namespace doris::snii::query { + +namespace { + +template +struct HyperscanDeleter { + template + void operator()(T* ptr) const { + deleter(ptr); + } +}; + +using HyperscanDatabasePtr = + std::unique_ptr>; +using HyperscanScratchPtr = + std::unique_ptr>; +using HyperscanCompileErrorPtr = + std::unique_ptr>; + +bool is_regex_metachar(char c) { + switch (c) { + case '.': + case '^': + case '$': + case '|': + case '(': + case ')': + case '[': + case ']': + case '*': + case '+': + case '?': + case '{': + case '}': + case '\\': + return true; + default: + return false; + } +} + +std::string literal_prefix_for_regex(std::string_view pattern) { + std::string out; + size_t i = 0; + if (!pattern.empty() && pattern.front() == '^') { + i = 1; + } + for (; i < pattern.size(); ++i) { + const char c = pattern[i]; + if (is_regex_metachar(c)) { + break; + } + out.push_back(c); + } + return out; +} + +} // namespace + +namespace internal { + +std::string regex_enum_prefix(std::string_view pattern, const re2::RE2& re) { + // Left-anchored patterns can yield a tighter enumeration prefix via RE2's + // PossibleMatchRange than the conservative literal scan (which stops at the + // first metacharacter). The prefix only bounds how many dictionary terms are + // enumerated. Unanchored patterns cannot use a prefix because Hyperscan may + // match the pattern after the beginning of a dictionary term. + if (pattern.empty() || pattern.front() != '^') { + return {}; + } + + if (re.ok()) { + std::string min_prefix; + std::string max_prefix; + if (re.PossibleMatchRange(&min_prefix, &max_prefix, 256) && !min_prefix.empty() && + !max_prefix.empty() && min_prefix.front() == max_prefix.front()) { + const auto mismatch_pair = std::ranges::mismatch(min_prefix, max_prefix); + const auto common_len = + static_cast(std::distance(min_prefix.begin(), mismatch_pair.in1)); + if (common_len > 0) { + return min_prefix.substr(0, common_len); + } + } + } + return literal_prefix_for_regex(pattern); +} + +} // namespace internal + +Status regexp_query(const reader::LogicalIndexReader& idx, std::string_view pattern, + std::vector* const docids, int32_t max_expansions) { + if (docids == nullptr) { + return Status::Error("regexp_query: null out"); + } + docids->clear(); + VectorDocIdSink sink(*docids); + return regexp_query(idx, pattern, &sink, max_expansions); +} + +Status regexp_query(const reader::LogicalIndexReader& idx, std::string_view pattern, + std::vector* const docids, QueryProfile* profile, + int32_t max_expansions) { + QueryProfileScope profile_scope(idx.reader(), profile); + return regexp_query(idx, pattern, docids, max_expansions); +} + +Status regexp_query(const reader::LogicalIndexReader& idx, std::string_view pattern, + DocIdSink* const sink, int32_t max_expansions) { + if (sink == nullptr) { + return Status::Error("regexp_query: null sink"); + } + + const std::string compiled_pattern(pattern); + hs_database_t* raw_database = nullptr; + hs_compile_error_t* raw_compile_error = nullptr; + const auto compile_status = + hs_compile(compiled_pattern.c_str(), HS_FLAG_DOTALL | HS_FLAG_ALLOWEMPTY | HS_FLAG_UTF8, + HS_MODE_BLOCK, nullptr, &raw_database, &raw_compile_error); + HyperscanCompileErrorPtr compile_error(raw_compile_error); + if (compile_status != HS_SUCCESS) { + return Status::OK(); + } + HyperscanDatabasePtr database(raw_database); + + hs_scratch_t* raw_scratch = nullptr; + if (hs_alloc_scratch(database.get(), &raw_scratch) != HS_SUCCESS) { + return Status::MemoryAllocFailed("regexp_query: failed to allocate Hyperscan scratch"); + } + HyperscanScratchPtr scratch(raw_scratch); + + std::string enum_prefix; + if (!pattern.empty() && pattern.front() == '^') { + re2::RE2::Options options; + options.set_log_errors(false); + const re2::RE2 re(re2::StringPiece(pattern.data(), pattern.size()), options); + enum_prefix = internal::regex_enum_prefix(pattern, re); + } + const auto on_match = [](unsigned int, unsigned long long, unsigned long long, unsigned int, + void* context) -> int { + *static_cast(context) = true; + return 0; + }; + return internal::emit_expanded_docid_union( + idx, enum_prefix, + [&database, &scratch, on_match](std::string_view term) { + bool matched = false; + const auto scan_status = + hs_scan(database.get(), term.data(), static_cast(term.size()), 0, + scratch.get(), on_match, &matched); + DCHECK_EQ(scan_status, HS_SUCCESS); + return matched; + }, + sink, max_expansions); +} + +} // namespace doris::snii::query diff --git a/be/src/storage/index/snii/query/regexp_query.h b/be/src/storage/index/snii/query/regexp_query.h new file mode 100644 index 00000000000000..d992751e84215d --- /dev/null +++ b/be/src/storage/index/snii/query/regexp_query.h @@ -0,0 +1,43 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/query/docid_sink.h" +#include "storage/index/snii/query/query_profile.h" +#include "storage/index/snii/reader/logical_index_reader.h" + +// regexp_query -- MATCH_REGEXP semantics over dictionary terms. Hyperscan applies +// the same unanchored matching semantics and flags as the V3 inverted index. An +// invalid pattern produces no matches. Matching terms are executed as a sorted +// deduplicated docid union. +namespace doris::snii::query { + +Status regexp_query(const reader::LogicalIndexReader& idx, std::string_view pattern, + std::vector* const docids, int32_t max_expansions = 0); +Status regexp_query(const reader::LogicalIndexReader& idx, std::string_view pattern, + std::vector* const docids, QueryProfile* profile, + int32_t max_expansions = 0); +Status regexp_query(const reader::LogicalIndexReader& idx, std::string_view pattern, + DocIdSink* const sink, int32_t max_expansions = 0); + +} // namespace doris::snii::query diff --git a/be/src/storage/index/snii/query/scoring_query.cpp b/be/src/storage/index/snii/query/scoring_query.cpp new file mode 100644 index 00000000000000..96afa8832b6237 --- /dev/null +++ b/be/src/storage/index/snii/query/scoring_query.cpp @@ -0,0 +1,913 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/query/scoring_query.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/encoding/byte_source.h" +#include "storage/index/snii/format/dict_entry.h" +#include "storage/index/snii/format/format_constants.h" +#include "storage/index/snii/format/frq_pod.h" +#include "storage/index/snii/format/frq_prelude.h" +#include "storage/index/snii/io/batch_range_fetcher.h" +#include "storage/index/snii/reader/dict_block_cache.h" +#include "storage/index/snii/reader/windowed_posting.h" + +namespace doris::snii::query { + +using format::DictEntry; +using format::DictEntryEnc; +using format::DictEntryKind; +using format::FrqPreludeReader; +using format::WindowMeta; +using reader::LogicalIndexReader; + +namespace { + +// One scored posting for one term in one doc. +struct TermPosting { + uint32_t docid = 0; + double score = 0.0; +}; + +// One window's block-max upper bound and the docid range it covers. block_max is +// true when max_score came from the frq_prelude columns (vs the exact-score +// fallback); both are valid upper bounds, so it is informational only. +struct WindowBound { + uint32_t first_docid = 0; // inclusive + uint32_t last_docid = 0; // inclusive + double max_score = 0.0; // block-max upper bound for any doc in this window + bool block_max = false; +}; + +// All scored postings of one query term plus its block-max metadata. +struct TermCursor { + std::vector postings; // ascending docid, exact per-doc scores + std::vector windows; // ascending, covering all postings + size_t pos = 0; // DAAT cursor into postings +}; + +uint32_t current_doc(const TermCursor& c) { + return c.pos < c.postings.size() ? c.postings[c.pos].docid + : std::numeric_limits::max(); +} + +// Reads one slim .frq window's bytes for a slim pod_ref/inline entry (prelude +// stripped). Windowed entries are handled separately via the prelude decode. +Status fetch_slim_window_bytes(const LogicalIndexReader& idx, const DictEntry& entry, + uint64_t frq_base, std::vector* window_owned, + Slice* window) { + if (entry.kind == DictEntryKind::kInline) { + *window = Slice(entry.frq_bytes); + return Status::OK(); + } + uint64_t win_abs = 0; + uint64_t win_len = 0; + RETURN_IF_ERROR(idx.resolve_frq_window(entry, frq_base, &win_abs, &win_len)); + io::BatchRangeFetcher fetcher(idx.reader()); + const size_t h = fetcher.add(win_abs, win_len); + RETURN_IF_ERROR(fetcher.fetch()); + Slice got = fetcher.get(h); + window_owned->assign(got.data(), got.data() + got.size()); + *window = Slice(*window_owned); + return Status::OK(); +} + +// Reads a windowed entry's frq_prelude (block-max columns live here). +Status fetch_prelude(const LogicalIndexReader& idx, const DictEntry& entry, uint64_t frq_base, + FrqPreludeReader* out) { + const auto& region = idx.section_refs().posting_region; + const uint64_t prelude_abs = region.offset + frq_base + entry.frq_off_delta; + io::BatchRangeFetcher fetcher(idx.reader()); + const size_t h = fetcher.add(prelude_abs, entry.prelude_len); + RETURN_IF_ERROR(fetcher.fetch()); + return FrqPreludeReader::open(fetcher.get(h), out); +} + +// Builds per-window block-max bounds from a windowed entry's prelude. Each +// WindowMeta carries the window's max_freq / max_norm and its covered docid +// range (win_base+1 .. last_docid), so bounds come straight from the directory. +Status build_window_bounds(const FrqPreludeReader& prelude, const ScorerContext& ctx, double avgdl, + const Bm25Params& params, std::vector* windows) { + const uint32_t n = prelude.n_windows(); + for (uint32_t w = 0; w < n; ++w) { + WindowMeta m; + RETURN_IF_ERROR(prelude.window(w, &m)); + if (m.doc_count == 0) continue; + WindowBound wb; + wb.first_docid = static_cast(m.win_base) + (w == 0 ? 0u : 1u); + wb.last_docid = m.last_docid; + wb.max_score = ctx.max_score(m.max_freq, m.max_norm, avgdl, params); + wb.block_max = true; + windows->push_back(wb); + } + return Status::OK(); +} + +// Fallback single window covering all postings, bounded by the exact max score +// (always a valid upper bound, so pruning stays correct). +void single_window_fallback(const std::vector& postings, + std::vector* windows) { + if (postings.empty()) return; + WindowBound wb; + wb.first_docid = postings.front().docid; + wb.last_docid = postings.back().docid; + wb.block_max = false; + for (const auto& p : postings) wb.max_score = std::max(wb.max_score, p.score); + windows->push_back(wb); +} + +// Computes exact per-doc BM25 scores from decoded (docid, freq) vectors. +Status score_decoded(const stats::SniiStatsProvider& stats, const ScorerContext& ctx, double avgdl, + const Bm25Params& params, const std::vector& docids, + const std::vector& freqs, std::vector* out) { + out->reserve(docids.size()); + for (size_t i = 0; i < docids.size(); ++i) { + uint8_t norm = 0; + RETURN_IF_ERROR(stats.encoded_norm(docids[i], &norm)); + const uint32_t tf = i < freqs.size() ? freqs[i] : 1; + out->push_back({docids[i], ctx.score(tf, norm, avgdl, params)}); + } + return Status::OK(); +} + +// Decodes a slim/inline term's single .frq window ([dd_region][freq_region]) into +// docids/freqs using the entry's region metadata. +Status decode_slim(const LogicalIndexReader& idx, const DictEntry& entry, uint64_t frq_base, + std::vector* docids, std::vector* freqs) { + std::vector owned; + Slice window; + RETURN_IF_ERROR(fetch_slim_window_bytes(idx, entry, frq_base, &owned, &window)); + const uint64_t dd_len = entry.dd_meta.disk_len; + if (dd_len > window.size()) { + return Status::Error( + "scoring_query: slim dd region exceeds window"); + } + Slice dd_region = window.subslice(0, static_cast(dd_len)); + RETURN_IF_ERROR(format::decode_dd_region(dd_region, entry.dd_meta, + /*win_base=*/0, docids)); + // G16-c freq-dropped segments (write_freq == false) carry a zero-length + // freq region on slim/inline entries. Fail with the SEMANTIC error and NOT + // with INVERTED_INDEX_FILE_CORRUPTED: the Doris segment iterator silently + // downgrades that code to a non-index evaluation, which would mask a + // by-design layout as data corruption once BM25 runs over mixed segments. + if (window.size() == static_cast(dd_len) && !docids->empty()) { + return Status::Error( + "scoring_query: freqs requested but the slim entry has no freq region " + "(freq-dropped positions index)"); + } + Slice freq_region = window.subslice(static_cast(dd_len), + window.size() - static_cast(dd_len)); + return format::decode_freq_region(freq_region, entry.freq_meta, docids->size(), freqs); +} + +// Builds the cursor for a windowed term: tiles all windows for exact scores and +// reads the prelude once for true per-window block-max bounds. +Status build_windowed_cursor(const LogicalIndexReader& idx, const stats::SniiStatsProvider& stats, + const ScorerContext& ctx, const DictEntry& entry, uint64_t frq_base, + uint64_t prx_base, double avgdl, const Bm25Params& params, + TermCursor* cursor) { + reader::DecodedPosting posting; + // Scoring needs freqs for BM25: fetch the FULL windows (want_freq=true). + RETURN_IF_ERROR(reader::read_windowed_posting(idx, entry, frq_base, prx_base, + /*want_positions=*/false, + /*want_freq=*/true, &posting)); + RETURN_IF_ERROR(score_decoded(stats, ctx, avgdl, params, posting.docids, posting.freqs, + &cursor->postings)); + FrqPreludeReader prelude; + if (fetch_prelude(idx, entry, frq_base, &prelude).ok()) { + RETURN_IF_ERROR(build_window_bounds(prelude, ctx, avgdl, params, &cursor->windows)); + } + return Status::OK(); +} + +Status build_resolved_cursor(const LogicalIndexReader& idx, const stats::SniiStatsProvider& stats, + const ScorerContext& ctx, const DictEntry& entry, uint64_t frq_base, + uint64_t prx_base, double avgdl, const Bm25Params& params, + TermCursor* cursor) { + const bool windowed = + entry.kind == DictEntryKind::kPodRef && entry.enc == DictEntryEnc::kWindowed; + if (windowed) { + RETURN_IF_ERROR(build_windowed_cursor(idx, stats, ctx, entry, frq_base, prx_base, avgdl, + params, cursor)); + } else { + std::vector docids; + std::vector freqs; + RETURN_IF_ERROR(decode_slim(idx, entry, frq_base, &docids, &freqs)); + RETURN_IF_ERROR(score_decoded(stats, ctx, avgdl, params, docids, freqs, &cursor->postings)); + } + if (cursor->windows.empty()) { + single_window_fallback(cursor->postings, &cursor->windows); + } + return Status::OK(); +} + +Status accumulate_decoded_candidate_scores(const stats::SniiStatsProvider& stats, + const ScorerContext& scorer, double avgdl, + const Bm25Params& params, + const std::vector& docids, + const std::vector& freqs, + std::span candidates, + std::span scores) { + DCHECK_EQ(candidates.size(), scores.size()); + DCHECK_EQ(docids.size(), freqs.size()); + size_t doc_index = 0; + size_t candidate_index = 0; + while (doc_index < docids.size() && candidate_index < candidates.size()) { + if (docids[doc_index] < candidates[candidate_index]) { + ++doc_index; + continue; + } + if (candidates[candidate_index] < docids[doc_index]) { + ++candidate_index; + continue; + } + uint8_t norm = 0; + RETURN_IF_ERROR(stats.encoded_norm(docids[doc_index], &norm)); + scores[candidate_index] += scorer.score(freqs[doc_index], norm, avgdl, params); + ++doc_index; + ++candidate_index; + } + return Status::OK(); +} + +struct CandidateWindowWork { + WindowMeta meta; + size_t candidate_begin = 0; + size_t candidate_end = 0; + size_t dd_handle = 0; + size_t freq_handle = 0; +}; + +Status accumulate_windowed_candidate_scores(const LogicalIndexReader& idx, + const stats::SniiStatsProvider& stats, + const DictEntry& entry, uint64_t frq_base, + uint64_t prx_base, + const std::vector& candidates, + const ScorerContext& scorer, double avgdl, + const Bm25Params& params, std::vector* scores) { + FrqPreludeReader prelude; + RETURN_IF_ERROR(reader::fetch_windowed_prelude(idx, entry, frq_base, &prelude)); + + std::vector windows; + const uint32_t window_count = prelude.n_windows(); + const bool candidates_cover_segment = + static_cast(candidates.size()) == stats.doc_count() && !candidates.empty() && + candidates.front() == 0 && + static_cast(candidates.back()) + 1 == stats.doc_count(); + bool scan_all = candidates_cover_segment; + if (scan_all) { + windows.resize(window_count); + std::iota(windows.begin(), windows.end(), 0); + } else { + prelude.select_covering_windows(candidates, &windows); + scan_all = windows.size() == window_count; + } + if (windows.empty()) { + return Status::OK(); + } + + // A sparse set must not be re-expanded into a near-full posting by merging + // across unselected windows. Dense/all-window reads retain the existing + // same-term coalescing policy and therefore the existing request shape. + const uint64_t coalesce_gap = scan_all ? reader::kSameTermCoalesceGap : 0; + io::BatchRangeFetcher fetcher(idx.reader(), coalesce_gap); + std::vector work; + work.reserve(windows.size()); + size_t candidate_cursor = 0; + for (uint32_t window : windows) { + CandidateWindowWork item; + RETURN_IF_ERROR(prelude.window(window, &item.meta)); + const uint32_t first_docid = + window == 0 ? 0 : static_cast(item.meta.win_base + 1); + while (candidate_cursor < candidates.size() && candidates[candidate_cursor] < first_docid) { + ++candidate_cursor; + } + item.candidate_begin = candidate_cursor; + while (candidate_cursor < candidates.size() && + candidates[candidate_cursor] <= item.meta.last_docid) { + ++candidate_cursor; + } + item.candidate_end = candidate_cursor; + + reader::WindowAbsRange range; + RETURN_IF_ERROR(reader::windowed_window_range( + idx, entry, frq_base, prx_base, prelude, window, + /*want_positions=*/false, /*want_freq=*/true, &range)); + item.dd_handle = fetcher.add(range.dd_off, range.dd_len); + item.freq_handle = fetcher.add(range.freq_off, range.freq_len); + work.push_back(std::move(item)); + } + RETURN_IF_ERROR(fetcher.fetch()); + + std::vector docids; + std::vector freqs; + std::vector> positions; + for (const auto& item : work) { + docids.clear(); + freqs.clear(); + RETURN_IF_ERROR(reader::decode_window_slices( + item.meta, fetcher.get(item.dd_handle), fetcher.get(item.freq_handle), Slice(), + /*want_positions=*/false, /*want_freq=*/true, &docids, &freqs, &positions)); + const size_t candidate_count = item.candidate_end - item.candidate_begin; + RETURN_IF_ERROR(accumulate_decoded_candidate_scores( + stats, scorer, avgdl, params, docids, freqs, + std::span(candidates) + .subspan(item.candidate_begin, candidate_count), + std::span(*scores).subspan(item.candidate_begin, candidate_count))); + } + return Status::OK(); +} + +Status accumulate_resolved_candidate_scores(const LogicalIndexReader& idx, + const stats::SniiStatsProvider& stats, + const DictEntry& entry, uint64_t frq_base, + uint64_t prx_base, + const std::vector& candidates, + const ScorerContext& scorer, double avgdl, + const Bm25Params& params, std::vector* scores) { + const bool windowed = + entry.kind == DictEntryKind::kPodRef && entry.enc == DictEntryEnc::kWindowed; + if (windowed) { + return accumulate_windowed_candidate_scores(idx, stats, entry, frq_base, prx_base, + candidates, scorer, avgdl, params, scores); + } + + std::vector docids; + std::vector freqs; + RETURN_IF_ERROR(decode_slim(idx, entry, frq_base, &docids, &freqs)); + return accumulate_decoded_candidate_scores(stats, scorer, avgdl, params, docids, freqs, + candidates, *scores); +} + +// Builds the cursor for one term: postings with exact scores + window bounds. +Status build_cursor(const LogicalIndexReader& idx, const stats::SniiStatsProvider& stats, + const std::string& term, const Bm25Params& params, bool* found, + TermCursor* cursor) { + DictEntry entry; + uint64_t frq_base = 0; + uint64_t prx_base = 0; + RETURN_IF_ERROR(idx.lookup(term, found, &entry, &frq_base, &prx_base)); + if (!*found) return Status::OK(); + + const ScorerContext ctx = ScorerContext::make(stats.indexed_doc_count(), entry.df); + return build_resolved_cursor(idx, stats, ctx, entry, frq_base, prx_base, stats.avgdl(), params, + cursor); +} + +// Block-max upper bound for a term at a given docid: the max_score of the window +// covering docid (windows are ascending and contiguous). Beyond the last window +// the bound is 0 (the term cannot contribute). +double term_bound_at(const TermCursor& c, uint32_t docid) { + // Windows are ascending and contiguous; the first window whose last_docid is + // >= docid covers it. Its block-max is a valid upper bound for any contained + // doc, so it also bounds gaps between windows. + for (const auto& w : c.windows) { + if (docid <= w.last_docid) return w.max_score; + } + return 0.0; +} + +// Min-heap keyed on score (smallest at top) maintaining the top-K. +struct TopK { + explicit TopK(uint32_t k) : k_(k) {} + void offer(uint32_t docid, double score) { + if (heap_.size() < k_) { + heap_.push({score, docid}); + return; + } + if (heap_.empty()) return; + const Entry& worst = heap_.top(); // lowest score; ties: largest docid + const bool better = score > worst.first || (score == worst.first && docid < worst.second); + if (better) { + heap_.pop(); + heap_.push({score, docid}); + } + } + double threshold() const { return heap_.size() < k_ ? -1.0 : heap_.top().first; } + + using Entry = std::pair; + struct Cmp { + bool operator()(const Entry& a, const Entry& b) const { + if (a.first != b.first) return a.first > b.first; // min-score at top + return a.second < b.second; // for ties, largest docid at top (evictable) + } + }; + uint32_t k_; + std::priority_queue, Cmp> heap_; +}; + +void drain_sorted(TopK* topk, std::vector* out) { + std::vector all; + while (!topk->heap_.empty()) { + all.push_back({topk->heap_.top().second, topk->heap_.top().first}); + topk->heap_.pop(); + } + std::sort(all.begin(), all.end(), [](const ScoredDoc& a, const ScoredDoc& b) { + if (a.score != b.score) return a.score > b.score; + return a.docid < b.docid; + }); + *out = std::move(all); +} + +Status build_cursors(const LogicalIndexReader& idx, const stats::SniiStatsProvider& stats, + const std::vector& terms, const Bm25Params& params, + std::vector* cursors) { + for (const auto& term : terms) { + bool found = false; + TermCursor c; + RETURN_IF_ERROR(build_cursor(idx, stats, term, params, &found, &c)); + if (found && !c.postings.empty()) cursors->push_back(std::move(c)); + } + return Status::OK(); +} + +} // namespace + +Status scoring_query_candidates(const LogicalIndexReader& idx, + const stats::SniiStatsProvider& segment_stats, + const std::vector& terms, + const roaring::Roaring& final_candidates, double collection_avgdl, + const Bm25Params& params, std::vector* out) { + if (out == nullptr) { + return Status::Error( + "scoring_query_candidates: null out"); + } + out->clear(); + if (final_candidates.isEmpty()) { + return Status::OK(); + } + if (!(collection_avgdl > 0.0)) { + return Status::Error( + "scoring_query_candidates: collection avgdl must be positive"); + } + + std::vector candidate_docids; + candidate_docids.reserve(final_candidates.cardinality()); + for (uint32_t docid : final_candidates) { + candidate_docids.push_back(docid); + } + std::vector candidate_scores(candidate_docids.size(), 0.0); + reader::DictBlockCache dict_block_cache; + + for (const auto& term : terms) { + bool found = false; + DictEntry entry; + uint64_t frq_base = 0; + uint64_t prx_base = 0; + RETURN_IF_ERROR(idx.lookup(term.physical_term, &found, &entry, &frq_base, &prx_base, + &dict_block_cache)); + if (!found) { + continue; + } + + const ScorerContext scorer = ScorerContext::from_idf(term.idf); + RETURN_IF_ERROR(accumulate_resolved_candidate_scores( + idx, segment_stats, entry, frq_base, prx_base, candidate_docids, scorer, + collection_avgdl, params, &candidate_scores)); + } + + std::vector scored_candidates; + scored_candidates.reserve(candidate_docids.size()); + for (size_t i = 0; i < candidate_docids.size(); ++i) { + scored_candidates.push_back({.docid = candidate_docids[i], .score = candidate_scores[i]}); + } + *out = std::move(scored_candidates); + return Status::OK(); +} + +Status scoring_query_exhaustive(const LogicalIndexReader& idx, + const stats::SniiStatsProvider& stats, + const std::vector& terms, uint32_t k, + const Bm25Params& params, std::vector* out) { + if (out == nullptr) + return Status::Error("scoring_query: null out"); + out->clear(); + if (k == 0) return Status::OK(); + + std::vector cursors; + RETURN_IF_ERROR(build_cursors(idx, stats, terms, params, &cursors)); + + std::unordered_map scores; + for (const auto& c : cursors) + for (const auto& p : c.postings) scores[p.docid] += p.score; + + std::vector all; + all.reserve(scores.size()); + for (const auto& [docid, score] : scores) all.push_back({docid, score}); + std::sort(all.begin(), all.end(), [](const ScoredDoc& a, const ScoredDoc& b) { + if (a.score != b.score) return a.score > b.score; + return a.docid < b.docid; + }); + if (all.size() > k) all.resize(k); + *out = std::move(all); + return Status::OK(); +} + +namespace { + +// --- Phase C: selective-fetch (lazy window) WAND ----------------------------- +// +// A LazyTermCursor knows its per-window block-max bounds + docid ranges from the +// frq_prelude WITHOUT fetching any .frq window. Each window's exact (docid,score) +// postings are decoded on first access and cached, so a window is fetched at most +// once and ONLY when the WAND control flow touches a posting in it. Combined with +// window-level SkipTo (advance past whole windows whose last_docid < target via +// the prelude, never fetching them), the offer sequence is byte-identical to the +// eager scoring_query_wand path -- only the bytes read differ. +// +// Soundness: a window is fetched only when lazy_current_doc/lazy_skip_to land the +// cursor inside it, i.e. it covers a candidate the WAND pivot already proved can +// reach the running theta (bound >= theta). lazy_skip_to jumps the cursor to the +// SAME posting (first docid >= target) the eager per-doc walk would, so pivots, +// alignments and offers are identical to the eager path; only windows the eager +// path read-through-but-never-offered-from are skipped. Windows whose block-max +// bound never reaches theta are never the pivot, so never fetched. + +// One query term's lazily-fetched scoring state. +struct LazyTermCursor { + const LogicalIndexReader* idx = nullptr; + const stats::SniiStatsProvider* stats = nullptr; + ScorerContext ctx = ScorerContext::make(1, 1); + Bm25Params params; + DictEntry entry; + uint64_t frq_base = 0; + uint64_t prx_base = 0; + FrqPreludeReader prelude; + bool windowed = false; // false => slim/inline single block already materialized + + std::vector windows; // ascending; from prelude (or slim fallback) + std::vector postings; // sparse: only fetched windows are filled + std::vector win_start; // prefix offsets, size = windows.size()+1 + std::vector fetched; // size = windows.size() + size_t pos = 0; // virtual cursor over all windows' postings +}; + +// Total posting count across all windows (the virtual stream length). +uint32_t total_postings(const LazyTermCursor& c) { + return c.win_start.empty() ? 0 : c.win_start.back(); +} + +// Index of the window whose virtual range contains posting index p (p < total). +uint32_t window_of(const LazyTermCursor& c, uint32_t p) { + const auto it = std::upper_bound(c.win_start.begin(), c.win_start.end(), p); + return static_cast((it - c.win_start.begin()) - 1); +} + +// Fetches + decodes window w into the cursor's posting cache (idempotent). Only +// reached when the WAND proves window w can still contribute to the top-K. +Status materialize_window(LazyTermCursor* c, uint32_t w) { + if (c->fetched[w]) return Status::OK(); + WindowMeta meta; + RETURN_IF_ERROR(c->prelude.window(w, &meta)); + reader::WindowAbsRange r; + RETURN_IF_ERROR(reader::windowed_window_range( + *c->idx, c->entry, c->frq_base, c->prx_base, c->prelude, w, + /*want_positions=*/false, /*want_freq=*/true, &r)); + // Scoring needs docids + freqs: fetch the window's dd sub-range AND freq sub-range. + io::BatchRangeFetcher fetcher(c->idx->reader(), reader::kSameTermCoalesceGap); + const size_t dh = fetcher.add(r.dd_off, r.dd_len); + const size_t fh = fetcher.add(r.freq_off, r.freq_len); + RETURN_IF_ERROR(fetcher.fetch()); + std::vector docids; + std::vector freqs; + std::vector> pos; + RETURN_IF_ERROR(reader::decode_window_slices(meta, fetcher.get(dh), fetcher.get(fh), Slice(), + /*want_positions=*/false, + /*want_freq=*/true, &docids, &freqs, &pos)); + if (docids.size() != c->win_start[w + 1] - c->win_start[w]) { + return Status::Error( + "scoring_query: selective window doc-count drift"); + } + std::vector scored; + RETURN_IF_ERROR( + score_decoded(*c->stats, c->ctx, c->stats->avgdl(), c->params, docids, freqs, &scored)); + std::copy(scored.begin(), scored.end(), c->postings.begin() + c->win_start[w]); + c->fetched[w] = 1; + return Status::OK(); +} + +// Current docid at the cursor, fetching the covering window if needed. Exhausted +// cursor -> UINT32_MAX. +Status lazy_current_doc(LazyTermCursor* c, uint32_t* docid) { + if (c->pos >= total_postings(*c)) { + *docid = std::numeric_limits::max(); + return Status::OK(); + } + const uint32_t w = window_of(*c, static_cast(c->pos)); + RETURN_IF_ERROR(materialize_window(c, w)); + *docid = c->postings[c->pos].docid; + return Status::OK(); +} + +// Advances pos to the first posting with docid >= target, skipping ENTIRE windows +// whose last_docid < target WITHOUT fetching them (prelude-only), then fetching +// just the landing window. Lands on the same posting the eager per-doc walk would. +Status lazy_skip_to(LazyTermCursor* c, uint32_t target) { + const uint32_t total = total_postings(*c); + while (c->pos < total) { + const uint32_t w = window_of(*c, static_cast(c->pos)); + if (c->windows[w].last_docid >= target) break; + c->pos = c->win_start[w + 1]; // skip this window entirely (no fetch) + } + if (c->pos >= total) return Status::OK(); + const uint32_t w = window_of(*c, static_cast(c->pos)); + RETURN_IF_ERROR(materialize_window(c, w)); + while (c->pos < total && c->postings[c->pos].docid < target) ++c->pos; + return Status::OK(); +} + +// Initializes a lazy windowed cursor from the prelude alone: per-window block-max +// bounds + ranges + cache slots, with NO .frq window fetched. +Status build_lazy_windowed(LazyTermCursor* c) { + RETURN_IF_ERROR(reader::fetch_windowed_prelude(*c->idx, c->entry, c->frq_base, &c->prelude)); + RETURN_IF_ERROR( + build_window_bounds(c->prelude, c->ctx, c->stats->avgdl(), c->params, &c->windows)); + // build_window_bounds keeps only non-empty windows, in window order. Build the + // matching prefix-sum of doc_counts over those same non-empty windows so the + // bound list, win_start and fetched stay 1:1. + const uint32_t nb = static_cast(c->windows.size()); + c->win_start.assign(nb + 1, 0); + c->fetched.assign(nb, 0); + uint32_t bi = 0; + uint32_t acc = 0; + for (uint32_t w = 0; w < c->prelude.n_windows() && bi < nb; ++w) { + WindowMeta meta; + RETURN_IF_ERROR(c->prelude.window(w, &meta)); + if (meta.doc_count == 0) continue; + acc += meta.doc_count; + c->win_start[++bi] = acc; + } + c->postings.assign(acc, TermPosting {}); + return Status::OK(); +} + +// Initializes a slim/inline cursor: its single window is small, so fetch + score +// it eagerly (exactly as the existing path). One bound covers all its postings. +Status build_lazy_slim(LazyTermCursor* c) { + std::vector docids; + std::vector freqs; + RETURN_IF_ERROR(decode_slim(*c->idx, c->entry, c->frq_base, &docids, &freqs)); + RETURN_IF_ERROR(score_decoded(*c->stats, c->ctx, c->stats->avgdl(), c->params, docids, freqs, + &c->postings)); + single_window_fallback(c->postings, &c->windows); + c->win_start = {0, static_cast(c->postings.size())}; + c->fetched.assign(1, 1); // already materialized + return Status::OK(); +} + +// Builds a LazyTermCursor for one term: prelude-only for windowed terms (no .frq +// fetched), fully-materialized single window for slim/inline (small). +Status build_lazy_cursor(const LogicalIndexReader& idx, const stats::SniiStatsProvider& stats, + const std::string& term, const Bm25Params& params, bool* found, + LazyTermCursor* c) { + uint64_t prx_base = 0; + RETURN_IF_ERROR(idx.lookup(term, found, &c->entry, &c->frq_base, &prx_base)); + if (!*found) return Status::OK(); + c->idx = &idx; + c->stats = &stats; + c->params = params; + c->prx_base = prx_base; + c->ctx = ScorerContext::make(stats.indexed_doc_count(), c->entry.df); + c->windowed = + c->entry.kind == DictEntryKind::kPodRef && c->entry.enc == DictEntryEnc::kWindowed; + return c->windowed ? build_lazy_windowed(c) : build_lazy_slim(c); +} + +Status selective_build_cursors(const LogicalIndexReader& idx, const stats::SniiStatsProvider& stats, + const std::vector& terms, const Bm25Params& params, + std::vector* cursors) { + for (const auto& term : terms) { + bool found = false; + LazyTermCursor c; + RETURN_IF_ERROR(build_lazy_cursor(idx, stats, term, params, &found, &c)); + if (found && total_postings(c) > 0) cursors->push_back(std::move(c)); + } + return Status::OK(); +} + +// Block-max upper bound for a lazy cursor at docid: block_max of the window +// covering docid (ascending, contiguous). Beyond the last window -> 0. Same +// semantics as term_bound_at over the eager cursor's window list. +double lazy_term_bound_at(const LazyTermCursor& c, uint32_t docid) { + for (const auto& w : c.windows) { + if (docid <= w.last_docid) return w.max_score; + } + return 0.0; +} + +// Sorts cursors ascending by current docid (materializing each cursor's current +// covering window), returning the smallest current docid via *front. +Status selective_sort_by_doc(std::vector* cursors, uint32_t* front) { + std::vector cur(cursors->size()); + for (size_t i = 0; i < cursors->size(); ++i) { + RETURN_IF_ERROR(lazy_current_doc(&(*cursors)[i], &cur[i])); + } + std::vector order(cursors->size()); + for (size_t i = 0; i < order.size(); ++i) order[i] = i; + std::sort(order.begin(), order.end(), [&](size_t a, size_t b) { return cur[a] < cur[b]; }); + std::vector sorted; + sorted.reserve(cursors->size()); + for (size_t i : order) sorted.push_back(std::move((*cursors)[i])); + *cursors = std::move(sorted); + *front = order.empty() ? std::numeric_limits::max() : cur[order.front()]; + return Status::OK(); +} + +// Finds the pivot term: the first cursor (current-docid order) at which the +// accumulated block-max bound reaches theta. >= keeps boundary ties (matching the +// exhaustive total order). *found=false when no remaining doc can beat theta. +Status selective_pivot(std::vector* cursors, double theta, size_t* pivot, + uint32_t* pivot_doc, bool* found) { + double bound = 0.0; + *found = false; + for (size_t i = 0; i < cursors->size(); ++i) { + uint32_t d = 0; + RETURN_IF_ERROR(lazy_current_doc(&(*cursors)[i], &d)); + if (d == std::numeric_limits::max()) break; + bound += lazy_term_bound_at((*cursors)[i], d); + if (bound >= theta) { + *pivot = i; + *pivot_doc = d; + *found = true; + return Status::OK(); + } + } + return Status::OK(); +} + +// Scores the aligned pivot doc exactly (summing all cursors AT pivot_doc) and +// advances those cursors by one posting. +Status selective_score_pivot(std::vector* cursors, uint32_t pivot_doc, TopK* topk) { + double doc_score = 0.0; + for (auto& c : *cursors) { + uint32_t d = 0; + RETURN_IF_ERROR(lazy_current_doc(&c, &d)); + if (d == pivot_doc) { + doc_score += c.postings[c.pos].score; // window already materialized + ++c.pos; + } + } + topk->offer(pivot_doc, doc_score); + return Status::OK(); +} + +// Advances the first lagging cursor (current doc < pivot_doc) up to pivot_doc. +Status selective_advance_lagging(std::vector* cursors, uint32_t pivot_doc) { + for (auto& c : *cursors) { + uint32_t d = 0; + RETURN_IF_ERROR(lazy_current_doc(&c, &d)); + if (d < pivot_doc) { + RETURN_IF_ERROR(lazy_skip_to(&c, pivot_doc)); + return Status::OK(); + } + } + return Status::OK(); +} + +// One WAND iteration body: sort, pick pivot, then either score (aligned) or skip +// a lagging cursor forward. *done=true ends the loop. +Status selective_step(std::vector* cursors, TopK* topk, bool* done) { + uint32_t front = 0; + RETURN_IF_ERROR(selective_sort_by_doc(cursors, &front)); + if (cursors->empty() || front == std::numeric_limits::max()) { + *done = true; + return Status::OK(); + } + size_t pivot = 0; + uint32_t pivot_doc = 0; + bool found_pivot = false; + RETURN_IF_ERROR(selective_pivot(cursors, topk->threshold(), &pivot, &pivot_doc, &found_pivot)); + if (!found_pivot) { + *done = true; + return Status::OK(); + } + if (front == pivot_doc) { + return selective_score_pivot(cursors, pivot_doc, topk); + } + return selective_advance_lagging(cursors, pivot_doc); +} + +Status selective_wand_loop(std::vector* cursors, TopK* topk) { + bool done = false; + while (!done) { + RETURN_IF_ERROR(selective_step(cursors, topk, &done)); + } + return Status::OK(); +} + +} // namespace + +Status scoring_query_wand_selective(const LogicalIndexReader& idx, + const stats::SniiStatsProvider& stats, + const std::vector& terms, uint32_t k, + const Bm25Params& params, std::vector* out) { + if (out == nullptr) + return Status::Error("scoring_query: null out"); + out->clear(); + if (k == 0) return Status::OK(); + + std::vector cursors; + RETURN_IF_ERROR(selective_build_cursors(idx, stats, terms, params, &cursors)); + + TopK topk(k); + RETURN_IF_ERROR(selective_wand_loop(&cursors, &topk)); + drain_sorted(&topk, out); + return Status::OK(); +} + +Status scoring_query_wand(const LogicalIndexReader& idx, const stats::SniiStatsProvider& stats, + const std::vector& terms, uint32_t k, + const Bm25Params& params, std::vector* out) { + if (out == nullptr) + return Status::Error("scoring_query: null out"); + out->clear(); + if (k == 0) return Status::OK(); + + std::vector cursors; + RETURN_IF_ERROR(build_cursors(idx, stats, terms, params, &cursors)); + + TopK topk(k); + // Document-at-a-time WAND with block-max bounds. + while (true) { + // Sort cursors by current docid (ascending; exhausted cursors sink). + std::sort(cursors.begin(), cursors.end(), [](const TermCursor& a, const TermCursor& b) { + return current_doc(a) < current_doc(b); + }); + if (cursors.empty() || + current_doc(cursors.front()) == std::numeric_limits::max()) { + break; + } + + const double theta = topk.threshold(); + // Accumulate block-max upper bounds in docid order to find the pivot term. + double bound = 0.0; + size_t pivot = 0; + bool found_pivot = false; + for (size_t i = 0; i < cursors.size(); ++i) { + const uint32_t d = current_doc(cursors[i]); + if (d == std::numeric_limits::max()) break; + bound += term_bound_at(cursors[i], d); + // Use >= (not >) so a doc whose upper bound only TIES the K-th threshold is + // still explored and exact-scored: under the (score desc, docid asc) total + // order a tie can still evict the current K-th entry (smaller docid wins), + // exactly as the exhaustive path would. Strict > would wrongly prune ties. + if (bound >= theta) { + pivot = i; + found_pivot = true; + break; + } + } + if (!found_pivot) break; // no doc can beat the threshold anymore. + + const uint32_t pivot_doc = current_doc(cursors[pivot]); + if (current_doc(cursors.front()) == pivot_doc) { + // All cursors at the pivot doc are aligned: score it exactly. + double doc_score = 0.0; + for (auto& c : cursors) { + if (current_doc(c) == pivot_doc) { + doc_score += c.postings[c.pos].score; + ++c.pos; + } + } + topk.offer(pivot_doc, doc_score); + } else { + // Advance a lagging cursor toward pivot_doc (skip docs it cannot win on). + for (auto& c : cursors) { + if (current_doc(c) < pivot_doc) { + while (c.pos < c.postings.size() && c.postings[c.pos].docid < pivot_doc) { + ++c.pos; + } + break; + } + } + } + } + drain_sorted(&topk, out); + return Status::OK(); +} + +} // namespace doris::snii::query diff --git a/be/src/storage/index/snii/query/scoring_query.h b/be/src/storage/index/snii/query/scoring_query.h new file mode 100644 index 00000000000000..e8e83a7881efdb --- /dev/null +++ b/be/src/storage/index/snii/query/scoring_query.h @@ -0,0 +1,98 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/query/bm25_scorer.h" +#include "storage/index/snii/reader/logical_index_reader.h" +#include "storage/index/snii/stats/snii_stats_provider.h" + +// scoring_query -- top-K BM25 scored retrieval over one logical index for one or +// more query terms. Two entry points produce IDENTICAL rankings: +// - scoring_query_exhaustive(): scores every candidate document (the baseline +// correctness oracle). +// - scoring_query_wand(): a block-max / WAND-style optimization that uses the +// per-window max_freq / max_norm columns from the frq_prelude to bound each +// window's best possible score and SKIP windows that cannot enter the +// current top-K. A window without block-max stats (slim/inline entries or a +// missing prelude) is never pruned, so the result still equals the +// exhaustive ranking. +// +// Results are sorted by score descending; ties are broken by ascending docid so +// the ordering is deterministic and the two paths compare equal. +namespace doris::snii::query { + +// One scored hit. +struct ScoredDoc { + uint32_t docid = 0; + double score = 0.0; +}; + +// One logical scoring clause after its plain term has been routed to this +// segment's physical key. IDF remains collection-scoped and is never derived +// from the physical term's segment-local dictionary entry. +struct CollectionScoringTerm { + std::string physical_term; + double idf = 0.0; +}; + +// Scores every document in final_candidates using collection-scoped IDF and +// avgdl plus segment-local TF/norm. Results are returned in ascending docid +// order. Repeated terms are repeated scoring clauses. This path deliberately +// does not use the segment-local WAND bounds below. +Status scoring_query_candidates(const reader::LogicalIndexReader& idx, + const stats::SniiStatsProvider& segment_stats, + const std::vector& terms, + const roaring::Roaring& final_candidates, double collection_avgdl, + const Bm25Params& params, std::vector* out); + +// Exhaustive baseline: score every doc that contains any query term, return the +// top-k by score. params controls k1/b. Unknown terms are skipped. +Status scoring_query_exhaustive(const reader::LogicalIndexReader& idx, + const stats::SniiStatsProvider& stats, + const std::vector& terms, uint32_t k, + const Bm25Params& params, std::vector* out); + +// WAND-style block-max pruning. MUST return the same top-k as the exhaustive +// path. Windows whose block-max upper bound cannot beat the current k-th score +// are skipped; windows lacking block-max stats are scored fully. +Status scoring_query_wand(const reader::LogicalIndexReader& idx, + const stats::SniiStatsProvider& stats, + const std::vector& terms, uint32_t k, + const Bm25Params& params, std::vector* out); + +// SELECTIVE-FETCH block-max WAND (design spec section 5, "Phase C"). Same WAND / +// theta / >= tie machinery as scoring_query_wand, but it DEFERS the .frq window +// fetch: for each windowed term it first reads ONLY the frq_prelude (block-max +// columns), then fetches a term's .frq window lazily and at most once -- and ONLY +// when the running block-max bound proves a doc in that window can still reach the +// top-K (bound >= theta). A window the bound rules out is never fetched. The +// result (top-K docids AND scores, INCLUDING ties) is byte-identical to +// scoring_query_exhaustive / scoring_query_wand; only the bytes read differ. +// Slim/inline terms (no prelude) are fetched fully, exactly as today. +Status scoring_query_wand_selective(const reader::LogicalIndexReader& idx, + const stats::SniiStatsProvider& stats, + const std::vector& terms, uint32_t k, + const Bm25Params& params, std::vector* out); + +} // namespace doris::snii::query diff --git a/be/src/storage/index/snii/query/sloppy_phrase_matcher.cpp b/be/src/storage/index/snii/query/sloppy_phrase_matcher.cpp new file mode 100644 index 00000000000000..d7e1ede9261c50 --- /dev/null +++ b/be/src/storage/index/snii/query/sloppy_phrase_matcher.cpp @@ -0,0 +1,266 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/query/internal/sloppy_phrase_matcher.h" + +#include +#include + +#include "common/check.h" + +namespace doris::snii::query::internal { + +SloppyPhraseMatcher::SloppyPhraseMatcher(std::span phrase_plan_index, + std::span position_offsets, uint32_t slop, + bool ordered) + : phrase_plan_index_(phrase_plan_index.begin(), phrase_plan_index.end()), + position_offsets_(position_offsets.begin(), position_offsets.end()), + slop_(slop), + ordered_(ordered), + clauses_(phrase_plan_index.size()) { + DORIS_CHECK_EQ(phrase_plan_index_.size(), position_offsets_.size()); + DORIS_CHECK_GT(phrase_plan_index_.size(), 1); + DORIS_CHECK_GT(slop_, 0); + heap_.reserve(phrase_plan_index_.size()); + for (size_t i = 0; i < phrase_plan_index_.size() && !has_repeats_; ++i) { + for (size_t j = 0; j < i; ++j) { + has_repeats_ = phrase_plan_index_[i] == phrase_plan_index_[j]; + if (has_repeats_) { + break; + } + } + } +} + +float SloppyPhraseMatcher::match(std::span positions, + bool collect_frequency) { + DCHECK_EQ(positions.size(), clauses_.size()); + return ordered_ ? match_ordered(positions, collect_frequency) + : match_unordered(positions, collect_frequency); +} + +bool SloppyPhraseMatcher::initialize_unordered(std::span positions) { + heap_.clear(); + end_ = std::numeric_limits::min(); + for (size_t i = 0; i < clauses_.size(); ++i) { + Clause& clause = clauses_[i]; + clause.positions = positions[i]; + DCHECK(clause.positions.first != clause.positions.second); + clause.raw_position = *clause.positions.first; + clause.next = clause.positions.first + 1; + clause.adjusted_position = static_cast(clause.raw_position) - position_offsets_[i]; + clause.has_position = true; + } + + if (has_repeats_) { + for (size_t i = 0; i < clauses_.size(); ++i) { + size_t preceding_repeats = 0; + for (size_t j = 0; j < i; ++j) { + preceding_repeats += phrase_plan_index_[i] == phrase_plan_index_[j]; + } + for (size_t repeat = 0; repeat < preceding_repeats; ++repeat) { + if (!advance_clause(i, false)) { + positioned_ = false; + return false; + } + } + } + } + + for (size_t i = 0; i < clauses_.size(); ++i) { + end_ = std::max(end_, clauses_[i].adjusted_position); + heap_.push_back(i); + } + rebuild_heap(); + positioned_ = true; + return true; +} + +bool SloppyPhraseMatcher::advance_clause(size_t clause_index, bool update_end) { + Clause& clause = clauses_[clause_index]; + if (clause.next == clause.positions.second) { + return false; + } + clause.raw_position = *clause.next++; + clause.adjusted_position = + static_cast(clause.raw_position) - position_offsets_[clause_index]; + if (update_end) { + end_ = std::max(end_, clause.adjusted_position); + } + return true; +} + +size_t SloppyPhraseMatcher::collision(size_t clause_index) const { + const Clause& clause = clauses_[clause_index]; + for (size_t i = 0; i < clauses_.size(); ++i) { + if (i != clause_index && phrase_plan_index_[i] == phrase_plan_index_[clause_index] && + clauses_[i].raw_position == clause.raw_position) { + return i; + } + } + return clauses_.size(); +} + +bool SloppyPhraseMatcher::clause_less(size_t left, size_t right) const { + const Clause& left_clause = clauses_[left]; + const Clause& right_clause = clauses_[right]; + if (left_clause.adjusted_position != right_clause.adjusted_position) { + return left_clause.adjusted_position < right_clause.adjusted_position; + } + if (position_offsets_[left] != position_offsets_[right]) { + return position_offsets_[left] < position_offsets_[right]; + } + return left < right; +} + +bool SloppyPhraseMatcher::clause_greater(size_t left, size_t right) const { + return left != right && !clause_less(left, right); +} + +bool SloppyPhraseMatcher::advance_repeat_collisions(size_t clause_index) { + size_t current = clause_index; + size_t other = collision(current); + while (other != clauses_.size()) { + current = clause_less(current, other) ? current : other; + if (!advance_clause(current, true)) { + return false; + } + other = collision(current); + } + rebuild_heap(); + return true; +} + +void SloppyPhraseMatcher::rebuild_heap() { + const auto greater = [this](size_t left, size_t right) { return clause_greater(left, right); }; + std::make_heap(heap_.begin(), heap_.end(), greater); +} + +size_t SloppyPhraseMatcher::pop_heap() { + const auto greater = [this](size_t left, size_t right) { return clause_greater(left, right); }; + std::pop_heap(heap_.begin(), heap_.end(), greater); + const size_t result = heap_.back(); + heap_.pop_back(); + return result; +} + +void SloppyPhraseMatcher::push_heap(size_t clause) { + const auto greater = [this](size_t left, size_t right) { return clause_greater(left, right); }; + heap_.push_back(clause); + std::push_heap(heap_.begin(), heap_.end(), greater); +} + +bool SloppyPhraseMatcher::next_unordered_match(uint64_t* match_width) { + if (!positioned_ || heap_.size() < 2) { + return false; + } + size_t clause = pop_heap(); + *match_width = static_cast(end_ - clauses_[clause].adjusted_position); + int64_t next_position = clauses_[heap_.front()].adjusted_position; + while (advance_clause(clause, true)) { + if (has_repeats_ && !advance_repeat_collisions(clause)) { + break; + } + if (clauses_[clause].adjusted_position > next_position) { + push_heap(clause); + if (*match_width <= slop_) { + return true; + } + clause = pop_heap(); + next_position = clauses_[heap_.front()].adjusted_position; + *match_width = static_cast(end_ - clauses_[clause].adjusted_position); + } else { + *match_width = std::min( + *match_width, static_cast(end_ - clauses_[clause].adjusted_position)); + } + } + positioned_ = false; + return *match_width <= slop_; +} + +float SloppyPhraseMatcher::match_unordered(std::span positions, + bool collect_frequency) { + if (!initialize_unordered(positions)) { + return 0.0F; + } + float frequency = 0.0F; + uint64_t match_width = 0; + while (next_unordered_match(&match_width)) { + if (!collect_frequency) { + return 1.0F; + } + frequency += 1.0F / (1.0F + static_cast(match_width)); + } + return frequency; +} + +bool SloppyPhraseMatcher::advance_ordered_to(size_t clause_index, int64_t target) { + Clause& clause = clauses_[clause_index]; + while (!clause.has_position || static_cast(clause.raw_position) < target) { + if (clause.next == clause.positions.second) { + return false; + } + clause.raw_position = *clause.next++; + clause.has_position = true; + } + return true; +} + +float SloppyPhraseMatcher::match_ordered(std::span positions, + bool collect_frequency) { + for (size_t i = 0; i < clauses_.size(); ++i) { + clauses_[i].positions = positions[i]; + clauses_[i].next = positions[i].first; + clauses_[i].has_position = false; + } + + float frequency = 0.0F; + Clause& first = clauses_.front(); + while (first.next != first.positions.second) { + first.raw_position = *first.next++; + first.has_position = true; + int64_t previous_start = + static_cast(first.raw_position) - position_offsets_.front(); + uint64_t match_width = 0; + bool all_terms_positioned = true; + for (size_t i = 1; i < clauses_.size(); ++i) { + const int64_t target = previous_start + position_offsets_[i]; + if (!advance_ordered_to(i, target)) { + all_terms_positioned = false; + break; + } + const int64_t current_start = + static_cast(clauses_[i].raw_position) - position_offsets_[i]; + DCHECK_GE(current_start, previous_start); + match_width += static_cast(current_start - previous_start); + if (match_width > slop_) { + all_terms_positioned = false; + break; + } + previous_start = current_start; + } + if (all_terms_positioned) { + if (!collect_frequency) { + return 1.0F; + } + frequency += 1.0F / (1.0F + static_cast(match_width)); + } + } + return frequency; +} + +} // namespace doris::snii::query::internal diff --git a/be/src/storage/index/snii/query/term_expansion.cpp b/be/src/storage/index/snii/query/term_expansion.cpp new file mode 100644 index 00000000000000..ea4b682df783bd --- /dev/null +++ b/be/src/storage/index/snii/query/term_expansion.cpp @@ -0,0 +1,168 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/query/internal/term_expansion.h" + +#include +#include +#include +#include + +#include "storage/index/inverted/common_grams/common_grams_key_codec.h" +#include "storage/index/snii/format/phrase_bigram.h" +#include "storage/index/snii/query/internal/docid_posting_reader.h" +#include "storage/index/snii/query/internal/docid_union.h" +#include "storage/index/snii/query/internal/plain_term_routing.h" +#include "storage/index/snii/reader/dict_block_cache.h" + +namespace doris::snii::query::internal { +namespace { + +Status legacy_raw_prefix_exists(const reader::LogicalIndexReader& idx, std::string_view prefix, + bool* exists, reader::DictBlockCache* cache) { + DORIS_CHECK(exists != nullptr); + *exists = false; + return idx.visit_prefix_terms( + prefix, + [&](reader::LogicalIndexReader::PrefixHit&&, bool* stop) -> Status { + *exists = true; + *stop = true; + return Status::OK(); + }, + cache); +} + +Status prove_legacy_raw_has_no_reserved_terms(const reader::LogicalIndexReader& idx, + reader::DictBlockCache* cache) { + bool exists = false; + RETURN_IF_ERROR(legacy_raw_prefix_exists(idx, segment_v2::inverted_index::CG_V1_MARKER, &exists, + cache)); + if (!exists) { + RETURN_IF_ERROR( + legacy_raw_prefix_exists(idx, format::kPhraseBigramTermMarker, &exists, cache)); + } + if (exists) { + return Status::Error( + "SNII legacy raw expansion overlaps an existing internal term namespace"); + } + return Status::OK(); +} + +} // namespace + +Status visit_expanded_plain_terms(const reader::LogicalIndexReader& idx, + std::string_view enum_prefix, const TermMatcher& matches, + const reader::LogicalIndexReader::PrefixHitVisitor& visitor, + int32_t max_expansions) { + if (!matches || !visitor) { + return Status::Error( + "term_expansion: null matcher or visitor"); + } + + std::string physical_prefix; + bool representable = false; + reader::DictBlockCache dict_cache(/*max_entries=*/1); + const auto version = plain_term_key_version(idx); + if (version == segment_v2::inverted_index::PlainTermKeyVersion::kLegacyRaw && + enum_prefix.empty()) { + RETURN_IF_ERROR(prove_legacy_raw_has_no_reserved_terms(idx, &dict_cache)); + representable = true; + } else { + RETURN_IF_ERROR( + route_plain_enumeration_prefix(idx, enum_prefix, &physical_prefix, &representable)); + } + if (!representable) { + return Status::OK(); + } + + int32_t count = 0; + bool stop_expansion = false; + std::string decoded_scratch; + const auto visit_hit = [&](reader::LogicalIndexReader::PrefixHit&& hit, bool* stop) -> Status { + std::string_view logical_term; + if (version != segment_v2::inverted_index::PlainTermKeyVersion::kEscapedV1 || + !hit.term.starts_with(segment_v2::inverted_index::PLAIN_ESCAPE_PREFIX)) { + if (version == segment_v2::inverted_index::PlainTermKeyVersion::kEscapedV1) { + DCHECK(!segment_v2::inverted_index::is_internal_term_key(hit.term)); + } + logical_term = hit.term; + decoded_scratch.clear(); + } else { + auto decoded = segment_v2::inverted_index::decode_plain_term_view(hit.term, version, + &decoded_scratch); + if (!decoded.has_value()) { + return std::move(decoded.error()); + } + logical_term = *decoded; + } + if (!matches(logical_term)) { + return Status::OK(); + } + if (!decoded_scratch.empty()) { + hit.term = decoded_scratch; + } + bool visitor_stop = false; + RETURN_IF_ERROR(visitor(std::move(hit), &visitor_stop)); + ++count; + *stop = visitor_stop || (max_expansions > 0 && count >= max_expansions); + stop_expansion = *stop; + return Status::OK(); + }; + + if (version == segment_v2::inverted_index::PlainTermKeyVersion::kEscapedV1 && + physical_prefix.empty()) { + RETURN_IF_ERROR(idx.visit_term_range( + /*lower_inclusive=*/ {}, segment_v2::inverted_index::INTERNAL_TERM_NAMESPACE_BEGIN, + visit_hit, &dict_cache)); + if (!stop_expansion && (max_expansions <= 0 || count < max_expansions)) { + RETURN_IF_ERROR( + idx.visit_term_range(segment_v2::inverted_index::INTERNAL_TERM_NAMESPACE_END, + /*upper_exclusive=*/std::nullopt, visit_hit, &dict_cache)); + } + } else { + RETURN_IF_ERROR(idx.visit_prefix_terms( + physical_prefix, + [&](reader::LogicalIndexReader::PrefixHit&& hit, bool* stop) -> Status { + if (version == segment_v2::inverted_index::PlainTermKeyVersion::kEscapedV1) { + DCHECK(!segment_v2::inverted_index::is_internal_term_key(hit.term)); + } + return visit_hit(std::move(hit), stop); + }, + &dict_cache)); + } + return Status::OK(); +} + +Status emit_expanded_docid_union(const reader::LogicalIndexReader& idx, + std::string_view enum_prefix, const TermMatcher& matches, + DocIdSink* const sink, int32_t max_expansions) { + if (sink == nullptr) { + return Status::Error("term_expansion: null sink"); + } + + std::vector postings; + RETURN_IF_ERROR(visit_expanded_plain_terms( + idx, enum_prefix, matches, + [&](reader::LogicalIndexReader::PrefixHit&& hit, bool*) { + postings.push_back({std::move(hit.entry), hit.frq_base, hit.prx_base}); + return Status::OK(); + }, + max_expansions)); + return emit_docid_union(idx, postings, sink); +} + +} // namespace doris::snii::query::internal diff --git a/be/src/storage/index/snii/query/term_query.cpp b/be/src/storage/index/snii/query/term_query.cpp new file mode 100644 index 00000000000000..633229eab67e97 --- /dev/null +++ b/be/src/storage/index/snii/query/term_query.cpp @@ -0,0 +1,58 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/query/term_query.h" + +#include + +#include "storage/index/snii/format/dict_entry.h" +#include "storage/index/snii/query/internal/docid_posting_reader.h" + +namespace doris::snii::query { + +using format::DictEntry; +using reader::LogicalIndexReader; + +Status term_query(const LogicalIndexReader& idx, std::string_view term, + std::vector* docids) { + if (docids == nullptr) + return Status::Error("term_query: null out"); + docids->clear(); + VectorDocIdSink sink(*docids); + return term_query(idx, term, &sink); +} + +Status term_query(const LogicalIndexReader& idx, std::string_view term, DocIdSink* sink) { + if (sink == nullptr) + return Status::Error("term_query: null sink"); + + bool found = false; + DictEntry entry; + uint64_t frq_base = 0; + uint64_t prx_base = 0; + RETURN_IF_ERROR(idx.lookup(term, &found, &entry, &frq_base, &prx_base)); + if (!found) return Status::OK(); + return internal::read_docid_posting(idx, entry, frq_base, prx_base, sink); +} + +Status term_query(const LogicalIndexReader& idx, std::string_view term, + std::vector* docids, QueryProfile* profile) { + QueryProfileScope profile_scope(idx.reader(), profile); + return term_query(idx, term, docids); +} + +} // namespace doris::snii::query diff --git a/be/src/storage/index/snii/query/term_query.h b/be/src/storage/index/snii/query/term_query.h new file mode 100644 index 00000000000000..7296acd9c60702 --- /dev/null +++ b/be/src/storage/index/snii/query/term_query.h @@ -0,0 +1,41 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/query/docid_sink.h" +#include "storage/index/snii/query/query_profile.h" +#include "storage/index/snii/reader/logical_index_reader.h" + +// term_query -- the simplest SNII query: return the sorted docid set that +// contains term. It runs the term lookup on the logical index, then issues a +// single batched .frq range read (one serial round) to decode the postings. +// Absent term -> empty result (OK status). +namespace doris::snii::query { + +Status term_query(const reader::LogicalIndexReader& idx, std::string_view term, + std::vector* docids); +Status term_query(const reader::LogicalIndexReader& idx, std::string_view term, DocIdSink* sink); +Status term_query(const reader::LogicalIndexReader& idx, std::string_view term, + std::vector* docids, QueryProfile* profile); + +} // namespace doris::snii::query diff --git a/be/src/storage/index/snii/query/wildcard_query.cpp b/be/src/storage/index/snii/query/wildcard_query.cpp new file mode 100644 index 00000000000000..e8313c499797aa --- /dev/null +++ b/be/src/storage/index/snii/query/wildcard_query.cpp @@ -0,0 +1,77 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/query/wildcard_query.h" + +#include +#include +#include +#include + +#include "storage/index/snii/query/internal/term_expansion.h" +#include "storage/index/snii/query/internal/wildcard_matcher.h" + +namespace doris::snii::query { + +namespace { + +std::string literal_prefix_for_wildcard(std::string_view pattern) { + std::string out; + for (char c : pattern) { + if (c == '*' || c == '?') { + break; + } + out.push_back(c); + } + return out; +} + +} // namespace + +Status wildcard_query(const reader::LogicalIndexReader& idx, std::string_view pattern, + std::vector* const docids, int32_t max_expansions) { + if (docids == nullptr) { + return Status::Error("wildcard_query: null out"); + } + docids->clear(); + VectorDocIdSink sink(*docids); + return wildcard_query(idx, pattern, &sink, max_expansions); +} + +Status wildcard_query(const reader::LogicalIndexReader& idx, std::string_view pattern, + std::vector* const docids, QueryProfile* profile, + int32_t max_expansions) { + QueryProfileScope profile_scope(idx.reader(), profile); + return wildcard_query(idx, pattern, docids, max_expansions); +} + +Status wildcard_query(const reader::LogicalIndexReader& idx, std::string_view pattern, + DocIdSink* const sink, int32_t max_expansions) { + if (sink == nullptr) { + return Status::Error("wildcard_query: null sink"); + } + const std::string enum_prefix = literal_prefix_for_wildcard(pattern); + // Request-scoped matcher: its two DP scratch rows are reused across every + // visited dictionary term, so the whole-dictionary scan triggered by a + // leading wildcard performs O(1) scratch allocations instead of O(2N). + internal::WildcardMatcher<> matcher(pattern); + return internal::emit_expanded_docid_union( + idx, enum_prefix, [&matcher](std::string_view term) { return matcher(term); }, sink, + max_expansions); +} + +} // namespace doris::snii::query diff --git a/be/src/storage/index/snii/query/wildcard_query.h b/be/src/storage/index/snii/query/wildcard_query.h new file mode 100644 index 00000000000000..66c08b18ae270e --- /dev/null +++ b/be/src/storage/index/snii/query/wildcard_query.h @@ -0,0 +1,42 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/query/docid_sink.h" +#include "storage/index/snii/query/query_profile.h" +#include "storage/index/snii/reader/logical_index_reader.h" + +// wildcard_query -- MATCH_WILDCARD semantics over dictionary terms. `*` matches +// any byte sequence, `?` matches one byte, and all other bytes match literally. +// Matching terms are executed as a sorted deduplicated docid union. +namespace doris::snii::query { + +Status wildcard_query(const reader::LogicalIndexReader& idx, std::string_view pattern, + std::vector* const docids, int32_t max_expansions = 0); +Status wildcard_query(const reader::LogicalIndexReader& idx, std::string_view pattern, + std::vector* const docids, QueryProfile* profile, + int32_t max_expansions = 0); +Status wildcard_query(const reader::LogicalIndexReader& idx, std::string_view pattern, + DocIdSink* const sink, int32_t max_expansions = 0); + +} // namespace doris::snii::query diff --git a/be/src/storage/index/snii/reader/dict_block_cache.h b/be/src/storage/index/snii/reader/dict_block_cache.h new file mode 100644 index 00000000000000..18c65978ee9263 --- /dev/null +++ b/be/src/storage/index/snii/reader/dict_block_cache.h @@ -0,0 +1,124 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/format/dict_block.h" + +// DictBlockCache -- a REQUEST-SCOPED (per-query) MRU cache of decoded DICT +// blocks, keyed by block ordinal. +// +// Why request-scoped (and not a reader-level shared cache): the same DICT block +// is decoded once per LogicalIndexReader::lookup() today, so a multi-term query +// (phrase / boolean / conjunction) whose terms fall in the same block re-runs +// the zstd decompress + CRC verify + anchor parse for every term. Threading one +// of these caches through a single query's lookups collapses that to a single +// decode per unique block. +// +// CONCURRENCY: this object carries NO shared mutable state and is intentionally +// NOT thread-safe. It is meant to live on one query's stack/context and be used +// by a single thread; concurrent queries each own a separate cache. The shared +// LogicalIndexReader therefore stays const and lock-free -- no lock is ever held +// across a decode/IO. (The cross-query, lock-striped variant that would let +// queries share decoded blocks is deferred to the T26 concurrency work.) +namespace doris::snii::reader { + +// A decoded DICT block with stable backing storage. Heap-allocated and owned by +// a shared_ptr so the embedded DictBlockReader's Slice into `bytes` stays valid +// for the whole lifetime of any pin handed to a caller -- even after the block +// has been evicted from the cache. +struct DecodedDictBlock { + std::vector bytes; // decompressed (or raw) block bytes + format::DictBlockReader reader; // its Slice points into `bytes` +}; + +class DictBlockCache { +public: + // Loads (decodes) the block for an ordinal into a freshly heap-allocated + // DecodedDictBlock. Always invoked OUTSIDE any cache bookkeeping; it performs + // the file read + optional zstd decompress + CRC/anchor parse. + using Loader = std::function*)>; + + // A small fixed bound is enough for a single query: it only needs to keep the + // handful of distinct blocks touched while resolving one query's terms. + static constexpr size_t kDefaultMaxEntries = 8; + + DictBlockCache() = default; + explicit DictBlockCache(size_t max_entries) + : max_entries_(max_entries == 0 ? 1 : max_entries) {} + + // Returns the decoded block for `ordinal`, invoking `loader` only on a miss. + // The returned pin keeps the block alive for the caller's use regardless of + // any later eviction. On a hit, `loader` is not called (no re-decode). + Status get_or_load(uint32_t ordinal, const Loader& loader, + std::shared_ptr* out) { + if (auto it = index_.find(ordinal); it != index_.end()) { + order_.splice(order_.begin(), order_, it->second); // promote to MRU + *out = it->second->block; + return Status::OK(); + } + + std::shared_ptr loaded; + // decode happens here, never under a lock (explicit Status, header-safe: + // RETURN_IF_ERROR would need a bare `Status` in scope). + if (Status st = loader(&loaded); !st.ok()) { + return st; + } + order_.push_front(Entry {.ordinal = ordinal, .block = loaded}); + index_[ordinal] = order_.begin(); + evict_overflow(); + *out = std::move(loaded); + return Status::OK(); + } + + // Number of resident (non-evicted) entries -- bounded by max_entries(). + size_t size() const { return index_.size(); } + size_t max_entries() const { return max_entries_; } + +private: + struct Entry { + uint32_t ordinal = 0; + std::shared_ptr block; + }; + + // Drops least-recently-used entries until the bound holds. Evicting only + // releases the cache's reference; any pin a caller still holds keeps the + // block (and its reader's Slice) alive. + void evict_overflow() { + while (index_.size() > max_entries_) { + const Entry& victim = order_.back(); + index_.erase(victim.ordinal); + order_.pop_back(); + } + } + + size_t max_entries_ = kDefaultMaxEntries; + std::list order_; // front = most recently used + std::unordered_map::iterator> index_; +}; + +} // namespace doris::snii::reader diff --git a/be/src/storage/index/snii/reader/logical_index_reader.cpp b/be/src/storage/index/snii/reader/logical_index_reader.cpp new file mode 100644 index 00000000000000..7b14e509e88754 --- /dev/null +++ b/be/src/storage/index/snii/reader/logical_index_reader.cpp @@ -0,0 +1,1106 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/reader/logical_index_reader.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "storage/index/snii/encoding/byte_source.h" +#include "storage/index/snii/encoding/crc32c.h" +#include "storage/index/snii/encoding/section_framer.h" +#include "storage/index/snii/encoding/varint.h" +#include "storage/index/snii/encoding/zstd_codec.h" +#include "storage/index/snii/format/dict_block.h" +#include "storage/index/snii/format/dict_block_directory.h" +#include "storage/index/snii/format/metadata_blob.h" +#include "storage/index/snii/format/norms_pod.h" +#include "storage/index/snii/format/null_bitmap.h" +#include "storage/index/snii/io/batch_range_fetcher.h" +#include "storage/index/snii/reader/dict_block_cache.h" + +namespace doris::snii::reader { + +struct LogicalIndexReader::NormsCacheState { + struct Data { + std::vector bytes; + format::NormsPodReader reader; + }; + + std::mutex mutex; + std::unique_ptr ready; + std::shared_future in_flight; +}; + +using format::BlockRef; +using format::bsbf_hash; +using format::DictBlockDirectoryReader; +using format::DictBlockReader; +using format::DictEntry; +using format::IndexTier; +using format::kBsbfBytesPerBlock; +using format::kBsbfHeaderSize; +using format::RegionRef; +using format::SampledTermIndexReader; + +namespace { +constexpr uint64_t kMaxDictBlockUncompBytes = 256ULL * 1024 * 1024; +constexpr uint64_t kDefaultDictResidentMaxBytes = 256ULL * 1024; +constexpr size_t kMaxDictLookupBatchRuns = 16; +constexpr uint64_t kMaxDictLookupBatchBytes = 4ULL * 1024 * 1024; +// Conservatively covers make_shared's control block and allocator bookkeeping +// for the state/data/vector allocations. The framed norms bytes are charged +// separately at their exact validated length. +constexpr size_t kNormsAllocationOverheadCharge = 128; + +// L0/L1 tiering threshold (bytes). Defaults to kBsbfResidentMaxBytes; the env +// SNII_BSBF_RESIDENT_MAX overrides it for tuning and for exercising the +// on-demand L1 path in tests without a 250K-term corpus. Read fresh each open. +uint64_t bsbf_resident_max_bytes() { + const char* s = std::getenv("SNII_BSBF_RESIDENT_MAX"); + if (s != nullptr) { + char* end = nullptr; + const unsigned long long v = std::strtoull(s, &end, 10); + if (end != s) { + return v; + } + } + return format::kBsbfResidentMaxBytes; +} + +uint64_t dict_resident_max_bytes() { + const char* s = std::getenv("SNII_DICT_RESIDENT_MAX"); + if (s != nullptr) { + char* end = nullptr; + const unsigned long long v = std::strtoull(s, &end, 10); + if (end != s) { + return v; + } + } + return kDefaultDictResidentMaxBytes; +} + +Status checked_size(uint64_t value, const char* error, size_t* out) { + if (value > std::numeric_limits::max()) { + return Status::Error(error); + } + *out = static_cast(value); + return Status::OK(); +} + +Status validate_norms_region(io::FileReader* reader, const RegionRef& norms, uint64_t doc_count, + size_t* length) { + *length = 0; + if (norms.length == 0) { + return Status::OK(); + } + const uint64_t file_size = reader->size(); + if (norms.offset > file_size || norms.length > file_size - norms.offset) { + return Status::Error( + "logical_index: norms region past end of file"); + } + if (doc_count > std::numeric_limits::max()) { + return Status::Error( + "logical_index: norms doc count exceeds uint32"); + } + const uint64_t payload_length = varint_len(doc_count) + doc_count; + const uint64_t expected_length = + 1 + varint_len(payload_length) + payload_length + sizeof(uint32_t); + if (norms.length != expected_length) { + return Status::Error( + "logical_index: norms region length mismatch"); + } + if (norms.length > std::numeric_limits::max()) { + return Status::Error( + "logical_index: norms region exceeds cache charge bounds"); + } + *length = static_cast(norms.length); + return Status::OK(); +} + +Status validate_null_bitmap_frame(Slice framed) { + ByteSource source(framed); + FramedSection section; + RETURN_IF_ERROR(SectionFramer::read(source, §ion)); + if (section.type != format::kNullBitmapSectionType) { + return Status::Error( + "logical_index: invalid null bitmap section type"); + } + if (source.remaining() != 0) { + return Status::Error( + "logical_index: trailing null bitmap frame bytes"); + } + + // NullBitmapReader owns the Roaring validation. Parse only the two prefix + // varints here to pin the region to exactly one complete payload; otherwise + // a valid frame could silently carry ignored bytes after roaring_bytes. + ByteSource payload(section.payload); + uint64_t doc_count = 0; + RETURN_IF_ERROR(payload.get_varint64(&doc_count)); + if (doc_count > std::numeric_limits::max()) { + return Status::Error( + "logical_index: null bitmap doc count exceeds uint32"); + } + uint64_t roaring_size = 0; + RETURN_IF_ERROR(payload.get_varint64(&roaring_size)); + if (roaring_size != payload.remaining()) { + return Status::Error( + "logical_index: null bitmap payload length mismatch"); + } + return Status::OK(); +} + +Status dict_block_memory_bytes(const BlockRef& ref, uint64_t* out) { + if ((ref.flags & format::block_ref_flags::kZstd) == 0) { + *out = ref.length; + return Status::OK(); + } + if (ref.uncomp_len == 0 || ref.uncomp_len > kMaxDictBlockUncompBytes) { + return Status::Error( + "dict block: zstd uncomp_len out of range"); + } + *out = ref.uncomp_len; + return Status::OK(); +} + +Status checked_memory_add(uint64_t lhs, uint64_t rhs, const char* message, uint64_t* out) { + if (rhs > std::numeric_limits::max() - lhs) { + return Status::Error(message); + } + *out = lhs + rhs; + return Status::OK(); +} + +Status checked_memory_mul(uint64_t lhs, uint64_t rhs, const char* message, uint64_t* out) { + if (lhs != 0 && rhs > std::numeric_limits::max() / lhs) { + return Status::Error(message); + } + *out = lhs * rhs; + return Status::OK(); +} + +// Decompresses a zstd dict block from its on-disk bytes into *out. +Status zstd_decompress_dict_block(Slice on_disk, const BlockRef& ref, std::vector* out) { + uint64_t memory_bytes = 0; + RETURN_IF_ERROR(dict_block_memory_bytes(ref, &memory_bytes)); + size_t uncomp_len = 0; + RETURN_IF_ERROR( + checked_size(memory_bytes, "dict block: zstd length out of range", &uncomp_len)); + return zstd_decompress(on_disk, uncomp_len, out); +} + +// Materializes the usable (uncompressed) bytes of a dict block from a view over +// its on-disk bytes -- a raw block is copied, a zstd block is decompressed. Used +// by the resident single-range path, where on_disk is a sub-slice of the shared +// region buffer (so a raw block must be copied, not aliased). +Status decompress_dict_block_payload(Slice on_disk, const BlockRef& ref, + std::vector* out) { + if ((ref.flags & format::block_ref_flags::kZstd) == 0) { + out->assign(on_disk.data(), on_disk.data() + on_disk.size()); + return Status::OK(); + } + return zstd_decompress_dict_block(on_disk, ref, out); +} + +Status read_dict_block_bytes(io::FileReader* reader, const BlockRef& ref, + std::vector* out) { + size_t read_len = 0; + RETURN_IF_ERROR(checked_size(ref.length, "dict block: on-disk length out of range", &read_len)); + + std::vector block_bytes; + RETURN_IF_ERROR(reader->read_at(ref.offset, read_len, &block_bytes)); + if (block_bytes.size() != read_len) { + return Status::Error( + "dict block: short read"); + } + + // Raw on-demand block: move the freshly read buffer in (no copy). + if ((ref.flags & format::block_ref_flags::kZstd) == 0) { + *out = std::move(block_bytes); + return Status::OK(); + } + return zstd_decompress_dict_block(Slice(block_bytes), ref, out); +} + +Status open_dict_block(io::FileReader* reader, const BlockRef& ref, IndexTier tier, + bool has_positions, std::vector* bytes, DictBlockReader* out) { + RETURN_IF_ERROR(read_dict_block_bytes(reader, ref, bytes)); + return DictBlockReader::open(Slice(*bytes), tier, has_positions, out); +} + +// Validates that block `ref` lies fully within dict_region and returns its byte +// range relative to the start of the region. Defends the single-range resident +// read against a corrupt directory ref (offset before the region, or a range +// that runs past it) before it is used to index the region buffer. +Status slice_dict_block_in_region(const BlockRef& ref, const RegionRef& dict_region, + size_t region_len, size_t* rel_off, size_t* len) { + if (ref.offset < dict_region.offset) { + return Status::Error( + "dict block: ref before dict region"); + } + size_t rel = 0; + size_t block_len = 0; + RETURN_IF_ERROR( + checked_size(ref.offset - dict_region.offset, "dict block: ref offset OOR", &rel)); + RETURN_IF_ERROR(checked_size(ref.length, "dict block: ref length OOR", &block_len)); + if (rel > region_len || block_len > region_len - rel) { + return Status::Error( + "dict block: ref past dict region"); + } + *rel_off = rel; + *len = block_len; + return Status::OK(); +} +} // namespace + +Status LogicalIndexReader::load_resident_dict_blocks() { + resident_dict_blocks_.clear(); + + const uint64_t max_bytes = dict_resident_max_bytes(); + if (max_bytes == 0 || dbd_.n_blocks() == 0) { + return Status::OK(); + } + + uint64_t total_bytes = 0; + for (uint32_t ord = 0; ord < dbd_.n_blocks(); ++ord) { + BlockRef ref {}; + RETURN_IF_ERROR(dbd_.get(ord, &ref)); + uint64_t block_bytes = 0; + RETURN_IF_ERROR(dict_block_memory_bytes(ref, &block_bytes)); + if (block_bytes > max_bytes - total_bytes) { + return Status::OK(); + } + total_bytes += block_bytes; + } + + // The resident blocks are physically contiguous within dict_region, so read + // the whole region in a SINGLE range read (was one read_at per block -> up to + // ~4 serial S3 rounds on a cold open) and decode each block from a sub-slice. + // The region buffer is <= the resident byte cap (<=256KB) and freed on return; + // each ResidentDictBlock keeps its own decoded copy. + const RegionRef& dict_region = section_refs().dict_region; + size_t region_len = 0; + RETURN_IF_ERROR( + checked_size(dict_region.length, "dict region: length out of range", ®ion_len)); + std::vector region; + RETURN_IF_ERROR(reader_->read_at(dict_region.offset, region_len, ®ion)); + if (region.size() != region_len) { + return Status::Error( + "dict region: short read"); + } + + resident_dict_blocks_.reserve(dbd_.n_blocks()); + for (uint32_t ord = 0; ord < dbd_.n_blocks(); ++ord) { + BlockRef ref {}; + RETURN_IF_ERROR(dbd_.get(ord, &ref)); + size_t rel_off = 0; + size_t block_len = 0; + RETURN_IF_ERROR( + slice_dict_block_in_region(ref, dict_region, region_len, &rel_off, &block_len)); + const Slice on_disk(region.data() + rel_off, block_len); + ResidentDictBlock block; + RETURN_IF_ERROR(decompress_dict_block_payload(on_disk, ref, &block.bytes)); + RETURN_IF_ERROR( + DictBlockReader::open(Slice(block.bytes), tier_, has_positions_, &block.reader)); + resident_dict_blocks_.push_back(std::move(block)); + } + return Status::OK(); +} + +Status LogicalIndexReader::dict_block_reader_for_ordinal( + uint32_t ordinal, DictBlockCache* cache, std::shared_ptr* pin, + const DictBlockReader** out) const { + pin->reset(); + if (!resident_dict_blocks_.empty()) { + if (resident_dict_blocks_.size() != dbd_.n_blocks() || + ordinal >= resident_dict_blocks_.size()) { + return Status::Error( + "logical_index: incomplete resident dict"); + } + // Resident blocks live for the reader lifetime: no pin needed. + *out = &resident_dict_blocks_[ordinal].reader; + return Status::OK(); + } + + // On-demand: decode into a heap-allocated DecodedDictBlock held by *pin so the + // reader's Slice never dangles. The loader (file read + optional zstd + CRC + + // anchor parse) runs OUTSIDE any cache bookkeeping; on a cache hit it is not + // called, so a block shared by several terms of one query decodes only once. + DictBlockCache::Loader loader = [&](std::shared_ptr* slot) -> Status { + BlockRef ref {}; + RETURN_IF_ERROR(dbd_.get(ordinal, &ref)); + auto block = std::make_shared(); + RETURN_IF_ERROR(open_dict_block(reader_, ref, tier_, has_positions_, &block->bytes, + &block->reader)); + *slot = std::move(block); + return Status::OK(); + }; + if (cache != nullptr) { + RETURN_IF_ERROR(cache->get_or_load(ordinal, loader, pin)); + } else { + RETURN_IF_ERROR(loader(pin)); + } + *out = &(*pin)->reader; + return Status::OK(); +} + +Status LogicalIndexReader::load_resident_bsbf() { + // Block-split bloom XFilter -- gated on RESIDENCY (P1 cold-read fix, see + // docs/perf/P1-cold-read-amplification.md). The bloom is set up and used ONLY + // when the whole (small) filter fits under the resident cap: it is read in + // full, verified, and kept in memory so probes are in-memory and enter the + // Doris searcher cache with the rest of the logical-index metadata. + // + // When NON-resident (the common case for a real text column, where the filter + // is many MB) the bloom is skipped ENTIRELY: not even the 28B header is read, + // and has_bsbf_ stays false. Every term then falls through to sti -> dict, + // which yields the true found/absent. At 1 MiB cache-block granularity a + // non-resident bloom never saves a physical block (an absent term still costs + // one dict block either way), so its 28B header + per-term 32B probes were pure + // cold read amplification. + const RegionRef& bsbf = core_.section_refs.bsbf; + if (open_mode_ != LogicalIndexOpenMode::kQuery || bsbf.length == 0 || + bsbf.length > bsbf_resident_max_bytes()) { + return Status::OK(); + } + if (bsbf.length <= kBsbfHeaderSize) { + return Status::Error( + "logical_index: bsbf section too small"); + } + const uint64_t num_bytes = bsbf.length - kBsbfHeaderSize; + std::vector head; + RETURN_IF_ERROR(reader_->read_at(bsbf.offset, bsbf.length, &head)); + if (head.size() < bsbf.length) { + return Status::Error( + "logical_index: short bsbf resident read"); + } + RETURN_IF_ERROR(format::BsbfHeader::parse(Slice(head.data(), kBsbfHeaderSize), bsbf.offset, + &bsbf_header_)); + // Cross-check the header geometry against the section ref. + if (bsbf_header_.num_bytes != num_bytes) { + return Status::Error( + "logical_index: bsbf header/section size mismatch"); + } + const Slice bitset(head.data() + kBsbfHeaderSize, bsbf_header_.num_bytes); + if (crc32c(bitset) != bsbf_header_.bitset_crc) { + return Status::Error( + "logical_index: bsbf bitset crc mismatch"); + } + bsbf_resident_bitset_.assign(bitset.data(), bitset.data() + bitset.size()); + has_bsbf_ = true; + bsbf_resident_ = true; + return Status::OK(); +} + +Status LogicalIndexReader::open(io::FileReader* file_reader, Slice core_frame, Slice sti_blob, + Slice dbd_blob, LogicalIndexReader* out, + LogicalIndexOpenMode open_mode) { + if (out == nullptr) { + return Status::Error("logical_index: null out"); + } + *out = LogicalIndexReader {}; + if (file_reader == nullptr) { + return Status::Error("logical_index: null file reader"); + } + if (core_frame.empty() || sti_blob.empty() || dbd_blob.empty()) { + return Status::Error( + "logical_index: empty mandatory metadata blob"); + } + + LogicalIndexReader candidate; + candidate.reader_ = file_reader; + candidate.open_mode_ = open_mode; + RETURN_IF_ERROR(format::decode_core_metadata(core_frame, &candidate.core_)); + candidate.tier_ = format::tier_of(candidate.core_.index_config); + candidate.has_positions_ = format::has_positions(candidate.core_.index_config); + size_t norms_length = 0; + RETURN_IF_ERROR(validate_norms_region(file_reader, candidate.core_.section_refs.norms, + candidate.core_.stats.doc_count, &norms_length)); + if (norms_length != 0) { + constexpr size_t fixed_charge = sizeof(NormsCacheState) + sizeof(NormsCacheState::Data) + + kNormsAllocationOverheadCharge; + if (norms_length > std::numeric_limits::max() - fixed_charge) { + return Status::Error( + "logical_index: norms cache charge overflow"); + } + candidate.norms_reserved_charge_ = fixed_charge + norms_length; + candidate.norms_cache_ = std::make_shared(); + } + // Raw frames alias the group read and compressed carriers materialize into + // transient scratch. Both readers own their decoded state. + { + std::vector scratch; + Slice frame; + RETURN_IF_ERROR(format::materialize_metadata_blob( + sti_blob, format::SectionType::kSampledTermIndex, + format::SectionType::kSampledTermIndexZstd, &scratch, &frame)); + RETURN_IF_ERROR(SampledTermIndexReader::open(frame, &candidate.sti_)); + } + { + std::vector scratch; + Slice frame; + RETURN_IF_ERROR(format::materialize_metadata_blob( + dbd_blob, format::SectionType::kDictBlockDirectory, + format::SectionType::kDictBlockDirectoryZstd, &scratch, &frame)); + RETURN_IF_ERROR(DictBlockDirectoryReader::open(frame, &candidate.dbd_)); + } + if (candidate.sti_.n_blocks() != candidate.dbd_.n_blocks()) { + return Status::Error( + "logical_index: sampled-term index and block directory count mismatch"); + } + if (open_mode == LogicalIndexOpenMode::kQuery) { + RETURN_IF_ERROR(candidate.load_resident_dict_blocks()); + } + RETURN_IF_ERROR(candidate.load_resident_bsbf()); + *out = std::move(candidate); + return Status::OK(); +} + +size_t LogicalIndexReader::memory_usage() const { + size_t bytes = sizeof(*this) + bsbf_resident_bitset_.capacity(); + if (core_.common_grams_metadata) { + const auto& common_grams = *core_.common_grams_metadata; + bytes += format::std_string_heap_bytes(common_grams.common_grams_dictionary_identity); + bytes += format::std_string_heap_bytes(common_grams.base_analyzer_fingerprint); + bytes += format::std_string_heap_bytes(common_grams.common_grams_fingerprint); + } + bytes += sti_.heap_bytes(); + bytes += dbd_.heap_bytes(); + for (const auto& block : resident_dict_blocks_) { + bytes += sizeof(block) + block.bytes.capacity() + block.reader.heap_bytes(); + } + // Norms are loaded lazily, but the searcher-cache charge is fixed when this + // reader is inserted. Reserve their complete framed size and heap state up + // front so later allocation is already accounted for. Saturation prevents + // an already-large resident metadata charge from wrapping around. + if (norms_reserved_charge_ > std::numeric_limits::max() - bytes) { + return std::numeric_limits::max(); + } + bytes += norms_reserved_charge_; + return bytes; +} + +Status LogicalIndexReader::open_norms(format::NormsPodReader* out) const { + if (out == nullptr) { + return Status::Error( + "logical_index: null norms reader"); + } + if (reader_ == nullptr) { + return Status::Error("logical_index: not opened"); + } + const RegionRef& norms = section_refs().norms; + if (norms.length == 0) { + return Status::Error( + "logical_index: index has no norms"); + } + DORIS_CHECK(norms_cache_ != nullptr); + + std::shared_future in_flight; + std::shared_ptr> completion; + { + std::lock_guard lock(norms_cache_->mutex); + if (norms_cache_->ready != nullptr) { + *out = norms_cache_->ready->reader; + return Status::OK(); + } + if (norms_cache_->in_flight.valid()) { + in_flight = norms_cache_->in_flight; + } else { + completion = std::make_shared>(); + in_flight = completion->get_future().share(); + norms_cache_->in_flight = in_flight; + } + } + + if (completion == nullptr) { + const Status status = in_flight.get(); + RETURN_IF_ERROR(status); + std::lock_guard lock(norms_cache_->mutex); + DORIS_CHECK(norms_cache_->ready != nullptr); + *out = norms_cache_->ready->reader; + return Status::OK(); + } + + auto data = std::make_unique(); + const size_t read_len = static_cast(norms.length); + Status status = reader_->read_at(norms.offset, read_len, &data->bytes); + if (status.ok() && data->bytes.size() != read_len) { + status = Status::Error( + "logical_index: short norms read"); + } + if (status.ok()) { + status = format::NormsPodReader::open(Slice(data->bytes), &data->reader); + } + { + std::lock_guard lock(norms_cache_->mutex); + if (status.ok()) { + norms_cache_->ready = std::move(data); + *out = norms_cache_->ready->reader; + } + norms_cache_->in_flight = std::shared_future {}; + } + completion->set_value(status); + return status; +} + +void LogicalIndexReader::release_compaction_norms() const { + DORIS_CHECK(open_mode_ == LogicalIndexOpenMode::kCompaction); + DORIS_CHECK(norms_cache_ != nullptr); + std::lock_guard lock(norms_cache_->mutex); + DORIS_CHECK(!norms_cache_->in_flight.valid()); + norms_cache_->ready.reset(); +} + +Status LogicalIndexReader::read_null_docids( + std::vector* out, const NullDocidsDecodeReservation& reserve_decode) const { + if (out == nullptr) { + return Status::Error( + "logical_index: null null-docids output"); + } + out->clear(); + if (reader_ == nullptr) { + return Status::Error("logical_index: not opened"); + } + + const RegionRef& ref = section_refs().null_bitmap; + if (ref.length == 0) { + if (stats().null_count != 0) { + return Status::Error( + "logical_index: null bitmap section missing"); + } + return Status::OK(); + } + + const uint64_t file_size = reader_->size(); + if (ref.offset > file_size || ref.length > file_size - ref.offset) { + return Status::Error( + "logical_index: null bitmap region past end of file"); + } + size_t read_len = 0; + RETURN_IF_ERROR( + checked_size(ref.length, "logical_index: null bitmap length out of range", &read_len)); + std::vector bytes; + RETURN_IF_ERROR(reader_->read_at(ref.offset, read_len, &bytes)); + if (bytes.size() != read_len) { + return Status::Error( + "logical_index: short null bitmap read"); + } + RETURN_IF_ERROR(validate_null_bitmap_frame(Slice(bytes))); + + uint64_t decoded_memory_bytes = 0; + RETURN_IF_ERROR( + format::NullBitmapReader::decoded_memory_bytes(Slice(bytes), &decoded_memory_bytes)); + if (reserve_decode) { + RETURN_IF_ERROR(reserve_decode(decoded_memory_bytes)); + } + + format::NullBitmapReader null_bitmap_reader; + RETURN_IF_ERROR(format::NullBitmapReader::open(Slice(bytes), &null_bitmap_reader)); + if (null_bitmap_reader.doc_count() != stats().doc_count) { + return Status::Error( + "logical_index: null bitmap doc count mismatch"); + } + if (null_bitmap_reader.null_count() != stats().null_count) { + return Status::Error( + "logical_index: null bitmap cardinality mismatch"); + } + + out->reserve(null_bitmap_reader.null_count()); + null_bitmap_reader.append_docids(*out); + for (uint32_t docid : *out) { + if (docid >= stats().doc_count) { + out->clear(); + return Status::Error( + "logical_index: null docid outside document domain"); + } + } + return Status::OK(); +} + +Status LogicalIndexReader::null_docids_scan_memory(NullDocidsScanMemory* out) const { + if (out == nullptr) { + return Status::Error( + "logical_index: null null-docids scan memory out"); + } + *out = NullDocidsScanMemory {}; + if (reader_ == nullptr) { + return Status::Error("logical_index: not opened"); + } + const RegionRef& ref = section_refs().null_bitmap; + if (ref.length == 0) { + if (stats().null_count != 0) { + return Status::Error( + "logical_index: null bitmap section missing"); + } + return Status::OK(); + } + const uint64_t file_size = reader_->size(); + if (ref.offset > file_size || ref.length > file_size - ref.offset) { + return Status::Error( + "logical_index: null bitmap region past end of file"); + } + size_t frame_bytes = 0; + RETURN_IF_ERROR(checked_size(ref.length, "logical_index: null bitmap length out of range", + &frame_bytes)); + RETURN_IF_ERROR(checked_memory_mul(stats().null_count, sizeof(uint32_t), + "logical_index: null docids output memory overflows", + &out->output_bytes)); + + out->frame_bytes = frame_bytes; + return Status::OK(); +} + +Status LogicalIndexReader::lookup(std::string_view term, bool* found, DictEntry* entry, + uint64_t* frq_base, uint64_t* prx_base, + DictBlockCache* cache) const { + *found = false; + if (reader_ == nullptr) { + return Status::Error("logical_index: not opened"); + } + + bool maybe = false; + uint32_t ordinal = 0; + RETURN_IF_ERROR(locate_candidate_dict_block(term, &maybe, &ordinal)); + if (!maybe) { + return Status::OK(); + } + + // Use a resident small-DICT block when present; otherwise read the DICT + // block on demand and parse it with the same validation path used at open. + // `pin` keeps an on-demand block alive through find_term (resident: null). + const DictBlockReader* br = nullptr; + std::shared_ptr pin; + RETURN_IF_ERROR(dict_block_reader_for_ordinal(ordinal, cache, &pin, &br)); + + bool hit = false; + RETURN_IF_ERROR(br->find_term(term, &hit, entry)); + if (!hit) { + return Status::OK(); + } + + *found = true; + *frq_base = br->frq_base(); + *prx_base = br->prx_base(); + return Status::OK(); +} + +Status LogicalIndexReader::locate_candidate_dict_block(std::string_view term, bool* maybe_present, + uint32_t* ordinal) const { + *maybe_present = false; + // A DEFINITELY-ABSENT term returns without a DICT read. The bloom is + // consulted only when resident; otherwise STI/DICT remains authoritative. + if (has_bsbf_) { + const uint64_t h = bsbf_hash(term); + bool maybe = true; + if (bsbf_resident_) { + const uint32_t blk = format::bsbf_block_index(h, bsbf_header_.num_blocks); + maybe = format::bsbf_block_contains( + h, + bsbf_resident_bitset_.data() + static_cast(blk) * kBsbfBytesPerBlock); + } + if (!maybe) { + return Status::OK(); + } + } + return sti_.locate(term, maybe_present, ordinal); +} + +Status LogicalIndexReader::collect_batch_lookup_groups( + const std::vector& terms, std::vector* candidates, + std::vector* groups) const { + candidates->clear(); + candidates->reserve(terms.size()); + for (size_t i = 0; i < terms.size(); ++i) { + bool maybe = false; + uint32_t ordinal = 0; + RETURN_IF_ERROR(locate_candidate_dict_block(terms[i], &maybe, &ordinal)); + if (maybe) { + candidates->push_back({i, ordinal}); + } + } + + auto by_ordinal = [](const BatchLookupCandidate& lhs, const BatchLookupCandidate& rhs) { + return lhs.ordinal < rhs.ordinal; + }; + if (!std::ranges::is_sorted(*candidates, by_ordinal)) { + std::ranges::sort(*candidates, by_ordinal); + } + + groups->clear(); + for (size_t begin = 0; begin < candidates->size();) { + const uint32_t ordinal = (*candidates)[begin].ordinal; + size_t end = begin + 1; + while (end < candidates->size() && (*candidates)[end].ordinal == ordinal) { + ++end; + } + groups->push_back({ordinal, begin, end}); + begin = end; + } + return Status::OK(); +} + +Status LogicalIndexReader::resolve_batch_lookup_group( + const std::vector& terms, const std::vector& candidates, + const BatchLookupGroup& group, const DictBlockReader& block_reader, + std::vector* results) { + for (size_t i = group.begin; i < group.end; ++i) { + const size_t term_index = candidates[i].term_index; + BatchLookupResult& result = (*results)[term_index]; + RETURN_IF_ERROR(block_reader.find_term(terms[term_index], &result.found, &result.entry)); + if (result.found) { + result.frq_base = block_reader.frq_base(); + result.prx_base = block_reader.prx_base(); + } + } + return Status::OK(); +} + +Status LogicalIndexReader::lookup_batch_on_demand( + const std::vector& terms, const std::vector& candidates, + const std::vector& groups, + std::vector* results) const { + for (size_t wave_begin = 0; wave_begin < groups.size();) { + std::vector pending; + pending.reserve(kMaxDictLookupBatchRuns); + uint64_t pending_bytes = 0; + uint64_t pending_end = 0; + size_t pending_runs = 0; + size_t wave_end = wave_begin; + while (wave_end < groups.size()) { + BlockRef ref {}; + RETURN_IF_ERROR(dbd_.get(groups[wave_end].ordinal, &ref)); + const uint64_t ref_end = ref.offset + ref.length; + const bool starts_new_run = pending.empty() || ref.offset > pending_end; + if (!pending.empty() && ((starts_new_run && pending_runs == kMaxDictLookupBatchRuns) || + ref.length > kMaxDictLookupBatchBytes || + pending_bytes > kMaxDictLookupBatchBytes - ref.length)) { + break; + } + pending.push_back({wave_end, ref, 0}); + pending_bytes += ref.length; + if (starts_new_run) { + ++pending_runs; + } + pending_end = std::max(pending_end, ref_end); + ++wave_end; + } + DORIS_CHECK(!pending.empty()); + io::BatchRangeFetcher fetcher(reader_, /*coalesce_gap=*/0); + for (PendingBatchLookupBlock& block : pending) { + block.handle = fetcher.add(block.ref.offset, block.ref.length); + } + RETURN_IF_ERROR(fetcher.fetch()); + + for (const PendingBatchLookupBlock& block : pending) { + const Slice on_disk = fetcher.get(block.handle); + std::vector decoded; + Slice payload = on_disk; + if ((block.ref.flags & format::block_ref_flags::kZstd) != 0) { + RETURN_IF_ERROR(zstd_decompress_dict_block(on_disk, block.ref, &decoded)); + payload = Slice(decoded); + } + DictBlockReader block_reader; + RETURN_IF_ERROR(DictBlockReader::open(payload, tier_, has_positions_, &block_reader)); + RETURN_IF_ERROR(resolve_batch_lookup_group(terms, candidates, groups[block.group_index], + block_reader, results)); + } + wave_begin = wave_end; + } + return Status::OK(); +} + +Status LogicalIndexReader::lookup_batch(const std::vector& terms, + std::vector* results) const { + DORIS_CHECK(results != nullptr); + DCHECK(std::ranges::is_sorted(terms)); + DCHECK(std::adjacent_find(terms.begin(), terms.end()) == terms.end()); + results->assign(terms.size(), BatchLookupResult {}); + if (reader_ == nullptr) { + return Status::Error("logical_index: not opened"); + } + + std::vector candidates; + std::vector groups; + RETURN_IF_ERROR(collect_batch_lookup_groups(terms, &candidates, &groups)); + if (groups.empty()) { + return Status::OK(); + } + + // Resident dictionaries always stay zero-I/O. One-block batches keep the + // existing synchronous read-through path. + if (!resident_dict_blocks_.empty() || groups.size() == 1) { + for (const BatchLookupGroup& group : groups) { + const DictBlockReader* block_reader = nullptr; + std::shared_ptr pin; + RETURN_IF_ERROR(dict_block_reader_for_ordinal(group.ordinal, /*cache=*/nullptr, &pin, + &block_reader)); + RETURN_IF_ERROR( + resolve_batch_lookup_group(terms, candidates, group, *block_reader, results)); + } + return Status::OK(); + } + + return lookup_batch_on_demand(terms, candidates, groups, results); +} + +Status LogicalIndexReader::decode_dict_block(uint32_t ordinal, std::vector* entries, + uint64_t* frq_base, uint64_t* prx_base) const { + if (entries == nullptr || frq_base == nullptr || prx_base == nullptr) { + return Status::Error( + "logical_index: null decode_dict_block out"); + } + if (reader_ == nullptr) { + return Status::Error("logical_index: not opened"); + } + entries->clear(); + // Same resolution path lookup() uses (resident block or one on-demand range + // read + zstd + CRC); `pin` keeps an on-demand block alive through + // decode_all. No request-scoped cache: a sequential full scan touches each + // block exactly once. + const DictBlockReader* br = nullptr; + std::shared_ptr pin; + RETURN_IF_ERROR(dict_block_reader_for_ordinal(ordinal, /*cache=*/nullptr, &pin, &br)); + RETURN_IF_ERROR(br->decode_all(entries)); + *frq_base = br->frq_base(); + *prx_base = br->prx_base(); + return Status::OK(); +} + +Status LogicalIndexReader::dict_block_scan_memory(uint32_t ordinal, + DictBlockScanMemory* out) const { + if (out == nullptr) { + return Status::Error( + "logical_index: null dict block scan memory out"); + } + *out = DictBlockScanMemory {}; + if (reader_ == nullptr) { + return Status::Error("logical_index: not opened"); + } + + BlockRef ref {}; + RETURN_IF_ERROR(dbd_.get(ordinal, &ref)); + uint64_t plain_bytes = 0; + RETURN_IF_ERROR(dict_block_memory_bytes(ref, &plain_bytes)); + + // During a compressed decode the fetched bytes coexist with the plain block. + // The reader then owns the plain bytes plus anchor vectors/strings. Counting + // every entry as an anchor is conservative for all valid anchor intervals. + uint64_t anchor_slots = 0; + RETURN_IF_ERROR(checked_memory_mul(ref.n_entries, sizeof(uint32_t) + sizeof(std::string), + "logical_index: dict decode anchor memory overflows", + &anchor_slots)); + uint64_t decode_bytes = 0; + RETURN_IF_ERROR(checked_memory_add(ref.length, plain_bytes, + "logical_index: dict decode bytes overflow", &decode_bytes)); + RETURN_IF_ERROR(checked_memory_add(decode_bytes, anchor_slots, + "logical_index: dict decode slots overflow", &decode_bytes)); + RETURN_IF_ERROR(checked_memory_add(decode_bytes, plain_bytes, + "logical_index: dict decode terms overflow", &decode_bytes)); + + uint64_t entry_slots = 0; + RETURN_IF_ERROR(checked_memory_mul(ref.n_entries, sizeof(DictEntry), + "logical_index: dict entry slots overflow", &entry_slots)); + uint64_t entry_payload = 0; + RETURN_IF_ERROR(checked_memory_mul(plain_bytes, 2, "logical_index: dict entry payload overflow", + &entry_payload)); + RETURN_IF_ERROR(checked_memory_add(entry_slots, entry_payload, + "logical_index: dict entries memory overflow", + &out->entries_bytes)); + out->decode_bytes = decode_bytes; + return Status::OK(); +} + +Status LogicalIndexReader::visit_prefix_terms(std::string_view prefix, + const PrefixHitVisitor& visitor, + DictBlockCache* cache) const { + if (!visitor) { + return Status::Error( + "logical_index: null prefix visitor"); + } + if (reader_ == nullptr) { + return Status::Error("logical_index: not opened"); + } + + // Seek the start block: the SampledTermIndex block whose first term <= prefix + // (terms with `prefix` are >= prefix, so they begin in that block or later). + // If the prefix sorts before every sample (or is empty), start at block 0. + uint32_t start = 0; + if (!prefix.empty()) { + bool maybe = false; + uint32_t ordinal = 0; + RETURN_IF_ERROR(sti_.locate(prefix, &maybe, &ordinal)); + if (maybe) { + start = ordinal; + } + } + + for (uint32_t ord = start; ord < dbd_.n_blocks(); ++ord) { + const DictBlockReader* br = nullptr; + std::shared_ptr pin; + RETURN_IF_ERROR(dict_block_reader_for_ordinal(ord, cache, &pin, &br)); + + // Stream this block's prefix range: anchor-jump past pre-prefix segments, + // decode only the bodies we keep, and stop at the first term past the + // range (decode_all materialized every entry of every scanned block). + // The visitor still owns final term acceptance, so results are identical; + // `br`/`pin` stay alive across this synchronous call. + bool prefix_exhausted = false; + bool visitor_stopped = false; + RETURN_IF_ERROR(br->visit_prefix_range( + prefix, /*accept_key=*/ {}, + [&](DictEntry&& e, bool* stop) -> Status { + PrefixHit hit; + hit.term = e.term; + hit.entry = std::move(e); + hit.frq_base = br->frq_base(); + hit.prx_base = br->prx_base(); + RETURN_IF_ERROR(visitor(std::move(hit), stop)); + visitor_stopped = *stop; + return Status::OK(); + }, + &prefix_exhausted)); + if (visitor_stopped || prefix_exhausted) { + return Status::OK(); + } + } + return Status::OK(); +} + +Status LogicalIndexReader::visit_term_range(std::string_view lower_inclusive, + std::optional upper_exclusive, + const PrefixHitVisitor& visitor, + DictBlockCache* cache) const { + if (!visitor) { + return Status::Error( + "logical_index: null range visitor"); + } + if (reader_ == nullptr) { + return Status::Error("logical_index: not opened"); + } + if (upper_exclusive.has_value() && *upper_exclusive <= lower_inclusive) { + return Status::OK(); + } + if (upper_exclusive.has_value()) { + bool upper_reaches_dictionary = false; + uint32_t upper_ordinal = 0; + RETURN_IF_ERROR(sti_.locate(*upper_exclusive, &upper_reaches_dictionary, &upper_ordinal)); + if (!upper_reaches_dictionary) { + return Status::OK(); + } + } + + uint32_t start = 0; + if (!lower_inclusive.empty()) { + bool maybe = false; + uint32_t ordinal = 0; + RETURN_IF_ERROR(sti_.locate(lower_inclusive, &maybe, &ordinal)); + if (maybe) { + start = ordinal; + } + } + + for (uint32_t ord = start; ord < dbd_.n_blocks(); ++ord) { + const DictBlockReader* block_reader = nullptr; + std::shared_ptr pin; + RETURN_IF_ERROR(dict_block_reader_for_ordinal(ord, cache, &pin, &block_reader)); + + bool range_exhausted = false; + bool visitor_stopped = false; + RETURN_IF_ERROR(block_reader->visit_term_range( + lower_inclusive, upper_exclusive, /*accept_key=*/ {}, + [&](DictEntry&& entry, bool* stop) -> Status { + PrefixHit hit; + hit.term = entry.term; + hit.entry = std::move(entry); + hit.frq_base = block_reader->frq_base(); + hit.prx_base = block_reader->prx_base(); + RETURN_IF_ERROR(visitor(std::move(hit), stop)); + visitor_stopped = *stop; + return Status::OK(); + }, + &range_exhausted)); + if (visitor_stopped || range_exhausted) { + return Status::OK(); + } + } + return Status::OK(); +} + +Status LogicalIndexReader::prefix_terms(std::string_view prefix, std::vector* const out, + int32_t max_terms, DictBlockCache* cache) const { + if (out == nullptr) { + return Status::Error("logical_index: null out"); + } + out->clear(); + return visit_prefix_terms( + prefix, + [&](PrefixHit&& hit, bool* stop) { + out->push_back(std::move(hit)); + *stop = max_terms > 0 && out->size() >= static_cast(max_terms); + return Status::OK(); + }, + cache); +} + +namespace { + +// Validates a pod_ref window locator against the posting region and returns the +// absolute window range (after the prelude). Rejects corrupt locators rather +// than letting size_t underflow / uint64 overflow reach read_at. +Status resolve_window(const format::RegionRef& section, uint64_t base, uint64_t off_delta, + uint64_t total_len, uint64_t prelude_len, uint64_t* abs_off, uint64_t* len) { + if (prelude_len > total_len) { + return Status::Error( + "logical_index: prelude_len exceeds window len"); + } + const uint64_t in_region = base + off_delta; + if (in_region < base) { + return Status::Error( + "logical_index: locator overflow"); + } + if (in_region > section.length || total_len > section.length - in_region) { + return Status::Error( + "logical_index: window past posting region"); + } + *abs_off = section.offset + in_region + prelude_len; + *len = total_len - prelude_len; + return Status::OK(); +} + +} // namespace + +Status LogicalIndexReader::resolve_frq_window(const format::DictEntry& entry, uint64_t frq_base, + uint64_t* abs_off, uint64_t* len) const { + return resolve_window(section_refs().posting_region, frq_base, entry.frq_off_delta, + entry.frq_len, entry.prelude_len, abs_off, len); +} + +Status LogicalIndexReader::resolve_prx_window(const format::DictEntry& entry, uint64_t prx_base, + uint64_t* abs_off, uint64_t* len) const { + // .prx windows carry no prelude (prelude_len = 0); both spans live in the + // same posting region (prx span precedes frq span for the same term). + return resolve_window(section_refs().posting_region, prx_base, entry.prx_off_delta, + entry.prx_len, 0, abs_off, len); +} + +} // namespace doris::snii::reader diff --git a/be/src/storage/index/snii/reader/logical_index_reader.h b/be/src/storage/index/snii/reader/logical_index_reader.h new file mode 100644 index 00000000000000..f7fdee0a902f18 --- /dev/null +++ b/be/src/storage/index/snii/reader/logical_index_reader.h @@ -0,0 +1,285 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/format/bsbf.h" +#include "storage/index/snii/format/core_metadata.h" +#include "storage/index/snii/format/dict_block.h" +#include "storage/index/snii/format/dict_block_directory.h" +#include "storage/index/snii/format/dict_entry.h" +#include "storage/index/snii/format/format_constants.h" +#include "storage/index/snii/format/metadata_blob.h" +#include "storage/index/snii/format/sampled_term_index.h" +#include "storage/index/snii/format/stats_block.h" +#include "storage/index/snii/io/file_reader.h" + +// LogicalIndexReader -- read-side counterpart of LogicalIndexWriter for one +// logical index. It owns decoded Core, SampledTermIndex, and DICT block-directory +// state from one adjacent metadata group, and resolves a query term to its +// DictEntry through the documented lookup flow: +// XFilter (reject absent) -> SampledTermIndex (candidate block ordinal) -> +// DICT block directory (block range) -> resident small-DICT block or one +// range read of the DICT block -> DictBlockReader::find_term. +// +// lookup() also returns the block's frq_base/prx_base (captured by the +// DictBlockReader) so callers can resolve a pod_ref entry's absolute .frq/.prx +// offsets via the writer's contract. Both deltas index into the SAME +// interleaved posting region (prx_base == frq_base; the prx span precedes the +// frq span): +// abs_frq = posting_region.offset + frq_base + entry.frq_off_delta +// abs_prx = posting_region.offset + prx_base + entry.prx_off_delta +// +// The reader retains no raw metadata-group bytes after open. +namespace doris::snii::format { +class NormsPodReader; +} + +namespace doris::snii::reader { + +// Forward-declared: this widely-included header only names DictBlockCache* and +// shared_ptr*; the full definitions are pulled into the +// .cpp and into tests that construct a cache. Keeps the request-scoped cache +// header out of the ~500 TUs that transitively include this one. +struct DecodedDictBlock; +class DictBlockCache; + +enum class LogicalIndexOpenMode : uint8_t { + kQuery, + kCompaction, +}; + +struct DictBlockScanMemory { + uint64_t decode_bytes = 0; + uint64_t entries_bytes = 0; +}; + +struct NullDocidsScanMemory { + uint64_t frame_bytes = 0; + uint64_t output_bytes = 0; +}; + +class LogicalIndexReader { +public: + LogicalIndexReader() = default; + + // Parses one mandatory Core/STI/DBD metadata group and binds the reader to + // file_reader. The reader retains decoded state, not the input byte slices. + static Status open(io::FileReader* file_reader, Slice core_frame, Slice sti_blob, + Slice dbd_blob, LogicalIndexReader* out, + LogicalIndexOpenMode open_mode = LogicalIndexOpenMode::kQuery); + + // Resolves term to a DictEntry. *found=false when the term is absent (XFilter + // rejection, out-of-range sample, or DICT-block miss). On a hit, *entry is + // filled and *frq_base / *prx_base carry the candidate block's bases. + // + // `cache` is an optional REQUEST-SCOPED DictBlockCache: when a single query + // threads one cache through its per-term lookups, an on-demand DICT block hit + // by several terms is decoded once instead of once per term. nullptr keeps the + // pre-existing behavior (each lookup materializes its own block). The cache is + // caller-owned, single-threaded, and never mutates this (const) reader. + Status lookup(std::string_view term, bool* found, format::DictEntry* entry, uint64_t* frq_base, + uint64_t* prx_base, DictBlockCache* cache = nullptr) const; + + struct BatchLookupResult { + bool found = false; + format::DictEntry entry; + uint64_t frq_base = 0; + uint64_t prx_base = 0; + }; + + // Resolves one sorted, duplicate-free term batch. Terms are first mapped to + // candidate DICT ordinals through the same XFilter/STI path as lookup(), then + // distinct on-demand blocks are fetched concurrently in bounded waves. + // Results stay aligned with `terms`; absent terms have found=false. + Status lookup_batch(const std::vector& terms, + std::vector* results) const; + + // One enumerated term whose key has the requested prefix, with its DictEntry + // and the owning DICT block's frq/prx bases (for posting resolution). + struct PrefixHit { + std::string term; + format::DictEntry entry; + uint64_t frq_base = 0; + uint64_t prx_base = 0; + }; + + using PrefixHitVisitor = std::function; + + // Ordered term enumeration: every term with `prefix`, in lexicographic order, + // by seeking the start DICT block via the SampledTermIndex and scanning + // forward across contiguous blocks until the terms pass the prefix range. + // Empty prefix enumerates all terms. This is the contiguous-DICT-block design + // the term-anchor layout was built for (MATCH_PHRASE_PREFIX / prefix / range + // queries). The visitor form avoids materializing all hits when callers only + // need a bounded expansion. + Status visit_prefix_terms(std::string_view prefix, const PrefixHitVisitor& visitor, + DictBlockCache* cache = nullptr) const; + Status visit_term_range(std::string_view lower_inclusive, + std::optional upper_exclusive, + const PrefixHitVisitor& visitor, DictBlockCache* cache = nullptr) const; + Status prefix_terms(std::string_view prefix, std::vector* const out, + int32_t max_terms = 0, DictBlockCache* cache = nullptr) const; + + // ---- Sequential whole-dictionary access (T2.3, compaction index merge) ---- + // Number of DICT blocks in this index (0 for an empty dictionary). + uint32_t n_dict_blocks() const { return dbd_.n_blocks(); } + // Decodes EVERY entry of DICT block `ordinal` in lexicographic order into + // *entries (each self-contained, owning its term and any inline posting + // bytes) and returns the block's frq/prx bases. One block is materialized + // at a time so a full-dictionary scan (SniiSegmentTermCursor) holds a single + // block's entries, never the whole vocabulary. ordinal must be + // < n_dict_blocks(). + Status decode_dict_block(uint32_t ordinal, std::vector* entries, + uint64_t* frq_base, uint64_t* prx_base) const; + // Returns conservative pre-allocation charges for the on-demand block decode + // and its fully materialized DictEntry vector. Compaction cursors reserve both + // before decoding so MEM_LIMIT_EXCEEDED is returned before the normal block + // allocations whenever the shared merge cap cannot admit them. + Status dict_block_scan_memory(uint32_t ordinal, DictBlockScanMemory* out) const; + + // Resolves a pod_ref entry's absolute .frq / .prx window byte range, + // validating the locator against the posting_region length (defends against + // corrupt entries: prelude_len > frq_len underflow, or off_delta+len past the + // region). Both windows resolve against the single posting_region. *abs_off + // is the absolute file offset of the window (after prelude); *len its byte + // length. + Status resolve_frq_window(const format::DictEntry& entry, uint64_t frq_base, uint64_t* abs_off, + uint64_t* len) const; + Status resolve_prx_window(const format::DictEntry& entry, uint64_t prx_base, uint64_t* abs_off, + uint64_t* len) const; + + const format::SectionRefs& section_refs() const { return core_.section_refs; } + const format::StatsBlock& stats() const { return core_.stats; } + format::IndexTier tier() const { return tier_; } + bool has_positions() const { return has_positions_; } + LogicalIndexOpenMode open_mode() const { return open_mode_; } + const segment_v2::inverted_index::CommonGramsSegmentMetadata* common_grams_metadata() const { + return core_.common_grams_metadata ? &*core_.common_grams_metadata : nullptr; + } + format::CommonGramsPostingPolicy common_grams_posting_policy() const { + return core_.common_grams_posting_policy; + } + io::FileReader* reader() const { return reader_; } + + // Returns a reader over the validated norms section. The first call reads + // and validates the section; later calls share the immutable reader-owned + // bytes. The full on-disk section is reserved in memory_usage() before this + // LogicalIndexReader enters the searcher cache, so lazy loading cannot make + // the cache under-report its eventual resident size. + Status open_norms(format::NormsPodReader* out) const; + // Compaction scans one source norm vector at a time. This charge matches the + // reader's full cache accounting; release_compaction_norms() drops the loaded + // frame after its values have been scattered into destination vectors. + size_t compaction_norms_cache_charge() const { return norms_reserved_charge_; } + void release_compaction_norms() const; + + // Reads and validates the sparse null-bitmap side POD, then returns its + // docids in ascending order. Work is O(null_count), not O(doc_count), which + // lets compaction remap NULL rows without scanning the complete document + // domain. A missing section is valid only when StatsBlock::null_count is 0. + using NullDocidsDecodeReservation = std::function; + Status read_null_docids(std::vector* out, + const NullDocidsDecodeReservation& reserve_decode = + NullDocidsDecodeReservation()) const; + Status null_docids_scan_memory(NullDocidsScanMemory* out) const; + size_t memory_usage() const; + +private: + struct NormsCacheState; + struct BatchLookupCandidate { + size_t term_index = 0; + uint32_t ordinal = 0; + }; + struct BatchLookupGroup { + uint32_t ordinal = 0; + size_t begin = 0; + size_t end = 0; + }; + struct PendingBatchLookupBlock { + size_t group_index = 0; + format::BlockRef ref; + size_t handle = 0; + }; + io::FileReader* reader_ = nullptr; + format::IndexTier tier_ = format::IndexTier::kT1; + bool has_positions_ = false; + LogicalIndexOpenMode open_mode_ = LogicalIndexOpenMode::kQuery; + format::CoreMetadata core_; + format::SampledTermIndexReader sti_; + format::DictBlockDirectoryReader dbd_; + format::BsbfHeader bsbf_header_; // resident header (from section ref) + bool has_bsbf_ = false; + // L0 tiering: when the bsbf section is small (<= kBsbfResidentMaxBytes) its + // whole bitset is loaded here at open -> in-memory probe, no per-lookup + // round. Larger filters keep only the parsed header here, so the small + // header enters Doris searcher cache and lookup reads just one 32-byte body + // block for an L1 probe. + bool bsbf_resident_ = false; + std::vector bsbf_resident_bitset_; + + // Small DICT blocks are opened once with the index so exact lookups avoid an + // otherwise serial S3 round for the term dictionary. Empty means the + // dictionary exceeded the resident threshold and lookup/prefix enumeration + // read blocks on demand. Each DictBlockReader holds a Slice into the owning + // bytes. + struct ResidentDictBlock { + std::vector bytes; + format::DictBlockReader reader; + }; + Status load_resident_dict_blocks(); + Status load_resident_bsbf(); + // Resolves the DictBlockReader for `ordinal`. Resident blocks return a pointer + // into the reader-owned resident set with *pin left null (stable for the reader + // lifetime). On-demand blocks are decoded (optionally via the request-scoped + // `cache`) into a heap-allocated DecodedDictBlock; *pin holds it alive so *out + // never dangles under a later cache eviction. Callers must keep *pin alive for + // as long as they use *out. + Status dict_block_reader_for_ordinal(uint32_t ordinal, DictBlockCache* cache, + std::shared_ptr* pin, + const format::DictBlockReader** out) const; + Status locate_candidate_dict_block(std::string_view term, bool* maybe_present, + uint32_t* ordinal) const; + Status collect_batch_lookup_groups(const std::vector& terms, + std::vector* candidates, + std::vector* groups) const; + static Status resolve_batch_lookup_group(const std::vector& terms, + const std::vector& candidates, + const BatchLookupGroup& group, + const format::DictBlockReader& block_reader, + std::vector* results); + Status lookup_batch_on_demand(const std::vector& terms, + const std::vector& candidates, + const std::vector& groups, + std::vector* results) const; + std::vector resident_dict_blocks_; + std::shared_ptr norms_cache_; + size_t norms_reserved_charge_ = 0; +}; + +} // namespace doris::snii::reader diff --git a/be/src/storage/index/snii/reader/snii_segment_reader.cpp b/be/src/storage/index/snii/reader/snii_segment_reader.cpp new file mode 100644 index 00000000000000..da28d2245024fd --- /dev/null +++ b/be/src/storage/index/snii/reader/snii_segment_reader.cpp @@ -0,0 +1,371 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/reader/snii_segment_reader.h" + +#include +#include +#include +#include +#include +#include + +#include "storage/index/snii/encoding/crc32c.h" +#include "storage/index/snii/format/bootstrap_header.h" +#include "storage/index/snii/format/core_metadata.h" +#include "storage/index/snii/format/tail_pointer.h" + +namespace doris::snii::reader { +namespace { + +Status corrupted(std::string_view message) { + return Status::Error(message); +} + +Status read_tail_pointer(io::FileReader* reader, format::TailPointer* tail, + uint64_t* footer_offset) { + const size_t footer_size = format::tail_pointer_size(); + const uint64_t total = reader->size(); + if (total < footer_size) { + return corrupted("segment: file smaller than tail pointer"); + } + *footer_offset = total - footer_size; + std::vector bytes; + RETURN_IF_ERROR(reader->read_at(*footer_offset, footer_size, &bytes)); + return format::decode_tail_pointer(Slice(bytes), tail); +} + +// Proves Core -> STI -> DBD are adjacent and end at or before directory_offset, so open_index() can +// cover the whole group with one range read. It also bounds core.length + sti.length + dbd.length by +// directory_offset (itself bounded by the file size), which is why open_index() may sum and narrow +// those on-disk 64-bit lengths to size_t without a further overflow check. +Status validate_metadata_group(const format::LogicalIndexMetadataRef& entry, + uint64_t directory_offset) { + const auto& core = entry.core_metadata; + const auto& sti = entry.sampled_term_index; + const auto& dbd = entry.dict_block_directory; + if (core.offset > directory_offset || core.length > directory_offset - core.offset) { + return corrupted("segment: Core metadata reference is outside metadata area"); + } + const uint64_t sti_offset = core.offset + core.length; + if (sti.offset != sti_offset) { + return corrupted("segment: STI metadata is not adjacent to Core metadata"); + } + if (sti.length > directory_offset - sti.offset) { + return corrupted("segment: STI metadata reference is outside metadata area"); + } + const uint64_t dbd_offset = sti.offset + sti.length; + if (dbd.offset != dbd_offset) { + return corrupted("segment: DBD metadata is not adjacent to STI metadata"); + } + if (dbd.length > directory_offset - dbd.offset) { + return corrupted("segment: DBD metadata reference is outside metadata area"); + } + return Status::OK(); +} + +// Blob entries carry no metadata group; their files must simply live before +// the directory. (Cold files sit in the data area, hot files between the text +// metadata groups and the directory -- the reader only needs the upper bound.) +Status validate_blob_files(const format::LogicalIndexMetadataRef& entry, + uint64_t directory_offset) { + for (const format::NamedBlobFileRef& file : entry.files) { + if (file.offset > directory_offset || file.length > directory_offset - file.offset) { + return corrupted("segment: blob file reference is outside the container data area"); + } + } + return Status::OK(); +} + +Status find_metadata_ref(const format::MetadataDirectory& directory, uint64_t index_id, + std::string_view suffix, const format::LogicalIndexMetadataRef** out) { + *out = directory.find(index_id, suffix); + if (*out == nullptr) { + return Status::Error( + "segment: logical index not found"); + } + return Status::OK(); +} + +// Guards a text-only entry point against a blob entry (and vice versa): the +// caller reached the wrong reader for this entry's kind, which is a usage +// error, not corruption. +Status require_inverted(const format::LogicalIndexMetadataRef& entry) { + if (entry.kind != format::LogicalIndexKind::kInverted) { + return Status::Error( + "segment: logical index is not a text inverted index"); + } + return Status::OK(); +} + +// Blob logical indexes are outside the physical-prefix inherit model: their +// HOT files live inside the metadata area (between the text metadata groups +// and the directory), which a prefix copy does not cover, and their zeroed +// core_metadata offsets would collapse the metadata_area_begin computation. +// A rewrite over such a container must fail loudly BEFORE reading any group, +// whichever keys the caller keeps -- silently dropping or mis-copying an index +// is never acceptable. Lifting this needs a hot-file re-emission step in +// SniiCompoundWriter::inherit (mirroring how metadata groups are re-emitted at +// new offsets). +Status reject_blob_container_for_rewrite(const format::MetadataDirectory& directory) { + for (const format::LogicalIndexMetadataRef& entry : directory.entries()) { + if (entry.kind != format::LogicalIndexKind::kInverted) { + return Status::Error( + "segment: rewrite snapshot over a container with blob logical indexes is " + "not supported"); + } + } + return Status::OK(); +} + +} // namespace + +Status SniiSegmentReader::open(io::FileReader* const reader, SniiSegmentReader* const out) { + if (reader == nullptr) { + return Status::Error("segment: null reader"); + } + if (out == nullptr) { + return Status::Error("segment: null out"); + } + *out = {}; + + // The per-segment bootstrap header at offset zero remains an inspect-tool record and is + // intentionally not read here. The footer validates the exact format version and its own CRC, + // and it locates the metadata directory. Reading only the file tail avoids an otherwise + // redundant offset-zero cache block or remote round trip on cold queries. Future incompatible + // evolution must bump the footer format version rather than rely on a min-reader-version change + // under a stable format version. + format::TailPointer tail; + uint64_t footer_offset = 0; + RETURN_IF_ERROR(read_tail_pointer(reader, &tail, &footer_offset)); + if (tail.directory_offset > footer_offset || + tail.directory_length > footer_offset - tail.directory_offset) { + return corrupted("segment: metadata directory reference overlaps footer or EOF"); + } + if (tail.directory_length > static_cast(std::numeric_limits::max())) { + return corrupted("segment: metadata directory exceeds protobuf parse limit"); + } + // The protobuf parse limit above already bounds the length, so narrowing it is safe. + const auto directory_length = static_cast(tail.directory_length); + + std::vector directory_bytes; + RETURN_IF_ERROR(reader->read_at(tail.directory_offset, directory_length, &directory_bytes)); + if (crc32c(Slice(directory_bytes)) != tail.directory_crc32c) { + return corrupted("segment: metadata directory crc32c mismatch"); + } + + format::MetadataDirectory directory; + RETURN_IF_ERROR(format::MetadataDirectory::decode(Slice(directory_bytes), &directory)); + for (const auto& entry : directory.entries()) { + if (entry.kind == format::LogicalIndexKind::kInverted) { + RETURN_IF_ERROR(validate_metadata_group(entry, tail.directory_offset)); + } else { + RETURN_IF_ERROR(validate_blob_files(entry, tail.directory_offset)); + } + } + + out->reader_ = reader; + out->directory_offset_ = tail.directory_offset; + out->directory_ = std::move(directory); + return Status::OK(); +} + +Status SniiSegmentReader::index_exists(uint64_t index_id, std::string_view suffix, + bool* const exists) const { + if (exists == nullptr) { + return Status::Error("segment: null exists out"); + } + if (reader_ == nullptr) { + return Status::Error("segment: not opened"); + } + *exists = directory_.find(index_id, suffix) != nullptr; + return Status::OK(); +} + +Status SniiSegmentReader::open_index(uint64_t index_id, std::string_view suffix, + LogicalIndexReader* const out, + LogicalIndexOpenMode open_mode) const { + if (out == nullptr) { + return Status::Error("segment: null index out"); + } + *out = {}; + if (reader_ == nullptr) { + return Status::Error("segment: not opened"); + } + const format::LogicalIndexMetadataRef* entry = nullptr; + RETURN_IF_ERROR(find_metadata_ref(directory_, index_id, suffix, &entry)); + RETURN_IF_ERROR(require_inverted(*entry)); + + // Safe to sum and narrow: open() ran validate_metadata_group on every directory entry. + const auto core_length = static_cast(entry->core_metadata.length); + const auto sti_length = static_cast(entry->sampled_term_index.length); + const auto dbd_length = static_cast(entry->dict_block_directory.length); + const size_t group_length = core_length + sti_length + dbd_length; + std::vector group; + RETURN_IF_ERROR(reader_->read_at(entry->core_metadata.offset, group_length, &group)); + DORIS_CHECK_EQ(group.size(), group_length); + const Slice bytes(group); + return LogicalIndexReader::open( + reader_, bytes.subslice(0, core_length), bytes.subslice(core_length, sti_length), + bytes.subslice(core_length + sti_length, dbd_length), out, open_mode); +} + +// Loads one kept logical index into *out and extends *physical_prefix_end to +// cover every section it references. Fails -- never silently drops an index -- +// on a missing key, a corrupt metadata blob, a section reference outside the +// physical area, or a doc-count disagreement with the segment. +Status SniiSegmentReader::load_inherited_index(const LogicalIndexKey& key, + uint64_t segment_doc_count, + uint64_t metadata_area_begin, + InheritedLogicalIndex* const out, + uint64_t* const physical_prefix_end) const { + const format::LogicalIndexMetadataRef* entry = nullptr; + RETURN_IF_ERROR(find_metadata_ref(directory_, key.index_id, key.index_suffix, &entry)); + + // Safe to sum and narrow: open() ran validate_metadata_group on every entry. + out->index_id = entry->index_id; + out->index_suffix = entry->index_suffix; + out->core_length = static_cast(entry->core_metadata.length); + out->sampled_term_index_length = static_cast(entry->sampled_term_index.length); + out->dict_block_directory_length = static_cast(entry->dict_block_directory.length); + const size_t group_length = + out->core_length + out->sampled_term_index_length + out->dict_block_directory_length; + RETURN_IF_ERROR( + reader_->read_at(entry->core_metadata.offset, group_length, &out->metadata_group)); + DORIS_CHECK_EQ(out->metadata_group.size(), group_length); + + // Decoding validates the frame crc, so a damaged live index fails the whole + // rewrite instead of being silently carried over or dropped. + format::CoreMetadata core; + RETURN_IF_ERROR(format::decode_core_metadata( + Slice(out->metadata_group.data(), out->core_length), &core)); + if (core.stats.doc_count != segment_doc_count) { + return corrupted("segment: inherited logical index doc count disagrees with segment"); + } + out->section_refs = core.section_refs; + out->doc_count = core.stats.doc_count; + + for (const format::RegionRef& region : + {core.section_refs.dict_region, core.section_refs.posting_region, core.section_refs.norms, + core.section_refs.null_bitmap, core.section_refs.bsbf}) { + if (region.offset > metadata_area_begin || + region.length > metadata_area_begin - region.offset) { + return corrupted("segment: inherited section reference is outside the physical area"); + } + *physical_prefix_end = std::max(*physical_prefix_end, region.offset + region.length); + } + return Status::OK(); +} + +Status SniiSegmentReader::prepare_rewrite_snapshot(const std::vector& keep, + uint64_t segment_doc_count, + SniiRewriteSnapshot* const out) const { + if (out == nullptr) { + return Status::Error("segment: null snapshot out"); + } + *out = {}; + if (reader_ == nullptr) { + return Status::Error("segment: not opened"); + } + + RETURN_IF_ERROR(reject_blob_container_for_rewrite(directory_)); + + // The query path skips the bootstrap header because the tail already gates the + // container version. A rewrite copies those bytes into the new container, so it + // must not carry a header it never checked. + std::vector bootstrap_bytes; + RETURN_IF_ERROR(reader_->read_at(0, format::kBootstrapHeaderSize, &bootstrap_bytes)); + format::BootstrapHeader bootstrap; + RETURN_IF_ERROR(format::decode_bootstrap_header(Slice(bootstrap_bytes), &bootstrap)); + + // Metadata groups follow every physical section, so the first one marks the end + // of the physical area. Sections must live strictly before it. + uint64_t metadata_area_begin = directory_offset_; + for (const auto& entry : directory_.entries()) { + metadata_area_begin = std::min(metadata_area_begin, entry.core_metadata.offset); + } + + // The bootstrap header is part of every inherited prefix, even when the kept + // indexes reference no section at all. + uint64_t physical_prefix_end = format::kBootstrapHeaderSize; + std::vector inherited; + inherited.reserve(keep.size()); + for (const LogicalIndexKey& key : keep) { + for (const InheritedLogicalIndex& seen : inherited) { + if (seen.index_id == key.index_id && seen.index_suffix == key.index_suffix) { + return Status::Error( + "segment: logical index requested twice in one rewrite snapshot"); + } + } + InheritedLogicalIndex kept; + RETURN_IF_ERROR(load_inherited_index(key, segment_doc_count, metadata_area_begin, &kept, + &physical_prefix_end)); + inherited.push_back(std::move(kept)); + } + + out->physical_prefix_end_ = physical_prefix_end; + out->inherited_ = std::move(inherited); + return Status::OK(); +} + +bool SniiSegmentReader::has_blob_index() const { + return std::any_of(directory_.entries().begin(), directory_.entries().end(), + [](const format::LogicalIndexMetadataRef& entry) { + return entry.kind != format::LogicalIndexKind::kInverted; + }); +} + +Status SniiSegmentReader::blob_entry(uint64_t index_id, std::string_view suffix, + const format::LogicalIndexMetadataRef** out) const { + if (out == nullptr) { + return Status::Error("segment: null entry out"); + } + *out = nullptr; + if (reader_ == nullptr) { + return Status::Error("segment: not opened"); + } + const format::LogicalIndexMetadataRef* entry = nullptr; + RETURN_IF_ERROR(find_metadata_ref(directory_, index_id, suffix, &entry)); + if (entry->kind == format::LogicalIndexKind::kInverted) { + return Status::Error( + "segment: logical index is not a blob index"); + } + *out = entry; + return Status::OK(); +} + +Status SniiSegmentReader::section_refs_for_index(uint64_t index_id, std::string_view suffix, + format::SectionRefs* const out) const { + if (out == nullptr) { + return Status::Error("segment: null section refs out"); + } + if (reader_ == nullptr) { + return Status::Error("segment: not opened"); + } + const format::LogicalIndexMetadataRef* entry = nullptr; + RETURN_IF_ERROR(find_metadata_ref(directory_, index_id, suffix, &entry)); + RETURN_IF_ERROR(require_inverted(*entry)); + std::vector core_bytes; + RETURN_IF_ERROR(reader_->read_at(entry->core_metadata.offset, entry->core_metadata.length, + &core_bytes)); + format::CoreMetadata core; + RETURN_IF_ERROR(format::decode_core_metadata(Slice(core_bytes), &core)); + *out = core.section_refs; + return Status::OK(); +} + +} // namespace doris::snii::reader diff --git a/be/src/storage/index/snii/reader/snii_segment_reader.h b/be/src/storage/index/snii/reader/snii_segment_reader.h new file mode 100644 index 00000000000000..f0ab723015576f --- /dev/null +++ b/be/src/storage/index/snii/reader/snii_segment_reader.h @@ -0,0 +1,157 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/format/core_metadata.h" +#include "storage/index/snii/format/metadata_directory.h" +#include "storage/index/snii/io/file_reader.h" +#include "storage/index/snii/reader/logical_index_reader.h" + +// SniiSegmentReader -- entry point for the SNII segment read path. It opens a +// single .idx container through a (possibly metered) io::FileReader and exposes +// its logical indexes. open() reads only the file tail: +// 1. the fixed tail pointer (last tail_pointer_size() bytes), which also gates +// the container format_version ('TAIL' magic + format_version exact-match + +// tail crc), and +// 2. the raw protobuf logical-index metadata directory. +// The bootstrap header at offset 0 is still WRITTEN on disk (for inspect tooling) +// but is intentionally NOT read at open: its only runtime role (the container +// version gate) is already covered, more strictly, by the tail pointer, so +// skipping it avoids a redundant offset-0 cache block / remote round-trip per +// segment on cold queries. +// Per-index metadata groups are read lazily by open_index() so opening one logical +// index does not read every other logical index's metadata. +// +// open_index() then materializes one LogicalIndexReader from the metadata group +// of a given (index_id, suffix); query functions operate on that reader. +namespace doris::snii::reader { + +// Identifies one logical index inside a container. +struct LogicalIndexKey { + uint64_t index_id = 0; + std::string index_suffix; +}; + +// One logical index a rewrite inherits unchanged from its source container. +struct InheritedLogicalIndex { + uint64_t index_id = 0; + std::string index_suffix; + // Section references as recorded on disk. They stay valid in the rewritten + // container because the physical prefix is copied to the SAME offsets. + format::SectionRefs section_refs; + uint64_t doc_count = 0; + // The on-disk [Core][STI][DBD] run, verbatim. A rewrite re-emits these bytes + // without decoding or re-encoding any postings. + std::vector metadata_group; + size_t core_length = 0; + size_t sampled_term_index_length = 0; + size_t dict_block_directory_length = 0; +}; + +// Immutable, fully validated view of one container as an inheritance source for a +// rewrite (BUILD INDEX on SNII). It exposes only what the writer needs: how many +// leading bytes to copy, and the metadata of the logical indexes carried over. +// Encoding details stay inside the reader and the writer. +class SniiRewriteSnapshot { +public: + SniiRewriteSnapshot() = default; + + // Copying [0, physical_prefix_end) reproduces the bootstrap header and every + // physical section the inherited indexes reference. It never covers a metadata + // group, the directory, padding or the tail. + uint64_t physical_prefix_end() const { return physical_prefix_end_; } + const std::vector& inherited() const { return inherited_; } + +private: + friend class SniiSegmentReader; + + uint64_t physical_prefix_end_ = 0; + std::vector inherited_; +}; + +class SniiSegmentReader { +public: + SniiSegmentReader() = default; + + // Reads the tail pointer + raw metadata directory from reader (the offset-0 + // bootstrap header is not read; the tail pointer gates the container version). + // reader must outlive the returned SniiSegmentReader and every + // LogicalIndexReader opened from it. reader == nullptr / out == nullptr -> + // InvalidArgument; structural problems -> Corruption / Unsupported. + static Status open(io::FileReader* const reader, SniiSegmentReader* const out); + + uint32_t n_logical_indexes() const { return static_cast(directory_.size()); } + + Status index_exists(uint64_t index_id, std::string_view suffix, bool* const exists) const; + + // Loads the adjacent Core/STI/DBD group for (index_id, suffix) and builds a + // LogicalIndexReader bound to the same FileReader. Absent index -> NotFound. + Status open_index(uint64_t index_id, std::string_view suffix, LogicalIndexReader* const out, + LogicalIndexOpenMode open_mode = LogicalIndexOpenMode::kQuery) const; + Status section_refs_for_index(uint64_t index_id, std::string_view suffix, + format::SectionRefs* const out) const; + + // Looks up a BLOB logical index entry (kind != kInverted) and exposes its + // validated directory entry (kind + named-file table). Absent -> NotFound; + // a text inverted entry under that key -> Unsupported (kind mismatch, not + // a lookup miss). The pointer stays valid for this reader's lifetime. + Status blob_entry(uint64_t index_id, std::string_view suffix, + const format::LogicalIndexMetadataRef** out) const; + + // True when the container holds at least one blob logical index. A rewrite + // driven by the text index list must consult this BEFORE deciding it has + // nothing to carry over, or it would drop those entries silently. + bool has_blob_index() const; + + // Builds a rewrite snapshot describing exactly the logical indexes in `keep`. + // `segment_doc_count` is the segment's row count: every kept logical index must + // agree with it. Fails -- never silently drops an index -- on a missing or + // duplicated key, a corrupt bootstrap header or metadata blob, a section + // reference outside the physical area, or a doc-count disagreement. + Status prepare_rewrite_snapshot(const std::vector& keep, + uint64_t segment_doc_count, + SniiRewriteSnapshot* const out) const; + + io::FileReader* reader() const { return reader_; } + + // Exclusive upper bound of the container's data area: physical sections, + // metadata groups and blob files all live strictly before it. Pass this to + // SniiBlobDirectory::open so the shim bounds blob files against the same + // limit open() validated them with, not against the whole file size. + uint64_t directory_offset() const { return directory_offset_; } + +private: + // One kept index of a rewrite snapshot; see the .cpp for the contract. + Status load_inherited_index(const LogicalIndexKey& key, uint64_t segment_doc_count, + uint64_t metadata_area_begin, InheritedLogicalIndex* const out, + uint64_t* const physical_prefix_end) const; + + io::FileReader* reader_ = nullptr; + // Start of the raw metadata directory. Together with the directory entries it + // bounds the metadata area, which is where the physical section area ends. + uint64_t directory_offset_ = 0; + format::MetadataDirectory directory_; +}; + +} // namespace doris::snii::reader diff --git a/be/src/storage/index/snii/reader/windowed_posting.cpp b/be/src/storage/index/snii/reader/windowed_posting.cpp new file mode 100644 index 00000000000000..17d931640fb428 --- /dev/null +++ b/be/src/storage/index/snii/reader/windowed_posting.cpp @@ -0,0 +1,308 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/reader/windowed_posting.h" + +#include +#include + +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/encoding/byte_source.h" +#include "storage/index/snii/format/frq_pod.h" +#include "storage/index/snii/format/frq_prelude.h" +#include "storage/index/snii/format/prx_pod.h" +#include "storage/index/snii/io/batch_range_fetcher.h" + +namespace doris::snii::reader { + +using format::DictEntry; +using format::FrqPreludeReader; +using format::FrqRegionMeta; +using format::WindowMeta; + +namespace { + +// Resolves the absolute file offset of the prelude bytes for a windowed entry. +// The frq span lives in the interleaved posting region (after the term's prx span). +uint64_t prelude_abs(const LogicalIndexReader& idx, const DictEntry& entry, uint64_t frq_base) { + const auto& region = idx.section_refs().posting_region; + return region.offset + frq_base + entry.frq_off_delta; +} + +// Validates that [off, off+len) fits within [0, total). +Status in_bounds(uint64_t off, uint64_t len, uint64_t total) { + if (off > total || len > total - off) { + return Status::Error( + "windowed_posting: range out of section"); + } + return Status::OK(); +} + +// Block geometry of a windowed entry's grouped .frq payload (all offsets absolute). +struct BlockGeometry { + uint64_t dd_block_off = 0; // absolute start of the dd-block + uint64_t dd_block_len = 0; + uint64_t freq_block_off = 0; // absolute start of the freq-block + uint64_t freq_block_len = 0; + uint64_t frq_region_len = 0; // entry.frq_len - prelude_len (dd-block + freq-block) +}; + +// Derives the dd-block / freq-block absolute ranges from the entry + prelude, +// validating they tile the post-prelude .frq region exactly. +Status resolve_blocks(const LogicalIndexReader& idx, const DictEntry& entry, uint64_t frq_base, + const FrqPreludeReader& prelude, BlockGeometry* g) { + if (entry.prelude_len > entry.frq_len) { + return Status::Error( + "windowed_posting: prelude_len exceeds frq_len"); + } + const uint64_t frq_window_start = prelude_abs(idx, entry, frq_base) + entry.prelude_len; + g->frq_region_len = entry.frq_len - entry.prelude_len; + g->dd_block_len = prelude.dd_block_len(); + g->freq_block_len = prelude.freq_block_len(); + // dd-block + freq-block must fit exactly within the post-prelude region. + if (g->dd_block_len > g->frq_region_len || + g->freq_block_len > g->frq_region_len - g->dd_block_len) { + return Status::Error( + "windowed_posting: blocks exceed frq region"); + } + g->dd_block_off = frq_window_start; + g->freq_block_off = frq_window_start + g->dd_block_len; + return Status::OK(); +} + +// Per-window decode state for the full-posting path. +struct WindowSlices { + WindowMeta meta; + Slice dd_region; + Slice freq_region; + Slice prx_window; +}; + +// Carves window w's dd (and freq when want_freq) sub-slices out of the fetched +// blocks, validating each locator against its block length. +Status carve_region_slices(const WindowMeta& m, Slice dd_block, Slice freq_block, bool want_freq, + WindowSlices* out) { + RETURN_IF_ERROR(in_bounds(m.dd_off, m.dd_disk_len, dd_block.size())); + out->dd_region = + dd_block.subslice(static_cast(m.dd_off), static_cast(m.dd_disk_len)); + if (!want_freq) { + return Status::OK(); + } + RETURN_IF_ERROR(in_bounds(m.freq_off, m.freq_disk_len, freq_block.size())); + out->freq_region = freq_block.subslice(static_cast(m.freq_off), + static_cast(m.freq_disk_len)); + return Status::OK(); +} + +// Decodes window w from the fetched blocks (+ optional prx slice) and appends to out. +Status append_window(const WindowSlices& ws, bool want_positions, bool want_freq, + DecodedPosting* out) { + std::vector docids, freqs; + std::vector> pos; + RETURN_IF_ERROR(decode_window_slices(ws.meta, ws.dd_region, ws.freq_region, ws.prx_window, + want_positions, want_freq, &docids, &freqs, &pos)); + out->docids.insert(out->docids.end(), docids.begin(), docids.end()); + out->freqs.insert(out->freqs.end(), freqs.begin(), freqs.end()); + if (want_positions) { + for (auto& v : pos) { + out->positions.push_back(std::move(v)); + } + } + return Status::OK(); +} + +} // namespace + +Status fetch_windowed_prelude(const LogicalIndexReader& idx, const DictEntry& entry, + uint64_t frq_base, FrqPreludeReader* prelude) { + if (entry.prelude_len == 0) { + return Status::Error( + "windowed_posting: windowed entry has no prelude"); + } + if (entry.prelude_len > entry.frq_len) { + return Status::Error( + "windowed_posting: prelude_len exceeds frq_len"); + } + const uint64_t prelude_offset = prelude_abs(idx, entry, frq_base); + io::BatchRangeFetcher fetcher(idx.reader()); + const size_t h = fetcher.add(prelude_offset, entry.prelude_len); + RETURN_IF_ERROR(fetcher.fetch()); + return FrqPreludeReader::open(fetcher.get(h), prelude); +} + +Status windowed_window_range(const LogicalIndexReader& idx, const DictEntry& entry, + uint64_t frq_base, uint64_t prx_base, const FrqPreludeReader& prelude, + uint32_t w, bool want_positions, bool want_freq, WindowAbsRange* out) { + if (out == nullptr) { + return Status::Error("windowed_posting: null range"); + } + *out = WindowAbsRange {}; + BlockGeometry g; + RETURN_IF_ERROR(resolve_blocks(idx, entry, frq_base, prelude, &g)); + WindowMeta meta; + RETURN_IF_ERROR(prelude.window(w, &meta)); + + // dd sub-range within the dd-block. + RETURN_IF_ERROR(in_bounds(meta.dd_off, meta.dd_disk_len, g.dd_block_len)); + out->dd_off = g.dd_block_off + meta.dd_off; + out->dd_len = meta.dd_disk_len; + + if (want_freq) { + // Symmetric to the positions guard below: a G16 freq-elided posting + // (freq-dropped index or prune-mode bigram) declares has_freq=false in + // its prelude flags. INVALID_ARGUMENT and not FILE_CORRUPTED: the + // Doris segment iterator silently downgrades the corruption code to a + // non-index evaluation, which would mask this by-design layout. + if (!prelude.has_freq()) { + return Status::Error( + "windowed_posting: freqs requested but prelude has none"); + } + RETURN_IF_ERROR(in_bounds(meta.freq_off, meta.freq_disk_len, g.freq_block_len)); + out->freq_off = g.freq_block_off + meta.freq_off; + out->freq_len = meta.freq_disk_len; + } + + if (!want_positions) { + return Status::OK(); + } + if (!prelude.has_prx()) { + return Status::Error( + "windowed_posting: positions requested but prelude has none"); + } + const uint64_t prx_region_start = + idx.section_refs().posting_region.offset + prx_base + entry.prx_off_delta; + RETURN_IF_ERROR(in_bounds(meta.prx_off, meta.prx_len, entry.prx_len)); + out->prx_off = prx_region_start + meta.prx_off; + out->prx_len = meta.prx_len; + return Status::OK(); +} + +Status decode_window_slices(const WindowMeta& meta, Slice dd_region, Slice freq_region, + Slice prx_window, bool want_positions, bool want_freq, + std::vector* docids, std::vector* freqs, + std::vector>* positions) { + FrqRegionMeta dd_meta; + dd_meta.zstd = meta.dd_zstd; + dd_meta.uncomp_len = meta.dd_uncomp_len; + dd_meta.disk_len = meta.dd_disk_len; + dd_meta.crc = meta.crc_dd; + dd_meta.verify_crc = meta.verify_crc; + RETURN_IF_ERROR(format::decode_dd_region(dd_region, dd_meta, meta.win_base, docids)); + if (docids->size() != meta.doc_count) { + return Status::Error( + "windowed_posting: frq doc_count mismatch"); + } + if (want_freq) { + FrqRegionMeta freq_meta; + freq_meta.zstd = meta.freq_zstd; + freq_meta.uncomp_len = meta.freq_uncomp_len; + freq_meta.disk_len = meta.freq_disk_len; + freq_meta.crc = meta.crc_freq; + freq_meta.verify_crc = meta.verify_crc; + RETURN_IF_ERROR(format::decode_freq_region(freq_region, freq_meta, meta.doc_count, freqs)); + } else { + freqs->clear(); + } + if (!want_positions) { + return Status::OK(); + } + + ByteSource psrc(prx_window); + RETURN_IF_ERROR(format::read_prx_window(&psrc, positions)); + if (!psrc.eof()) { + return Status::Error( + "windowed_posting: trailing bytes after prx frame"); + } + if (positions->size() != docids->size()) { + return Status::Error( + "windowed_posting: prx/frq doc-count mismatch"); + } + return Status::OK(); +} + +namespace { + +// Fetches the dd-block (always), the freq-block (when want_freq) and the whole .prx +// region (when want_positions) of a windowed entry in ONE batch and returns the +// in-memory block slices. The dd-block is a single contiguous range -> the +// docid-only / phrase path reads it as one Range GET (the byte-saving core). +Status fetch_blocks(const LogicalIndexReader& idx, const DictEntry& entry, uint64_t prx_base, + const BlockGeometry& g, bool want_positions, bool want_freq, + io::BatchRangeFetcher* fetcher, size_t* dd_h, size_t* freq_h, size_t* prx_h) { + *dd_h = fetcher->add(g.dd_block_off, g.dd_block_len); + if (want_freq) { + *freq_h = fetcher->add(g.freq_block_off, g.freq_block_len); + } + if (want_positions) { + const uint64_t prx_region_start = + idx.section_refs().posting_region.offset + prx_base + entry.prx_off_delta; + *prx_h = fetcher->add(prx_region_start, entry.prx_len); + } + return fetcher->fetch(); +} + +} // namespace + +Status read_windowed_posting(const LogicalIndexReader& idx, const DictEntry& entry, + uint64_t frq_base, uint64_t prx_base, bool want_positions, + bool want_freq, DecodedPosting* out) { + if (out == nullptr) { + return Status::Error("windowed_posting: null out"); + } + *out = DecodedPosting {}; + + FrqPreludeReader prelude; + RETURN_IF_ERROR(fetch_windowed_prelude(idx, entry, frq_base, &prelude)); + if (want_positions && !prelude.has_prx()) { + return Status::Error( + "windowed_posting: positions requested but prelude has none"); + } + // G16 freq-elided postings (freq-dropped index or prune-mode bigram) + // declare has_freq=false; fail with the semantic error instead of a deep + // region-decode corruption. INVALID_ARGUMENT so the Doris segment + // iterator's corruption downgrade never masks this by-design layout. + if (want_freq && !prelude.has_freq()) { + return Status::Error( + "windowed_posting: freqs requested but prelude has none"); + } + BlockGeometry g; + RETURN_IF_ERROR(resolve_blocks(idx, entry, frq_base, prelude, &g)); + + io::BatchRangeFetcher fetcher(idx.reader()); + size_t dd_h = 0, freq_h = 0, prx_h = 0; + RETURN_IF_ERROR(fetch_blocks(idx, entry, prx_base, g, want_positions, want_freq, &fetcher, + &dd_h, &freq_h, &prx_h)); + const Slice dd_block = fetcher.get(dd_h); + const Slice freq_block = want_freq ? fetcher.get(freq_h) : Slice(); + const Slice prx_region = want_positions ? fetcher.get(prx_h) : Slice(); + + const uint32_t n = prelude.n_windows(); + for (uint32_t w = 0; w < n; ++w) { + WindowSlices ws; + RETURN_IF_ERROR(prelude.window(w, &ws.meta)); + RETURN_IF_ERROR(carve_region_slices(ws.meta, dd_block, freq_block, want_freq, &ws)); + if (want_positions) { + RETURN_IF_ERROR(in_bounds(ws.meta.prx_off, ws.meta.prx_len, prx_region.size())); + ws.prx_window = prx_region.subslice(static_cast(ws.meta.prx_off), + static_cast(ws.meta.prx_len)); + } + RETURN_IF_ERROR(append_window(ws, want_positions, want_freq, out)); + } + return Status::OK(); +} + +} // namespace doris::snii::reader diff --git a/be/src/storage/index/snii/reader/windowed_posting.h b/be/src/storage/index/snii/reader/windowed_posting.h new file mode 100644 index 00000000000000..dee527430aff08 --- /dev/null +++ b/be/src/storage/index/snii/reader/windowed_posting.h @@ -0,0 +1,121 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include "common/status.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/format/dict_entry.h" +#include "storage/index/snii/format/frq_prelude.h" +#include "storage/index/snii/reader/logical_index_reader.h" + +// WindowedPostingReader -- shared read-side decode of a windowed term's posting +// from its two-level frq_prelude + GROUPED dd-block / freq-block (design 1.6). +// +// A windowed pod_ref entry's .frq payload is laid out +// [prelude][dd-block][freq-block] +// where the dd-block concatenates every window's dd_region and the freq-block +// every window's freq_region. The docs-only prefix [prelude][dd-block] is ONE +// contiguous run. This helper: +// 1. range-fetches the prelude (prelude_len bytes) and parses the directory, +// 2. range-fetches the WHOLE dd-block in ONE contiguous range (and, for +// scoring, +// the whole freq-block in one more range), +// 3. decodes each window's dd region (and freq region) from the in-memory +// blocks +// via the prelude metadata (dd_off/dd_disk_len, freq_off/freq_disk_len), +// and concatenates the per-window docids / freqs / positions. +// +// The slim/inline single-window path is handled by the term/phrase/scoring +// callers directly; this helper is for enc=windowed entries only. +namespace doris::snii::reader { + +// Coalesce gap (bytes) used when batch-fetching MULTIPLE dd sub-ranges of the +// SAME term (the phrase window-skip path): dd regions of one term are +// contiguous in the dd-block, so merging reads separated by <= this gap into +// one physical Range GET trades a little over-read for fewer remote GETs (the +// design's higher-priority metric). Only applied to same-term multi-window +// batches, never to cross-term. +inline constexpr uint64_t kSameTermCoalesceGap = 16 * 1024; + +// Full decoded posting for one windowed term (docids ascending across windows). +struct DecodedPosting { + std::vector docids; + std::vector freqs; // aligned with docids + std::vector> positions; // aligned; empty when no prx +}; + +// Decodes the entire windowed posting. want_positions requires the index to +// have positions (and the entry to carry prx). want_freq selects whether the +// freq-block is fetched + decoded: when false ONLY the contiguous +// [prelude][dd-block] prefix is fetched (docid-only / phrase callers) and +// DecodedPosting.freqs stays empty; when true the freq-block is additionally +// fetched (scoring). Returns Corruption on any prelude/block inconsistency +// (doc-count mismatch, out-of-range offsets). +Status read_windowed_posting(const LogicalIndexReader& idx, const format::DictEntry& entry, + uint64_t frq_base, uint64_t prx_base, bool want_positions, + bool want_freq, DecodedPosting* out); + +// --- Sub-block (window) skipping helpers (shared with phrase / selective WAND) +// -- +// +// These expose the per-window dd/freq/prx addressing within the grouped blocks +// so the skip path can fetch ONLY the windows covering candidate docids (their +// dd sub-ranges within the dd-block, near-contiguous and coalesce-friendly) +// instead of the whole posting, without duplicating the offset arithmetic. + +// Absolute file byte ranges of one window's regions. dd is always valid; freq +// is valid only when want_freq; prx is valid only when want_positions (and +// has_prx). +struct WindowAbsRange { + uint64_t dd_off = 0; + uint64_t dd_len = 0; + uint64_t freq_off = 0; + uint64_t freq_len = 0; + uint64_t prx_off = 0; + uint64_t prx_len = 0; +}; + +// Fetches + parses the two-level prelude of a windowed entry (one batched +// read). +Status fetch_windowed_prelude(const LogicalIndexReader& idx, const format::DictEntry& entry, + uint64_t frq_base, format::FrqPreludeReader* prelude); + +// Computes the absolute file ranges of window w's dd region (and freq region +// when want_freq, and .prx window when want_positions), fully validated against +// the POD sections (anti-DoS: rejects out-of-range offsets and overflowing +// locators). +Status windowed_window_range(const LogicalIndexReader& idx, const format::DictEntry& entry, + uint64_t frq_base, uint64_t prx_base, + const format::FrqPreludeReader& prelude, uint32_t w, + bool want_positions, bool want_freq, WindowAbsRange* out); + +// Decodes one window's docids (and per-doc positions when want_positions, and +// per-doc freqs when want_freq) from already-fetched byte slices: dd_region is +// the window's dd sub-slice; freq_region its freq sub-slice (ignored when +// !want_freq); prx_window its .prx bytes. The decoded docids are absolute +// (win_base applied). Returns Corruption on any doc-count mismatch between the +// prelude, dd/freq and prx. +Status decode_window_slices(const format::WindowMeta& meta, Slice dd_region, Slice freq_region, + Slice prx_window, bool want_positions, bool want_freq, + std::vector* docids, std::vector* freqs, + std::vector>* positions); + +} // namespace doris::snii::reader diff --git a/be/src/storage/index/snii/snii_bkd_index_reader.cpp b/be/src/storage/index/snii/snii_bkd_index_reader.cpp new file mode 100644 index 00000000000000..a5b6d84cfde223 --- /dev/null +++ b/be/src/storage/index/snii/snii_bkd_index_reader.cpp @@ -0,0 +1,216 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/snii_bkd_index_reader.h" + +#include "common/check.h" +#include "runtime/runtime_state.h" +#include "storage/index/bkd_field_encoding.h" +#include "storage/index/index_file_reader.h" +#include "storage/index/inverted/inverted_index_iterator.h" +#include "storage/index/snii/format/null_bitmap.h" +#include "storage/index/snii/snii_bkd_query.h" +#include "storage/index/snii/snii_doris_adapter.h" +#include "storage/key_coder.h" +#include "util/time.h" + +namespace doris::segment_v2 { + +namespace { +::doris::snii::Slice slice_of(const std::string& bytes) { + return ::doris::snii::Slice(reinterpret_cast(bytes.data()), bytes.size()); +} +} // namespace + +Status SniiBkdIndexReader::new_iterator(std::unique_ptr* iterator) { + if (*iterator == nullptr) { + *iterator = InvertedIndexIterator::create_unique(); + } + dynamic_cast(iterator->get()) + ->add_reader(InvertedIndexReaderType::BKD, + std::dynamic_pointer_cast(shared_from_this())); + return Status::OK(); +} + +Status SniiBkdIndexReader::_get_searcher(const IndexQueryContextPtr& context, + InvertedIndexCacheHandle* cache_handle, + std::unique_ptr<::doris::snii::bkd::BkdSearcher>* uncached, + const ::doris::snii::bkd::BkdSearcher** searcher) { + DORIS_CHECK(cache_handle != nullptr); + DORIS_CHECK(uncached != nullptr); + DORIS_CHECK(searcher != nullptr); + + const bool enable_searcher_cache = + context->runtime_state != nullptr && + context->runtime_state->query_options().enable_inverted_index_searcher_cache; + const auto index_file_key = _index_file_reader->get_index_file_cache_key(&_index_meta); + InvertedIndexSearcherCache::CacheKey searcher_cache_key(index_file_key); + + if (enable_searcher_cache) { + SCOPED_RAW_TIMER(&context->stats->inverted_index_lookup_timer); + if (InvertedIndexSearcherCache::instance()->lookup(searcher_cache_key, cache_handle)) { + context->stats->inverted_index_searcher_cache_hit++; + *searcher = cache_handle->get_snii_bkd_searcher(); + if (*searcher == nullptr) { + return Status::InternalError("SNII searcher cache entry has no BKD searcher"); + } + return Status::OK(); + } + } + + SCOPED_RAW_TIMER(&context->stats->inverted_index_searcher_open_timer); + context->stats->inverted_index_searcher_cache_miss++; + RETURN_IF_ERROR( + _index_file_reader->init(config::inverted_index_read_buffer_size, context->io_ctx)); + auto opened = DORIS_TRY(_index_file_reader->open_snii_bkd_index(&_index_meta, context->io_ctx)); + + if (!enable_searcher_cache) { + *searcher = opened.get(); + *uncached = std::move(opened); + return Status::OK(); + } + + const size_t reader_size = std::max(opened->memory_usage(), 1); + auto* cache_value = new InvertedIndexSearcherCache::CacheValue( + std::move(opened), reader_size, UnixMillis(), _index_file_reader); + InvertedIndexSearcherCache::instance()->insert(searcher_cache_key, cache_value, cache_handle); + *searcher = cache_handle->get_snii_bkd_searcher(); + if (*searcher == nullptr) { + return Status::InternalError("SNII searcher cache insert produced empty BKD searcher"); + } + return Status::OK(); +} + +Status SniiBkdIndexReader::_encode_query_value(const ::doris::snii::bkd::BkdSearcher& searcher, + const Field& query_value, std::string* out) { + // The type comes out of the index HEADER, so the coder that reads is the one + // that wrote. Taking it from the query's own type instead would compare + // correctly-encoded bytes in the wrong order (INV-1). + const FieldType type = searcher.reader->header().field_type; + if (!is_scalar_type(type)) { + return Status::Error( + "unsupported bkd index type {}", static_cast(type)); + } + return encode_bkd_field_ascending(type, query_value, get_key_coder(type), out); +} + +Status SniiBkdIndexReader::query(const IndexQueryContextPtr& context, + const std::string& column_name, const Field& query_value, + InvertedIndexQueryType query_type, + std::shared_ptr& bit_map, + const InvertedIndexAnalyzerCtx* /*analyzer_ctx*/) { + SCOPED_RAW_TIMER(&context->stats->inverted_index_query_timer); + snii_doris::DorisSniiFileReader::ScopedIOContext io_context_scope(context->io_ctx); + + InvertedIndexCacheHandle searcher_cache_handle; + std::unique_ptr<::doris::snii::bkd::BkdSearcher> uncached; + const ::doris::snii::bkd::BkdSearcher* searcher = nullptr; + RETURN_IF_ERROR(_get_searcher(context, &searcher_cache_handle, &uncached, &searcher)); + + std::string query_str; + RETURN_IF_ERROR(_encode_query_value(*searcher, query_value, &query_str)); + + // The interval is resolved BEFORE the query cache is consulted: an + // unsupported shape must be refused, not answered from a cache entry some + // other predicate left behind under the same key. + BkdQueryBounds bounds; + RETURN_IF_ERROR(build_bkd_query_bounds(query_type, slice_of(query_str), &bounds)); + + auto index_file_key = _index_file_reader->get_index_file_cache_key(&_index_meta); + InvertedIndexQueryCache::CacheKey cache_key {index_file_key, column_name, query_type, + query_str}; + auto* cache = InvertedIndexQueryCache::instance(); + InvertedIndexQueryCacheHandle cache_handler; + if (handle_query_cache(context, cache, cache_key, &cache_handler, bit_map)) { + return Status::OK(); + } + + RETURN_IF_ERROR(searcher->reader->range(bounds.lower, bounds.lower_inclusive, bounds.upper, + bounds.upper_inclusive, bit_map.get())); + bit_map->runOptimize(); + // insert_query_cache, not cache->insert: it is the one that honours + // enable_inverted_index_query_cache, so a query that opted out of the cache + // does not populate it anyway. + insert_query_cache(context, cache, cache_key, bit_map, &cache_handler); + return Status::OK(); +} + +Status SniiBkdIndexReader::try_query(const IndexQueryContextPtr& context, + const std::string& /*column_name*/, const Field& query_value, + InvertedIndexQueryType query_type, size_t* count) { + DORIS_CHECK(count != nullptr); + snii_doris::DorisSniiFileReader::ScopedIOContext io_context_scope(context->io_ctx); + + InvertedIndexCacheHandle searcher_cache_handle; + std::unique_ptr<::doris::snii::bkd::BkdSearcher> uncached; + const ::doris::snii::bkd::BkdSearcher* searcher = nullptr; + RETURN_IF_ERROR(_get_searcher(context, &searcher_cache_handle, &uncached, &searcher)); + + std::string query_str; + RETURN_IF_ERROR(_encode_query_value(*searcher, query_value, &query_str)); + BkdQueryBounds bounds; + RETURN_IF_ERROR(build_bkd_query_bounds(query_type, slice_of(query_str), &bounds)); + + uint64_t estimate = 0; + RETURN_IF_ERROR(searcher->reader->estimate_cardinality( + bounds.lower, bounds.lower_inclusive, bounds.upper, bounds.upper_inclusive, &estimate)); + *count = estimate; + return Status::OK(); +} + +Status SniiBkdIndexReader::read_null_bitmap(const IndexQueryContextPtr& context, + InvertedIndexQueryCacheHandle* cache_handle, + lucene::store::Directory* /*dir*/) { + SCOPED_RAW_TIMER(&context->stats->inverted_index_query_null_bitmap_timer); + auto index_file_key = _index_file_reader->get_index_file_cache_key(&_index_meta); + InvertedIndexQueryCache::CacheKey cache_key { + index_file_key, "", InvertedIndexQueryType::UNKNOWN_QUERY, "null_bitmap"}; + auto* cache = InvertedIndexQueryCache::instance(); + if (cache->lookup(cache_key, cache_handle)) { + return Status::OK(); + } + + snii_doris::DorisSniiFileReader::ScopedIOContext io_context_scope(context->io_ctx); + InvertedIndexCacheHandle searcher_cache_handle; + std::unique_ptr<::doris::snii::bkd::BkdSearcher> uncached; + const ::doris::snii::bkd::BkdSearcher* searcher = nullptr; + RETURN_IF_ERROR(_get_searcher(context, &searcher_cache_handle, &uncached, &searcher)); + + auto null_bitmap = std::make_shared(); + if (searcher->null_bitmap_length > 0) { + std::vector bytes; + // Through the SEARCHER's own file, mirroring the text reader + // (snii_index_reader.cpp uses logical_reader->reader()->read_at). Going + // via _index_file_reader reads through THIS object's member, which on a + // searcher-cache hit may belong to a different, never-init()-ed + // IndexFileReader -- the Segment cache and the searcher cache evict on + // different axes, so a surviving searcher paired with a reloaded segment + // is ordinary under memory pressure, and it produced + // INVERTED_INDEX_FILE_NOT_FOUND for a file that is open right there. + RETURN_IF_ERROR(searcher->reader->reader()->read_at(searcher->null_bitmap_offset, + searcher->null_bitmap_length, &bytes)); + ::doris::snii::format::NullBitmapReader reader; + RETURN_IF_ERROR(::doris::snii::format::NullBitmapReader::open(::doris::snii::Slice(bytes), + &reader)); + reader.copy_to(null_bitmap.get()); + null_bitmap->runOptimize(); + } + cache->insert(cache_key, null_bitmap, cache_handle); + return Status::OK(); +} + +} // namespace doris::segment_v2 diff --git a/be/src/storage/index/snii/snii_bkd_index_reader.h b/be/src/storage/index/snii/snii_bkd_index_reader.h new file mode 100644 index 00000000000000..d80e44855a39dd --- /dev/null +++ b/be/src/storage/index/snii/snii_bkd_index_reader.h @@ -0,0 +1,78 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include "common/status.h" +#include "storage/index/inverted/inverted_index_reader.h" +#include "storage/index/snii/snii_bkd_searcher.h" + +namespace doris::segment_v2 { + +// Doris read-path adapter for the SNII-native BKD (design 10 / task P3-2b): the +// numeric counterpart of SniiIndexReader, and the drop-in replacement for the +// CLucene-backed BkdIndexReader on SNII segments. +// +// It reports type() == BKD on purpose. The predicate layer routes on exactly +// that (comparison_predicate.h and in_list_predicate.h both refuse to push a +// numeric comparison down unless the iterator has a BKD reader), so a distinct +// reader type would silently disable index acceleration for every numeric +// column in the format rather than fail loudly. +// +// Nothing here catches a CLuceneError, because nothing under it can throw one: +// this reader reaches the SNII-native core and no third-party index library. +class SniiBkdIndexReader final : public InvertedIndexReader { + ENABLE_FACTORY_CREATOR(SniiBkdIndexReader); + +public: + SniiBkdIndexReader(const TabletIndex* index_meta, + const std::shared_ptr& index_file_reader) + : InvertedIndexReader(index_meta, index_file_reader) {} + ~SniiBkdIndexReader() override = default; + + Status new_iterator(std::unique_ptr* iterator) override; + Status query(const IndexQueryContextPtr& context, const std::string& column_name, + const Field& query_value, InvertedIndexQueryType query_type, + std::shared_ptr& bit_map, + const InvertedIndexAnalyzerCtx* analyzer_ctx = nullptr) override; + Status try_query(const IndexQueryContextPtr& context, const std::string& column_name, + const Field& query_value, InvertedIndexQueryType query_type, + size_t* count) override; + Status read_null_bitmap(const IndexQueryContextPtr& context, + InvertedIndexQueryCacheHandle* cache_handle, + lucene::store::Directory* dir = nullptr) override; + InvertedIndexReaderType type() override { return InvertedIndexReaderType::BKD; } + +private: + // Resolves the opened index, from the searcher cache when the query allows + // it. `uncached` owns the reader when caching is off, `cache_handle` owns it + // otherwise; either way *searcher points at whichever is alive. + Status _get_searcher(const IndexQueryContextPtr& context, + InvertedIndexCacheHandle* cache_handle, + std::unique_ptr<::doris::snii::bkd::BkdSearcher>* uncached, + const ::doris::snii::bkd::BkdSearcher** searcher); + + // Encodes `query_value` with the key coder of the INDEX's OWN field type + // (INV-1) -- the one read out of the header, never the query's own type. + static Status _encode_query_value(const ::doris::snii::bkd::BkdSearcher& searcher, + const Field& query_value, std::string* out); +}; + +} // namespace doris::segment_v2 diff --git a/be/src/storage/index/snii/snii_bkd_index_writer.cpp b/be/src/storage/index/snii/snii_bkd_index_writer.cpp new file mode 100644 index 00000000000000..11fd8a7e9b8f3d --- /dev/null +++ b/be/src/storage/index/snii/snii_bkd_index_writer.cpp @@ -0,0 +1,229 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/snii_bkd_index_writer.h" + +#include +#include + +#include "common/check.h" +#include "storage/index/index_file_writer.h" +#include "storage/index/snii/bkd/bkd_types.h" +#include "storage/index/snii/format/metadata_directory.h" +#include "storage/index/snii/format/null_bitmap.h" +#include "storage/index/snii/writer/snii_compound_writer.h" +#include "storage/key_coder.h" +#include "storage/tablet/tablet_schema.h" +#include "storage/types.h" + +namespace doris::segment_v2 { + +namespace { +constexpr const char* kDataFileName = "bkd_data"; +constexpr const char* kIndexFileName = "bkd_index"; +constexpr const char* kNullsFileName = "bkd_nulls"; + +// A BlobFileSource over bytes the adapter already holds in RAM. The container +// pulls positionally, so the range is bounds-checked here rather than trusted: +// a short read reported as OK would be checksummed and sealed as the payload. +::doris::snii::writer::BlobFileSource resident_source(std::string name, + std::shared_ptr> bytes) { + ::doris::snii::writer::BlobFileSource source; + source.name = std::move(name); + source.length = bytes->size(); + source.read_fn = [bytes](uint64_t offset, size_t len, uint8_t* out) { + if (offset > bytes->size() || len > bytes->size() - offset) { + return Status::IOError("bkd blob staging read out of range"); + } + std::memcpy(out, bytes->data() + offset, len); + return Status::OK(); + }; + return source; +} +} // namespace + +SniiBkdIndexColumnWriter::SniiBkdIndexColumnWriter(IndexFileWriter* index_file_writer, + const TabletIndex* index_meta, + FieldType value_type) + : _index_file_writer(index_file_writer), _index_meta(index_meta), _value_type(value_type) {} + +SniiBkdIndexColumnWriter::~SniiBkdIndexColumnWriter() = default; + +Status SniiBkdIndexColumnWriter::init() { + if (!field_is_numeric_type(_value_type)) { + return Status::Error( + "SNII BKD index does not support field type {}", static_cast(_value_type)); + } + // Both resolved from the SAME FieldType, which is also the one recorded in + // the index header (INV-1): the stride the source array is walked with and + // the encoder the points are built with can never disagree. + // field_is_numeric_type() is WIDER than the set this index can actually + // encode: it admits UNSIGNED_TINYINT and UNSIGNED_SMALLINT, for which + // field_type_size LOG(FATAL)s and get_key_coder returns nullptr. No FE type + // maps to either today, so this is unreachable through DDL -- but the gate + // must not depend on that staying true, and a null coder would be a + // dereference rather than an error. + _value_key_coder = get_key_coder(_value_type); + if (_value_key_coder == nullptr) { + return Status::Error( + "SNII BKD index has no key coder for field type {}", static_cast(_value_type)); + } + _value_size = cast_set(field_type_size(_value_type)); + + ::doris::snii::bkd::BkdBuilderOptions options; + options.bytes_per_dim = _value_size; + options.field_type = _value_type; + return ::doris::snii::bkd::BkdBuilder::create(options, &_builder); +} + +Status SniiBkdIndexColumnWriter::_add_value(const void* value, uint32_t docid) { + DORIS_CHECK(_builder != nullptr); + std::string encoded; + _value_key_coder->full_encode_ascending(value, &encoded); + // full_encode_ascending is length-preserving for every numeric type; a + // disagreement here would mean the header's bytes_per_dim describes points + // the builder never produced, so it is asserted rather than tolerated. + DORIS_CHECK_EQ(encoded.size(), _value_size); + return _builder->add( + docid, + ::doris::snii::Slice(reinterpret_cast(encoded.data()), encoded.size())); +} + +Status SniiBkdIndexColumnWriter::add_values(const std::string /*name*/, const void* values, + size_t count) { + DORIS_CHECK(values != nullptr || count == 0); + const auto* cursor = static_cast(values); + for (size_t i = 0; i < count; ++i) { + RETURN_IF_ERROR(_add_value(cursor, _rid)); + cursor += _value_size; + ++_rid; + } + return Status::OK(); +} + +Status SniiBkdIndexColumnWriter::add_array_values(size_t field_size, const void* value_ptr, + const uint8_t* null_map, + const uint8_t* offsets_ptr, size_t count) { + if (count == 0) { + return Status::OK(); + } + DORIS_CHECK(value_ptr != nullptr); + DORIS_CHECK(offsets_ptr != nullptr); + // The element width comes from the caller's array layout, but the points it + // produces are the index's own field type, so a mismatch would silently + // reinterpret the payload. + DORIS_CHECK_EQ(field_size, _value_size); + + const auto* offsets = reinterpret_cast(offsets_ptr); + const auto* elements = static_cast(value_ptr); + size_t element = 0; + for (size_t i = 0; i < count; ++i) { + const size_t row_elements = offsets[i + 1] - offsets[i]; + for (size_t j = 0; j < row_elements; ++j, ++element) { + if (null_map != nullptr && null_map[element] == 1) { + continue; + } + // One row contributing several points is a first-class case; the + // builder keys on (value, doc_id), so the row id does NOT advance + // between them. + RETURN_IF_ERROR(_add_value(elements + element * _value_size, _rid)); + } + // A row that produced no point is NOT recorded as NULL here. An empty + // array, and an array whose every element is NULL, are both non-null + // arrays that simply cannot match a comparison -- and the index already + // says so by holding no point for them. Marking them NULL would make + // `col IS NULL` true for a row holding []. Array-LEVEL nulls arrive + // separately through add_array_nulls, which is their only source. + ++_rid; + } + return Status::OK(); +} + +Status SniiBkdIndexColumnWriter::add_nulls(uint32_t count) { + for (uint32_t i = 0; i < count; ++i) { + _null_docids.push_back(_rid); + ++_rid; + } + return Status::OK(); +} + +Status SniiBkdIndexColumnWriter::add_array_nulls(const uint8_t* null_map, size_t num_rows) { + DORIS_CHECK(null_map != nullptr || num_rows == 0); + // Called for the SAME rows add_array_values already walked, so it must not + // advance the row id -- it only records which of those rows were NULL at the + // array level. + DORIS_CHECK_GE(_rid, num_rows); + const uint32_t first_rid = _rid - cast_set(num_rows); + for (size_t i = 0; i < num_rows; ++i) { + if (null_map[i] == 1) { + _null_docids.push_back(first_rid + cast_set(i)); + } + } + return Status::OK(); +} + +Status SniiBkdIndexColumnWriter::finish() { + DORIS_CHECK(_builder != nullptr); + DORIS_CHECK(_index_file_writer != nullptr); + + // bkd_data is sized by the point count, so it is staged through a temp file + // rather than held in RAM; the two hot sub-files are small by construction. + std::unique_ptr<::doris::snii::bkd::StagedBlobFile> data; + RETURN_IF_ERROR(::doris::snii::bkd::StagedBlobFile::create(kDataFileName, &data)); + + ::doris::snii::ByteSink index_sink; + ::doris::snii::bkd::BkdStats stats; + RETURN_IF_ERROR(_builder->finish(data.get(), &index_sink, &stats)); + RETURN_IF_ERROR(data->finalize()); + + ::doris::snii::format::NullBitmapWriter null_writer; + null_writer.add_many(_null_docids); + ::doris::snii::ByteSink null_sink; + RETURN_IF_ERROR(null_writer.finish(_rid, &null_sink)); + + // The container pulls at IndexFileWriter::finish_close(), long after this + // returns, so the staging file has to outlive this object. Ownership moves + // into the read callback itself. + _data = std::move(data); + std::shared_ptr<::doris::snii::bkd::StagedBlobFile> staged = _data; + ::doris::snii::writer::BlobFileSource cold; + cold.name = kDataFileName; + cold.length = staged->bytes_written(); + cold.read_fn = [staged](uint64_t offset, size_t len, uint8_t* out) { + return staged->read_at(offset, len, out); + }; + + std::vector<::doris::snii::writer::BlobFileSource> hot; + hot.push_back(resident_source(kIndexFileName, + std::make_shared>(index_sink.take()))); + hot.push_back(resident_source(kNullsFileName, + std::make_shared>(null_sink.take()))); + + return _index_file_writer->add_snii_blob_index(_index_meta, + ::doris::snii::format::LogicalIndexKind::kBkd, + {std::move(cold)}, std::move(hot)); +} + +void SniiBkdIndexColumnWriter::close_on_error() { + // The builder unlinks its own spilled runs; the staging file removes itself + // on destruction. Dropping both here means an aborted segment leaves no temp + // file behind even if this writer is kept alive for a while. + _builder.reset(); + _data.reset(); +} + +} // namespace doris::segment_v2 diff --git a/be/src/storage/index/snii/snii_bkd_index_writer.h b/be/src/storage/index/snii/snii_bkd_index_writer.h new file mode 100644 index 00000000000000..62571b86fe85e1 --- /dev/null +++ b/be/src/storage/index/snii/snii_bkd_index_writer.h @@ -0,0 +1,100 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/index_writer.h" +#include "storage/index/snii/bkd/bkd_builder.h" +#include "storage/index/snii/bkd/staged_blob_file.h" +#include "storage/index/snii/encoding/byte_sink.h" +#include "storage/olap_common.h" + +namespace doris { +class KeyCoder; +class TabletIndex; + +namespace segment_v2 { +class IndexFileWriter; + +// Doris write-path adapter for the SNII-native BKD (design 10): the numeric +// counterpart of SniiIndexColumnWriter, which serves the text side of the same +// container. +// +// It is NOT a template on FieldType, unlike the CLucene-era +// InvertedIndexColumnWriter. Everything that path needed the type +// parameter for -- the value stride and the encoder -- is available at runtime +// from the index's own FieldType (field_type_size + get_key_coder), and +// resolving both from ONE source is exactly what INV-1 asks for: an index +// encoded with a coder other than its recorded field_type's is self-consistent +// but compares in the wrong order, and no round-trip test can see it. +// +// The three sub-files it registers on the container: +// bkd_data (cold) -- leaf blocks, sized by the point count, so it is staged +// through a temp file rather than held in RAM; +// bkd_index (hot) -- the framed header/bounds/splits/leaf directory; +// bkd_nulls (hot) -- the SNII null-bitmap POD. NULL rows own no point (a +// NULL that leaked in as a point would answer `col > x`), +// so they are carried here and nowhere else. +class SniiBkdIndexColumnWriter final : public IndexColumnWriter { +public: + SniiBkdIndexColumnWriter(IndexFileWriter* index_file_writer, const TabletIndex* index_meta, + FieldType value_type); + ~SniiBkdIndexColumnWriter() override; + + Status init() override; + Status add_values(const std::string name, const void* values, size_t count) override; + Status add_array_values(size_t field_size, const void* value_ptr, const uint8_t* null_map, + const uint8_t* offsets_ptr, size_t count) override; + Status add_nulls(uint32_t count) override; + Status add_array_nulls(const uint8_t* null_map, size_t num_rows) override; + Status finish() override; + int64_t size() const override { return 0; } + void close_on_error() override; + +private: + // Encodes one CppType-wide value at `value` through the index's own key + // coder and appends it as a point for `docid`. + Status _add_value(const void* value, uint32_t docid); + + IndexFileWriter* _index_file_writer = nullptr; + const TabletIndex* _index_meta = nullptr; + const FieldType _value_type; + // sizeof(CppType) for _value_type: both the source stride and the point + // width, which is why they cannot disagree. + uint32_t _value_size = 0; + const KeyCoder* _value_key_coder = nullptr; + + // Segment-local row id of the NEXT row to arrive. Advanced by both value + // runs and null runs, so it is the row count once the column is exhausted. + uint32_t _rid = 0; + std::vector _null_docids; + + std::unique_ptr<::doris::snii::bkd::BkdBuilder> _builder; + // Staged bkd_data, kept alive until the container has pulled its bytes at + // IndexFileWriter::finish_close(). Destroying it earlier would unlink the + // temp file out from under the pull. + std::shared_ptr<::doris::snii::bkd::StagedBlobFile> _data; +}; + +} // namespace segment_v2 +} // namespace doris diff --git a/be/src/storage/index/snii/snii_bkd_query.cpp b/be/src/storage/index/snii/snii_bkd_query.cpp new file mode 100644 index 00000000000000..62f9ab6a2f7451 --- /dev/null +++ b/be/src/storage/index/snii/snii_bkd_query.cpp @@ -0,0 +1,61 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/snii_bkd_query.h" + +#include "common/check.h" + +namespace doris::segment_v2 { + +Status build_bkd_query_bounds(InvertedIndexQueryType query_type, snii::Slice value, + BkdQueryBounds* out) { + DORIS_CHECK(out != nullptr); + *out = BkdQueryBounds(); + switch (query_type) { + case InvertedIndexQueryType::EQUAL_QUERY: + out->lower = value; + out->lower_inclusive = true; + out->upper = value; + out->upper_inclusive = true; + return Status::OK(); + case InvertedIndexQueryType::LESS_THAN_QUERY: + out->upper = value; + out->upper_inclusive = false; + return Status::OK(); + case InvertedIndexQueryType::LESS_EQUAL_QUERY: + out->upper = value; + out->upper_inclusive = true; + return Status::OK(); + case InvertedIndexQueryType::GREATER_THAN_QUERY: + out->lower = value; + out->lower_inclusive = false; + return Status::OK(); + case InvertedIndexQueryType::GREATER_EQUAL_QUERY: + out->lower = value; + out->lower_inclusive = true; + return Status::OK(); + default: + break; + } + // Refused, not approximated: the caller keeps the predicate and evaluates it + // normally. Answering an unsupported shape with some nearby interval would be + // a wrong answer with no error attached. + return Status::Error( + "bkd index does not support query type {}", static_cast(query_type)); +} + +} // namespace doris::segment_v2 diff --git a/be/src/storage/index/snii/snii_bkd_query.h b/be/src/storage/index/snii/snii_bkd_query.h new file mode 100644 index 00000000000000..af0fa835d920f9 --- /dev/null +++ b/be/src/storage/index/snii/snii_bkd_query.h @@ -0,0 +1,65 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "common/status.h" +#include "storage/index/inverted/inverted_index_query_type.h" +#include "storage/index/snii/common/slice.h" + +// Predicate -> interval translation for the SNII-native BKD (design 7.1 / 10). +// +// This is the whole of what the old implementation spread across five template +// specializations of InvertedIndexVisitor, each with its own matches() and +// compare(). One interval primitive needs one translation, and it is a pure +// function of the query type -- no index, no I/O, no KeyCoder. +// +// ENCODING IS NOT DONE HERE ON PURPOSE. The caller passes bytes already produced +// by the KeyCoder of the INDEX's own field_type (INV-1); resolving that type is +// the reader's job because only the reader has read the header. Keeping the two +// apart is what makes this testable without an index at all. +namespace doris::segment_v2 { + +// A closed-or-open interval in sortable-byte space, in the shape +// BkdReader::range takes. An EMPTY bound is an unbounded side. +struct BkdQueryBounds { + snii::Slice lower; + bool lower_inclusive = true; + snii::Slice upper; + bool upper_inclusive = true; +}; + +// Maps one supported predicate onto its interval. +// +// The open side is left UNBOUNDED rather than pinned to the type's minimum or +// maximum. The old implementation had to pin it -- its bounds were always closed +// and strictness lived in matches() -- which meant every one-sided query carried +// a type-limit encode it never used. Here the strictness is the interval's own, +// so `<` and `<=` differ by a flag and nothing else. +// +// `value` must be exactly the index's bytes_per_dim; that is the caller's +// invariant and is checked by BkdReader::range itself. +// +// Anything outside {EQUAL, LESS_THAN, LESS_EQUAL, GREATER_THAN, GREATER_EQUAL} +// comes back as INVERTED_INDEX_NOT_SUPPORTED so the caller falls back to a +// normal predicate rather than silently answering the wrong question. In +// particular RANGE_QUERY and LIST_QUERY exist in the enum but are produced only +// by the SEARCH DSL and never reach a BKD reader. +Status build_bkd_query_bounds(InvertedIndexQueryType query_type, snii::Slice value, + BkdQueryBounds* out); + +} // namespace doris::segment_v2 diff --git a/be/src/storage/index/snii/snii_bkd_searcher.h b/be/src/storage/index/snii/snii_bkd_searcher.h new file mode 100644 index 00000000000000..f74934d4bd4e64 --- /dev/null +++ b/be/src/storage/index/snii/snii_bkd_searcher.h @@ -0,0 +1,48 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include "storage/index/snii/bkd/bkd_reader.h" + +// What one opened SNII BKD logical index amounts to, and therefore what the +// searcher cache holds on to. +// +// The BkdReader alone is not enough: the null bitmap is a THIRD sub-file of the +// same blob entry, and re-resolving the container directory to find it on every +// read_null_bitmap would undo the point of caching. Its extent is 16 bytes of +// metadata, so it rides along. +namespace doris::snii::bkd { + +struct BkdSearcher { + std::unique_ptr reader; + // Extent of the bkd_nulls sub-file inside the container. length == 0 means + // the column had no NULL row -- a legal state, not a missing section. + uint64_t null_bitmap_offset = 0; + uint64_t null_bitmap_length = 0; + + // Real resident cost, for the searcher cache's accounting. The null extent + // is not counted because its bytes are not held here. + size_t memory_usage() const { + return sizeof(*this) + (reader != nullptr ? reader->memory_usage() : 0); + } +}; + +} // namespace doris::snii::bkd diff --git a/be/src/storage/index/snii/snii_blob_directory.cpp b/be/src/storage/index/snii/snii_blob_directory.cpp new file mode 100644 index 00000000000000..628ab0ef85e7aa --- /dev/null +++ b/be/src/storage/index/snii/snii_blob_directory.cpp @@ -0,0 +1,264 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/snii_blob_directory.h" + +#include + +#include + +#include "common/check.h" + +namespace doris::segment_v2::snii_doris { +namespace { + +// IndexInput over one blob sub-file. readInternal fills the caller's buffer +// through DorisSniiFileReader::read_into -- one positional read, no vector +// round-trip -- so BufferedIndexInput's large-read bypass (len >= bufferSize +// goes straight to the destination) costs no extra resident copy. read_into is +// stateless, so clones share nothing but the reader handle: no base-stream +// mutex (unlike CSIndexInput, whose shared base cursor needs one). +class SniiBlobIndexInput final : public lucene::store::BufferedIndexInput { +public: + SniiBlobIndexInput(std::shared_ptr reader, uint64_t blob_offset, + int64_t blob_length, int32_t buffer_size) + : BufferedIndexInput(buffer_size), + _reader(std::move(reader)), + _blob_offset(blob_offset), + _blob_length(blob_length) {} + + // A clone deliberately does NOT inherit _io_ctx (it stays null), matching + // CSIndexInput. The context is a borrowed, non-owning pointer whose lifetime + // is the caller's stack frame; a clone can outlive it (bkd_reader keeps + // per-intersect-state clones), so inheriting it would leave a dangling + // pointer that readInternal would dereference. Consumers that need a context + // on a clone set it explicitly -- bkd_reader's intersect_state does exactly + // that right after cloning. + SniiBlobIndexInput(const SniiBlobIndexInput& other) + : BufferedIndexInput(other), + _reader(other._reader), + _blob_offset(other._blob_offset), + _blob_length(other._blob_length) {} + + lucene::store::IndexInput* clone() const override { return _CLNEW SniiBlobIndexInput(*this); } + void close() override {} + int64_t length() const override { return _blob_length; } + const char* getDirectoryType() const override { return SniiBlobDirectory::getClassName(); } + const char* getObjectName() const override { return getClassName(); } + static const char* getClassName() { return "SniiBlobIndexInput"; } + void setIoContext(const void* io_ctx) override { + _io_ctx = static_cast(io_ctx); + } + +protected: + void readInternal(uint8_t* b, const int32_t len) override { + const int64_t start = getFilePointer(); + if (len < 0 || start > _blob_length || len > _blob_length - start) { + _CLTHROWA(CL_ERR_IO, "read past EOF"); + } + // IOContext is a thread-local SCOPE on the SNII read stack, not a + // parameter: an explicitly injected context is pushed around this one + // read; otherwise the reader's own default (INDEX-partition) context + // applies inside read_into. + Status status; + if (_io_ctx != nullptr) { + DorisSniiFileReader::ScopedIOContext scoped(_io_ctx); + status = _reader->read_into(_blob_offset + static_cast(start), b, + static_cast(len)); + } else { + status = _reader->read_into(_blob_offset + static_cast(start), b, + static_cast(len)); + } + if (!status.ok()) { + // CLuceneError STRDUPs the message; the temporary is safe. + _CLTHROWA(CL_ERR_IO, status.to_string().c_str()); + } + } + void seekInternal(const int64_t /*pos*/) override {} + +private: + std::shared_ptr _reader; + uint64_t _blob_offset; + int64_t _blob_length; + const io::IOContext* _io_ctx = nullptr; +}; + +} // namespace + +Status SniiBlobDirectory::open(std::shared_ptr reader, + const snii::format::LogicalIndexMetadataRef& entry, + uint64_t data_area_end, SniiBlobDirectoryPtr* out, + int32_t read_buffer_size) { + if (out == nullptr) { + return Status::Error("blob directory: null out"); + } + out->reset(); + if (reader == nullptr) { + return Status::Error("blob directory: null reader"); + } + if (entry.kind == snii::format::LogicalIndexKind::kInverted) { + return Status::Error( + "blob directory: entry is a text inverted index"); + } + if (entry.files.empty()) { + return Status::Error( + "blob directory: entry has no files"); + } + if (data_area_end > reader->size()) { + return Status::Error( + fmt::format("blob directory: data area end {} exceeds container size {}", + data_area_end, reader->size())); + } + for (const snii::format::NamedBlobFileRef& file : entry.files) { + if (file.offset > data_area_end || file.length > data_area_end - file.offset) { + return Status::Error(fmt::format( + "blob directory: file {} [{}, +{}) is outside the container data area of " + "{} bytes", + file.name, file.offset, file.length, data_area_end)); + } + } + if (read_buffer_size <= 0) { + read_buffer_size = lucene::store::BufferedIndexInput::BUFFER_SIZE; + } + out->reset(new SniiBlobDirectory(std::move(reader), entry, read_buffer_size)); + return Status::OK(); +} + +SniiBlobDirectory::SniiBlobDirectory(std::shared_ptr reader, + snii::format::LogicalIndexMetadataRef entry, + int32_t read_buffer_size) + : _reader(std::move(reader)), + _entry(std::move(entry)), + _read_buffer_size(read_buffer_size) {} + +SniiBlobDirectory::~SniiBlobDirectory() = default; + +const snii::format::NamedBlobFileRef* SniiBlobDirectory::find_file(const char* name) const { + // A null name is a caller bug, not an absent file: returning "absent" would + // hide it, and every caller then formats `name` into its error message. + DORIS_CHECK(name != nullptr); + for (const snii::format::NamedBlobFileRef& file : _entry.files) { + if (file.name == name) { + return &file; + } + } + return nullptr; +} + +bool SniiBlobDirectory::list(std::vector* names) const { + DORIS_CHECK(names != nullptr); + for (const snii::format::NamedBlobFileRef& file : _entry.files) { + names->push_back(file.name); + } + return true; +} + +bool SniiBlobDirectory::fileExists(const char* name) const { + return find_file(name) != nullptr; +} + +int64_t SniiBlobDirectory::fileModified(const char* /*name*/) const { + return 0; +} + +int64_t SniiBlobDirectory::fileLength(const char* name) const { + const snii::format::NamedBlobFileRef* file = find_file(name); + if (file == nullptr) { + const std::string message = fmt::format("File does not exist in SNII blob entry: {}", name); + _CLTHROWA(CL_ERR_IO, message.c_str()); // CLuceneError STRDUPs the message + } + return static_cast(file->length); +} + +bool SniiBlobDirectory::openInput(const char* name, lucene::store::IndexInput*& ret, + CLuceneError& err, int32_t bufferSize) { + ret = nullptr; + if (_closed) { + err.set(CL_ERR_IO, "SniiBlobDirectory is already closed"); + return false; + } + const snii::format::NamedBlobFileRef* file = find_file(name); + if (file == nullptr) { + // Report-and-return rather than throw, matching DorisCompoundReader: this + // is the 4-arg overload's contract. Note that CLucene's 2-arg + // Directory::openInput re-throws this err (Directory.cpp), and + // bkd_reader::open() uses THAT overload -- so a genuinely missing file + // still surfaces as a CLuceneError to the searcher, which is correct: a + // missing sub-file is an error. It is a ZERO-LENGTH sub-file, handled + // below, that must not be confused with corruption. + err.set(CL_ERR_IO, fmt::format("SNII blob entry (index_id={}, suffix={}) has no file '{}'", + _entry.index_id, _entry.index_suffix, name) + .c_str()); + return false; + } + if (bufferSize <= 0) { + bufferSize = _read_buffer_size; + } + // A 0-length entry yields a length()==0 input on purpose: an empty BKD + // segment stores 0-byte `bkd` / `bkd_index` files, and throwing here (the + // FSIndexInput EmptyIndexSegment behavior) would turn "empty" into + // "corrupt" (design 2026-07-28 §3.4). + ret = _CLNEW SniiBlobIndexInput(_reader, file->offset, static_cast(file->length), + bufferSize); + return true; +} + +void SniiBlobDirectory::renameFile(const char* /*from*/, const char* /*to*/) { + _CLTHROWA(CL_ERR_UnsupportedOperation, + "UnsupportedOperationException: SniiBlobDirectory::renameFile"); +} + +void SniiBlobDirectory::touchFile(const char* /*name*/) { + _CLTHROWA(CL_ERR_UnsupportedOperation, + "UnsupportedOperationException: SniiBlobDirectory::touchFile"); +} + +lucene::store::IndexOutput* SniiBlobDirectory::createOutput(const char* /*name*/) { + _CLTHROWA(CL_ERR_UnsupportedOperation, + "UnsupportedOperationException: SniiBlobDirectory::createOutput"); +} + +bool SniiBlobDirectory::doDeleteFile(const char* /*name*/) { + _CLTHROWA(CL_ERR_UnsupportedOperation, + "UnsupportedOperationException: SniiBlobDirectory::doDeleteFile"); +} + +// Never throws: ~bkd_reader (implicitly noexcept) calls close() whenever it +// owns the directory, with no try/catch anywhere on that path. Both statements +// below are noexcept. Releasing the reader here (rather than at destruction) +// drops this directory's hold on the segment file handle at the point the +// caller asked for it; already-opened inputs keep their own shared_ptr and stay +// valid, exactly as CLucene expects. +void SniiBlobDirectory::close() { + _closed = true; + _reader.reset(); +} + +std::string SniiBlobDirectory::toString() const { + return fmt::format("SniiBlobDirectory(index_id={}, suffix={})", _entry.index_id, + _entry.index_suffix); +} + +const char* SniiBlobDirectory::getClassName() { + return "SniiBlobDirectory"; +} + +const char* SniiBlobDirectory::getObjectName() const { + return getClassName(); +} + +} // namespace doris::segment_v2::snii_doris diff --git a/be/src/storage/index/snii/snii_blob_directory.h b/be/src/storage/index/snii/snii_blob_directory.h new file mode 100644 index 00000000000000..285915d5331704 --- /dev/null +++ b/be/src/storage/index/snii/snii_blob_directory.h @@ -0,0 +1,108 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include // IWYU pragma: keep +#include +#include + +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/inverted/inverted_index_common.h" +#include "storage/index/snii/format/metadata_directory.h" +#include "storage/index/snii/snii_doris_adapter.h" + +class CLuceneError; + +namespace doris::segment_v2::snii_doris { + +class SniiBlobDirectory; +// lucene::store::Directory is LUCENE_REFBASE: consumers (e.g. bkd_reader) +// _CL_POINTER / _CLDECDELETE it, so the owner must release through the same +// refcount -- a plain `delete` would double-free against a holder's DEC. +using SniiBlobDirectoryPtr = std::unique_ptr; + +// Read-only lucene::store::Directory over ONE blob logical index entry of an +// SNII container (design 2026-07-28 §5.1). It serves the entry's named files +// as BufferedIndexInputs whose readInternal lands directly in the caller's +// buffer via DorisSniiFileReader::read_into -- no per-refill allocation, no +// second GiB-scale buffer on whole-blob loads. This is the ONLY new read +// component blob indexes need: CLucene's bkd_reader and faiss's IOReader wrap +// plain Directory/IndexInput and run on it unmodified. +// +// Contract pins: +// * openInput on a 0-length entry returns a length()==0 input (an empty BKD +// segment stores 0-byte `bkd`/`bkd_index`; throwing would make +// inverted_index_searcher misreport "empty" as "corrupt"); +// * close() NEVER throws (~bkd_reader is implicitly noexcept and calls it +// when close_directory=true); +// * write operations throw UnsupportedOperation, mirroring +// DorisCompoundReader; +// * clones are independent (read_into is a stateless positional read; no +// base-stream mutex needed, unlike CSIndexInput). +class SniiBlobDirectory : public lucene::store::Directory { +public: + // Validates the entry (blob kind; every file inside [0, data_area_end)) and + // builds the directory. `reader` is shared into every opened input, so the + // directory and its inputs may outlive the caller's reference. + // + // `data_area_end` is the container's metadata-directory offset, i.e. the + // exclusive upper bound of the region blob bytes may occupy — obtain it from + // SniiSegmentReader::directory_offset(). Bounding against the whole file + // size instead would accept a corrupt entry pointing into the directory or + // the tail. + static Status open(std::shared_ptr reader, + const snii::format::LogicalIndexMetadataRef& entry, uint64_t data_area_end, + SniiBlobDirectoryPtr* out, int32_t read_buffer_size = -1); + + ~SniiBlobDirectory() override; + + bool list(std::vector* names) const override; + bool fileExists(const char* name) const override; + int64_t fileModified(const char* name) const override; + int64_t fileLength(const char* name) const override; + bool openInput(const char* name, lucene::store::IndexInput*& ret, CLuceneError& err, + int32_t bufferSize = -1) override; + void renameFile(const char* from, const char* to) override; + void touchFile(const char* name) override; + lucene::store::IndexOutput* createOutput(const char* name) override; + void close() override; + std::string toString() const override; + static const char* getClassName(); + const char* getObjectName() const override; + +protected: + bool doDeleteFile(const char* name) override; + +private: + SniiBlobDirectory(std::shared_ptr reader, + snii::format::LogicalIndexMetadataRef entry, int32_t read_buffer_size); + + const snii::format::NamedBlobFileRef* find_file(const char* name) const; + + std::shared_ptr _reader; + snii::format::LogicalIndexMetadataRef _entry; + int32_t _read_buffer_size; + bool _closed = false; +}; + +} // namespace doris::segment_v2::snii_doris diff --git a/be/src/storage/index/snii/snii_blob_staging_directory.cpp b/be/src/storage/index/snii/snii_blob_staging_directory.cpp new file mode 100644 index 00000000000000..c8b39bdb23de01 --- /dev/null +++ b/be/src/storage/index/snii/snii_blob_staging_directory.cpp @@ -0,0 +1,212 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/snii_blob_staging_directory.h" + +#include + +#include +#include + +#include "common/check.h" +#include "common/status.h" + +namespace doris::segment_v2::snii_doris { + +// Appends into one staged buffer. BufferedIndexOutput already batches the +// writer's byte-at-a-time calls, so flushBuffer sees 64 KiB chunks and the +// buffer grows geometrically through vector::insert. +// +// The buffer is held by shared_ptr because a blob source handed to the container +// keeps it alive on its own: an output may be closed and destroyed, and the +// directory itself dropped, long before finish() pulls the bytes. +class SniiBlobStagingDirectory::StagingIndexOutput final + : public lucene::store::BufferedIndexOutput { +public: + explicit StagingIndexOutput(std::shared_ptr buffer) : _buffer(std::move(buffer)) {} + + ~StagingIndexOutput() override { + // MUST close here, qualified. ~BufferedIndexOutput also calls close() if + // the caller did not -- but by then this subobject is gone, so the call + // resolves to BufferedIndexOutput::close() -> flush() -> the PURE virtual + // flushBuffer, i.e. __cxa_pure_virtual and a BE abort. Closing first + // frees the base buffer, which is exactly what stops the base destructor + // from trying. Same reason DorisFSDirectory::FSIndexOutput does it. + try { + StagingIndexOutput::close(); + } catch (const CLuceneError&) { + // A destructor may not throw. Nothing here can fail anyway -- + // flushBuffer only appends to a vector -- but the base close() is + // declared throwing, so the guard has to exist. + } + } + + void close() override { BufferedIndexOutput::close(); } + + int64_t length() const override { return static_cast(_buffer->size()); } + +protected: + void flushBuffer(const uint8_t* b, const int32_t size) override { + // flush() is also called with an empty buffer (on close, and on every + // seek), which is not an error. + if (b == nullptr || size <= 0) { + return; + } + _buffer->insert(_buffer->end(), b, b + size); + } + +private: + const std::shared_ptr _buffer; +}; + +SniiBlobStagingDirectory::~SniiBlobStagingDirectory() = default; + +const char* SniiBlobStagingDirectory::getClassName() { + return "SniiBlobStagingDirectory"; +} + +const char* SniiBlobStagingDirectory::getObjectName() const { + return getClassName(); +} + +const std::shared_ptr* SniiBlobStagingDirectory::find_file( + const char* name) const { + // A null name is a caller bug, not an absent file; reporting "absent" would + // hide it, and every caller formats `name` into its error message. + DORIS_CHECK(name != nullptr); + const auto it = _files.find(name); + return it == _files.end() ? nullptr : &it->second; +} + +bool SniiBlobStagingDirectory::list(std::vector* names) const { + DORIS_CHECK(names != nullptr); + for (const auto& [name, buffer] : _files) { + names->push_back(name); + } + return true; +} + +bool SniiBlobStagingDirectory::fileExists(const char* name) const { + return find_file(name) != nullptr; +} + +int64_t SniiBlobStagingDirectory::fileModified(const char* name) const { + // Nothing staged has a modification time, but an ABSENT name is still an + // error -- reporting 0 for it would be the one silently-successful answer in + // a class where every other lookup fails loudly. + if (find_file(name) == nullptr) { + const std::string message = + fmt::format("File does not exist in the SNII staging directory: {}", name); + _CLTHROWA(CL_ERR_IO, message.c_str()); // CLuceneError STRDUPs the message + } + return 0; +} + +int64_t SniiBlobStagingDirectory::fileLength(const char* name) const { + const std::shared_ptr* buffer = find_file(name); + if (buffer == nullptr) { + const std::string message = + fmt::format("File does not exist in the SNII staging directory: {}", name); + _CLTHROWA(CL_ERR_IO, message.c_str()); // CLuceneError STRDUPs the message + } + return static_cast((*buffer)->size()); +} + +bool SniiBlobStagingDirectory::openInput(const char* name, lucene::store::IndexInput*& ret, + CLuceneError& err, int32_t /*bufferSize*/) { + ret = nullptr; + // Refused rather than implemented: a staged index is never read back from + // here. Once it is sealed the container owns the bytes and SniiBlobDirectory + // serves them; serving a second, pre-seal copy would be a way for a reader to + // silently disagree with what was written. + err.set(CL_ERR_UnsupportedOperation, + fmt::format("SniiBlobStagingDirectory is write-only; cannot open '{}'", name).c_str()); + return false; +} + +void SniiBlobStagingDirectory::renameFile(const char* /*from*/, const char* /*to*/) { + _CLTHROWA(CL_ERR_UnsupportedOperation, + "UnsupportedOperationException: SniiBlobStagingDirectory::renameFile"); +} + +void SniiBlobStagingDirectory::touchFile(const char* /*name*/) { + _CLTHROWA(CL_ERR_UnsupportedOperation, + "UnsupportedOperationException: SniiBlobStagingDirectory::touchFile"); +} + +lucene::store::IndexOutput* SniiBlobStagingDirectory::createOutput(const char* name) { + DORIS_CHECK(name != nullptr); + // Same semantics as a filesystem directory: creating an existing name + // truncates it. The buffer is replaced rather than cleared, so a blob source + // already taken over the old content keeps reading the old content instead of + // seeing it mutate underneath. + auto buffer = std::make_shared(); + _files[name] = buffer; + return _CLNEW StagingIndexOutput(std::move(buffer)); +} + +bool SniiBlobStagingDirectory::doDeleteFile(const char* name) { + DORIS_CHECK(name != nullptr); + // Reports whether anything was actually removed: Directory::deleteFile turns + // false into an error, and claiming to have deleted a file that was never + // staged would hide a caller's wrong name. + return _files.erase(name) > 0; +} + +void SniiBlobStagingDirectory::close() { + // Deliberately keeps the staged files: close() is what a CLucene writer calls + // when it is done producing, and the harvest happens afterwards. The buffers + // die with the directory, or with the last blob source over them. +} + +std::string SniiBlobStagingDirectory::toString() const { + return fmt::format("SniiBlobStagingDirectory(files={}, bytes={})", _files.size(), + staged_bytes()); +} + +std::vector SniiBlobStagingDirectory::blob_sources() const { + std::vector sources; + sources.reserve(_files.size()); + // std::map iterates in name order, which is the order the filesystem harvest + // produced by sorting list(). Two builds of one index therefore lay their + // sub-files out identically in the container. + for (const auto& [name, buffer] : _files) { + sources.push_back(snii::writer::BlobFileSource { + .name = name, + .length = buffer->size(), + .read_fn = [buffer](uint64_t offset, size_t len, uint8_t* out) -> Status { + if (offset > buffer->size() || len > buffer->size() - offset) { + return Status::Error( + "SNII staging read [{}, +{}) is outside the staged {} bytes", + offset, len, buffer->size()); + } + std::memcpy(out, buffer->data() + offset, len); + return Status::OK(); + }}); + } + return sources; +} + +uint64_t SniiBlobStagingDirectory::staged_bytes() const { + uint64_t total = 0; + for (const auto& [name, buffer] : _files) { + total += buffer->size(); + } + return total; +} + +} // namespace doris::segment_v2::snii_doris diff --git a/be/src/storage/index/snii/snii_blob_staging_directory.h b/be/src/storage/index/snii/snii_blob_staging_directory.h new file mode 100644 index 00000000000000..a1f4256a3e971f --- /dev/null +++ b/be/src/storage/index/snii/snii_blob_staging_directory.h @@ -0,0 +1,102 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include // IWYU pragma: keep +#include +#include + +#include +#include +#include +#include +#include + +#include "storage/index/snii/writer/snii_compound_writer.h" + +class CLuceneError; + +namespace doris::segment_v2::snii_doris { + +// Write-only, memory-backed lucene::store::Directory: the staging area an ANN +// index is built into before it is sealed as a blob logical index of a SNII +// container. +// +// WHY IT EXISTS. A SNII container cannot take the faiss bytes as they are +// produced -- blob payloads are streamed by SniiCompoundWriter::finish(), after +// the text physical sections -- so the bytes must be parked somewhere in +// between. Borrowing a CLucene filesystem directory for that, as the V1/V2 +// formats do, buys two problems SNII has no use for: the temp directory is +// removed by exactly one call, so every early return on the close path leaks it +// until a BE restart wipes the tmp dir; and that call, deleteDirectory(), +// throws CLuceneError, which must not cross a Status-returning close. Parking +// the bytes here removes both -- nothing lands on disk, and there is nothing to +// delete. +// +// SCOPE. Only the write side is real. The faiss writer uses createOutput() and +// toString() and nothing else, and a sealed ANN blob is read back through +// SniiBlobDirectory over the container, never through here -- so openInput() +// refuses rather than pretending to serve a staged file. +// +// THREADING. One directory belongs to one ANN column writer and is harvested +// later, on the close path; the two are sequenced, never concurrent. No locking. +class SniiBlobStagingDirectory : public lucene::store::Directory { +public: + SniiBlobStagingDirectory() = default; + ~SniiBlobStagingDirectory() override; + + bool list(std::vector* names) const override; + bool fileExists(const char* name) const override; + int64_t fileModified(const char* name) const override; + int64_t fileLength(const char* name) const override; + bool openInput(const char* name, lucene::store::IndexInput*& ret, CLuceneError& err, + int32_t bufferSize = -1) override; + void renameFile(const char* from, const char* to) override; + void touchFile(const char* name) override; + lucene::store::IndexOutput* createOutput(const char* name) override; + void close() override; + std::string toString() const override; + static const char* getClassName(); + const char* getObjectName() const override; + + // Blob sources over the staged buffers, in name order -- the same order the + // filesystem harvest produced by sorting list(), so the container lays two + // builds of one index out identically. + // + // Each source keeps its buffer alive on its own, so the sources stay valid + // after this directory is destroyed: SniiCompoundWriter::finish() pulls them + // long after the ANN writer is gone. + std::vector blob_sources() const; + + // Bytes currently held in memory across every staged sub-file. + uint64_t staged_bytes() const; + +protected: + bool doDeleteFile(const char* name) override; + +private: + class StagingIndexOutput; + + using Buffer = std::vector; + + const std::shared_ptr* find_file(const char* name) const; + + std::map> _files; +}; + +} // namespace doris::segment_v2::snii_doris diff --git a/be/src/storage/index/snii/snii_doris_adapter.cpp b/be/src/storage/index/snii/snii_doris_adapter.cpp new file mode 100644 index 00000000000000..62543ea04f0d31 --- /dev/null +++ b/be/src/storage/index/snii/snii_doris_adapter.cpp @@ -0,0 +1,436 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/snii_doris_adapter.h" + +#include + +#include +#include +#include +#include + +#include "common/cast_set.h" +#include "cpp/sync_point.h" +#include "runtime/exec_env.h" +#include "runtime/thread_context.h" +#include "util/countdown_latch.h" +#include "util/threadpool.h" + +namespace doris::segment_v2::snii_doris { +namespace { +// Per-call cap on concurrently dispatched physical segment reads. A coalesced +// batch of at most this many segments is served as a single concurrent round +// (mirrors the "at most one serial round" contract that the test-side +// MeteredFileReader measures, and the S3 standalone reader's 16-way fan-out). +constexpr size_t kMaxConcurrentReads = 16; +} // namespace + +thread_local const io::IOContext* DorisSniiFileReader::_scoped_io_ctx = nullptr; +ThreadPool* DorisSniiFileReader::_io_pool_for_test = nullptr; + +Status DorisSniiFileWriter::append(::doris::snii::Slice data) { + if (_writer == nullptr) { + return Status::Error("doris writer is null"); + } + return _writer->append(Slice(reinterpret_cast(data.data()), data.size())); +} + +Status DorisSniiFileWriter::finalize() { + if (_writer == nullptr) { + return Status::Error("doris writer is null"); + } + return Status::OK(); +} + +uint64_t DorisSniiFileWriter::bytes_written() const { + return _writer == nullptr ? 0 : _writer->bytes_appended(); +} + +DorisSniiFileReader::DorisSniiFileReader(io::FileReaderSPtr reader, const io::IOContext* io_ctx, + bool direct_remote_io) + : _reader(std::move(reader)), + _direct_remote_io(direct_remote_io), + _default_io_ctx(_make_index_io_context(io_ctx)) {} + +io::IOContext DorisSniiFileReader::_make_index_io_context(const io::IOContext* io_ctx) { + io::IOContext index_io_ctx; + if (io_ctx != nullptr) { + index_io_ctx = *io_ctx; + } + index_io_ctx.is_inverted_index = true; + // is_index_data is inherited from io_ctx: META scopes set it true at the source + // (index_file_reader), non-meta reads default to false. + // + // Every SNII remote read is funnelled through here, and nothing else is, so this + // is where the session variable becomes a cache policy. Deciding it at the shared + // consult site instead would need an is_snii bit on IOContext and would silently + // catch CLucene reads, which clone the same upstream context. + if (index_io_ctx.inverted_index_snii_read_no_write_file_cache && + index_io_ctx.reader_type == ReaderType::READER_QUERY && !index_io_ctx.is_warmup) { + index_io_ctx.file_cache_miss_policy = io::FileCacheMissPolicy::REMOTE_ONLY_ON_MISS; + } + return index_io_ctx; +} + +DorisSniiFileReader::ScopedIOContext::ScopedIOContext(const io::IOContext* io_ctx) + : _previous(_scoped_io_ctx), _io_ctx(DorisSniiFileReader::_make_index_io_context(io_ctx)) { + _scoped_io_ctx = &_io_ctx; +} + +DorisSniiFileReader::ScopedIOContext::~ScopedIOContext() { + _scoped_io_ctx = _previous; +} + +Status DorisSniiFileReader::read_at(uint64_t offset, size_t len, std::vector* out) { + if (out == nullptr) { + return Status::Error("output buffer is null"); + } + RETURN_IF_ERROR(_check_read_range(offset, len)); + RETURN_IF_ERROR(_read_at(offset, len, out, _current_io_ctx())); + if (len > 0) { + _record_read_stats(cast_set(len), cast_set(len), 1, 1); + } + return Status::OK(); +} + +// NOLINTNEXTLINE(readability-non-const-parameter): out is the SNII read output buffer. +Status DorisSniiFileReader::read_into(uint64_t offset, uint8_t* out, size_t out_len) { + if (out_len == 0) { + return Status::OK(); + } + if (out == nullptr) { + return Status::Error("output buffer is null"); + } + RETURN_IF_ERROR(_check_read_range(offset, out_len)); + TEST_SYNC_POINT_RETURN_WITH_VALUE("DorisSniiFileReader::_read_at", + Status::IOError("injected SNII read failure"), offset, + out_len); + DCHECK(_reader != nullptr); + size_t bytes_read = 0; + RETURN_IF_ERROR(_reader->read_at(offset, Slice(out, out_len), &bytes_read, _current_io_ctx())); + if (bytes_read != out_len) { + return Status::Error(fmt::format( + "short read at offset {}, expect {}, got {}", offset, out_len, bytes_read)); + } + _record_read_stats(cast_set(out_len), cast_set(out_len), 1, 1); + return Status::OK(); +} + +// NOLINTNEXTLINE(readability-non-const-parameter): out is the SNII read output buffer. +Status DorisSniiFileReader::_read_at(uint64_t offset, size_t len, std::vector* out, + const io::IOContext* io_ctx) const { + TEST_SYNC_POINT_RETURN_WITH_VALUE("DorisSniiFileReader::_read_at", + Status::IOError("injected SNII read failure"), offset, len); + DCHECK(_reader != nullptr); + DCHECK(out != nullptr); + DCHECK(_check_read_range(offset, len).ok()); + if (len == 0) { + out->clear(); + return Status::OK(); + } + out->resize(len); + size_t bytes_read = 0; + auto status = _reader->read_at(offset, Slice(out->data(), len), &bytes_read, io_ctx); + if (!status.ok()) { + return status; + } + if (bytes_read != len) { + return Status::Error( + fmt::format("short read at offset {}, expect {}, got {}", offset, len, bytes_read)); + } + return Status::OK(); +} + +// NOLINTBEGIN(readability-non-const-parameter): outs is the SNII batch read output buffer. +Status DorisSniiFileReader::read_batch(const std::vector<::doris::snii::io::Range>& ranges, + std::vector>* outs) { + if (outs == nullptr) { + return Status::Error("output buffers is null"); + } + outs->clear(); + outs->resize(ranges.size()); + if (ranges.empty()) { + return Status::OK(); + } + + // ----- Phase 1: plan (serial, lock-free) ----- + // No section-classification lock exists on this reader, so the whole plan is + // a plain in-memory scan; the NO-IO-UNDER-LOCK red line holds trivially (no + // lock is taken anywhere in read_batch). + struct IndexedRange { + uint64_t offset = 0; + size_t len = 0; + size_t index = 0; + }; + int64_t request_bytes = 0; + std::vector sorted; + sorted.reserve(ranges.size()); + for (size_t i = 0; i < ranges.size(); ++i) { + RETURN_IF_ERROR(_check_read_range(ranges[i].offset, ranges[i].len)); + request_bytes += cast_set(ranges[i].len); + if (ranges[i].len == 0) { + continue; + } + sorted.push_back({ranges[i].offset, ranges[i].len, i}); + } + if (sorted.empty()) { + return Status::OK(); + } + // F27: callers (BatchRangeFetcher::fetch) already pass offset-sorted ranges; + // only pay for a sort when the input is actually out of order. + auto by_offset = [](const IndexedRange& lhs, const IndexedRange& rhs) { + return lhs.offset < rhs.offset; + }; + if (!std::ranges::is_sorted(sorted, by_offset)) { + std::ranges::sort(sorted, by_offset); + } + + // Coalesce the sorted ranges into disjoint physical segments. + struct Seg { + uint64_t offset = 0; + size_t len = 0; + size_t begin = 0; // first index into `sorted` covered by this segment + size_t end = 0; // one-past-last index into `sorted` + bool single = false; + }; + constexpr uint64_t max_coalesced_gap = 4096; + constexpr uint64_t max_coalesced_read = 1ULL << 20; + std::vector segs; + for (size_t begin = 0; begin < sorted.size();) { + uint64_t read_offset = sorted[begin].offset; + uint64_t read_end = sorted[begin].offset + sorted[begin].len; + size_t end = begin + 1; + while (end < sorted.size()) { + const uint64_t next_end = sorted[end].offset + sorted[end].len; + if ((sorted[end].offset > read_end && + sorted[end].offset - read_end > max_coalesced_gap) || + next_end - read_offset > max_coalesced_read) { + break; + } + read_end = std::max(read_end, next_end); + ++end; + } + Seg seg; + seg.offset = read_offset; + seg.len = cast_set(read_end - read_offset); + seg.begin = begin; + seg.end = end; + // A single-range group exactly covers its segment, so it can be read + // straight into the caller's output slot with no temp + no second copy. + seg.single = (end == begin + 1); + segs.push_back(seg); + begin = end; + } + + // Resolve per-segment target buffers, io contexts and the shared sink on the + // calling thread: workers (which run on tracker-less pool threads) must not + // allocate, and per-segment private cache-stat slots keep disjoint physical + // reads from racing on the shared FileCacheStatistics. + const size_t num_segs = segs.size(); + const io::IOContext* base_io_ctx = _current_io_ctx(); + std::vector> tmp_bufs(num_segs); + std::vector*> targets(num_segs); + std::vector seg_stats(num_segs); + std::vector seg_io_ctx(num_segs); + std::vector seg_status(num_segs); + int64_t read_bytes = 0; + for (size_t s = 0; s < num_segs; ++s) { + const Seg& seg = segs[s]; + std::vector* target = + seg.single ? &(*outs)[sorted[seg.begin].index] : &tmp_bufs[s]; + target->resize(seg.len); + targets[s] = target; + seg_io_ctx[s] = *base_io_ctx; + seg_io_ctx[s].file_cache_stats = + base_io_ctx->file_cache_stats != nullptr ? &seg_stats[s] : nullptr; + read_bytes += cast_set(seg.len); + } + + // ----- Phase 2: physical reads (lock-free; concurrent when a pool exists) ----- + auto run_segment = [&](size_t s) { + seg_status[s] = _read_at(segs[s].offset, segs[s].len, targets[s], &seg_io_ctx[s]); + }; + ThreadPool* pool = _select_io_pool(); + if (pool != nullptr && num_segs > 1) { + // Carried onto the pool threads below. They are "Orphan" threads with no MemTrackerLimiter + // of their own (buffered_reader.cpp:426), and the read does allocate down there: in cloud + // mode _read_at reaches CachedRemoteFileReader::read_at_impl -> + // _read_from_indirect_cache -> _read_remote_blocks_into_cache -> _execute_remote_read -> + // _execute_s3_fallback, which does `new char[span_size]`. Without this the span is charged + // to no tracker at all, which memory_orphan_check() (on by default) treats as a bug. + // Same shape as the hedged-read path in cached_remote_file_reader.cpp:500-505. + const std::shared_ptr parent_resource_ctx = + thread_context()->resource_ctx(); + for (size_t base = 0; base < num_segs; base += kMaxConcurrentReads) { + const size_t wave_end = std::min(base + kMaxConcurrentReads, num_segs); + ::doris::CountDownLatch latch(cast_set(wave_end - base)); + for (size_t s = base; s < wave_end; ++s) { + Status submit_st = + pool->submit_func([&run_segment, &latch, s, parent_resource_ctx]() { + std::unique_ptr attach_task; + if (parent_resource_ctx != nullptr) { + attach_task = std::make_unique(parent_resource_ctx); + } + run_segment(s); + latch.count_down(); + }); + if (!submit_st.ok()) { + // Pool full/shut down: read this segment inline; never skip + // the count_down or the latch would not drain. + run_segment(s); + latch.count_down(); + } + } + latch.wait(); + } + } else { + // Serial fallback: no executor (e.g. tools without ExecEnv) or a single + // segment (avoids micro-batch scheduling overhead). + for (size_t s = 0; s < num_segs; ++s) { + run_segment(s); + } + } + + // ----- Phase 3: merge stats, first-error, scatter, account (serial) ----- + // Fold every segment's private stats back FIRST: physical IO that already + // happened (including partial work inside a segment that then failed) must + // reach the query profile even when another segment of this batch errors. + if (base_io_ctx->file_cache_stats != nullptr) { + for (size_t s = 0; s < num_segs; ++s) { + _merge_file_cache_statistics(base_io_ctx->file_cache_stats, seg_stats[s]); + } + } + Status first_error = Status::OK(); + int64_t completed_request_bytes = 0; + int64_t completed_read_bytes = 0; + for (size_t s = 0; s < num_segs; ++s) { + if (!seg_status[s].ok()) { + if (first_error.ok()) { + first_error = seg_status[s]; + } + continue; + } + completed_read_bytes += cast_set(segs[s].len); + for (size_t i = segs[s].begin; i < segs[s].end; ++i) { + completed_request_bytes += cast_set(sorted[i].len); + } + } + if (!first_error.ok()) { + // Record only what actually completed so the logical counters stay + // truthful for the failed batch; ranges/rounds reflect what was issued. + _record_read_stats(completed_request_bytes, completed_read_bytes, + cast_set(num_segs), + cast_set(_compute_num_waves(num_segs))); + return first_error; + } + for (size_t s = 0; s < num_segs; ++s) { + const Seg& seg = segs[s]; + if (seg.single) { + continue; // already read in place + } + const std::vector& bytes = tmp_bufs[s]; + for (size_t i = seg.begin; i < seg.end; ++i) { + const uint64_t pos = sorted[i].offset - seg.offset; + auto& out = (*outs)[sorted[i].index]; + out.assign(bytes.begin() + cast_set(pos), + bytes.begin() + cast_set(pos + sorted[i].len)); + } + } + _record_read_stats(request_bytes, read_bytes, cast_set(num_segs), + cast_set(_compute_num_waves(num_segs))); + return Status::OK(); +} +// NOLINTEND(readability-non-const-parameter) + +uint64_t DorisSniiFileReader::size() const { + return _reader == nullptr ? 0 : _reader->size(); +} + +const io::IOContext* DorisSniiFileReader::_current_io_ctx() const { + return _scoped_io_ctx != nullptr ? _scoped_io_ctx : &_default_io_ctx; +} + +void DorisSniiFileReader::_record_read_stats(int64_t request_bytes, int64_t read_bytes, + int64_t range_read_count, + int64_t serial_read_rounds) const { + const auto* io_ctx = _current_io_ctx(); + if (io_ctx->file_cache_stats == nullptr) { + return; + } + auto* stats = io_ctx->file_cache_stats; + stats->inverted_index_request_bytes += request_bytes; + stats->inverted_index_read_bytes += read_bytes; + stats->inverted_index_range_read_count += range_read_count; + stats->inverted_index_serial_read_rounds += serial_read_rounds; + if (_direct_remote_io) { + // No CachedRemoteFileReader below us: every byte this reader fetched was a + // direct remote GET, so account it as physical remote IO here. + stats->inverted_index_remote_physical_read_bytes += read_bytes; + } +} + +void DorisSniiFileReader::set_io_thread_pool_for_test(ThreadPool* pool) { + _io_pool_for_test = pool; +} + +ThreadPool* DorisSniiFileReader::_select_io_pool() { + if (_io_pool_for_test != nullptr) { + return _io_pool_for_test; + } + if (ExecEnv::ready()) { + return ExecEnv::GetInstance()->buffered_reader_prefetch_thread_pool(); + } + return nullptr; +} + +size_t DorisSniiFileReader::_compute_num_waves(size_t seg_count) { + if (seg_count == 0) { + return 0; + } + return (seg_count + kMaxConcurrentReads - 1) / kMaxConcurrentReads; +} + +void DorisSniiFileReader::_merge_file_cache_statistics(io::FileCacheStatistics* dst, + const io::FileCacheStatistics& src) { + if (dst == nullptr) { + return; + } + // Delegate to the canonical field list so per-wave private stats can never + // silently drop fields the general layer adds (e.g. write_cache_io_timer, + // remote_only_on_miss_*), which a hand-rolled copy here used to do. + dst->merge_from(src); +} + +Status DorisSniiFileReader::_check_read_range(uint64_t offset, size_t len) const { + if (_reader == nullptr) { + return Status::Error("doris reader is null"); + } + if (offset > std::numeric_limits::max() - len) { + return Status::Error( + fmt::format("read range overflows: offset {}, len {}", offset, len)); + } + const uint64_t end = offset + len; + if (end > _reader->size()) { + return Status::Error( + fmt::format("read range exceeds file size: offset {}, len {}, file size {}", offset, + len, _reader->size())); + } + return Status::OK(); +} + +} // namespace doris::segment_v2::snii_doris diff --git a/be/src/storage/index/snii/snii_doris_adapter.h b/be/src/storage/index/snii/snii_doris_adapter.h new file mode 100644 index 00000000000000..cdcd9365ca79cd --- /dev/null +++ b/be/src/storage/index/snii/snii_doris_adapter.h @@ -0,0 +1,119 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include "common/status.h" +#include "io/fs/file_reader.h" +#include "io/fs/file_writer.h" +#include "io/io_common.h" +#include "storage/index/snii/io/file_reader.h" +#include "storage/index/snii/io/file_writer.h" +#include "util/slice.h" + +namespace doris { +class ThreadPool; +} // namespace doris + +namespace doris::segment_v2::snii_doris { + +class DorisSniiFileWriter final : public ::doris::snii::io::FileWriter { +public: + explicit DorisSniiFileWriter(io::FileWriter* writer) : _writer(writer) {} + + Status append(::doris::snii::Slice data) override; + Status finalize() override; + uint64_t bytes_written() const override; + +private: + io::FileWriter* _writer = nullptr; +}; + +class DorisSniiFileReader final : public ::doris::snii::io::FileReader { +public: + class ScopedIOContext { + public: + explicit ScopedIOContext(const io::IOContext* io_ctx); + ~ScopedIOContext(); + + ScopedIOContext(const ScopedIOContext&) = delete; + ScopedIOContext& operator=(const ScopedIOContext&) = delete; + + private: + const io::IOContext* _previous = nullptr; + io::IOContext _io_ctx; + }; + + // `direct_remote_io` marks a reader whose byte ranges are served straight from + // remote storage with no CachedRemoteFileReader in between (NO_CACHE on a + // non-local filesystem). Only that layer would normally + // account physical remote bytes, so this reader then counts its own reads as + // physical remote IO. Never set it for cached or local readers: the former + // double-counts, the latter reports local disk as remote fetch volume. + explicit DorisSniiFileReader(io::FileReaderSPtr reader, const io::IOContext* io_ctx = nullptr, + bool direct_remote_io = false); + + Status read_at(uint64_t offset, size_t len, std::vector* out) override; + // Fills the caller's buffer directly (no vector round-trip) under the same + // scoped-IOContext resolution as read_at. The blob read shim + // (SniiBlobIndexInput) refills through this, so BufferedIndexInput's + // large-read bypass lands blob bytes straight in the destination array. + Status read_into(uint64_t offset, uint8_t* out, size_t out_len) override; + Status read_batch(const std::vector<::doris::snii::io::Range>& ranges, + std::vector>* outs) override; + uint64_t size() const override; + + // Test-only: inject (or clear with nullptr) the thread pool used to fan out + // batch segment reads. nullptr (default) routes to the BE buffered-reader + // prefetch pool when ExecEnv is ready, otherwise a serial fallback. + static void set_io_thread_pool_for_test(ThreadPool* pool); + +private: + static io::IOContext _make_index_io_context(const io::IOContext* io_ctx); + Status _check_read_range(uint64_t offset, size_t len) const; + Status _read_at(uint64_t offset, size_t len, std::vector* out, + const io::IOContext* io_ctx) const; + const io::IOContext* _current_io_ctx() const; + void _record_read_stats(int64_t request_bytes, int64_t read_bytes, int64_t range_read_count, + int64_t serial_read_rounds) const; + + // Selects the executor for parallel batch segment reads: the test seam if + // set, else the BE prefetch pool when ExecEnv is ready, else nullptr (the + // caller then reads segments serially). + static ThreadPool* _select_io_pool(); + // Number of dependent serial dispatch waves for `seg_count` physical + // segments given the per-call concurrency cap. This is the F19 metric: + // a coalesced batch of <= cap segments is a single concurrent round. + static size_t _compute_num_waves(size_t seg_count); + // Folds a per-segment private stats slot back into the shared sink (disjoint + // physical reads never race on it). Delegates to FileCacheStatistics::merge_from + // so the field list can never drift from io_common.h. + static void _merge_file_cache_statistics(io::FileCacheStatistics* dst, + const io::FileCacheStatistics& src); + + io::FileReaderSPtr _reader; + const bool _direct_remote_io = false; + io::IOContext _default_io_ctx; + static thread_local const io::IOContext* _scoped_io_ctx; + // Test seam for the batch-read executor; see set_io_thread_pool_for_test. + static ThreadPool* _io_pool_for_test; +}; + +} // namespace doris::segment_v2::snii_doris diff --git a/be/src/storage/index/snii/snii_index_reader.cpp b/be/src/storage/index/snii/snii_index_reader.cpp new file mode 100644 index 00000000000000..d701c314efb367 --- /dev/null +++ b/be/src/storage/index/snii/snii_index_reader.cpp @@ -0,0 +1,1188 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/snii_index_reader.h" + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/config.h" +#include "runtime/exec_env.h" +#include "runtime/query_context.h" +#include "runtime/runtime_profile.h" +#include "runtime/runtime_state.h" +#include "storage/index/index_file_reader.h" +#include "storage/index/index_reader_helper.h" +#include "storage/index/inverted/analyzer/analyzer.h" +#include "storage/index/inverted/analyzer/segment_analyzer_context.h" +#include "storage/index/inverted/common/single_flight.h" +#include "storage/index/inverted/inverted_index_cache.h" +#include "storage/index/inverted/inverted_index_iterator.h" +#include "storage/index/inverted/token_filter/common_grams_filter.h" +#include "storage/index/snii/format/null_bitmap.h" +#include "storage/index/snii/query/boolean_query.h" +#include "storage/index/snii/query/count_query.h" +#include "storage/index/snii/query/docid_sink.h" +#include "storage/index/snii/query/internal/plain_term_routing.h" +#include "storage/index/snii/query/phrase_query.h" +#include "storage/index/snii/query/prefix_query.h" +#include "storage/index/snii/query/regexp_query.h" +#include "storage/index/snii/query/scoring_query.h" +#include "storage/index/snii/query/term_query.h" +#include "storage/index/snii/query/wildcard_query.h" +#include "storage/index/snii/reader/logical_index_reader.h" +#include "storage/index/snii/snii_doris_adapter.h" +#include "storage/index/snii/snii_prx_profile.h" +#include "storage/index/snii/stats/snii_stats_provider.h" +#include "util/defer_op.h" +#include "util/time.h" + +#ifdef BE_TEST +namespace doris::snii::testing { +namespace { + +std::atomic prx_execution_profile_scope_constructions {0}; +std::atomic prx_execution_profile_scope_flushes {0}; + +} // namespace + +void record_prx_execution_profile_scope_construction() { + prx_execution_profile_scope_constructions.fetch_add(1, std::memory_order_relaxed); +} + +void record_prx_execution_profile_scope_flush() { + prx_execution_profile_scope_flushes.fetch_add(1, std::memory_order_relaxed); +} + +void reset_prx_execution_profile_scope_counters() { + prx_execution_profile_scope_constructions.store(0, std::memory_order_relaxed); + prx_execution_profile_scope_flushes.store(0, std::memory_order_relaxed); +} + +uint64_t prx_execution_profile_scope_construction_count() { + return prx_execution_profile_scope_constructions.load(std::memory_order_relaxed); +} + +uint64_t prx_execution_profile_scope_flush_count() { + return prx_execution_profile_scope_flushes.load(std::memory_order_relaxed); +} + +} // namespace doris::snii::testing +#endif + +namespace doris::segment_v2 { + +namespace { + +class RoaringDocIdSink final : public ::doris::snii::query::DocIdSink { +public: + explicit RoaringDocIdSink(roaring::Roaring* bitmap) : _bitmap(bitmap) { + DCHECK(_bitmap != nullptr); + } + + Status append_sorted(std::span docids) override { + if (!docids.empty()) { + _bitmap->addMany(docids.size(), docids.data()); + } + return Status::OK(); + } + + Status append_range(uint32_t first, uint64_t last_exclusive) override { + if (last_exclusive > first) { + _bitmap->addRange(first, last_exclusive); + } + return Status::OK(); + } + + // Roaring addMany/addRange deduplicate and order natively, so multi-term OR + // can stream each posting straight into the bitmap (no per-term vector + merge). + bool dedups() const override { return true; } + +private: + roaring::Roaring* _bitmap; +}; + +struct SniiQueryExecutionResult { + std::shared_ptr bitmap; + std::vector<::doris::snii::query::PhraseMatch> phrase_matches; +}; + +std::vector to_terms(const InvertedIndexQueryInfo& query_info) { + std::vector terms; + terms.reserve(query_info.term_infos.size()); + for (const auto& term_info : query_info.term_infos) { + DCHECK(term_info.is_single_term()); + terms.push_back(term_info.get_single_term()); + } + return terms; +} + +bool uses_plain_term_frequency_scoring(InvertedIndexQueryType query_type, + const InvertedIndexQueryInfo& query_info) { + return query_type == InvertedIndexQueryType::MATCH_ANY_QUERY || + query_type == InvertedIndexQueryType::MATCH_ALL_QUERY || + (query_type == InvertedIndexQueryType::MATCH_PHRASE_QUERY && + query_info.term_infos.size() == 1); +} + +bool uses_phrase_frequency_scoring(InvertedIndexQueryType query_type, + const InvertedIndexQueryInfo& query_info) { + return query_info.term_infos.size() > 1 && + (query_type == InvertedIndexQueryType::MATCH_PHRASE_QUERY || + query_type == InvertedIndexQueryType::MATCH_PHRASE_PREFIX_QUERY); +} + +Status score_plain_term_candidates(const IndexQueryContextPtr& context, + std::string_view column_name, + const InvertedIndexQueryInfo& query_info, + const ::doris::snii::reader::LogicalIndexReader& logical_reader, + const ::doris::snii::stats::SniiStatsProvider& segment_stats, + const roaring::Roaring& final_candidates) { + DORIS_CHECK(context->collection_statistics != nullptr); + DORIS_CHECK(context->collection_similarity != nullptr); + + const std::wstring field_name = StringUtil::string_to_wstring(std::string(column_name)); + const double collection_avgdl = + context->collection_statistics->get_or_calculate_avg_dl(field_name); + std::vector<::doris::snii::query::CollectionScoringTerm> scoring_terms; + scoring_terms.reserve(query_info.term_infos.size()); + for (const auto& term_info : query_info.term_infos) { + DORIS_CHECK(term_info.is_single_term()); + std::string physical_term; + bool representable = false; + RETURN_IF_ERROR(::doris::snii::query::internal::route_query_term( + logical_reader, term_info, &physical_term, &representable)); + if (!representable) { + continue; + } + const std::string& logical_term = term_info.get_single_term(); + const double idf = context->collection_statistics->get_or_calculate_idf( + field_name, StringUtil::string_to_wstring(logical_term)); + scoring_terms.push_back({.physical_term = std::move(physical_term), .idf = idf}); + } + DORIS_CHECK(final_candidates.isEmpty() || !scoring_terms.empty()); + + std::vector<::doris::snii::query::ScoredDoc> scored_docs; + RETURN_IF_ERROR(::doris::snii::query::scoring_query_candidates( + logical_reader, segment_stats, scoring_terms, final_candidates, collection_avgdl, + ::doris::snii::query::Bm25Params {}, &scored_docs)); + for (const auto& scored_doc : scored_docs) { + context->collection_similarity->collect(scored_doc.docid, + static_cast(scored_doc.score)); + } + return Status::OK(); +} + +Status score_phrase_matches(const IndexQueryContextPtr& context, std::string_view column_name, + InvertedIndexQueryType query_type, + const InvertedIndexQueryInfo& query_info, + const ::doris::snii::reader::LogicalIndexReader& logical_reader, + const ::doris::snii::stats::SniiStatsProvider& segment_stats, + const roaring::Roaring& final_candidates, + const std::vector<::doris::snii::query::PhraseMatch>& matches) { + DORIS_CHECK(context->collection_statistics != nullptr); + DORIS_CHECK(context->collection_similarity != nullptr); + DORIS_CHECK(uses_phrase_frequency_scoring(query_type, query_info)); + DORIS_CHECK_EQ(final_candidates.cardinality(), matches.size()); + + const std::wstring field_name = StringUtil::string_to_wstring(std::string(column_name)); + const double collection_avgdl = + context->collection_statistics->get_or_calculate_avg_dl(field_name); + const size_t idf_term_count = query_type == InvertedIndexQueryType::MATCH_PHRASE_PREFIX_QUERY + ? query_info.term_infos.size() - 1 + : query_info.term_infos.size(); + double idf_sum = 0.0; + for (size_t i = 0; i < idf_term_count; ++i) { + const auto& term_info = query_info.term_infos[i]; + DORIS_CHECK(term_info.is_single_term()); + idf_sum += context->collection_statistics->get_or_calculate_idf( + field_name, StringUtil::string_to_wstring(term_info.get_single_term())); + } + + const auto scorer = ::doris::snii::query::ScorerContext::from_idf(idf_sum); + std::vector<::doris::snii::query::ScoredDoc> scored_docs; + scored_docs.reserve(matches.size()); + for (const auto& match : matches) { + DCHECK(final_candidates.contains(match.docid)); + DCHECK_NE(match.frequency, 0); + uint8_t norm = 0; + RETURN_IF_ERROR(segment_stats.encoded_norm(match.docid, &norm)); + scored_docs.push_back({.docid = match.docid, + .score = scorer.score(match.frequency, norm, collection_avgdl, + ::doris::snii::query::Bm25Params {})}); + } + for (const auto& scored_doc : scored_docs) { + context->collection_similarity->collect(scored_doc.docid, + static_cast(scored_doc.score)); + } + return Status::OK(); +} + +void parse_phrase_slop(std::string* query, InvertedIndexQueryInfo* query_info) { + DCHECK(query != nullptr); + DCHECK(query_info != nullptr); + const auto is_digits = [](std::string_view str) { + return std::all_of(str.begin(), str.end(), [](unsigned char c) { return std::isdigit(c); }); + }; + + const size_t last_space_pos = query->find_last_of(' '); + if (last_space_pos == std::string::npos) { + return; + } + const size_t tilde_pos = last_space_pos + 1; + if (tilde_pos >= query->size() - 1 || (*query)[tilde_pos] != '~') { + return; + } + + const size_t slop_pos = tilde_pos + 1; + std::string_view slop_str(query->data() + slop_pos, query->size() - slop_pos); + if (slop_str.empty()) { + return; + } + + bool ordered = false; + if (slop_str.size() == 1) { + if (!std::isdigit(static_cast(slop_str[0]))) { + return; + } + } else if (slop_str.back() == '+') { + ordered = true; + slop_str.remove_suffix(1); + } + + if (!is_digits(slop_str)) { + return; + } + auto result = std::from_chars(slop_str.begin(), slop_str.end(), query_info->slop); + if (result.ec != std::errc()) { + return; + } + query_info->ordered = ordered; + *query = query->substr(0, last_space_pos); +} + +std::shared_ptr docids_to_bitmap(const std::vector& docids) { + auto result = std::make_shared(); + if (!docids.empty()) { + result->addMany(docids.size(), docids.data()); + } + result->runOptimize(); + return result; +} + +// Runs `compute` under single-flight keyed by `key`: concurrent identical queries collapse to a +// single execution and the followers reuse the leader's bitmap. `compute(out)` fills *out and +// returns its Status; on overall success *result receives the bitmap. See SingleFlight for why +// this matters under a cold cache with parallel scanners hitting the same segment. +template +Status run_query_single_flight( + ::doris::segment_v2::inverted_index::SingleFlight< + std::pair>>& flight, + const std::string& key, std::shared_ptr* result, +#ifdef BE_TEST + SniiIndexReader::SingleFlightFollowerJoinedObserver follower_joined_observer, + void* follower_joined_opaque, + SniiIndexReader::SingleFlightLeaderBeforeComputeObserver leader_before_compute_observer, + void* leader_before_compute_opaque, +#endif + Compute&& compute) { + auto follower = flight.join_or_lead(key); + if (follower.has_value()) { +#ifdef BE_TEST + if (follower_joined_observer != nullptr) { + follower_joined_observer(follower_joined_opaque); + } +#endif + auto [leader_status, leader_bitmap] = follower->get(); + if (leader_status.ok() && leader_bitmap != nullptr) { + *result = std::move(leader_bitmap); + return Status::OK(); + } + // Leader failed; fall through and compute independently (rare error path). + } + const bool is_leader = !follower.has_value(); +#ifdef BE_TEST + if (is_leader && leader_before_compute_observer != nullptr) { + leader_before_compute_observer(leader_before_compute_opaque); + } +#endif + + Status status = Status::OK(); + std::shared_ptr bitmap; + { + // Publish to any waiting followers on every exit path (including errors). + DEFER(if (is_leader) { flight.publish(key, std::make_pair(status, bitmap)); }); + status = compute(&bitmap); + } + RETURN_IF_ERROR(status); + *result = std::move(bitmap); + return Status::OK(); +} + +Status execute_snii_query(const ::doris::snii::reader::LogicalIndexReader& logical_reader, + InvertedIndexQueryType query_type, + const InvertedIndexQueryInfo& query_info, std::string_view search_str, + const std::vector& terms, int32_t max_expansions, + bool collect_phrase_frequency, SniiQueryExecutionResult* result, + ::doris::snii::query::QueryProfile* profile) { + result->bitmap = std::make_shared(); + result->phrase_matches.clear(); + DORIS_CHECK(!collect_phrase_frequency || uses_phrase_frequency_scoring(query_type, query_info)); + RoaringDocIdSink sink(result->bitmap.get()); + std::vector docids; + bool emitted_to_sink = false; + Status status; + switch (query_type) { + case InvertedIndexQueryType::EQUAL_QUERY: + case InvertedIndexQueryType::MATCH_ANY_QUERY: + status = terms.size() == 1 + ? ::doris::snii::query::term_query(logical_reader, terms.front(), &sink) + : ::doris::snii::query::boolean_or(logical_reader, terms, &sink); + emitted_to_sink = true; + break; + case InvertedIndexQueryType::MATCH_ALL_QUERY: + if (terms.size() == 1) { + status = ::doris::snii::query::term_query(logical_reader, terms.front(), &sink); + emitted_to_sink = true; + } else { + status = ::doris::snii::query::boolean_and(logical_reader, terms, &docids); + } + break; + case InvertedIndexQueryType::MATCH_PHRASE_QUERY: + if (terms.size() == 1) { + status = ::doris::snii::query::term_query(logical_reader, terms.front(), &sink); + emitted_to_sink = true; + } else { + status = collect_phrase_frequency + ? ::doris::snii::query::phrase_query_with_frequencies( + logical_reader, terms, &result->phrase_matches, profile, + {.slop = static_cast(query_info.slop), + .ordered = query_info.ordered}) + : ::doris::snii::query::phrase_query( + logical_reader, terms, &docids, profile, + {.slop = static_cast(query_info.slop), + .ordered = query_info.ordered}); + } + break; + case InvertedIndexQueryType::MATCH_PHRASE_PREFIX_QUERY: + if (terms.size() == 1) { + status = ::doris::snii::query::prefix_query(logical_reader, terms.front(), &sink, + max_expansions); + emitted_to_sink = true; + } else { + status = collect_phrase_frequency + ? ::doris::snii::query::phrase_prefix_query_with_frequencies( + logical_reader, terms, &result->phrase_matches, profile, + max_expansions) + : ::doris::snii::query::phrase_prefix_query( + logical_reader, terms, &docids, profile, max_expansions); + } + break; + case InvertedIndexQueryType::MATCH_REGEXP_QUERY: + status = ::doris::snii::query::regexp_query(logical_reader, search_str, &sink, + max_expansions); + emitted_to_sink = true; + break; + case InvertedIndexQueryType::WILDCARD_QUERY: + status = ::doris::snii::query::wildcard_query(logical_reader, search_str, &sink, + max_expansions); + emitted_to_sink = true; + break; + case InvertedIndexQueryType::LESS_THAN_QUERY: + case InvertedIndexQueryType::LESS_EQUAL_QUERY: + case InvertedIndexQueryType::GREATER_THAN_QUERY: + case InvertedIndexQueryType::GREATER_EQUAL_QUERY: + case InvertedIndexQueryType::RANGE_QUERY: + return Status::Error( + "SNII inverted index storage format does not support BKD/range query"); + case InvertedIndexQueryType::MATCH_PHRASE_EDGE_QUERY: + // SNII has no native edge-phrase operator yet. V3 answers this through + // PhraseEdgeQuery, and a row implementation exists (match_phrase_edge), so + // downgrade to scalar evaluation instead of failing the query outright. + return Status::Error( + "SNII does not implement MATCH_PHRASE_EDGE; evaluating by function"); + default: + return Status::Error( + "SNII unsupported inverted index query type {}", query_type_to_string(query_type)); + } + RETURN_IF_ERROR(status); + if (collect_phrase_frequency) { + for (const auto& match : result->phrase_matches) { + result->bitmap->add(match.docid); + } + result->bitmap->runOptimize(); + } else if (emitted_to_sink) { + result->bitmap->runOptimize(); + } else { + result->bitmap = docids_to_bitmap(docids); + } + return Status::OK(); +} + +} // namespace + +Status SniiIndexReader::new_iterator(std::unique_ptr* iterator) { + if (*iterator == nullptr) { + *iterator = InvertedIndexIterator::create_unique(); + } + dynamic_cast(iterator->get()) + ->add_reader(_reader_type, + dynamic_pointer_cast(shared_from_this())); + return Status::OK(); +} + +Status SniiIndexReader::_parse_query_terms( + const IndexQueryContextPtr& context, std::string search_str, + InvertedIndexQueryType query_type, const InvertedIndexAnalyzerCtx* analyzer_ctx, + InvertedIndexQueryInfo* query_info, + std::optional purpose_override) { + DCHECK(query_info != nullptr); + if (query_type == InvertedIndexQueryType::MATCH_REGEXP_QUERY || + query_type == InvertedIndexQueryType::WILDCARD_QUERY) { + query_info->term_infos.emplace_back(search_str, 0); + return Status::OK(); + } + if (query_type == InvertedIndexQueryType::MATCH_PHRASE_QUERY) { + parse_phrase_slop(&search_str, query_info); + } + + const bool actual_similarity = + context->collection_similarity && + IndexReaderHelper::is_need_similarity_score(query_type, &_index_meta); + const auto purpose = purpose_override.value_or(inverted_index::select_analysis_purpose( + query_type, query_info->slop, actual_similarity)); + SCOPED_RAW_TIMER(&context->stats->inverted_index_analyzer_timer); + try { + if (analyzer_ctx != nullptr && !analyzer_ctx->requires_analysis()) { + query_info->term_infos.emplace_back(search_str); + } else { + auto analyzer = analyzer_ctx == nullptr ? nullptr : analyzer_ctx->get_analyzer(purpose); + if (analyzer != nullptr) { + auto reader = inverted_index::InvertedIndexAnalyzer::create_reader( + analyzer_ctx->char_filter_map); + reader->init(search_str.data(), static_cast(search_str.size()), true); + query_info->term_infos = inverted_index::InvertedIndexAnalyzer::get_analyse_result( + reader, analyzer.get()); + } else { + query_info->term_infos = inverted_index::InvertedIndexAnalyzer::get_analyse_result( + search_str, _index_meta.properties(), purpose); + } + } + } catch (const CLuceneError& e) { + return Status::Error( + "SNII analyze query failed: {}", e.what()); + } catch (const Exception& e) { + return Status::Error( + "SNII analyze query failed: {}", e.what()); + } + return Status::OK(); +} + +Status SniiIndexReader::_get_logical_reader( + const IndexQueryContextPtr& context, InvertedIndexCacheHandle* searcher_cache_handle, + std::unique_ptr<::doris::snii::reader::LogicalIndexReader>* uncached_reader, + const ::doris::snii::reader::LogicalIndexReader** logical_reader) { + DCHECK(searcher_cache_handle != nullptr); + DCHECK(uncached_reader != nullptr); + DCHECK(logical_reader != nullptr); + + const bool enable_searcher_cache = + context->runtime_state != nullptr && + context->runtime_state->query_options().enable_inverted_index_searcher_cache; + const auto index_file_key = _index_file_reader->get_index_file_cache_key(&_index_meta); + InvertedIndexSearcherCache::CacheKey searcher_cache_key(index_file_key); + + bool cache_hit = false; + if (enable_searcher_cache) { + SCOPED_RAW_TIMER(&context->stats->inverted_index_lookup_timer); + cache_hit = InvertedIndexSearcherCache::instance()->lookup(searcher_cache_key, + searcher_cache_handle); + } + + if (cache_hit) { + context->stats->inverted_index_searcher_cache_hit++; + *logical_reader = searcher_cache_handle->get_snii_logical_reader(); + if (*logical_reader == nullptr) { + return Status::InternalError("SNII searcher cache entry has no logical reader"); + } + return Status::OK(); + } + + SCOPED_RAW_TIMER(&context->stats->inverted_index_searcher_open_timer); + context->stats->inverted_index_searcher_cache_miss++; +#ifdef BE_TEST + if (_searcher_open_observer != nullptr) { + _searcher_open_observer(_searcher_open_opaque); + } +#endif + RETURN_IF_ERROR( + _index_file_reader->init(config::inverted_index_read_buffer_size, context->io_ctx)); + auto opened_reader = + DORIS_TRY(_index_file_reader->open_snii_index(&_index_meta, context->io_ctx)); + + if (!enable_searcher_cache) { + *logical_reader = opened_reader.get(); + *uncached_reader = std::move(opened_reader); + return Status::OK(); + } + + const size_t reader_size = std::max(opened_reader->memory_usage(), 1); + auto* cache_value = new InvertedIndexSearcherCache::CacheValue( + std::move(opened_reader), reader_size, UnixMillis(), _index_file_reader); + InvertedIndexSearcherCache::instance()->insert(searcher_cache_key, cache_value, + searcher_cache_handle); + *logical_reader = searcher_cache_handle->get_snii_logical_reader(); + if (*logical_reader == nullptr) { + return Status::InternalError("SNII searcher cache insert produced empty logical reader"); + } + return Status::OK(); +} + +Status SniiIndexReader::query(const IndexQueryContextPtr& context, const std::string& column_name, + const Field& query_value, InvertedIndexQueryType query_type, + std::shared_ptr& bit_map, + const InvertedIndexAnalyzerCtx* analyzer_ctx) { + return _query(context, column_name, query_value, query_type, bit_map, nullptr, analyzer_ctx); +} + +Status SniiIndexReader::query_with_null_bitmap( + const IndexQueryContextPtr& context, const std::string& column_name, + const Field& query_value, InvertedIndexQueryType query_type, + std::shared_ptr& bit_map, + InvertedIndexQueryCacheHandle* null_bitmap_cache_handle, + const InvertedIndexAnalyzerCtx* analyzer_ctx) { + DORIS_CHECK(null_bitmap_cache_handle != nullptr); + return _query(context, column_name, query_value, query_type, bit_map, null_bitmap_cache_handle, + analyzer_ctx); +} + +Status SniiIndexReader::_query(const IndexQueryContextPtr& context, const std::string& column_name, + const Field& query_value, InvertedIndexQueryType query_type, + std::shared_ptr& bit_map, + InvertedIndexQueryCacheHandle* null_bitmap_cache_handle, + const InvertedIndexAnalyzerCtx* analyzer_ctx) { + const bool track_requested_null_time = null_bitmap_cache_handle != nullptr; + const int64_t query_ns_before = + track_requested_null_time ? context->stats->inverted_index_query_timer : 0; + int64_t requested_null_ns = 0; + DEFER({ + if (!track_requested_null_time) { + return; + } + const int64_t inclusive_query_ns = + context->stats->inverted_index_query_timer - query_ns_before; + DORIS_CHECK_GE(inclusive_query_ns, 0); + const int64_t exclusive_query_ns = + inclusive_query_ns > requested_null_ns ? inclusive_query_ns - requested_null_ns : 0; + context->stats->inverted_index_query_timer = query_ns_before + exclusive_query_ns; + }); + SCOPED_RAW_TIMER(&context->stats->inverted_index_query_timer); + const std::string search_str = query_value.get(); + const auto finish_query = + [&](const ::doris::snii::reader::LogicalIndexReader* reader) -> Status { + if (null_bitmap_cache_handle == nullptr) { + return Status::OK(); + } + const int64_t null_ns_before = context->stats->inverted_index_query_null_bitmap_timer; + Status status = _read_null_bitmap(context, null_bitmap_cache_handle, reader); + const int64_t null_ns_after = context->stats->inverted_index_query_null_bitmap_timer; + DORIS_CHECK_GE(null_ns_after, null_ns_before); + requested_null_ns += null_ns_after - null_ns_before; + return status; + }; + + if (int ignore_above = + std::stoi(get_parser_ignore_above_value_from_properties(_index_meta.properties())); + _reader_type == InvertedIndexReaderType::STRING_TYPE && search_str.size() > ignore_above) { + return Status::Error( + "query value is too long, evaluate skipped."); + } + + const bool actual_similarity = + context->collection_similarity && + IndexReaderHelper::is_need_similarity_score(query_type, &_index_meta); + const int32_t max_expansions = + context->runtime_state == nullptr + ? 50 + : context->runtime_state->query_options().inverted_index_max_expansions; + InvertedIndexQueryInfo query_info; + std::string plain_analysis_str = search_str; + if (query_type == InvertedIndexQueryType::MATCH_PHRASE_QUERY) { + parse_phrase_slop(&plain_analysis_str, &query_info); + } + const bool common_grams_phrase_shape = + (query_type == InvertedIndexQueryType::MATCH_PHRASE_QUERY && query_info.slop == 0) || + query_type == InvertedIndexQueryType::MATCH_PHRASE_PREFIX_QUERY; + const bool common_grams_query_eligible = common_grams_phrase_shape && !actual_similarity; + const bool raw_pattern_query = query_type == InvertedIndexQueryType::MATCH_REGEXP_QUERY || + query_type == InvertedIndexQueryType::WILDCARD_QUERY; + // Lucene-style CommonGrams: the plan decision is local to the segment and query. Snapshot the + // switch once so this query's plan and cache identity use the same mode. + const bool common_grams_query_plan_enabled = config::enable_common_grams_query_plan; + const inverted_index::CommonGramsPlanCostModel common_grams_cost_model { + .position_verify_factor = + static_cast(config::common_grams_position_verify_factor), + .common_grams_cost_ratio_percent = + static_cast(config::common_grams_plan_cost_ratio_percent)}; + const auto has_common_grams_analyzer = [](const InvertedIndexAnalyzerCtx* ctx) { + return ctx != nullptr && ctx->analyzer_provider != nullptr && + ctx->analyzer_provider->uses_common_grams() && + ctx->has_complete_common_grams_identity(); + }; + const bool safety_requires_plain = !common_grams_query_plan_enabled; + // The raw cache key cannot prove whether the immutable segment analyzer has CommonGrams until + // its metadata is open. Delay every eligible forced-plain lookup, then restore ordinary cache + // access below only for a segment that cannot contain gram terms. + const bool initial_force_plain = common_grams_query_eligible && safety_requires_plain; + const bool initial_allow_result_cache = !actual_similarity && !initial_force_plain; + const bool defer_result_cache_lookup = !actual_similarity && !initial_allow_result_cache; + const InvertedIndexRawQuerySemantic raw_semantic { + .raw_query_bytes = search_str, + .query_type = query_type, + .slop = query_info.slop, + .ordered = query_info.ordered, + .max_expansions = max_expansions, + .common_grams_query_plan_enabled = common_grams_query_plan_enabled}; + const auto index_file_key = _index_file_reader->get_index_file_cache_key(&_index_meta); + InvertedIndexQueryCache::CacheKey cache_key {index_file_key, column_name, query_type, + raw_semantic.encode()}; + std::string single_flight_key = cache_key.encode(); + auto* cache = InvertedIndexQueryCache::instance(); + InvertedIndexQueryCacheHandle cache_handler; + bool allow_result_cache = initial_allow_result_cache; + if (handle_query_cache(context, cache, cache_key, &cache_handler, bit_map, + allow_result_cache)) { + return finish_query(nullptr); + } + + snii_doris::DorisSniiFileReader::ScopedIOContext io_context_scope(context->io_ctx); + InvertedIndexCacheHandle searcher_cache_handle; + std::unique_ptr<::doris::snii::reader::LogicalIndexReader> uncached_reader; + const ::doris::snii::reader::LogicalIndexReader* logical_reader = nullptr; + RETURN_IF_ERROR(_get_logical_reader(context, &searcher_cache_handle, &uncached_reader, + &logical_reader)); + + std::optional rebuilt_analyzer_context; + const auto* common_grams_metadata = logical_reader->common_grams_metadata(); + if (!raw_pattern_query) { + auto rebuilt_result = inverted_index::maybe_rebuild_segment_analyzer_context( + analyzer_ctx, common_grams_metadata, _index_meta.properties(), + ExecEnv::GetInstance()->index_policy_mgr()); + if (!rebuilt_result.has_value()) { + if (common_grams_query_eligible) { + ++context->stats->snii_stats.common_grams_fallback_base_analyzer_mismatch; + } + return std::move(rebuilt_result.error()); + } + rebuilt_analyzer_context = std::move(*rebuilt_result); + } + const InvertedIndexAnalyzerCtx* effective_analyzer_context = + rebuilt_analyzer_context ? &*rebuilt_analyzer_context : analyzer_ctx; + const bool effective_common_grams_configured = + common_grams_query_eligible && has_common_grams_analyzer(effective_analyzer_context); + const bool segment_may_contain_common_grams = + common_grams_metadata != nullptr && common_grams_metadata->common_grams_coverage != + inverted_index::CommonGramsCoverage::kNone; + const bool force_plain = + common_grams_query_eligible && safety_requires_plain && + (effective_common_grams_configured || segment_may_contain_common_grams); + allow_result_cache = !actual_similarity && !force_plain; + if (defer_result_cache_lookup && allow_result_cache && + handle_query_cache(context, cache, cache_key, &cache_handler, bit_map, + allow_result_cache)) { + return finish_query(logical_reader); + } + InvertedIndexQueryInfo execution_query_info = query_info; + const auto plain_purpose = common_grams_query_eligible + ? std::optional(inverted_index::AnalysisPurpose::kPlainQuery) + : std::nullopt; + RETURN_IF_ERROR(_parse_query_terms(context, plain_analysis_str, query_type, + effective_analyzer_context, &execution_query_info, + plain_purpose)); + if (execution_query_info.term_infos.empty()) { + auto msg = fmt::format("token parser result is empty for SNII query '{}'", search_str); + if (is_match_query(query_type)) { + LOG(WARNING) << msg; + bit_map = std::make_shared(); + insert_query_cache(context, cache, cache_key, bit_map, &cache_handler, + allow_result_cache); + return finish_query(logical_reader); + } + return Status::Error(msg); + } + if (execution_query_info.has_common_gram()) { + return Status::Error( + "CommonGrams term escaped the plain query analyzer"); + } + if (actual_similarity && query_type == InvertedIndexQueryType::MATCH_PHRASE_PREFIX_QUERY && + execution_query_info.term_infos.size() == 1) { + return Status::Error( + "SNII scoring does not support a single-token phrase-prefix query"); + } + std::vector terms = to_terms(execution_query_info); + + // G02 count-only fast path: the SegmentIterator asserted (via the context + // flag) that only the match COUNT of this predicate matters, so eligible + // shapes are answered from dict-entry df without decoding postings. Placed + // AFTER the query-cache lookup (a cached row-accurate bitmap is free and + // counts correctly) and BEFORE single-flight; the fabricated [0, df) bitmap + // is returned early and NEVER inserted into the query cache or published to + // single-flight followers -- both are keyed identically to row-accurate + // queries and must only ever serve real row ids. + if (context->count_on_index_fastpath) { + bool count_handled = false; + std::shared_ptr count_bitmap; + RETURN_IF_ERROR(_try_count_only_fastpath(context, query_type, execution_query_info, terms, + &count_handled, &count_bitmap, logical_reader)); + if (count_handled) { + bit_map = std::move(count_bitmap); + RETURN_IF_ERROR(finish_query(logical_reader)); + // G03 reply: tell the SegmentIterator the bitmap is count-shaped + // (cardinality exact, row ids fabricated) so it may short-circuit + // row emission. Deliberately NOT set on the cache-hit return above + // or on the decode path below -- those bitmaps are row-accurate + // and keep today's emission. + context->count_on_index_fastpath_hit = true; + return Status::OK(); + } + } + + // Under a cold cache, parallel scanners _lazy_init the same segment concurrently and each + // would otherwise miss the searcher/query caches and redundantly open + decode this segment's + // index. Collapse identical concurrent queries into one shared execution (see SingleFlight). + static ::doris::segment_v2::inverted_index::SingleFlight< + std::pair>> + query_single_flight; + std::shared_ptr result_bitmap; + std::vector<::doris::snii::query::PhraseMatch> phrase_matches; + auto* phrase_matches_out = + actual_similarity && uses_phrase_frequency_scoring(query_type, execution_query_info) + ? &phrase_matches + : nullptr; + Status single_flight_status; + if (!allow_result_cache) { + single_flight_status = + _compute_query_bitmap(context, + {.query_type = query_type, + .query_info = execution_query_info, + .search_str = search_str, + .max_expansions = max_expansions, + .common_grams_query_shape = common_grams_query_eligible, + .force_plain = force_plain, + .common_grams_cost_model = common_grams_cost_model, + .analyzer_ctx = effective_analyzer_context, + .physical_raw_query_key = single_flight_key, + .logical_reader = logical_reader}, + &terms, &result_bitmap, phrase_matches_out); + } else { + DORIS_CHECK(phrase_matches_out == nullptr); + single_flight_status = run_query_single_flight( + query_single_flight, single_flight_key, &result_bitmap, +#ifdef BE_TEST + _single_flight_follower_joined_observer, _single_flight_follower_joined_opaque, + _single_flight_leader_before_compute_observer, + _single_flight_leader_before_compute_opaque, +#endif + [&](std::shared_ptr* out) { + auto status = _compute_query_bitmap( + context, + {.query_type = query_type, + .query_info = execution_query_info, + .search_str = search_str, + .max_expansions = max_expansions, + .common_grams_query_shape = common_grams_query_eligible, + .force_plain = force_plain, + .common_grams_cost_model = common_grams_cost_model, + .analyzer_ctx = effective_analyzer_context, + .physical_raw_query_key = single_flight_key, + .logical_reader = logical_reader}, + &terms, out, nullptr); + if (status.ok()) { + insert_query_cache(context, cache, cache_key, *out, &cache_handler, + allow_result_cache); + } + return status; + }); + } + RETURN_IF_ERROR(single_flight_status); + DORIS_CHECK(result_bitmap != nullptr); + if (actual_similarity && !result_bitmap->isEmpty()) { + ::doris::snii::stats::SniiStatsProvider segment_stats; + RETURN_IF_ERROR( + ::doris::snii::stats::SniiStatsProvider::open(logical_reader, &segment_stats)); + if (phrase_matches_out != nullptr) { + RETURN_IF_ERROR(score_phrase_matches(context, column_name, query_type, + execution_query_info, *logical_reader, + segment_stats, *result_bitmap, phrase_matches)); + } else if (uses_plain_term_frequency_scoring(query_type, execution_query_info)) { + RETURN_IF_ERROR(score_plain_term_candidates(context, column_name, execution_query_info, + *logical_reader, segment_stats, + *result_bitmap)); + } + } + bit_map = result_bitmap; + return finish_query(logical_reader); +} + +Status SniiIndexReader::_compute_query_bitmap( + const IndexQueryContextPtr& context, const SniiQueryBitmapRequest& request, + std::vector* preanalyzed_terms, std::shared_ptr* out, + std::vector<::doris::snii::query::PhraseMatch>* phrase_matches) { + // Bound once so the body below reads the same as before the request object was introduced; + // renaming 71 uses would have buried the actual change. + const InvertedIndexQueryType query_type = request.query_type; + const InvertedIndexQueryInfo& request_query_info = request.query_info; + const bool common_grams_query_shape = request.common_grams_query_shape; + const bool force_plain = request.force_plain; + const inverted_index::CommonGramsPlanCostModel common_grams_cost_model = + request.common_grams_cost_model; + const InvertedIndexAnalyzerCtx* analyzer_ctx = request.analyzer_ctx; + const std::string_view physical_raw_query_key = request.physical_raw_query_key; + const std::string_view search_str = request.search_str; + const int32_t max_expansions = request.max_expansions; + const ::doris::snii::reader::LogicalIndexReader* logical_reader = request.logical_reader; + + DORIS_CHECK(preanalyzed_terms != nullptr); + DORIS_CHECK(logical_reader != nullptr); + DORIS_CHECK(request_query_info.term_infos.size() == preanalyzed_terms->size()); + if (phrase_matches != nullptr) { + phrase_matches->clear(); + } + const auto* common_grams_metadata = logical_reader->common_grams_metadata(); + InvertedIndexQueryInfo query_info = request_query_info; + std::vector routed_terms = *preanalyzed_terms; + auto* terms = &routed_terms; + + const auto* common_grams_identity = + analyzer_ctx == nullptr ? nullptr : analyzer_ctx->get_common_grams_identity(); + const bool common_grams_configured = common_grams_query_shape && analyzer_ctx != nullptr && + analyzer_ctx->analyzer_provider != nullptr && + analyzer_ctx->analyzer_provider->uses_common_grams() && + analyzer_ctx->has_complete_common_grams_identity(); + const bool common_grams_candidate = + common_grams_configured && query_info.term_infos.size() >= 2 && !force_plain; + const bool common_grams_forced_plain = + common_grams_configured && query_info.term_infos.size() >= 2 && force_plain; + enum class CommonGramsPlainFallback : uint8_t { kNone, kNoGram, kIncompatible, kKillSwitch }; + CommonGramsPlainFallback common_grams_plain_fallback = + common_grams_forced_plain ? CommonGramsPlainFallback::kKillSwitch + : CommonGramsPlainFallback::kNone; + const bool common_grams_compatible = + common_grams_metadata != nullptr && common_grams_identity != nullptr && + (logical_reader->common_grams_posting_policy() == + ::doris::snii::format::CommonGramsPostingPolicy::kHybridV1 + ? inverted_index::is_common_grams_query_compatible( + *common_grams_metadata, *common_grams_identity, + inverted_index::CommonGramsCoverage::kMixed) + : inverted_index::is_common_grams_query_compatible(*common_grams_metadata, + *common_grams_identity)); + const auto* common_grams_word_set = + common_grams_configured ? analyzer_ctx->analyzer_provider->common_grams_word_set() + : nullptr; + const auto common_grams_query_mode = + query_type == InvertedIndexQueryType::MATCH_PHRASE_QUERY + ? inverted_index::CommonGramsQueryMode::kExact + : inverted_index::CommonGramsQueryMode::kPhrasePrefix; + const bool proven_no_common_gram = + common_grams_candidate && common_grams_compatible && common_grams_word_set != nullptr && + !inverted_index::common_grams_query_may_use_gram( + *preanalyzed_terms, common_grams_query_mode, *common_grams_word_set); + if (proven_no_common_gram && query_type == InvertedIndexQueryType::MATCH_PHRASE_QUERY) { + DORIS_CHECK(phrase_matches == nullptr); + ::doris::snii::SniiPrxExecutionProfileScope execution_profile(*context->stats); + InvertedIndexQueryInfo empty_gram_query_info; + std::vector docids; + RETURN_IF_ERROR(::doris::snii::query::planned_exact_phrase_query( + *logical_reader, query_info, empty_gram_query_info, common_grams_identity, &docids, + execution_profile.profile(), nullptr, common_grams_cost_model, + ::doris::snii::query::CommonGramsPlanDebugOverride::kNone)); + *out = docids_to_bitmap(docids); + return Status::OK(); + } + if (proven_no_common_gram) { + common_grams_plain_fallback = CommonGramsPlainFallback::kNoGram; + } else if (common_grams_candidate && common_grams_compatible) { + DORIS_CHECK(phrase_matches == nullptr); + DORIS_CHECK(!physical_raw_query_key.empty()); + const auto debug_override = ::doris::snii::query::common_grams_plan_debug_override(); + ::doris::snii::SniiPrxExecutionProfileScope execution_profile(*context->stats); + + // The gram-side analysis always runs. Without a memoized plan choice nothing can tell us + // in advance that the plain plan wins, and one analyzer pass over a phrase string is noise + // next to the posting decode that follows it. + InvertedIndexQueryInfo gram_query_info; + const auto gram_purpose = query_type == InvertedIndexQueryType::MATCH_PHRASE_QUERY + ? inverted_index::AnalysisPurpose::kExactPhraseQuery + : inverted_index::AnalysisPurpose::kPhrasePrefixQuery; + RETURN_IF_ERROR(_parse_query_terms(context, std::string(search_str), query_type, + analyzer_ctx, &gram_query_info, gram_purpose)); + + std::vector docids; + if (query_type == InvertedIndexQueryType::MATCH_PHRASE_QUERY) { + RETURN_IF_ERROR(::doris::snii::query::planned_exact_phrase_query( + *logical_reader, query_info, gram_query_info, common_grams_identity, &docids, + execution_profile.profile(), nullptr, common_grams_cost_model, debug_override)); + } else { + DORIS_CHECK(query_type == InvertedIndexQueryType::MATCH_PHRASE_PREFIX_QUERY); + RETURN_IF_ERROR(::doris::snii::query::planned_phrase_prefix_query( + *logical_reader, query_info, gram_query_info, common_grams_identity, &docids, + execution_profile.profile(), max_expansions, nullptr, common_grams_cost_model, + debug_override)); + } + *out = docids_to_bitmap(docids); + return Status::OK(); + } else if (common_grams_candidate) { + common_grams_plain_fallback = CommonGramsPlainFallback::kIncompatible; + } + switch (query_type) { + case InvertedIndexQueryType::EQUAL_QUERY: + case InvertedIndexQueryType::MATCH_ANY_QUERY: + case InvertedIndexQueryType::MATCH_ALL_QUERY: + case InvertedIndexQueryType::MATCH_PHRASE_QUERY: { + bool all_representable = false; + RETURN_IF_ERROR(::doris::snii::query::internal::route_query_terms( + *logical_reader, query_info, terms, &all_representable)); + if (terms->empty() && (query_type == InvertedIndexQueryType::EQUAL_QUERY || + query_type == InvertedIndexQueryType::MATCH_ANY_QUERY)) { + *out = std::make_shared(); + return Status::OK(); + } + if (!all_representable && (query_type == InvertedIndexQueryType::MATCH_ALL_QUERY || + query_type == InvertedIndexQueryType::MATCH_PHRASE_QUERY)) { + *out = std::make_shared(); + return Status::OK(); + } + break; + } + default: + break; + } + SniiQueryExecutionResult query_result; + const bool phrase_can_decode_prx = query_type == InvertedIndexQueryType::MATCH_PHRASE_QUERY; + const bool needs_prx_profile = + terms->size() > 1 && (phrase_can_decode_prx || + query_type == InvertedIndexQueryType::MATCH_PHRASE_PREFIX_QUERY); + if (needs_prx_profile) { + ::doris::snii::SniiPrxExecutionProfileScope execution_profile(*context->stats); + const Status execution_status = execute_snii_query( + *logical_reader, query_type, query_info, search_str, *terms, max_expansions, + phrase_matches != nullptr, &query_result, execution_profile.profile()); + if (common_grams_plain_fallback != CommonGramsPlainFallback::kNone) { + auto& plan_stats = execution_profile.profile()->phrase_query_stats; + ++plan_stats.common_grams_candidate_queries; + ++plan_stats.common_grams_plain_plans; + switch (common_grams_plain_fallback) { + case CommonGramsPlainFallback::kNoGram: + ++plan_stats.common_grams_fallback_no_gram; + break; + case CommonGramsPlainFallback::kIncompatible: + ++plan_stats.common_grams_fallback_incompatible; + break; + case CommonGramsPlainFallback::kKillSwitch: + ++plan_stats.common_grams_fallback_kill_switch; + break; + case CommonGramsPlainFallback::kNone: + break; + } + } + RETURN_IF_ERROR(execution_status); + } else { + RETURN_IF_ERROR(execute_snii_query(*logical_reader, query_type, query_info, search_str, + *terms, max_expansions, phrase_matches != nullptr, + &query_result, nullptr)); + } + *out = std::move(query_result.bitmap); + if (phrase_matches != nullptr) { + *phrase_matches = std::move(query_result.phrase_matches); + } + return Status::OK(); +} + +#ifdef BE_TEST +Status SniiIndexReader::_compute_query_bitmap(const IndexQueryContextPtr& context, + InvertedIndexQueryType query_type, + const InvertedIndexQueryInfo& query_info, + std::string_view search_str, + std::vector* terms, + int32_t max_expansions, + std::shared_ptr* out) { + snii_doris::DorisSniiFileReader::ScopedIOContext io_context_scope(context->io_ctx); + InvertedIndexCacheHandle searcher_cache_handle; + std::unique_ptr<::doris::snii::reader::LogicalIndexReader> uncached_reader; + const ::doris::snii::reader::LogicalIndexReader* logical_reader = nullptr; + RETURN_IF_ERROR(_get_logical_reader(context, &searcher_cache_handle, &uncached_reader, + &logical_reader)); + return _compute_query_bitmap(context, + {.query_type = query_type, + .query_info = query_info, + .search_str = search_str, + .max_expansions = max_expansions, + .logical_reader = logical_reader}, + terms, out, nullptr); +} +#endif + +Status SniiIndexReader::_try_count_only_fastpath( + const IndexQueryContextPtr& context, InvertedIndexQueryType query_type, + const InvertedIndexQueryInfo& query_info, const std::vector& terms, + bool* handled, std::shared_ptr* out, + const ::doris::snii::reader::LogicalIndexReader* preopened_reader) { + *handled = false; + // Shape guard: only exact-term query types. Prefix/regexp/wildcard/ + // phrase-prefix expand the term set, so no single dict entry carries the + // count; range types never reach SNII anyway. + switch (query_type) { + case InvertedIndexQueryType::EQUAL_QUERY: + case InvertedIndexQueryType::MATCH_ANY_QUERY: + case InvertedIndexQueryType::MATCH_ALL_QUERY: + case InvertedIndexQueryType::MATCH_PHRASE_QUERY: + break; + default: + return Status::OK(); + } + if (terms.size() != 1) { + // Multi-term MATCH_ANY (OR) / MATCH_ALL (AND) counts are not derivable + // from per-term dfs (overlap unknown), and phrases require positional + // verification, so execute the normal query path. + return Status::OK(); + } + + snii_doris::DorisSniiFileReader::ScopedIOContext io_context_scope(context->io_ctx); + InvertedIndexCacheHandle searcher_cache_handle; + std::unique_ptr<::doris::snii::reader::LogicalIndexReader> uncached_reader; + const ::doris::snii::reader::LogicalIndexReader* logical_reader = preopened_reader; + if (logical_reader == nullptr) { + RETURN_IF_ERROR(_get_logical_reader(context, &searcher_cache_handle, &uncached_reader, + &logical_reader)); + } + + std::string physical_term_scratch; + std::string_view physical_term; + bool representable = false; + DORIS_CHECK(query_info.term_infos.size() == 1); + RETURN_IF_ERROR(::doris::snii::query::internal::route_query_term_view( + *logical_reader, query_info.term_infos.front(), &physical_term_scratch, &physical_term, + &representable)); + uint64_t count = 0; + if (representable) { + RETURN_IF_ERROR( + ::doris::snii::query::count_only_term_df(*logical_reader, physical_term, &count)); + } + + // Null handling. df is the exact match count REGARDLESS of nulls: the + // writer adds no tokens for a null doc (scalar add_nulls; a NULL array row + // is an empty range), so postings -- and therefore df -- never include + // null rows, exactly matching MATCH's "null never matches" semantics. The + // fabricated bitmap however flows through FunctionMatchBase -> + // InvertedIndexResultBitmap::mask_out_null, which subtracts the segment's + // REAL null bitmap from it; a dense [0, df) range colliding with null row + // ids would be shrunk below df. So on a segment WITH a null bitmap, load + // it (query-cache backed, the same read the normal MATCH path performs) + // and fabricate df ids DISJOINT from it, making that subtraction a + // provable no-op. Segments without a null section (the writer omits it + // when no row is null) keep the trivial [0, df) range. + auto result = std::make_shared(); + if (count > 0 && logical_reader->section_refs().null_bitmap.length > 0) { + InvertedIndexQueryCacheHandle null_bitmap_cache_handle; + RETURN_IF_ERROR(_read_null_bitmap(context, &null_bitmap_cache_handle, logical_reader)); + std::shared_ptr nulls = null_bitmap_cache_handle.get_bitmap(); + // Fall through on a missing bitmap behind the cache handle or a + // fabrication failure (df + null count breaching the docid domain): + // both mean a corrupt index, and the row-accurate decode -- which + // intersects real ids -- must own the answer rather than a blind + // fabrication. The count_fastpath_hits test seam already counted the + // dict lookup above; production correctness is unaffected. + if (nulls == nullptr) { + return Status::OK(); + } + if (!nulls->isEmpty()) { + if (!::doris::snii::query::fabricate_null_disjoint_count_bitmap(count, *nulls, + result.get()) + .ok()) { + return Status::OK(); + } + } else { + result->addRange(0, count); + } + } else if (count > 0) { + result->addRange(0, count); + } + *out = std::move(result); + *handled = true; + return Status::OK(); +} + +Status SniiIndexReader::try_query(const IndexQueryContextPtr& /*context*/, + const std::string& /*column_name*/, const Field& /*query_value*/, + InvertedIndexQueryType /*query_type*/, size_t* /*count*/) { + return Status::Error("SNII does not support try_query"); +} + +Status SniiIndexReader::read_null_bitmap(const IndexQueryContextPtr& context, + InvertedIndexQueryCacheHandle* cache_handle, + lucene::store::Directory* /*dir*/) { + return _read_null_bitmap(context, cache_handle, nullptr); +} + +Status SniiIndexReader::_read_null_bitmap( + const IndexQueryContextPtr& context, InvertedIndexQueryCacheHandle* cache_handle, + const ::doris::snii::reader::LogicalIndexReader* preopened_reader) { + SCOPED_RAW_TIMER(&context->stats->inverted_index_query_null_bitmap_timer); + auto index_file_key = _index_file_reader->get_index_file_cache_key(&_index_meta); + InvertedIndexQueryCache::CacheKey cache_key { + index_file_key, "", InvertedIndexQueryType::UNKNOWN_QUERY, "null_bitmap"}; + auto* cache = InvertedIndexQueryCache::instance(); + if (cache->lookup(cache_key, cache_handle)) { + return Status::OK(); + } + + snii_doris::DorisSniiFileReader::ScopedIOContext io_context_scope(context->io_ctx); + InvertedIndexCacheHandle searcher_cache_handle; + std::unique_ptr<::doris::snii::reader::LogicalIndexReader> uncached_reader; + const ::doris::snii::reader::LogicalIndexReader* logical_reader = preopened_reader; + if (logical_reader == nullptr) { + RETURN_IF_ERROR(_get_logical_reader(context, &searcher_cache_handle, &uncached_reader, + &logical_reader)); + } + auto null_bitmap = std::make_shared(); + const auto& ref = logical_reader->section_refs().null_bitmap; + if (ref.length > 0) { + std::vector bytes; + RETURN_IF_ERROR(logical_reader->reader()->read_at(ref.offset, ref.length, &bytes)); + ::doris::snii::format::NullBitmapReader reader; + RETURN_IF_ERROR(::doris::snii::format::NullBitmapReader::open(::doris::snii::Slice(bytes), + &reader)); + reader.copy_to(null_bitmap.get()); + null_bitmap->runOptimize(); + } + cache->insert(cache_key, null_bitmap, cache_handle); + return Status::OK(); +} + +} // namespace doris::segment_v2 diff --git a/be/src/storage/index/snii/snii_index_reader.h b/be/src/storage/index/snii/snii_index_reader.h new file mode 100644 index 00000000000000..aef3d12d904e68 --- /dev/null +++ b/be/src/storage/index/snii/snii_index_reader.h @@ -0,0 +1,180 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "storage/index/inverted/common_grams/common_grams_query_cost.h" +#include "storage/index/inverted/inverted_index_query_type.h" +#include "storage/index/inverted/inverted_index_reader.h" + +namespace doris::snii::reader { +class LogicalIndexReader; +} // namespace doris::snii::reader + +namespace doris::snii::query { +struct PhraseMatch; +} // namespace doris::snii::query + +namespace doris::segment_v2 { + +// One query plus the plan the caller chose for it. This is a parameter object rather than a +// parameter list because _compute_query_bitmap() took fourteen positional arguments, two of them +// adjacent bools (common_grams_query_shape, force_plain) that no call site could tell apart +// without counting commas. +struct SniiQueryBitmapRequest { + InvertedIndexQueryType query_type; + const InvertedIndexQueryInfo& query_info; + std::string_view search_str; + int32_t max_expansions = 0; + + // Plan decisions the caller has already made. Both bools are false on the plain path. + bool common_grams_query_shape = false; + bool force_plain = false; + inverted_index::CommonGramsPlanCostModel common_grams_cost_model {}; + const InvertedIndexAnalyzerCtx* analyzer_ctx = nullptr; + // Identifies the physical query for the single-flight key; empty when unused. + std::string_view physical_raw_query_key {}; + + const ::doris::snii::reader::LogicalIndexReader* logical_reader = nullptr; +}; + +class SniiIndexReader final : public InvertedIndexReader { + ENABLE_FACTORY_CREATOR(SniiIndexReader); + +public: +#ifdef BE_TEST + using SingleFlightFollowerJoinedObserver = void (*)(void*) noexcept; + using SingleFlightLeaderBeforeComputeObserver = void (*)(void*) noexcept; + using SearcherOpenObserver = void (*)(void*) noexcept; +#endif + + SniiIndexReader(const TabletIndex* index_meta, + const std::shared_ptr& index_file_reader, + InvertedIndexReaderType reader_type) + : InvertedIndexReader(index_meta, index_file_reader), _reader_type(reader_type) {} + + Status new_iterator(std::unique_ptr* iterator) override; + Status query(const IndexQueryContextPtr& context, const std::string& column_name, + const Field& query_value, InvertedIndexQueryType query_type, + std::shared_ptr& bit_map, + const InvertedIndexAnalyzerCtx* analyzer_ctx = nullptr) override; + Status query_with_null_bitmap(const IndexQueryContextPtr& context, + const std::string& column_name, const Field& query_value, + InvertedIndexQueryType query_type, + std::shared_ptr& bit_map, + InvertedIndexQueryCacheHandle* null_bitmap_cache_handle, + const InvertedIndexAnalyzerCtx* analyzer_ctx = nullptr) override; + Status try_query(const IndexQueryContextPtr& context, const std::string& column_name, + const Field& query_value, InvertedIndexQueryType query_type, + size_t* count) override; + Status read_null_bitmap(const IndexQueryContextPtr& context, + InvertedIndexQueryCacheHandle* cache_handle, + lucene::store::Directory* dir = nullptr) override; + InvertedIndexReaderType type() override { return _reader_type; } + +#ifdef BE_TEST + void set_single_flight_follower_joined_observer_for_test( + SingleFlightFollowerJoinedObserver observer, void* opaque) { + _single_flight_follower_joined_observer = observer; + _single_flight_follower_joined_opaque = opaque; + } + void set_single_flight_leader_before_compute_observer_for_test( + SingleFlightLeaderBeforeComputeObserver observer, void* opaque) { + _single_flight_leader_before_compute_observer = observer; + _single_flight_leader_before_compute_opaque = opaque; + } + void set_searcher_open_observer_for_test(SearcherOpenObserver observer, void* opaque) { + _searcher_open_observer = observer; + _searcher_open_opaque = opaque; + } +#endif + +private: + Status _query(const IndexQueryContextPtr& context, const std::string& column_name, + const Field& query_value, InvertedIndexQueryType query_type, + std::shared_ptr& bit_map, + InvertedIndexQueryCacheHandle* null_bitmap_cache_handle, + const InvertedIndexAnalyzerCtx* analyzer_ctx); + Status _parse_query_terms( + const IndexQueryContextPtr& context, std::string search_str, + InvertedIndexQueryType query_type, const InvertedIndexAnalyzerCtx* analyzer_ctx, + InvertedIndexQueryInfo* query_info, + std::optional purpose_override = std::nullopt); + Status _get_logical_reader( + const IndexQueryContextPtr& context, InvertedIndexCacheHandle* searcher_cache_handle, + std::unique_ptr<::doris::snii::reader::LogicalIndexReader>* uncached_reader, + const ::doris::snii::reader::LogicalIndexReader** logical_reader); + Status _read_null_bitmap(const IndexQueryContextPtr& context, + InvertedIndexQueryCacheHandle* cache_handle, + const ::doris::snii::reader::LogicalIndexReader* preopened_reader); + // Opens the segment index and runs the query, producing the result bitmap. Invoked as the + // single-flight "compute" step by query(); see SingleFlight for the concurrency rationale. + Status _compute_query_bitmap(const IndexQueryContextPtr& context, + const SniiQueryBitmapRequest& request, + std::vector* terms, + std::shared_ptr* out, + std::vector<::doris::snii::query::PhraseMatch>* phrase_matches); +#ifdef BE_TEST + Status _compute_query_bitmap(const IndexQueryContextPtr& context, + InvertedIndexQueryType query_type, + const InvertedIndexQueryInfo& query_info, + std::string_view search_str, std::vector* terms, + int32_t max_expansions, std::shared_ptr* out); +#endif + // G02 count-only fast path. Only called when the caller (SegmentIterator) + // set context->count_on_index_fastpath, i.e. the match count alone decides + // the scan result. On *handled = true, *out is a bitmap of cardinality df + // (row ids NOT real) built from a single exact term's dict-entry df WITHOUT + // decoding postings. On a segment without a null bitmap the fabricated ids + // are the dense range [0, df); on a segment WITH one they are the first df + // NON-NULL row ids (see + // fabricate_null_disjoint_count_bitmap) so that the unconditional + // FunctionMatchBase -> mask_out_null subtraction of the real null bitmap + // is a no-op and the cardinality stays df -- which is already the exact + // match count, because postings never contain null docs. Falls through + // (*handled = false) for every other shape: every multi-term query + // (including phrase and OR/AND), prefix/regexp/wildcard/phrase-prefix + // expansion. Multi-term sloppy phrases fall through with every other + // multi-term shape; a single-term phrase remains exactly one posting df. + // On *handled = true, query() also raises + // context->count_on_index_fastpath_hit (G03) so the SegmentIterator may + // short-circuit row emission for the count-shaped bitmap. + Status _try_count_only_fastpath( + const IndexQueryContextPtr& context, InvertedIndexQueryType query_type, + const InvertedIndexQueryInfo& query_info, const std::vector& terms, + bool* handled, std::shared_ptr* out, + const ::doris::snii::reader::LogicalIndexReader* preopened_reader = nullptr); + + InvertedIndexReaderType _reader_type; +#ifdef BE_TEST + SingleFlightFollowerJoinedObserver _single_flight_follower_joined_observer = nullptr; + void* _single_flight_follower_joined_opaque = nullptr; + SingleFlightLeaderBeforeComputeObserver _single_flight_leader_before_compute_observer = nullptr; + void* _single_flight_leader_before_compute_opaque = nullptr; + SearcherOpenObserver _searcher_open_observer = nullptr; + void* _searcher_open_opaque = nullptr; +#endif +}; + +} // namespace doris::segment_v2 diff --git a/be/src/storage/index/snii/snii_index_writer.cpp b/be/src/storage/index/snii/snii_index_writer.cpp new file mode 100644 index 00000000000000..d6ada539077e44 --- /dev/null +++ b/be/src/storage/index/snii/snii_index_writer.cpp @@ -0,0 +1,512 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/snii_index_writer.h" + +#include + +#include +#include +#include +#include + +#include "common/cast_set.h" +#include "common/config.h" +#include "common/logging.h" +#include "storage/index/index_file_writer.h" +#include "storage/index/inverted/analyzer/analyzer.h" +#include "storage/index/inverted/common_grams/common_grams_key_codec.h" +#include "storage/index/inverted/query/query_info.h" +#include "storage/index/inverted/token_filter/common_grams_filter.h" +#include "storage/index/snii/query/bm25_scorer.h" +#include "storage/index/snii/writer/global_memory_limiter.h" +#include "storage/index/snii/writer/snii_build_memory_tracker.h" +#include "storage/tablet/tablet_schema.h" + +namespace doris::segment_v2 { +namespace { + +Status validate_common_grams_metadata_seed( + const inverted_index::CommonGramsSegmentMetadata& metadata) { + using namespace inverted_index; + auto status = validate_common_grams_segment_metadata(metadata); + if (!status.ok() || metadata.plain_term_key_version != PlainTermKeyVersion::kEscapedV1 || + metadata.common_grams_coverage != CommonGramsCoverage::kComplete || + metadata.common_grams_semantics_version != COMMON_GRAMS_SEMANTICS_VERSION_V1 || + metadata.common_grams_key_version != COMMON_GRAMS_KEY_VERSION_V1 || + metadata.scoring_coverage != ScoringCoverage::kComplete || + metadata.scoring_stats_version != COMMON_GRAMS_SCORING_STATS_VERSION_V1 || + metadata.norm_semantics_version != COMMON_GRAMS_NORM_SEMANTICS_VERSION_V1) { + return Status::Error( + "SNII CommonGrams metadata identity seed is incomplete or incompatible"); + } + return Status::OK(); +} + +} // namespace + +SniiIndexColumnWriter::SniiIndexColumnWriter( + IndexFileWriter* index_file_writer, const TabletIndex* index_meta, FieldType value_type, + std::optional common_grams_metadata_seed) + : _index_file_writer(index_file_writer), + _index_meta(index_meta), + _is_char(value_type == FieldType::OLAP_FIELD_TYPE_CHAR), + _common_grams_build_enabled(config::enable_common_grams_index_build), + _common_grams_metadata_seed(std::move(common_grams_metadata_seed)) {} + +Status SniiIndexColumnWriter::init() { + _should_analyzer = + inverted_index::InvertedIndexAnalyzer::should_analyzer(_index_meta->properties()); + _has_positions = get_parser_phrase_support_string_from_properties(_index_meta->properties()) == + INVERTED_INDEX_PARSER_PHRASE_SUPPORT_YES; + _config = _has_positions ? ::doris::snii::format::IndexConfig::kDocsPositions + : ::doris::snii::format::IndexConfig::kDocsOnly; + auto ignore_above_value = + get_parser_ignore_above_value_from_properties(_index_meta->properties()); + _ignore_above = cast_set(std::stoul(ignore_above_value)); + const auto spill_threshold = + static_cast(config::inverted_index_ram_buffer_size * 1024 * 1024); + // The consume_release callback mirrors this writer's live build bytes into + // the process-wide SNII index-build observation tracker, so ingestion shows + // up as its own line in Doris's memory picture (the allocation hook alone + // only knows which THREAD allocated). + _memory_reporter = std::make_unique<::doris::snii::writer::MemoryReporter>( + ::doris::snii::writer::snii_build_consume_release( + ::doris::snii::writer::BuildMemoryPopulation::kRegistered), + spill_threshold, ::doris::snii::writer::MemoryReporter::CapPolicy::kSpillThreshold); + _term_buffer = std::make_unique<::doris::snii::writer::SpimiTermBuffer>( + _has_positions, spill_threshold, _memory_reporter.get()); + // G09: join the PROCESS-WIDE build-RAM limiter. The per-writer spill threshold above + // bounds one writer; a load keeps (tablets x concurrency) writers alive at + // once, none of which may ever reach it -- the global registry bounds their + // SUM by asking the largest buffers to spill early (advisory flags honored + // on each writer's own thread; byte-identical output). Registration is + // UNCONDITIONAL: the limiter re-reads its trigger (SNII's share of the + // process limit, plus the process-level backstops) at every decision, so an + // admin enabling or disabling the share mid-load takes effect for writers + // that are already running -- it is not latched here. + // G09 anti-storm knobs (see the config comments): the forced-spill floor + // gates both the owner-side honor (a request is a pending no-op until the + // reclaimable arena regrows past it) and the limiter's victim eligibility, + // and the run-file cap merge-compacts a writer's spill runs so the final + // k-way merge's fd fan-in stays bounded. Applied unconditionally -- the + // floor also protects test-seam requests, and the cap also bounds + // per-writer gate-2 runs when the global limiter is off. + _term_buffer->set_forced_spill_min_arena_bytes( + static_cast(std::max(config::snii_forced_spill_min_arena_bytes, 0))); + _term_buffer->set_max_run_files( + static_cast(std::max(config::snii_spill_max_run_files_per_buffer, 0))); + auto* global_limiter = ::doris::snii::writer::GlobalMemoryLimiter::instance(); + global_limiter->set_min_victim_arena_bytes(config::snii_forced_spill_min_arena_bytes); + _term_buffer->attach_global_limiter(global_limiter); + _analyzer_config.analyzer_name = get_analyzer_name_from_properties(_index_meta->properties()); + _analyzer_config.parser_type = get_inverted_index_parser_type_from_string( + get_parser_string_from_properties(_index_meta->properties())); + _analyzer_config.parser_mode = + get_parser_mode_string_from_properties(_index_meta->properties()); + _analyzer_config.char_filter_map = + get_parser_char_filter_map_from_properties(_index_meta->properties()); + _analyzer_config.lower_case = + get_parser_lowercase_from_properties(_index_meta->properties()); + _analyzer_config.stop_words = get_parser_stopwords_from_properties(_index_meta->properties()); + try { + _char_string_reader = inverted_index::InvertedIndexAnalyzer::create_reader( + _analyzer_config.char_filter_map); + if (_should_analyzer) { + auto analyzer_provider = + inverted_index::InvertedIndexAnalyzer::create_analyzer_provider( + &_analyzer_config); + const bool policy_uses_common_grams = analyzer_provider->uses_common_grams(); + _uses_common_grams = _common_grams_build_enabled && policy_uses_common_grams; + if (policy_uses_common_grams && !_uses_common_grams) { + _common_grams_metadata_seed.reset(); + } + if (_uses_common_grams) { + if (!_has_positions) { + close_on_error(); + return Status::Error( + "SNII CommonGrams requires phrase positions"); + } + const auto base_analyzer_fingerprint = + analyzer_provider->base_analyzer_fingerprint(); + if (base_analyzer_fingerprint.empty()) { + close_on_error(); + return Status::Error( + "SNII CommonGrams analyzer has no immutable base fingerprint"); + } + const auto* identity = analyzer_provider->common_grams_identity(); + if (identity == nullptr) { + close_on_error(); + return Status::Error( + "SNII CommonGrams analyzer has no immutable dictionary identity"); + } + if (_common_grams_metadata_seed.has_value()) { + if (!inverted_index::common_grams_identity_matches(*_common_grams_metadata_seed, + *identity)) { + close_on_error(); + return Status::Error( + "SNII CommonGrams metadata seed does not match the analyzer " + "identity"); + } + } else { + _common_grams_metadata_seed = + inverted_index::make_common_grams_segment_metadata(*identity); + } + auto status = validate_common_grams_metadata_seed(*_common_grams_metadata_seed); + if (!status.ok()) { + close_on_error(); + return status; + } + _config = ::doris::snii::format::IndexConfig::kDocsPositionsScoring; + } else if (_common_grams_metadata_seed.has_value()) { + close_on_error(); + return Status::Error( + "SNII CommonGrams metadata cannot be attached to a plain analyzer"); + } + _analyzer = analyzer_provider->get_analyzer( + _uses_common_grams ? inverted_index::AnalysisPurpose::kSniiTransientIndex + : inverted_index::AnalysisPurpose::kPlainQuery); + } else if (_common_grams_metadata_seed.has_value()) { + close_on_error(); + return Status::Error( + "SNII CommonGrams metadata cannot be attached to a keyword analyzer"); + } + } catch (const CLuceneError& e) { + return Status::Error( + "SNII create analyzer failed: {}", e.what()); + } catch (const Exception& e) { + return Status::Error( + "SNII create analyzer failed: {}", e.what()); + } + if (_uses_common_grams) { + _term_buffer->enable_common_gram_pair_keys(); + } + return Status::OK(); +} + +void SniiIndexColumnWriter::set_direct_load(bool is_direct_load) { + // The PRX compression-tier hint must be stable for one index. The first + // pre-row call wins; repeat or late calls are ignored and logged. + DCHECK(!_direct_load_marked && _rid == 0); + if (_direct_load_marked || _rid != 0) { + LOG_EVERY_N(WARNING, 100) << "SNII set_direct_load(" << is_direct_load + << ") ignored (already_marked=" << _direct_load_marked + << ", rows_fed=" << _rid << ") for index " + << (_index_meta != nullptr ? _index_meta->index_id() : -1) + << "; keeping the first-captured PRX tier decision"; + return; + } + _direct_load_marked = true; + _is_direct_load = is_direct_load; +} + +Status SniiIndexColumnWriter::_add_value_tokens(const Slice& value, uint32_t docid, + uint32_t position_base, uint32_t* max_position, + uint32_t* semantic_length) { + DCHECK(max_position != nullptr); + DCHECK(semantic_length != nullptr); + *max_position = position_base; + *semantic_length = 0; + const size_t logical_size = _is_char ? strnlen(value.data, value.size) : value.size; + const std::string_view logical_value(value.data, logical_size); + if ((!_should_analyzer && logical_value.size() > _ignore_above) || + (_should_analyzer && logical_value.empty())) { + return Status::OK(); + } + + // T1a: tokens STREAM from the analyzer straight into the SPIMI buffer as + // string_views (the buffer interns the bytes into its own storage) -- no + // per-row vector and no per-token std::string materialization + // (the old get_analyse_result lane; profile: 3.4-4.7% of import CPU burned + // in token realloc). Golden-byte pins (snii_writer_golden_bytes_test.cpp) + // hold this path byte-identical to the materializing one it replaced. + auto consume_token = [&](std::string_view term, int32_t token_position, bool retain_positions) { + const uint32_t position = + _has_positions ? position_base + cast_set(token_position) : 0; + _term_buffer->add_token(term, docid, position, retain_positions); + *max_position = std::max(*max_position, position); + }; + + if (!_should_analyzer) { + // Keyword lane: the whole value is one exact-match token at position 0 + // (an EMPTY value is a valid keyword token, mirrored from the old lane). + consume_token(logical_value, 0, _has_positions); + } else { + try { + _char_string_reader->init(logical_value.data(), cast_set(logical_value.size()), + false); + if (_uses_common_grams) { + auto* token_stream = _analyzer->reusableTokenStream(L"", _char_string_reader); + if (_common_grams_filter == nullptr) { + _common_grams_filter = + dynamic_cast(token_stream); + DORIS_CHECK(_common_grams_filter != nullptr); + } else { + DCHECK_EQ(static_cast(_common_grams_filter), + token_stream); + } + _common_grams_filter->reset(); + + int32_t position = 0; + std::optional<::doris::snii::writer::ClassifiedPlainTerm> previous; + size_t previous_logical_term_size = 0; + inverted_index::SniiCommonGramsIndexEvent event; + while (_common_grams_filter->next_snii_index_event(&event)) { + const auto current = _term_buffer->intern_classified_plain_term( + event.plain_term, event.logical_term, + _common_grams_filter->common_words()); + const bool has_preceding_gram = + previous.has_value() && (previous->is_common || current.is_common) && + inverted_index::common_gram_component_sizes_encodable( + previous_logical_term_size, event.logical_term.size()); + const uint32_t gram_position = position_base + cast_set(position); + const uint32_t physical_position = + position_base + cast_set(position + 1); + if (has_preceding_gram) { + DCHECK(previous.has_value()); + _term_buffer->add_common_gram_and_plain( + previous->id, current.id, docid, gram_position, physical_position, + previous->is_common && current.is_common); + } else { + _term_buffer->add_plain_token(current.id, docid, physical_position); + } + ++position; + ++*semantic_length; + *max_position = std::max(*max_position, physical_position); + previous = current; + previous_logical_term_size = event.logical_term.size(); + } + } else { + std::unique_ptr owned_token_stream( + _analyzer->tokenStream(L"", _char_string_reader)); + auto* token_stream = owned_token_stream.get(); + // EXACT InvertedIndexAnalyzer::get_analyse_result semantics, + // including the subtle one: an empty token's position increment is + // dropped WITH the token (not accumulated into the next). + lucene::analysis::Token token; + int32_t position = 0; + while (token_stream->next(&token)) { + if (token.termLength() != 0) { + const std::string_view term(token.termBuffer(), + token.termLength()); + position += token.getPositionIncrement(); + consume_token(term, position, _has_positions); + } + } + token_stream->close(); + } + } catch (const CLuceneError& e) { + return _latch_analysis_failure(Status::Error( + "SNII analyze value failed: {}", e.what())); + } catch (const Exception& e) { + return _latch_analysis_failure(Status::Error( + "SNII analyze value failed: {}", e.what())); + } + } + return Status::OK(); +} + +Status SniiIndexColumnWriter::add_values(const std::string /*name*/, const void* values, + size_t count) { + if (!_failure_status.ok()) { + return _failure_status; + } + const auto* v = reinterpret_cast(values); + for (size_t i = 0; i < count; ++i) { + uint32_t max_position = 0; + uint32_t semantic_length = 0; + RETURN_IF_ERROR(_add_value_tokens(*v, _rid, 0, &max_position, &semantic_length)); + if (_uses_common_grams) { + _encoded_norms.push_back(::doris::snii::query::encode_norm(semantic_length)); + _report_encoded_norms_capacity(); + _scoring_token_count += semantic_length; + } + ++v; + ++_rid; + } + return Status::OK(); +} + +Status SniiIndexColumnWriter::add_array_values(size_t field_size, const void* value_ptr, + const uint8_t* nested_null_map, + const uint8_t* offsets_ptr, size_t count) { + if (!_failure_status.ok()) { + return _failure_status; + } + if (_uses_common_grams) { + return _latch_analysis_failure(Status::Error( + "SNII CommonGrams does not support ARRAY fields")); + } + if (count == 0) { + return Status::OK(); + } + const auto* offsets = reinterpret_cast(offsets_ptr); + size_t start_off = 0; + for (size_t i = 0; i < count; ++i) { + auto array_elem_size = offsets[i + 1] - offsets[i]; + uint32_t position_base = 0; + for (auto j = start_off; j < start_off + array_elem_size; ++j) { + if (nested_null_map != nullptr && nested_null_map[j] == 1) { + continue; + } + const auto* value = reinterpret_cast( + reinterpret_cast(value_ptr) + j * field_size); + uint32_t max_position = position_base; + uint32_t semantic_length = 0; + RETURN_IF_ERROR(_add_value_tokens(*value, _rid, position_base, &max_position, + &semantic_length)); + position_base = max_position + 1; + } + start_off += array_elem_size; + ++_rid; + } + return Status::OK(); +} + +void SniiIndexColumnWriter::_report_null_docids_capacity(bool release_all) { + if (_memory_reporter == nullptr) { + return; + } + const int64_t now = + release_all ? 0 : static_cast(_null_docids.capacity() * sizeof(uint32_t)); + if (now != _null_docids_charged_bytes) { + _memory_reporter->report(now - _null_docids_charged_bytes); + _null_docids_charged_bytes = now; + } +} + +void SniiIndexColumnWriter::_report_encoded_norms_capacity(bool release_all) { + if (_memory_reporter == nullptr) { + return; + } + const int64_t now = release_all ? 0 : static_cast(_encoded_norms.capacity()); + if (now != _encoded_norms_charged_bytes) { + _memory_reporter->report(now - _encoded_norms_charged_bytes); + _encoded_norms_charged_bytes = now; + } +} + +Status SniiIndexColumnWriter::add_nulls(uint32_t count) { + if (!_failure_status.ok()) { + return _failure_status; + } + // GEOMETRIC BULK reserve -- never an exact one: append_nullable calls + // add_nulls once per NULL RUN (thousands to millions of calls on a large + // interleaved-null segment), and an exact reserve(size()+count) caps + // capacity at "just enough" -- the NEXT call then reallocates and memcpys + // the WHOLE array, defeating geometric growth and turning total memcpy + // quadratic: O(runs x array_bytes). On an agentlogs full-compaction segment + // (12.4M rows, 22% interleaved nulls) that was TBs of memcpy per tablet -- + // the compaction ran 8+x slower than V3 (whose add_nulls is a roaring + // addRange). Doubling on overflow keeps the O(count) amortization AND makes + // one large run pay at most one reallocation. + const size_t need = _null_docids.size() + count; + if (need > _null_docids.capacity()) { + _null_docids.reserve(std::max(need, _null_docids.capacity() * 2)); + } + for (uint32_t i = 0; i < count; ++i) { + _null_docids.push_back(_rid + i); + } + _rid += count; + if (_uses_common_grams) { + _encoded_norms.insert(_encoded_norms.end(), count, ::doris::snii::query::encode_norm(0)); + _report_encoded_norms_capacity(); + } + _report_null_docids_capacity(); + return Status::OK(); +} + +Status SniiIndexColumnWriter::add_array_nulls(const uint8_t* null_map, size_t num_rows) { + if (!_failure_status.ok()) { + return _failure_status; + } + DCHECK(_rid >= num_rows); + if (num_rows == 0 || null_map == nullptr) { + return Status::OK(); + } + const auto first_row = _rid - num_rows; + for (size_t i = 0; i < num_rows; ++i) { + if (null_map[i] == 1) { + _null_docids.push_back(cast_set(first_row + i)); + } + } + _report_null_docids_capacity(); + return Status::OK(); +} + +Status SniiIndexColumnWriter::finish() { + if (!_failure_status.ok()) { + return _failure_status; + } + DCHECK(_term_buffer != nullptr); + auto status = _term_buffer->status(); + if (!status.ok()) { + return Status::InternalError("SNII term buffer error: {}", status.to_string()); + } + // Ownership of _null_docids hands off to the flush below (transient, + // flush-scoped); release the accumulation-phase charge so the retained + // reporter (and the observation tracker behind it) balances to zero. + _report_null_docids_capacity(/*release_all=*/true); + IndexFileWriter::SniiAddIndexOptions options {}; + options.is_direct_load = _is_direct_load; + if (_uses_common_grams) { + options.encoded_norms = std::move(_encoded_norms); + options.common_grams_metadata = _build_common_grams_metadata(); + options.common_grams_posting_policy = + ::doris::snii::format::CommonGramsPostingPolicy::kHybridV1; + } + status = _index_file_writer->add_snii_index( + _index_meta, cast_set(_rid), std::move(_null_docids), _term_buffer.get(), + _config, std::move(options), _memory_reporter.get()); + _report_encoded_norms_capacity(/*release_all=*/true); + RETURN_IF_ERROR(status); + _index_file_writer->retain_snii_memory_reporter(std::move(_memory_reporter)); + _term_buffer.reset(); + return Status::OK(); +} + +inverted_index::CommonGramsSegmentMetadata SniiIndexColumnWriter::_build_common_grams_metadata() + const { + DORIS_CHECK(_uses_common_grams); + DORIS_CHECK(_common_grams_metadata_seed.has_value()); + auto metadata = *_common_grams_metadata_seed; + metadata.common_grams_coverage = inverted_index::CommonGramsCoverage::kMixed; + metadata.scoring_doc_count = _rid; + metadata.scoring_token_count = _scoring_token_count; + return metadata; +} + +Status SniiIndexColumnWriter::_latch_analysis_failure(Status status) { + DORIS_CHECK(!status.ok()); + DORIS_CHECK(_failure_status.ok()); + _failure_status = std::move(status); + close_on_error(); + return _failure_status; +} + +void SniiIndexColumnWriter::close_on_error() { + _term_buffer.reset(); + // Balance the observation-tracker mirror before dropping the reporter. + _report_null_docids_capacity(/*release_all=*/true); + _report_encoded_norms_capacity(/*release_all=*/true); + _memory_reporter.reset(); + _null_docids.clear(); + std::vector().swap(_encoded_norms); + _scoring_token_count = 0; +} + +} // namespace doris::segment_v2 diff --git a/be/src/storage/index/snii/snii_index_writer.h b/be/src/storage/index/snii/snii_index_writer.h new file mode 100644 index 00000000000000..76fbd287401945 --- /dev/null +++ b/be/src/storage/index/snii/snii_index_writer.h @@ -0,0 +1,144 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include + +#include "storage/index/index_writer.h" +#include "storage/index/inverted/common_grams/common_grams_segment_metadata.h" +#include "storage/index/inverted/inverted_index_parser.h" +#include "storage/index/inverted/query/query_info.h" +#include "storage/index/inverted/util/reader.h" +#include "storage/index/snii/format/format_constants.h" +#include "storage/index/snii/writer/memory_reporter.h" +#include "storage/index/snii/writer/spimi_term_buffer.h" +#include "util/slice.h" + +namespace lucene::analysis { +class Analyzer; +} + +namespace doris::segment_v2::inverted_index { +class CommonGramsFilter; +} + +namespace doris::segment_v2 { + +class SniiIndexColumnWriter final : public IndexColumnWriter { +public: + SniiIndexColumnWriter(IndexFileWriter* index_file_writer, const TabletIndex* index_meta, + FieldType value_type, + std::optional + common_grams_metadata_seed = std::nullopt); + ~SniiIndexColumnWriter() override = default; + + Status init() override; + void set_direct_load(bool is_direct_load) override; + Status add_values(const std::string name, const void* values, size_t count) override; + Status add_array_values(size_t field_size, const void* value_ptr, + const uint8_t* nested_null_map, const uint8_t* offsets_ptr, + size_t count) override; + Status add_nulls(uint32_t count) override; + Status add_array_nulls(const uint8_t* null_map, size_t num_rows) override; + Status finish() override; + int64_t size() const override { return 0; } + void close_on_error() override; + +#ifdef BE_TEST + // TEST-ONLY view of the accumulated null docids: the growth-policy + // regression pin asserts add_nulls keeps geometric growth (an exact + // reserve(size+count) per null RUN made total memcpy quadratic -- the + // agentlogs full-compaction pathology). + const std::vector& null_docids_for_test() const { return _null_docids; } + ::doris::snii::writer::SpimiTermBuffer* term_buffer_for_test() const { + return _term_buffer.get(); + } + ::doris::snii::writer::MemoryReporter* memory_reporter_for_test() const { + return _memory_reporter.get(); + } + const std::vector& encoded_norms_for_test() const { return _encoded_norms; } + uint64_t scoring_token_count_for_test() const { return _scoring_token_count; } + ::doris::snii::format::IndexConfig config_for_test() const { return _config; } + bool has_common_grams_metadata_seed_for_test() const { + return _common_grams_metadata_seed.has_value(); + } + inverted_index::CommonGramsSegmentMetadata common_grams_metadata_for_test() const { + return _build_common_grams_metadata(); + } + void set_analysis_for_test(inverted_index::ReaderPtr reader, + std::shared_ptr analyzer) { + _should_analyzer = true; + _char_string_reader = std::move(reader); + _analyzer = std::move(analyzer); + } +#endif + +private: + Status _add_value_tokens(const Slice& value, uint32_t docid, uint32_t position_base, + uint32_t* max_position, uint32_t* semantic_length); + inverted_index::CommonGramsSegmentMetadata _build_common_grams_metadata() const; + // Mirrors _null_docids' capacity into _memory_reporter (delta-charged); + // release_all zeroes the charge (finish() handoff / close_on_error()). + void _report_null_docids_capacity(bool release_all = false); + void _report_encoded_norms_capacity(bool release_all = false); + Status _latch_analysis_failure(Status status); + + IndexFileWriter* _index_file_writer = nullptr; + const TabletIndex* _index_meta = nullptr; + bool _should_analyzer = false; + bool _has_positions = false; + const bool _is_char; + const bool _common_grams_build_enabled; + bool _uses_common_grams = false; + // Latch: set_direct_load() ran. The first call wins; a repeat or late call + // is ignored (and logged) so one index keeps one stable compression-tier + // decision. + bool _direct_load_marked = false; + // Captured by set_direct_load() under the same latch: this writer serves a + // stream/broker load (DataWriteType::TYPE_DIRECT). Consumed at finish() to + // route the prx region to the load-tier zstd level (patch C, + // config::snii_prx_zstd_level_direct_load). + bool _is_direct_load = false; + uint32_t _ignore_above = 0; + uint32_t _rid = 0; + ::doris::snii::format::IndexConfig _config = ::doris::snii::format::IndexConfig::kDocsOnly; + InvertedIndexAnalyzerConfig _analyzer_config; + inverted_index::ReaderPtr _char_string_reader; + std::shared_ptr _analyzer; + inverted_index::CommonGramsFilter* _common_grams_filter = nullptr; + std::unique_ptr<::doris::snii::writer::MemoryReporter> _memory_reporter; + std::unique_ptr<::doris::snii::writer::SpimiTermBuffer> _term_buffer; + std::vector _null_docids; + std::vector _encoded_norms; + uint64_t _scoring_token_count = 0; + std::optional _common_grams_metadata_seed; + // Bytes of _null_docids capacity currently mirrored into _memory_reporter + // (and through it the SNII index-build observation tracker). Re-charged on + // growth in add_nulls / add_array_nulls, released in finish() / close_on_error() -- + // without it a large interleaved-null segment accumulates untracked RSS the + // G09 limiter cannot see. + int64_t _null_docids_charged_bytes = 0; + int64_t _encoded_norms_charged_bytes = 0; + Status _failure_status = Status::OK(); +}; + +} // namespace doris::segment_v2 diff --git a/be/src/storage/index/snii/snii_prx_profile.h b/be/src/storage/index/snii/snii_prx_profile.h new file mode 100644 index 00000000000000..36218500a537e7 --- /dev/null +++ b/be/src/storage/index/snii/snii_prx_profile.h @@ -0,0 +1,313 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include "runtime/runtime_profile.h" +#include "storage/index/snii/format/prx_decode_stats.h" +#include "storage/index/snii/query/query_profile.h" +#include "storage/olap_common.h" + +namespace doris::snii { + +#ifdef BE_TEST +namespace testing { + +void record_prx_execution_profile_scope_construction(); +void record_prx_execution_profile_scope_flush(); +void reset_prx_execution_profile_scope_counters(); +uint64_t prx_execution_profile_scope_construction_count(); +uint64_t prx_execution_profile_scope_flush_count(); + +} // namespace testing +#endif + +inline void add_prx_decode_stats(OlapReaderStatistics* target, + const format::PrxDecodeStats& delta) { + SniiQueryStats& stats = target->snii_stats; + stats.prx_raw_frames += static_cast(delta.raw_frames); + stats.prx_zstd_frames += static_cast(delta.zstd_frames); + stats.prx_pfor_frames += static_cast(delta.pfor_frames); + stats.prx_plaintext_bytes += static_cast(delta.plaintext_bytes); + stats.prx_total_docs += static_cast(delta.total_docs); + stats.prx_selected_docs += static_cast(delta.selected_docs); + stats.prx_total_positions += static_cast(delta.total_positions); + stats.prx_selected_positions += static_cast(delta.selected_positions); + stats.prx_fetch_ns += static_cast(delta.fetch_ns); + stats.prx_decode_ns += static_cast(delta.decode_ns); + stats.prx_phrase_verify_ns += static_cast(delta.phrase_verify_ns); +} + +inline void add_phrase_query_stats(OlapReaderStatistics* target, + const format::PhraseQueryExecutionStats& delta) { + SniiQueryStats& stats = target->snii_stats; + stats.phrase_candidate_docs += static_cast(delta.exact_candidate_docs); + stats.phrase_candidate_visits += static_cast(delta.exact_candidate_visits); + stats.prx_streaming_frames += static_cast(delta.prx_streaming_frames); + stats.phrase_prefix_leading_candidate_docs += + static_cast(delta.prefix_leading_candidate_docs); + stats.phrase_prefix_tail_candidate_visits += + static_cast(delta.prefix_tail_candidate_visits); + stats.common_grams_candidate_queries += + static_cast(delta.common_grams_candidate_queries); + stats.common_grams_plain_plans += static_cast(delta.common_grams_plain_plans); + stats.common_grams_gram_plans += static_cast(delta.common_grams_gram_plans); + stats.common_grams_fallback_no_gram += + static_cast(delta.common_grams_fallback_no_gram); + stats.common_grams_fallback_incompatible += + static_cast(delta.common_grams_fallback_incompatible); + stats.common_grams_fallback_kill_switch += + static_cast(delta.common_grams_fallback_kill_switch); + stats.common_grams_fallback_cost += static_cast(delta.common_grams_fallback_cost); + stats.common_grams_fallback_base_analyzer_mismatch += + static_cast(delta.common_grams_fallback_base_analyzer_mismatch); + stats.common_grams_fallback_prefix_tail_empty += + static_cast(delta.common_grams_fallback_prefix_tail_empty); + stats.common_grams_authoritative_empty += + static_cast(delta.common_grams_authoritative_empty); + stats.common_grams_plain_posting_bytes += + static_cast(delta.common_grams_plain_posting_bytes); + stats.common_grams_gram_posting_bytes += + static_cast(delta.common_grams_gram_posting_bytes); + stats.common_grams_plain_estimated_candidate_df += + static_cast(delta.common_grams_plain_estimated_candidate_df); + stats.common_grams_gram_estimated_candidate_df += + static_cast(delta.common_grams_gram_estimated_candidate_df); + stats.common_grams_plain_estimated_cost += + static_cast(delta.common_grams_plain_estimated_cost); + stats.common_grams_gram_estimated_cost += + static_cast(delta.common_grams_gram_estimated_cost); + stats.common_grams_planning_ns += static_cast(delta.common_grams_planning_ns); +} + +// Exists only around an actual SNII compute execution. Query-cache hits, +// count-only fast paths, and single-flight followers never construct this +// scope, so they cannot contribute another execution's PRX work. The destructor +// flushes already-committed frames on both success and early error returns. +class SniiPrxExecutionProfileScope { +public: + explicit SniiPrxExecutionProfileScope(OlapReaderStatistics& target) : target_(target) { +#ifdef BE_TEST + testing::record_prx_execution_profile_scope_construction(); +#endif + } + ~SniiPrxExecutionProfileScope() { + add_prx_decode_stats(&target_, profile_.prx_decode_stats); + add_phrase_query_stats(&target_, profile_.phrase_query_stats); +#ifdef BE_TEST + testing::record_prx_execution_profile_scope_flush(); +#endif + } + + SniiPrxExecutionProfileScope(const SniiPrxExecutionProfileScope&) = delete; + SniiPrxExecutionProfileScope& operator=(const SniiPrxExecutionProfileScope&) = delete; + SniiPrxExecutionProfileScope(SniiPrxExecutionProfileScope&&) = delete; + SniiPrxExecutionProfileScope& operator=(SniiPrxExecutionProfileScope&&) = delete; + + query::QueryProfile* profile() { return &profile_; } + +private: + OlapReaderStatistics& target_; + query::QueryProfile profile_; +}; + +class SniiPrxRuntimeProfileCounters { +public: + static constexpr std::array counter_names() { + return {"SniiPrxRawFrames", + "SniiPrxZstdFrames", + "SniiPrxPforFrames", + "SniiPrxPlaintextBytes", + "SniiPrxTotalDocs", + "SniiPrxSelectedDocs", + "SniiPrxTotalPositions", + "SniiPrxSelectedPositions", + "SniiPrxFetchTime", + "SniiPrxInclusiveDecodeTime", + "SniiPrxExclusivePhraseVerifyTime"}; + } + + void initialize(RuntimeProfile* profile) { + raw_frames_ = profile->add_nonzero_counter("SniiPrxRawFrames", TUnit::UNIT, + RuntimeProfile::ROOT_COUNTER, 1); + zstd_frames_ = profile->add_nonzero_counter("SniiPrxZstdFrames", TUnit::UNIT, + RuntimeProfile::ROOT_COUNTER, 1); + pfor_frames_ = profile->add_nonzero_counter("SniiPrxPforFrames", TUnit::UNIT, + RuntimeProfile::ROOT_COUNTER, 1); + plaintext_bytes_ = profile->add_nonzero_counter("SniiPrxPlaintextBytes", TUnit::BYTES, + RuntimeProfile::ROOT_COUNTER, 1); + total_docs_ = profile->add_nonzero_counter("SniiPrxTotalDocs", TUnit::UNIT, + RuntimeProfile::ROOT_COUNTER, 1); + selected_docs_ = profile->add_nonzero_counter("SniiPrxSelectedDocs", TUnit::UNIT, + RuntimeProfile::ROOT_COUNTER, 1); + total_positions_ = profile->add_nonzero_counter("SniiPrxTotalPositions", TUnit::UNIT, + RuntimeProfile::ROOT_COUNTER, 1); + selected_positions_ = profile->add_nonzero_counter("SniiPrxSelectedPositions", TUnit::UNIT, + RuntimeProfile::ROOT_COUNTER, 1); + fetch_ns_ = profile->add_nonzero_counter("SniiPrxFetchTime", TUnit::TIME_NS, + RuntimeProfile::ROOT_COUNTER, 1); + decode_ns_ = profile->add_nonzero_counter("SniiPrxInclusiveDecodeTime", TUnit::TIME_NS, + RuntimeProfile::ROOT_COUNTER, 1); + phrase_verify_ns_ = + profile->add_nonzero_counter("SniiPrxExclusivePhraseVerifyTime", TUnit::TIME_NS, + RuntimeProfile::ROOT_COUNTER, 1); + } + + void update(const OlapReaderStatistics& stats) const { + const SniiQueryStats& s = stats.snii_stats; + COUNTER_UPDATE(raw_frames_, s.prx_raw_frames); + COUNTER_UPDATE(zstd_frames_, s.prx_zstd_frames); + COUNTER_UPDATE(pfor_frames_, s.prx_pfor_frames); + COUNTER_UPDATE(plaintext_bytes_, s.prx_plaintext_bytes); + COUNTER_UPDATE(total_docs_, s.prx_total_docs); + COUNTER_UPDATE(selected_docs_, s.prx_selected_docs); + COUNTER_UPDATE(total_positions_, s.prx_total_positions); + COUNTER_UPDATE(selected_positions_, s.prx_selected_positions); + COUNTER_UPDATE(fetch_ns_, s.prx_fetch_ns); + COUNTER_UPDATE(decode_ns_, s.prx_decode_ns); + COUNTER_UPDATE(phrase_verify_ns_, s.prx_phrase_verify_ns); + } + +private: + RuntimeProfile::Counter* raw_frames_ = nullptr; + RuntimeProfile::Counter* zstd_frames_ = nullptr; + RuntimeProfile::Counter* pfor_frames_ = nullptr; + RuntimeProfile::Counter* plaintext_bytes_ = nullptr; + RuntimeProfile::Counter* total_docs_ = nullptr; + RuntimeProfile::Counter* selected_docs_ = nullptr; + RuntimeProfile::Counter* total_positions_ = nullptr; + RuntimeProfile::Counter* selected_positions_ = nullptr; + RuntimeProfile::Counter* fetch_ns_ = nullptr; + RuntimeProfile::Counter* decode_ns_ = nullptr; + RuntimeProfile::Counter* phrase_verify_ns_ = nullptr; +}; + +class SniiPhraseRuntimeProfileCounters { +public: + void initialize(RuntimeProfile* profile) { + candidate_docs_ = profile->add_nonzero_counter("SniiPhraseCandidateDocs", TUnit::UNIT, + RuntimeProfile::ROOT_COUNTER, 1); + candidate_visits_ = profile->add_nonzero_counter("SniiPhraseCandidateVisits", TUnit::UNIT, + RuntimeProfile::ROOT_COUNTER, 1); + streaming_prx_frames_ = profile->add_nonzero_counter( + "SniiPhraseStreamingPrxFrames", TUnit::UNIT, RuntimeProfile::ROOT_COUNTER, 1); + prefix_leading_candidate_docs_ = + profile->add_nonzero_counter("SniiPhrasePrefixLeadingCandidateDocs", TUnit::UNIT, + RuntimeProfile::ROOT_COUNTER, 1); + prefix_tail_candidate_visits_ = + profile->add_nonzero_counter("SniiPhrasePrefixTailCandidateVisits", TUnit::UNIT, + RuntimeProfile::ROOT_COUNTER, 1); + common_grams_candidate_queries_ = profile->add_nonzero_counter( + "SniiCommonGramsCandidateQueries", TUnit::UNIT, RuntimeProfile::ROOT_COUNTER, 1); + common_grams_plain_plans_ = profile->add_nonzero_counter( + "SniiCommonGramsPlainPlans", TUnit::UNIT, RuntimeProfile::ROOT_COUNTER, 1); + common_grams_gram_plans_ = profile->add_nonzero_counter( + "SniiCommonGramsGramPlans", TUnit::UNIT, RuntimeProfile::ROOT_COUNTER, 1); + common_grams_fallback_no_gram_ = profile->add_nonzero_counter( + "SniiCommonGramsFallbackNoGram", TUnit::UNIT, RuntimeProfile::ROOT_COUNTER, 1); + common_grams_fallback_incompatible_ = + profile->add_nonzero_counter("SniiCommonGramsFallbackIncompatible", TUnit::UNIT, + RuntimeProfile::ROOT_COUNTER, 1); + common_grams_fallback_kill_switch_ = profile->add_nonzero_counter( + "SniiCommonGramsFallbackKillSwitch", TUnit::UNIT, RuntimeProfile::ROOT_COUNTER, 1); + common_grams_fallback_cost_ = profile->add_nonzero_counter( + "SniiCommonGramsFallbackCost", TUnit::UNIT, RuntimeProfile::ROOT_COUNTER, 1); + common_grams_fallback_base_analyzer_mismatch_ = + profile->add_nonzero_counter("SniiCommonGramsFallbackBaseAnalyzerMismatch", + TUnit::UNIT, RuntimeProfile::ROOT_COUNTER, 1); + common_grams_fallback_prefix_tail_empty_ = + profile->add_nonzero_counter("SniiCommonGramsFallbackPrefixTailEmpty", TUnit::UNIT, + RuntimeProfile::ROOT_COUNTER, 1); + common_grams_authoritative_empty_ = profile->add_nonzero_counter( + "SniiCommonGramsAuthoritativeEmpty", TUnit::UNIT, RuntimeProfile::ROOT_COUNTER, 1); + common_grams_plain_posting_bytes_ = profile->add_nonzero_counter( + "SniiCommonGramsPlainPostingBytes", TUnit::BYTES, RuntimeProfile::ROOT_COUNTER, 1); + common_grams_gram_posting_bytes_ = profile->add_nonzero_counter( + "SniiCommonGramsGramPostingBytes", TUnit::BYTES, RuntimeProfile::ROOT_COUNTER, 1); + common_grams_plain_estimated_candidate_df_ = + profile->add_nonzero_counter("SniiCommonGramsPlainEstimatedCandidateDf", + TUnit::UNIT, RuntimeProfile::ROOT_COUNTER, 1); + common_grams_gram_estimated_candidate_df_ = + profile->add_nonzero_counter("SniiCommonGramsGramEstimatedCandidateDf", TUnit::UNIT, + RuntimeProfile::ROOT_COUNTER, 1); + common_grams_plain_estimated_cost_ = profile->add_nonzero_counter( + "SniiCommonGramsPlainEstimatedCost", TUnit::UNIT, RuntimeProfile::ROOT_COUNTER, 1); + common_grams_gram_estimated_cost_ = profile->add_nonzero_counter( + "SniiCommonGramsGramEstimatedCost", TUnit::UNIT, RuntimeProfile::ROOT_COUNTER, 1); + common_grams_planning_ns_ = profile->add_nonzero_counter( + "SniiCommonGramsPlanningTime", TUnit::TIME_NS, RuntimeProfile::ROOT_COUNTER, 1); + } + + void update(const OlapReaderStatistics& stats) const { + const SniiQueryStats& s = stats.snii_stats; + COUNTER_UPDATE(candidate_docs_, s.phrase_candidate_docs); + COUNTER_UPDATE(candidate_visits_, s.phrase_candidate_visits); + COUNTER_UPDATE(streaming_prx_frames_, s.prx_streaming_frames); + COUNTER_UPDATE(prefix_leading_candidate_docs_, s.phrase_prefix_leading_candidate_docs); + COUNTER_UPDATE(prefix_tail_candidate_visits_, s.phrase_prefix_tail_candidate_visits); + COUNTER_UPDATE(common_grams_candidate_queries_, s.common_grams_candidate_queries); + COUNTER_UPDATE(common_grams_plain_plans_, s.common_grams_plain_plans); + COUNTER_UPDATE(common_grams_gram_plans_, s.common_grams_gram_plans); + COUNTER_UPDATE(common_grams_fallback_no_gram_, s.common_grams_fallback_no_gram); + COUNTER_UPDATE(common_grams_fallback_incompatible_, s.common_grams_fallback_incompatible); + COUNTER_UPDATE(common_grams_fallback_kill_switch_, s.common_grams_fallback_kill_switch); + COUNTER_UPDATE(common_grams_fallback_cost_, s.common_grams_fallback_cost); + COUNTER_UPDATE(common_grams_fallback_base_analyzer_mismatch_, + s.common_grams_fallback_base_analyzer_mismatch); + COUNTER_UPDATE(common_grams_fallback_prefix_tail_empty_, + s.common_grams_fallback_prefix_tail_empty); + COUNTER_UPDATE(common_grams_authoritative_empty_, s.common_grams_authoritative_empty); + COUNTER_UPDATE(common_grams_plain_posting_bytes_, s.common_grams_plain_posting_bytes); + COUNTER_UPDATE(common_grams_gram_posting_bytes_, s.common_grams_gram_posting_bytes); + COUNTER_UPDATE(common_grams_plain_estimated_candidate_df_, + s.common_grams_plain_estimated_candidate_df); + COUNTER_UPDATE(common_grams_gram_estimated_candidate_df_, + s.common_grams_gram_estimated_candidate_df); + COUNTER_UPDATE(common_grams_plain_estimated_cost_, s.common_grams_plain_estimated_cost); + COUNTER_UPDATE(common_grams_gram_estimated_cost_, s.common_grams_gram_estimated_cost); + COUNTER_UPDATE(common_grams_planning_ns_, s.common_grams_planning_ns); + } + +private: + RuntimeProfile::Counter* candidate_docs_ = nullptr; + RuntimeProfile::Counter* candidate_visits_ = nullptr; + RuntimeProfile::Counter* streaming_prx_frames_ = nullptr; + RuntimeProfile::Counter* prefix_leading_candidate_docs_ = nullptr; + RuntimeProfile::Counter* prefix_tail_candidate_visits_ = nullptr; + RuntimeProfile::Counter* common_grams_candidate_queries_ = nullptr; + RuntimeProfile::Counter* common_grams_plain_plans_ = nullptr; + RuntimeProfile::Counter* common_grams_gram_plans_ = nullptr; + RuntimeProfile::Counter* common_grams_fallback_no_gram_ = nullptr; + RuntimeProfile::Counter* common_grams_fallback_incompatible_ = nullptr; + RuntimeProfile::Counter* common_grams_fallback_kill_switch_ = nullptr; + RuntimeProfile::Counter* common_grams_fallback_cost_ = nullptr; + RuntimeProfile::Counter* common_grams_fallback_base_analyzer_mismatch_ = nullptr; + RuntimeProfile::Counter* common_grams_fallback_prefix_tail_empty_ = nullptr; + RuntimeProfile::Counter* common_grams_authoritative_empty_ = nullptr; + RuntimeProfile::Counter* common_grams_plain_posting_bytes_ = nullptr; + RuntimeProfile::Counter* common_grams_gram_posting_bytes_ = nullptr; + RuntimeProfile::Counter* common_grams_plain_estimated_candidate_df_ = nullptr; + RuntimeProfile::Counter* common_grams_gram_estimated_candidate_df_ = nullptr; + RuntimeProfile::Counter* common_grams_plain_estimated_cost_ = nullptr; + RuntimeProfile::Counter* common_grams_gram_estimated_cost_ = nullptr; + RuntimeProfile::Counter* common_grams_planning_ns_ = nullptr; +}; + +} // namespace doris::snii diff --git a/be/src/storage/index/snii/snii_query_stats.h b/be/src/storage/index/snii/snii_query_stats.h new file mode 100644 index 00000000000000..7c1947f3dd641c --- /dev/null +++ b/be/src/storage/index/snii/snii_query_stats.h @@ -0,0 +1,67 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +namespace doris::snii { + +// Populated by snii_prx_profile.h from a per-execution query::QueryProfile, and read back out by +// SniiPrxRuntimeProfileCounters / SniiPhraseRuntimeProfileCounters into the scan node's +// RuntimeProfile. Lives in its own header, mirroring InvertedIndexStatistics +// (storage/index/inverted/inverted_index_stats.h), so OlapReaderStatistics -- shared by every +// storage format -- holds one named field here instead of one field per counter. +struct SniiQueryStats { + int64_t prx_raw_frames = 0; + int64_t prx_zstd_frames = 0; + int64_t prx_pfor_frames = 0; + int64_t prx_plaintext_bytes = 0; + int64_t prx_total_docs = 0; + int64_t prx_selected_docs = 0; + int64_t prx_total_positions = 0; + int64_t prx_selected_positions = 0; + int64_t prx_fetch_ns = 0; + int64_t prx_decode_ns = 0; + int64_t prx_phrase_verify_ns = 0; + + int64_t phrase_candidate_docs = 0; + int64_t phrase_candidate_visits = 0; + int64_t prx_streaming_frames = 0; + int64_t phrase_prefix_leading_candidate_docs = 0; + int64_t phrase_prefix_tail_candidate_visits = 0; + + int64_t common_grams_candidate_queries = 0; + int64_t common_grams_plain_plans = 0; + int64_t common_grams_gram_plans = 0; + int64_t common_grams_fallback_no_gram = 0; + int64_t common_grams_fallback_incompatible = 0; + int64_t common_grams_fallback_kill_switch = 0; + int64_t common_grams_fallback_cost = 0; + int64_t common_grams_fallback_base_analyzer_mismatch = 0; + int64_t common_grams_fallback_prefix_tail_empty = 0; + int64_t common_grams_authoritative_empty = 0; + int64_t common_grams_plain_posting_bytes = 0; + int64_t common_grams_gram_posting_bytes = 0; + int64_t common_grams_plain_estimated_candidate_df = 0; + int64_t common_grams_gram_estimated_candidate_df = 0; + int64_t common_grams_plain_estimated_cost = 0; + int64_t common_grams_gram_estimated_cost = 0; + int64_t common_grams_planning_ns = 0; +}; + +} // namespace doris::snii diff --git a/be/src/storage/index/snii/stats/snii_stats_provider.cpp b/be/src/storage/index/snii/stats/snii_stats_provider.cpp new file mode 100644 index 00000000000000..e12fe6d5f73d80 --- /dev/null +++ b/be/src/storage/index/snii/stats/snii_stats_provider.cpp @@ -0,0 +1,145 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/stats/snii_stats_provider.h" + +#include + +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/format/dict_entry.h" +#include "storage/index/snii/format/format_constants.h" +#include "storage/index/snii/format/stats_block.h" + +namespace doris::snii::stats { + +using format::DictEntry; +using format::NormsPodReader; +using format::RegionRef; + +namespace { + +// Resolves a term's DictEntry. *found=false for an absent term (OK status). +Status lookup_entry(const reader::LogicalIndexReader& idx, std::string_view term, bool* found, + DictEntry* entry) { + uint64_t frq_base = 0; + uint64_t prx_base = 0; + return idx.lookup(term, found, entry, &frq_base, &prx_base); +} + +} // namespace + +Status SniiStatsProvider::open(const reader::LogicalIndexReader* idx, SniiStatsProvider* out) { + return open_impl(idx, out, true); +} + +#ifdef BE_TEST +Status SniiStatsProvider::open_legacy_for_test(const reader::LogicalIndexReader* idx, + SniiStatsProvider* out) { + return open_impl(idx, out, false); +} +#endif + +Status SniiStatsProvider::open_impl(const reader::LogicalIndexReader* idx, SniiStatsProvider* out, + bool require_semantic_metadata) { + if (idx == nullptr || out == nullptr) { + return Status::Error("stats_provider: null argument"); + } + out->idx_ = idx; + const auto& sb = idx->stats(); + out->doc_count_ = sb.doc_count; + out->indexed_doc_count_ = sb.indexed_doc_count; + out->sum_total_term_freq_ = sb.sum_total_term_freq; + + const RegionRef& norms = idx->section_refs().norms; + const auto* metadata = idx->common_grams_metadata(); + if (metadata != nullptr) { + using namespace segment_v2::inverted_index; + RETURN_IF_ERROR(validate_snii_scoring_metadata( + metadata, sb.doc_count, sb.sum_total_term_freq, + idx->tier() == format::IndexTier::kT3, idx->has_positions(), norms.length != 0)); + out->doc_count_ = metadata->scoring_doc_count; + out->indexed_doc_count_ = metadata->scoring_doc_count; + out->sum_total_term_freq_ = metadata->scoring_token_count; + } else if (require_semantic_metadata) { + RETURN_IF_ERROR(segment_v2::inverted_index::validate_snii_scoring_metadata( + nullptr, sb.doc_count, sb.sum_total_term_freq, + idx->tier() == format::IndexTier::kT3, idx->has_positions(), norms.length != 0)); + } + if (norms.length == 0) { + out->has_norms_ = false; + return Status::OK(); + } + + RETURN_IF_ERROR(idx->open_norms(&out->norms_reader_)); + if (metadata != nullptr && out->norms_reader_.doc_count() != metadata->scoring_doc_count) { + return Status::Error( + "snii_stats: semantic norms doc count {} differs from scoring doc count {}", + out->norms_reader_.doc_count(), metadata->scoring_doc_count); + } + out->has_norms_ = true; + return Status::OK(); +} + +double SniiStatsProvider::avgdl() const { + const uint64_t denom = std::max(1, indexed_doc_count_); + return static_cast(sum_total_term_freq_) / static_cast(denom); +} + +Status SniiStatsProvider::doc_freq(std::string_view term, uint64_t* df) const { + if (df == nullptr) + return Status::Error("stats_provider: null df"); + *df = 0; + bool found = false; + DictEntry entry; + RETURN_IF_ERROR(lookup_entry(*idx_, term, &found, &entry)); + if (found) *df = entry.df; + return Status::OK(); +} + +Status SniiStatsProvider::total_term_freq(std::string_view term, uint64_t* ttf) const { + if (ttf == nullptr) + return Status::Error("stats_provider: null ttf"); + *ttf = 0; + bool found = false; + DictEntry entry; + RETURN_IF_ERROR(lookup_entry(*idx_, term, &found, &entry)); + if (!found) return Status::OK(); + // tier>=T2 entries carry the total term frequency directly in ttf_delta (the + // LogicalIndexWriter stores ttf there, not a delta from df). G16-f blocks + // (kNoTermStats: freq-dropped index) omit it -- fail with the semantic + // error instead of silently returning the default 0 (mixed old/new + // segments would otherwise disagree on the same term). + if (!entry.term_stats_present) { + return Status::Error( + "snii_stats: ttf requested but the dict block carries no term stats " + "(freq-dropped index)"); + } + *ttf = entry.ttf_delta; + return Status::OK(); +} + +Status SniiStatsProvider::encoded_norm(uint32_t docid, uint8_t* out) const { + if (out == nullptr) + return Status::Error("stats_provider: null out"); + if (!has_norms_) { + return Status::Error( + "stats_provider: index has no norms"); + } + return norms_reader_.try_encoded_norm(docid, out); +} + +} // namespace doris::snii::stats diff --git a/be/src/storage/index/snii/stats/snii_stats_provider.h b/be/src/storage/index/snii/stats/snii_stats_provider.h new file mode 100644 index 00000000000000..284a6ecc07cb71 --- /dev/null +++ b/be/src/storage/index/snii/stats/snii_stats_provider.h @@ -0,0 +1,92 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include "common/status.h" +#include "storage/index/snii/format/norms_pod.h" +#include "storage/index/snii/reader/logical_index_reader.h" + +// SniiStatsProvider -- exposes the native SNII scoring statistics required by +// BM25, sourced directly from the on-disk structures of one logical index: +// - semantic segment-level counts from CommonGrams scoring metadata. +// - per-term df / ttf from the term's DictEntry (resolved through the reader's +// lookup flow). The LogicalIndexWriter stores ttf directly in ttf_delta for +// tier>=T2 entries, so total_term_freq returns entry.ttf_delta. +// - per-doc length normalization byte (encoded_norm) from the norms POD, +// lazily loaded and validated once by LogicalIndexReader, then shared by +// every stats provider for that cached logical index. +// +// avgdl() = sum_total_term_freq / max(1, indexed_doc_count): the average document +// length used by BM25 length normalization. The provider performs no scoring; it +// only surfaces the statistics so query::Bm25Scorer can combine them. +namespace doris::snii::stats { + +class SniiStatsProvider { +public: + SniiStatsProvider() = default; + + // Binds to idx and acquires its shared validated norms view when the index + // carries scoring norms. idx must outlive this provider. Complete CommonGrams + // scoring metadata requires compatible semantic statistics and norms. + static Status open(const reader::LogicalIndexReader* idx, SniiStatsProvider* out); + +#ifdef BE_TEST + // Legacy scorer fixtures predate semantic scoring metadata. Production + // callers must use open() and fail closed when the proof is absent. + static Status open_legacy_for_test(const reader::LogicalIndexReader* idx, + SniiStatsProvider* out); +#endif + + // Segment-level semantic counts persisted by the SNII writer. + uint64_t doc_count() const { return doc_count_; } + uint64_t indexed_doc_count() const { return indexed_doc_count_; } + uint64_t sum_total_term_freq() const { return sum_total_term_freq_; } + + // Average document length: sum_total_term_freq / max(1, indexed_doc_count). + double avgdl() const; + + // Per-term document frequency. Absent term -> *df = 0 (OK status). + Status doc_freq(std::string_view term, uint64_t* df) const; + + // Per-term total term frequency (ttf = df + ttf_delta at tier>=T2). Absent + // term -> *ttf = 0 (OK status). + Status total_term_freq(std::string_view term, uint64_t* ttf) const; + + // 1-byte encoded doc-length norm for docid (raw byte from the norms POD). + // Out-of-range docid -> InvalidArgument; index without norms -> InvalidArgument. + Status encoded_norm(uint32_t docid, uint8_t* out) const; + + bool has_norms() const { return has_norms_; } + +private: + static Status open_impl(const reader::LogicalIndexReader* idx, SniiStatsProvider* out, + bool require_semantic_metadata); + + const reader::LogicalIndexReader* idx_ = nullptr; + uint64_t doc_count_ = 0; + uint64_t indexed_doc_count_ = 0; + uint64_t sum_total_term_freq_ = 0; + bool has_norms_ = false; + // Zero-copy view into the immutable bytes owned by idx_ after open_norms(). + format::NormsPodReader norms_reader_; +}; + +} // namespace doris::snii::stats diff --git a/be/src/storage/index/snii/writer/compact_posting_pool.cpp b/be/src/storage/index/snii/writer/compact_posting_pool.cpp new file mode 100644 index 00000000000000..9cef1a38b9f572 --- /dev/null +++ b/be/src/storage/index/snii/writer/compact_posting_pool.cpp @@ -0,0 +1,120 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/writer/compact_posting_pool.h" + +#include +#include +#include + +namespace doris::snii::writer { + +// Gentle (~1.5x) many-level payload-capacity schedule. Starting at 5 bytes with a +// slow ramp keeps the over-allocated FINAL slice small for the millions of low-df +// terms (the dominant arena-overhead source) while still reaching multi-KiB slices +// for high-df chains in a bounded number of hops (so the per-slice 4-byte forward +// pointer stays a small fraction of a large chain's bytes). +const uint32_t CompactPostingPool::kSliceSizes[kLevelCount] = { + 5, 8, 12, 18, 27, 40, 60, 90, 135, 202, 303, 455, 683, 1024, 1536, 2304}; +const uint8_t CompactPostingPool::kNextLevel[kLevelCount] = {1, 2, 3, 4, 5, 6, 7, 8, + 9, 10, 11, 12, 13, 14, 15, 15}; + +CompactPostingPool::CompactPostingPool() = default; + +uint32_t CompactPostingPool::kSliceSizes_level0() { + return kSliceSizes[0]; +} + +uint32_t CompactPostingPool::kSliceSize_at(int level) { + return kSliceSizes[level]; +} + +uint8_t CompactPostingPool::kNextLevel_at(int level) { + return kNextLevel[level]; +} + +void CompactPostingPool::reset() { + std::vector>().swap(blocks_); + next_offset_ = 0; + payload_bytes_ = 0; +} + +uint32_t CompactPostingPool::alloc_run(uint32_t bytes) { + const uint32_t in_block = next_offset_ & kBlockMask; + // A fresh block is needed when (a) there is no tail block yet, (b) the run does + // not fit in the current tail block's remaining space, or (c) next_offset_ sits + // exactly on a block boundary whose block has not been allocated (a previous run + // that exactly filled the tail leaves next_offset_ == blocks_.size()*kBlockSize, + // so in_block == 0 must NOT be mistaken for an empty fresh block). + const bool tail_exists = (next_offset_ >> kBlockShift) < blocks_.size(); + const bool need_block = !tail_exists || in_block + bytes > kBlockSize; + // Hard invariant (see arena_bytes()): the uint32 offset must never wrap. The spimi + // accumulator force-spills below 4 GiB, but enforce it here too -- in release as + // well as debug -- so any direct user of the pool fails loudly instead of silently + // aliasing block 0. We are a library: throw and let the caller decide how to + // handle it, rather than aborting the process. The run starts either in the + // current tail or at a new block's base; compute that start in 64 bits before the + // uint32 arithmetic can wrap. + const uint64_t run_start = + need_block ? static_cast(blocks_.size()) * kBlockSize : next_offset_; + if (run_start + bytes > UINT32_MAX) { + throw std::overflow_error( + "snii: CompactPostingPool arena exceeded the 4 GiB uint32 offset limit; " + "the caller must spill before this point"); + } + if (need_block) { + blocks_.emplace_back(kBlockSize, 0); + next_offset_ = static_cast((blocks_.size() - 1) * kBlockSize); + } + const uint32_t off = next_offset_; + next_offset_ += bytes; + return off; +} + +uint32_t CompactPostingPool::alloc_slice(int level, uint32_t* slice_end) { + const uint32_t cap = kSliceSizes[level]; + const uint32_t first = alloc_run(cap + kPtrBytes); + *slice_end = first + cap; + // Zero the forward pointer so a not-yet-extended tail slice reads next_head == 0. + std::memset(at(*slice_end), 0, kPtrBytes); + return first; +} + +uint32_t CompactPostingPool::read_ptr(uint32_t slice_end) const { + uint32_t v; + std::memcpy(&v, at(slice_end), sizeof(v)); + return v; +} + +void CompactPostingPool::write_ptr(uint32_t slice_end, uint32_t next_head) { + std::memcpy(at(slice_end), &next_head, sizeof(next_head)); +} + +uint32_t CompactPostingPool::start_chain(SliceWriter* w, uint8_t* level) { + *level = 0; + const uint32_t head = alloc_slice(0, &w->slice_end); + w->cur = head; + return head; +} + +CompactPostingPool::Cursor::Cursor(const CompactPostingPool* pool, uint32_t head, uint64_t budget) + : pool_(pool), cur_(head), level_(0), budget_(budget) { + // The first slice is level 0; its payload region ends kSliceSizes[0] bytes in. + slice_end_ = head + CompactPostingPool::kSliceSizes[0]; +} + +} // namespace doris::snii::writer diff --git a/be/src/storage/index/snii/writer/compact_posting_pool.h b/be/src/storage/index/snii/writer/compact_posting_pool.h new file mode 100644 index 00000000000000..38adf4363b6528 --- /dev/null +++ b/be/src/storage/index/snii/writer/compact_posting_pool.h @@ -0,0 +1,333 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace doris::snii::writer { + +// SEGMENTED BYTE ARENA with per-term SLICED runs (a ByteBlockPool, after Lucene). +// +// WHY: the SPIMI accumulator's bulk memory is the per-term posting bytes. Backing +// each term with its own std::vector pays two taxes that dominate peak +// RSS at scale: (1) geometric-growth doubling slack (~1.17x of the live payload), +// and (2) a 24-32 B vector/struct header per term (hundreds of thousands of +// terms). This pool removes both: all term bytes live in a few large fixed-size +// blocks (so slack is ~one block, amortized to ~1.05x), and a term needs only two +// 32-bit cursors of live state (chain head for reads + write head for appends). +// +// HOW (slices): a term's bytes are not stored contiguously. They live in a chain +// of SLICES of geometrically growing payload capacity (the kSliceSizes schedule: +// 4, 8, 16, ... bytes of payload). Each slice is laid out as +// [ payload bytes ... ][ 4-byte forward pointer ] +// The forward pointer holds the absolute offset of the next slice's first payload +// byte (0 while the slice is still the tail of the chain). When a slice's payload +// region fills, the writer allocates a larger slice, stores its head into the old +// slice's 4 pointer bytes, and keeps appending. A reader walks the chain by +// reading payload bytes until a slice boundary, then following the pointer. +// +// Both writer and reader recompute each slice's capacity from the chain's slice +// INDEX (0, 1, 2, ...) via the deterministic schedule, so neither needs to store +// per-slice sizes. The writer carries the current slice's end offset in its +// SliceWriter handle; the reader recomputes capacities as it advances. +// +// Offsets are GLOBAL absolute byte indices into the logical concatenation of all +// blocks: offset = block_index * kBlockSize + byte_in_block. kBlockSize is a power +// of two, so offset -> (block, byte) is a shift/mask. +class CompactPostingPool { +public: + // Block size (power of two). 32 KiB blocks keep per-block tail waste tiny (it + // matters at the smaller 1M scale where the whole arena is only tens of MiB) and + // bound the outer vector header cost; at the 5M scale a few thousand + // blocks is still cheap. Empirically the lowest peak across both scales. + static constexpr uint32_t kBlockShift = 15; + static constexpr uint32_t kBlockSize = 1u << kBlockShift; // 32 KiB + static constexpr uint32_t kBlockMask = kBlockSize - 1; + + // Per-slice forward-pointer width (absolute uint32 next-slice offset). + static constexpr uint32_t kPtrBytes = 4; + + // Geometric slice payload-capacity schedule and the level transition. Level i + // slices hold kSliceSizes[i] payload bytes; on overflow the chain advances to + // kNextLevel[i] (capping at the largest level). A GENTLE (~1.5x) many-level + // schedule starting small minimizes the over-allocated final slice (the + // dominant arena overhead) while keeping the per-slice forward-pointer count + // bounded for high-df chains. + static constexpr int kLevelCount = 16; + + CompactPostingPool(); + + CompactPostingPool(const CompactPostingPool&) = delete; + CompactPostingPool& operator=(const CompactPostingPool&) = delete; + + // Payload capacity (bytes) of a fresh level-0 slice. Exposed for tests that need + // to fill exactly one slice without hardcoding the schedule. + static uint32_t kSliceSizes_level0(); + + // Payload capacity of the slice at `level`, and the level a chain advances to when + // that slice overflows. Exposed (like kSliceSizes_level0) so tests can simulate the + // arena's bump allocator exactly -- e.g. to construct an EXACT block-boundary fill -- + // without hardcoding the private schedule. `level` must be in [0, kLevelCount). + static uint32_t kSliceSize_at(int level); + static uint8_t kNextLevel_at(int level); + + // Live append handle for one term's chain. POD, 8 bytes: the absolute write + // cursor and the absolute end of the current slice's payload region. The chain's + // current slice LEVEL is kept by the caller (a uint8, packed alongside its other + // flags) so this handle stays 8 bytes -- shaving the per-term accumulator. `head` + // (the chain's first payload offset) is also stored by the CALLER (the read entry + // point); start_chain returns it. + struct SliceWriter { + uint32_t cur = 0; // next byte to write (absolute) + uint32_t slice_end = 0; // one-past-last payload byte of the current slice + }; + + // Begins a fresh chain, initializing `w` to its first (level-0) slice and + // *level to 0, and returns the chain head (absolute first payload offset). + uint32_t start_chain(SliceWriter* w, uint8_t* level); + + // Appends one payload byte to the chain described by `w` / `*level`, growing the + // chain with a new linked slice (and advancing *level) when the current slice's + // payload region is exhausted. + void append_byte(SliceWriter* w, uint8_t* level, uint8_t value); + // Encodes and appends one unsigned LEB128 value. The common case copies all + // encoded bytes into the current slice in one operation; only a slice-boundary + // value enters the cold allocation loop. + void append_varint(SliceWriter* w, uint8_t* level, uint64_t value); + + // Total live payload bytes ever written across all chains (excludes slice + // forward-pointer overhead). Drives the spill-threshold estimate only. + uint64_t payload_bytes() const { return payload_bytes_; } + + // Bytes the arena currently occupies (block_count * kBlockSize). The pool + // addresses bytes with a uint32 offset (next_offset_), so the arena MUST stay + // below 4 GiB or alloc_run wraps and silently aliases block 0. The accumulator + // watches this to force a safety spill before the wrap; alloc_run also enforces it + // directly (throws std::overflow_error on a would-be wrap) so a direct user of the + // pool fails loudly rather than silently corrupting. + // Hard invariant: a single CompactPostingPool never exceeds UINT32_MAX bytes. + uint64_t arena_bytes() const { return static_cast(blocks_.size()) << kBlockShift; } + + // Releases ALL blocks back to the OS. Called after the accumulator is fully + // drained (or before a spill's next fill) so no input-side bytes stay resident. + void reset(); + + // ---- Reader ---------------------------------------------------------------- + // Forward cursor over one term's chain, yielding its payload bytes in write + // order by walking the slice forward pointers. + // + // CONTRACT of the `budget` ctor argument (single, unambiguous meaning): + // `budget` is an UPPER BOUND on the number of bytes this cursor may yield. It + // is NOT required to equal the exact payload length: passing the exact length + // is fine, and so is passing any value >= it (the production caller passes the + // chain's write-head offset, which always bounds the payload from above). The + // cursor is SELF-TERMINATING: once it walks off the last written byte it sees + // the tail slice's zero forward pointer and stops, regardless of how much + // budget remains. So an over-large budget can never make next() read past the + // chain (no aliasing of block 0, no off-chain access) -- the budget is purely a + // secondary cap. has_next() is therefore a reliable "more bytes remain" + // predicate for ANY budget >= the true length: it becomes false at the smaller + // of (budget exhausted, chain tail reached). + class Cursor { + public: + Cursor(const CompactPostingPool* pool, uint32_t head, uint64_t budget); + + // True while the cursor can still yield a REAL payload byte: the budget is not + // spent AND the cursor has not reached the chain tail. It peeks the tail forward + // pointer at a slice boundary so it never reports a phantom trailing byte, making + // has_next()/next() a safe loop for any budget >= the true payload length. + bool has_next() const; + // Yields the next payload byte. Returns 0 (and yields no more) once the chain + // tail is reached or the budget is spent -- never reads past the chain. + uint8_t next(); + // Decodes one unsigned LEB128 value directly from the current slice, following + // a slice pointer only when the encoded value straddles a boundary. + uint64_t read_varint(); + + private: + const CompactPostingPool* pool_; + uint32_t cur_; // absolute read cursor + uint32_t slice_end_; // one-past-last payload byte of the current slice + uint32_t level_; // current slice level + uint64_t budget_; // remaining byte budget (upper bound on bytes to yield) + }; + + // Builds a cursor over the chain at `head`. `budget` is an UPPER BOUND on bytes to + // read (see Cursor's contract): the exact payload length or anything larger. The + // production caller passes the write-head offset, which always bounds the payload + // from above; the cursor self-terminates at the chain tail regardless. + Cursor cursor(uint32_t head, uint64_t budget) const { return Cursor(this, head, budget); } + +private: + static const uint32_t kSliceSizes[kLevelCount]; + static const uint8_t kNextLevel[kLevelCount]; + + uint8_t* at(uint32_t off) { return &blocks_[off >> kBlockShift][off & kBlockMask]; } + const uint8_t* at(uint32_t off) const { return &blocks_[off >> kBlockShift][off & kBlockMask]; } + + // Reads/writes the 4-byte forward pointer at the END of a slice whose payload + // region ends at `slice_end` (pointer occupies [slice_end, slice_end+4)). + uint32_t read_ptr(uint32_t slice_end) const; + void write_ptr(uint32_t slice_end, uint32_t next_head); + + // Reserves `bytes` contiguous bytes from the arena tail (a fresh block if the + // current tail cannot hold them) and returns the first reserved absolute offset. + // `bytes` must be <= kBlockSize. + uint32_t alloc_run(uint32_t bytes); + + // Allocates a slice at `level` (payload region + 4 pointer bytes), zeroes its + // forward pointer, and returns the first payload offset; sets *slice_end. + uint32_t alloc_slice(int level, uint32_t* slice_end); + + std::vector> blocks_; // fixed kBlockSize blocks + uint32_t next_offset_ = 0; // global bump pointer (absolute) into the tail block + uint64_t payload_bytes_ = 0; +}; + +// ---- Inlined per-byte hot paths -------------------------------------------- +// append_byte (one call per encoded payload byte during SPIMI ingest) and +// Cursor::has_next/next (one call per arena byte during finalize drain/merge) +// are the writer's two hottest per-byte loops. Their callers live in a DIFFERENT +// translation unit (spimi_term_buffer.cpp), and the build has no LTO/IPO/unity +// build, so an out-of-line .cpp definition forces a non-inlinable cross-TU call +// (plus stack spill of cur_/slice_end_/budget_) on every single byte. Defining +// the bodies inline HERE lets the caller's TU inline them, keeping the cursor +// state in registers and eliminating the per-byte call/ret. They are placed +// AFTER the class so every private member they touch (at, read_ptr, alloc_slice, +// write_ptr, kSliceSizes, kNextLevel, payload_bytes_) is already declared and +// visible; Cursor is a nested class, so it may use the enclosing pool's privates +// (read_ptr/at) through pool_. The cold slice-overflow helpers (alloc_slice, +// write_ptr) stay out-of-line in the .cpp -- only the per-byte work is inlined. +// Pure code move from the .cpp: behavior and produced bytes are identical. + +inline void CompactPostingPool::append_byte(SliceWriter* w, uint8_t* level, uint8_t value) { + if (w->cur == w->slice_end) { + // Current slice payload region is full: grow the chain with a larger slice and + // record the link in the old slice's trailing pointer bytes. + const uint8_t next_level = kNextLevel[*level]; + uint32_t new_end = 0; + const uint32_t new_head = alloc_slice(next_level, &new_end); + write_ptr(w->slice_end, new_head); + *level = next_level; + w->cur = new_head; + w->slice_end = new_end; + } + *at(w->cur) = value; + ++w->cur; + ++payload_bytes_; +} + +inline void CompactPostingPool::append_varint(SliceWriter* w, uint8_t* level, uint64_t value) { + std::array encoded {}; + size_t size = 0; + do { + uint8_t byte = static_cast(value & 0x7fU); + value >>= 7; + if (value != 0) { + byte |= 0x80U; + } + encoded[size++] = byte; + } while (value != 0); + + size_t copied = 0; + while (copied < size) { + if (w->cur == w->slice_end) { + const uint8_t next_level = kNextLevel[*level]; + uint32_t new_end = 0; + const uint32_t new_head = alloc_slice(next_level, &new_end); + write_ptr(w->slice_end, new_head); + *level = next_level; + w->cur = new_head; + w->slice_end = new_end; + } + const size_t available = w->slice_end - w->cur; + const size_t count = std::min(size - copied, available); + std::memcpy(at(w->cur), encoded.data() + copied, count); + w->cur += static_cast(count); + copied += count; + payload_bytes_ += count; + } +} + +inline bool CompactPostingPool::Cursor::has_next() const { + if (budget_ == 0) { + return false; + } + // At a slice boundary, the chain continues only if the forward pointer is non-zero; + // a zero pointer is the tail marker (offset 0 is never a valid next-slice head). Peek + // it so has_next() never reports a phantom byte that next() would have to fabricate. + if (cur_ == slice_end_) { + return pool_->read_ptr(slice_end_) != 0; + } + return true; +} + +inline uint8_t CompactPostingPool::Cursor::next() { + // Budget guard: the caller's stated upper bound is spent -- yield nothing more. + if (budget_ == 0) { + return 0; + } + if (cur_ == slice_end_) { + // Reached this slice's payload boundary. Follow the forward pointer to the next + // slice -- UNLESS it is zero, which marks the CHAIN TAIL (offset 0 is always the + // pool's very first slice, never a valid *next*-slice head, so a zero pointer is + // unambiguously "no more slices"). Without this tail check, an over-reading caller + // would follow the zero pointer to offset 0 and alias block 0's bytes (or read an + // unallocated block) -- UB. Stopping here makes the cursor self-terminating and + // safe regardless of how large a budget the caller passed. + const uint32_t next_head = pool_->read_ptr(slice_end_); + if (next_head == 0) { + budget_ = 0; // chain exhausted: no further bytes exist + return 0; + } + level_ = CompactPostingPool::kNextLevel[level_]; + cur_ = next_head; + slice_end_ = next_head + CompactPostingPool::kSliceSizes[level_]; + } + const uint8_t v = *pool_->at(cur_); + ++cur_; + --budget_; + return v; +} + +inline uint64_t CompactPostingPool::Cursor::read_varint() { + uint64_t result = 0; + uint32_t shift = 0; + for (;;) { + uint8_t byte; + if (budget_ != 0 && cur_ != slice_end_) { + byte = *pool_->at(cur_); + ++cur_; + --budget_; + } else { + byte = next(); + } + result |= static_cast(byte & 0x7fU) << shift; + if ((byte & 0x80U) == 0) { + return result; + } + shift += 7; + } +} + +} // namespace doris::snii::writer diff --git a/be/src/storage/index/snii/writer/global_memory_limiter.cpp b/be/src/storage/index/snii/writer/global_memory_limiter.cpp new file mode 100644 index 00000000000000..360aeb4834b38c --- /dev/null +++ b/be/src/storage/index/snii/writer/global_memory_limiter.cpp @@ -0,0 +1,296 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/writer/global_memory_limiter.h" + +#include +#include +#include +#include + +#include "common/config.h" +#include "common/logging.h" +#include "runtime/memory/global_memory_arbitrator.h" +#include "runtime/memory/mem_tracker.h" +#include "storage/index/snii/writer/snii_build_memory_tracker.h" +#include "util/mem_info.h" + +namespace doris::snii::writer { + +int64_t BuildMemorySignals::overage() const { + const int64_t over_share = build_share_bytes > 0 ? build_consumption - build_share_bytes : 0; + // The maximum, NOT the sum -- and this is a correctness requirement, not a + // stylistic one. The terms overlap: SNII's own bytes are inside + // build_consumption AND inside the process_memory_usage() that produces + // process_above_soft_mem_limit. Summing would count them twice and demand + // reclaiming memory that does not exist. Each term is the bytes needed to + // clear ONE pressure source; reclaiming the deepest clears the shallower. + return std::max( + {over_share, sys_avail_below_warning_water_mark, process_above_soft_mem_limit, 0}); +} + +int64_t calc_process_max_snii_build_memory(int64_t process_mem_limit) { + // <= 0 is Doris's "no limit" convention (-1), and INT64_MAX appears on + // unconfigured/unlimited cgroups: neither yields a meaningful share, and + // multiplying either would overflow. + if (process_mem_limit <= 0 || process_mem_limit == std::numeric_limits::max()) { + return -1; + } + const int32_t percent = std::clamp(config::snii_index_build_max_memory_limit_percent, 0, 100); + if (percent <= 0) { + return 0; // share trigger explicitly disabled + } + // Multiply first where it is provably safe (percent <= 100, so the product + // fits whenever the limit is under INT64_MAX/100 -- true of any real BE), + // which keeps the share exact. Above that, divide first to stay in range; + // the sub-100-byte rounding is irrelevant at that scale. + const int64_t share = process_mem_limit <= std::numeric_limits::max() / 100 + ? process_mem_limit * percent / 100 + : process_mem_limit / 100 * percent; + // FLOOR AGAINST THE PER-WRITER CAP: inverted_index_ram_buffer_size (512 MiB + // by default) is what ONE writer may hold before it spills on its own. A + // share below a few writers' worth of that would put small BEs permanently + // over the share the moment two writers exist -- back-pressure that can + // never be relieved rather than a limit. Four writers' worth is the + // smallest share at which the mechanism can express "some writers are fine, + // this one is not". + const auto per_writer_cap = + static_cast(config::inverted_index_ram_buffer_size * 1024 * 1024); + return std::max(share, 4 * per_writer_cap); +} + +BuildMemorySignals read_build_memory_signals() { + BuildMemorySignals signals; + // The RECLAIMABLE population only. Index-merge compaction charges the same + // observation tracker but registers no writer and holds no posting arena, + // so including it would charge ingestion writers for memory they do not + // hold -- picking the wrong victim, or demanding an overage no arena can + // cover. Its own kHardLimit reservation policy bounds it instead. + // + // ONE atomic load, and no arithmetic across populations. The registered + // bytes are maintained directly for exactly this reason: deriving them as + // (tracker - unregistered) would sample two independently-updated atomics + // that cannot be read as one snapshot, and a read landing mid-update would + // both invent overages and hide real ones. See snii_registered_build_bytes. + signals.build_consumption = snii_registered_build_bytes(); + signals.build_share_bytes = calc_process_max_snii_build_memory(MemInfo::mem_limit()); + // The byte-valued form of the two conditions is_exceed_soft_mem_limit() + // tests, computed directly (as MemTableMemoryLimiter does) because the + // decision needs HOW MANY bytes are missing, not just whether. Reading them + // directly also keeps this off the logging side effect that the boolean + // helper performs, which would fire on every writer report. + signals.sys_avail_below_warning_water_mark = MemInfo::sys_mem_available_warning_water_mark() - + GlobalMemoryArbitrator::sys_mem_available(); + signals.process_above_soft_mem_limit = + GlobalMemoryArbitrator::process_memory_usage() - MemInfo::soft_mem_limit(); + return signals; +} + +namespace { +// Names the term that produced overage(), so the shortfall warning cannot +// misattribute process-wide pressure to SNII's own persistent structures. +const char* deepest_pressure_source(const BuildMemorySignals& signals) { + const int64_t over_share = signals.build_share_bytes > 0 + ? signals.build_consumption - signals.build_share_bytes + : 0; + if (over_share >= signals.sys_avail_below_warning_water_mark && + over_share >= signals.process_above_soft_mem_limit) { + return "SNII index-build is over its own share"; + } + if (signals.sys_avail_below_warning_water_mark >= signals.process_above_soft_mem_limit) { + return "system available memory is below its warning water mark (not SNII's own memory)"; + } + return "process memory usage is above the soft limit (not SNII's own memory)"; +} +} // namespace + +GlobalMemoryLimiter::GlobalMemoryLimiter() : signals_(&read_build_memory_signals) {} + +GlobalMemoryLimiter* GlobalMemoryLimiter::instance() { + // Intentionally leaked (never destroyed): buffers un-register from their + // destructors, which may run during static teardown at process exit -- a + // destroyed registry there would be use-after-free. A leaked singleton + // makes the "limiter outlives every attached buffer" contract + // unconditional. + static auto* g_instance = [] { + auto* limiter = new GlobalMemoryLimiter(); +#ifdef BE_TEST + // TEST ISOLATION: registration is unconditional, so every unit test + // that builds a SniiIndexColumnWriter joins THIS singleton. With the + // production reader that would import the HOST's memory state -- an + // ASAN container already above the process soft limit would fire forced + // spills inside tests that never asked for one, including the + // golden-bytes tests. Default the singleton to "no pressure"; a test + // that wants the real reader installs it explicitly. + limiter->set_signals_provider([] { return BuildMemorySignals {}; }); +#endif + return limiter; + }(); + return g_instance; +} + +void GlobalMemoryLimiter::set_signals_provider(SignalsFn signals) { + std::lock_guard guard(mutex_); + signals_ = std::move(signals); +} + +void GlobalMemoryLimiter::register_buffer(std::atomic* spill_flag, int64_t arena_bytes) { + std::lock_guard guard(mutex_); + entries_[spill_flag] = arena_bytes; +} + +void GlobalMemoryLimiter::report(std::atomic* spill_flag, int64_t arena_bytes) { + std::lock_guard guard(mutex_); + auto it = entries_.find(spill_flag); + if (it == entries_.end()) { + // Not registered (or already unregistered): ignore rather than + // resurrect an entry nobody will remove. + return; + } + it->second = arena_bytes; + request_spills_locked(); +} + +void GlobalMemoryLimiter::unregister_buffer(std::atomic* spill_flag) { + std::lock_guard guard(mutex_); + entries_.erase(spill_flag); +} + +size_t GlobalMemoryLimiter::registered_count() const { + std::lock_guard guard(mutex_); + return entries_.size(); +} + +bool GlobalMemoryLimiter::reclaim_shortfall() const { + std::lock_guard guard(mutex_); + return reclaim_shortfall_; +} + +int64_t GlobalMemoryLimiter::eligible_arena_bytes() const { + std::lock_guard guard(mutex_); + const int64_t victim_floor = + std::max(min_victim_arena_bytes_.load(std::memory_order_relaxed), 1); + int64_t total = 0; + for (const auto& [flag, arena] : entries_) { + if (arena >= victim_floor) { + total += arena; + } + } + return total; +} + +void GlobalMemoryLimiter::request_spills_locked() { + const BuildMemorySignals signals = signals_(); + const int64_t overage = signals.overage(); + if (overage <= 0) { + // Pressure gone is the ordinary way an episode ends -- writers drain, a query + // finishes, RSS falls back. Clearing here is what keeps reclaim_shortfall() a + // statement about the CURRENT arena, and what lets the next episode log again: + // the recovery branch below only runs while an overage still exists, so without + // this the latch would survive for the life of the process and silence every + // later episode. + if (reclaim_shortfall_) { + reclaim_shortfall_ = false; + LOG(INFO) << "SNII index-build memory pressure cleared; reclaim shortfall episode " + "over."; + } + return; + } + // FORCED-SPILL FLOOR + PER-BUFFER COOLDOWN: only buffers holding at least + // the floor of RECLAIMABLE arena are eligible victims -- flagging a + // smaller arena would cut a tiny run and reclaim next to nothing, and a + // buffer that just honored a forced spill (arena ~0) stays exempt until + // its arena regrows past the floor. Never below one byte: an empty arena + // has nothing to write to a run. + const int64_t victim_floor = + std::max(min_victim_arena_bytes_.load(std::memory_order_relaxed), 1); + // Largest RECLAIMABLE consumers first: victims are ranked by their + // spillable ARENA (what the forced spill frees), not by any + // persistent-dominated resident total. n is the live writer count of the + // process (at most a few hundred), and this only runs while over the + // target, so the sort under the mutex is bounded, allocation-light work. + std::vector*>> by_arena; + by_arena.reserve(entries_.size()); + int64_t reclaimable = 0; + for (const auto& [flag, arena] : entries_) { + if (arena >= victim_floor) { + by_arena.emplace_back(arena, flag); + reclaimable += arena; + } + } + // SHORTFALL IS NOT A REASON TO DO NOTHING. The overage may be larger than + // everything SNII could free -- most obviously when it comes from the + // process-level terms, which measure memory SNII does not hold at all. + // Refusing to flag in that state would make SNII least willing to give + // memory back exactly when the system needs it most, and it is a control + // loop that cannot recover: nothing is reclaimed, consumption keeps + // growing, the ratio only worsens. So flag BEST EFFORT and record the + // shortfall for observability. + // + // What actually prevents the conc=16 storm is the victim FLOOR, not any + // reachability judgement: every eligible victim holds >= the floor of + // arena, so every forced run is at least floor-sized; after honoring, the + // victim's arena is ~0 and the cooldown keeps it ineligible until it + // regrows a full floor; and the run-file cap merge-compacts what survives. + // Flagging is therefore bounded to one >= floor-sized run per floor of + // arena growth per buffer -- the intended back-pressure. + const bool shortfall = reclaimable < overage; + if (shortfall && !reclaim_shortfall_) { + reclaim_shortfall_ = true; + LOG(WARNING) << "SNII index-build cannot fully relieve memory pressure: reclaimable " + << "posting arena across " << by_arena.size() << " eligible writers (" + << reclaimable << " B of " << entries_.size() << " registered) is short of " + << "the " << overage << " B overage; spilling every eligible arena anyway. " + << "Deepest pressure source: " << deepest_pressure_source(signals) + << " (over_share=" + << (signals.build_share_bytes > 0 + ? signals.build_consumption - signals.build_share_bytes + : 0) + << " B, sys_avail_below_water_mark=" + << signals.sys_avail_below_warning_water_mark + << " B, process_above_soft_limit=" << signals.process_above_soft_mem_limit + << " B; reclaimable build_consumption=" << signals.build_consumption + << " B, share=" << signals.build_share_bytes << " B)."; + } else if (!shortfall && reclaim_shortfall_) { + reclaim_shortfall_ = false; // episode over; a relapse will log once again + LOG(INFO) << "SNII index-build can cover the current overage again: " << reclaimable + << " B of reclaimable arena vs a " << overage << " B overage."; + } + std::sort(by_arena.begin(), by_arena.end(), + [](const auto& a, const auto& b) { return a.first > b.first; }); + int64_t covered = 0; + for (const auto& [arena, flag] : by_arena) { + if (covered >= overage) { + break; + } + // An ALREADY-pending flag (set by an earlier report, owner not yet at + // its next token) counts toward the covered sum without a fresh store: + // re-flagging it would be a no-op, and skipping the store avoids + // dirtying the owner's cache line every over-share report. + if (!flag->load(std::memory_order_relaxed)) { + flag->store(true, std::memory_order_relaxed); + } + // Count the ARENA toward coverage: it is all a forced spill of this + // victim can actually reclaim. When the eligible arenas fall short the + // loop simply exhausts them -- every flagged victim still cuts a + // >= floor-sized run, and the cooldown keeps any one buffer from being + // re-victimized before it has a floor's worth of arena again. + covered += arena; + } +} + +} // namespace doris::snii::writer diff --git a/be/src/storage/index/snii/writer/global_memory_limiter.h b/be/src/storage/index/snii/writer/global_memory_limiter.h new file mode 100644 index 00000000000000..003aba82731f06 --- /dev/null +++ b/be/src/storage/index/snii/writer/global_memory_limiter.h @@ -0,0 +1,229 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include +#include +#include +#include +#include + +namespace doris::snii::writer { + +// Byte-valued memory-pressure signals the forced-spill decision is judged +// against. Production fills these from Doris's own global memory state; unit +// tests inject deterministic values. +// +// The three pressure sources mirror MemTableMemoryLimiter's hard-limit test: +// the subsystem's own share first, then the two process-level backstops. The +// ORDER matters -- SNII's own share is set well below the backstops so SNII +// sheds its own memory before the global valve, which trips late by design. +struct BuildMemorySignals { + // Live SNII index-build bytes -- the SniiIndexBuild observation tracker's + // consumption, covering ingestion writers and index-merge compaction alike. + int64_t build_consumption = 0; + // SNII's share of the process memory limit. <= 0 disables the share + // trigger; the backstops below still apply. + int64_t build_share_bytes = 0; + // Bytes by which system available memory sits BELOW its warning water mark + // (<= 0 when the system is not short). + int64_t sys_avail_below_warning_water_mark = 0; + // Bytes by which process memory usage sits ABOVE the process soft limit + // (<= 0 when the process is not over it). + int64_t process_above_soft_mem_limit = 0; + + // Bytes that must be reclaimed to clear every active pressure source; 0 + // when nothing is under pressure. The maximum, not the sum: satisfying the + // deepest deficit satisfies the others. + int64_t overage() const; +}; + +// SNII's index-build share of `process_mem_limit`, from the mutable percent +// config (snii_index_build_max_memory_limit_percent). Returns 0 when the share +// trigger is disabled, and -1 for an unlimited process (mirroring +// MemInfo::mem_limit()'s -1 convention), which also disables the share trigger. +int64_t calc_process_max_snii_build_memory(int64_t process_mem_limit); + +// Production signal reader: the SniiIndexBuild tracker, the share above, and +// Doris's system/process memory state. +BuildMemorySignals read_build_memory_signals(); + +// Process-wide SNII build-RAM limiter (G09) -- the index-build analogue of +// Doris's MemTableMemoryLimiter. Every live SPIMI accumulator registers here +// and forwards its SPILLABLE arena bytes through its existing debounced report +// path; when SNII's index-build memory as a whole crosses its share of the +// process limit (or the process itself comes under pressure), the limiter +// requests spills from the largest-ARENA eligible buffers until the flagged +// (reclaimable) arena sum covers the overage. +// +// WHY: the per-writer gate-2 cap (e.g. 512 MiB) bounds ONE writer, but a load +// keeps (tablets x concurrency) writers alive at once -- wikipedia at +// concurrency 16 held 100+ writers at 300-500 MB each (~41 GiB), none of which +// ever reached its own cap, so per-writer spilling never fired. This registry +// bounds the SUM. +// +// WHERE THE SUM COMES FROM: not from the limiter. The bytes are already +// counted, twice over -- Doris's allocation hook charges the thread's +// MemTrackerLimiter, and every MemoryReporter mirrors its live bytes into the +// SniiIndexBuild observation tracker. The limiter reads that tracker instead of +// maintaining a third, narrower sum of its own; the registry exists only for +// what a tracker cannot express: WHICH writer to ask, and how much of its +// memory is actually reclaimable. +// +// ASYNC-SAFE REQUESTS: the SPIMI structures are single-threaded, so the +// limiter must never spill on the reporting thread. A request is a relaxed +// atomic FLAG on the target buffer (SpimiTermBuffer::global_spill_requested_) +// that the OWNER's next add_token / maybe_spill_after_token observes and +// honors on its own thread (bypassing the G08 per-writer anti-churn floor but +// still requiring the FORCED-SPILL FLOOR of reclaimable arena -- see below -- +// so every forced run is worth its fixed costs). Flags are ADVISORY: the +// owner may have just spilled or drained -- the flag is then a (harmless) +// no-op or one extra floor-sized run. The limiter itself only ever takes its +// registry mutex and flips atomics; it never blocks a reporting thread beyond +// that mutex and never calls back into a buffer. +// +// LIFETIME: buffers un-register in their destructor. register / report / +// unregister all serialize on the registry mutex, and flags are only ever set +// UNDER that mutex, so once unregister_buffer returns no thread can touch the +// (about-to-die) flag again. The limiter must outlive every attached buffer +// (trivial for the process singleton; test-local instances are declared before +// the buffers they serve). +// +// SPILLING RECLAIMS ARENA, NOT PERSISTENT MEMORY: a forced spill releases only +// the buffer's posting ARENA; the persistent vocab / pair-map / slot structures +// (~100-500 MB per wikipedia writer) survive it. The share is a back-pressure +// valve over the reclaimable arenas, not a hard cap on resident RSS. Three +// defenses keep an unreachable target from degenerating into a forced-spill +// storm (the conc=16 wikipedia field failure: every report re-flagged every +// buffer, each honoring with one 32 KiB arena block -> thousands of tiny runs +// per buffer -> EMFILE re-opening them for the k-way merge -> failed loads): +// * VICTIMS BY ARENA: victims are selected by their reported SPILLABLE arena +// bytes -- the only bytes a forced spill can actually reclaim -- never by +// a persistent-dominated resident total, and only buffers whose arena is +// at least min_victim_arena_bytes (config snii_forced_spill_min_arena_bytes) +// are eligible. Every forced run is therefore at least floor-sized. +// * PER-BUFFER COOLDOWN: right after a buffer honors a forced spill its +// arena is ~0, below the victim floor, so it is EXEMPT from new flags +// until the arena regrows past the floor. No timer state: the eligibility +// rule IS the cooldown. +// Those two defenses -- and NOT any judgement about whether the overage is +// reachable -- are what bound the work: flagging costs at most one +// >= floor-sized run per floor of arena growth per buffer. The limiter +// therefore always flags BEST EFFORT, even when the eligible arenas fall short +// of the overage. Refusing to flag on a shortfall would make SNII least willing +// to give memory back exactly when the system is most short of it, and the +// overage frequently exceeds anything SNII holds simply because two of its +// three terms measure whole-process pressure. reclaim_shortfall() reports the +// condition (logged once per episode) instead of acting on it. +class GlobalMemoryLimiter { +public: + // Signals provider; swapped out by unit tests. Invoked under the registry + // mutex, so it must be cheap and must not call back into the limiter. + using SignalsFn = std::function; + + // Victim floor default (mirrors config snii_forced_spill_min_arena_bytes): + // a buffer is only ever asked to force-spill once its reclaimable arena + // holds at least this much, so no forced run is smaller than this. + static constexpr int64_t kDefaultMinVictimArenaBytes = 64LL << 20; // 64 MiB + + // Local instances are constructible for unit tests; production code uses + // the process singleton below. + GlobalMemoryLimiter(); + GlobalMemoryLimiter(const GlobalMemoryLimiter&) = delete; + GlobalMemoryLimiter& operator=(const GlobalMemoryLimiter&) = delete; + + // Process singleton (never destroyed before the writers that use it). + static GlobalMemoryLimiter* instance(); + + // Replaces the memory-pressure signals the decision is judged against. + // Production keeps the default (read_build_memory_signals); tests inject + // deterministic values. + void set_signals_provider(SignalsFn signals); + + // Victim-eligibility floor over a buffer's reported SPILLABLE arena bytes + // (see the class comment). Refreshed from the mutable BE config at each + // writer init. Values < 1 behave as 1 (an empty arena is never a victim -- + // there would be nothing to write to the run). + void set_min_victim_arena_bytes(int64_t bytes) { + min_victim_arena_bytes_.store(bytes, std::memory_order_relaxed); + } + int64_t min_victim_arena_bytes() const { + return min_victim_arena_bytes_.load(std::memory_order_relaxed); + } + + // True while the eligible reclaimable arena is short of the overage: every + // eligible victim is being flagged and it still will not be enough. Purely + // observability -- it never gates flagging. + bool reclaim_shortfall() const; + + // Adds `spill_flag` (the owning buffer's advisory request flag; also the + // entry's identity) with its SPILLABLE arena bytes -- what a forced spill + // can reclaim, and the victim selection key. Re-registering an + // already-registered flag just updates its bytes. Never sets flags itself: + // a single registration cannot create NEW pressure worth reacting to + // before the buffer's first report. + void register_buffer(std::atomic* spill_flag, int64_t arena_bytes); + + // Updates the entry's spillable-arena bytes (an ABSOLUTE total, not a delta + // -- self-healing across any missed report) and re-decides. When SNII's + // build memory is over its share (or the process is under pressure), sets + // the request flags of the largest-ARENA eligible entries (arena >= the + // victim floor; see the class comment) -- counting entries whose flag is + // ALREADY pending toward the covered sum, so an in-flight request is not + // amplified -- until the flagged ARENA bytes cover the overage or the + // eligible victims run out. A report for a flag that is not registered is + // ignored. + void report(std::atomic* spill_flag, int64_t arena_bytes); + + // Removes the entry. After this returns, the limiter never touches + // `spill_flag` again -- safe to destroy the owning buffer. + void unregister_buffer(std::atomic* spill_flag); + + // Reclaimable arena summed over the ELIGIBLE entries (arena >= the victim + // floor) -- the exact quantity a round of forced spills could free, and + // the left-hand side of the reachability test. Tests and observability. + int64_t eligible_arena_bytes() const; + size_t registered_count() const; + +private: + // Called with mutex_ held on every report: reads the pressure signals, + // and if there is an overage, sorts the ELIGIBLE entries (arena >= victim + // floor) by ARENA descending and flags from the top until the flagged arena + // sum covers it or the victims run out. O(n log n) over the live writer + // count (at most a few hundred) -- bounded work under the mutex, no I/O, + // no callbacks. + void request_spills_locked(); + + mutable std::mutex mutex_; + std::atomic min_victim_arena_bytes_ {kDefaultMinVictimArenaBytes}; + // All below guarded by mutex_. + SignalsFn signals_; + // Shortfall episode latch: set (with ONE warning log) when the eligible + // arena is short of the overage, cleared when a later report finds it + // sufficient again -- so a relapse logs again, but a sustained episode logs + // exactly once. Observability only; flagging proceeds either way. + bool reclaim_shortfall_ = false; + // Live writers, keyed by their advisory flag: value is the buffer's last + // reported SPILLABLE arena bytes (the victim selection key and the only + // bytes a forced spill can reclaim). + phmap::flat_hash_map*, int64_t> entries_; +}; + +} // namespace doris::snii::writer diff --git a/be/src/storage/index/snii/writer/logical_index_writer.cpp b/be/src/storage/index/snii/writer/logical_index_writer.cpp new file mode 100644 index 00000000000000..63cd04bd3914c9 --- /dev/null +++ b/be/src/storage/index/snii/writer/logical_index_writer.cpp @@ -0,0 +1,952 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/writer/logical_index_writer.h" + +#include +#include +#include +#include +#include +#include + +#include "storage/index/inverted/common_grams/common_grams_key_codec.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/encoding/crc32c.h" +#include "storage/index/snii/encoding/varint.h" +#include "storage/index/snii/encoding/zstd_codec.h" +#include "storage/index/snii/format/bsbf.h" +#include "storage/index/snii/format/dict_block.h" +#include "storage/index/snii/format/dict_block_directory.h" +#include "storage/index/snii/format/frq_pod.h" +#include "storage/index/snii/format/norms_pod.h" +#include "storage/index/snii/format/null_bitmap.h" +#include "storage/index/snii/format/prx_pod.h" +#include "storage/index/snii/writer/posting_window_emitter.h" + +namespace doris::snii::writer { + +using format::BlockRef; +using format::DictBlockBuilder; +using format::DictBlockDirectoryBuilder; +using format::DictEntry; +using format::DictEntryEnc; +using format::DictEntryKind; +using format::SampledTermIndexBuilder; +using format::SectionRefs; +using segment_v2::inverted_index::CG_V1_MARKER; +using segment_v2::inverted_index::CommonGramsCoverage; +using segment_v2::inverted_index::ScoringCoverage; +using segment_v2::inverted_index::validate_common_grams_segment_metadata; + +namespace { + +// Target false-positive probability for the block-split bloom XFilter. Sizes +// the filter via Parquet OptimalNumOfBytes; L0 keeps the probe in memory and L1 +// keeps the per-query cost at one 32-byte block. +constexpr double kBsbfFpp = 0.01; +// Force-raw level for .frq dd/freq regions. Their plaintext is PFOR-bit-packed +// doc-deltas/freqs -- already high-entropy, so zstd shrinks ~30 MB of input by +// <0.1 MiB while burning ~0.4s CPU (and an extra crc pass over the compressed +// bytes) at 5M. We force raw here and keep zstd only on .prx (which compresses +// ~77%). Output stays self-describing: the region meta records zstd=false. +constexpr int kRawFrqRegion = 0; +// zstd level for whole-DICT-block compression comes from +// SniiIndexInput::dict_block_zstd_level (default 3: ~40% on the 64KiB +// front-coded blocks at ~120 MiB/s encode / ~600 MiB/s decode; higher levels +// trade import CPU for size, decode speed unchanged). G16-h made it (and the +// .prx auto level) caller-tunable. + +using format::FrqRegionMeta; + +// Fused single-pass term-level freq statistics: total_freq (running sum) and +// max_freq (running max) in ONE scan, reused by validate_term (has_prx +// position-count budget), stats_.sum_total_term_freq, and the DictEntry +// ttf_delta/max_freq. Byte-identical to the former separate SumOf/MaxOf scans: +// same left-to-right accumulation order and the same max init of 0, so a freq of +// 0 never lowers the max. Complete CommonGrams entries bypass this helper: +// their ttf is the already-known PRX position count and max_freq is not stored. +FreqStats fuse_freq_stats(const std::vector& freqs) { +#ifdef BE_TEST + testing::note_term_freq_scan(); +#endif + FreqStats fs; + for (uint32_t f : freqs) { + fs.total_freq += f; + fs.max_freq = std::max(f, fs.max_freq); + } + return fs; +} + +// Default window doc count by df: high-df windowed terms combine kFrqBaseUnit +// units into larger (kAdaptiveWindowDocs) windows. PRX limits may subsequently +// recut one of these default windows at document boundaries. +uint32_t adaptive_window_docs(uint32_t df) { + return df >= format::kAdaptiveWindowDfThreshold ? format::kAdaptiveWindowDocs + : format::kFrqBaseUnit; +} + +bool fits_prx_window_shape(uint64_t doc_count, uint64_t position_count, + const format::PrxWindowLimits& limits) { + return doc_count <= limits.max_docs && position_count <= limits.max_positions; +} + +} // namespace + +// The only encoder for TermPostingSource input. It borrows the writer's reusable +// posting buffer, streams PRX windows directly to the final sink, and stages grouped DD +// and frequency regions without retaining the complete term. +class StreamingTermEncoder { +public: + StreamingTermEncoder(LogicalIndexWriter* writer, StreamedTermPostings* postings, + bool declared_common_gram, bool term_has_freq, bool term_has_prx, + TermPostingBuffer* buffer, uint64_t frq_base, uint64_t prx_base) + : writer_(writer), + postings_(postings), + declared_common_gram_(declared_common_gram), + term_has_freq_(term_has_freq), + term_has_prx_(term_has_prx), + buffer_(buffer), + frq_base_(frq_base), + prx_base_(prx_base), + emitter_(WindowEmitterOptions { + .posting_out = writer->posting_out_, + .posting_region_offset = writer->posting_off0_, + .frq_base = frq_base, + .prx_base = prx_base, + .encoded_norms = writer->encoded_norms_, + .has_freq = term_has_freq, + .has_prx = term_has_prx, + .prx_zstd_level = writer->prx_zstd_level_, + .prx_window_limits = writer->prx_window_limits_, + .term_frequency_source = + declared_common_gram ? (postings->retain_positions + ? TermFrequencySource::kPositions + : TermFrequencySource::kDocuments) + : TermFrequencySource::kFrequenciesOrDocuments, + .memory_reporter = writer->memory_reporter_, + }) { + DCHECK(buffer_ != nullptr); + DCHECK(buffer_->empty()); + } + + ~StreamingTermEncoder() { + buffer_->clear_reuse_and_release_excess(format::kAdaptiveWindowDfThreshold); + } + + Status encode(DictEntry* entry, FreqStats* stats) { + if (postings_->source == nullptr) { + return Status::Error( + "logical_index: streamed term has a null posting source"); + } + if (writer_->has_prx_ && !postings_->retain_positions && !declared_common_gram_) { + return Status::Error( + "logical_index: only a declared CommonGrams term may omit positions"); + } +#ifdef BE_TEST + if (!declared_common_gram_) { + testing::note_term_freq_scan(); + } +#endif + bool exhausted = false; + RETURN_IF_ERROR(fill(format::kAdaptiveWindowDfThreshold, &exhausted)); + if (exhausted && buffer_->document_count() < format::kAdaptiveWindowDfThreshold) { + entry->term = std::move(postings_->term); + entry->df = total_docs_; + entry->ttf_delta = stats_.total_freq; + entry->max_freq = stats_.max_freq; + RETURN_IF_ERROR(encode_small(entry)); + *stats = stats_; + return Status::OK(); + } + + if (!buffer_->empty()) { + RETURN_IF_ERROR(encode_windowed_buffer(format::kAdaptiveWindowDocs)); + } + while (!exhausted) { + RETURN_IF_ERROR(fill(format::kAdaptiveWindowDocs, &exhausted)); + if (!buffer_->empty()) { + RETURN_IF_ERROR(encode_windowed_buffer(format::kAdaptiveWindowDocs)); + } + } + + entry->term = std::move(postings_->term); + entry->df = total_docs_; + entry->ttf_delta = stats_.total_freq; + entry->max_freq = stats_.max_freq; + RETURN_IF_ERROR(finish_windowed(entry)); + *stats = stats_; + return Status::OK(); + } + +private: + Status fill(uint32_t target_docs, bool* exhausted) { + buffer_->clear_reuse(); + position_offsets_.clear(); + *exhausted = false; + RETURN_IF_ERROR(postings_->source->fill(target_docs, buffer_, exhausted)); + const size_t count = buffer_->document_count(); + if (count > target_docs) { + return Status::Error( + "logical_index: posting source exceeded target_docs"); + } + if (!*exhausted && count != target_docs) { + return Status::Error( + "logical_index: posting source returned a short non-terminal fill"); + } + if (count == 0 && !*exhausted) { + return Status::Error( + "logical_index: posting source returned empty before EOF"); + } + return validate_and_accumulate(); + } + + Status validate_and_accumulate() { + const auto docids = buffer_->docids(); + const auto freqs = buffer_->freqs(); + const auto positions = buffer_->positions_flat(); + if (postings_->retain_positions && freqs.size() != docids.size()) { + return Status::Error( + "logical_index: positioned source must provide one freq per docid"); + } + if (!postings_->retain_positions && !freqs.empty() && freqs.size() != docids.size()) { + return Status::Error( + "logical_index: docs-only source freqs must be empty or parallel"); + } + if (postings_->retain_positions && + positions.size() > std::numeric_limits::max()) { + return Status::Error( + "logical_index: one source fill exceeds uint32 position offsets"); + } + if (postings_->retain_positions) { + position_offsets_.resize(freqs.size() + 1); + } + // One fused pass serves both the positions-count validation and the + // frequency statistics below. fill() has already capped the buffer at + // target_docs (at most the adaptive window sizes), so a uint64 sum of + // uint32 frequencies cannot overflow within one fill; the per-term + // accumulation below keeps its overflow check. + uint64_t fill_freq_sum = 0; + uint32_t fill_max_freq = 0; + for (size_t doc = 0; doc < freqs.size(); ++doc) { + const uint32_t freq = freqs[doc]; + fill_freq_sum += freq; + fill_max_freq = std::max(fill_max_freq, freq); + if (postings_->retain_positions) { + position_offsets_[doc + 1] = static_cast(fill_freq_sum); + } + } + if (postings_->retain_positions) { + if (fill_freq_sum != positions.size()) { + return Status::Error( + "logical_index: source positions count must equal sum(freqs)"); + } + } else { + if (!positions.empty()) { + return Status::Error( + "logical_index: docs-only source must not provide positions"); + } + } + + for (uint32_t docid : docids) { + if (last_docid_.has_value() && docid <= *last_docid_) { + return Status::Error( + "logical_index: source docids must be strictly ascending"); + } + if (docid >= writer_->doc_count_) { + return Status::Error( + "logical_index: source docid must be less than doc_count"); + } + last_docid_ = docid; + } + if (docids.size() > std::numeric_limits::max() - total_docs_) { + return Status::Error( + "logical_index: source document count exceeds uint32"); + } + total_docs_ += static_cast(docids.size()); + + if (declared_common_gram_) { + const uint64_t increment = + postings_->retain_positions ? positions.size() : docids.size(); + if (increment > std::numeric_limits::max() - stats_.total_freq) { + return Status::Error( + "logical_index: source total frequency overflow"); + } + stats_.total_freq += increment; + } else if (freqs.empty()) { + if (docids.size() > std::numeric_limits::max() - stats_.total_freq) { + return Status::Error( + "logical_index: source total frequency overflow"); + } + stats_.total_freq += docids.size(); + } else { + if (fill_freq_sum > std::numeric_limits::max() - stats_.total_freq) { + return Status::Error( + "logical_index: source total frequency overflow"); + } + stats_.total_freq += fill_freq_sum; + stats_.max_freq = std::max(stats_.max_freq, fill_max_freq); + } + return Status::OK(); + } + + Status encode_small(DictEntry* entry) { + if (total_docs_ >= format::kSlimDfThreshold || + (term_has_prx_ && + !fits_prx_window_shape(total_docs_, stats_.total_freq, writer_->prx_window_limits_))) { + RETURN_IF_ERROR(encode_windowed_buffer(adaptive_window_docs(total_docs_))); + return finish_windowed(entry); + } + + std::vector prx_window; + if (term_has_prx_) { + ByteSink sink; + format::PrxWindowBuildOutcome outcome = format::PrxWindowBuildOutcome::kBuilt; + RETURN_IF_ERROR(format::try_build_prx_window_flat( + buffer_->positions_flat(), buffer_->freqs(), -writer_->prx_zstd_level_, + writer_->prx_window_limits_, &sink, &outcome)); + if (outcome == format::PrxWindowBuildOutcome::kNeedsSplit) { + RETURN_IF_ERROR(encode_windowed_buffer(format::kFrqBaseUnit)); + return finish_windowed(entry); + } + prx_window = sink.take(); + } + + ByteSink frq_sink; + FrqRegionMeta dd_meta; + FrqRegionMeta freq_meta {}; + RETURN_IF_ERROR(format::build_dd_region(buffer_->docids(), /*win_base=*/0, kRawFrqRegion, + &frq_sink, &dd_meta)); + if (term_has_freq_) { + RETURN_IF_ERROR(format::build_freq_region(buffer_->freqs(), kRawFrqRegion, &frq_sink, + &freq_meta)); + } + std::vector frq_window = frq_sink.take(); + entry->enc = DictEntryEnc::kSlim; + entry->dd_meta = dd_meta; + entry->freq_meta = freq_meta; + if (frq_window.size() <= format::kDefaultInlineThreshold) { + entry->kind = DictEntryKind::kInline; + entry->inline_dd_disk_len = dd_meta.disk_len; + entry->frq_bytes = std::move(frq_window); + if (term_has_prx_) entry->prx_bytes = std::move(prx_window); + return Status::OK(); + } + + entry->kind = DictEntryKind::kPodRef; + entry->frq_docs_len = dd_meta.disk_len; + if (term_has_prx_) { + const uint64_t prx_off = writer_->posting_size(); + RETURN_IF_ERROR(writer_->posting_out_->append(Slice(prx_window))); + entry->prx_off_delta = prx_off - prx_base_; + entry->prx_len = writer_->posting_size() - prx_off; + } + const uint64_t frq_off = writer_->posting_size(); + RETURN_IF_ERROR(writer_->posting_out_->append(Slice(frq_window))); + entry->frq_off_delta = frq_off - frq_base_; + entry->frq_len = writer_->posting_size() - frq_off; + return Status::OK(); + } + + Status encode_windowed_buffer(uint32_t unit) { + DCHECK_GT(unit, 0); + for (size_t begin = 0; begin < buffer_->document_count(); begin += unit) { + const size_t count = + std::min(buffer_->document_count() - begin, static_cast(unit)); + const auto offsets = + term_has_prx_ + ? std::span(position_offsets_).subspan(begin, count + 1) + : std::span {}; + const auto positions = + term_has_prx_ ? buffer_->positions_flat().subspan( + offsets.front(), + static_cast(offsets.back() - offsets.front())) + : std::span {}; + RETURN_IF_ERROR(emitter_.emit_window(PostingRunView { + .docids = buffer_->docids().subspan(begin, count), + .freqs = buffer_->freqs().empty() ? std::span {} + : buffer_->freqs().subspan(begin, count), + .position_offsets = offsets, + .positions_flat = positions, + })); + } + return Status::OK(); + } + + Status finish_windowed(DictEntry* entry) { + TermAggregateStats emitted_stats; + RETURN_IF_ERROR(emitter_.finish_term(entry, &emitted_stats)); + DCHECK_EQ(emitted_stats.df, total_docs_); + DCHECK_EQ(emitted_stats.total_freq, stats_.total_freq); + DCHECK_EQ(emitted_stats.max_freq, stats_.max_freq); + return Status::OK(); + } + + LogicalIndexWriter* writer_; + StreamedTermPostings* postings_; + bool declared_common_gram_ = false; + bool term_has_freq_ = false; + bool term_has_prx_ = false; + TermPostingBuffer* buffer_ = nullptr; + uint64_t frq_base_ = 0; + uint64_t prx_base_ = 0; + WindowEmitter emitter_; + std::vector position_offsets_; + FreqStats stats_; + std::optional last_docid_; + uint32_t total_docs_ = 0; +}; + +namespace testing { +#ifdef BE_TEST +namespace { +// Function-local-static op-count seam backing term_freq_scans(). One atomic, +// relaxed: the writer build path is single-threaded, so only the COUNT matters, +// not ordering (the atomic keeps it race-clean if a test ever parallelizes). +std::atomic& term_freq_scan_counter() { + static std::atomic counter {0}; + return counter; +} +} // namespace +#endif + +void note_term_freq_scan() { +#ifdef BE_TEST + term_freq_scan_counter().fetch_add(1, std::memory_order_relaxed); +#endif +} +uint64_t term_freq_scans() { +#ifdef BE_TEST + return term_freq_scan_counter().load(std::memory_order_relaxed); +#else + return 0; +#endif +} +void reset_term_freq_scans() { +#ifdef BE_TEST + term_freq_scan_counter().store(0, std::memory_order_relaxed); +#endif +} + +// Forwards to the real fused helper so pure boundary tests exercise production +// code (not a test-local re-implementation). +FreqStats fuse_freq_stats_for_test(const std::vector& freqs) { + return fuse_freq_stats(freqs); +} +} // namespace testing + +LogicalIndexWriter::LogicalIndexWriter(const SniiIndexInput& in) + : LogicalIndexWriter(in, TrackedNullDocids(std::vector(in.null_docids))) {} + +LogicalIndexWriter::LogicalIndexWriter(const SniiIndexInput& in, TrackedNullDocids null_docids) + : index_id_(in.index_id), + index_suffix_(in.index_suffix), + index_config_(in.config), + tier_(format::tier_of(in.config)), + has_prx_(format::has_positions(in.config)), + // G16-c: the caller can drop freq layout entirely (in.write_freq == + // false) on a freq-capable tier -- see SniiIndexInput::write_freq. + has_freq_(format::tier_of(in.config) >= format::IndexTier::kT2 && in.write_freq), + has_norms_(format::has_scoring(in.config)), + doc_count_(in.doc_count), + null_docids_(std::move(null_docids)), + terms_(in.terms), + term_source_(in.term_source), + encoded_norms_(in.encoded_norms), + common_grams_metadata_(in.common_grams_metadata), + common_grams_posting_policy_(in.common_grams_posting_policy), + target_dict_block_bytes_(in.target_dict_block_bytes != 0 + ? in.target_dict_block_bytes + : format::kDefaultTargetDictBlockBytes), + dict_block_zstd_level_(in.dict_block_zstd_level), + prx_zstd_level_(in.prx_zstd_level), + prx_window_limits_(in.prx_window_limits), + memory_reporter_(in.mem_reporter), + dict_buf_(in.dict_resident_cap_bytes, "dict", in.mem_reporter), + norms_section_reservation_(in.mem_reporter == nullptr + ? MemoryReporter::Reservation() + : in.mem_reporter->make_reservation()), + null_bitmap_section_reservation_(in.mem_reporter == nullptr + ? MemoryReporter::Reservation() + : in.mem_reporter->make_reservation()), + term_hashes_reservation_(in.mem_reporter == nullptr + ? MemoryReporter::Reservation() + : in.mem_reporter->make_reservation()), + bsbf_bytes_reservation_(in.mem_reporter == nullptr + ? MemoryReporter::Reservation() + : in.mem_reporter->make_reservation()) {} + +Status LogicalIndexWriter::reserve_term_hash_for_append() { + if (memory_reporter_ == nullptr || term_hashes_.size() < term_hashes_.capacity()) { + return Status::OK(); + } + size_t target_capacity = term_hashes_.capacity() == 0 ? 1 : term_hashes_.capacity() * 2; + if (target_capacity < term_hashes_.capacity() || + target_capacity > std::numeric_limits::max() / sizeof(uint64_t)) { + return Status::Error( + "logical_index: term hash capacity overflow"); + } + MemoryReporter::Reservation replacement; + RETURN_IF_ERROR(term_hashes_reservation_.prepare_replacement(target_capacity * sizeof(uint64_t), + &replacement)); + term_hashes_.reserve(target_capacity); + DCHECK_EQ(term_hashes_.capacity(), target_capacity); + term_hashes_reservation_ = std::move(replacement); + return Status::OK(); +} + +// Serializes the current open block, zstd-compresses it (the dict region is the +// single largest section -- term keys + entry meta + inline postings -- and the +// 64KiB blocks compress ~40%), streams the compressed bytes into the dict +// scratch file, and records a directory entry. The block-level crc32c +// (rec.checksum) covers the UNCOMPRESSED bytes, so DictBlockReader::open +// verifies integrity after the reader decompresses. A compressed block also +// shrinks the bytes a term lookup fetches from S3 -- aligning with the +// read-byte thesis. If zstd does not shrink a (tiny) block, it is stored raw so +// a lookup never pays a pointless decompress. +Status LogicalIndexWriter::flush_block(DictBlockBuilder* block, std::string first_term) { + std::vector plain_bytes = block->finish_owned(); + const Slice plain(plain_bytes); + BlockRecord rec; + rec.rel_offset = dict_buf_.size(); + rec.n_entries = block->n_entries(); + rec.checksum = crc32c(plain); // crc over UNCOMPRESSED block bytes + rec.first_term = std::move(first_term); + + std::vector comp; + Status zs = zstd_compress(plain, dict_block_zstd_level_, &comp); + if (zs.ok() && comp.size() < plain.size()) { + rec.flags = format::block_ref_flags::kZstd; + rec.uncomp_len = static_cast(plain.size()); + rec.length = static_cast(comp.size()); + RETURN_IF_ERROR(dict_buf_.append_move(std::move(comp))); + } else { + rec.flags = 0; + rec.uncomp_len = 0; + rec.length = static_cast(plain.size()); + RETURN_IF_ERROR(dict_buf_.append_move(std::move(plain_bytes))); + } + blocks_.push_back(std::move(rec)); + return Status::OK(); +} + +// Running state for the in-flight DICT block while terms stream past. +struct LogicalIndexWriter::BlockState { + explicit BlockState(MemoryReporter* memory_reporter) : transfer_buffer(memory_reporter) {} + + std::unique_ptr block; + std::string block_first_term; + uint64_t frq_base = 0; + uint64_t prx_base = 0; + bool term_stats = true; + TermPostingBuffer transfer_buffer; +}; + +// Out-of-line so unique_ptr sees the complete type (see header). +LogicalIndexWriter::~LogicalIndexWriter() = default; + +Status LogicalIndexWriter::process_term(StreamedTermPostings& tp, BlockState* st) { + const bool is_declared_common_gram = + common_grams_metadata_.has_value() && tp.term.starts_with(CG_V1_MARKER) && + (common_grams_metadata_->common_grams_coverage == CommonGramsCoverage::kComplete || + common_grams_posting_policy_ == format::CommonGramsPostingPolicy::kHybridV1); + const bool term_has_prx = has_prx_ && tp.retain_positions; + const bool term_has_freq = has_freq_ && !is_declared_common_gram; + + if (st->block && st->term_stats != term_has_freq) { + RETURN_IF_ERROR(flush_block(st->block.get(), st->block_first_term)); + st->block.reset(); + } + if (!st->block) { + const uint64_t base = posting_size(); + st->frq_base = base; + st->prx_base = base; + st->term_stats = term_has_freq; + st->block = std::make_unique(tier_, has_prx_, st->frq_base, st->prx_base, + /*anchor_interval=*/16, + /*term_stats=*/term_has_freq); + st->block_first_term = tp.term; + } + + RETURN_IF_ERROR(reserve_term_hash_for_append()); + const uint64_t term_hash = format::bsbf_hash(tp.term); + DictEntry entry; + FreqStats stats; + StreamingTermEncoder encoder(this, &tp, is_declared_common_gram, term_has_freq, term_has_prx, + &st->transfer_buffer, st->frq_base, st->prx_base); + RETURN_IF_ERROR(encoder.encode(&entry, &stats)); + + term_hashes_.push_back(term_hash); + ++term_count_; + stats_.sum_total_term_freq += stats.total_freq; + st->block->add_entry(std::move(entry)); + if (st->block->estimated_bytes() >= target_dict_block_bytes_) { + RETURN_IF_ERROR(flush_block(st->block.get(), st->block_first_term)); + st->block.reset(); + } + return Status::OK(); +} + +Status LogicalIndexWriter::build_blocks() { + BlockState st(memory_reporter_); + if (term_source_ != nullptr) { + RETURN_IF_ERROR(term_source_->for_each_term_sorted( + [&](StreamedTermPostings&& tp) { return process_term(tp, &st); })); + } else { + for (const auto& tp : terms_) { + SpanTermPostingSource source(tp.docids, tp.freqs, tp.positions_flat); + StreamedTermPostings streamed { + .term = tp.term, .retain_positions = tp.retain_positions, .source = &source}; + RETURN_IF_ERROR(process_term(streamed, &st)); + } + } + if (st.block) RETURN_IF_ERROR(flush_block(st.block.get(), st.block_first_term)); + return Status::OK(); +} + +Status LogicalIndexWriter::prepare_build(io::FileWriter* posting_out) { + if (posting_out == nullptr) { + return Status::Error( + "logical_index: null posting sink"); + } + RETURN_IF_ERROR(format::validate_prx_window_limits(prx_window_limits_)); + if (has_norms_ && encoded_norms_.size() != doc_count_) { + return Status::Error( + "logical_index: norms length must equal doc_count"); + } + for (size_t i = 0; i < null_docids_.size(); ++i) { + if (null_docids_[i] >= doc_count_) { + return Status::Error( + "logical_index: null docid must be less than doc_count"); + } + if (i != 0 && null_docids_[i] <= null_docids_[i - 1]) { + return Status::Error( + "logical_index: null docids must be strictly ascending"); + } + } + if (common_grams_metadata_) { + RETURN_IF_ERROR(validate_common_grams_segment_metadata(*common_grams_metadata_)); + if (common_grams_metadata_->common_grams_coverage == CommonGramsCoverage::kComplete && + !has_prx_) { + return Status::Error( + "logical_index: complete CommonGrams metadata requires positions"); + } + if (common_grams_metadata_->scoring_coverage == ScoringCoverage::kComplete) { + if (!has_norms_ || !has_freq_) { + return Status::Error( + "logical_index: complete scoring metadata requires frequencies and " + "semantic norms"); + } + if (common_grams_metadata_->scoring_doc_count != doc_count_) { + return Status::Error( + "logical_index: scoring doc count must equal doc_count"); + } + } + } + if (common_grams_posting_policy_ == format::CommonGramsPostingPolicy::kHybridV1 && + (!common_grams_metadata_ || + common_grams_metadata_->common_grams_coverage != CommonGramsCoverage::kMixed || + !has_prx_)) { + return Status::Error( + "logical_index: hybrid CommonGrams postings require mixed metadata and positions"); + } + // The interleaved posting region streams STRAIGHT into the container output + // (no temp round-trip): posting_size() is the region-relative byte count, + // derived from the output offset advanced since this index's region began. + // The DICT region is staged in dict_buf_ (tiered: RAM under the cap = + // spill-only; spills above it) since it must land contiguously after the + // concurrently-streamed posting region. + posting_out_ = posting_out; + posting_off0_ = posting_out->bytes_written(); + return Status::OK(); +} + +Status LogicalIndexWriter::finalize_build() { + if (common_grams_metadata_ && + common_grams_metadata_->scoring_coverage == ScoringCoverage::kComplete) { + if (common_grams_metadata_->scoring_token_count > stats_.sum_total_term_freq) { + return Status::Error( + "logical_index: semantic scoring token count exceeds physical term frequency"); + } + if (stats_.sum_total_term_freq != 0 && common_grams_metadata_->scoring_token_count == 0) { + return Status::Error( + "logical_index: non-empty physical postings have zero semantic scoring tokens"); + } + if (common_grams_metadata_->plain_term_key_version == + ::doris::segment_v2::inverted_index::PlainTermKeyVersion::kRawNoInternal && + common_grams_metadata_->common_grams_coverage == CommonGramsCoverage::kNone && + common_grams_metadata_->scoring_token_count != stats_.sum_total_term_freq) { + return Status::Error( + "logical_index: semantic plain token count must equal physical term frequency"); + } + } + // Seal the dict buffer so a spilled temp is flushed before + // stream_dict_region_into reads it back. A no-op for a RAM-resident dict. + RETURN_IF_ERROR(dict_buf_.seal()); + + stats_.doc_count = doc_count_; + stats_.indexed_doc_count = doc_count_ - static_cast(null_docids_.size()); + stats_.term_count = term_count_; + stats_.null_count = static_cast(null_docids_.size()); + + if (has_norms_) { + const size_t payload_size = varint_len(encoded_norms_.size()) + encoded_norms_.size(); + const size_t section_size = 1 + varint_len(payload_size) + payload_size + sizeof(uint32_t); + MemoryReporter::Reservation build_reservation = + memory_reporter_ == nullptr ? MemoryReporter::Reservation() + : memory_reporter_->make_reservation(); + if (memory_reporter_ != nullptr) { + RETURN_IF_ERROR(build_reservation.set_bytes(payload_size)); + RETURN_IF_ERROR(norms_section_reservation_.set_bytes(section_size)); + } + ByteSink nsink; + format::NormsPodWriter::finish(encoded_norms_, &nsink); + norms_section_ = nsink.take(); + DORIS_CHECK_EQ(norms_section_.capacity(), section_size); + if (memory_reporter_ != nullptr) { + DORIS_CHECK_EQ(norms_section_reservation_.bytes(), norms_section_.capacity()); + } + } + + if (!null_docids_.empty()) { + MemoryReporter::Reservation bitmap_build_reservation = + memory_reporter_ == nullptr ? MemoryReporter::Reservation() + : memory_reporter_->make_reservation(); + if (memory_reporter_ != nullptr) { + RETURN_IF_ERROR(bitmap_build_reservation.set_bytes( + format::NullBitmapWriter::build_memory_upper_bound( + std::span(null_docids_.data(), null_docids_.size())))); + } + format::NullBitmapWriter null_writer; + null_writer.add_many(std::span(null_docids_.data(), null_docids_.size())); + null_docids_.release(); + + format::NullBitmapSerializationSizes sizes; + RETURN_IF_ERROR(null_writer.serialization_sizes(doc_count_, &sizes)); + if (sizes.roaring_bytes > std::numeric_limits::max() - sizes.payload_bytes) { + return Status::Error( + "logical_index: null bitmap scratch size overflows"); + } + MemoryReporter::Reservation scratch_reservation = + memory_reporter_ == nullptr ? MemoryReporter::Reservation() + : memory_reporter_->make_reservation(); + if (memory_reporter_ != nullptr) { + RETURN_IF_ERROR( + scratch_reservation.set_bytes(sizes.roaring_bytes + sizes.payload_bytes)); + RETURN_IF_ERROR(null_bitmap_section_reservation_.set_bytes(sizes.framed_bytes)); + } + ByteSink null_sink; + RETURN_IF_ERROR(null_writer.finish(doc_count_, &null_sink)); + null_bitmap_section_ = null_sink.take(); + DORIS_CHECK_EQ(null_bitmap_section_.size(), sizes.framed_bytes); + if (memory_reporter_ != nullptr) { + DORIS_CHECK_LE(null_bitmap_section_.capacity(), + null_bitmap_section_reservation_.bytes()); + } + } + null_docids_.release(); + + // Build the absent-term filter (block-split bloom, Parquet-canonical) from + // the per-term keys (no retained strings) as a [28B header][bitset] blob; the + // compound writer places it as a PHYSICAL section probed one 32-byte block on + // demand. + bsbf_bytes_.clear(); + bsbf_built_ = false; + if (!term_hashes_.empty()) { + const uint32_t bitset_bytes = format::bsbf_optimal_num_bytes( + static_cast(term_hashes_.size()), kBsbfFpp); + const size_t serialized_bytes = format::kBsbfHeaderSize + bitset_bytes; + MemoryReporter::Reservation builder_reservation = + memory_reporter_ == nullptr ? MemoryReporter::Reservation() + : memory_reporter_->make_reservation(); + if (memory_reporter_ != nullptr) { + RETURN_IF_ERROR(builder_reservation.set_bytes(bitset_bytes)); + RETURN_IF_ERROR(bsbf_bytes_reservation_.set_bytes(serialized_bytes)); + } + format::BsbfBuilder bf; + RETURN_IF_ERROR(format::BsbfBuilder::create(static_cast(term_hashes_.size()), + kBsbfFpp, &bf)); + DCHECK_EQ(bf.resident_capacity_bytes(), bitset_bytes); + for (uint64_t k : term_hashes_) bf.insert(k); + ByteSink bsink; + bsink.reserve(serialized_bytes); + RETURN_IF_ERROR(bf.serialize(&bsink)); + bsbf_bytes_ = bsink.take(); + DCHECK_EQ(bsbf_bytes_.capacity(), serialized_bytes); + bsbf_built_ = true; + } + std::vector().swap(term_hashes_); // release + term_hashes_reservation_.reset(); + + return Status::OK(); +} + +void LogicalIndexWriter::release_bsbf_bytes() { + std::vector().swap(bsbf_bytes_); + bsbf_bytes_reservation_.reset(); +} + +void LogicalIndexWriter::release_null_bitmap_bytes() { + std::vector().swap(null_bitmap_section_); + null_bitmap_section_reservation_.reset(); +} + +void LogicalIndexWriter::release_norms_bytes() { + std::vector().swap(norms_section_); + norms_section_reservation_.reset(); +} + +Status LogicalIndexWriter::build(io::FileWriter* posting_out) { + // Single-session invariant: a writer that ran (or is running) a streamed + // session must not also build() -- the posting sink anchor and the dict + // buffer are one-shot. + if (stream_phase_ != StreamPhase::kIdle) { + return Status::Error( + "logical_index: build() on a writer with a streamed session"); + } + // prepare_build is pure entry validation up to its final sink-anchor + // assignments, so a failure there leaves the writer clean (still kIdle). + RETURN_IF_ERROR(prepare_build(posting_out)); + // Poison-by-default past this point: build_blocks/finalize_build may fail + // AFTER posting bytes hit the sink or term state advanced, and a dirty + // writer must never accept a later begin_streamed/build (single-session + + // crash-safety invariant 6). Only full success seals to kFinished. + stream_phase_ = StreamPhase::kFailed; + RETURN_IF_ERROR(build_blocks()); + RETURN_IF_ERROR(finalize_build()); + // Claim the (only) session so a later begin_streamed/push_term errors out. + stream_phase_ = StreamPhase::kFinished; + return Status::OK(); +} + +Status LogicalIndexWriter::begin_streamed(io::FileWriter* posting_out) { + if (stream_phase_ == StreamPhase::kFailed) { + return Status::Error( + "logical_index: begin_streamed on a failed writer (a prior session error left " + "partial state; allocate a fresh writer)"); + } + if (stream_phase_ != StreamPhase::kIdle) { + return Status::Error( + "logical_index: begin_streamed on an already-claimed writer session"); + } + RETURN_IF_ERROR(prepare_build(posting_out)); + stream_state_ = std::make_unique(memory_reporter_); + stream_phase_ = StreamPhase::kActive; + return Status::OK(); +} + +Status LogicalIndexWriter::push_term(StreamedTermPostings&& tp) { + if (stream_phase_ != StreamPhase::kActive) { + return Status::Error( + "logical_index: push_term without an active streamed session"); + } + if (has_pushed_term_ && tp.term <= last_pushed_term_) { + return Status::Error( + "logical_index: pushed terms must be strictly increasing ('{}' after '{}')", + tp.term, last_pushed_term_); + } + last_pushed_term_ = tp.term; + has_pushed_term_ = true; + Status status = process_term(tp, stream_state_.get()); + if (!status.ok()) { + stream_phase_ = StreamPhase::kFailed; + stream_state_.reset(); + } + return status; +} + +Status LogicalIndexWriter::finish_streamed() { + if (stream_phase_ == StreamPhase::kFinished) { + return Status::Error( + "logical_index: finish_streamed on an already-finished session"); + } + if (stream_phase_ == StreamPhase::kFailed) { + return Status::Error( + "logical_index: finish_streamed on a failed session (a prior push/finish error " + "poisoned the writer; the partial output must be discarded)"); + } + if (stream_phase_ != StreamPhase::kActive) { + return Status::Error( + "logical_index: finish_streamed without begin_streamed"); + } + // Poison-by-default: a failed trailing flush or finalize leaves partial + // output, so only full success below seals the session to kFinished. + stream_phase_ = StreamPhase::kFailed; + // Trailing-block flush mirrors the tail of build_blocks(); the shared + // finalize_build() then seals the dict buffer and materializes the + // stats/norms/null-bitmap/BSBF sections. + if (stream_state_->block) { + RETURN_IF_ERROR(flush_block(stream_state_->block.get(), stream_state_->block_first_term)); + } + stream_state_.reset(); + RETURN_IF_ERROR(finalize_build()); + stream_phase_ = StreamPhase::kFinished; + return Status::OK(); +} + +Status LogicalIndexWriter::finish_metadata(const SectionRefs& abs_refs, uint64_t dict_region_offset, + SerializedMetadataGroup* out) const { + if (out == nullptr) { + return Status::Error( + "logical_index: null metadata output"); + } + *out = {}; + + SampledTermIndexBuilder sti; + for (const auto& b : blocks_) sti.add_block_first_term(b.first_term); + ByteSink sti_sink; + sti.finish(&sti_sink); + + DictBlockDirectoryBuilder dir; + for (const auto& b : blocks_) { + BlockRef ref; + ref.offset = dict_region_offset + b.rel_offset; + ref.length = b.length; + ref.n_entries = b.n_entries; + ref.flags = b.flags; + ref.checksum = b.checksum; + ref.uncomp_len = b.uncomp_len; + dir.add(ref); + } + ByteSink dir_sink; + dir.finish(&dir_sink); + + ByteSink sti_blob; + RETURN_IF_ERROR( + format::encode_metadata_blob(sti_sink.view(), format::SectionType::kSampledTermIndex, + format::SectionType::kSampledTermIndexZstd, &sti_blob)); + out->sampled_term_index = sti_blob.take(); + + ByteSink dbd_blob; + RETURN_IF_ERROR( + format::encode_metadata_blob(dir_sink.view(), format::SectionType::kDictBlockDirectory, + format::SectionType::kDictBlockDirectoryZstd, &dbd_blob)); + out->dict_block_directory = dbd_blob.take(); + + format::CoreMetadata core; + core.index_config = index_config_; + core.stats = stats_; + core.section_refs = abs_refs; + core.common_grams_metadata = common_grams_metadata_; + core.common_grams_posting_policy = common_grams_posting_policy_; + ByteSink core_sink; + RETURN_IF_ERROR(format::encode_core_metadata(core, &core_sink)); + out->core = core_sink.take(); + return Status::OK(); +} + +} // namespace doris::snii::writer diff --git a/be/src/storage/index/snii/writer/logical_index_writer.h b/be/src/storage/index/snii/writer/logical_index_writer.h new file mode 100644 index 00000000000000..c93c58d8b722ac --- /dev/null +++ b/be/src/storage/index/snii/writer/logical_index_writer.h @@ -0,0 +1,459 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/inverted/common_grams/common_grams_segment_metadata.h" +#include "storage/index/snii/encoding/byte_sink.h" +#include "storage/index/snii/format/core_metadata.h" +#include "storage/index/snii/format/dict_block.h" +#include "storage/index/snii/format/dict_block_directory.h" +#include "storage/index/snii/format/dict_entry.h" +#include "storage/index/snii/format/format_constants.h" +#include "storage/index/snii/format/metadata_blob.h" +#include "storage/index/snii/format/prx_pod.h" +#include "storage/index/snii/format/sampled_term_index.h" +#include "storage/index/snii/format/stats_block.h" +#include "storage/index/snii/io/file_writer.h" +#include "storage/index/snii/writer/memory_reporter.h" +#include "storage/index/snii/writer/spillable_byte_buffer.h" +#include "storage/index/snii/writer/spimi_term_buffer.h" +#include "storage/index/snii/writer/term_posting_source.h" + +// LogicalIndexWriter -- builds the per-logical-index section bytes (interleaved +// posting region + DICT block region) plus the SampledTermIndex and DICT block +// directory metadata for ONE logical index. It owns the in-memory section bytes, +// runtime statistics, and references needed by the container orchestrator +// (SniiCompoundWriter) to resolve absolute offsets and emit the Core/STI/DBD +// metadata group. +// +// This module deliberately produces ONLY relative bytes/structures: it has no +// knowledge of the absolute file position where the sections will land. The +// orchestrator stitches the absolute offsets in afterward (append-only, no +// seek-back). See snii_compound_writer.h for the precise offset contract. +// +// POSTING REGION (single interleaved sink): the former separate .frq POD and .prx +// POD are merged into ONE posting region. For each pod_ref term, in term order, the +// writer appends its prx span FIRST then its frq span, contiguously: +// posting region = concat over pod_ref terms of [prx span][frq span]. +// The prx span is empty when !has_prx (docs-only / keyword tier). INLINE terms +// append NOTHING to the posting region. +// +// Per-term encoding policy (v1): +// df >= kSlimDfThreshold (512), or a lower-df term whose positions cannot fit +// one configured reader-safe PRX window: WINDOWED pod_ref. The term's [prx +// windows] are appended to the posting region first, then its +// [prelude][dd-block][freq-block] frq span. The DictEntry records frq/prx +// off_delta+len relative to frq_base/prx_base (see below). +// Other df < kSlimDfThreshold terms: SLIM. The postings are encoded as a +// single .frq window (and .prx window). If the encoded .frq bytes are small +// (<= kDefaultInlineThreshold), they are stored INLINE inside the DictEntry +// (kind=inline); otherwise the term's [prx][frq] spans are appended to the +// posting region as a slim pod_ref (kind=pod_ref, enc=slim, no prelude). +// +// frq_base / prx_base convention (DOCUMENTED CONTRACT): +// For each DICT block, frq_base == prx_base == the running byte offset into THIS +// index's posting region at the moment the block opens (the posting-region size +// when the block's first POD-backed entry is appended). A windowed/slim pod_ref +// entry then sets frq_off_delta = (offset of its frq span within the posting +// region) - frq_base, so the reader computes the absolute file offset as +// section_refs.posting_region.offset + frq_base + frq_off_delta. +// prx_base / prx_off_delta follow the identical rule against the SAME region. +// Because [prx][frq] are written contiguously per term, a writer-side property +// holds when has_prx: frq_off_delta == prx_off_delta + prx_len. The reader does +// NOT rely on it -- each delta is resolved independently. +// Inline entries carry no off_delta (bytes live in the entry). +namespace doris::snii::writer { + +class SniiStreamedIndexSession; +class StreamingTermEncoder; + +struct SerializedMetadataGroup { + std::vector core; + std::vector sampled_term_index; + std::vector dict_block_directory; +}; + +// Inputs describing one logical index to be written. +struct SniiIndexInput { + uint64_t index_id = 0; + std::string index_suffix; + format::IndexConfig config = format::IndexConfig::kDocsPositions; + uint32_t doc_count = 0; + std::vector null_docids; + // Per-doc 1-byte encoded norm (length doc_count); only consumed when the + // config has scoring. May be empty otherwise. + std::vector encoded_norms; + // G16-h: zstd levels for the dict-block whole-block compression and the + // .prx window auto mode (both default 3 == the historical constants). + // Higher levels trade import CPU for size; decode speed is unaffected. + int dict_block_zstd_level = 3; + int prx_zstd_level = 3; + // Internal writer policy. Production callers keep the reader limits; unit + // tests may only tighten them to exercise extreme-window behavior without + // allocating hundreds of MiB. + format::PrxWindowLimits prx_window_limits = format::kReaderPrxWindowLimits; + // G16-c: whether freq-capable (tier>=T2) postings lay out freq regions at + // all. Freq bytes serve ONLY BM25 scoring (want_freq=true lives solely in + // scoring_query), so the CALLER resolves the policy -- the Doris adapter + // passes has_scoring(config) || config::snii_positions_index_write_freq, + // i.e. plain kDocsPositions indexes drop freq unless the escape hatch is + // set. Defaults to true so the core library and existing callers keep the + // full T2 layout unless they opt out. The drop is value-driven on disk + // (windowed prelude flags bit0; slim/inline zero-length freq regions), so + // readers need no index-level flag. Ignored for docs-only configs. + bool write_freq = true; + // Lexicographically sorted terms with ascending-docid postings. Used when + // `term_source` is null (callers that already hold a materialized vector, + // e.g. unit tests). The writer reads but does not retain these. + std::vector terms; + // Optional streaming term source. When non-null, the writer DRAINS it via + // SpimiTermBuffer::for_each_term_sorted so that only one term's postings is + // materialized at a time (avoiding the full TermPostings vector and its + // second-copy peak). `terms` is ignored when this is set. The buffer is + // consumed (emptied) by build(); the caller must keep it alive until build() + // returns and must not reuse it afterwards. + SpimiTermBuffer* term_source = nullptr; + // Target DICT block size in bytes; a block is cut once its estimate reaches + // this. 0 uses kDefaultTargetDictBlockBytes. Smaller values yield more blocks + // (and a finer-grained sampled-term index). + uint32_t target_dict_block_bytes = 0; + // Maximum resident capacity of the staged DICT region before it spills. + // Ordinary builds keep the default unlimited local cap and use their shared + // spill-threshold reporter. Streamed compaction sets a bounded watermark so + // reclaimable DICT blocks cannot consume the hard-capped term workspace. + uint64_t dict_resident_cap_bytes = std::numeric_limits::max(); + // Optional writer-level build-RAM reporter (one per SniiCompoundWriter = one + // segment inverted index). When non-null, the dict buffer reports its REAL + // resident-byte deltas (positive on grow, negative on spill). The SPIMI side + // (arena + slot index) reports through the SAME reporter, injected directly at + // the term_source's construction by the caller. null in bench / unit tests -> no + // reporting. NEVER report live_bytes_ (a gated estimate); report + // arena_bytes()+slot_of_+dict ram_bytes_. + MemoryReporter* mem_reporter = nullptr; + // Optional persisted CommonGrams capability and semantic scoring stats. + // Missing metadata preserves the legacy SNII image and cannot be treated as + // compatibility proof by readers. + std::optional common_grams_metadata; + // Optional per-term CommonGrams postings shape. HybridV1 requires Mixed + // coverage metadata and a positions-capable logical index. + format::CommonGramsPostingPolicy common_grams_posting_policy = + format::CommonGramsPostingPolicy::kNone; +}; + +// Move-only ownership of a NULL-docid allocation and its precharged bytes. +// Reservation is declared first so destruction always frees the vector before +// returning its charge. Move assignment is intentionally forbidden because its +// default member order would release the destination charge before its vector. +class TrackedNullDocids { +public: + explicit TrackedNullDocids(std::vector&& docids) : docids_(std::move(docids)) {} + TrackedNullDocids(MemoryReporter::Reservation&& reservation, std::vector&& docids) + : reservation_(std::move(reservation)), docids_(std::move(docids)) {} + + TrackedNullDocids(const TrackedNullDocids&) = delete; + TrackedNullDocids& operator=(const TrackedNullDocids&) = delete; + TrackedNullDocids(TrackedNullDocids&&) noexcept = default; + TrackedNullDocids& operator=(TrackedNullDocids&&) = delete; + + bool empty() const { return docids_.empty(); } + size_t size() const { return docids_.size(); } + const uint32_t* data() const { return docids_.data(); } + uint32_t operator[](size_t index) const { return docids_[index]; } + auto begin() const { return docids_.begin(); } + auto end() const { return docids_.end(); } + + void release() { + std::vector().swap(docids_); + reservation_.reset(); + } + +private: + MemoryReporter::Reservation reservation_; + std::vector docids_; +}; + +// Move-only ownership of a destination norms allocation and its precharged +// bytes. Streamed sessions adopt both together so the vector remains accounted +// for until the writer has materialized the norms section. +class TrackedEncodedNorms { +public: + explicit TrackedEncodedNorms(std::vector&& norms) : norms_(std::move(norms)) {} + TrackedEncodedNorms(MemoryReporter::Reservation&& reservation, std::vector&& norms) + : reservation_(std::move(reservation)), norms_(std::move(norms)) {} + + TrackedEncodedNorms(const TrackedEncodedNorms&) = delete; + TrackedEncodedNorms& operator=(const TrackedEncodedNorms&) = delete; + TrackedEncodedNorms(TrackedEncodedNorms&&) noexcept = default; + TrackedEncodedNorms& operator=(TrackedEncodedNorms&&) = delete; + + bool empty() const { return norms_.empty(); } + size_t size() const { return norms_.size(); } + uint8_t operator[](size_t index) const { return norms_[index]; } + auto begin() const { return norms_.begin(); } + auto end() const { return norms_.end(); } + + void release() { + std::vector().swap(norms_); + reservation_.reset(); + } + +private: + friend class SniiStreamedIndexSession; + MemoryReporter::Reservation reservation_; + std::vector norms_; +}; + +// Term-level frequency statistics. Ordinary terms compute sum(freqs) and +// max(freqs) in one fused scan. Complete CommonGrams entries derive total_freq +// from their required PRX position count and leave max_freq at 0 because their +// statless DICT block does not serialize it. +struct FreqStats { + uint64_t total_freq = 0; + uint32_t max_freq = 0; +}; + +// Builds and holds the section bytes + meta sub-sections for one logical index. +class LogicalIndexWriter { +public: + explicit LogicalIndexWriter(const SniiIndexInput& in); + // Out-of-line: stream_state_ points at the private nested BlockState, which + // is incomplete here (unique_ptr needs the complete type at destruction). + ~LogicalIndexWriter(); + + // Builds DICT blocks, the interleaved posting region, sampled-term index, dict + // directory, stats and bsbf. The posting region is written STRAIGHT into + // `posting_out` as terms are produced (no temp round-trip for the bulk); the + // orchestrator captures its absolute offset/length from posting_out->bytes_written() + // around this call. Must be called once before the accessors below. Returns + // InvalidArgument on a null sink or inconsistent input (e.g. norms/doc_count + // mismatch when scoring is enabled, or non-ascending docids). + Status build(io::FileWriter* posting_out); + + // Streamed three-phase alternative to build() (T2.1, the compaction index + // merge fast path): the CALLER produces terms one at a time (k-way merge + // over source segments) instead of handing the writer a term source. + // begin_streamed(sink) -> push_term(tp) x N -> finish_streamed() + // push_term funnels through the SAME process_term choke point build() + // drains through (bigram prune gates, shape validation, encode), and the + // setup/finalize steps are shared with build() -- so the produced bytes are + // IDENTICAL to a build() fed the same terms in the same order (the T2 + // byte-golden invariant). A writer instance runs EXACTLY ONE session: + // build() and begin_streamed are mutually exclusive, push_term after + // finish_streamed and a second finish_streamed are errors -- a half-fed + // session can never masquerade as a sealed index (crash-safety invariant 6). + // Failure poisons the session: any push_term/finish_streamed/build failure + // past entry validation (e.g. a posting-sink append error mid-encode) moves + // the writer to a terminal failed state where every subsequent + // push_term/finish_streamed/build/begin_streamed is rejected, so a caller + // that swallows an error can never seal (or re-claim) a corrupt index. + // Entry rejections themselves (term-order / postings-shape / phase checks) + // do not poison an active session EXCEPT postings-shape violations, which + // fail inside the shared process_term and conservatively poison too; only + // the term-order guard is explicitly recoverable (see the UT contract). + Status begin_streamed(io::FileWriter* posting_out); + // Consumes tp synchronously. Entry validation: terms must arrive in STRICTLY + // increasing lexicographic order (the one invariant process_term cannot see + // -- DICT blocks, the sampled term index and the reader's binary search all + // assume it; equal terms are rejected too, the upstream merge must have + // combined duplicates). The streaming encoder rejects invalid per-term + // posting shapes. Returns InvalidArgument on any violation. + Status push_term(StreamedTermPostings&& tp); + Status finish_streamed(); + + // DICT region byte length (relative; orchestrator decides its absolute offset). The + // DICT region (zstd-compressed blocks) is built into a tiered buffer during build() + // -- it must land contiguously AFTER the posting region (streamed concurrently), so + // it cannot stream directly. The buffer stays in RAM while small (spill-only build) + // and spills to a temp once it crosses the RAM cap (bounded peak RSS for a huge + // dict). Its bytes are emitted via stream_dict_region_into below. The posting region + // went straight to the output during build(), so it has no length accessor here -- + // the orchestrator measures it directly. norms stays in RAM (1 byte/doc). + uint64_t dict_region_size() const { return dict_buf_.size(); } + const std::vector& norms_bytes() const { return norms_section_; } + const std::vector& null_bitmap_bytes() const { return null_bitmap_section_; } + // Block-split bloom XFilter blob ([28B header][bitset]); empty when no terms. + const std::vector& bsbf_bytes() const { return bsbf_bytes_; } + bool has_bsbf() const { return bsbf_built_; } + void release_bsbf_bytes(); + void release_null_bitmap_bytes(); + void release_norms_bytes(); + bool has_null_bitmap() const { return !null_bitmap_section_.empty(); } + + // Streams the DICT region (RAM or spilled temp) into the append-only container + // after its posting region. + Status stream_dict_region_into(io::FileWriter* out) { + return dict_buf_.stream_into_and_release(out); + } + + bool has_prx() const { return has_prx_; } + bool has_norms() const { return has_norms_; } + format::IndexTier tier() const { return tier_; } + uint64_t index_id() const { return index_id_; } + const std::string& index_suffix() const { return index_suffix_; } + + // Builds the three mandatory v1 metadata blobs. The orchestrator writes them + // contiguously as Core -> STI -> DBD and publishes their absolute references + // only after all three appends succeed. + Status finish_metadata(const format::SectionRefs& abs_refs, uint64_t dict_region_offset, + SerializedMetadataGroup* out) const; + +private: + friend class SniiStreamedIndexSession; + LogicalIndexWriter(const SniiIndexInput& in, TrackedNullDocids null_docids); + + // One DICT block's directory record. The block's serialized bytes are appended to + // the in-RAM dict buffer as soon as the block is cut; only this compact summary + // (offset within the dict region + length + entry count + checksum) is kept to + // build the DICT block directory at finish_metadata time. The absolute file offset is + // computed as dict_region_offset + rel_offset. + struct BlockRecord { + uint64_t rel_offset = 0; // byte offset of this block within the dict region + uint64_t length = 0; // ON-DISK block length (compressed when flags&kZstd) + uint32_t n_entries = 0; + uint32_t checksum = 0; // crc32c of the UNCOMPRESSED block bytes + uint8_t flags = 0; // block_ref_flags::* (kZstd when block is compressed) + uint64_t uncomp_len = 0; // uncompressed block length (when flags&kZstd) + std::string first_term; + }; + + // Shared entry/exit of build() and the streamed session, extracted so the + // two paths are byte-identical BY CONSTRUCTION (not by parallel-maintained + // copies). prepare_build validates the sink/norms/CommonGrams identity and + // anchors the posting region; finalize_build re-checks the stats-dependent + // CommonGrams invariants, seals the dict buffer and materializes + // stats/norms/null-bitmap/BSBF. + Status prepare_build(io::FileWriter* posting_out); + Status finalize_build(); + // Iterates terms (from the streaming source or the materialized vector), + // splitting DICT blocks by target size and filling PODs + blocks_. + Status build_blocks(); + // Per-term driver shared by every producer. It validates the term, opens a + // block if needed, encodes it, and cuts the block at the target size. + struct BlockState; + Status process_term(StreamedTermPostings& tp, BlockState* st); + Status reserve_term_hash_for_append(); + // Region-relative byte count of the posting bytes written so far (the offset basis + // for frq_base/prx_base + frq_off_delta/prx_off_delta). During build() the only + // writes to posting_out_ are this index's posting region, so the count is the + // output offset advanced since the region began. + uint64_t posting_size() const { return posting_out_->bytes_written() - posting_off0_; } + // Serializes the current open block, streams its bytes into the dict scratch + // file, and records a compact directory entry (no block bytes retained). + Status flush_block(format::DictBlockBuilder* block, std::string first_term); + + uint64_t index_id_; + std::string index_suffix_; + format::IndexConfig index_config_; + format::IndexTier tier_; + bool has_prx_; + bool has_freq_; // tier >= T2: a freq region is encoded per window + bool has_norms_; + uint32_t doc_count_; + TrackedNullDocids null_docids_; + const std::vector& terms_; // materialized fallback (may be empty) + SpimiTermBuffer* term_source_; // streaming source (null => use terms_) + uint64_t term_count_ = 0; // distinct terms actually consumed + const std::vector& encoded_norms_; + std::optional common_grams_metadata_; + format::CommonGramsPostingPolicy common_grams_posting_policy_ = + format::CommonGramsPostingPolicy::kNone; + + uint32_t target_dict_block_bytes_; + // G16-h: zstd levels (dict whole-block / prx auto mode), from SniiIndexInput. + int dict_block_zstd_level_ = 3; + int prx_zstd_level_ = 3; + format::PrxWindowLimits prx_window_limits_ = format::kReaderPrxWindowLimits; + // The DICT region (zstd-compressed blocks) is staged here as blocks flush. It must + // land contiguously AFTER the posting region (which streams concurrently to the + // output), so it cannot stream directly; the orchestrator streams it into the + // container right after the posting region. It has NO independent local cap -- it + // spills to a temp via the writer's shared tracked-memory budget (the + // MemoryReporter from SniiIndexInput, null off-Doris). This reporter covers + // explicitly reserved structures; codec-local scratch remains governed by + // its format window limits rather than being represented as a whole-writer + // RSS hard cap. + MemoryReporter* memory_reporter_ = nullptr; + SpillableByteBuffer dict_buf_; + // The interleaved [prx][frq] posting region streams STRAIGHT into the container + // output during build() -- no temp. posting_out_ is the container writer (borrowed + // for the duration of build); posting_off0_ is its absolute offset when this index's + // region began, so posting_size() = bytes_written() - posting_off0_. + io::FileWriter* posting_out_ = nullptr; + uint64_t posting_off0_ = 0; + MemoryReporter::Reservation norms_section_reservation_; + std::vector norms_section_; + MemoryReporter::Reservation null_bitmap_section_reservation_; + std::vector null_bitmap_section_; + + std::vector blocks_; + MemoryReporter::Reservation term_hashes_reservation_; + MemoryReporter::Reservation bsbf_bytes_reservation_; + // One 8-byte XXH64 (seed 0) filter key per term, collected during the build pass + // so the whole-vocabulary string copy is never retained. + std::vector term_hashes_; + format::StatsBlock stats_; + std::vector bsbf_bytes_; // serialized block-split bloom XFilter section + bool bsbf_built_ = false; + + // Streamed-session state (T2.1). kIdle until build()/begin_streamed claims + // the writer; build() jumps straight to kFinished on completion so the two + // entry points can never interleave on one instance (single-session + // invariant). kFailed is the poison state: any failure PAST entry + // validation (process_term inside push_term, build_blocks inside build(), + // flush/finalize inside either finish path) may have left partial posting + // bytes in the sink or partial term state (term_hashes_/stats_), so the + // writer transitions to kFailed and every subsequent + // push/finish/build/begin is rejected -- a half-fed session can never + // masquerade as a sealed index (crash-safety invariant 6), and a dirty + // writer can never be re-claimed for a second session. Pure entry + // rejections (phase check, term-order guard, prepare_build validation) + // mutate nothing and therefore do NOT poison. + enum class StreamPhase : uint8_t { kIdle, kActive, kFinished, kFailed }; + StreamPhase stream_phase_ = StreamPhase::kIdle; + std::unique_ptr stream_state_; // live only while kActive + std::string last_pushed_term_; // strict term-order entry guard + bool has_pushed_term_ = false; + + friend class StreamingTermEncoder; +}; + +// TEST-ONLY observability seam (mirrors the reader-side decode-counter and the +// SPIMI vocab-materialization patterns). term_freq_scans() returns a +// process-global count of term-level fused freqs scans. Ordinary terms call +// fuse_freq_stats exactly once; complete CommonGrams entries call it zero times. +// note_term_freq_scan() bumps the counter; reset_term_freq_scans() zeroes it +// between tests; fuse_freq_stats_for_test() exposes the real fused helper so +// pure boundary tests exercise production code. Process-global; reset between +// tests. Not part of the production API. +namespace testing { +void note_term_freq_scan(); +uint64_t term_freq_scans(); +void reset_term_freq_scans(); +FreqStats fuse_freq_stats_for_test(const std::vector& freqs); +} // namespace testing + +} // namespace doris::snii::writer diff --git a/be/src/storage/index/snii/writer/memory_reporter.h b/be/src/storage/index/snii/writer/memory_reporter.h new file mode 100644 index 00000000000000..fdc4c1bcd2da6b --- /dev/null +++ b/be/src/storage/index/snii/writer/memory_reporter.h @@ -0,0 +1,259 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include + +#include "common/check.h" +#include "common/logging.h" +#include "common/status.h" + +namespace doris::snii::writer { + +// Per-WRITER accurate byte counter for build-time RAM (one per SniiCompoundWriter = +// one per segment's inverted index). Legacy modules report resident-byte deltas +// after their allocation; hard-gated modules own a Reservation that atomically +// pre-charges before allocation. current_bytes() is their shared live total. +// consume_release mirrors successful changes into the process-wide SNII +// index-build OBSERVATION tracker; it is null off-Doris (bench / unit tests), +// where only the local atomic is updated. +class MemoryReporter { +public: + // The callback may be invoked concurrently and from Reservation destructors; + // it must be thread-safe and must not throw. Null off-Doris. + // + // It must NEVER charge a MemTrackerLimiter: Doris's jemalloc allocation hook + // has already attributed these bytes to the MemTrackerLimiter attached to + // the allocating thread, so an explicit charge would count them twice. The + // only valid target is a plain MemTracker used for classified observation + // (see snii_build_consume_release). + using ConsumeReleaseFn = std::function; + + enum class CapPolicy : uint8_t { + // Reservations fail before an allocation would cross the cap. Native + // compaction uses this policy so an over-budget merge can fall back to + // the raw-column rebuild path without exceeding its bounded workspace. + kHardLimit, + // The cap is a spill trigger, not an allocation limit. Ordinary index + // ingestion uses this policy because persistent vocabulary structures + // can exceed the reclaimable posting-arena threshold by design. + kSpillThreshold, + }; + + // Move-only ownership of bytes pre-charged against this reporter. Growing a + // reservation atomically charges before allocation. Hard-limit reporters + // reject an over-cap charge without changing state; spill-threshold reporters + // retain exact accounting above the threshold. Callers must release/shrink the + // physical buffer before lowering the reservation. A Reservation borrows its + // reporter, which must outlive it. + class Reservation { + public: + Reservation() = default; + Reservation(const Reservation&) = delete; + Reservation& operator=(const Reservation&) = delete; + Reservation(Reservation&& other) noexcept; + Reservation& operator=(Reservation&& other) noexcept; + ~Reservation(); + + Status set_bytes(uint64_t target_bytes); + // Pre-charges an independent allocation while this Reservation keeps + // covering the old one. After the physical replacement succeeds, move + // `replacement` back into this Reservation to release the old charge. + Status prepare_replacement(uint64_t target_bytes, Reservation* replacement) const; + void reset(); + uint64_t bytes() const { return bytes_; } + + private: + friend class MemoryReporter; + explicit Reservation(MemoryReporter* owner) : owner_(owner) {} + + MemoryReporter* owner_ = nullptr; + uint64_t bytes_ = 0; + }; + + // cap_bytes is the shared gate-2 threshold (0 = unlimited). Hard-limit + // reporters reject reservations before their covered allocations cross it. + // Spill-threshold reporters keep exact accounting above it so over_cap() can + // drive reclaim without turning irreducible vocabulary growth into an import + // failure. + explicit MemoryReporter(ConsumeReleaseFn consume_release = nullptr, uint64_t cap_bytes = 0, + CapPolicy cap_policy = CapPolicy::kHardLimit) + : consume_release_(std::move(consume_release)), + cap_bytes_(cap_bytes), + cap_policy_(cap_policy) {} + + MemoryReporter(const MemoryReporter&) = delete; + MemoryReporter& operator=(const MemoryReporter&) = delete; + + // TERMINAL DRAIN. Reservations are RAII and balance themselves, but the + // legacy report() path is manual, so a caller that dies between report(+X) + // and report(-X) leaks X. That used to be harmless -- consume_release_ was + // null in production, so the residue died with this object's atomic. It is + // not harmless now: the mirrored bytes are a PROCESS-WIDE counter that also + // feeds the build-RAM decision, so a few MiB of residue per segment would + // become permanent phantom pressure on a long-lived BE. + // + // Warn rather than DCHECK. An unbalanced reporter is worth knowing about, + // but it is a legitimate end state for this observe-only type -- callers + // use report() for transient accounting they may abandon on an error path + // -- so aborting debug builds over it would be wrong. A warning also + // reaches RELEASE builds, which is where an unnoticed leak would actually + // accumulate; a DCHECK would not. + ~MemoryReporter() { + const int64_t remaining = current_.load(std::memory_order_relaxed); + if (remaining != 0 && consume_release_) { + LOG(WARNING) << "SNII MemoryReporter destroyed with " << remaining + << " unbalanced bytes; draining them so they do not accumulate in the " + << "process-wide index-build counter."; + consume_release_(-remaining); + } + } + + Reservation make_reservation() { return Reservation(this); } + + // Observe-only legacy path: delta > 0 grows, delta < 0 shrinks/frees. New + // hard-gated allocations must use Reservation instead. + void report(int64_t delta) { + if (delta == 0) return; + DCHECK_NE(delta, std::numeric_limits::min()); + int64_t current = current_.load(std::memory_order_relaxed); + while (true) { + DCHECK_GE(current, 0); + if (delta > 0) { + DCHECK_LE(delta, std::numeric_limits::max() - current); + } else { + DCHECK_GE(current, -delta); + } + const int64_t desired = current + delta; + if (current_.compare_exchange_weak(current, desired, std::memory_order_relaxed, + std::memory_order_relaxed)) { + if (consume_release_) consume_release_(delta); + return; + } + } + } + + int64_t current_bytes() const { return current_.load(std::memory_order_relaxed); } + + // True once all reported/reserved build RAM reaches the shared spill threshold. + bool over_cap() const { + const int64_t current = current_bytes(); + DCHECK_GE(current, 0); + return cap_bytes_ != 0 && static_cast(current) >= cap_bytes_; + } + uint64_t cap_bytes() const { return cap_bytes_; } + +private: + Status try_acquire(uint64_t bytes); + void release(uint64_t bytes); + + std::atomic current_ {0}; + ConsumeReleaseFn consume_release_; + uint64_t cap_bytes_ = 0; + CapPolicy cap_policy_ = CapPolicy::kHardLimit; +}; + +inline MemoryReporter::Reservation::Reservation(Reservation&& other) noexcept + : owner_(std::exchange(other.owner_, nullptr)), bytes_(std::exchange(other.bytes_, 0)) {} + +inline MemoryReporter::Reservation& MemoryReporter::Reservation::operator=( + Reservation&& other) noexcept { + if (this != &other) { + reset(); + owner_ = std::exchange(other.owner_, nullptr); + bytes_ = std::exchange(other.bytes_, 0); + } + return *this; +} + +inline MemoryReporter::Reservation::~Reservation() { + reset(); +} + +inline Status MemoryReporter::Reservation::set_bytes(uint64_t target_bytes) { + DORIS_CHECK(owner_ != nullptr); + if (target_bytes > bytes_) { + RETURN_IF_ERROR(owner_->try_acquire(target_bytes - bytes_)); + } else if (target_bytes < bytes_) { + owner_->release(bytes_ - target_bytes); + } + bytes_ = target_bytes; + return Status::OK(); +} + +inline Status MemoryReporter::Reservation::prepare_replacement(uint64_t target_bytes, + Reservation* replacement) const { + DORIS_CHECK(owner_ != nullptr); + DORIS_CHECK(replacement != nullptr); + DORIS_CHECK(replacement->owner_ == nullptr); + Reservation pending(owner_); + RETURN_IF_ERROR(pending.set_bytes(target_bytes)); + *replacement = std::move(pending); + return Status::OK(); +} + +inline void MemoryReporter::Reservation::reset() { + if (owner_ != nullptr && bytes_ != 0) { + owner_->release(bytes_); + bytes_ = 0; + } +} + +inline Status MemoryReporter::try_acquire(uint64_t bytes) { + if (bytes == 0) { + return Status::OK(); + } + int64_t current = current_.load(std::memory_order_relaxed); + while (true) { + DCHECK_GE(current, 0); + const uint64_t current_bytes = static_cast(current); + const bool exceeds_cap = cap_policy_ == CapPolicy::kHardLimit && cap_bytes_ != 0 && + (current_bytes > cap_bytes_ || bytes > cap_bytes_ - current_bytes); + const bool exceeds_counter = + bytes > static_cast(std::numeric_limits::max()) - current_bytes; + if (exceeds_cap || exceeds_counter) { + return Status::Error( + "SNII memory reservation exceeds limit: request={} current={} cap={}", bytes, + current_bytes, cap_bytes_); + } + const int64_t desired = current + static_cast(bytes); + if (current_.compare_exchange_weak(current, desired, std::memory_order_relaxed, + std::memory_order_relaxed)) { + if (consume_release_) { + consume_release_(static_cast(bytes)); + } + return Status::OK(); + } + } +} + +inline void MemoryReporter::release(uint64_t bytes) { + DCHECK_LE(bytes, static_cast(std::numeric_limits::max())); + const int64_t delta = static_cast(bytes); + const int64_t previous = current_.fetch_sub(delta, std::memory_order_relaxed); + DCHECK_GE(previous, delta); + if (consume_release_) { + consume_release_(-delta); + } +} + +} // namespace doris::snii::writer diff --git a/be/src/storage/index/snii/writer/posting_window_emitter.cpp b/be/src/storage/index/snii/writer/posting_window_emitter.cpp new file mode 100644 index 00000000000000..cef62b5e14f538 --- /dev/null +++ b/be/src/storage/index/snii/writer/posting_window_emitter.cpp @@ -0,0 +1,579 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/writer/posting_window_emitter.h" + +#include +#include +#include +#include +#include +#include + +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/encoding/byte_sink.h" +#include "storage/index/snii/format/dict_entry.h" +#include "storage/index/snii/format/format_constants.h" +#include "storage/index/snii/format/frq_pod.h" +#include "storage/index/snii/format/frq_prelude.h" +#include "storage/index/snii/io/file_writer.h" +#include "storage/index/snii/writer/spillable_byte_buffer.h" + +namespace doris::snii::writer { + +namespace { + +constexpr int kEmitterRawFrqRegion = 0; +constexpr uint32_t kPreludeGroupSize = 64; + +struct PostingWindowPlan { + size_t doc_begin = 0; + size_t doc_count = 0; + uint64_t position_begin = 0; + uint64_t position_count = 0; + uint32_t max_freq = 0; +}; + +bool emitter_fits_prx_window_shape(uint64_t doc_count, uint64_t position_count, + const format::PrxWindowLimits& limits) { + return doc_count <= limits.max_docs && position_count <= limits.max_positions; +} + +// Five bytes is the maximum encoded width of every uint32 field in the raw +// payload. This gate is used only after an exact build requests a split. +bool conservatively_fits_prx_window(uint64_t doc_count, uint64_t position_count, + const format::PrxWindowLimits& limits) { + return emitter_fits_prx_window_shape(doc_count, position_count, limits) && + 1 + doc_count + position_count <= limits.max_uncomp_bytes / 5; +} + +uint8_t window_max_norm(std::span norms, std::span docs) { + if (norms.empty() || docs.empty()) { + return 0; + } +#ifdef BE_TEST + testing::note_window_norm_doc_visits(docs.size()); +#endif + uint8_t best = 0xFF; + for (uint32_t docid : docs) { + DCHECK_LT(docid, norms.size()); + best = std::min(best, norms[docid]); + } + return best == 0xFF ? 0 : best; +} + +Status build_prelude(const std::vector& windows, bool has_freq, bool has_prx, + std::vector* output) { + format::FrqPreludeColumns columns; + columns.has_freq = has_freq; + columns.has_prx = has_prx; + columns.group_size = kPreludeGroupSize; + columns.windows = windows; + ByteSink sink; + RETURN_IF_ERROR(format::build_frq_prelude(columns, &sink)); + *output = sink.take(); + return Status::OK(); +} + +Status checked_add(uint64_t increment, uint64_t* value) { + if (increment > std::numeric_limits::max() - *value) { + return Status::Error( + "window emitter: term frequency overflow"); + } + *value += increment; + return Status::OK(); +} + +} // namespace + +class WindowEmitter::Impl { +public: + explicit Impl(WindowEmitterOptions options) + : options_(options), + dd_stager_(std::numeric_limits::max(), "term_dd", options.memory_reporter), + freq_stager_(std::numeric_limits::max(), "term_freq", + options.memory_reporter) { + if (options_.posting_out != nullptr && + options_.posting_out->bytes_written() >= options_.posting_region_offset) { + prx_off_ = options_.posting_out->bytes_written() - options_.posting_region_offset; + posting_offset_valid_ = true; + } + } + + Status emit_window(const PostingRunView& run) { + if (phase_ != Phase::kActive) { + return phase_error("emit_window"); + } + Status status = emit_window_impl(run); + if (!status.ok()) { + phase_ = Phase::kFailed; + } + return status; + } + + Status finish_term(format::DictEntry* entry, TermAggregateStats* stats) { + if (phase_ != Phase::kActive) { + return phase_error("finish_term"); + } + if (entry == nullptr || stats == nullptr) { + phase_ = Phase::kFailed; + return Status::Error( + "window emitter: null finish output"); + } + if (windows_.empty()) { + phase_ = Phase::kFailed; + return Status::Error( + "window emitter: cannot finish an empty term"); + } + Status status = finish_term_impl(entry); + if (!status.ok()) { + phase_ = Phase::kFailed; + return status; + } + *stats = stats_; + phase_ = Phase::kFinished; +#ifdef BE_TEST + finished_term_counter().fetch_add(1, std::memory_order_relaxed); +#endif + return Status::OK(); + } + +private: + enum class Phase : uint8_t { kActive, kFinished, kFailed }; + + Status phase_error(std::string_view operation) const { + return Status::Error( + "window emitter: {} after {}", operation, + phase_ == Phase::kFailed ? "failure" : "finish"); + } + + Status posting_size(uint64_t* size) const { + if (options_.posting_out == nullptr) { + return Status::Error( + "window emitter: null posting sink"); + } + if (!posting_offset_valid_ || + options_.posting_out->bytes_written() < options_.posting_region_offset) { + return Status::Error( + "window emitter: invalid posting region offset"); + } + *size = options_.posting_out->bytes_written() - options_.posting_region_offset; + return Status::OK(); + } + + Status validate_run(const PostingRunView& run) const { + if (options_.posting_out == nullptr) { + return Status::Error( + "window emitter: null posting sink"); + } + if (run.docids.empty()) { + return Status::Error( + "window emitter: empty posting window"); + } + if ((!run.freqs.empty() || options_.has_freq || options_.has_prx) && + run.freqs.size() != run.docids.size()) { + return Status::Error( + "window emitter: frequency shape must match documents"); + } + if (options_.term_frequency_source == TermFrequencySource::kPositions && + !options_.has_prx) { + return Status::Error( + "window emitter: position-derived statistics require PRX offsets"); + } + if (options_.has_prx) { + if (run.position_offsets.size() != run.docids.size() + 1) { + return Status::Error( + "window emitter: position offsets must have docs plus one entries"); + } + if (run.position_offsets.front() > run.position_offsets.back() || + run.position_offsets.back() - run.position_offsets.front() != + run.positions_flat.size()) { + return Status::Error( + "window emitter: position offsets differ from the position run"); + } + } else if (!run.position_offsets.empty() || !run.positions_flat.empty()) { + return Status::Error( + "window emitter: positions require a PRX term"); + } + if (last_input_docid_.has_value() && run.docids.front() <= *last_input_docid_) { + return Status::Error( + "window emitter: posting windows must be strictly ordered"); + } + return Status::OK(); + } + + Status accumulate_constant_stats(const PostingRunView& run) { + if (run.docids.size() > std::numeric_limits::max() - stats_.df) { + return Status::Error( + "window emitter: document frequency overflow"); + } + stats_.df += static_cast(run.docids.size()); + switch (options_.term_frequency_source) { + case TermFrequencySource::kDocuments: + return checked_add(run.docids.size(), &stats_.total_freq); + case TermFrequencySource::kPositions: + return checked_add(run.position_offsets.back() - run.position_offsets.front(), + &stats_.total_freq); + case TermFrequencySource::kFrequenciesOrDocuments: + if (run.freqs.empty()) { + return checked_add(run.docids.size(), &stats_.total_freq); + } + return Status::OK(); + } + __builtin_unreachable(); + } + + uint64_t position_count(const PostingRunView& run, size_t begin, size_t count) const { + if (!options_.has_prx) { + return 0; + } + return run.position_offsets[begin + count] - run.position_offsets[begin]; + } + + Status emit_window_impl(const PostingRunView& run) { + RETURN_IF_ERROR(validate_run(run)); + RETURN_IF_ERROR(accumulate_constant_stats(run)); + + const bool accumulate_frequencies = + options_.term_frequency_source == TermFrequencySource::kFrequenciesOrDocuments && + !run.freqs.empty(); + if (!options_.has_prx && !options_.has_freq && !accumulate_frequencies) { + RETURN_IF_ERROR( + emit_planned(run, make_plan(run, 0, run.docids.size(), /*max_freq=*/0))); + last_input_docid_ = run.docids.back(); + return Status::OK(); + } + + size_t window_begin = 0; + uint32_t window_max_freq = 0; + for (size_t doc = 0; doc < run.docids.size(); ++doc) { + const uint64_t document_positions = options_.has_prx ? position_count(run, doc, 1) : 0; + if (options_.has_prx && (run.position_offsets[doc + 1] < run.position_offsets[doc] || + document_positions != run.freqs[doc])) { + return Status::Error( + "window emitter: position offsets must match frequencies"); + } + if (options_.has_prx && document_positions > options_.prx_window_limits.max_positions) { + return Status::Error( + "window emitter: one document exceeds the PRX position limit"); + } + if (accumulate_frequencies) { + RETURN_IF_ERROR(checked_add(run.freqs[doc], &stats_.total_freq)); + stats_.max_freq = std::max(stats_.max_freq, run.freqs[doc]); + } + const uint64_t candidate_docs = doc - window_begin + 1; + const uint64_t candidate_positions = position_count(run, window_begin, candidate_docs); + if (doc != window_begin && options_.has_prx && + !emitter_fits_prx_window_shape(candidate_docs, candidate_positions, + options_.prx_window_limits)) { + RETURN_IF_ERROR(emit_planned( + run, make_plan(run, window_begin, doc - window_begin, window_max_freq))); + window_begin = doc; + window_max_freq = 0; + } + if (options_.has_freq) { +#ifdef BE_TEST + testing::note_window_freq_doc_visits(); +#endif + window_max_freq = std::max(window_max_freq, run.freqs[doc]); + } + } + RETURN_IF_ERROR(emit_planned( + run, + make_plan(run, window_begin, run.docids.size() - window_begin, window_max_freq))); + last_input_docid_ = run.docids.back(); + return Status::OK(); + } + + PostingWindowPlan make_plan(const PostingRunView& run, size_t begin, size_t count, + uint32_t max_freq) const { + return { + .doc_begin = begin, + .doc_count = count, + .position_begin = options_.has_prx ? run.position_offsets[begin] - + run.position_offsets.front() + : uint64_t {0}, + .position_count = position_count(run, begin, count), + .max_freq = max_freq, + }; + } + + Status emit_planned(const PostingRunView& run, const PostingWindowPlan& plan) { + format::PrxWindowBuildOutcome outcome = format::PrxWindowBuildOutcome::kBuilt; + RETURN_IF_ERROR(emit_physical_window(run, plan, &outcome)); + if (outcome == format::PrxWindowBuildOutcome::kBuilt) { + return Status::OK(); + } + + std::vector recut; + recut_window(run, plan, &recut); + for (const PostingWindowPlan& subplan : recut) { + outcome = format::PrxWindowBuildOutcome::kBuilt; + RETURN_IF_ERROR(emit_physical_window(run, subplan, &outcome)); + if (outcome == format::PrxWindowBuildOutcome::kNeedsSplit) { + return Status::Error( + "window emitter: one document exceeds the PRX byte limit"); + } + } + return Status::OK(); + } + + void recut_window(const PostingRunView& run, const PostingWindowPlan& input, + std::vector* output) const { + size_t window_begin = input.doc_begin; + uint32_t window_max_freq = 0; + const size_t input_end = input.doc_begin + input.doc_count; + for (size_t doc = input.doc_begin; doc < input_end; ++doc) { + const uint64_t candidate_docs = doc - window_begin + 1; + const uint64_t candidate_positions = position_count(run, window_begin, candidate_docs); + if (doc != window_begin && + !conservatively_fits_prx_window(candidate_docs, candidate_positions, + options_.prx_window_limits)) { + output->push_back( + make_plan(run, window_begin, doc - window_begin, window_max_freq)); + window_begin = doc; + window_max_freq = 0; + } + if (options_.has_freq) { +#ifdef BE_TEST + testing::note_window_freq_doc_visits(); +#endif + window_max_freq = std::max(window_max_freq, run.freqs[doc]); + } + } + output->push_back(make_plan(run, window_begin, input_end - window_begin, window_max_freq)); + } + + Status emit_physical_window(const PostingRunView& run, const PostingWindowPlan& plan, + format::PrxWindowBuildOutcome* outcome) { + const auto docs = run.docids.subspan(plan.doc_begin, plan.doc_count); + const auto freqs = run.freqs.empty() ? std::span {} + : run.freqs.subspan(plan.doc_begin, plan.doc_count); + format::WindowMeta window; + window.last_docid = docs.back(); + window.win_base = window_base_; + window.doc_count = static_cast(docs.size()); + window.max_freq = options_.has_freq ? plan.max_freq : 0; + window.max_norm = options_.has_freq ? window_max_norm(options_.encoded_norms, docs) : 0; + + if (options_.has_prx) { + const auto positions = + run.positions_flat.subspan(static_cast(plan.position_begin), + static_cast(plan.position_count)); + prx_scratch_.clear(); + RETURN_IF_ERROR(format::try_build_prx_window_flat( + positions, freqs, -options_.prx_zstd_level, options_.prx_window_limits, + &prx_scratch_, outcome)); + if (*outcome == format::PrxWindowBuildOutcome::kNeedsSplit) { + return Status::OK(); + } + window.prx_off = prx_total_len_; + window.prx_len = prx_scratch_.size(); + RETURN_IF_ERROR(options_.posting_out->append(prx_scratch_.view())); + prx_total_len_ += window.prx_len; + } else { + *outcome = format::PrxWindowBuildOutcome::kBuilt; + } + + ByteSink dd_sink; + format::FrqRegionMeta dd_meta; + window.dd_off = dd_stager_.size(); + RETURN_IF_ERROR(format::build_dd_region(docs, window_base_, kEmitterRawFrqRegion, &dd_sink, + &dd_meta)); + window.dd_zstd = dd_meta.zstd; + window.dd_disk_len = dd_meta.disk_len; + window.dd_uncomp_len = dd_meta.uncomp_len; + window.crc_dd = dd_meta.crc; + RETURN_IF_ERROR(dd_stager_.append_move(dd_sink.take())); + + if (options_.has_freq) { + ByteSink freq_sink; + format::FrqRegionMeta freq_meta; + window.freq_off = freq_stager_.size(); + RETURN_IF_ERROR( + format::build_freq_region(freqs, kEmitterRawFrqRegion, &freq_sink, &freq_meta)); + window.freq_zstd = freq_meta.zstd; + window.freq_disk_len = freq_meta.disk_len; + window.freq_uncomp_len = freq_meta.uncomp_len; + window.crc_freq = freq_meta.crc; + RETURN_IF_ERROR(freq_stager_.append_move(freq_sink.take())); + } + + windows_.push_back(window); + window_base_ = window.last_docid; +#ifdef BE_TEST + physical_window_counter().fetch_add(1, std::memory_order_relaxed); +#endif + return Status::OK(); + } + + Status finish_term_impl(format::DictEntry* entry) { + std::vector prelude; + RETURN_IF_ERROR(build_prelude(windows_, options_.has_freq, options_.has_prx, &prelude)); + entry->kind = format::DictEntryKind::kPodRef; + entry->enc = format::DictEntryEnc::kWindowed; + entry->has_sb = true; + entry->prelude_len = prelude.size(); + entry->frq_docs_len = entry->prelude_len + dd_stager_.size(); + + uint64_t frq_off = 0; + RETURN_IF_ERROR(posting_size(&frq_off)); + RETURN_IF_ERROR(options_.posting_out->append(Slice(prelude))); + RETURN_IF_ERROR(dd_stager_.seal()); + RETURN_IF_ERROR(dd_stager_.stream_into_and_release(options_.posting_out)); + RETURN_IF_ERROR(freq_stager_.seal()); + RETURN_IF_ERROR(freq_stager_.stream_into_and_release(options_.posting_out)); + entry->frq_off_delta = frq_off - options_.frq_base; + uint64_t end = 0; + RETURN_IF_ERROR(posting_size(&end)); + entry->frq_len = end - frq_off; + if (options_.has_prx) { + entry->prx_off_delta = prx_off_ - options_.prx_base; + entry->prx_len = prx_total_len_; + } + return Status::OK(); + } + +#ifdef BE_TEST + static std::atomic& finished_term_counter(); + static std::atomic& physical_window_counter(); +#endif + + WindowEmitterOptions options_; + SpillableByteBuffer dd_stager_; + SpillableByteBuffer freq_stager_; + std::vector windows_; + ByteSink prx_scratch_; + TermAggregateStats stats_; + std::optional last_input_docid_; + uint64_t prx_off_ = 0; + uint64_t prx_total_len_ = 0; + uint64_t window_base_ = 0; + bool posting_offset_valid_ = false; + Phase phase_ = Phase::kActive; +}; + +#ifdef BE_TEST +namespace { +std::atomic& window_norm_doc_visit_counter() { + static std::atomic counter {0}; + return counter; +} +std::atomic& window_freq_doc_visit_counter() { + static std::atomic counter {0}; + return counter; +} +std::atomic& emitter_finished_term_counter() { + static std::atomic counter {0}; + return counter; +} +std::atomic& emitter_physical_window_counter() { + static std::atomic counter {0}; + return counter; +} +} // namespace + +std::atomic& WindowEmitter::Impl::finished_term_counter() { + return emitter_finished_term_counter(); +} + +std::atomic& WindowEmitter::Impl::physical_window_counter() { + return emitter_physical_window_counter(); +} +#endif + +WindowEmitter::WindowEmitter(WindowEmitterOptions options) + : impl_(std::make_unique(options)) {} + +WindowEmitter::~WindowEmitter() = default; + +Status WindowEmitter::emit_window(const PostingRunView& window) { + return impl_->emit_window(window); +} + +Status WindowEmitter::finish_term(format::DictEntry* entry, TermAggregateStats* stats) { + return impl_->finish_term(entry, stats); +} + +namespace testing { + +void note_window_norm_doc_visits(uint64_t count) { +#ifdef BE_TEST + window_norm_doc_visit_counter().fetch_add(count, std::memory_order_relaxed); +#endif +} + +uint64_t window_norm_doc_visits() { +#ifdef BE_TEST + return window_norm_doc_visit_counter().load(std::memory_order_relaxed); +#else + return 0; +#endif +} + +void reset_window_norm_doc_visits() { +#ifdef BE_TEST + window_norm_doc_visit_counter().store(0, std::memory_order_relaxed); +#endif +} + +void note_window_freq_doc_visits() { +#ifdef BE_TEST + window_freq_doc_visit_counter().fetch_add(1, std::memory_order_relaxed); +#endif +} + +uint64_t window_freq_doc_visits() { +#ifdef BE_TEST + return window_freq_doc_visit_counter().load(std::memory_order_relaxed); +#else + return 0; +#endif +} + +void reset_window_freq_doc_visits() { +#ifdef BE_TEST + window_freq_doc_visit_counter().store(0, std::memory_order_relaxed); +#endif +} + +uint64_t window_emitter_finished_terms() { +#ifdef BE_TEST + return emitter_finished_term_counter().load(std::memory_order_relaxed); +#else + return 0; +#endif +} + +uint64_t window_emitter_physical_windows() { +#ifdef BE_TEST + return emitter_physical_window_counter().load(std::memory_order_relaxed); +#else + return 0; +#endif +} + +void reset_window_emitter_counters() { +#ifdef BE_TEST + emitter_finished_term_counter().store(0, std::memory_order_relaxed); + emitter_physical_window_counter().store(0, std::memory_order_relaxed); +#endif +} + +} // namespace testing + +} // namespace doris::snii::writer diff --git a/be/src/storage/index/snii/writer/posting_window_emitter.h b/be/src/storage/index/snii/writer/posting_window_emitter.h new file mode 100644 index 00000000000000..e6b0e3444b31fd --- /dev/null +++ b/be/src/storage/index/snii/writer/posting_window_emitter.h @@ -0,0 +1,112 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/format/prx_pod.h" + +namespace doris::snii { +namespace format { +struct DictEntry; +} +namespace io { +class FileWriter; +} +namespace writer { + +class MemoryReporter; + +// Synchronously borrowed posting arrays for one canonical posting window. +// position_offsets has docids.size()+1 entries when positions are present. The +// offsets may start above zero when the view borrows a sub-window; positions_flat +// covers exactly position_offsets.back()-position_offsets.front() values. +struct PostingRunView { + std::span docids; + std::span freqs; + std::span position_offsets; + std::span positions_flat; +}; + +struct TermAggregateStats { + uint32_t df = 0; + uint64_t total_freq = 0; + uint32_t max_freq = 0; +}; + +// CommonGrams entries define semantic term frequency from documents or +// positions rather than the transient physical frequency array. +enum class TermFrequencySource : uint8_t { + kFrequenciesOrDocuments, + kDocuments, + kPositions, +}; + +struct WindowEmitterOptions { + io::FileWriter* posting_out = nullptr; + uint64_t posting_region_offset = 0; + uint64_t frq_base = 0; + uint64_t prx_base = 0; + std::span encoded_norms; + bool has_freq = false; + bool has_prx = false; + int prx_zstd_level = 3; + format::PrxWindowLimits prx_window_limits = format::kReaderPrxWindowLimits; + TermFrequencySource term_frequency_source = TermFrequencySource::kFrequenciesOrDocuments; + MemoryReporter* memory_reporter = nullptr; +}; + +// The single owner of windowed DD/frequency/PRX encoding and prelude metadata. +// A failed emit poisons the instance; finish_term cannot publish a partial term. +class WindowEmitter { +public: + explicit WindowEmitter(WindowEmitterOptions options); + ~WindowEmitter(); + + WindowEmitter(const WindowEmitter&) = delete; + WindowEmitter& operator=(const WindowEmitter&) = delete; + WindowEmitter(WindowEmitter&&) = delete; + WindowEmitter& operator=(WindowEmitter&&) = delete; + + Status emit_window(const PostingRunView& window); + Status finish_term(format::DictEntry* entry, TermAggregateStats* stats); + +private: + class Impl; + std::unique_ptr impl_; +}; + +// Process-global test observability for the one emitter choke point and the +// existing bounded-work counters. Reset between tests. +namespace testing { +void note_window_norm_doc_visits(uint64_t count); +uint64_t window_norm_doc_visits(); +void reset_window_norm_doc_visits(); +void note_window_freq_doc_visits(); +uint64_t window_freq_doc_visits(); +void reset_window_freq_doc_visits(); +uint64_t window_emitter_finished_terms(); +uint64_t window_emitter_physical_windows(); +void reset_window_emitter_counters(); +} // namespace testing + +} // namespace writer +} // namespace doris::snii diff --git a/be/src/storage/index/snii/writer/snii_build_memory_tracker.cpp b/be/src/storage/index/snii/writer/snii_build_memory_tracker.cpp new file mode 100644 index 00000000000000..81b0c8347ad1ef --- /dev/null +++ b/be/src/storage/index/snii/writer/snii_build_memory_tracker.cpp @@ -0,0 +1,86 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/writer/snii_build_memory_tracker.h" + +#include + +#include "common/metrics/doris_metrics.h" +#include "common/metrics/metrics.h" +#include "runtime/memory/mem_tracker.h" + +namespace doris { +DEFINE_GAUGE_METRIC_PROTOTYPE_5ARG(snii_index_build_mem_consumption, MetricUnit::BYTES, "", + snii_index_build_mem_consumption, Labels({{"type", "load"}})); +} // namespace doris + +namespace doris::snii::writer { + +doris::MemTracker* snii_build_mem_tracker() { + // Intentionally leaked (never destroyed): MemoryReporters release their + // bytes from destructors that may run during static teardown at process + // exit, and a destroyed tracker there would be a use-after-free. Leaking + // makes the "tracker outlives every reporter" contract unconditional. + // + // The hook metric is registered exactly once, alongside the tracker, so the + // gauge exists as soon as anything charges it. It is never deregistered for + // the same reason the tracker is never destroyed. + static auto* tracker = [] { + auto* instance = new doris::MemTracker("SniiIndexBuild"); + REGISTER_HOOK_METRIC(snii_index_build_mem_consumption, + []() { return snii_build_mem_tracker()->consumption(); }); + return instance; + }(); + return tracker; +} + +namespace { +// The kRegistered subset of the tracker above -- see the header for why this is +// maintained directly instead of being derived by subtraction, and why it is a +// plain atomic rather than a second MemTracker. +std::atomic g_registered_build_bytes {0}; +} // namespace + +int64_t snii_registered_build_bytes() { + return g_registered_build_bytes.load(std::memory_order_relaxed); +} + +MemoryReporter::ConsumeReleaseFn snii_build_consume_release(BuildMemoryPopulation population) { + auto* tracker = snii_build_mem_tracker(); + // MemTracker and the subset counter are both thread-safe atomics, which is + // what MemoryReporter requires of this callback: it is invoked + // concurrently, from Reservation destructors, and must not throw. + // + // The registered path updates two of them, which is NOT atomic as a pair -- + // and does not need to be. The decision layer reads only + // g_registered_build_bytes; the tracker is observation. Because no consumer + // combines the two, there is no window in which they can disagree with each + // other in a way anything can observe. + const bool registered = population == BuildMemoryPopulation::kRegistered; + return [tracker, registered](int64_t delta) { + if (delta >= 0) { + tracker->consume(delta); + } else { + tracker->release(-delta); + } + if (registered) { + g_registered_build_bytes.fetch_add(delta, std::memory_order_relaxed); + } + }; +} + +} // namespace doris::snii::writer diff --git a/be/src/storage/index/snii/writer/snii_build_memory_tracker.h b/be/src/storage/index/snii/writer/snii_build_memory_tracker.h new file mode 100644 index 00000000000000..fd45c1b2784a85 --- /dev/null +++ b/be/src/storage/index/snii/writer/snii_build_memory_tracker.h @@ -0,0 +1,98 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "storage/index/snii/writer/memory_reporter.h" + +namespace doris { +class MemTracker; +} // namespace doris + +namespace doris::snii::writer { + +// Process-wide OBSERVATION tracker for SNII index-build RAM, the index-build +// analogue of MemTableMemoryLimiter's "AllMemTableMemory" tracker. +// +// WHAT IT IS NOT: it is NOT a MemTrackerLimiter and must never be turned into +// one. Doris charges these bytes AUTOMATICALLY -- the jemalloc allocation hook +// attributes every allocation on a thread to the MemTrackerLimiter attached by +// SCOPED_ATTACH_TASK (which is why the SNII IO pool threads attach at all). +// Enforcement and process accounting therefore already happen without SNII +// doing anything. Charging a MemTrackerLimiter here as well would count the +// same bytes twice. +// +// WHAT IT IS FOR: the hook knows only which THREAD allocated, so SNII's +// index-build memory is invisible as a category -- it disappears into whichever +// task tracker happened to be attached. This labelled MemTracker is a pure +// classified counter (thread-safe, no limit, cannot refuse an allocation) fed +// explicitly by every MemoryReporter, so index-build RAM shows up as its own +// line in the memory picture. +// +// IT IS NOT AN INPUT TO THE DECISION. The GlobalMemoryLimiter reads +// snii_registered_build_bytes() below, never this tracker. That separation is +// what makes the accounting safe under concurrency: charging a reporter touches +// this tracker AND (on the registered path) the counter below, which cannot be +// done atomically as a pair -- so the decision must never combine the two. It +// reads exactly one of them. +// +// Never destroyed: writers release bytes from destructors that can run at any +// point, including static teardown, so the tracker must outlive all of them. +doris::MemTracker* snii_build_mem_tracker(); + +// Which POPULATION a reporter's bytes belong to. The observation line is the +// same for both -- this only classifies whether a forced spill could ever +// reclaim any of these bytes, which is what the decision layer needs. +enum class BuildMemoryPopulation { + // Ingestion writers. Their SpimiTermBuffer registers with the + // GlobalMemoryLimiter and holds a reclaimable posting arena, so asking it + // to spill actually frees memory. + kRegistered, + // Index-merge compaction. It holds Reservation scratch only: it never + // constructs a SpimiTermBuffer and never registers, so the limiter has no + // lever over these bytes at all. Its own kHardLimit cap policy bounds them + // instead -- it refuses allocations, which a spill request cannot do. + kUnregistered, +}; + +// A MemoryReporter consume_release callback that mirrors the reporter's live +// bytes into the tracker above. Every production MemoryReporter is built with +// one; off-Doris users (benchmarks, unit tests) pass null and keep only the +// reporter's local atomic. The population must be stated explicitly: guessing +// it wrong either charges unreclaimable bytes to the wrong victims or hides +// reclaimable ones from the decision. +MemoryReporter::ConsumeReleaseFn snii_build_consume_release(BuildMemoryPopulation population); + +// Live bytes of the kRegistered population -- the memory a forced spill could +// actually reclaim, and the ONLY build-memory input to the decision layer. +// +// A plain atomic rather than a second MemTracker on purpose: it must not look +// like an independent line in the memory picture that a reader might add to the +// tracker's total. It is a subset of that total, maintained alongside it. +// +// Maintained DIRECTLY rather than derived by subtracting the unregistered +// population from the tracker. Deriving it would mean the decision reads two +// independent atomics that are updated in two separate steps, and a read landing +// between those steps tears: while a compaction grows, its bytes would briefly +// look reclaimable and could trigger a spill no writer needed; while it +// releases, reclaimable would be understated and could even go negative, +// skipping a spill that was needed. No memory order fixes that -- two atomics +// cannot be sampled as one snapshot. Reading a single counter has no such +// window. +int64_t snii_registered_build_bytes(); + +} // namespace doris::snii::writer diff --git a/be/src/storage/index/snii/writer/snii_compound_writer.cpp b/be/src/storage/index/snii/writer/snii_compound_writer.cpp new file mode 100644 index 00000000000000..5472ed71b31550 --- /dev/null +++ b/be/src/storage/index/snii/writer/snii_compound_writer.cpp @@ -0,0 +1,749 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/writer/snii_compound_writer.h" + +#include + +#include +#include + +#include "common/config.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/encoding/byte_sink.h" +#include "storage/index/snii/encoding/crc32c.h" +#include "storage/index/snii/format/bootstrap_header.h" +#include "storage/index/snii/format/core_metadata.h" +#include "storage/index/snii/format/metadata_directory.h" +#include "storage/index/snii/format/tail_pointer.h" +#include "storage/index/snii/reader/snii_segment_reader.h" + +namespace doris::snii::writer { + +using format::BootstrapHeader; +using format::LogicalIndexMetadataRef; +using format::SectionRefs; +using format::TailPointer; + +SniiCompoundWriter::SniiCompoundWriter(io::FileWriter* out) : out_(out) {} + +Status SniiCompoundWriter::poison(Status status) { + DCHECK(!status.ok()); + if (failed_.ok()) { + failed_ = std::move(status); + } + return failed_; +} + +Status SniiCompoundWriter::append(const std::vector& bytes) { + if (bytes.empty()) return Status::OK(); + return out_->append(Slice(bytes)); +} + +// The bootstrap header occupies offset 0 and must precede the first posting region, +// which streams straight into the output during build(). Written lazily exactly once +// (on the first add, or in finish() for an empty container). +Status SniiCompoundWriter::ensure_bootstrap() { + if (!failed_.ok()) return failed_; + if (bootstrap_written_) return Status::OK(); + const Status status = write_bootstrap(); + if (!status.ok()) return poison(status); + bootstrap_written_ = true; + return Status::OK(); +} + +Status SniiCompoundWriter::inherit(const reader::SniiRewriteSnapshot& snapshot, + io::FileReader* source) { + if (out_ == nullptr) { + return Status::Error("compound: null file writer"); + } + if (source == nullptr) { + return Status::Error("compound: null inherit source"); + } + if (!failed_.ok()) { + return failed_; + } + // Every rejection below poisons the writer. The caller asked for a container + // holding the inherited indexes; sealing one without them would silently drop + // logical indexes the target schema requires. + if (finished_) { + return poison( + Status::Error("compound: inherit after finish")); + } + if (inherited_prefix_) { + return poison(Status::Error( + "compound: inherit called twice; one source prefix is copied at most once")); + } + if (bootstrap_written_ || out_->bytes_written() != 0) { + return poison(Status::Error( + "compound: inherit must be the first data operation; the copied prefix owns the " + "front of the container")); + } + + // Sequential copy through a fixed-size buffer: peak memory is one chunk no + // matter how large the source container is. + const uint64_t prefix_end = snapshot.physical_prefix_end(); + std::vector chunk; + while (out_->bytes_written() < prefix_end) { + const auto chunk_size = static_cast( + std::min(kInheritCopyChunkBytes, prefix_end - out_->bytes_written())); + Status status = source->read_at(out_->bytes_written(), chunk_size, &chunk); + if (!status.ok()) { + return poison(status); + } + DORIS_CHECK_EQ(chunk.size(), chunk_size); + status = append(chunk); + if (!status.ok()) { + return poison(status); + } + } + // The copied prefix starts with the source's bootstrap header, which the + // snapshot validated, so the container must not get a second one. + bootstrap_written_ = true; + inherited_prefix_ = true; + + inherited_.reserve(snapshot.inherited().size()); + for (const reader::InheritedLogicalIndex& index : snapshot.inherited()) { + InheritedGroup group; + group.index_id = index.index_id; + group.index_suffix = index.index_suffix; + group.metadata_group = index.metadata_group; + group.core_length = index.core_length; + group.sampled_term_index_length = index.sampled_term_index_length; + group.dict_block_directory_length = index.dict_block_directory_length; + DORIS_CHECK_EQ(group.metadata_group.size(), group.core_length + + group.sampled_term_index_length + + group.dict_block_directory_length); + inherited_.push_back(std::move(group)); + } + return Status::OK(); +} + +Status SniiCompoundWriter::add_logical_index(const SniiIndexInput& in) { + if (out_ == nullptr) + return Status::Error("compound: null file writer"); + if (finished_) + return Status::Error("compound: add after finish"); + if (!failed_.ok()) return failed_; + if (has_active_session()) + return Status::Error( + "compound: add_logical_index while a streamed index session is active (its " + "posting region streams straight into the output; interleaving would corrupt " + "both indexes)"); + for (const InheritedGroup& group : inherited_) { + if (group.index_id == in.index_id && group.index_suffix == in.index_suffix) { + return poison(Status::Error( + "compound: new logical index reuses an inherited key; the final directory " + "must hold each key exactly once")); + } + } + for (const PendingBlobIndex& blob : blobs_) { + if (blob.index_id == in.index_id && blob.index_suffix == in.index_suffix) { + return poison(Status::Error( + "compound: new logical index reuses a registered blob index key")); + } + } + RETURN_IF_ERROR(ensure_bootstrap()); + auto liw = std::make_unique(in); + Placement p; + // The posting region streams DIRECTLY into the container during build() -- no temp + // round-trip for the bulk -- followed immediately by this index's compact DICT + // trailer (produced interleaved into a temp, but laid out right after its posting + // region, preserving the per-index [posting][dict] layout). Offsets are read off + // the output writer (the single source of truth -- no separate cursor). + p.post_off = out_->bytes_written(); + Status status = liw->build(out_); + if (!status.ok()) return poison(status); + p.post_len = out_->bytes_written() - p.post_off; + p.dict_off = out_->bytes_written(); + status = liw->stream_dict_region_into(out_); + if (!status.ok()) return poison(status); + p.dict_len = out_->bytes_written() - p.dict_off; + indexes_.push_back(std::move(liw)); + placements_.push_back(p); + // liw has been moved from; write_index_aux_sections works off indexes_.back(). + status = write_index_aux_sections(indexes_.size() - 1); + if (!status.ok()) return poison(status); + return Status::OK(); +} + +// Argument validation for one blob registration: the kind must be one this +// writer can emit, and the named-file table must be non-empty with unique, +// named, readable entries. Rejecting an unknown kind HERE (rather than letting +// the directory encoder catch it inside finish()) keeps a multi-GiB blob from +// being copied before the failure surfaces, and reports a caller bug as +// INVALID_ARGUMENT instead of an Unsupported format problem. +Status SniiCompoundWriter::validate_blob_registration( + format::LogicalIndexKind kind, const std::vector& cold_files, + const std::vector& hot_files) { + if (kind != format::LogicalIndexKind::kBkd && kind != format::LogicalIndexKind::kAnn) { + return Status::Error( + fmt::format("compound: add_blob_index got kind {}; text indexes go through " + "add_logical_index and other kinds are not emittable", + static_cast(kind))); + } + if (cold_files.empty() && hot_files.empty()) { + return Status::Error( + "compound: blob index registered without files"); + } + for (const std::vector* files : {&cold_files, &hot_files}) { + for (const BlobFileSource& file : *files) { + if (file.name.empty()) { + return Status::Error( + "compound: blob file with empty name"); + } + if (file.length > 0 && !file.read_fn) { + return Status::Error( + "compound: blob file without a read function"); + } + size_t seen = 0; + for (const std::vector* other : {&cold_files, &hot_files}) { + for (const BlobFileSource& candidate : *other) { + seen += candidate.name == file.name ? 1 : 0; + } + } + if (seen != 1) { + return Status::Error( + "compound: duplicate blob file name"); + } + } + } + return Status::OK(); +} + +Status SniiCompoundWriter::add_blob_index(uint64_t index_id, std::string index_suffix, + format::LogicalIndexKind kind, + std::vector cold_files, + std::vector hot_files) { + if (out_ == nullptr) { + return Status::Error("compound: null file writer"); + } + if (finished_) { + return Status::Error("compound: add after finish"); + } + if (!failed_.ok()) return failed_; + // Registration writes nothing, so it is legal while a streamed session is + // active. Rejections that only mean "bad arguments" leave the writer clean; + // key collisions poison it (see below). + RETURN_IF_ERROR(validate_blob_registration(kind, cold_files, hot_files)); + for (const PendingBlobIndex& blob : blobs_) { + if (blob.index_id == index_id && blob.index_suffix == index_suffix) { + return Status::Error( + "compound: blob index key registered twice"); + } + } + // A key collision against an already-registered index means the caller's plan + // cannot produce a valid container (the directory must hold each key exactly + // once, and MetadataDirectory::find would silently shadow one of the two). + // Poison rather than reject cleanly, mirroring inherit() and + // add_logical_index: sealing a container that silently omits an index the + // schema requires is never acceptable. Catching it HERE also keeps a + // multi-GiB blob from being copied before the failure surfaces, and reports + // it as the caller bug it is instead of Corruption from the encoder's + // self-check inside finish(). + for (const InheritedGroup& group : inherited_) { + if (group.index_id == index_id && group.index_suffix == index_suffix) { + return poison(Status::Error( + "compound: blob index reuses an inherited key")); + } + } + for (const std::unique_ptr& text : indexes_) { + if (text->index_id() == index_id && text->index_suffix() == index_suffix) { + return poison(Status::Error( + "compound: blob index reuses a text logical index key")); + } + } + PendingBlobIndex blob; + blob.index_id = index_id; + blob.index_suffix = std::move(index_suffix); + blob.kind = kind; + blob.cold_files = std::move(cold_files); + blob.hot_files = std::move(hot_files); + blobs_.push_back(std::move(blob)); + return Status::OK(); +} + +Status SniiCompoundWriter::write_blob_files(const std::vector& files, + std::vector* refs) { + std::vector chunk; + for (const BlobFileSource& file : files) { + format::NamedBlobFileRef ref; + ref.name = file.name; + ref.offset = out_->bytes_written(); + ref.length = file.length; + uint32_t crc = 0; + uint64_t copied = 0; + while (copied < file.length) { + const auto n = static_cast( + std::min(kBlobCopyChunkBytes, file.length - copied)); + chunk.resize(n); + RETURN_IF_ERROR(file.read_fn(copied, n, chunk.data())); + crc = crc32c_extend(crc, Slice(chunk.data(), n)); + RETURN_IF_ERROR(out_->append(Slice(chunk.data(), n))); + copied += n; + } + ref.crc32c = crc; + DORIS_CHECK_EQ(out_->bytes_written(), ref.offset + ref.length); + refs->push_back(std::move(ref)); + } + return Status::OK(); +} + +void SniiCompoundWriter::release_blob_sources(std::vector* files) { + DORIS_CHECK(files != nullptr); + // swap-with-empty, not clear(): a source's read_fn may own the bytes it + // serves, and clear() would leave the vector's capacity -- and, for a + // std::function, nothing at all is guaranteed to be freed until the elements + // themselves are destroyed. + std::vector released; + files->swap(released); +} + +SniiIndexInput SniiStreamedIndexSession::attach_encoded_norms(SniiIndexInput in, + TrackedEncodedNorms* encoded_norms, + uint64_t reserved_bytes) { + DORIS_CHECK(encoded_norms != nullptr); + DORIS_CHECK(in.encoded_norms.empty()); + if (in.mem_reporter != nullptr) { + DORIS_CHECK_EQ(reserved_bytes, encoded_norms->norms_.capacity()); + } else { + DORIS_CHECK_EQ(reserved_bytes, 0); + } + in.encoded_norms = std::move(encoded_norms->norms_); + return in; +} + +SniiStreamedIndexSession::SniiStreamedIndexSession(SniiCompoundWriter* owner, SniiIndexInput in, + TrackedNullDocids null_docids, + TrackedEncodedNorms encoded_norms) + : owner_(owner), + encoded_norms_reservation_(std::move(encoded_norms.reservation_)), + input_(attach_encoded_norms(std::move(in), &encoded_norms, + encoded_norms_reservation_.bytes())), + // input_ (a member, initialized above) owns the vectors the writer + // keeps references into -- NOT the caller's already-moved-from `in`. + writer_(new LogicalIndexWriter(input_, std::move(null_docids))), + semantic_token_count_required_( + input_.common_grams_metadata.has_value() && + input_.common_grams_metadata->scoring_coverage == + segment_v2::inverted_index::ScoringCoverage::kComplete) {} + +Status SniiStreamedIndexSession::push_term(StreamedTermPostings&& tp) { + if (!owner_->failed_.ok()) return owner_->failed_; + if (finished_) { + return Status::Error( + "compound: push_term on a finished streamed index session"); + } + const Status status = writer_->push_term(std::move(tp)); + if (!status.ok()) return owner_->poison(status); + return Status::OK(); +} + +Status SniiStreamedIndexSession::set_semantic_token_count(uint64_t token_count) { + if (!owner_->failed_.ok()) return owner_->failed_; + if (finished_) { + return Status::Error( + "compound: semantic token count on a finished streamed index session"); + } + if (!semantic_token_count_required_) { + return Status::Error( + "compound: streamed index session has no complete semantic scoring metadata"); + } + if (semantic_token_count_set_) { + return Status::Error( + "compound: semantic token count was already set"); + } + DORIS_CHECK(input_.common_grams_metadata.has_value()); + DORIS_CHECK(writer_->common_grams_metadata_.has_value()); + input_.common_grams_metadata->scoring_token_count = token_count; + writer_->common_grams_metadata_->scoring_token_count = token_count; + semantic_token_count_set_ = true; + return Status::OK(); +} + +Status SniiStreamedIndexSession::finish() { + if (!owner_->failed_.ok()) return owner_->failed_; + if (finished_) { + return Status::Error( + "compound: finish on an already-finished streamed index session"); + } + if (semantic_token_count_required_ && !semantic_token_count_set_) { + return owner_->poison(Status::Error( + "compound: semantic token count must be set before streamed index finish")); + } + return owner_->finish_streamed_index(this); +} + +void SniiStreamedIndexSession::abort(const Status& cause) { + DCHECK(!cause.ok()); + static_cast(owner_->poison(cause)); +} + +Status SniiCompoundWriter::begin_streamed_index(SniiIndexInput in, + SniiStreamedIndexSession** session) { + std::vector null_docids; + null_docids.swap(in.null_docids); + MemoryReporter::Reservation null_docids_reservation = + in.mem_reporter == nullptr ? MemoryReporter::Reservation() + : in.mem_reporter->make_reservation(); + if (in.mem_reporter != nullptr) { + RETURN_IF_ERROR(null_docids_reservation.set_bytes( + static_cast(null_docids.capacity()) * sizeof(uint32_t))); + } + return begin_streamed_index( + std::move(in), + TrackedNullDocids(std::move(null_docids_reservation), std::move(null_docids)), session); +} + +Status SniiCompoundWriter::begin_streamed_index(SniiIndexInput in, TrackedNullDocids null_docids, + SniiStreamedIndexSession** session) { + std::vector encoded_norms; + encoded_norms.swap(in.encoded_norms); + MemoryReporter::Reservation encoded_norms_reservation = + in.mem_reporter == nullptr ? MemoryReporter::Reservation() + : in.mem_reporter->make_reservation(); + if (in.mem_reporter != nullptr) { + RETURN_IF_ERROR(encoded_norms_reservation.set_bytes(encoded_norms.capacity())); + } + return begin_streamed_index( + std::move(in), std::move(null_docids), + TrackedEncodedNorms(std::move(encoded_norms_reservation), std::move(encoded_norms)), + session); +} + +Status SniiCompoundWriter::begin_streamed_index(SniiIndexInput in, TrackedNullDocids null_docids, + TrackedEncodedNorms encoded_norms, + SniiStreamedIndexSession** session) { + if (session == nullptr) + return Status::Error( + "compound: null session out parameter"); + *session = nullptr; + if (out_ == nullptr) + return Status::Error("compound: null file writer"); + if (finished_) + return Status::Error("compound: begin after finish"); + if (!failed_.ok()) return failed_; + if (has_active_session()) + return Status::Error( + "compound: a streamed index session is already active (one at a time: its " + "posting region streams straight into the container output)"); + // A streamed session takes terms ONLY via push_term; a term source or a + // materialized vector would silently be ignored by the streamed writer. + if (in.term_source != nullptr || !in.terms.empty()) + return Status::Error( + "compound: a streamed index session must not carry a term source or " + "materialized terms"); + if (!in.null_docids.empty()) + return Status::Error( + "compound: tracked streamed NULL docids must not also be present in input"); + if (!in.encoded_norms.empty()) + return Status::Error( + "compound: tracked streamed norms must not also be present in input"); + RETURN_IF_ERROR(ensure_bootstrap()); + auto s = std::unique_ptr(new SniiStreamedIndexSession( + this, std::move(in), std::move(null_docids), std::move(encoded_norms))); + s->post_off_ = out_->bytes_written(); + RETURN_IF_ERROR(s->writer_->begin_streamed(out_)); + sessions_.push_back(std::move(s)); + *session = sessions_.back().get(); + return Status::OK(); +} + +Status SniiCompoundWriter::finish_streamed_index(SniiStreamedIndexSession* session) { + // Flushes the trailing DICT block and finalizes the stats / null-bitmap / + // BSBF sections (poisoning the logical writer on failure). + Status status = session->writer_->finish_streamed(); + if (!status.ok()) return poison(status); + // finalize materialized the framed norms section. Drop the source vector and + // its transferred charge before retaining that section for compound finish. + std::vector().swap(session->input_.encoded_norms); + session->encoded_norms_reservation_.reset(); + Placement p; + p.post_off = session->post_off_; + p.post_len = out_->bytes_written() - p.post_off; + p.dict_off = out_->bytes_written(); + status = session->writer_->stream_dict_region_into(out_); + if (!status.ok()) return poison(status); + p.dict_len = out_->bytes_written() - p.dict_off; + // The index joins the container (indexes_/placements_) here, but session->finished_ + // is not set until write_index_aux_sections below also succeeds. A failure ANYWHERE + // in this function -- finish_streamed()/stream_dict_region_into() above, or + // write_index_aux_sections below -- calls poison(), which sets failed_ before + // returning. finish() checks "if (!failed_.ok()) return failed_;" ahead of its + // has_active_session() gate, so a poisoned writer fails loudly on its own; it can + // never fall through to sealing a tail that silently omits an index whose posting + // bytes are already in the file. + indexes_.push_back(std::move(session->writer_)); + placements_.push_back(p); + status = write_index_aux_sections(indexes_.size() - 1); + if (!status.ok()) return poison(status); + session->finished_ = true; + return Status::OK(); +} + +Status SniiCompoundWriter::write_bootstrap() { + BootstrapHeader bh; + bh.tail_pointer_size = static_cast(format::tail_pointer_size()); + ByteSink sink; + RETURN_IF_ERROR(format::encode_bootstrap_header(bh, &sink)); + return append(sink.buffer()); +} + +// Writes one index's norms / null bitmap / bsbf directly after its [posting][dict] pair. +// Bytes are released as soon as they are on disk rather than being held until finish(), +// which also lowers import peak memory -- a content column's bsbf runs to MBs. +Status SniiCompoundWriter::write_index_aux_sections(size_t index) { + DORIS_CHECK_LT(index, indexes_.size()); + DORIS_CHECK_LT(index, placements_.size()); + LogicalIndexWriter& w = *indexes_[index]; + Placement& p = placements_[index]; + + if (w.has_norms() && !w.norms_bytes().empty()) { + p.norms_off = out_->bytes_written(); + RETURN_IF_ERROR(append(w.norms_bytes())); + p.norms_len = out_->bytes_written() - p.norms_off; + w.release_norms_bytes(); + } + if (w.has_null_bitmap()) { + p.null_off = out_->bytes_written(); + RETURN_IF_ERROR(append(w.null_bitmap_bytes())); + p.null_len = out_->bytes_written() - p.null_off; + w.release_null_bitmap_bytes(); + } + if (w.has_bsbf()) { + p.bsbf_off = out_->bytes_written(); + RETURN_IF_ERROR(append(w.bsbf_bytes())); + p.bsbf_len = out_->bytes_written() - p.bsbf_off; + w.release_bsbf_bytes(); + } + return Status::OK(); +} + +// Streams every registered blob's HOT files -- after all text metadata groups, +// physically adjacent within each entry, so a future open can fetch an entry's +// hot set with one range read (mirroring Core/STI/DBD adjacency) -- then appends +// one directory entry per blob holding its cold refs followed by its hot refs. +Status SniiCompoundWriter::write_blob_hot_files_and_entries( + std::vector* directory_entries) { + for (PendingBlobIndex& blob : blobs_) { + RETURN_IF_ERROR(write_blob_files(blob.hot_files, &blob.hot_refs)); + release_blob_sources(&blob.hot_files); // see the cold loop in finish() + } + for (PendingBlobIndex& blob : blobs_) { + LogicalIndexMetadataRef entry; + entry.index_id = blob.index_id; + entry.index_suffix = blob.index_suffix; + entry.kind = blob.kind; + entry.files.reserve(blob.cold_refs.size() + blob.hot_refs.size()); + entry.files.insert(entry.files.end(), blob.cold_refs.begin(), blob.cold_refs.end()); + entry.files.insert(entry.files.end(), blob.hot_refs.begin(), blob.hot_refs.end()); + directory_entries->push_back(std::move(entry)); + } + return Status::OK(); +} + +Status SniiCompoundWriter::write_tail() { + std::vector directory_entries; + directory_entries.reserve(inherited_.size() + indexes_.size() + blobs_.size()); + // Inherited metadata groups are re-emitted verbatim: their section references + // already point into the copied prefix, which landed at identical offsets, so + // no posting is decoded or re-encoded. Only the group's own position moves. + for (const InheritedGroup& group : inherited_) { + LogicalIndexMetadataRef entry; + entry.index_id = group.index_id; + entry.index_suffix = group.index_suffix; + const uint64_t core_offset = out_->bytes_written(); + entry.core_metadata = {.offset = core_offset, .length = group.core_length}; + entry.sampled_term_index = {.offset = core_offset + group.core_length, + .length = group.sampled_term_index_length}; + entry.dict_block_directory = { + .offset = core_offset + group.core_length + group.sampled_term_index_length, + .length = group.dict_block_directory_length}; + RETURN_IF_ERROR(append(group.metadata_group)); + DORIS_CHECK_EQ(out_->bytes_written(), core_offset + group.metadata_group.size()); + directory_entries.push_back(std::move(entry)); + } + for (size_t i = 0; i < indexes_.size(); ++i) { + const LogicalIndexWriter& w = *indexes_[i]; + const Placement& p = placements_[i]; + + SectionRefs refs; + refs.dict_region = {.offset = p.dict_off, .length = p.dict_len}; + refs.posting_region = {.offset = p.post_off, .length = p.post_len}; + refs.norms = {.offset = p.norms_off, .length = p.norms_len}; + refs.null_bitmap = {.offset = p.null_off, .length = p.null_len}; + refs.bsbf = {.offset = p.bsbf_off, .length = p.bsbf_len}; + + SerializedMetadataGroup group; + RETURN_IF_ERROR(w.finish_metadata(refs, p.dict_off, &group)); + + LogicalIndexMetadataRef entry; + entry.index_id = w.index_id(); + entry.index_suffix = w.index_suffix(); + entry.core_metadata = {.offset = out_->bytes_written(), .length = group.core.size()}; + RETURN_IF_ERROR(append(group.core)); + DORIS_CHECK_EQ(out_->bytes_written(), + entry.core_metadata.offset + entry.core_metadata.length); + + entry.sampled_term_index = {.offset = out_->bytes_written(), + .length = group.sampled_term_index.size()}; + DORIS_CHECK_EQ(entry.sampled_term_index.offset, + entry.core_metadata.offset + entry.core_metadata.length); + RETURN_IF_ERROR(append(group.sampled_term_index)); + DORIS_CHECK_EQ(out_->bytes_written(), + entry.sampled_term_index.offset + entry.sampled_term_index.length); + + entry.dict_block_directory = {.offset = out_->bytes_written(), + .length = group.dict_block_directory.size()}; + DORIS_CHECK_EQ(entry.dict_block_directory.offset, + entry.sampled_term_index.offset + entry.sampled_term_index.length); + RETURN_IF_ERROR(append(group.dict_block_directory)); + DORIS_CHECK_EQ(out_->bytes_written(), + entry.dict_block_directory.offset + entry.dict_block_directory.length); + directory_entries.push_back(std::move(entry)); + } + + RETURN_IF_ERROR(write_blob_hot_files_and_entries(&directory_entries)); + + ByteSink directory_sink; + RETURN_IF_ERROR(format::encode_metadata_directory(directory_entries, &directory_sink)); + const uint64_t directory_offset = out_->bytes_written(); + RETURN_IF_ERROR(append(directory_sink.buffer())); + const uint64_t directory_length = out_->bytes_written() - directory_offset; + + TailPointer tp; + tp.directory_offset = directory_offset; + tp.directory_length = directory_length; + tp.directory_crc32c = crc32c(directory_sink.view()); + ByteSink tail_sink; + RETURN_IF_ERROR(format::encode_tail_pointer(tp, &tail_sink)); + + // Pad the container up to a file-cache block boundary, but only when that padding is small + // relative to the container. + // + // Padding at all: s_align_size clamps the aligned window to the file end, and back-pads by a + // whole block when the clamp leaves it short (io/cache/cached_remote_file_reader.cpp). Ending + // on a boundary makes that condition false, so a read confined to the final block fetches one + // block instead of two. + // + // Only sometimes: the back-pad costs nothing when the query already fetches the preceding + // block for other reasons. So the saving is one-shot and bounded (at most last_partial per + // container) while the cost -- filler that every tail read pulls in -- scales with how much of + // the container a query touches. Both signs are measured, on the same wikipedia corpus: + // + // 53-62 MiB containers, pad 0.09%-1.28%: -2.2% bytes fetched over a 13-case sweep, + // -21% when the sweep is one targeted case + // ~4 MiB containers, pad 13.3%: the sweep reads nearly the whole container, so the back-pad + // was already free and the filler is pure addition -> +13.3% + // + // kMinPaddingLeverage is a judgement call, not a derived constant: it admits the measured + // 1.28% case with margin and rejects the 13.3% one. + // + // Expressed as a floor on the CONTAINER rather than a ratio on the padding. Since pad < block + // the two bound the cost identically, but the floor cannot overflow, and it keeps containers + // small enough to be PACKED out of the deal: in cloud mode a container below + // cloud::config::small_file_threshold_bytes (1 MiB) is appended into a shared object at an + // arbitrary offset (RowsetWriterContext wraps the fs in io::PackedFileSystem), where + // s_align_size works in packed coordinates and aligning the sub-file buys nothing at all. + // A 32-block floor clears that threshold by 32x. + // + // The padding goes BEFORE the tail pointer, which must stay the last thing in the file for the + // reader to find it. + // + // Caveat: the block size is read at WRITE time but the saving is realised at READ time. If a + // deployment changes file_cache_each_block_size afterwards, nothing breaks and no index becomes + // unreadable -- but the outcome is not symmetric, so "neutral" would be the wrong word. + // SHRINKING it is safe when the new size divides the old (256 KiB into 1 MiB keeps alignment). + // GROWING it (1 MiB -> 4 MiB, a normal S3 throughput tuning move) brings the back-pad back + // while the filler bytes stay on disk: strictly worse than never having padded, until + // compaction rewrites the container. + // + // Gated on enable_file_cache because the saving is realised only by CachedRemoteFileReader. + // That flag defaults to FALSE; without this check a storage-compute-coupled or local-filesystem + // deployment appends up to a block of zeros per container and never reads through a block cache + // at all. (exec_env_init only validates file_cache_each_block_size when the cache is on, so in + // that configuration the value here would also be entirely unvalidated.) + const int64_t block = config::file_cache_each_block_size; + if (config::enable_file_cache && block > 0) { + const uint64_t unpadded = out_->bytes_written() + tail_sink.buffer().size(); + const auto block_size = static_cast(block); + const uint64_t pad = (block_size - unpadded % block_size) % block_size; + // 2*pad < block is the cost/benefit test itself, and it is exact rather than a proxy. + // Measured in §6.7.1 of the design doc: the saving equals sum(last_partial) (2,290,341 vs + // 2,290,330 predicted) and the cost equals the bytes added to disk (+45,207,390 written vs + // +45,199,729 fetched). Since last_partial + pad == block, benefit = block - pad and + // cost ~ pad -- perfectly anticorrelated, and NEITHER depends on the container size. A + // 40 MiB container that overshoots a boundary by 100 B has last_partial = 100 and + // pad = 1,048,476: it clears any size-based gate while saving 100 bytes for a megabyte. + // On this project's own four measured containers the extra term trades 29% of the saving + // for a 76% cut in padding written (1.20:1 -> 3.55:1 benefit:cost). + if (pad > 0 && 2 * pad < block_size && unpadded / block_size >= kMinPaddingLeverage) { + // Never referenced by any SectionRef, so no reader ever reads it. + const std::vector filler(pad, 0); + RETURN_IF_ERROR(append(filler)); + } + } + + RETURN_IF_ERROR(append(tail_sink.buffer())); + return Status::OK(); +} + +Status SniiCompoundWriter::finish() { + if (out_ == nullptr) + return Status::Error("compound: null file writer"); + if (!failed_.ok()) { + return failed_; + } + if (finished_) + return Status::Error("compound: finish called twice"); + // Crash-safety invariant 6: a begun-but-unfinished streamed session already + // streamed posting bytes into the file but recorded no placement; sealing + // the container now would silently drop that index. Fail loudly -- the + // whole compaction round must fail and be retried. + if (has_active_session()) + return Status::Error( + "compound: finish with an unfinished streamed index session; the half-fed " + "index must never be sealed away silently"); + finished_ = true; + + RETURN_IF_ERROR(ensure_bootstrap()); // empty container still gets a header + // Aux sections were written per index at add/finish_streamed time, right after each + // index's [posting][dict] pair -- see write_index_aux_sections. + // Blob COLD files follow all text physical sections and precede the first + // metadata group, in registration order. + Status status; + for (PendingBlobIndex& blob : blobs_) { + status = write_blob_files(blob.cold_files, &blob.cold_refs); + if (!status.ok()) return poison(status); + // The sources are dead the instant their bytes are in the container and + // their extents are in cold_refs -- and a source can OWN its bytes (the + // ANN staging directory hands over shared buffers holding a whole faiss + // index), so holding the vector until this writer is destroyed would pin + // that memory across every remaining blob and, because a rowset build + // keeps one writer per segment alive until every segment has been closed, + // across every segment of the rowset. Released per blob rather than after + // the loop so a multi-blob container never holds two at once. + release_blob_sources(&blob.cold_files); + } + status = write_tail(); + if (!status.ok()) return poison(status); + status = out_->finalize(); + if (!status.ok()) return poison(status); + return Status::OK(); +} + +} // namespace doris::snii::writer diff --git a/be/src/storage/index/snii/writer/snii_compound_writer.h b/be/src/storage/index/snii/writer/snii_compound_writer.h new file mode 100644 index 00000000000000..8575bab5829a53 --- /dev/null +++ b/be/src/storage/index/snii/writer/snii_compound_writer.h @@ -0,0 +1,344 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/format/metadata_directory.h" +#include "storage/index/snii/io/file_reader.h" +#include "storage/index/snii/io/file_writer.h" +#include "storage/index/snii/writer/logical_index_writer.h" + +namespace doris::snii::reader { +// Only ever named by reference below, so a declaration is enough. Including +// snii_segment_reader.h here would instead splice the whole reader into this +// header's dependents -- and this header sits upstream of runtime/exec_env.h, +// so that is most of the backend. +class SniiRewriteSnapshot; +} // namespace doris::snii::reader + +// SniiCompoundWriter -- orchestrates a single-segment SNII container for one or +// more logical indexes, written front-to-back through an append-only +// io::FileWriter (no seek-back). It resolves all back-references by writing the +// metadata groups, raw directory, and fixed tail pointer LAST. +// +// CONTAINER LAYOUT PRODUCED (this is the on-disk contract the reader matches): +// [bootstrap_header] (kBootstrapHeaderSize bytes) +// for each logical index, in add order: +// [posting region] interleaved [prx][frq] per pod_ref term, term order +// (prx span empty when !has_prx) +// [DICT blocks region] concatenated DICT blocks, split by +// target_dict_block_bytes +// for each logical index, in add order: +// [norms POD] NormsPodWriter::finish (scoring only; else absent) +// [null bitmap POD] NullBitmapWriter::finish (when nulls exist) +// for each logical index, in add order: +// [Core metadata][SampledTermIndex blob][DICT block directory blob] +// [metadata directory] raw SniiMetadataDirectoryPB bytes +// [block padding] OPTIONAL run of zero bytes, see write_tail(). Written only for +// containers of >= kMinPaddingLeverage cache blocks, and only when the +// file cache is on, so that the container ends on a block boundary. +// Referenced by nothing and read by nobody -- but it means the metadata +// directory is NOT necessarily adjacent to the tail pointer, and that +// container length is not a pure function of the indexed content (two +// BEs with different file_cache_each_block_size produce different +// lengths for identical input). +// [tail_pointer] encode_tail_pointer at EOF +// +// (The posting region is streamed BEFORE the DICT region per index: postings are +// the large append-only term-ordered stream; the DICT region is the compact +// compressed trailer.) +// +// OFFSET CONVENTIONS (ABSOLUTE file offsets unless stated otherwise): +// - SectionRefs in each Core metadata record ABSOLUTE file offset+length of +// that index's posting, DICT, norms, null-bitmap, and BSBF regions. Absent +// regions are (0,0); a present-but-empty posting region (all-INLINE index) +// is (off, 0). +// - DictBlockDirectory entries record each DICT block's ABSOLUTE file offset + +// length. +// - A windowed/slim pod_ref entry's absolute .frq offset = +// section_refs.posting_region.offset + frq_base + frq_off_delta +// where frq_base is the posting-region-relative running offset captured at the +// block's open (see logical_index_writer.h). prx follows the identical rule +// against the SAME region (prx_base == frq_base). +// - tail_pointer.directory_offset/length point at the raw metadata directory. +namespace doris::snii::writer { + +// A container is padded up to a file-cache block boundary only when it spans at least this many +// blocks -- see the reasoning at the padding site in write_tail(). Exposed so tests can size their +// fixtures against the real threshold instead of a copy of it. +inline constexpr uint64_t kMinPaddingLeverage = 32; + +class SniiCompoundWriter; + +// One opaque sub-file of a blob logical index, registered by add_blob_index. +// `read_fn` MUST fill exactly `len` bytes at blob-relative `offset` into `out` +// or return an error -- a short read reported as OK would be checksummed and +// sealed as if it were the real payload, since the crc is computed over the +// same buffer this call fills. +// +// The signature is PURE Status by design: the future Doris adapter that wraps a +// staged third-party index directory is required to convert that library's +// exceptions into Status BEFORE calling in, because the snii core has no +// try/catch on the sealing path (an escaping exception would skip poison()) and +// takes no third-party index-library dependency (a guard test enforces this). +struct BlobFileSource { + std::string name; + uint64_t length = 0; + std::function read_fn; +}; + +// T2.2 (compaction index merge fast path): handle for ONE streamed logical-index +// session inside a SniiCompoundWriter, obtained from begin_streamed_index(). The +// caller pushes lexicographically sorted terms (the k-way merge output) and seals +// the index with finish(), which lays the [posting][dict] regions out exactly like +// add_logical_index. The handle is owned by the compound writer and stays valid +// for the writer's lifetime; it must not outlive it. +// +// CRASH SAFETY (invariant 6): a session that was begun but never successfully +// finished keeps the container permanently unsealable -- its posting bytes are +// already in the file, so SniiCompoundWriter::finish() fails loudly instead of +// writing a tail that silently omits the half-fed index. +class SniiStreamedIndexSession { +public: + SniiStreamedIndexSession(const SniiStreamedIndexSession&) = delete; + SniiStreamedIndexSession& operator=(const SniiStreamedIndexSession&) = delete; + SniiStreamedIndexSession(SniiStreamedIndexSession&&) = delete; + SniiStreamedIndexSession& operator=(SniiStreamedIndexSession&&) = delete; + + // Terms must arrive in strictly increasing lexicographic order with + // ascending-docid postings. Unlike the standalone LogicalIndexWriter API, + // every rejection is terminal here because posting bytes may already have + // entered the compound output; all later calls return the first error. + Status push_term(StreamedTermPostings&& tp); + // Binds the semantic (plain-token) count after the merge's single postings + // pass. Complete scoring metadata requires exactly one call, including for + // an empty destination whose count is zero. + Status set_semantic_token_count(uint64_t token_count); + // Seals this index: flushes the trailing DICT block, streams the DICT region + // right after the posting region and records the placements. A failed finish + // leaves the session unfinished (and the container unsealable) -- there is + // no retry, the whole compaction round must fail and be redone. + Status finish(); + // Makes the owning compound permanently unsealable. Merge plans call this + // on every destination when any source or sibling destination fails, because + // a successfully written prefix is not a complete logical index. + void abort(const Status& cause); + bool finished() const { return finished_; } + +private: + friend class SniiCompoundWriter; + SniiStreamedIndexSession(SniiCompoundWriter* owner, SniiIndexInput in, + TrackedNullDocids null_docids, TrackedEncodedNorms encoded_norms); + static SniiIndexInput attach_encoded_norms(SniiIndexInput in, + TrackedEncodedNorms* encoded_norms, + uint64_t reserved_bytes); + + SniiCompoundWriter* owner_; + // The reservation precedes input_ so input_.encoded_norms is destroyed + // before its charge is released. + MemoryReporter::Reservation encoded_norms_reservation_; + // Owns the input: LogicalIndexWriter keeps references into it (terms / + // encoded_norms), so it must live exactly as long as the writer. + SniiIndexInput input_; + std::unique_ptr writer_; + uint64_t post_off_ = 0; + bool semantic_token_count_required_ = false; + bool semantic_token_count_set_ = false; + bool finished_ = false; +}; + +class SniiCompoundWriter { +public: + explicit SniiCompoundWriter(io::FileWriter* out); + SniiCompoundWriter(const SniiCompoundWriter&) = delete; + SniiCompoundWriter& operator=(const SniiCompoundWriter&) = delete; + SniiCompoundWriter(SniiCompoundWriter&&) = delete; + SniiCompoundWriter& operator=(SniiCompoundWriter&&) = delete; + + // Size of the buffer the inherit copy streams through. Fixed, so peak memory + // of an inherit does not grow with the source container. Sized for sequential + // local reads -- generous next to inverted_index_read_buffer_size (4 KiB), and + // small enough that even a multi-GiB container costs a negligible number of + // reads. + static constexpr size_t kInheritCopyChunkBytes = 64U << 10; + // Same rationale for the blob file copy in finish(): peak memory is one + // chunk regardless of blob size (a GiB-scale ann.faiss must never be + // buffered whole inside the compound writer). + static constexpr size_t kBlobCopyChunkBytes = 64U << 10; + + // Carries a source container's unchanged logical indexes into this one + // (BUILD INDEX on SNII). It copies the source's validated physical prefix -- + // bootstrap header plus every section the inherited indexes reference -- + // verbatim, then registers their metadata groups so finish() re-emits them + // without decoding or re-encoding a single posting. Because the prefix lands + // at the SAME offsets, the inherited section references stay valid unchanged. + // + // MUST be the writer's first data operation: the copy owns the front of the + // file, so anything already written would be overwritten in meaning. A read + // or write failure poisons the writer, so finish() can never seal a container + // holding a partial prefix. + Status inherit(const reader::SniiRewriteSnapshot& snapshot, io::FileReader* source); + + // Buffers one logical index: builds its section bytes and meta sub-sections. + // The actual file writing happens in finish() (single front-to-back pass). + // The key (index_id, suffix) must not collide with an inherited one. + Status add_logical_index(const SniiIndexInput& in); + + // Registers one opaque BLOB logical index (kind must not be kInverted). + // Registration is pure bookkeeping -- NOT A BYTE is written here, so it is + // legal at any point before finish() (even while a streamed session is + // active). finish() streams cold_files into the data area after all text + // physical sections, and hot_files after the text metadata groups, + // physically adjacent per entry, recording absolute offsets + crc32c into + // the directory entry. A rejected registration leaves the writer clean; a + // copy failure during finish() poisons the container for good. + Status add_blob_index(uint64_t index_id, std::string index_suffix, + format::LogicalIndexKind kind, std::vector cold_files, + std::vector hot_files); + + // T2.2: begins a STREAMED logical-index session (the compaction merge fast + // path) -- the caller pushes pre-merged terms through *session instead of + // handing the writer a term source, so `in` must carry NO term_source and NO + // materialized terms. Only ONE session may be active at a time (its posting + // region streams straight into the container output, so a concurrent + // add_logical_index or second session would interleave bytes); both are + // rejected while a session is unfinished, as is finish(). The returned + // handle is owned by this writer and valid for its lifetime. + Status begin_streamed_index(SniiIndexInput in, SniiStreamedIndexSession** session); + Status begin_streamed_index(SniiIndexInput in, TrackedNullDocids null_docids, + SniiStreamedIndexSession** session); + Status begin_streamed_index(SniiIndexInput in, TrackedNullDocids null_docids, + TrackedEncodedNorms encoded_norms, + SniiStreamedIndexSession** session); + + // Writes bootstrap header + all index sections + adjacent metadata groups + + // raw directory + tail pointer, then finalizes the underlying writer. + Status finish(); + +private: + // Absolute placement of one index's sections, resolved during finish(). + struct Placement { + uint64_t dict_off = 0; + uint64_t dict_len = 0; + uint64_t post_off = 0; // interleaved [prx][frq] posting region (was frq + prx) + uint64_t post_len = 0; + uint64_t norms_off = 0; + uint64_t norms_len = 0; + uint64_t null_off = 0; + uint64_t null_len = 0; + uint64_t bsbf_off = 0; + uint64_t bsbf_len = 0; + }; + + // One logical index carried over from a source container: its raw + // [Core][STI][DBD] bytes plus the three lengths needed to rebuild the + // directory entry once the group's new position is known. + struct InheritedGroup { + uint64_t index_id = 0; + std::string index_suffix; + std::vector metadata_group; + size_t core_length = 0; + size_t sampled_term_index_length = 0; + size_t dict_block_directory_length = 0; + }; + + // One registered blob logical index awaiting finish(). cold/hot refs are + // resolved as the corresponding bytes stream out during finish(). + struct PendingBlobIndex { + uint64_t index_id = 0; + std::string index_suffix; + format::LogicalIndexKind kind = format::LogicalIndexKind::kInverted; + std::vector cold_files; + std::vector hot_files; + std::vector cold_refs; + std::vector hot_refs; + }; + + friend class SniiStreamedIndexSession; + + Status ensure_bootstrap(); + Status write_bootstrap(); + // Writes indexes_[index]'s norms/null-bitmap/bsbf immediately after its + // [posting][dict] pair and fills placements_[index]. Keeping one index's sections + // contiguous is what makes a single-index cold query touch one cache block instead + // of three; the previous layout grouped these by section type across all indexes. + // Must be called after indexes_/placements_ have been pushed for this index. + Status write_index_aux_sections(size_t index); + Status write_tail(); + Status append(const std::vector& bytes); + Status poison(Status status); + // Argument validation for one add_blob_index call; see the .cpp. + static Status validate_blob_registration(format::LogicalIndexKind kind, + const std::vector& cold_files, + const std::vector& hot_files); + // Streams `files` into the container at the current position through a + // fixed-size chunk buffer, recording each file's absolute placement and + // crc32c into *refs. Called from finish() only (cold then hot regions). + // Drops blob sources whose bytes are already in the container. Called as soon + // as write_blob_files has recorded their extents: a source may own the bytes + // it serves, so keeping it alive keeps that memory resident for the writer's + // whole life. + static void release_blob_sources(std::vector* files); + Status write_blob_files(const std::vector& files, + std::vector* refs); + // Emits the blob hot-file region and the blob directory entries; see the .cpp. + Status write_blob_hot_files_and_entries( + std::vector* directory_entries); + // Seals one streamed session: records its placements and adopts its + // LogicalIndexWriter into indexes_ (only on FULL success -- see the + // crash-safety note on SniiStreamedIndexSession). + Status finish_streamed_index(SniiStreamedIndexSession* session); + // An unfinished streamed session (begun but not successfully sealed): + // blocks add_logical_index, another begin_streamed_index and finish(). + bool has_active_session() const { return !sessions_.empty() && !sessions_.back()->finished(); } + + io::FileWriter* out_; + std::vector> indexes_; + // Streamed sessions in begin order (at most the last one is unfinished). + // Owned here so the raw handles returned to callers stay valid for the + // writer's lifetime; a finished session is inert (its writer moved out). + std::vector> sessions_; + // Per-index placement, fully resolved by the time add_logical_index / + // finish_streamed_index returns: post_off/post_len and dict_off/dict_len as each + // index's posting/DICT regions stream in, then norms/null/bsbf off+len via + // write_index_aux_sections immediately after. finish() no longer fills any of + // these fields -- it only reads placements_ to build the metadata directory. The + // absolute write offset is out_->bytes_written() (the single source of truth -- + // no separate cursor). + std::vector placements_; + // Logical indexes carried over by inherit(), in source directory order. They + // own no LogicalIndexWriter: their sections are already in the copied prefix. + std::vector inherited_; + // Blob logical indexes registered by add_blob_index(), in add order. Their + // bytes stream out during finish() only. + std::vector blobs_; + // inherit() ran successfully. Distinct from inherited_ being non-empty: a + // rewrite may drop every old index and still copy the bootstrap header. + bool inherited_prefix_ = false; + bool bootstrap_written_ = false; + bool finished_ = false; + Status failed_ = Status::OK(); +}; + +} // namespace doris::snii::writer diff --git a/be/src/storage/index/snii/writer/spill_run_codec.cpp b/be/src/storage/index/snii/writer/spill_run_codec.cpp new file mode 100644 index 00000000000000..b85f903c26e0f4 --- /dev/null +++ b/be/src/storage/index/snii/writer/spill_run_codec.cpp @@ -0,0 +1,937 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/writer/spill_run_codec.h" + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "storage/index/snii/encoding/varint.h" +#include "storage/index/snii/format/format_constants.h" + +namespace doris::snii::writer { + +namespace { + +// Flush staging at this exact bound. A large write buffer (4 MiB) collapses the +// per-flush write() syscall count by ~64x: at 64 KiB the 5M build issued +// ~8800 write()s to ext4 (~9s of syscall overhead) for ~553 MiB of runs, versus +// a raw dd of the same bytes taking ~1.2s. Wide terms are appended and flushed +// in chunks, so the staging allocation never grows with term width. +constexpr size_t kWriteFlushBytes = 1u << 22; // 4 MiB +// RunReader reads this much per disk fill; the window slides so a single record +// never needs the whole run in RAM (only the current term's encoded span). KEEP +// this small (64 KiB): a large read chunk x many open runs would inflate the +// merge-phase peak RSS at low spill thresholds (each reader holds a window). +constexpr size_t kReadChunkBytes = 1u << 16; // 64 KiB + +enum class RunPostingShape : uint8_t { + kDocsOnlyStatless = 0, + kDocsAndFreqs = 1, + kPositioned = 2, +}; + +RunPostingShape posting_shape(const TermPostings& tp) { + if (tp.retain_positions) { + return RunPostingShape::kPositioned; + } + return tp.freqs.empty() ? RunPostingShape::kDocsOnlyStatless : RunPostingShape::kDocsAndFreqs; +} + +// Writes the full byte range [data, data+len) to fd, looping over short writes. +Status write_all(int fd, const uint8_t* data, size_t len) { + size_t off = 0; + while (off < len) { + const ssize_t n = ::write(fd, data + off, len - off); + if (n < 0) { + if (errno == EINTR) continue; + return Status::Error(std::string("run write failed: ") + + std::strerror(errno)); + } + off += static_cast(n); + } + return Status::OK(); +} + +template +Status reserve_vector_for_size(std::vector* values, size_t target, + MemoryReporter* memory_reporter, + MemoryReporter::Reservation* reservation) { + if (target <= values->capacity()) { + return Status::OK(); + } + if (target > std::numeric_limits::max() / sizeof(T)) { + return Status::Error( + "run reader: vector byte capacity overflow"); + } + if (memory_reporter == nullptr) { + values->reserve(target); + return Status::OK(); + } + MemoryReporter::Reservation replacement; + const uint64_t target_bytes = static_cast(target) * sizeof(T); + RETURN_IF_ERROR(reservation->prepare_replacement(target_bytes, &replacement)); + values->reserve(target); + DCHECK_EQ(values->capacity(), target); + *reservation = std::move(replacement); + return Status::OK(); +} + +Status reserve_write_buffer_for_append(std::vector* buffer, size_t target, + MemoryReporter* memory_reporter, + MemoryReporter::Reservation* reservation) { + if (target <= buffer->capacity()) { + return Status::OK(); + } + if (target > kWriteFlushBytes) { + return Status::Error( + "run writer: staging buffer exceeds flush bound"); + } + + size_t capacity = std::max(buffer->capacity(), 1); + while (capacity < target) { + capacity = capacity > kWriteFlushBytes / 2 ? kWriteFlushBytes : capacity * 2; + } + return reserve_vector_for_size(buffer, capacity, memory_reporter, reservation); +} + +} // namespace + +// --------------------------------------------------------------------------- +// RunWriter +// --------------------------------------------------------------------------- + +RunWriter::RunWriter(MemoryReporter* memory_reporter) + : memory_reporter_(memory_reporter), + buffer_reservation_(memory_reporter == nullptr ? MemoryReporter::Reservation() + : memory_reporter->make_reservation()) {} + +RunWriter::~RunWriter() { + if (fd_ >= 0) ::close(fd_); +} + +Status RunWriter::open(const std::string& path) { + fd_ = ::open(path.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0600); + if (fd_ < 0) { + return Status::Error("run open(" + path + + "): " + std::strerror(errno)); + } + buf_.clear(); + return Status::OK(); +} + +Status RunWriter::flush() { + if (buf_.empty()) return Status::OK(); + RETURN_IF_ERROR(write_all(fd_, buf_.data(), buf_.size())); + buf_.clear(); + return Status::OK(); +} + +Status RunWriter::append_bytes(const uint8_t* data, size_t size) { + while (size != 0) { + if (buf_.size() == kWriteFlushBytes) { + RETURN_IF_ERROR(flush()); + } + const size_t count = std::min(size, kWriteFlushBytes - buf_.size()); + const size_t target = buf_.size() + count; + RETURN_IF_ERROR(reserve_write_buffer_for_append(&buf_, target, memory_reporter_, + &buffer_reservation_)); + buf_.insert(buf_.end(), data, data + count); + data += count; + size -= count; + } + return Status::OK(); +} + +Status RunWriter::append_varint(uint64_t value) { + uint8_t bytes[10]; + const size_t size = encode_varint64(value, bytes); + return append_bytes(bytes, size); +} + +Status RunWriter::append_raw_u32(const uint32_t* values, size_t count) { + if (count > std::numeric_limits::max() / sizeof(uint32_t)) { + return Status::Error( + "run writer: raw u32 byte count overflows size_t"); + } + return append_bytes(reinterpret_cast(values), count * sizeof(uint32_t)); +} + +void RunWriter::release_buffer() { + std::vector().swap(buf_); + buffer_reservation_.reset(); +} + +Status RunWriter::write_term(uint32_t term_id, const TermPostings& tp) { + DCHECK(tp.retain_positions || tp.positions_flat.empty()); + const RunPostingShape shape = posting_shape(tp); + const size_t doc_count = tp.document_count(); + if (shape != RunPostingShape::kDocsOnlyStatless) { + DCHECK_EQ(tp.docids.size(), tp.freqs.size()); + } + RETURN_IF_ERROR(append_varint(term_id)); + RETURN_IF_ERROR(append_varint(static_cast(shape))); + RETURN_IF_ERROR(append_varint(doc_count)); + // Docids are a RAW fixed-width u32 block (bulk memcpy), NOT per-value VInt. + // Per-value varint over ~60M docids cost ~1.5s of encode CPU on the spill feed + // side; raw is a single memcpy and the decode side becomes a memcpy too. Runs + // are PRIVATE temp files written then read back from page cache, so the modestly + // larger run (no delta packing) costs ~0 extra real I/O. Absolute docids are + // stored (the merge concatenates per-term across runs and re-deltas at encode). + RETURN_IF_ERROR(append_raw_u32(tp.docids.data(), tp.docids.size())); + if (shape != RunPostingShape::kDocsOnlyStatless) { + RETURN_IF_ERROR(append_raw_u32(tp.freqs.data(), tp.freqs.size())); + } + if (shape == RunPostingShape::kPositioned) { + const uint64_t n_pos = tp.positions_flat.size(); + RETURN_IF_ERROR(append_varint(n_pos)); + RETURN_IF_ERROR(append_raw_u32(tp.positions_flat.data(), tp.positions_flat.size())); + } + return Status::OK(); +} + +Status RunWriter::close() { + if (fd_ < 0) return Status::OK(); + RETURN_IF_ERROR(flush()); + const int fd = fd_; + fd_ = -1; + if (::close(fd) != 0) { + return Status::Error(std::string("run close: ") + + std::strerror(errno)); + } + release_buffer(); + return Status::OK(); +} + +// --------------------------------------------------------------------------- +// RunReader +// --------------------------------------------------------------------------- + +RunReader::RunReader(MemoryReporter* memory_reporter) + : memory_reporter_(memory_reporter), + window_reservation_(memory_reporter == nullptr ? MemoryReporter::Reservation() + : memory_reporter->make_reservation()), + docids_reservation_(memory_reporter == nullptr ? MemoryReporter::Reservation() + : memory_reporter->make_reservation()), + freqs_reservation_(memory_reporter == nullptr ? MemoryReporter::Reservation() + : memory_reporter->make_reservation()), + positions_reservation_(memory_reporter == nullptr ? MemoryReporter::Reservation() + : memory_reporter->make_reservation()) { +} + +RunReader::~RunReader() { + if (fd_ >= 0) ::close(fd_); +} + +Status RunReader::open(const std::string& path, bool has_positions) { + fd_ = ::open(path.c_str(), O_RDONLY); + if (fd_ < 0) { + return Status::Error("run reopen(" + path + + "): " + std::strerror(errno)); + } + // Record the run's byte size so every length decoded from the stream can be + // bounded against it before allocating (no record holds more u32s than the whole + // file). Honors the header's "lengths validated against the file size" contract, + // turning a corrupt/truncated length into Status::Corruption rather than an + // uncaught std::bad_alloc from a giant resize(). + struct stat st {}; + if (::fstat(fd_, &st) != 0) { + return Status::Error(std::string("run fstat: ") + + std::strerror(errno)); + } + file_size_ = static_cast(st.st_size); + bytes_read_ = 0; + has_positions_ = has_positions; + exhausted_ = false; + eof_ = false; + pos_ = 0; + pos_count_ = 0; + pos_remaining_ = 0; + window_.clear(); + return advance(); +} + +// Slides consumed bytes out of the window, then appends one disk chunk. +Status RunReader::fill() { + if (pos_ > 0) { + window_.erase(window_.begin(), window_.begin() + pos_); + pos_ = 0; + } + if (eof_) return Status::OK(); + if (bytes_read_ > file_size_) { + return Status::Error( + "run reader: bytes read exceed file size"); + } + const uint64_t remaining = file_size_ - bytes_read_; + if (remaining == 0) { + eof_ = true; + return Status::OK(); + } + const size_t read_size = static_cast( + std::min(remaining, static_cast(kReadChunkBytes))); + const size_t base = window_.size(); + if (base > std::numeric_limits::max() - read_size) { + return Status::Error( + "run reader: decode window capacity overflow"); + } + RETURN_IF_ERROR(reserve_vector_for_size(&window_, base + read_size, memory_reporter_, + &window_reservation_)); + window_.resize(base + read_size); + ssize_t n; + do { + n = ::read(fd_, window_.data() + base, read_size); + } while (n < 0 && errno == EINTR); + if (n < 0) + return Status::Error(std::string("run read: ") + + std::strerror(errno)); + window_.resize(base + static_cast(n)); + bytes_read_ += static_cast(n); + if (n == 0 || bytes_read_ == file_size_) eof_ = true; + return Status::OK(); +} + +// Buffered bytes available to the decoder right now (from pos_ to window end). +// fill() may slide the window (erasing consumed bytes), so callers must compare +// THIS quantity -- not window_.size() -- to decide whether more data arrived. +size_t RunReader::available() const { + return window_.size() - pos_; +} + +Status RunReader::ensure(size_t n) { + while (available() < n) { + const size_t had = available(); + RETURN_IF_ERROR(fill()); + if (available() == had && eof_) { + return Status::Error( + "run truncated: needed more bytes than available"); + } + } + return Status::OK(); +} + +// Streamed varint: decode from the current window; if it straddles the buffered +// boundary, top up from disk and retry. A varint is at most 10 bytes, so this +// loops at most a couple of times. Bounds-safe: decode_varint64 never reads past +// `end`, and a partial varint at true eof is reported as corruption. +Status RunReader::read_varint(uint64_t* v) { + while (true) { + const uint8_t* p = window_.data() + pos_; + const uint8_t* end = window_.data() + window_.size(); + const uint8_t* next = nullptr; + Status s = decode_varint64(p, end, v, &next); + if (s.ok()) { + pos_ += static_cast(next - p); + return Status::OK(); + } + if (eof_) + return Status::Error( + "run truncated: incomplete varint"); + const size_t had = available(); + RETURN_IF_ERROR(fill()); + if (available() == had && eof_) { + return Status::Error( + "run truncated: incomplete varint at eof"); + } + } +} + +// Streams `count` raw little-endian u32s from the window into `dst` (caller-owned +// storage of at least count*4 bytes), topping up the window from disk as needed. +// Copies whatever is buffered each pass (the window may hold only part of a large +// block), so a high-df term's freqs/positions stream through in 64 KiB chunks +// without ever needing the whole block resident at once. +Status RunReader::pull_raw_u32(uint8_t* dst, size_t count) { + if (count == 0) return Status::OK(); + if (count > std::numeric_limits::max() / sizeof(uint32_t)) { + return Status::Error( + "run: raw u32 byte count overflows size_t"); + } + size_t need = count * sizeof(uint32_t); + size_t written = 0; + while (need > 0) { + if (available() == 0) { + const size_t had = available(); + RETURN_IF_ERROR(fill()); + if (available() == had && eof_) { + return Status::Error( + "run truncated: needed more raw bytes than available"); + } + } + const size_t take = std::min(need, available()); + std::memcpy(dst + written, window_.data() + pos_, take); + pos_ += take; + written += take; + need -= take; + } + return Status::OK(); +} + +// Bulk-decodes `count` raw u32s into `out` (resized to count). +Status RunReader::read_raw_u32(size_t count, std::vector* out, + MemoryReporter::Reservation* reservation) { + // Bound `count` against the run's byte size BEFORE resize(): a record can never + // hold more u32s than the whole file. Rejects a corrupt/truncated length varint + // (which is otherwise an unbounded resize -> uncaught std::bad_alloc). + if (count > file_size_ / sizeof(uint32_t)) { + return Status::Error( + "run: raw u32 count exceeds file size"); + } + RETURN_IF_ERROR(reserve_vector_for_size(out, count, memory_reporter_, reservation)); + out->resize(count); + if (count == 0) return Status::OK(); + return pull_raw_u32(reinterpret_cast(out->data()), count); +} + +// Materializes the current term's deferred position block into positions_flat. +// A no-op once the positions are already drained (idempotent within a term). +Status RunReader::materialize_positions() { + if (pos_remaining_ == 0) { + current_.positions_flat.clear(); + return Status::OK(); + } + if (pos_remaining_ > std::numeric_limits::max()) { + return Status::Error( + "run: position count exceeds addressable memory"); + } + const size_t n = static_cast(pos_remaining_); + RETURN_IF_ERROR(read_raw_u32(n, ¤t_.positions_flat, &positions_reservation_)); + pos_remaining_ = 0; + return Status::OK(); +} + +// Streams the next `n` positions of the current term straight from the window. +Status RunReader::stream_positions(uint32_t* dst, size_t n) { + if (n == 0) return Status::OK(); + if (n > pos_remaining_) { + return Status::Error( + "run: stream_positions past block end"); + } + RETURN_IF_ERROR(pull_raw_u32(reinterpret_cast(dst), n)); + pos_remaining_ -= n; + return Status::OK(); +} + +// Discards any positions of the current term left unread, so the window cursor +// lands at the next record boundary before advance() reads the next term. +Status RunReader::skip_remaining_positions() { + if (pos_remaining_ == 0) return Status::OK(); + std::array scratch; + while (pos_remaining_ != 0) { + const size_t count = static_cast( + std::min(pos_remaining_, static_cast(scratch.size()))); + RETURN_IF_ERROR(pull_raw_u32(reinterpret_cast(scratch.data()), count)); + pos_remaining_ -= count; + } + return Status::OK(); +} + +Status RunReader::advance() { + // Drain any positions the owner left unread for the previous term so the window + // cursor lands at the next record boundary. + RETURN_IF_ERROR(skip_remaining_positions()); + // End-of-run detection: at a record boundary, if no bytes remain we are done. + if (available() == 0) { + RETURN_IF_ERROR(fill()); + if (available() == 0 && eof_) { + exhausted_ = true; + return Status::OK(); + } + } + uint64_t term_id = 0; + RETURN_IF_ERROR(read_varint(&term_id)); + if (term_id > UINT32_MAX) + return Status::Error( + "run term_id exceeds uint32"); + current_id_ = static_cast(term_id); + current_.term.clear(); // runs store only the id; owner resolves the string + + uint64_t encoded_shape = 0; + RETURN_IF_ERROR(read_varint(&encoded_shape)); + if (encoded_shape > static_cast(RunPostingShape::kPositioned)) { + return Status::Error( + "run: unknown posting shape"); + } + const auto shape = static_cast(encoded_shape); + if (shape == RunPostingShape::kPositioned && !has_positions_) { + return Status::Error( + "run: positioned record in docs-only run"); + } + if (shape == RunPostingShape::kDocsOnlyStatless && !has_positions_) { + return Status::Error( + "run: statless record requires a positioned mixed-shape run"); + } + + uint64_t n_docs = 0; + RETURN_IF_ERROR(read_varint(&n_docs)); + if (n_docs > file_size_ / sizeof(uint32_t) || n_docs > std::numeric_limits::max()) { + return Status::Error( + "run: document count exceeds file size or addressable memory"); + } + // Docids: RAW absolute u32 block (bulk read), matching the writer's AppendRawU32. + RETURN_IF_ERROR( + read_raw_u32(static_cast(n_docs), ¤t_.docids, &docids_reservation_)); + for (size_t i = 1; i < current_.docids.size(); ++i) { + if (current_.docids[i] <= current_.docids[i - 1]) { + return Status::Error( + "run: docids must be strictly ascending within one record"); + } + } + current_.freqs.clear(); + current_.positions_flat.clear(); + pos_count_ = 0; + pos_remaining_ = 0; + if (shape == RunPostingShape::kDocsOnlyStatless) { + current_.retain_positions = false; + return Status::OK(); + } + + // Freqs: RAW u32 block (bulk read), matching the writer's AppendRawU32. + RETURN_IF_ERROR( + read_raw_u32(static_cast(n_docs), ¤t_.freqs, &freqs_reservation_)); + uint64_t total_freq = 0; + for (uint32_t freq : current_.freqs) { + if (freq == 0) { + return Status::Error( + "run: frequency must be positive"); + } + if (freq > UINT64_MAX - total_freq) { + return Status::Error( + "run: frequency sum overflows uint64"); + } + total_freq += freq; + } + if (shape == RunPostingShape::kDocsAndFreqs) { + current_.retain_positions = false; + return Status::OK(); + } + + uint64_t n_pos = 0; + RETURN_IF_ERROR(read_varint(&n_pos)); + if (n_pos != total_freq) { + return Status::Error( + "run: position count does not match frequency sum"); + } + if (n_pos > file_size_ / sizeof(uint32_t)) { + return Status::Error( + "run: position count exceeds file size"); + } + // Positions are LAZY: record the block count and leave the window cursor parked + // at the block start. The owner picks materialize_positions() for explicit + // materialization or stream_positions() for bounded writer-owned windows. + current_.retain_positions = true; + pos_count_ = n_pos; + pos_remaining_ = n_pos; + return Status::OK(); +} + +// --------------------------------------------------------------------------- +// K-way merge +// --------------------------------------------------------------------------- + +namespace { + +// Min-heap entry: orders by the run's current term-id's PRECOMPUTED integer +// string-rank (rank[term_id] == its lexicographic rank over the dense vocabulary), +// tie-broken by run index so equal terms are gathered run-order (keeping +// concatenated docids ascending). The rank is a lexicographic bijection on a dense +// vocab, so ordering by the dense 4 B rank array reproduces the exact dictionary +// order a vocab-string compare would -- with an integer compare and zero random +// vocab string access in the inner loop. +struct HeapItem { + uint32_t term_id; + size_t run; +}; +struct HeapGreater { + const std::vector* rank; + bool operator()(const HeapItem& a, const HeapItem& b) const { + const uint32_t ra = (*rank)[a.term_id]; + const uint32_t rb = (*rank)[b.term_id]; + if (ra != rb) { + return ra > rb; + } // smaller rank first (lexicographic min-heap) + return a.run > b.run; // same term across runs: run-order tie-break + } +}; + +// Appends src's postings onto dst (run order). Later runs only cover docids +// >= dst's last, so docids stay ascending. COALESCE the boundary doc: if a spill +// fell BETWEEN two tokens of the same doc, that doc ends one run and begins the +// next with the SAME docid -- merge them (sum freqs, splice positions) so the +// merged term has exactly one entry per docid (matching the in-memory build). +// +// Positions are FLAT: doc order, partitioned by freqs. Because both dst and src +// already store doc-ordered flat positions, the common (no-boundary-overlap) case +// is a single bulk append. The boundary-overlap case must INSERT src's first +// doc's positions right after dst's last doc's positions so flat order stays +// consistent with the merged (coalesced) freqs. +void concat(TermPostings* dst, const TermPostings& src, RunPostingShape shape) { + if (src.docids.empty()) return; + const bool has_positions = shape == RunPostingShape::kPositioned; + const bool statless = shape == RunPostingShape::kDocsOnlyStatless; + DCHECK(posting_shape(src) == shape); + size_t start = 0; + size_t src_pos_start = 0; // flat offset of src positions to append after splice + if (!dst->docids.empty() && dst->docids.back() == src.docids.front()) { + const uint32_t head_fc = statless ? 0 : src.freqs.front(); + if (has_positions && head_fc != 0) { + // Splice src's first-doc positions in right after dst's last-doc positions. + // dst's last doc owns dst->freqs.back() entries at the tail of positions_flat + // BEFORE we bump that freq, so insert at end() (last doc is the tail run). + auto& flat = dst->positions_flat; + flat.insert(flat.end(), src.positions_flat.begin(), + src.positions_flat.begin() + head_fc); + } + if (!statless) { + dst->freqs.back() += head_fc; + } + src_pos_start = head_fc; + start = 1; // boundary doc folded in; append the rest + } + dst->docids.insert(dst->docids.end(), src.docids.begin() + start, src.docids.end()); + if (!statless) { + dst->freqs.insert(dst->freqs.end(), src.freqs.begin() + start, src.freqs.end()); + } + if (has_positions) { + dst->positions_flat.insert(dst->positions_flat.end(), + src.positions_flat.begin() + src_pos_start, + src.positions_flat.end()); + } +} + +class RunTermPostingSource final : public TermPostingSource { +public: + RunTermPostingSource(std::vector>* readers, + const std::vector* matching, RunPostingShape shape) + : readers_(readers), matching_(matching), shape_(shape) {} + + Status fill(uint32_t target_docs, TermPostingBuffer* out, bool* exhausted) override { + if (out == nullptr || exhausted == nullptr || target_docs == 0 || !out->empty()) { + return Status::Error( + "run posting source: invalid fill arguments"); + } + + Cursor planned = cursor_; + size_t document_count = 0; + size_t position_count = 0; + while (document_count < target_docs) { + normalize(&planned); + if (planned.run == matching_->size()) { + break; + } + const uint32_t docid = current_docid(planned); + uint64_t frequency = 0; + do { + const TermPostings& postings = current_postings(planned); + if (shape_ != RunPostingShape::kDocsOnlyStatless) { + frequency += postings.freqs[planned.doc]; + if (frequency > std::numeric_limits::max()) { + return Status::Error( + "run: coalesced frequency exceeds uint32"); + } + } + advance(&planned); + normalize(&planned); + if (planned.run == matching_->size()) { + break; + } + const uint32_t next_docid = current_docid(planned); + if (next_docid < docid) { + return Status::Error( + "run: docids overlap across spill runs"); + } + if (next_docid != docid) { + break; + } + } while (true); + if (shape_ == RunPostingShape::kPositioned) { + if (frequency > std::numeric_limits::max() - position_count) { + return Status::Error( + "run posting source: position window exceeds size_t"); + } + position_count += static_cast(frequency); + } + ++document_count; + } + + MutableTermPostingSpan destination; + const bool has_freqs = shape_ != RunPostingShape::kDocsOnlyStatless; + RETURN_IF_ERROR( + out->grow_uninitialized(document_count, has_freqs, position_count, &destination)); + size_t position_offset = 0; + for (size_t output = 0; output < document_count; ++output) { + normalize(&cursor_); + DCHECK_LT(cursor_.run, matching_->size()); + const uint32_t docid = current_docid(cursor_); + uint64_t frequency = 0; + do { + const TermPostings& postings = current_postings(cursor_); + const uint32_t run_frequency = has_freqs ? postings.freqs[cursor_.doc] : 0; + if (shape_ == RunPostingShape::kPositioned) { + RunReader* reader = (*readers_)[(*matching_)[cursor_.run]].get(); + RETURN_IF_ERROR(reader->stream_positions( + destination.positions_flat.data() + position_offset, run_frequency)); + position_offset += run_frequency; + } + frequency += run_frequency; + advance(&cursor_); + normalize(&cursor_); + if (cursor_.run == matching_->size() || current_docid(cursor_) != docid) { + break; + } + } while (true); + destination.docids[output] = docid; + if (has_freqs) { + destination.freqs[output] = static_cast(frequency); + } + } + DCHECK_EQ(position_offset, destination.positions_flat.size()); + normalize(&cursor_); + *exhausted = cursor_.run == matching_->size(); + return Status::OK(); + } + + bool exhausted() { + normalize(&cursor_); + return cursor_.run == matching_->size(); + } + +private: + struct Cursor { + size_t run = 0; + size_t doc = 0; + }; + + const TermPostings& current_postings(const Cursor& cursor) const { + return (*readers_)[(*matching_)[cursor.run]]->current(); + } + + uint32_t current_docid(const Cursor& cursor) const { + return current_postings(cursor).docids[cursor.doc]; + } + + void normalize(Cursor* cursor) const { + while (cursor->run < matching_->size() && + cursor->doc == current_postings(*cursor).docids.size()) { + ++cursor->run; + cursor->doc = 0; + } + } + + static void advance(Cursor* cursor) { ++cursor->doc; } + + std::vector>* readers_; + const std::vector* matching_; + RunPostingShape shape_; + Cursor cursor_; +}; + +} // namespace + +Status merge_run_sources(const std::vector& run_paths, + const std::vector& vocab, + const std::vector& string_rank, bool has_positions, + const StreamedTermConsumer& fn, TermKeyMaterializer materialize_term_key, + MemoryReporter* memory_reporter) { + if (string_rank.size() != vocab.size()) { + return Status::Error( + "merge_run_sources: string_rank/vocab size mismatch"); + } + std::vector> readers; + readers.reserve(run_paths.size()); + std::priority_queue, HeapGreater> heap( + HeapGreater {&string_rank}); + for (size_t i = 0; i < run_paths.size(); ++i) { + auto reader = std::make_unique(memory_reporter); + RETURN_IF_ERROR(reader->open(run_paths[i], has_positions)); + if (!reader->exhausted()) { + if (reader->current_id() >= vocab.size()) { + return Status::Error( + "run term_id out of vocab range"); + } + heap.push({reader->current_id(), i}); + } + readers.push_back(std::move(reader)); + } + + std::vector matching; + while (!heap.empty()) { + const uint32_t id = heap.top().term_id; + matching.clear(); + while (!heap.empty() && heap.top().term_id == id) { + matching.push_back(heap.top().run); + heap.pop(); + } + DCHECK(!matching.empty()); + const RunPostingShape shape = posting_shape(readers[matching.front()]->current()); + for (size_t run : matching) { + if (posting_shape(readers[run]->current()) != shape) { + return Status::Error( + "run: posting shape differs across matching terms"); + } + } + + RunTermPostingSource source(&readers, &matching, shape); + StreamedTermPostings postings {.term = materialize_term_key + ? materialize_term_key(vocab[id]) + : std::string(vocab[id]), + .retain_positions = shape == RunPostingShape::kPositioned, + .source = &source}; + RETURN_IF_ERROR(fn(std::move(postings))); + if (!source.exhausted()) { + return Status::Error( + "run posting source: consumer returned before term exhaustion"); + } + + for (size_t run : matching) { + RunReader* reader = readers[run].get(); + RETURN_IF_ERROR(reader->advance()); + if (!reader->exhausted()) { + if (reader->current_id() >= vocab.size()) { + return Status::Error( + "run term_id out of vocab range"); + } + heap.push({reader->current_id(), run}); + } + } + } + return Status::OK(); +} + +Status compact_runs(const std::vector& run_paths, + const std::vector& string_rank, bool has_positions, + const std::string& out_path, MemoryReporter* memory_reporter) { + // Same heap machinery as merge_run_sources, but the output is a RUN (records keyed + // by term-id, ordered by string rank -- the exact invariant every run file + // carries), not a resolved term stream: no vocab strings are needed, and + // positions are always materialized because the run codec serializes + // positions_flat directly. + std::vector> readers; + readers.reserve(run_paths.size()); + std::priority_queue, HeapGreater> heap( + HeapGreater {&string_rank}); + for (size_t i = 0; i < run_paths.size(); ++i) { + auto r = std::make_unique(memory_reporter); + RETURN_IF_ERROR(r->open(run_paths[i], has_positions)); + if (!r->exhausted()) { + if (r->current_id() >= string_rank.size()) { + return Status::Error( + "run term_id out of rank range"); + } + heap.push({r->current_id(), i}); + } + readers.push_back(std::move(r)); + } + + RunWriter w(memory_reporter); + RETURN_IF_ERROR(w.open(out_path)); + std::vector matching; // run indices contributing the current term + while (!heap.empty()) { + const uint32_t id = heap.top().term_id; + MemoryReporter::Reservation merged_docids_reservation = + memory_reporter == nullptr ? MemoryReporter::Reservation() + : memory_reporter->make_reservation(); + MemoryReporter::Reservation merged_freqs_reservation = + memory_reporter == nullptr ? MemoryReporter::Reservation() + : memory_reporter->make_reservation(); + MemoryReporter::Reservation merged_positions_reservation = + memory_reporter == nullptr ? MemoryReporter::Reservation() + : memory_reporter->make_reservation(); + TermPostings merged; + matching.clear(); + uint64_t total_docs = 0; + uint64_t total_pos = 0; + while (!heap.empty() && heap.top().term_id == id) { + const size_t ri = heap.top().run; + heap.pop(); + const RunReader* r = readers[ri].get(); + const uint64_t run_docs = r->current().docids.size(); + const uint64_t run_positions = r->current_pos_count(); + if (run_docs > std::numeric_limits::max() - total_docs || + run_positions > std::numeric_limits::max() - total_pos) { + return Status::Error( + "run compaction: merged posting size overflows uint64"); + } + total_docs += run_docs; + total_pos += run_positions; + matching.push_back(ri); + } + DCHECK(!matching.empty()); + const RunPostingShape shape = posting_shape(readers[matching.front()]->current()); + for (size_t ri : matching) { + if (posting_shape(readers[ri]->current()) != shape) { + return Status::Error( + "run: posting shape differs across matching terms"); + } + } + const bool term_has_positions = shape == RunPostingShape::kPositioned; + const bool statless = shape == RunPostingShape::kDocsOnlyStatless; + merged.retain_positions = term_has_positions; + if (total_docs > std::numeric_limits::max() || + total_pos > std::numeric_limits::max()) { + return Status::Error( + "run compaction: merged posting exceeds addressable memory"); + } + RETURN_IF_ERROR(reserve_vector_for_size(&merged.docids, static_cast(total_docs), + memory_reporter, &merged_docids_reservation)); + if (!statless) { + RETURN_IF_ERROR(reserve_vector_for_size(&merged.freqs, static_cast(total_docs), + memory_reporter, &merged_freqs_reservation)); + } + if (term_has_positions) { + RETURN_IF_ERROR(reserve_vector_for_size(&merged.positions_flat, + static_cast(total_pos), memory_reporter, + &merged_positions_reservation)); + } + // concat (WITH boundary-doc coalescing) is deliberately the SAME + // append the final merge applies: coalescing the seam between two + // adjacent input runs here yields exactly what the final merge would + // have produced from the uncompacted pair, so compaction is invisible + // in the emitted term stream. + for (size_t ri : matching) { + RunReader* r = readers[ri].get(); + if (term_has_positions) { + RETURN_IF_ERROR(r->materialize_positions()); + } + concat(&merged, r->current(), shape); + } + RETURN_IF_ERROR(w.write_term(id, merged)); + for (size_t ri : matching) { + RunReader* r = readers[ri].get(); + RETURN_IF_ERROR(r->advance()); + if (!r->exhausted()) { + if (r->current_id() >= string_rank.size()) { + return Status::Error( + "run term_id out of rank range"); + } + heap.push({r->current_id(), ri}); + } + } + } + return w.close(); +} + +} // namespace doris::snii::writer diff --git a/be/src/storage/index/snii/writer/spill_run_codec.h b/be/src/storage/index/snii/writer/spill_run_codec.h new file mode 100644 index 00000000000000..f63bd06d6a9156 --- /dev/null +++ b/be/src/storage/index/snii/writer/spill_run_codec.h @@ -0,0 +1,236 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/writer/spimi_term_buffer.h" + +namespace doris::snii::writer { + +using TermKeyMaterializer = std::function; + +// On-disk SPIMI "run" codec for the spill / k-way-merge out-of-core build path. +// +// A RUN is a self-describing file holding a sequence of terms keyed by TERM-ID, +// each followed by its postings, in this exact wire layout. The file is produced +// and consumed by THIS module only (a private temp file -- the on-disk INDEX is +// unaffected), so the format is chosen for cheap I/O: docids, freqs and positions +// are ALL RAW fixed-width little-endian u32 BLOCKS (bulk memcpy on both ends, +// ~10x cheaper than per-value varint -- which cost ~1.5s of encode CPU over the +// 5M build's ~60M docids and compressed those streams poorly anyway). Decode +// still validates every length against the file size. +// +// run := record* (term-ids ordered by vocab string, +// strictly ascending within a run) +// record := +// VInt term_id (index into the shared vocabulary) +// VInt shape (0=docs-only-statless, 1=docs+freq, +// 2=positioned) +// VInt n_docs +// u32 docid * n_docs (RAW LE absolute ascending docids) +// shape=1: u32 freq * n_docs (RAW LE, each >= 1) +// shape=2: u32 freq * n_docs, VInt n_pos, u32 position * n_pos +// (n_pos == sum(freqs)) +// +// Shape 0 is the CommonGrams docs-only set representation. It writes neither +// synthetic all-one frequencies nor n_pos; run files are private temporaries, +// so this does not change the persisted SNII index format. +// +// Decode is fully STREAMED: a RunReader reads a small fixed buffer at a time and +// materializes only the CURRENT term's postings, never the whole run. The k-way +// merge keeps one heap slot per run (each holding only its current term-id + +// that term's postings), so peak memory is bounded by the widest single term +// summed across the runs that contain it -- not by total postings. The merge +// orders runs by a PRECOMPUTED integer string-rank (term-id -> its lexicographic +// rank over the shared dense vocabulary): an integer compare that reproduces the +// exact lexicographic order without touching a vocab string in the inner loops. + +// Writes a sorted sequence of terms (by id) to one run file. Term-ids must be +// handed to write_term in vocab-string ascending order (the spill caller sorts +// before spilling). RAII: the file is flushed and closed on close(); the partial +// file is left for the owning SpimiTermBuffer to delete on its temp-path list. +class RunWriter { +public: + explicit RunWriter(MemoryReporter* memory_reporter = nullptr); + ~RunWriter(); + + RunWriter(const RunWriter&) = delete; + RunWriter& operator=(const RunWriter&) = delete; + + // Opens `path` for writing (truncating). Returns IoError on failure. + Status open(const std::string& path); + + // Appends one term's postings under `term_id`. Empty freqs denotes the + // docs-only-statless shape; otherwise freqs parallels docids. Positioned + // postings additionally hold sum(freqs) positions in document order. + Status write_term(uint32_t term_id, const TermPostings& tp); + + // Flushes the buffer and closes the file. Safe to call once; idempotent. + Status close(); + +private: + Status flush(); + Status append_bytes(const uint8_t* data, size_t size); + Status append_varint(uint64_t value); + Status append_raw_u32(const uint32_t* values, size_t count); + void release_buffer(); + + MemoryReporter* memory_reporter_ = nullptr; + MemoryReporter::Reservation buffer_reservation_; + int fd_ = -1; + std::vector buf_; // bounded staging buffer; flushed in fixed-size chunks +}; + +// Streamed reader over one run file. After open() the first term is loaded; +// current()/current_id() expose it; advance() loads the next (or marks +// exhausted). Only the current term's postings live in memory at a time. The +// current record's `term` string is left EMPTY -- runs store only the id; the +// owner resolves the string via the shared vocabulary. +// +// LAZY POSITIONS (peak-RSS optimization for the widest merged term): advance() +// loads term_id / docids / freqs and the position-block COUNT, but does NOT read +// the position bytes -- it leaves the decode window cursor parked at the start of +// the position block. The owner then chooses, per term: +// * materialize_positions(): bulk-reads the block into current().positions_flat +// (the default; behaves exactly as the old eager reader). +// * stream_positions(dst, n): pulls the next n positions straight from the +// window in 64 KiB chunks, never materializing the whole block -- used by the +// k-way merge source to decode directly into each writer-owned window. +// advance() drains any positions left unread from the previous term before the +// next record, so a partly-streamed (or skipped) term still lands at the right +// record boundary. The yielded byte sequence is identical either way. +class RunReader { +public: + explicit RunReader(MemoryReporter* memory_reporter = nullptr); + ~RunReader(); + + RunReader(const RunReader&) = delete; + RunReader& operator=(const RunReader&) = delete; + + // Opens `path`, loading the first record (if any). has_positions declares + // whether this run may contain positioned or mixed statless records. + Status open(const std::string& path, bool has_positions); + + bool exhausted() const { return exhausted_; } + const TermPostings& current() const { return current_; } + uint32_t current_id() const { return current_id_; } + + // Number of positions in the current term's (lazily-loaded) position block. + uint64_t current_pos_count() const { return pos_count_; } + // True once the current term's positions have been materialized OR fully + // streamed (i.e. nothing remains to read before advance()). + bool positions_drained() const { return pos_remaining_ == 0; } + + // Materializes the current term's position block into current().positions_flat + // (bulk read). Idempotent within a term: a no-op once positions are drained. + Status materialize_positions(); + // Streams the next `n` positions of the current term into dst[0..n) directly + // from the decode window (64 KiB chunks topped up on demand). Caller must not + // request more than positions_remaining(); each call advances the cursor. + Status stream_positions(uint32_t* dst, size_t n); + uint64_t positions_remaining() const { return pos_remaining_; } + + // Loads the next record into current(); sets exhausted() at end of file. Any + // positions of the current term left unread are skipped first. + Status advance(); + +private: + size_t available() const; // buffered bytes from pos_ to window end + Status fill(); // tops up the decode window from disk + Status ensure(size_t n); // guarantees >= n buffered bytes (or eof) + Status read_varint(uint64_t* v); // bounds-checked streamed varint + // Bulk-reads `count` RAW little-endian u32s from the window into `out` (resized + // to count). Bounds-checked against the run's true length (Corruption on EOF). + Status read_raw_u32(size_t count, std::vector* out, + MemoryReporter::Reservation* reservation); + // Streams `count` raw u32s from the window into dst (caller-owned, sized by the + // caller); shared by read_raw_u32 (into a vector) and stream_positions. + Status pull_raw_u32(uint8_t* dst, size_t count); + // Drains (and discards) any remaining positions of the current term so the + // window cursor lands at the next record boundary. + Status skip_remaining_positions(); + + MemoryReporter* memory_reporter_ = nullptr; + // Reservations precede their vectors so allocations are destroyed first. + MemoryReporter::Reservation window_reservation_; + MemoryReporter::Reservation docids_reservation_; + MemoryReporter::Reservation freqs_reservation_; + MemoryReporter::Reservation positions_reservation_; + int fd_ = -1; + bool has_positions_ = false; + bool exhausted_ = false; + uint64_t file_size_ = 0; // total run byte size (fstat at open); bounds lengths + uint64_t bytes_read_ = 0; // bytes pulled from fd; never exceeds file_size_ + std::vector window_; // sliding decode window + size_t pos_ = 0; // consumed offset within window_ + bool eof_ = false; // no more bytes on disk + uint32_t current_id_ = 0; // current record's term-id + uint64_t pos_count_ = 0; // current term's total position count (from n_pos) + uint64_t pos_remaining_ = 0; // positions still unread in the current block + TermPostings current_; +}; + +// K-way merges the given run files into a single term stream ordered by a +// PRECOMPUTED integer string-rank (string_rank[term_id] == the term-id's +// lexicographic rank over the dense vocabulary), invoking `fn` once per distinct +// term-id with its postings concatenated across all runs that contain it (in run +// order -> docids stay ascending) and its `term` resolved from `vocab` once. +// Because a dense vocab maps each id to a distinct string, the rank is a +// lexicographic bijection: ordering by the dense 4 B rank array (an integer +// compare) reproduces the EXACT order a vocab-string compare would -- but never +// reads a vocab string in the inner heap/gather loops. Only one bounded posting +// window is materialized at a time. Returns IoError/Corruption on bad run data, or +// InternalError when string_rank.size() != vocab.size(). has_positions must match +// how the runs were written. `vocab` (term-id -> string) and `string_rank` +// (term-id -> rank) are both borrowed and MUST be sized to the vocabulary. +// +// The source callback is synchronous: matching run readers remain parked on the +// current term until the callback returns. The source fills writer-owned windows +// directly and coalesces equal docids at run boundaries. A successful callback +// must exhaust the source. +Status merge_run_sources(const std::vector& run_paths, + const std::vector& vocab, + const std::vector& string_rank, bool has_positions, + const StreamedTermConsumer& fn, + TermKeyMaterializer materialize_term_key = {}, + MemoryReporter* memory_reporter = nullptr); + +// G09 run-file cap support: k-way merges `run_paths` into ONE new run file at +// `out_path`, keyed and ordered exactly like merge_run_sources (heap on +// string_rank[term_id]; per-term postings concatenated across runs in run +// order, boundary docs coalesced -- the same concat the final merge applies, +// so compact-then-merge emits the identical term stream as merging the +// originals). Positions are fully materialized for each term because the run +// codec serializes positions_flat. +// Every record's term-id must index string_rank (else Corruption). On error +// `out_path` may hold a partial file the caller must delete; the input runs +// are never modified. Opens run_paths.size() read fds + 1 write fd for the +// call's duration -- the caller (SpimiTermBuffer::compact_runs) bounds that +// fan-in with its run-count cap. +Status compact_runs(const std::vector& run_paths, + const std::vector& string_rank, bool has_positions, + const std::string& out_path, MemoryReporter* memory_reporter = nullptr); + +} // namespace doris::snii::writer diff --git a/be/src/storage/index/snii/writer/spillable_byte_buffer.h b/be/src/storage/index/snii/writer/spillable_byte_buffer.h new file mode 100644 index 00000000000000..2760600b00f019 --- /dev/null +++ b/be/src/storage/index/snii/writer/spillable_byte_buffer.h @@ -0,0 +1,305 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "io/fs/file_reader.h" +#include "io/fs/file_reader_writer_fwd.h" +#include "io/fs/file_writer.h" +#include "io/fs/local_file_system.h" +#include "io/fs/path.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/io/file_writer.h" +#include "storage/index/snii/writer/memory_reporter.h" +#include "storage/index/snii/writer/temp_dir.h" +#include "util/slice.h" + +namespace doris::snii::writer { + +// A tiered append buffer for one build-time section. While resident it holds the +// bytes as a CHAIN OF CHUNKS (one per append) rather than a single growing vector. +// Copying appends own a right-sized allocation. Move appends preserve the caller's +// allocation, including capacity slack, and account that retained capacity as resident +// memory. Once the resident capacity crosses `cap_bytes` the buffer SPILLS to a temp +// file (resolve_temp_dir()) and routes later appends there, so a huge section stays +// RSS-bounded at ~cap_bytes while a small one is RAM-only (zero disk, spill-only build). +// Append order/bytes are identical wherever they land; stream_into() reproduces the +// logical section bytes in order. RAII-removes the temp. (cap_bytes == UINT64_MAX +// disables spilling -> always RAM.) +class SpillableByteBuffer { +public: + // `reporter` is an OPTIONAL writer-level build-RAM reporter (null off-Doris / + // unit tests). Resident chunk capacities are hard-reserved before adoption or + // allocation. If the shared cap cannot admit the next chunk, the buffer spills + // its resident prefix first and writes that chunk directly to disk. Spilled + // bytes are not resident and hold no reservation. + SpillableByteBuffer(uint64_t cap_bytes, std::string tag, MemoryReporter* reporter = nullptr) + : cap_bytes_(cap_bytes), + tag_(std::move(tag)), + reporter_(reporter), + reservation_(reporter == nullptr ? MemoryReporter::Reservation() + : reporter->make_reservation()) {} + ~SpillableByteBuffer() { release_storage(); } + SpillableByteBuffer(const SpillableByteBuffer&) = delete; + SpillableByteBuffer& operator=(const SpillableByteBuffer&) = delete; + + // Total bytes appended so far (the offset basis for callers recording sub-offsets). + uint64_t size() const { + return consumed_ ? consumed_size_ : (spilled_ ? spilled_bytes_ : ram_bytes_); + } + + // Copying append (the Slice bytes are copied into a fresh chunk). + Status append(Slice bytes) { + if (consumed_) { + return Status::Error( + "spillable buffer: append after stream"); + } + if (spilled_) { + const ::doris::Slice s(bytes.data(), bytes.size()); + RETURN_IF_ERROR(to_snii(temp_writer_->appendv(&s, 1))); + spilled_bytes_ += bytes.size(); + return Status::OK(); + } + if (!bytes.empty()) { + bool keep_resident = true; + RETURN_IF_ERROR(reserve_resident_capacity(bytes.size(), &keep_resident)); + if (!keep_resident) { + const ::doris::Slice s(bytes.data(), bytes.size()); + RETURN_IF_ERROR(to_snii(temp_writer_->appendv(&s, 1))); + spilled_bytes_ += bytes.size(); + return Status::OK(); + } + std::vector chunk; + chunk.reserve(bytes.size()); + DCHECK_EQ(chunk.capacity(), bytes.size()); + chunk.insert(chunk.end(), bytes.data(), bytes.data() + bytes.size()); + chunks_.push_back(std::move(chunk)); + ram_bytes_ += bytes.size(); + ram_capacity_bytes_ += chunks_.back().capacity(); + } + if (over_cap()) return spill_to_disk(); + return Status::OK(); + } + + // Move append: the section ADOPTS the caller's vector without copying. The common + // dict path hands off each flushed block this way. Logical bytes remain v.size(), + // while resident accounting uses the capacity retained by the adopted vector. + Status append_move(std::vector&& v) { + if (consumed_) { + return Status::Error( + "spillable buffer: append after stream"); + } + if (spilled_) { + const ::doris::Slice s(v.data(), v.size()); + RETURN_IF_ERROR(to_snii(temp_writer_->appendv(&s, 1))); + spilled_bytes_ += v.size(); + return Status::OK(); + } + if (!v.empty()) { + const size_t logical_bytes = v.size(); + const size_t retained_capacity = v.capacity(); + bool keep_resident = true; + RETURN_IF_ERROR(reserve_resident_capacity(retained_capacity, &keep_resident)); + if (!keep_resident) { + const ::doris::Slice s(v.data(), v.size()); + RETURN_IF_ERROR(to_snii(temp_writer_->appendv(&s, 1))); + spilled_bytes_ += v.size(); + return Status::OK(); + } + chunks_.push_back(std::move(v)); + ram_bytes_ += logical_bytes; + ram_capacity_bytes_ += chunks_.back().capacity(); + DCHECK_EQ(chunks_.back().capacity(), retained_capacity); + } + if (over_cap()) return spill_to_disk(); + return Status::OK(); + } + + // Must be called once after the last append, before stream_into(): flushes the temp + // (if spilled) so it can be read back. A no-op for a RAM-resident buffer. + Status seal() { + if (consumed_) { + return Status::Error( + "spillable buffer: seal after stream"); + } + if (spilled_ && !sealed_) { + RETURN_IF_ERROR(to_snii(temp_writer_->close())); + sealed_ = true; + } + return Status::OK(); + } + + // Streams the whole section (RAM chunks or sealed temp) into `out`, in append order. + Status stream_into(io::FileWriter* out) const { + if (consumed_) { + return Status::Error( + "spillable buffer: stream called twice"); + } + if (!spilled_) { + for (const auto& c : chunks_) { + if (!c.empty()) RETURN_IF_ERROR(out->append(Slice(c))); + } + return Status::OK(); + } + ::doris::io::FileReaderSPtr reader; + RETURN_IF_ERROR( + to_snii(::doris::io::global_local_filesystem()->open_file(temp_path_, &reader))); + constexpr uint64_t kMaxChunk = 1u << 20; // bounded copy window (no whole-section reload) + size_t read_capacity = static_cast(std::min(kMaxChunk, spilled_bytes_)); + MemoryReporter::Reservation read_reservation = reporter_ == nullptr + ? MemoryReporter::Reservation() + : reporter_->make_reservation(); + if (reporter_ != nullptr) { + Status reserve_status = read_reservation.set_bytes(read_capacity); + while (!reserve_status.ok() && reserve_status.is() && + read_capacity > 1) { + read_capacity = std::max(1, read_capacity / 2); + reserve_status = read_reservation.set_bytes(read_capacity); + } + RETURN_IF_ERROR(reserve_status); + } + std::vector buf; + buf.reserve(read_capacity); + DCHECK_EQ(buf.capacity(), read_capacity); + for (uint64_t off = 0; off < spilled_bytes_; off += read_capacity) { + const uint64_t n = std::min(read_capacity, spilled_bytes_ - off); + buf.resize(static_cast(n)); + size_t bytes_read = 0; + RETURN_IF_ERROR(to_snii(reader->read_at( + off, ::doris::Slice(buf.data(), static_cast(n)), &bytes_read))); + if (bytes_read != n) { + return Status::Error( + "short read from spill scratch file"); + } + RETURN_IF_ERROR(out->append(Slice(buf.data(), static_cast(n)))); + } + return Status::OK(); + } + + // The staged bytes have no readers after they are copied into the compound + // output. Releasing them here, rather than in the logical-writer destructor, + // keeps multi-destination compaction from retaining one DICT image per + // completed index until rowset close. + Status stream_into_and_release(io::FileWriter* out) { + RETURN_IF_ERROR(stream_into(out)); + release_storage(); + return Status::OK(); + } + + bool spilled() const { return spilled_; } + +private: + Status reserve_resident_capacity(size_t additional_capacity, bool* keep_resident) { + DCHECK(keep_resident != nullptr); + *keep_resident = true; + if (reporter_ == nullptr) return Status::OK(); + if (additional_capacity > std::numeric_limits::max() - ram_capacity_bytes_) { + return Status::Error( + "spillable buffer: resident capacity overflows uint64"); + } + const Status reserved = reservation_.set_bytes(ram_capacity_bytes_ + additional_capacity); + if (reserved.ok()) return Status::OK(); + if (!reserved.is()) return reserved; + RETURN_IF_ERROR(spill_to_disk()); + *keep_resident = false; + return Status::OK(); + } + + // Gate-2 spill condition (UNIFIED): spill when the writer's TOTAL build RAM crosses + // the one shared cap (reporter_->over_cap()), with the local cap_bytes_ kept only as + // a defensive per-buffer hard ceiling (e.g. when no reporter is attached). + bool over_cap() const { + return (reporter_ != nullptr && reporter_->over_cap()) || ram_capacity_bytes_ >= cap_bytes_; + } + // Bridge a Doris IO Status into SNII's Status. R01 (status migration) is not done yet, + // so this buffer still returns Status; this mirrors snii_doris_adapter's + // to_snii_status (ok -> OK, otherwise IoError carrying the Doris message). + static Status to_snii(const Status& s) { + if (s.ok()) return Status::OK(); + return Status::Error(s.to_string_no_stack()); + } + Status spill_to_disk() { + temp_path_ = resolve_temp_dir() + "/snii_" + tag_ + "_" + std::to_string(::getpid()) + "_" + + std::to_string(reinterpret_cast(this)) + ".tmp"; + RETURN_IF_ERROR(to_snii( + ::doris::io::global_local_filesystem()->create_file(temp_path_, &temp_writer_))); + for (const auto& c : chunks_) { + if (!c.empty()) { + const ::doris::Slice s(c.data(), c.size()); + RETURN_IF_ERROR(to_snii(temp_writer_->appendv(&s, 1))); + } + } + spilled_bytes_ = ram_bytes_; + std::vector>().swap(chunks_); // reclaim the RAM immediately + ram_bytes_ = 0; + ram_capacity_bytes_ = 0; + reservation_.reset(); + spilled_ = true; + return Status::OK(); + } + + void release_storage() { + if (consumed_) { + return; + } + consumed_size_ = spilled_ ? spilled_bytes_ : ram_bytes_; + std::vector>().swap(chunks_); + ram_bytes_ = 0; + ram_capacity_bytes_ = 0; + reservation_.reset(); + + // A sealed temp writer is already closed; an unsealed error-path writer + // aborts on reset. In both cases remove the scratch path best-effort. + temp_writer_.reset(); + if (!temp_path_.empty()) { + std::remove(temp_path_.c_str()); + temp_path_.clear(); + } + spilled_bytes_ = 0; + consumed_ = true; + } + + uint64_t cap_bytes_; + std::string tag_; + MemoryReporter* reporter_ = nullptr; // optional build-RAM reporter (null off-Doris) + MemoryReporter::Reservation reservation_; // resident inner-vector capacities + std::vector> chunks_; // resident tier: one chunk per append + uint64_t ram_bytes_ = 0; // logical section bytes retained in RAM + uint64_t ram_capacity_bytes_ = 0; // resident capacities of the retained chunks + bool spilled_ = false; + bool sealed_ = false; + ::doris::io::FileWriterPtr temp_writer_; // Doris local writer for the spill scratch file + std::string temp_path_; + uint64_t spilled_bytes_ = 0; + uint64_t consumed_size_ = 0; + bool consumed_ = false; +}; + +} // namespace doris::snii::writer diff --git a/be/src/storage/index/snii/writer/spimi_term_buffer.cpp b/be/src/storage/index/snii/writer/spimi_term_buffer.cpp new file mode 100644 index 00000000000000..51150a573463a9 --- /dev/null +++ b/be/src/storage/index/snii/writer/spimi_term_buffer.cpp @@ -0,0 +1,2081 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/writer/spimi_term_buffer.h" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/exception.h" +#include "storage/index/inverted/common_grams/common_grams_key_codec.h" +#include "storage/index/inverted/common_grams/common_word_set.h" +#include "storage/index/snii/encoding/varint.h" +#include "storage/index/snii/format/format_constants.h" +#include "storage/index/snii/writer/global_memory_limiter.h" +#include "storage/index/snii/writer/spill_run_codec.h" +#include "storage/index/snii/writer/temp_dir.h" + +namespace doris::snii::writer { + +namespace { + +constexpr size_t kCommonGramPairKeySize = 10; +constexpr char kCommonGramPairKeyTag = 'P'; + +std::array EncodeCommonGramPairKey(PlainTermId left, + PlainTermId right) { + std::array key {}; + key[0] = segment_v2::inverted_index::INTERNAL_TERM_NAMESPACE_BEGIN.front(); + key[1] = kCommonGramPairKeyTag; + for (size_t byte = 0; byte < sizeof(uint32_t); ++byte) { + const size_t shift = (sizeof(uint32_t) - byte - 1) * 8; + key[2 + byte] = static_cast((left.value >> shift) & 0xffU); + key[6 + byte] = static_cast((right.value >> shift) & 0xffU); + } + return key; +} + +bool is_common_gram_pair_key(std::string_view key) { + return key.size() == kCommonGramPairKeySize && + key.front() == segment_v2::inverted_index::INTERNAL_TERM_NAMESPACE_BEGIN.front() && + key[1] == kCommonGramPairKeyTag; +} + +struct CommonGramPairIds { + PlainTermId left; + PlainTermId right; +}; + +uint32_t decode_big_endian_uint32_unchecked(const char* bytes) { + uint32_t value = 0; + for (size_t byte = 0; byte < sizeof(uint32_t); ++byte) { + value = (value << 8) | static_cast(bytes[byte]); + } + return value; +} + +#ifdef BE_TEST +std::atomic g_common_gram_pair_unchecked_decodes {0}; +std::atomic g_common_gram_trusted_plain_decodes {0}; +std::atomic g_common_gram_pair_cache_probes {0}; +std::atomic g_common_gram_pair_cache_pair_hits {0}; +std::atomic g_common_gram_pair_cache_same_doc_hits {0}; +std::atomic g_common_gram_native_pair_probes {0}; +std::atomic g_common_gram_native_pair_hits {0}; +std::atomic g_common_gram_native_pair_inserts {0}; +std::atomic g_common_gram_logical_validations {0}; +std::atomic g_common_gram_plain_cache_probes {0}; +std::atomic g_common_gram_plain_cache_hits {0}; +std::atomic g_common_gram_plain_intern_table_probes {0}; +std::atomic g_owned_term_full_byte_comparisons {0}; +std::atomic g_fail_next_owned_term_reserve {false}; +std::atomic g_fail_next_owned_term_emplace {false}; +std::atomic g_spill_gate_checks {0}; +std::atomic g_compact_chain_varint_decodes {0}; +#endif + +CommonGramPairIds decode_common_gram_pair_key_unchecked(std::string_view key) { + DCHECK(is_common_gram_pair_key(key)); +#ifdef BE_TEST + g_common_gram_pair_unchecked_decodes.fetch_add(1, std::memory_order_relaxed); +#endif + return { + .left = PlainTermId {.value = decode_big_endian_uint32_unchecked(key.data() + 2)}, + .right = PlainTermId {.value = decode_big_endian_uint32_unchecked(key.data() + 6)}, + }; +} + +class LogicalPlainKeyView { +public: + explicit LogicalPlainKeyView(std::string_view physical) : physical_(physical) { + escaped_ = !physical.empty() && + physical.front() == segment_v2::inverted_index::PLAIN_ESCAPE_PREFIX; + if (escaped_) { + DCHECK_GE(physical.size(), 2U); + DCHECK(physical[1] == 'E' || physical[1] == 'G'); + } + } + + size_t size() const { return physical_.size() - (escaped_ ? 1 : 0); } + + uint8_t operator[](size_t index) const { + DCHECK_LT(index, size()); + if (!escaped_) { + return static_cast(physical_[index]); + } + if (index == 0) { + return static_cast( + physical_[1] == 'E' ? segment_v2::inverted_index::PLAIN_ESCAPE_PREFIX : '\x1f'); + } + return static_cast(physical_[index + 1]); + } + +private: + std::string_view physical_; + bool escaped_ = false; +}; + +std::string_view decode_logical_plain_term_trusted(std::string_view physical, + std::string* scratch) { + DCHECK(scratch != nullptr); +#ifdef BE_TEST + g_common_gram_trusted_plain_decodes.fetch_add(1, std::memory_order_relaxed); +#endif + scratch->clear(); + if (physical.empty() || physical.front() != segment_v2::inverted_index::PLAIN_ESCAPE_PREFIX) { + DCHECK(!segment_v2::inverted_index::is_internal_term_key(physical)); + return physical; + } + + DCHECK_GE(physical.size(), 2U); + DCHECK(physical[1] == 'E' || physical[1] == 'G'); + scratch->reserve(physical.size() - 1); + scratch->push_back(physical[1] == 'E' + ? segment_v2::inverted_index::PLAIN_ESCAPE_PREFIX + : segment_v2::inverted_index::INTERNAL_TERM_NAMESPACE_BEGIN.front()); + scratch->append(physical.substr(2)); + return std::string_view(*scratch); +} + +int compare_logical_plain_keys(const LogicalPlainKeyView& left, const LogicalPlainKeyView& right) { + const size_t common = std::min(left.size(), right.size()); + for (size_t i = 0; i < common; ++i) { + if (left[i] != right[i]) { + return left[i] < right[i] ? -1 : 1; + } + } + if (left.size() == right.size()) { + return 0; + } + return left.size() < right.size() ? -1 : 1; +} + +} // namespace + +struct SpimiTermBuffer::CommonGramPairCache { + struct Entry { + uint64_t pair = 0; + uint32_t term_id = std::numeric_limits::max(); + uint32_t last_docid = 0; + }; + static_assert(sizeof(Entry) == 16); + + static constexpr size_t kEntryCount = 1024; + static constexpr uint64_t kHashMultiplier = 11400714819323198485ULL; + static constexpr uint32_t kInvalidTermId = std::numeric_limits::max(); + + static size_t index(uint64_t pair) { + return static_cast((pair * kHashMultiplier) >> 54); + } + + std::array entries; + static_assert(sizeof(std::array) == 16 * 1024); +}; + +struct SpimiTermBuffer::CommonGramPlainTermCache { + struct Entry { + uint32_t fingerprint = 0; + uint32_t term_id = std::numeric_limits::max(); + }; + static_assert(sizeof(Entry) == 8); + + static constexpr size_t kSetCount = 1024; + static constexpr uint32_t kInvalidTermId = std::numeric_limits::max(); + using Set = std::array; + static_assert((kSetCount & (kSetCount - 1)) == 0); + + static size_t index(size_t term_hash) { return term_hash & (kSetCount - 1); } + + static uint32_t fingerprint(size_t term_hash) { + const auto hash = static_cast(term_hash); + return static_cast(hash ^ (hash >> 32)); + } + + static bool matches(const Entry& entry, uint32_t expected_fingerprint, std::string_view term, + const std::vector& vocab) { + if (entry.term_id == kInvalidTermId || entry.fingerprint != expected_fingerprint) { + return false; + } + DCHECK_LT(entry.term_id, vocab.size()); + return std::string_view(vocab[entry.term_id]) == term; + } + + uint32_t find(size_t term_hash, std::string_view term, const std::vector& vocab) { + Set& set = sets[index(term_hash)]; + const uint32_t expected_fingerprint = fingerprint(term_hash); + if (matches(set[0], expected_fingerprint, term, vocab)) { + return set[0].term_id; + } + if (matches(set[1], expected_fingerprint, term, vocab)) { + std::swap(set[0], set[1]); + return set[0].term_id; + } + return kInvalidTermId; + } + + void remember(size_t term_hash, uint32_t term_id) { + Set& set = sets[index(term_hash)]; + set[1] = set[0]; + set[0] = Entry {.fingerprint = fingerprint(term_hash), .term_id = term_id}; + } + + std::array sets {}; + static_assert(sizeof(std::array) == 16 * 1024); +}; + +bool SpimiTermBuffer::OwnedVocabEq::operator()(uint32_t stored, + std::string_view probe) const noexcept { +#ifdef BE_TEST + g_owned_term_full_byte_comparisons.fetch_add(1, std::memory_order_relaxed); +#endif + DCHECK_LT(stored, vocab->size()); + return std::string_view((*vocab)[stored]) == probe; +} + +bool SpimiTermBuffer::OwnedVocabEq::operator()(std::string_view probe, + uint32_t stored) const noexcept { + return (*this)(stored, probe); +} + +#ifdef BE_TEST +size_t SpimiTermBuffer::owned_term_key_size_for_test() { + return sizeof(decltype(intern_)::key_type); +} + +void SpimiTermBuffer::set_owned_term_hash_mask_for_test(size_t mask) { + DORIS_CHECK(intern_.empty()); + intern_ = decltype(intern_)(0, OwnedVocabHash {.vocab = &owned_vocab_, .hash_mask = mask}, + OwnedVocabEq {&owned_vocab_}); +} +#endif + +namespace { + +// Process-unique temp path for a spill run under `dir` (pid + monotonic counter so +// parallel builds / multiple buffers never collide). +std::string make_run_path(const std::string& dir) { + static std::atomic counter {0}; + const uint64_t n = counter.fetch_add(1); + return dir + "/snii_spill_" + std::to_string(::getpid()) + "_" + std::to_string(n) + ".run"; +} + +// TEST-ONLY seam backing testing::vocab_string_materialization_count(). Bumped once +// per DISTINCT interned term (owned_vocab_.emplace_back), never per token. Relaxed: +// the build path is single-threaded, so only the COUNT matters, not ordering. +#ifdef BE_TEST +std::atomic g_vocab_materializations {0}; +#endif + +// G09 seam: spills that consumed a pending process-wide forced-spill request +// (the limiter flagged this buffer as one of the largest reclaimable-arena +// consumers while SNII was over its memory share). Incremented under BE_TEST only +// (per-token path shared by concurrent writers). +std::atomic g_global_forced_spills {0}; + +// G09 run-file cap seam: merge-compactions of a buffer's run list (always-on: +// at most one per cap-many spills, contention-free). +std::atomic g_run_compactions {0}; + +// Test seam for complete-vocabulary rank rebuilds. The increment is compiled +// out of production because ensure_string_rank() may run on the import path. +#ifdef BE_TEST +std::atomic g_string_rank_rebuilds {0}; +std::atomic g_dense_rank_inversions {0}; +std::atomic g_rank_comparison_sorts {0}; +#endif + +// G11 bench seam: when set (BE_TEST paths only), the add-path prefetch hints +// are skipped so the locality bench can A/B them in one process. Production +// builds never read it (the hint compiles in unconditionally there). +std::atomic g_bench_disable_g11_prefetch {false}; + +// G11 add-path prefetch gate: always-on in production; toggleable under +// BE_TEST for the in-process A/B bench. The branch is perfectly predicted, so +// the bench's OFF arm measures the pre-G11 code path faithfully. +inline bool g11_prefetch_enabled() { +#ifdef BE_TEST + return !g_bench_disable_g11_prefetch.load(std::memory_order_relaxed); +#else + return true; +#endif +} + +// G08: heap payload of one owned-vocab string -- 0 while it fits the SSO buffer +// (those bytes live inside the 32 B header owned_vocab_.capacity() charges), else +// the allocated buffer (capacity + NUL). The SSO capacity is probed from the +// running stdlib so the classification is exact, not hardcoded. +uint64_t string_heap_bytes(const std::string& s) { + static const size_t kSsoCapacity = std::string().capacity(); + return s.capacity() > kSsoCapacity ? static_cast(s.capacity()) + 1 : 0; +} + +void order_ids_by_dense_rank(std::vector* ids, const std::vector& rank) { + if (ids->size() == rank.size()) { + // Touched ids are unique. Equal cardinality therefore means the run covers + // the complete vocabulary, so invert the dense rank in linear time. + for (uint32_t term_id = 0; term_id < rank.size(); ++term_id) { + (*ids)[rank[term_id]] = term_id; + } +#ifdef BE_TEST + g_dense_rank_inversions.fetch_add(1, std::memory_order_relaxed); +#endif + return; + } + + std::ranges::sort(*ids, [&](uint32_t a, uint32_t b) { return rank[a] < rank[b]; }); +#ifdef BE_TEST + g_rank_comparison_sorts.fetch_add(1, std::memory_order_relaxed); +#endif +} + +} // namespace + +namespace testing { +void set_bench_disable_g11_prefetch(bool disabled) { + g_bench_disable_g11_prefetch.store(disabled, std::memory_order_relaxed); +} +uint64_t vocab_string_materialization_count() { +#ifdef BE_TEST + return g_vocab_materializations.load(std::memory_order_relaxed); +#else + return 0; +#endif +} +void reset_vocab_string_materialization_count() { +#ifdef BE_TEST + g_vocab_materializations.store(0, std::memory_order_relaxed); +#endif +} +uint64_t global_forced_spills() { + return g_global_forced_spills.load(std::memory_order_relaxed); +} +void reset_global_forced_spills() { + g_global_forced_spills.store(0, std::memory_order_relaxed); +} +uint64_t run_compactions() { + return g_run_compactions.load(std::memory_order_relaxed); +} +void reset_run_compactions() { + g_run_compactions.store(0, std::memory_order_relaxed); +} +uint64_t string_rank_rebuilds() { +#ifdef BE_TEST + return g_string_rank_rebuilds.load(std::memory_order_relaxed); +#else + return 0; +#endif +} +void reset_string_rank_rebuilds() { +#ifdef BE_TEST + g_string_rank_rebuilds.store(0, std::memory_order_relaxed); +#endif +} +uint64_t dense_rank_inversions() { +#ifdef BE_TEST + return g_dense_rank_inversions.load(std::memory_order_relaxed); +#else + return 0; +#endif +} +uint64_t rank_comparison_sorts() { +#ifdef BE_TEST + return g_rank_comparison_sorts.load(std::memory_order_relaxed); +#else + return 0; +#endif +} +void reset_rank_ordering_counts() { +#ifdef BE_TEST + g_dense_rank_inversions.store(0, std::memory_order_relaxed); + g_rank_comparison_sorts.store(0, std::memory_order_relaxed); +#endif +} +uint64_t common_gram_pair_unchecked_decode_count() { +#ifdef BE_TEST + return g_common_gram_pair_unchecked_decodes.load(std::memory_order_relaxed); +#else + return 0; +#endif +} +uint64_t common_gram_trusted_plain_decode_count() { +#ifdef BE_TEST + return g_common_gram_trusted_plain_decodes.load(std::memory_order_relaxed); +#else + return 0; +#endif +} +void reset_common_gram_pair_fast_path_counts() { +#ifdef BE_TEST + g_common_gram_pair_unchecked_decodes.store(0, std::memory_order_relaxed); + g_common_gram_trusted_plain_decodes.store(0, std::memory_order_relaxed); +#endif +} +uint64_t common_gram_pair_cache_probes() { +#ifdef BE_TEST + return g_common_gram_pair_cache_probes.load(std::memory_order_relaxed); +#else + return 0; +#endif +} +uint64_t common_gram_pair_cache_pair_hits() { +#ifdef BE_TEST + return g_common_gram_pair_cache_pair_hits.load(std::memory_order_relaxed); +#else + return 0; +#endif +} +uint64_t common_gram_pair_cache_same_doc_hits() { +#ifdef BE_TEST + return g_common_gram_pair_cache_same_doc_hits.load(std::memory_order_relaxed); +#else + return 0; +#endif +} +void reset_common_gram_pair_cache_counts() { +#ifdef BE_TEST + g_common_gram_pair_cache_probes.store(0, std::memory_order_relaxed); + g_common_gram_pair_cache_pair_hits.store(0, std::memory_order_relaxed); + g_common_gram_pair_cache_same_doc_hits.store(0, std::memory_order_relaxed); +#endif +} +uint64_t common_gram_native_pair_probes() { +#ifdef BE_TEST + return g_common_gram_native_pair_probes.load(std::memory_order_relaxed); +#else + return 0; +#endif +} +uint64_t common_gram_native_pair_hits() { +#ifdef BE_TEST + return g_common_gram_native_pair_hits.load(std::memory_order_relaxed); +#else + return 0; +#endif +} +uint64_t common_gram_native_pair_inserts() { +#ifdef BE_TEST + return g_common_gram_native_pair_inserts.load(std::memory_order_relaxed); +#else + return 0; +#endif +} +void reset_common_gram_native_pair_intern_counts() { +#ifdef BE_TEST + g_common_gram_native_pair_probes.store(0, std::memory_order_relaxed); + g_common_gram_native_pair_hits.store(0, std::memory_order_relaxed); + g_common_gram_native_pair_inserts.store(0, std::memory_order_relaxed); +#endif +} +uint64_t common_gram_logical_validation_count() { +#ifdef BE_TEST + return g_common_gram_logical_validations.load(std::memory_order_relaxed); +#else + return 0; +#endif +} +void reset_common_gram_logical_validation_count() { +#ifdef BE_TEST + g_common_gram_logical_validations.store(0, std::memory_order_relaxed); +#endif +} +uint64_t common_gram_plain_cache_probes() { +#ifdef BE_TEST + return g_common_gram_plain_cache_probes.load(std::memory_order_relaxed); +#else + return 0; +#endif +} +uint64_t common_gram_plain_cache_hits() { +#ifdef BE_TEST + return g_common_gram_plain_cache_hits.load(std::memory_order_relaxed); +#else + return 0; +#endif +} +uint64_t common_gram_plain_intern_table_probes() { +#ifdef BE_TEST + return g_common_gram_plain_intern_table_probes.load(std::memory_order_relaxed); +#else + return 0; +#endif +} +void reset_common_gram_plain_cache_counts() { +#ifdef BE_TEST + g_common_gram_plain_cache_probes.store(0, std::memory_order_relaxed); + g_common_gram_plain_cache_hits.store(0, std::memory_order_relaxed); + g_common_gram_plain_intern_table_probes.store(0, std::memory_order_relaxed); +#endif +} +uint64_t owned_term_full_byte_comparison_count() { +#ifdef BE_TEST + return g_owned_term_full_byte_comparisons.load(std::memory_order_relaxed); +#else + return 0; +#endif +} +void reset_owned_term_full_byte_comparison_count() { +#ifdef BE_TEST + g_owned_term_full_byte_comparisons.store(0, std::memory_order_relaxed); +#endif +} +void fail_next_owned_term_reserve() { +#ifdef BE_TEST + g_fail_next_owned_term_reserve.store(true, std::memory_order_relaxed); +#endif +} +void fail_next_owned_term_emplace() { +#ifdef BE_TEST + g_fail_next_owned_term_emplace.store(true, std::memory_order_relaxed); +#endif +} +uint64_t spill_gate_check_count() { +#ifdef BE_TEST + return g_spill_gate_checks.load(std::memory_order_relaxed); +#else + return 0; +#endif +} +void reset_spill_gate_check_count() { +#ifdef BE_TEST + g_spill_gate_checks.store(0, std::memory_order_relaxed); +#endif +} +uint64_t compact_chain_varint_decode_count() { +#ifdef BE_TEST + return g_compact_chain_varint_decodes.load(std::memory_order_relaxed); +#else + return 0; +#endif +} +void reset_compact_chain_varint_decode_count() { +#ifdef BE_TEST + g_compact_chain_varint_decodes.store(0, std::memory_order_relaxed); +#endif +} +} // namespace testing + +SpimiTermBuffer::SpimiTermBuffer(const std::vector* vocab, bool has_positions, + size_t spill_threshold_bytes, MemoryReporter* reporter) + : vocab_(vocab), + // Bind the equality functor to &owned_vocab_ even in borrowed mode: + // add_token(string_view) rejects before the functor can dereference it, + // and binding unconditionally keeps both constructors symmetric. + // Initialized in the member-init list (NOT the body): the functors are + // NESTED types, whose default-constructibility is not yet established at + // the point the flat set's default ctor would be needed. The + // (bucket_count, hash, equal) constructor sidesteps that entirely. + // owned_vocab_ is constructed before intern_ (declaration order) and the + // buffer is non-movable, so &owned_vocab_ is stable for the buffer's life. + intern_(0, OwnedVocabHash {.vocab = &owned_vocab_}, OwnedVocabEq {&owned_vocab_}), + has_positions_(has_positions), + spill_threshold_bytes_(spill_threshold_bytes), + mem_reporter_(reporter) { + // Borrowed-vocab mode: only the 4 B/id slot-index array is sized to the + // vocabulary; the Term pool (slots_) grows with the LIVE touched count, so an + // all-but-empty vocabulary costs ~4 B/id instead of ~80 B/id. + slot_of_.assign(vocab_->size(), 0); + // The vocab-sized slot index is resident immediately and survives spills; report + // its initial positive delta now. + report_arena_delta(); +} + +SpimiTermBuffer::SpimiTermBuffer(bool has_positions, size_t spill_threshold_bytes, + MemoryReporter* reporter) + : vocab_(&owned_vocab_), + // Owned-vocab mode: bind both functors to the sole vocabulary so stored + // ids can rehash and string probes resolve full term equality. + // Initialized in the member-init list (NOT the body): + // the functors are NESTED types whose default-constructibility is not yet + // established where the flat set's default ctor (whose noexcept spec inspects + // the functors) would be needed for a body assignment, so the + // (bucket_count, hash, equal) constructor is used instead. owned_vocab_ is + // constructed before intern_ (declaration order) and the buffer is + // non-movable, so &owned_vocab_ is stable for the buffer's life. + intern_(0, OwnedVocabHash {.vocab = &owned_vocab_}, OwnedVocabEq {&owned_vocab_}), + has_positions_(has_positions), + spill_threshold_bytes_(spill_threshold_bytes), + mem_reporter_(reporter) { + report_arena_delta(); +} + +SpimiTermBuffer::~SpimiTermBuffer() { + // G09: leave the process-wide registry FIRST. unregister_buffer removes the + // entry (and its bytes) under the registry mutex -- the same mutex every + // flag store is made under -- so once it returns, no other thread can touch + // global_spill_requested_ while this buffer dies. + if (global_limiter_ != nullptr) { + global_limiter_->unregister_buffer(&global_spill_requested_); + global_limiter_ = nullptr; + } + // Balance the writer-level / Doris tracker on the error path: if the buffer is + // destroyed while resident bytes were reported but not yet freed-and-reported + // (e.g. a build aborts before draining), return them here so nothing leaks. + if (mem_reporter_ != nullptr && reported_resident_ != 0) { + mem_reporter_->report(-reported_resident_); + reported_resident_ = 0; + } + cleanup_runs(); +} + +void SpimiTermBuffer::attach_global_limiter(GlobalMemoryLimiter* limiter) { + // At-most-once: a re-attach would leave a stale registry entry behind (the + // dtor un-registers only the current limiter). + if (limiter == nullptr || global_limiter_ != nullptr) { + return; + } + global_limiter_ = limiter; + // Race-safe vs report: registration and every report run on the OWNER's + // thread, strictly ordered; the registry serializes them against other + // buffers' calls internally. Register with the current spillable arena + // bytes (the victim-selection key) so the registry is exact from the first + // moment. The buffer's total memory needs no reporting here: it already + // reaches the limiter through the SniiIndexBuild observation tracker that + // mem_reporter_ feeds. + global_limiter_->register_buffer(&global_spill_requested_, + static_cast(pool_.arena_bytes())); +} + +void SpimiTermBuffer::report_arena_delta() { + if (mem_reporter_ == nullptr && global_limiter_ == nullptr) { + return; + } + // Diff the REAL resident bytes (resident_bytes()) against the last reported + // total; emit the signed delta exactly once. + const auto now = static_cast(resident_bytes()); + // Per-token zero-delta debounce: skip the locked fetch_add when resident is + // unchanged (the common case -- arena_bytes() grows only ~every 32 KiB block and + // the other charged structures grow by geometric capacity steps / per new term + // only, so most tokens see delta==0). A + // delta==0 report() is a no-op (current_.fetch_add(0) plus a mirrored + // consume_release(0)) and leaves reported_resident_ == now, so current_bytes(), + // every over_cap() result, and the gate-2 spill timing stay bit-for-bit identical. + // The spill gate still evaluates the writer-level UNIFIED total whenever the + // arena is large enough to reclaim, even if this buffer's local delta is 0: + // the shared dict buffer may have crossed the cap independently. + if (now == reported_resident_) { + return; + } + if (mem_reporter_ != nullptr) { + mem_reporter_->report(now - reported_resident_); + } + // G09: forward the current SPILLABLE arena bytes -- as an ABSOLUTE, + // self-healing value -- to the process-wide registry (the victim-selection + // key: only the arena is reclaimable by a forced spill; the persistent + // vocab/pair structures are not). The delta reported above has already + // moved the observation tracker the limiter judges the SUM by, so this + // report carries no total of its own. This is the limiter's decision point: + // report() flags the largest-arena eligible buffers (possibly this one) + // while SNII is over its share. It only ever takes the registry mutex and + // flips advisory atomics; no lock is held here while spilling (any spill + // this buffer performs happens AFTER this returns, back in + // maybe_spill_after_token, on this thread). + if (global_limiter_ != nullptr) { + global_limiter_->report(&global_spill_requested_, + static_cast(pool_.arena_bytes())); + } + reported_resident_ = now; +} + +size_t SpimiTermBuffer::unique_terms() const { + return live_term_count_; +} + +uint64_t SpimiTermBuffer::resident_bytes() const { + // Everything live is charged by CAPACITY (the reserved tail is resident RSS + // and survives spills). All reads are O(1), since this runs once per token. + uint64_t b = pool_.arena_bytes(); // posting chains: docs + prx payload + b += static_cast(slot_of_.capacity()) * sizeof(uint32_t); // vocab-sized slot index + b += static_cast(slots_.capacity()) * sizeof(Term); // live Term pool + b += static_cast(free_slots_.capacity()) * sizeof(uint32_t); + b += static_cast(touched_ids_.capacity()) * sizeof(uint32_t); + // Owned-vocab machinery (all zero in borrowed mode): string headers by vector + // capacity, heap payloads via the incrementally-maintained counter, and the + // intern set's entries at a fixed per-entry estimate (kept at the + // pre-G10 node-set value so the gate-2 spill points are unchanged; see the + // constant's comment). + b += static_cast(owned_vocab_.capacity()) * sizeof(std::string); + b += owned_vocab_heap_bytes_; + b += static_cast(common_word_classification_.capacity()) * + sizeof(CommonWordClassification); + b += static_cast(intern_.size() + common_gram_pair_intern_.size()) * + kInternEntryEstimateBytes; + b += common_gram_pair_cache_bytes_; + b += common_gram_plain_term_cache_bytes_; + // Cached lexicographic ranks survive spills and are included by capacity. + b += static_cast(string_rank_.capacity()) * sizeof(uint32_t); + return b; +} + +// Returns the live Term for `term_id`, claiming a pool slot on first touch (1 == +// new). Reuses a freed slot from free_slots_ when available; otherwise appends a +// fresh Term to slots_. slot_of_[term_id] holds (slot index + 1); 0 means empty. +SpimiTermBuffer::Term& SpimiTermBuffer::term_slot(uint32_t term_id, bool* new_term) { + uint32_t enc = slot_of_[term_id]; + if (enc != 0) { + *new_term = false; + return slots_[enc - 1]; + } + *new_term = true; + uint32_t slot; + if (!free_slots_.empty()) { + slot = free_slots_.back(); + free_slots_.pop_back(); + } else { + slot = static_cast(slots_.size()); + slots_.emplace_back(); + } + slot_of_[term_id] = slot + 1; + return slots_[slot]; +} + +void SpimiTermBuffer::put_varint(Term* t, uint64_t v) { + if (t->head == kNoChain) { + t->head = pool_.start_chain(&t->w, &t->level); + } + if (v < 0x80U) { + pool_.append_byte(&t->w, &t->level, static_cast(v)); + return; + } + pool_.append_varint(&t->w, &t->level, v); +} + +void SpimiTermBuffer::accumulate_without_spill_gate(uint32_t term_id, uint32_t docid, uint32_t pos, + PostingChainShape shape) { + const bool retain_positions = shape == PostingChainShape::kTaggedPositioned; + const bool statless_common_gram = shape == PostingChainShape::kStatlessDocsOnly; + DCHECK(!retain_positions || has_positions_); + bool new_term = false; + Term& t = term_slot(term_id, &new_term); + if (new_term) { + t.shape = shape; + touched_ids_.push_back(term_id); + ++live_term_count_; + } else { + DCHECK(t.shape == shape); + } + // Docs-only accelerator postings are sets. Tokens for one input document are + // contiguous on the writer path, so discard repeated occurrences before they + // allocate arena bytes or enter spill/sort/posting encoding. + if (!retain_positions && t.started && t.cur_docid == docid) { + ++total_tokens_; + return; + } + // A token starts a new doc unless it continues the most-recent doc for this term. + const bool first_token = !t.started; + const bool new_doc = first_token || t.cur_docid != docid; + // A statless CommonGram singleton owns no chain. On its second distinct doc, + // backfill the first absolute docid before appending the current delta. + if (statless_common_gram && !first_token && t.head == kNoChain) { + DCHECK_EQ(t.ntok, 1U); + DCHECK_EQ(t.ndocs, 1U); + put_varint(&t, zigzag_encode(static_cast(t.cur_docid))); + } + + // Positioned and ordinary docs-only terms retain the tagged token stream used + // to reconstruct frequency. A statless CommonGram is already deduplicated per + // document and has no frequency, so its new_doc tag would be the constant 1; + // omit it and store only the document delta. + if (!statless_common_gram) { + // Widen to 64-bit so a full 32-bit position survives the shift. + const uint64_t tagged = retain_positions + ? ((static_cast(pos) << 1) | (new_doc ? 1U : 0U)) + : (new_doc ? 1U : 0U); + put_varint(&t, tagged); + } else { + DCHECK(new_doc); + } + if (new_doc) { + // Out-of-order docids are tolerated (zigzag delta is signed) and reordered at + // finalize; flag them so to_postings sorts. The delta base is the previous + // distinct doc (cur_docid), which is 0 for the very first doc (started==false). + const int64_t base = t.started ? static_cast(t.cur_docid) : 0; + if (t.started && docid < t.cur_docid) { + t.sorted = false; + } + const int64_t delta = static_cast(docid) - base; + if (!first_token || !statless_common_gram) { + put_varint(&t, zigzag_encode(delta)); + } + t.cur_docid = docid; + t.started = true; + // Exact new-doc group count; out-of-order coalescing can only shrink it. + ++t.ndocs; + } + ++t.ntok; + ++total_tokens_; +} + +void SpimiTermBuffer::accumulate(uint32_t term_id, uint32_t docid, uint32_t pos, + bool retain_positions) { + accumulate_without_spill_gate(term_id, docid, pos, + retain_positions ? PostingChainShape::kTaggedPositioned + : PostingChainShape::kTaggedDocsOnly); + maybe_spill_after_token(); +} + +// Per-input-token gate-2 tail. Ordinary adds invoke it after one posting; the +// fused CommonGrams path invokes it after its gram and right plain posting. It +// reports the token's REAL resident growth FIRST so the writer's unified total +// (reporter_->current_bytes()) reflects it before the gate check (single-source +// diff; cheap: a subtraction + relaxed atomic add), then evaluates the spill triggers: +// * Gate-2 (UNIFIED): with a reporter attached, trigger on the writer's TOTAL +// build RAM (arena + vocab structures + dict) crossing the one +// configured cap -- the same total and cap every buffer of this writer +// shares, not a per-buffer threshold. Off Doris (no reporter) fall back to +// the local spill_threshold_bytes_ against resident_bytes(). +// * G08 anti-churn floor: a gate-2 spill reclaims ONLY the posting arena +// (pool_.reset()); the vocab / slot structures resident_bytes() +// now also charges SURVIVE it. Once those persistent bytes alone exceed the +// cap, an unconditioned +// trigger would spill EVERY subsequent token -- one-block runs, k-way-merge +// and spill-fixed-cost blowup. Honor the cap only when at least a quarter of +// it is reclaimable arena: peak stays bounded at persistent + cap/4 and no +// run is smaller than cap/4, while the one-block minimum keeps small caps +// (tests, tiny configs) spilling on the first block exactly as before. +// * Hard arena safety stop, active even in unlimited mode and BYPASSING the +// floor: when the arena nears the 4 GiB uint32-offset limit, spill now -- +// without it a single >4 GiB in-memory segment wraps alloc_run and silently +// corrupts data. A forced spill + final k-way merge stays byte-identical +// regardless of when it fires. +// spill_to_run() resets the arena and reports its negative internally, so the +// unified total drops (and the trigger self-rearms) after each spill. +void SpimiTermBuffer::maybe_spill_after_token() { +#ifdef BE_TEST + g_spill_gate_checks.fetch_add(1, std::memory_order_relaxed); +#endif + constexpr uint64_t kArenaSpillCap = 0xE0000000ULL; // 3.5 GiB, < UINT32_MAX margin + const bool global_requested = global_spill_requested_.load(std::memory_order_relaxed); + const bool arena_near_limit = pool_.arena_bytes() >= kArenaSpillCap; + report_arena_delta(); + const uint64_t gate_cap = + mem_reporter_ != nullptr ? mem_reporter_->cap_bytes() : spill_threshold_bytes_; + const bool arena_worth_spilling = + pool_.arena_bytes() >= std::max(CompactPostingPool::kBlockSize, gate_cap / 4); + // G09: the process-wide limiter flagged this buffer (one of the + // largest-ARENA eligible consumers while SNII index-build memory was over + // its share). Honored HERE, on the owner's own thread -- never on the + // reporting thread that set the flag. The G08 anti-churn floor (cap/4) is + // deliberately BYPASSED (each victim's arena is below cap/4 by + // construction: it never reached its per-writer gate -- that is exactly + // why the global sum grew), but the FORCED-SPILL FLOOR + // (snii_forced_spill_min_arena_bytes, >= one arena block so a run is + // writable) still applies: a forced spill reclaims ONLY the arena, so + // honoring below the floor would cut a tiny run for near-zero relief. + // Below the floor the request is a NO-OP that stays PENDING -- it is NOT + // retried as a spill each token -- and is honored once the arena regrows + // past the floor (the limiter's victim selection applies the same floor, + // so a below-floor flag only arises from a floor/config race or a test + // seam). A request that finds the owner already drained is never observed + // again -- an advisory no-op (the dtor un-registers) -- and a stale + // re-request after a spill costs at most one extra floor-sized run + // (double-spill is harmless, byte-identical output). + const bool global_spill_now = + global_requested && + pool_.arena_bytes() >= std::max(CompactPostingPool::kBlockSize, + forced_spill_min_arena_bytes_); + const bool over_cap = !global_spill_now && !arena_near_limit && arena_worth_spilling && + (mem_reporter_ != nullptr ? mem_reporter_->over_cap() + : (spill_threshold_bytes_ != 0 && + resident_bytes() >= spill_threshold_bytes_)); + if ((over_cap || global_spill_now || arena_near_limit) && spill_status_.ok()) { + if (global_requested) { + // Consume the request BEFORE spilling: this spill releases exactly + // the arena a forced spill would, so it satisfies the request no + // matter which trigger won the OR above. + global_spill_requested_.store(false, std::memory_order_relaxed); +#ifdef BE_TEST + // Seam under BE_TEST only: per-token path shared by every + // concurrent writer. + g_global_forced_spills.fetch_add(1, std::memory_order_relaxed); +#endif + } + spill_status_ = spill_to_run(); + } +} + +void SpimiTermBuffer::add_token(uint32_t term_id, uint32_t docid, uint32_t pos) { + add_token(term_id, docid, pos, has_positions_); +} + +void SpimiTermBuffer::add_token(uint32_t term_id, uint32_t docid, uint32_t pos, + bool retain_positions) { + // Hot path: a pooled slot lookup + a couple of pushes. No hashing, no string + // construction per token. Reject (and latch) an out-of-range id. + if (term_id >= slot_of_.size()) { + if (spill_status_.ok()) { + spill_status_ = Status::Error( + "spimi: term_id out of vocab range"); + } + return; + } + accumulate(term_id, docid, pos, retain_positions); +} + +void SpimiTermBuffer::add_token(std::string_view term, uint32_t docid, uint32_t pos) { + add_token(term, docid, pos, has_positions_); +} + +void SpimiTermBuffer::add_token(std::string_view term, uint32_t docid, uint32_t pos, + bool retain_positions) { + // Compatibility path: intern the term into the owned vocabulary on first + // occurrence, then accumulate by its id. ONLY valid in OWNED-vocab mode. In + // BORROWED-vocab mode vocab_ points at the caller's vector, NOT &owned_vocab_: + // interning here would grow owned_vocab_ / intern_ / slot_of_ out of step with + // the active (borrowed) vocab, so the new id indexes the WRONG string and writes + // a slot_of_ entry the borrowed-vocab build never reconciles -- silent + // corruption. Reject (and latch) instead of forwarding by a bogus id. + if (vocab_ != &owned_vocab_) { + if (spill_status_.ok()) { + spill_status_ = Status::Error( + "spimi: add_token(string_view) requires owned-vocab mode"); + } + return; + } + DCHECK(!common_gram_pair_keys_); + const uint32_t term_id = find_or_intern_owned_term(term); + accumulate(term_id, docid, pos, retain_positions); +} + +PlainTermId SpimiTermBuffer::intern_plain_term(std::string_view physical_plain_term) { + DCHECK(common_gram_pair_keys_); + DCHECK(vocab_ == &owned_vocab_); + DCHECK(!physical_plain_term.empty()); + DCHECK(!segment_v2::inverted_index::is_internal_term_key(physical_plain_term)); + const size_t term_hash = intern_.hash(physical_plain_term); + uint32_t term_id = find_interned_plain_term(physical_plain_term, term_hash); + if (term_id == CommonGramPlainTermCache::kInvalidTermId) { + term_id = intern_owned_term(std::string(physical_plain_term), term_hash); + remember_plain_term(term_hash, term_id); + } + return PlainTermId {.value = term_id}; +} + +PlainTermId SpimiTermBuffer::intern_plain_term(std::string_view physical_plain_term, + std::string_view logical_plain_term) { + DCHECK(common_gram_pair_keys_); + DCHECK(vocab_ == &owned_vocab_); + DCHECK(!physical_plain_term.empty()); + DCHECK(!logical_plain_term.empty()); + DCHECK(!segment_v2::inverted_index::is_internal_term_key(physical_plain_term)); + + const size_t term_hash = intern_.hash(physical_plain_term); + uint32_t term_id = find_interned_plain_term(physical_plain_term, term_hash); + if (term_id != CommonGramPlainTermCache::kInvalidTermId) { + return PlainTermId {.value = term_id}; + } + +#ifdef BE_TEST + g_common_gram_logical_validations.fetch_add(1, std::memory_order_relaxed); +#endif + auto validation = segment_v2::inverted_index::validate_common_grams_logical_term( + logical_plain_term, "input token"); + if (!validation.ok()) { + throw Exception(validation); + } + if (physical_plain_term.size() > segment_v2::inverted_index::COMMON_GRAM_MAX_ENCODED_BYTES) { + throw Exception(Status::Error( + "CommonGrams escaped plain term would exceed the 16383-byte key limit; " + "set enable_common_grams_index_build=false and retry the import in a new " + "transaction")); + } + term_id = intern_owned_term(std::string(physical_plain_term), term_hash); + remember_plain_term(term_hash, term_id); + return PlainTermId {.value = term_id}; +} + +ClassifiedPlainTerm SpimiTermBuffer::intern_classified_plain_term( + std::string_view physical_plain_term, std::string_view logical_plain_term, + const segment_v2::inverted_index::CommonWordSet& common_words) { + DCHECK(common_gram_pair_keys_); + const PlainTermId id = intern_plain_term(physical_plain_term, logical_plain_term); + DCHECK_EQ(common_word_classification_.size(), owned_vocab_.size()); + DCHECK_LT(id.value, common_word_classification_.size()); + CommonWordClassification& classification = common_word_classification_[id.value]; + if (classification == CommonWordClassification::kUnknown) { + classification = common_words.contains(logical_plain_term) + ? CommonWordClassification::kCommon + : CommonWordClassification::kNotCommon; + } + return ClassifiedPlainTerm { + .id = id, + .is_common = classification == CommonWordClassification::kCommon, + }; +} + +void SpimiTermBuffer::add_plain_token(PlainTermId term_id, uint32_t docid, uint32_t pos) { + DCHECK(common_gram_pair_keys_); + DCHECK_LT(term_id.value, owned_vocab_.size()); + DCHECK(!segment_v2::inverted_index::is_internal_term_key(owned_vocab_[term_id.value])); + accumulate(term_id.value, docid, pos, has_positions_); +} + +void SpimiTermBuffer::add_common_gram_without_spill_gate(PlainTermId left, PlainTermId right, + uint32_t docid, uint32_t pos, + bool retain_positions) { + DCHECK(common_gram_pair_keys_); + DCHECK_LT(left.value, owned_vocab_.size()); + DCHECK_LT(right.value, owned_vocab_.size()); + DCHECK(!segment_v2::inverted_index::is_internal_term_key(owned_vocab_[left.value])); + DCHECK(!segment_v2::inverted_index::is_internal_term_key(owned_vocab_[right.value])); + DCHECK(common_gram_pair_cache_ != nullptr); + const PostingChainShape shape = retain_positions ? PostingChainShape::kTaggedPositioned + : PostingChainShape::kStatlessDocsOnly; + const uint64_t pair = (static_cast(left.value) << 32) | right.value; + CommonGramPairCache::Entry& entry = + common_gram_pair_cache_->entries[CommonGramPairCache::index(pair)]; +#ifdef BE_TEST + g_common_gram_pair_cache_probes.fetch_add(1, std::memory_order_relaxed); +#endif + if (entry.term_id != CommonGramPairCache::kInvalidTermId && entry.pair == pair) { +#ifdef BE_TEST + g_common_gram_pair_cache_pair_hits.fetch_add(1, std::memory_order_relaxed); +#endif + DCHECK_LT(entry.term_id, slot_of_.size()); + if (!retain_positions && entry.last_docid == docid) { +#ifdef BE_TEST + g_common_gram_pair_cache_same_doc_hits.fetch_add(1, std::memory_order_relaxed); +#endif + ++total_tokens_; + return; + } + entry.last_docid = docid; + accumulate_without_spill_gate(entry.term_id, docid, pos, shape); + return; + } + + const uint32_t term_id = find_or_intern_common_gram_pair(left, right, pair); + DCHECK_NE(term_id, CommonGramPairCache::kInvalidTermId); + entry = CommonGramPairCache::Entry {.pair = pair, .term_id = term_id, .last_docid = docid}; + accumulate_without_spill_gate(term_id, docid, pos, shape); +} + +void SpimiTermBuffer::add_common_gram(PlainTermId left, PlainTermId right, uint32_t docid, + uint32_t pos, bool retain_positions) { + add_common_gram_without_spill_gate(left, right, docid, pos, retain_positions); + maybe_spill_after_token(); +} + +void SpimiTermBuffer::add_common_gram_and_plain(PlainTermId left, PlainTermId right, uint32_t docid, + uint32_t gram_pos, uint32_t plain_pos, + bool retain_gram_positions) { + add_common_gram_without_spill_gate(left, right, docid, gram_pos, retain_gram_positions); + DCHECK_LT(right.value, owned_vocab_.size()); + DCHECK(!segment_v2::inverted_index::is_internal_term_key(owned_vocab_[right.value])); + accumulate_without_spill_gate(right.value, docid, plain_pos, + has_positions_ ? PostingChainShape::kTaggedPositioned + : PostingChainShape::kTaggedDocsOnly); + maybe_spill_after_token(); +} + +void SpimiTermBuffer::enable_common_gram_pair_keys() { + DORIS_CHECK(vocab_ == &owned_vocab_); + DORIS_CHECK_EQ(total_tokens_, 0); + DORIS_CHECK(owned_vocab_.empty()); + DORIS_CHECK(!common_gram_pair_keys_); + DORIS_CHECK(common_word_classification_.empty()); + auto pair_cache = std::make_unique(); + auto plain_term_cache = std::make_unique(); + common_gram_pair_keys_ = true; + common_gram_pair_cache_ = std::move(pair_cache); + common_gram_pair_cache_bytes_ = sizeof(CommonGramPairCache); + common_gram_plain_term_cache_ = std::move(plain_term_cache); + common_gram_plain_term_cache_bytes_ = sizeof(CommonGramPlainTermCache); + report_arena_delta(); +} + +uint32_t SpimiTermBuffer::find_or_intern_owned_term(std::string_view term) { + static_assert(std::is_same_v); + DCHECK_LE(term.size(), std::numeric_limits::max()); + const size_t term_hash = intern_.hash(term); + const auto found = intern_.find(term, term_hash); + if (found != intern_.end()) { + const uint32_t term_id = *found; + if (g11_prefetch_enabled()) { + __builtin_prefetch(slot_of_.data() + term_id); + } + return term_id; + } + return intern_owned_term(std::string(term), term_hash); +} + +uint32_t SpimiTermBuffer::find_or_intern_common_gram_pair(PlainTermId left, PlainTermId right, + uint64_t pair) { + DCHECK(common_gram_pair_keys_); + DCHECK_LT(left.value, owned_vocab_.size()); + DCHECK_LT(right.value, owned_vocab_.size()); + DCHECK(!segment_v2::inverted_index::is_internal_term_key(owned_vocab_[left.value])); + DCHECK(!segment_v2::inverted_index::is_internal_term_key(owned_vocab_[right.value])); +#ifdef BE_TEST + g_common_gram_native_pair_probes.fetch_add(1, std::memory_order_relaxed); +#endif + const auto found = common_gram_pair_intern_.find(pair); + if (found != common_gram_pair_intern_.end()) { +#ifdef BE_TEST + g_common_gram_native_pair_hits.fetch_add(1, std::memory_order_relaxed); +#endif + return found->second; + } + + const auto key = EncodeCommonGramPairKey(left, right); + const uint32_t term_id = append_owned_vocab_term(std::string(key.data(), key.size())); + const auto [inserted_it, inserted] = common_gram_pair_intern_.try_emplace(pair, term_id); + DCHECK(inserted); + DCHECK_EQ(inserted_it->second, term_id); +#ifdef BE_TEST + g_common_gram_native_pair_inserts.fetch_add(1, std::memory_order_relaxed); +#endif + return term_id; +} + +uint32_t SpimiTermBuffer::find_interned_plain_term(std::string_view term, size_t term_hash) { + DCHECK(common_gram_pair_keys_); + DCHECK(common_gram_plain_term_cache_ != nullptr); +#ifdef BE_TEST + g_common_gram_plain_cache_probes.fetch_add(1, std::memory_order_relaxed); +#endif + const uint32_t cached_term_id = + common_gram_plain_term_cache_->find(term_hash, term, owned_vocab_); + if (cached_term_id != CommonGramPlainTermCache::kInvalidTermId) { +#ifdef BE_TEST + g_common_gram_plain_cache_hits.fetch_add(1, std::memory_order_relaxed); +#endif + if (g11_prefetch_enabled()) { + __builtin_prefetch(slot_of_.data() + cached_term_id); + } + return cached_term_id; + } + +#ifdef BE_TEST + g_common_gram_plain_intern_table_probes.fetch_add(1, std::memory_order_relaxed); +#endif + const auto found = intern_.find(term, term_hash); + if (found == intern_.end()) { + return CommonGramPlainTermCache::kInvalidTermId; + } + const uint32_t term_id = *found; + remember_plain_term(term_hash, term_id); + if (g11_prefetch_enabled()) { + __builtin_prefetch(slot_of_.data() + term_id); + } + return term_id; +} + +void SpimiTermBuffer::remember_plain_term(size_t term_hash, uint32_t term_id) { + DCHECK(common_gram_pair_keys_); + DCHECK(common_gram_plain_term_cache_ != nullptr); + DCHECK_LT(term_id, owned_vocab_.size()); + common_gram_plain_term_cache_->remember(term_hash, term_id); +} + +bool SpimiTermBuffer::transient_term_less(uint32_t left_id, uint32_t right_id) const { + const std::vector& v = vocab(); + const std::string_view left = v[left_id]; + const std::string_view right = v[right_id]; + if (!common_gram_pair_keys_) { + return left < right; + } + + const bool left_is_pair = is_common_gram_pair_key(left); + const bool right_is_pair = is_common_gram_pair_key(right); + if (left_is_pair != right_is_pair) { + const std::string_view plain = left_is_pair ? right : left; + DCHECK(!plain.empty()); + DCHECK(!segment_v2::inverted_index::is_internal_term_key(plain)); + const bool pair_sorts_first = + static_cast( + segment_v2::inverted_index::INTERNAL_TERM_NAMESPACE_BEGIN.front()) < + static_cast(plain.front()); + return left_is_pair ? pair_sorts_first : !pair_sorts_first; + } + if (!left_is_pair) { + return left < right; + } + + const CommonGramPairIds left_ids = decode_common_gram_pair_key_unchecked(left); + const CommonGramPairIds right_ids = decode_common_gram_pair_key_unchecked(right); + DCHECK_LT(left_ids.left.value, v.size()); + DCHECK_LT(left_ids.right.value, v.size()); + DCHECK_LT(right_ids.left.value, v.size()); + DCHECK_LT(right_ids.right.value, v.size()); + + const LogicalPlainKeyView left_left_key(v[left_ids.left.value]); + const LogicalPlainKeyView left_right_key(v[left_ids.right.value]); + const LogicalPlainKeyView right_left_key(v[right_ids.left.value]); + const LogicalPlainKeyView right_right_key(v[right_ids.right.value]); + if (left_left_key.size() != right_left_key.size()) { + return left_left_key.size() < right_left_key.size(); + } + const int left_component_order = compare_logical_plain_keys(left_left_key, right_left_key); + if (left_component_order != 0) { + return left_component_order < 0; + } + return compare_logical_plain_keys(left_right_key, right_right_key) < 0; +} + +std::string SpimiTermBuffer::materialize_transient_term(std::string_view term) const { + if (!is_common_gram_pair_key(term)) { + return std::string(term); + } + + DCHECK(common_gram_pair_keys_); + const CommonGramPairIds ids = decode_common_gram_pair_key_unchecked(term); + DCHECK_LT(ids.left.value, owned_vocab_.size()); + DCHECK_LT(ids.right.value, owned_vocab_.size()); + DCHECK(!is_common_gram_pair_key(owned_vocab_[ids.left.value])); + DCHECK(!is_common_gram_pair_key(owned_vocab_[ids.right.value])); + + std::string left_scratch; + std::string right_scratch; + const std::string_view left = + decode_logical_plain_term_trusted(owned_vocab_[ids.left.value], &left_scratch); + const std::string_view right = + decode_logical_plain_term_trusted(owned_vocab_[ids.right.value], &right_scratch); + std::string output; + [[maybe_unused]] const bool encoded = + segment_v2::inverted_index::try_encode_common_gram_prevalidated(left, right, output); + DCHECK(encoded); + return output; +} + +// Prepared first-time insertion stores the string before emplace so every +// stored id remains resolvable during later growth rehashes. +uint32_t SpimiTermBuffer::intern_owned_term(std::string&& term_str, size_t term_hash) { + const size_t next_vocab_size = owned_vocab_.size() + 1; + DCHECK_LE(next_vocab_size, std::numeric_limits::max()); + + size_t target_capacity = owned_vocab_.capacity(); + if (target_capacity < next_vocab_size) { + target_capacity = target_capacity <= std::numeric_limits::max() / 2 + ? std::max(next_vocab_size, target_capacity * 2) + : next_vocab_size; + } + + // Prepare append-only vectors geometrically before publishing a vocabulary + // id. A later reserve may throw after an earlier vector already changed + // capacity, so the catch path must settle that resident delta before + // propagating the failure. + try { + owned_vocab_.reserve(target_capacity); +#ifdef BE_TEST + if (g_fail_next_owned_term_reserve.exchange(false, std::memory_order_relaxed)) { + throw std::bad_alloc(); + } +#endif + slot_of_.reserve(target_capacity); + if (common_gram_pair_keys_) { + common_word_classification_.reserve(target_capacity); + } + } catch (...) { + report_arena_delta(); + throw; + } + report_arena_delta(); + + const uint32_t term_id = append_owned_vocab_term(std::move(term_str)); + static_assert(std::is_nothrow_copy_constructible_v); + + const auto rollback_append = [&]() { +#ifdef BE_TEST + g_vocab_materializations.fetch_sub(1, std::memory_order_relaxed); +#endif + owned_vocab_heap_bytes_ -= string_heap_bytes(owned_vocab_.back()); + slot_of_.pop_back(); + if (common_gram_pair_keys_) { + common_word_classification_.pop_back(); + } + owned_vocab_.pop_back(); + report_arena_delta(); + }; + + // phmap allocates a growth table before publishing the prepared slot, and + // constructing this trivial key cannot throw. If allocation fails, the old + // table is intact and only the preceding vocabulary append needs rollback. + const auto [it, inserted] = [&]() { + try { +#ifdef BE_TEST + if (g_fail_next_owned_term_emplace.exchange(false, std::memory_order_relaxed)) { + throw std::bad_alloc(); + } +#endif + return intern_.emplace_with_hash(term_hash, term_id); + } catch (...) { + rollback_append(); + throw; + } + }(); + if (!inserted) { + rollback_append(); + } + DCHECK(inserted); + DCHECK_EQ(*it, term_id); + return term_id; +} + +uint32_t SpimiTermBuffer::append_owned_vocab_term(std::string&& term_str) { + const uint32_t term_id = static_cast(owned_vocab_.size()); + owned_vocab_.emplace_back(std::move(term_str)); + if (common_gram_pair_keys_) { + common_word_classification_.push_back(CommonWordClassification::kUnknown); + DCHECK_EQ(common_word_classification_.size(), owned_vocab_.size()); + } + slot_of_.push_back(0); // vocab grows: new id starts with no live slot + // G08: credit the stored string's heap payload (0 for SSO); the header is + // charged via owned_vocab_.capacity(). + owned_vocab_heap_bytes_ += string_heap_bytes(owned_vocab_[term_id]); +#ifdef BE_TEST + g_vocab_materializations.fetch_add(1, std::memory_order_relaxed); +#endif + return term_id; +} + +namespace { + +// Reorders a term's flat arrays into ascending-docid order, COALESCING any +// same-docid groups so the result has exactly one entry per docid -- matching the +// k-way-merge path's boundary-doc coalescing and the writer's strictly-ascending +// precondition. Only invoked for the rare term that received out-of-order docids +// (the common ascending path leaves t.sorted true and skips it). +// +// A docid may REVISIT (e.g. feed 5,1,5): the chain holds two separate doc-groups +// for doc 5. A STABLE sort keeps equal-docid groups in arrival order, then the +// coalesce pass sums their freqs and concatenates their positions in that same +// (document/arrival) order -- so the merged positions stay consistent with the +// merged freqs, exactly as the run-order merge would have produced. +template +Status reserve_tracked_vector(std::vector* values, size_t target, + MemoryReporter* memory_reporter, + MemoryReporter::Reservation* reservation) { + if (target <= values->capacity()) { + return Status::OK(); + } + if (target > std::numeric_limits::max() / sizeof(T)) { + return Status::Error( + "spimi materialization: vector byte capacity overflow"); + } + if (memory_reporter == nullptr) { + values->reserve(target); + return Status::OK(); + } + MemoryReporter::Reservation replacement; + RETURN_IF_ERROR(reservation->prepare_replacement(static_cast(target) * sizeof(T), + &replacement)); + values->reserve(target); + DCHECK_EQ(values->capacity(), target); + *reservation = std::move(replacement); + return Status::OK(); +} + +Status sort_by_docid(std::vector* docids, std::vector* freqs, + std::vector* positions_flat, bool has_positions, + MemoryReporter* memory_reporter, + MemoryReporter::Reservation* docids_reservation, + MemoryReporter::Reservation* freqs_reservation, + MemoryReporter::Reservation* positions_reservation) { + const size_t n = docids->size(); + MemoryReporter::Reservation order_reservation = memory_reporter == nullptr + ? MemoryReporter::Reservation() + : memory_reporter->make_reservation(); + MemoryReporter::Reservation pos_off_reservation = memory_reporter == nullptr + ? MemoryReporter::Reservation() + : memory_reporter->make_reservation(); + MemoryReporter::Reservation sorted_docids_reservation = + memory_reporter == nullptr ? MemoryReporter::Reservation() + : memory_reporter->make_reservation(); + MemoryReporter::Reservation sorted_freqs_reservation = + memory_reporter == nullptr ? MemoryReporter::Reservation() + : memory_reporter->make_reservation(); + MemoryReporter::Reservation sorted_positions_reservation = + memory_reporter == nullptr ? MemoryReporter::Reservation() + : memory_reporter->make_reservation(); + std::vector order; + RETURN_IF_ERROR(reserve_tracked_vector(&order, n, memory_reporter, &order_reservation)); + order.resize(n); + std::iota(order.begin(), order.end(), 0); + // The original index breaks equal-doc ties, preserving arrival order without + // stable_sort's implementation-owned allocation. + std::ranges::sort(order, [&](size_t a, size_t b) { + if ((*docids)[a] != (*docids)[b]) { + return (*docids)[a] < (*docids)[b]; + } + return a < b; + }); + + std::vector pos_off; + if (has_positions) { + RETURN_IF_ERROR(reserve_tracked_vector(&pos_off, n, memory_reporter, &pos_off_reservation)); + pos_off.resize(n); + uint32_t running = 0; + for (size_t i = 0; i < n; ++i) { + pos_off[i] = running; + running += (*freqs)[i]; + } + } + std::vector nd, nf, np; + RETURN_IF_ERROR(reserve_tracked_vector(&nd, n, memory_reporter, &sorted_docids_reservation)); + RETURN_IF_ERROR(reserve_tracked_vector(&nf, n, memory_reporter, &sorted_freqs_reservation)); + if (has_positions) { + RETURN_IF_ERROR(reserve_tracked_vector(&np, positions_flat->size(), memory_reporter, + &sorted_positions_reservation)); + } + for (size_t k : order) { + // Coalesce a revisited docid into the previous entry (it sorts adjacent now): + // sum freqs and append this group's positions right after the prior group's, + // so flat doc order stays partitioned by the merged freqs. + if (!nd.empty() && nd.back() == (*docids)[k]) { + if (has_positions) { + nf.back() += (*freqs)[k]; + } + } else { + nd.push_back((*docids)[k]); + nf.push_back((*freqs)[k]); + } + if (has_positions) { + np.insert(np.end(), positions_flat->begin() + pos_off[k], + positions_flat->begin() + pos_off[k] + (*freqs)[k]); + } + } + docids->swap(nd); + freqs->swap(nf); + std::swap(*docids_reservation, sorted_docids_reservation); + std::swap(*freqs_reservation, sorted_freqs_reservation); + if (has_positions) { + positions_flat->swap(np); + std::swap(*positions_reservation, sorted_positions_reservation); + } + return Status::OK(); +} + +} // namespace + +namespace { + +// Decodes one varint from a pool chain cursor. The chain was written by +// encode_varint*, so the same LEB128 continuation-bit loop reconstructs it. +uint64_t decode_chain_varint(CompactPostingPool::Cursor* c) { +#ifdef BE_TEST + g_compact_chain_varint_decodes.fetch_add(1, std::memory_order_relaxed); +#endif + return c->read_varint(); +} + +} // namespace + +// Decodes the compact tagged chain directly into caller-owned posting windows. +class SpimiTermBuffer::ArenaTermPostingSource final : public TermPostingSource { +public: + ArenaTermPostingSource(const CompactPostingPool* pool, const Term& term) + : shape_(term.shape), + remaining_docs_(term.ndocs), + remaining_tokens_(term.ntok), + inline_docid_(term.cur_docid) { + if (term.head != kNoChain) { + doc_cursor_.emplace(pool->cursor(term.head, term.w.cur)); + } + } + + Status fill(uint32_t target_docs, TermPostingBuffer* out, bool* exhausted) override { + if (out == nullptr || exhausted == nullptr || target_docs == 0 || !out->empty()) { + return Status::Error( + "spimi arena source: invalid fill arguments"); + } + const uint32_t count = std::min(target_docs, remaining_docs_); + if (count == 0) { + *exhausted = true; + return Status::OK(); + } + + if (shape_ == PostingChainShape::kStatlessDocsOnly) { + MutableTermPostingSpan destination; + RETURN_IF_ERROR(out->grow_uninitialized(count, /*has_freqs=*/false, + /*position_count=*/0, &destination)); + for (uint32_t i = 0; i < count; ++i) { + if (!doc_cursor_) { + DCHECK_EQ(remaining_docs_, 1U); + destination.docids[i] = inline_docid_; + } else { + absolute_docid_ += zigzag_decode(decode_chain_varint(&*doc_cursor_)); + destination.docids[i] = static_cast(absolute_docid_); + } + } + remaining_tokens_ -= count; + } else { + RETURN_IF_ERROR(fill_tagged(count, out)); + } + + remaining_docs_ -= count; + *exhausted = remaining_docs_ == 0; + if (*exhausted) { + DCHECK_EQ(remaining_tokens_, 0U); + DCHECK(!pending_new_doc_); + } + return Status::OK(); + } + + bool exhausted() const { return remaining_docs_ == 0; } + +private: + Status fill_tagged(uint32_t count, TermPostingBuffer* out) { + const bool has_positions = shape_ == PostingChainShape::kTaggedPositioned; + const bool terminal_fill = count == remaining_docs_; + const size_t position_count = has_positions && terminal_fill ? remaining_tokens_ : 0; + MutableTermPostingSpan documents; + RETURN_IF_ERROR( + out->grow_uninitialized(count, /*has_freqs=*/true, position_count, &documents)); + size_t position_index = 0; + for (uint32_t i = 0; i < count; ++i) { + uint64_t tagged = 0; + if (pending_new_doc_) { + tagged = pending_tagged_; + pending_new_doc_ = false; + } else { + DCHECK_GT(remaining_tokens_, 0U); + tagged = decode_chain_varint(&*doc_cursor_); + } + DCHECK_NE(tagged & 1U, 0U); + absolute_docid_ += zigzag_decode(decode_chain_varint(&*doc_cursor_)); + documents.docids[i] = static_cast(absolute_docid_); + uint32_t frequency = 0; + while (true) { + if (has_positions) { + if (terminal_fill) { + documents.positions_flat[position_index++] = + static_cast(tagged >> 1); + } else { + RETURN_IF_ERROR(out->append_position(static_cast(tagged >> 1))); + } + } + ++frequency; + --remaining_tokens_; + if (remaining_tokens_ == 0) { + break; + } + tagged = decode_chain_varint(&*doc_cursor_); + if ((tagged & 1U) != 0) { + pending_tagged_ = tagged; + pending_new_doc_ = true; + break; + } + } + documents.freqs[i] = frequency; + } + DCHECK_EQ(position_index, documents.positions_flat.size()); + return Status::OK(); + } + + PostingChainShape shape_; + std::optional doc_cursor_; + uint32_t remaining_docs_ = 0; + uint32_t remaining_tokens_ = 0; + uint32_t inline_docid_ = 0; + int64_t absolute_docid_ = 0; + uint64_t pending_tagged_ = 0; + bool pending_new_doc_ = false; +}; + +Status SpimiTermBuffer::to_postings(std::string term, Term&& t, + TrackedTermPostings* tracked) const { + DCHECK(tracked != nullptr); + TermPostings& postings = tracked->postings; + DCHECK(postings.docids.empty()); + DCHECK(postings.freqs.empty()); + DCHECK(postings.positions_flat.empty()); + postings.term = std::move(term); + postings.retain_positions = t.shape == PostingChainShape::kTaggedPositioned; + if (t.ntok == 0) { + return Status::OK(); + } + + RETURN_IF_ERROR(reserve_tracked_vector(&postings.docids, t.ndocs, mem_reporter_, + &tracked->docids_reservation)); + if (t.shape != PostingChainShape::kStatlessDocsOnly) { + RETURN_IF_ERROR(reserve_tracked_vector(&postings.freqs, t.ndocs, mem_reporter_, + &tracked->freqs_reservation)); + } + if (t.shape == PostingChainShape::kTaggedPositioned) { + RETURN_IF_ERROR(reserve_tracked_vector(&postings.positions_flat, t.ntok, mem_reporter_, + &tracked->positions_reservation)); + } + + ArenaTermPostingSource source(&pool_, t); + TermPostingBuffer buffer(mem_reporter_); + bool exhausted = false; + while (!exhausted) { + buffer.clear_reuse(); + RETURN_IF_ERROR(source.fill(format::kAdaptiveWindowDocs, &buffer, &exhausted)); + postings.docids.insert(postings.docids.end(), buffer.docids().begin(), + buffer.docids().end()); + postings.freqs.insert(postings.freqs.end(), buffer.freqs().begin(), buffer.freqs().end()); + postings.positions_flat.insert(postings.positions_flat.end(), + buffer.positions_flat().begin(), + buffer.positions_flat().end()); + } + if (!t.sorted && t.shape == PostingChainShape::kStatlessDocsOnly) { + std::ranges::sort(postings.docids); + postings.docids.erase(std::unique(postings.docids.begin(), postings.docids.end()), + postings.docids.end()); + } else if (!t.sorted) { + RETURN_IF_ERROR(sort_by_docid(&postings.docids, &postings.freqs, &postings.positions_flat, + postings.retain_positions, mem_reporter_, + &tracked->docids_reservation, &tracked->freqs_reservation, + &tracked->positions_reservation)); + } + return Status::OK(); +} + +void SpimiTermBuffer::ensure_string_rank() const { + const std::vector& v = vocab(); + if (string_rank_.size() == v.size()) { + return; // already built for the current append-only vocabulary + } + // Build the complete rank required by the first spill and by k-way merge + // paths. Ordinary spills with a stale rank deliberately do not call here. + if (!common_gram_pair_keys_) { + std::vector order(v.size()); + std::iota(order.begin(), order.end(), 0U); + std::ranges::sort(order, [&](uint32_t a, uint32_t b) { return transient_term_less(a, b); }); + string_rank_.assign(v.size(), 0U); + for (uint32_t rank = 0; rank < order.size(); ++rank) { + string_rank_[order[rank]] = rank; + } + } else { + size_t pair_count = 0; + for (const std::string& term : v) { + pair_count += is_common_gram_pair_key(term); + } + + std::vector plain_order; + std::vector pair_order; + plain_order.reserve(v.size() - pair_count); + pair_order.reserve(pair_count); + for (uint32_t term_id = 0; term_id < v.size(); ++term_id) { + if (is_common_gram_pair_key(v[term_id])) { + pair_order.push_back(term_id); + } else { + plain_order.push_back(term_id); + } + } + + // EscapedV1 preserves logical byte order: 0x1e maps to 0x1eE and 0x1f + // maps to 0x1eG. One physical sort therefore supplies both the final plain + // order and the logical component rank used by a gram's right term. + std::ranges::sort(plain_order, + [&](uint32_t left, uint32_t right) { return v[left] < v[right]; }); + string_rank_.assign(v.size(), 0U); + for (uint32_t rank = 0; rank < plain_order.size(); ++rank) { + string_rank_[plain_order[rank]] = rank; + } + + // Decode each transient pair exactly once. The low word remains its term id; + // the high word temporarily carries the left plain id, while the pair's + // unused rank slot carries its right component's logical rank. + for (uint64_t& decorated_pair : pair_order) { + const uint32_t pair_term_id = static_cast(decorated_pair); + const CommonGramPairIds ids = decode_common_gram_pair_key_unchecked(v[pair_term_id]); + DCHECK_LT(ids.left.value, v.size()); + DCHECK_LT(ids.right.value, v.size()); + DCHECK(!is_common_gram_pair_key(v[ids.left.value])); + DCHECK(!is_common_gram_pair_key(v[ids.right.value])); + string_rank_[pair_term_id] = string_rank_[ids.right.value]; + decorated_pair = (static_cast(ids.left.value) << 32) | pair_term_id; + } + + // The physical gram key orders its left component by fixed-width encoded + // length, then by logical bytes. Stable per-length offsets convert the + // already-logically-sorted plain ids into that compound dense rank without + // another comparison sort. + std::vector next_length_rank( + segment_v2::inverted_index::COMMON_GRAM_MAX_ENCODED_BYTES + 1, 0U); + for (uint32_t plain_id : plain_order) { + const size_t logical_size = LogicalPlainKeyView(v[plain_id]).size(); + DCHECK_LT(logical_size, next_length_rank.size()); + ++next_length_rank[logical_size]; + } + uint32_t next_rank = 0; + for (uint32_t& length_count : next_length_rank) { + const uint32_t count = length_count; + length_count = next_rank; + next_rank += count; + } + DCHECK_EQ(next_rank, plain_order.size()); + for (uint32_t plain_id : plain_order) { + const size_t logical_size = LogicalPlainKeyView(v[plain_id]).size(); + string_rank_[plain_id] = next_length_rank[logical_size]++; + } + for (uint64_t& decorated_pair : pair_order) { + const uint32_t left_plain_id = static_cast(decorated_pair >> 32); + const uint32_t pair_term_id = static_cast(decorated_pair); + decorated_pair = + (static_cast(string_rank_[left_plain_id]) << 32) | pair_term_id; + } + + std::ranges::sort(pair_order, [&](uint64_t left, uint64_t right) { + const uint32_t left_component_rank = static_cast(left >> 32); + const uint32_t right_component_rank = static_cast(right >> 32); + if (left_component_rank != right_component_rank) { + return left_component_rank < right_component_rank; + } + return string_rank_[static_cast(left)] < + string_rank_[static_cast(right)]; + }); + + // No EscapedV1 plain key enters 0x1f, so every materialized gram forms one + // contiguous namespace group between the two physical-plain ranges. + const auto pair_position = std::lower_bound( + plain_order.begin(), plain_order.end(), segment_v2::inverted_index::CG_V1_MARKER, + [&](uint32_t plain_id, std::string_view marker) { return v[plain_id] < marker; }); + uint32_t final_rank = 0; + for (auto it = plain_order.begin(); it != pair_position; ++it) { + string_rank_[*it] = final_rank++; + } + for (uint64_t decorated_pair : pair_order) { + string_rank_[static_cast(decorated_pair)] = final_rank++; + } + for (auto it = pair_position; it != plain_order.end(); ++it) { + string_rank_[*it] = final_rank++; + } + DCHECK_EQ(final_rank, v.size()); + } +#ifdef BE_TEST + g_string_rank_rebuilds.fetch_add(1, std::memory_order_relaxed); +#endif +} + +std::vector SpimiTermBuffer::sorted_ids() const { + std::vector ids = touched_ids_; + const std::vector& v = vocab(); + if (string_rank_.empty()) { + // Preserve the fixed-vocabulary fast path: the first spill pays once for + // a complete rank, then every later spill is integer-only until vocab grows. + ensure_string_rank(); + } + if (string_rank_.size() == v.size()) { + order_ids_by_dense_rank(&ids, string_rank_); + } else { + // Vocabulary grew after the last complete rank. A run needs only its touched + // terms in lexical order; defer the O(vocab log vocab) rebuild until a k-way + // merge needs rank lookups for arbitrary ids. Reserve the same persistent + // rank capacity the old rebuild allocated so resident accounting and later + // spill-trigger timing remain unchanged. + string_rank_.reserve(v.size()); + std::ranges::sort(ids, [&](uint32_t a, uint32_t b) { return transient_term_less(a, b); }); + } + return ids; +} + +void SpimiTermBuffer::release_term(uint32_t term_id) { + const uint32_t enc = slot_of_[term_id]; + DCHECK_NE(enc, 0U); + const uint32_t slot = enc - 1; + slots_[slot] = Term(); // free this term's arrays; the empty Term slot is reusable + free_slots_.push_back(slot); + slot_of_[term_id] = 0; + --live_term_count_; +} + +Status SpimiTermBuffer::drain_sorted_streamed(const StreamedTermConsumer& fn) { + const std::vector& v = vocab(); + ensure_string_rank(); + report_arena_delta(); + order_ids_by_dense_rank(&touched_ids_, string_rank_); + intern_ = decltype(intern_)(0, OwnedVocabHash {.vocab = &owned_vocab_}, + OwnedVocabEq {&owned_vocab_}); + common_gram_pair_intern_ = decltype(common_gram_pair_intern_)(); + std::vector().swap(common_word_classification_); + std::vector().swap(string_rank_); + report_arena_delta(); + + constexpr size_t kSlotIndexPrefetchDistance = 32; + constexpr size_t kTermPrefetchDistance = 16; + Status callback_status = Status::OK(); + for (size_t ordinal = 0; ordinal < touched_ids_.size(); ++ordinal) { + if (ordinal + kSlotIndexPrefetchDistance < touched_ids_.size()) { + const uint32_t future_id = touched_ids_[ordinal + kSlotIndexPrefetchDistance]; + __builtin_prefetch(slot_of_.data() + future_id); + } + if (ordinal + kTermPrefetchDistance < touched_ids_.size()) { + const uint32_t future_id = touched_ids_[ordinal + kTermPrefetchDistance]; + const uint32_t future_enc = slot_of_[future_id]; + DCHECK_NE(future_enc, 0U); + __builtin_prefetch(slots_.data() + future_enc - 1); + __builtin_prefetch(v.data() + future_id); + } + const uint32_t id = touched_ids_[ordinal]; + const uint32_t enc = slot_of_[id]; + DCHECK_NE(enc, 0U); + Term term = slots_[enc - 1]; + slots_[enc - 1] = Term(); + slot_of_[id] = 0; + --live_term_count_; + + std::string output_term = materialize_transient_term(v[id]); + if (term.sorted) { + ArenaTermPostingSource source(&pool_, term); + StreamedTermPostings postings { + .term = std::move(output_term), + .retain_positions = term.shape == PostingChainShape::kTaggedPositioned, + .source = &source}; + callback_status = fn(std::move(postings)); + if (callback_status.ok() && !source.exhausted()) { + callback_status = Status::Error( + "spimi arena source: consumer returned before term exhaustion"); + } + } else { + TrackedTermPostings materialized(mem_reporter_); + callback_status = to_postings(std::move(output_term), std::move(term), &materialized); + if (callback_status.ok()) { + SpanTermPostingSource source(materialized.postings.docids, + materialized.postings.freqs, + materialized.postings.positions_flat); + StreamedTermPostings postings { + .term = std::move(materialized.postings.term), + .retain_positions = materialized.postings.retain_positions, + .source = &source}; + callback_status = fn(std::move(postings)); + if (callback_status.ok() && !source.exhausted()) { + callback_status = Status::Error( + "spimi span source: consumer returned before term exhaustion"); + } + } + } + if (!callback_status.ok()) { + break; + } + } + + pool_.reset(); + std::vector().swap(slots_); + std::vector().swap(free_slots_); + std::vector().swap(slot_of_); + std::vector().swap(touched_ids_); + live_term_count_ = 0; + std::vector().swap(owned_vocab_); + owned_vocab_heap_bytes_ = 0; + common_gram_pair_cache_.reset(); + common_gram_pair_cache_bytes_ = 0; + common_gram_plain_term_cache_.reset(); + common_gram_plain_term_cache_bytes_ = 0; + report_arena_delta(); + return callback_status; +} + +Status SpimiTermBuffer::drain_to_writer(RunWriter* w) { + Status st = Status::OK(); + const std::vector& v = vocab(); + // Spill writes by term-id (no string IO). Iterate touched ids in vocab-string + // order so each run is sorted; the k-way merge re-orders runs by the same key. + for (uint32_t id : sorted_ids()) { + const uint32_t enc = slot_of_[id]; + DCHECK_NE(enc, 0U); + Term term = slots_[enc - 1]; + release_term(id); + if (st.ok()) { + TrackedTermPostings materialized(mem_reporter_); + st = to_postings(v[id], std::move(term), &materialized); + if (st.ok()) { + st = w->write_term(id, materialized.postings); + } + } + } + touched_ids_.clear(); + pool_.reset(); // all chains decoded into the run; free the arena for the refill + // The spill returns the arena to 0; slot_of_ keeps its capacity (survives + // the spill). Report the arena-drop negative now so the gate-2 spill is balanced + // immediately, not deferred to the next token. + report_arena_delta(); + return st; +} + +Status SpimiTermBuffer::compact_runs() { + if (run_paths_.size() < 2) { + return Status::OK(); + } + // The compaction heap can encounter any id held by an earlier run, so it + // requires a complete rank for the current vocabulary. New append-only ids + // can shift existing lexicographic ranks, hence the explicit refresh here. + ensure_string_rank(); + const std::string out_path = make_run_path(resolve_temp_dir()); + Status s = + writer::compact_runs(run_paths_, string_rank_, has_positions_, out_path, mem_reporter_); + if (!s.ok()) { + std::remove(out_path.c_str()); // drop the partial output; inputs intact + return s; + } + // The compacted run REPLACES its inputs at the FRONT of the run order: + // it holds exactly runs [0..n) merged in run order, and any later run only + // covers strictly-later docids, so per-term run-order concatenation (the + // k-way merge invariant) is preserved. + for (const std::string& p : run_paths_) { + std::remove(p.c_str()); + } + run_paths_.clear(); + run_paths_.push_back(out_path); + g_run_compactions.fetch_add(1, std::memory_order_relaxed); + return Status::OK(); +} + +Status SpimiTermBuffer::spill_to_run() { + // G09 run-file cap: a buffer must never accumulate unbounded run files -- + // the final k-way merge (re)opens ALL of them simultaneously and holds + // the fds for its whole duration, so unbounded runs across ~100 + // concurrent writers exhausted the BE nofile rlimit ('Too many open + // files' at run reopen). At the cap, merge-compact the existing runs into + // one before cutting the new run: the merge fan-in (and its fd count) is + // bounded by cap + 1 per buffer. + if (max_run_files_ != 0 && run_paths_.size() >= max_run_files_) { + RETURN_IF_ERROR(compact_runs()); + } + const std::string dir = resolve_temp_dir(); + // Best-effort space pre-check: fail with a clear, early error rather than a + // mid-write IoError that leaves a half-written run. Best-effort only (TOCTOU; on + // tmpfs this reports RAM). The ARENA -- not full resident_bytes(), which since + // G08 also charges vocabulary structures a run never contains -- is what the + // run re-encodes, and its block slack makes it a conservative over-estimate of + // the run's on-disk size. + const uint64_t arena = pool_.arena_bytes(); + const uint64_t avail = temp_dir_available_bytes(dir); + if (avail < arena) { + return Status::Error( + "spimi: insufficient temp space in '" + dir + "' to spill ~" + + std::to_string(arena) + " B (~" + std::to_string(avail) + + " B free); set SNII_TEMP_DIR/TMPDIR to a larger disk"); + } + const std::string path = make_run_path(dir); + RunWriter w(mem_reporter_); + RETURN_IF_ERROR(w.open(path)); + run_paths_.push_back(path); // tracked for cleanup even if a later step fails + RETURN_IF_ERROR(drain_to_writer(&w)); + // The drain emptied touched_ids_ and released every live slot while retaining + // capacity for the next fill. + return w.close(); +} + +Status SpimiTermBuffer::prepare_run_merge(TermKeyMaterializer* materializer) { + if (!touched_ids_.empty()) { + Status status = spill_to_run(); + if (!status.ok() && spill_status_.ok()) { + spill_status_ = status; + } + } + if (!spill_status_.ok()) { + return spill_status_; + } + + std::vector().swap(slots_); + std::vector().swap(free_slots_); + std::vector().swap(slot_of_); + std::vector().swap(touched_ids_); + common_gram_pair_cache_.reset(); + common_gram_pair_cache_bytes_ = 0; + common_gram_plain_term_cache_.reset(); + common_gram_plain_term_cache_bytes_ = 0; + common_gram_pair_intern_ = decltype(common_gram_pair_intern_)(); + std::vector().swap(common_word_classification_); + report_arena_delta(); + + ensure_string_rank(); + report_arena_delta(); + intern_ = decltype(intern_)(0, OwnedVocabHash {.vocab = &owned_vocab_}, + OwnedVocabEq {&owned_vocab_}); + report_arena_delta(); + if (common_gram_pair_keys_) { + *materializer = [this](std::string_view term) { return materialize_transient_term(term); }; + } + return Status::OK(); +} + +void SpimiTermBuffer::finish_run_merge() { + std::vector().swap(owned_vocab_); + owned_vocab_heap_bytes_ = 0; + std::vector().swap(string_rank_); + report_arena_delta(); +} + +Status SpimiTermBuffer::merge_runs_streamed(const StreamedTermConsumer& fn) { + TermKeyMaterializer materializer; + RETURN_IF_ERROR(prepare_run_merge(&materializer)); + Status status = merge_run_sources(run_paths_, vocab(), string_rank_, has_positions_, fn, + std::move(materializer), mem_reporter_); + finish_run_merge(); + return status; +} + +Status SpimiTermBuffer::for_each_term_sorted(const StreamedTermConsumer& fn) { + // Single-drain contract: a second call would re-merge the (still-present) run + // files and re-emit every term, or emit nothing in the in-memory path. Return + // an error and emit NOTHING rather than produce a wrong second stream. + if (drained_) { + return Status::Error( + "spimi: already drained (single-drain contract)"); + } + drained_ = true; + if (run_paths_.empty() && spill_status_.ok()) { + return drain_sorted_streamed(fn); + } + return merge_runs_streamed(fn); +} + +std::vector SpimiTermBuffer::finalize_sorted() { + std::vector out; + out.reserve(touched_ids_.size()); + Status status = for_each_term_sorted([&out](StreamedTermPostings&& streamed) { + TermPostings materialized; + materialized.term = std::move(streamed.term); + materialized.retain_positions = streamed.retain_positions; + TermPostingBuffer buffer(nullptr); + bool exhausted = false; + while (!exhausted) { + buffer.clear_reuse(); + RETURN_IF_ERROR( + streamed.source->fill(format::kAdaptiveWindowDocs, &buffer, &exhausted)); + materialized.docids.insert(materialized.docids.end(), buffer.docids().begin(), + buffer.docids().end()); + materialized.freqs.insert(materialized.freqs.end(), buffer.freqs().begin(), + buffer.freqs().end()); + materialized.positions_flat.insert(materialized.positions_flat.end(), + buffer.positions_flat().begin(), + buffer.positions_flat().end()); + } + out.push_back(std::move(materialized)); + return Status::OK(); + }); + if (!status.ok() && spill_status_.ok()) { + spill_status_ = status; + std::vector().swap(out); + } + return out; +} + +void SpimiTermBuffer::cleanup_runs() { + for (const std::string& p : run_paths_) { + std::remove(p.c_str()); + } + run_paths_.clear(); +} + +} // namespace doris::snii::writer diff --git a/be/src/storage/index/snii/writer/spimi_term_buffer.h b/be/src/storage/index/snii/writer/spimi_term_buffer.h new file mode 100644 index 00000000000000..40286cb6d64e29 --- /dev/null +++ b/be/src/storage/index/snii/writer/spimi_term_buffer.h @@ -0,0 +1,738 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/writer/compact_posting_pool.h" +#include "storage/index/snii/writer/memory_reporter.h" +#include "storage/index/snii/writer/term_posting_source.h" + +namespace doris::segment_v2::inverted_index { +class CommonWordSet; +} + +namespace doris::snii::writer { + +using StreamedTermConsumer = std::function; + +// G11: compiled-in marker for the per-token prefetch candidate (the locality +// bench keys its in-process A/B test off this). +#define SNII_G11_PREFETCH 1 + +class GlobalMemoryLimiter; // G09 process-wide build-RAM registry (see below) + +struct PlainTermId { + uint32_t value = 0; +}; + +struct ClassifiedPlainTerm { + PlainTermId id; + bool is_common = false; +}; + +// One term's posting list: docids ascending, with parallel freqs and (when +// positions are enabled) a single FLAT positions buffer. +// +// positions_flat holds every position for the term in document order, partitioned +// by freqs: doc i owns the next freqs[i] entries. This is the SAME layout the +// accumulator stores natively, so no per-doc vector-of-vectors is ever built on +// the build/merge hot path (that vector-of-vectors was the dominant peak-RSS +// driver for high-df terms). doc_positions(i) returns a non-owning span view of +// doc i's positions for consumers that want per-doc access (e.g. the prx window +// builder, tests). positions_flat is empty when positions are disabled. +struct TermPostings { + std::string term; + std::vector docids; // absolute docids + std::vector freqs; + std::vector positions_flat; // empty when positions disabled + // Per-term posting shape. A positioned logical index may mix ordinary terms + // with docs-only accelerator terms; the latter carry one posting per doc and + // deliberately omit frequencies/positions from the final index. + bool retain_positions = true; + + size_t document_count() const { return docids.size(); } + + // Byte offset of doc i's first position within positions_flat (prefix sum of + // freqs). O(i) -- callers iterating all docs should track a running offset. + size_t pos_offset(size_t doc_index) const { + size_t off = 0; + for (size_t i = 0; i < doc_index; ++i) { + off += freqs[i]; + } + return off; + } + // Non-owning view of doc i's positions (length freqs[i]) into positions_flat. + std::span doc_positions(size_t doc_index) const { + const size_t off = pos_offset(doc_index); + return {positions_flat.data() + off, freqs[doc_index]}; + } + + // Rebuilds the per-doc position lists (for callers/tests wanting per-doc access) + // from positions_flat partitioned by freqs. O(total positions); allocates. + std::vector> positions_per_doc() const { + std::vector> out(freqs.size()); + size_t off = 0; + for (size_t i = 0; i < freqs.size(); ++i) { + out[i].assign(positions_flat.begin() + off, positions_flat.begin() + off + freqs[i]); + off += freqs[i]; + } + return out; + } + + // Sets the flat positions from per-doc lists (convenience for tests / callers + // that produce per-doc positions). Does NOT touch freqs; the caller is expected + // to keep freqs[i] == per_doc[i].size() consistent (the writer validates this). + void set_positions_per_doc(const std::vector>& per_doc) { + positions_flat.clear(); + for (const auto& d : per_doc) { + positions_flat.insert(positions_flat.end(), d.begin(), d.end()); + } + } +}; + +// In-memory SPIMI (Single-Pass In-Memory Indexing) accumulator for one logical +// index. Records term occurrences and produces lexicographically sorted terms +// with ascending-docid posting lists. +// +// TERM-ID ACCUMULATION (no per-token string work): tokens are accumulated by an +// INTEGER term-id, not by hashing/constructing a std::string per token. The +// caller supplies a VOCABULARY mapping term-id -> term string; the buffer keeps +// a DENSE std::vector indexed by term-id, so the hot add_token path is a +// vector index + a couple of pushes -- no hashing, no allocation per token. The +// vocabulary is resolved to strings only once per distinct term at finalize. +// +// Two construction modes: +// * BORROWED vocab (the fast path): pass a non-null `vocab` that the caller +// owns and keeps alive; add_token(term_id, ...) indexes straight into it. +// * OWNED vocab (compatibility): pass a null `vocab`; the string-keyed +// add_token(string_view, ...) interns each new term into an internal owned +// vocabulary (assigning ids in first-seen order) and forwards to the id +// path. Existing callers that feed strings keep working unchanged. +// +// SPILL / K-WAY MERGE (out-of-core, bounds input RAM): when a non-zero +// spill_threshold_bytes is set, the REAL resident accumulator size (see +// resident_bytes(): the posting arena PLUS every live vocab / slot / rank +// structure, G08) is compared against the threshold as tokens arrive. Once it +// crosses the threshold and enough reclaimable posting arena has accumulated, +// the buffer SORTS its current terms, +// writes a self-describing sorted RUN to a temp file, and CLEARS memory. Each +// run record is keyed by the TERM-ID (varint); the k-way merge orders runs by +// the id's VOCAB STRING so the merged stream stays lexicographic. Because +// tokens arrive in globally ascending docid order, a term that reappears in a +// later run only covers strictly-later docids, so concatenating its postings in +// run order during the final merge keeps docids ascending. for_each_term_sorted +// flushes the residual buffer as a final run, then k-way merges all runs +// materializing only ONE merged term at a time -> peak memory stays bounded by +// the threshold (plus the widest single term), NOT by total postings. With the +// default threshold 0 (unlimited) the path is exactly the in-memory behavior. +// +// Internal representation is a COMPACT TAGGED VARINT byte stream per term, held in +// a shared SEGMENTED ARENA (CompactPostingPool), NOT per-term uint32 vectors. Each +// term owns ONE arena chain holding a stream of per-TOKEN entries in arrival +// order: positioned and ordinary docs-only tokens contribute +// varint((pos << 1) | new_doc_bit); when new_doc_bit is set, a +// zigzag-varint(docid - prev_docid) immediately follows. Statless CommonGrams are +// deduplicated per document and store only that document delta, omitting the +// constant new_doc tag. Frequencies are otherwise recovered as the count of +// consecutive same-doc tokens. This drops +// the entire freq stream and the second (positions) chain versus a freq/prox split, +// so the payload is ~3.4x smaller than raw uint32 docids/freqs/positions, and the +// shared arena removes per-vector doubling slack and per-term vector headers. Each +// positioned and ordinary docs-only tokens append straight into the chain. +// Stateless CommonGrams keep a singleton doc id inline and backfill it only when a +// second document arrives. The other live per-term state is the current doc id (to +// detect a doc change) and the delta base. +// The production writer drains each chain through a bounded TermPostingSource. +// to_postings() remains only for explicit materialization in run maintenance and +// test/finalize helpers. positions_flat stays empty (and pos is tagged as 0) when +// positions are disabled; freq still counts. +// +// Duplicate vocab strings: the vocab is assumed to map each id to a DISTINCT +// string (a dense vocabulary). If two ids share a string they sort adjacently +// but are emitted as two separate terms; callers must not rely on coalescing. +class SpimiTermBuffer { +public: + // BORROWED-vocab constructor: `vocab` maps term-id -> term string and is + // borrowed (NOT owned) -- the caller must keep it alive for the buffer's + // lifetime. add_token(term_id, ...) accumulates by id with no string work. + // spill_threshold_bytes is the gate-2 internal buffer cap (e.g. 512 MiB), + // sourced from config; == 0 means unlimited (pure in-memory, default). A + // positive value is a soft spill threshold for the REAL resident accumulator + // size (resident_bytes(): arena + every live vocab/slot/rank structure, G08), + // triggering a spill once enough reclaimable arena has accumulated -- NOT a + // hard cap on persistent vocabulary memory or the old per-token estimate. + // `reporter` is the OPTIONAL writer-level build-RAM reporter (null off-Doris / + // unit tests). When non-null, the accumulator reports its REAL resident-byte + // deltas -- resident_bytes() diffs -- positive on grow, negative on every + // reset/free, exactly once. NEVER reports live_bytes_ (a gated estimate that + // feeds only the spill threshold). + explicit SpimiTermBuffer(const std::vector* vocab, bool has_positions, + size_t spill_threshold_bytes = 0, MemoryReporter* reporter = nullptr); + + // OWNED-vocab (compatibility) constructor: no external vocab. The string-keyed + // add_token interns terms into an internal vocabulary on first occurrence. + explicit SpimiTermBuffer(bool has_positions, size_t spill_threshold_bytes = 0, + MemoryReporter* reporter = nullptr); + + ~SpimiTermBuffer(); + + SpimiTermBuffer(const SpimiTermBuffer&) = delete; + SpimiTermBuffer& operator=(const SpimiTermBuffer&) = delete; + + // Records one token by TERM-ID: term `term_id` occurs in `docid` at `pos`. + // `term_id` must be in [0, vocab_size). An out-of-range id latches an + // InvalidArgument into status() and is ignored. For a given term, docids are + // expected to arrive in non-decreasing order, and positions within a docid in + // ascending order; out-of-order docids (INCLUDING a REVISITED docid -- the same + // docid appearing again after a different one) are tolerated and reordered at + // finalize: sort_by_docid stably sorts by docid and COALESCES same-docid groups + // (summing freqs, concatenating positions in document order), so the emitted + // postings have exactly ONE strictly-ascending entry per docid -- matching the + // k-way merge path and the writer's strictly-ascending precondition. + void add_token(uint32_t term_id, uint32_t docid, uint32_t pos); + void add_token(uint32_t term_id, uint32_t docid, uint32_t pos, bool retain_positions); + + // Compatibility overload: records one token by TERM STRING. Valid ONLY on an + // OWNED-vocab buffer before enable_common_gram_pair_keys(); interns `term` into + // the internal vocabulary on first occurrence, then forwards by id. Pair-key + // mode must use the typed plain/gram APIs below so a physical gram and its + // transient pair key cannot become two ids for the same logical term. Called on + // a BORROWED-vocab buffer it is REJECTED (latches InvalidArgument, token ignored) + // -- interning would grow the owned vocab out of step with the borrowed one and + // corrupt the build. Interning probes a heterogeneous (string_view-keyed) set, + // so a repeat token for an already-seen term allocates NOTHING; a std::string is + // materialized only on a term's FIRST occurrence (stored once in owned_vocab_). + // The id overload remains the hot path (no hashing at all); prefer that and + // reserve this for tests / legacy string-fed callers. + void add_token(std::string_view term, uint32_t docid, uint32_t pos); + void add_token(std::string_view term, uint32_t docid, uint32_t pos, bool retain_positions); + + // SNII CommonGrams fast path. Plain terms are interned once and returned as + // stable ids; each gram occurrence hashes a fixed 10-byte pair of those ids + // instead of constructing and hashing the variable-length physical gram key. + PlainTermId intern_plain_term(std::string_view physical_plain_term); + // Production CommonGrams path. A physical-key hit proves the injectively mapped + // logical term was validated previously; a miss validates exactly once before + // materializing the vocabulary entry. + PlainTermId intern_plain_term(std::string_view physical_plain_term, + std::string_view logical_plain_term); + ClassifiedPlainTerm intern_classified_plain_term( + std::string_view physical_plain_term, std::string_view logical_plain_term, + const segment_v2::inverted_index::CommonWordSet& common_words); + void add_plain_token(PlainTermId term_id, uint32_t docid, uint32_t pos); + void add_common_gram(PlainTermId left, PlainTermId right, uint32_t docid, uint32_t pos, + bool retain_positions); + void add_common_gram_and_plain(PlainTermId left, PlainTermId right, uint32_t docid, + uint32_t gram_pos, uint32_t plain_pos, + bool retain_gram_positions); + void enable_common_gram_pair_keys(); + + // G09: joins the PROCESS-WIDE build-RAM registry. Registers this buffer's + // current SPILLABLE arena bytes with `limiter` and forwards every + // subsequent (debounced, see report_arena_delta) arena total to it; the + // destructor un-registers. The buffer's total memory reaches the limiter by + // another route -- the observation tracker its MemoryReporter feeds -- so + // the registry carries only what a forced spill could reclaim. When SNII's + // index-build memory crosses its share of the process memory limit (or the + // process itself comes under pressure), the limiter may set this buffer's + // ADVISORY spill-request flag from ANOTHER thread; the flag is observed -- + // and the forced spill run ON THIS BUFFER'S OWN THREAD -- by the next + // add_token's maybe_spill_after_token (see there for the honor rule). + // Call at most once, right after construction (extra calls are ignored); + // `limiter` must outlive this buffer. Null / never attached = the G08 + // per-writer behavior, byte-identical. + void attach_global_limiter(GlobalMemoryLimiter* limiter); + + // TEST-ONLY: G09 advisory-flag observability -- read the pending flag, and + // plant a request directly (what the limiter does cross-thread) so the + // owner-honors-at-next-token contract is testable without a registry. + bool global_spill_requested_for_test() const { + return global_spill_requested_.load(std::memory_order_relaxed); + } + void request_global_spill_for_test() { + global_spill_requested_.store(true, std::memory_order_relaxed); + } + + // G09 forced-spill floor (config snii_forced_spill_min_arena_bytes): a + // pending process-wide forced-spill request is honored only once the + // reclaimable posting arena holds at least this much (never below one + // arena block, so a run is always writable). A request planted while the + // arena is below the floor is a NO-OP that stays PENDING -- it is NOT + // retried as a spill every token -- and is honored when the arena regrows + // past the floor. THE FLOOR IS THE ANTI-STORM DEFENSE: without it, a + // process-wide target the persistent vocabulary/slot structures alone + // exceed re-flagged every buffer on every report and each honored with a + // single 32 KiB arena block -- thousands of tiny runs per buffer, EMFILE at + // the k-way merge reopen, failed loads (the conc=16 wikipedia field storm). + // With it, flagging costs at most one >= floor-sized run per floor of arena + // growth per buffer, which is the intended back-pressure. + static constexpr uint64_t kDefaultForcedSpillMinArenaBytes = 64ULL << 20; // 64 MiB + void set_forced_spill_min_arena_bytes(uint64_t bytes) { forced_spill_min_arena_bytes_ = bytes; } + uint64_t forced_spill_min_arena_bytes() const { return forced_spill_min_arena_bytes_; } + + // G09 run-file cap (config snii_spill_max_run_files_per_buffer): when a + // new spill would grow the accumulated run-file count past this cap, the + // existing runs are first MERGE-COMPACTED into one (a k-way merge of the + // run files back into a single fresh run; term stream byte-identical, the + // old files deleted) so the buffer never holds more than the cap + 1 run + // files. Bounds both the final k-way merge's fan-in and -- decisively -- + // its OPEN FILE DESCRIPTORS: every run of a buffer is (re)opened + // simultaneously and held open for the whole merge, so unbounded run + // counts across ~100 concurrent writers exhausted the BE nofile rlimit + // ('Too many open files' at run reopen). 0 disables the cap. + static constexpr size_t kDefaultMaxRunFilesPerBuffer = 64; + void set_max_run_files(size_t cap) { max_run_files_ = cap; } + size_t max_run_files() const { return max_run_files_; } + + // Number of DISTINCT terms accumulated so far (touched ids still resident). + size_t unique_terms() const; + uint64_t total_tokens() const { return total_tokens_; } + bool has_positions() const { return has_positions_; } + + // OK unless an add_token validation error (out-of-range term-id, wrong vocab + // mode) was latched. for_each_term_sorted now returns its own I/O Status + // directly; callers that use add_token's latch-and-report pattern MUST check + // this after draining to surface input-side validation errors. + [[nodiscard]] Status status() const { return spill_status_; } + + // TEST-ONLY: number of spill run files currently HELD (== 0 in pure + // in-memory mode). Lets tests assert that a gate-2 spill actually fired + // once the REAL resident size crossed the configured cap. NOTE: a G09 + // run-cap merge-compaction (see set_max_run_files) collapses the list to + // ONE file, so the count is not monotonic. Not part of the production API. + size_t run_count_for_test() const { return run_paths_.size(); } + + // TEST-ONLY: the REAL resident accumulator bytes the gate-2 trigger and the + // MemoryReporter see (resident_bytes()). Lets the G08 accounting tests assert + // coverage and monotonicity without widening access to the private + // accounting. Not part of + // the production API. + uint64_t resident_bytes_for_test() const { return resident_bytes(); } + // TEST-ONLY: the SPILLABLE posting-arena bytes forwarded to the G09 registry + // as this buffer's victim-selection key. Not part of the production API. + uint64_t arena_bytes_for_test() const { return pool_.arena_bytes(); } + size_t string_rank_capacity_for_test() const { return string_rank_.capacity(); } +#ifdef BE_TEST + static size_t hash_term_bytes_for_test(std::string_view term) { return hash_term_bytes(term); } + static size_t owned_term_key_size_for_test(); + void set_owned_term_hash_mask_for_test(size_t mask); +#endif + + // Materializes all terms sorted lexicographically; each term's docids are + // ascending. Convenience wrapper around for_each_term_sorted that keeps the + // whole result alive at once. Prefer for_each_term_sorted for low peak memory. + // The returned vectors are caller-owned compatibility output and are not + // charged to this buffer's internal MemoryReporter after the callback returns. + // MUST be called at most once: it drains internal state. A SECOND drain (a + // repeat call, or a finalize_sorted after a for_each_term_sorted, or vice versa) + // returns EMPTY and latches an error into status() rather than re-emitting. + std::vector finalize_sorted(); + + // Streams terms to `fn` in lexicographic order. Each source is borrowed for + // the synchronous callback and fills the writer-owned transfer buffer. The + // callback must exhaust the source before returning success. + // MUST be called at most once: it drains internal state. A SECOND drain invokes + // `fn` zero times and returns an Internal error (a re-merge of the still-present + // run files would otherwise re-emit every term). Returns non-OK on spill/merge + // I/O or corruption errors, or if a prior add_token latched a validation error + // into status(). + Status for_each_term_sorted(const StreamedTermConsumer& fn); + +private: + struct CommonGramPairCache; + struct CommonGramPlainTermCache; + + enum class PostingChainShape : uint8_t { + kTaggedPositioned, + kTaggedDocsOnly, + kStatlessDocsOnly, + }; + + // Compact per-term accumulator: ONE tagged-varint arena chain plus a few cursors. + // A statless CommonGram keeps its first distinct doc inline in cur_docid and + // starts a chain only when a second doc arrives. For other posting shapes, a + // sentinel chain head marks an empty term. ntok / ndocs bound the decode loop + // and size reserves. + // Total 28 B per live term. + static constexpr uint32_t kNoChain = 0xFFFFFFFFU; + struct Term { + uint32_t head = kNoChain; // chain read entry point + CompactPostingPool::SliceWriter w; // chain cursor (8 B) + uint32_t ntok = 0; // total tokens (entries) in the chain + uint32_t cur_docid = 0; // most-recent doc id: detects doc change AND + // is the zigzag delta base for the next doc + // Exact count of new-doc groups in the chain (one per new_doc tag). It + // bounds the decode reserves and equals the distinct-doc count while the + // input remains sorted; a later out-of-order coalesce can only shrink it. + uint32_t ndocs = 0; + PostingChainShape shape = PostingChainShape::kTaggedPositioned; + uint8_t level = 0; // current slice level of w (packed here, not in w) + bool started = false; // false until the first token is accumulated + bool sorted = true; // false if a docid arrived out of ascending order + }; + static_assert(sizeof(CompactPostingPool::SliceWriter) == 8, + "SliceWriter must stay 8 bytes to keep Term compact"); + static_assert(sizeof(Term) == 28, "Term must stay compact for high-cardinality imports"); + + struct TrackedTermPostings { + explicit TrackedTermPostings(MemoryReporter* reporter) + : docids_reservation(reporter == nullptr ? MemoryReporter::Reservation() + : reporter->make_reservation()), + freqs_reservation(reporter == nullptr ? MemoryReporter::Reservation() + : reporter->make_reservation()), + positions_reservation(reporter == nullptr ? MemoryReporter::Reservation() + : reporter->make_reservation()) {} + + TrackedTermPostings(const TrackedTermPostings&) = delete; + TrackedTermPostings& operator=(const TrackedTermPostings&) = delete; + + // Reservations precede the posting vectors so physical allocations are + // destroyed before their charges are released. + MemoryReporter::Reservation docids_reservation; + MemoryReporter::Reservation freqs_reservation; + MemoryReporter::Reservation positions_reservation; + TermPostings postings; + }; + + // The active vocabulary (term-id -> string): either the borrowed pointer or, + // in owned mode, &owned_vocab_. Always non-null after construction. + const std::vector& vocab() const { return *vocab_; } + + // Accumulates one already-validated token into the per-id Term and checks the + // spill gate once for that input token. + void accumulate(uint32_t term_id, uint32_t docid, uint32_t pos, bool retain_positions); + void accumulate_without_spill_gate(uint32_t term_id, uint32_t docid, uint32_t pos, + PostingChainShape shape); + void add_common_gram_without_spill_gate(PlainTermId left, PlainTermId right, uint32_t docid, + uint32_t pos, bool retain_positions); + + // Per-token gate-2 tail of accumulate(): reports the token's resident growth, + // then spills when the unified cap / local threshold fires with a worthwhile + // reclaimable arena (the G08 anti-churn floor), when the G09 process-wide + // limiter's advisory request flag is pending (honored here, on the owner's + // own thread; bypasses the G08 floor but requires one allocated arena block + // so a run is writable), or when the arena nears its hard 4 GiB offset + // limit. Every public add path invokes this gate once; the fused CommonGrams + // path invokes it after appending both the gram and its right plain token. + void maybe_spill_after_token(); + + Status to_postings(std::string term, Term&& t, TrackedTermPostings* tracked) const; + class ArenaTermPostingSource; + + // Returns the touched term-ids sorted by their vocab string (lexicographic). + // The first spill builds the full integer string-rank. Later spills reuse it + // while the append-only vocabulary is unchanged. If the vocabulary grew, an + // ordinary spill sorts only this run's touched ids by string; run compaction + // and final merge rebuild the full rank when they actually require it. + std::vector sorted_ids() const; + // Builds string_rank_ (term-id -> lexicographic rank) for the current complete + // vocabulary. Idempotent until the append-only vocabulary grows. + void ensure_string_rank() const; + Status drain_sorted_streamed(const StreamedTermConsumer& fn); + // Spills the current buffer to a fresh sorted run file and clears memory. + Status spill_to_run(); + // G09 run-file cap enforcement (see set_max_run_files): merge-compacts the + // current run files into ONE fresh run (same term stream, ids ordered by + // the current string rank), deletes the old + // files and replaces run_paths_ with the compacted one. Called by + // spill_to_run before opening a new run once the cap is reached. + Status compact_runs(); + // Writes all current terms (sorted) to an already-open RunWriter, draining. + Status drain_to_writer(class RunWriter* w); + // REAL resident accumulator bytes -- the single source of truth for the gate-2 + // spill trigger and every MemoryReporter delta. G08: sums EVERY live input-side + // structure -- the posting arena (docs+prx payload) + // plus the vocab-sized slot index, the Term slot pool + free/touched lists, the + // owned vocabulary (headers by capacity + string heap payloads via + // owned_vocab_heap_bytes_) and its intern set, plus the cached string ranks. + // Capacity, not size, throughout: the reserved tail is resident RSS and + // survives spills. + uint64_t resident_bytes() const; + // Reports the signed change in REAL resident bytes (resident_bytes()) to + // mem_reporter_ since the previous call, then caches the new total. + // Single-source diff: every grow/reset/free emits EXACTLY ONE delta + // (self-balancing -> impossible to double-count or miss a negative). No-op when + // mem_reporter_ is null. + void report_arena_delta(); + Status merge_runs_streamed(const StreamedTermConsumer& fn); + Status prepare_run_merge(std::function* materializer); + void finish_run_merge(); + // Deletes every temp run file; called from the destructor (RAII cleanup). + void cleanup_runs(); + // Frees a drained term's accumulator (id leaves the touched set). + void release_term(uint32_t term_id); + + // Stores a first-seen owned-vocabulary term under a stable id. + uint32_t append_owned_vocab_term(std::string&& term_str); + uint32_t intern_owned_term(std::string&& term_str, size_t term_hash); + uint32_t find_or_intern_owned_term(std::string_view term); + uint32_t find_or_intern_common_gram_pair(PlainTermId left, PlainTermId right, uint64_t pair); + uint32_t find_interned_plain_term(std::string_view term, size_t term_hash); + void remember_plain_term(size_t term_hash, uint32_t term_id); + bool transient_term_less(uint32_t left_id, uint32_t right_id) const; + std::string materialize_transient_term(std::string_view term) const; + + const std::vector* vocab_; // active vocab (borrowed or &owned_) + std::vector owned_vocab_; // owned mode: interned term strings + + enum class CommonWordClassification : uint8_t { + kUnknown, + kNotCommon, + kCommon, + }; + // Stable semantic classification keyed by owned term id. Pair ids remain + // kUnknown; physical plain ids are classified once from their logical bytes. + std::vector common_word_classification_; + + // G08: running sum of the owned vocab strings' HEAP payloads (0 for SSO + // strings -- their bytes live inside the headers owned_vocab_.capacity() + // already charges; capacity+1 for heap strings). Maintained incrementally by + // intern_owned_term so resident_bytes() stays O(1); terminal drains zero it + // when owned_vocab_ is released. + uint64_t owned_vocab_heap_bytes_ = 0; + + // G08: fixed per-entry estimate for one intern-set entry. Sized for the + // pre-G10 NODE-based set (16 B next-ptr+id node, its malloc chunk rounding, + // and an amortized bucket-array share) and deliberately UNCHANGED by the G10 + // swap to the flat set: resident_bytes() feeds the gate-2 spill trigger, so + // keeping the constant keeps the resident-byte sequence -- and therefore + // every spill point and the drained output -- bit-identical to the prior + // build. It still OVER-approximates the 4-byte flat key plus control bytes + // and load-factor slack, which can only fire the gate earlier, never overshoot. + // Deterministic so the accounting tests can reason about it, and ZERO for an + // empty set so an untouched (borrowed-mode) buffer charges nothing for it. + static constexpr uint64_t kInternEntryEstimateBytes = 48; + + // The table slot stores only the stable vocabulary id. String probes hash + // their bytes once; stored ids rehash through the sole owned vocabulary. + // Equality always resolves term identity from the complete bytes, so hash + // collisions are harmless and only add comparisons. + static size_t hash_term_bytes(std::string_view s) noexcept { + return std::hash {}(s); + } + + struct OwnedVocabHash { + using is_transparent = void; + const std::vector* vocab = nullptr; + size_t hash_mask = std::numeric_limits::max(); + size_t operator()(std::string_view term) const noexcept { + return hash_term_bytes(term) & hash_mask; + } + size_t operator()(uint32_t term_id) const noexcept { + return (*this)(std::string_view((*vocab)[term_id])); + } + }; + struct OwnedVocabEq { + using is_transparent = void; + const std::vector* vocab = nullptr; + bool operator()(uint32_t left, uint32_t right) const noexcept { return left == right; } + bool operator()(uint32_t stored, std::string_view probe) const noexcept; + bool operator()(std::string_view probe, uint32_t stored) const noexcept; + }; + // One flat table is the sole ordinary-term admission path. Heterogeneous + // probes avoid temporary strings; prepared insertion materializes only a miss. + // A failed table insertion rolls back the preceding vocabulary append, and no + // iterator survives a mutation. + phmap::flat_hash_set intern_; + + // CommonGram pair ids are already a canonical, collision-free key. Keep them + // out of the string-content intern table: an L0 cache miss probes this native + // map and materializes the 10-byte transient vocabulary key only for a new + // pair. The map survives ordinary spills because the persistent vocabulary + // and term ids do; terminal drains release it before output reservations. + phmap::flat_hash_map common_gram_pair_intern_; + + bool has_positions_; + bool common_gram_pair_keys_ = false; + std::unique_ptr common_gram_pair_cache_; + uint64_t common_gram_pair_cache_bytes_ = 0; + std::unique_ptr common_gram_plain_term_cache_; + uint64_t common_gram_plain_term_cache_bytes_ = 0; + size_t spill_threshold_bytes_; // 0 => unlimited (no spilling) + uint64_t total_tokens_ = 0; + + // POOLED accumulators (replaces a dense vocab-sized std::vector, which + // cost ~80 B per vocab id even for the ~empty majority -- the single largest + // input-phase memory line). slot_of_ is the only vocab-sized array: a 4 B index + // per id (0 == no live Term; otherwise slot index + 1). slots_ holds ONE Term + // per CURRENTLY-LIVE id, so its size tracks the live touched count, not the + // vocabulary. On first touch an id claims a slot (reusing a freed one from + // free_slots_ when available, else appending). release_term frees the slot back + // to the pool and clears slot_of_[id]. touched_ids_ lists every live id so + // finalize/spill iterate touched ids without scanning the whole vocabulary. + // present_[id] is now (slot_of_[id] != 0). The hot add path is still a vector + // index + a couple of pushes: no hashing, no per-token allocation. + std::vector slot_of_; // vocab-sized: id -> slot index + 1 (0=empty) + std::vector slots_; // live Term pool (size ~ live touched count) + std::vector free_slots_; // recycled slot indices (drained terms) + std::vector touched_ids_; + size_t live_term_count_ = 0; // present (non-drained) terms; == unique_terms() + + // Shared arena backing every live term's DOC and POS varint byte chains. Holds + // the bulk of the accumulator's memory in a few large blocks (no per-term vector + // headers, no per-vector doubling slack) -- the compact-RSS win. + CompactPostingPool pool_; + + // Optional writer-level build-RAM reporter (null off-Doris / unit tests) and the + // last resident-byte total it was told about. report_arena_delta() diffs the live + // total (arena_bytes() + slot_of_.capacity()*4) against reported_resident_. + MemoryReporter* mem_reporter_ = nullptr; + int64_t reported_resident_ = 0; + + // ---- G09 process-wide limiter hookup (null / false = feature off) -------- + // The registry this buffer joined via attach_global_limiter (borrowed; must + // outlive the buffer), and the ADVISORY forced-spill request flag the + // limiter sets from other threads (only ever under the registry mutex; the + // owner reads it relaxed on its own thread each token). The flag pointer + // doubles as the buffer's registry identity. + GlobalMemoryLimiter* global_limiter_ = nullptr; + std::atomic global_spill_requested_ {false}; + // G09 forced-spill floor / run-file cap (see the public setters above). + uint64_t forced_spill_min_arena_bytes_ = kDefaultForcedSpillMinArenaBytes; + size_t max_run_files_ = kDefaultMaxRunFilesPerBuffer; + + // Returns the live Term for `term_id`, claiming a pool slot on first touch. + Term& term_slot(uint32_t term_id, bool* new_term); + + // Appends one varint to a term's chain, lazily starting the chain on first use + // (so an untouched term costs no arena bytes). + void put_varint(Term* t, uint64_t v); + + std::vector run_paths_; // spilled run temp files (deleted in dtor) + Status spill_status_; // first spill / range error, at finalize + bool drained_ = false; // set once finalize_sorted/for_each_term_sorted has run; + // a second drain would (spilled path) re-merge the run + // files and re-emit every term, or (in-memory path) emit + // nothing -- both wrong. Guard against the double-drain. + + // Lazily-built vocab-sized map: term-id -> its lexicographic rank among all + // vocab strings. `size() == vocab().size()` means the rank is current; a + // smaller non-zero size is a stale rank retained after vocabulary growth. + // Its capacity is still advanced on stale ordinary spills so resident-byte + // accounting and spill-trigger timing keep the prior full-rank charge. + mutable std::vector string_rank_; +}; + +// TEST-ONLY observability seam (mirrors the reader-side decode-counter pattern). +// Counts how many times a vocabulary string is MATERIALIZED into owned_vocab_ during +// owned-mode interning. With single-store interning this is bumped EXACTLY ONCE per +// DISTINCT term (the owned_vocab_.emplace_back) and NEVER per token -- so feeding the +// same term M times still materializes it once, and the per-token temporary probe +// string is gone entirely. Writer tests use it for deterministic allocation +// assertions (count == distinct terms). Process-global; reset between tests. Not part +// of the production API. +namespace testing { +// G11 bench seam (honored under BE_TEST only): disables the add-path +// prefetch hints so the locality bench can A/B them within ONE process. +// Production builds prefetch unconditionally. +void set_bench_disable_g11_prefetch(bool disabled); + +uint64_t vocab_string_materialization_count(); +void reset_vocab_string_materialization_count(); + +// G09 process-wide limiter seam: spills that observed -- and cleared -- a +// PENDING global forced-spill request at the moment they fired (whether or not +// the per-writer gate would also have spilled that token; the request was +// consumed either way). Incremented under BE_TEST only because the check sits +// on the per-token path of every +// concurrent writer). Deterministic on the single-threaded build path; reset +// between tests. Not part of the production API. +uint64_t global_forced_spills(); +void reset_global_forced_spills(); + +// G09 run-file cap seam: merge-compactions of a buffer's accumulated spill +// runs (each collapses the whole run list into one file). Always-on relaxed +// atomic (a compaction is rare -- at most once per cap-many spills -- so +// contention is a non-issue, unlike the per-token seams above). Deterministic +// on the single-threaded build path; reset between tests. Not part of the +// production API. +uint64_t run_compactions(); +void reset_run_compactions(); + +// Number of complete-vocabulary lexicographic rank rebuilds. Ordinary spills +// with a stale rank must not increment it; run compaction and final merge may. +uint64_t string_rank_rebuilds(); +void reset_string_rank_rebuilds(); + +// Complete touched vocabularies invert the dense rank in O(N); partial runs +// retain comparison sorting. Both counters compile out of production. +uint64_t dense_rank_inversions(); +uint64_t rank_comparison_sorts(); +void reset_rank_ordering_counts(); + +// CommonGrams pair-key terminal-ordering seam. Both counters compile out of +// production: tests use them to prove terminal sorting/materialization takes the +// trusted fixed-key path instead of re-running generic key validation. +uint64_t common_gram_pair_unchecked_decode_count(); +uint64_t common_gram_trusted_plain_decode_count(); +void reset_common_gram_pair_fast_path_counts(); + +// CommonGrams pair direct-cache seam. The counters compile out of production; +// tests use them to prove repeated pairs bypass key encoding and the intern table. +// Docs-only pairs additionally suppress same-document repeats, while positioned +// pairs reuse the cached term id and still accumulate every position. +uint64_t common_gram_pair_cache_probes(); +uint64_t common_gram_pair_cache_pair_hits(); +uint64_t common_gram_pair_cache_same_doc_hits(); +void reset_common_gram_pair_cache_counts(); + +// Native CommonGram pair-interner seam. Normal production pair ingestion must +// never route a transient pair key through the generic string-content table. +uint64_t common_gram_native_pair_probes(); +uint64_t common_gram_native_pair_hits(); +uint64_t common_gram_native_pair_inserts(); +void reset_common_gram_native_pair_intern_counts(); + +uint64_t common_gram_logical_validation_count(); +void reset_common_gram_logical_validation_count(); + +// CommonGrams plain-term hot-cache seam: total cache probes, cache hits, and +// fallbacks that reached the global intern table. +uint64_t common_gram_plain_cache_probes(); +uint64_t common_gram_plain_cache_hits(); +uint64_t common_gram_plain_intern_table_probes(); +void reset_common_gram_plain_cache_counts(); + +// Counts equality checks that must dereference owned vocabulary bytes after the +// inline length and prefix checks. Short terms must never increment this counter. +uint64_t owned_term_full_byte_comparison_count(); +void reset_owned_term_full_byte_comparison_count(); +void fail_next_owned_term_reserve(); +void fail_next_owned_term_emplace(); + +uint64_t spill_gate_check_count(); +void reset_spill_gate_check_count(); + +// Counts compact-chain varint decodes during arena source consumption. Tests use +// this to prevent positioned sources from replaying the token chain. +uint64_t compact_chain_varint_decode_count(); +void reset_compact_chain_varint_decode_count(); +} // namespace testing + +} // namespace doris::snii::writer diff --git a/be/src/storage/index/snii/writer/temp_dir.cpp b/be/src/storage/index/snii/writer/temp_dir.cpp new file mode 100644 index 00000000000000..23da77c6a2ecff --- /dev/null +++ b/be/src/storage/index/snii/writer/temp_dir.cpp @@ -0,0 +1,41 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/writer/temp_dir.h" + +#include +#include +#include + +#include "runtime/exec_env.h" +#include "storage/index/index_writer.h" // segment_v2::TmpFileDirs (full definition) + +namespace doris::snii::writer { + +std::string resolve_temp_dir() { + // Use Doris's configured spill/scratch dirs (the same source the inverted-index + // writer uses; see index_file_writer.cpp). SNII spills/section temp files live in + // a dedicated "snii" subdirectory so they do not crowd the tmp root alongside + // every other component's files. + auto dir = ExecEnv::GetInstance()->get_tmp_file_dirs()->get_tmp_file_dir(); + dir /= "snii"; + std::error_code ec; + std::filesystem::create_directories(dir, ec); + return dir.native(); +} + +} // namespace doris::snii::writer diff --git a/be/src/storage/index/snii/writer/temp_dir.h b/be/src/storage/index/snii/writer/temp_dir.h new file mode 100644 index 00000000000000..1b249087cbd213 --- /dev/null +++ b/be/src/storage/index/snii/writer/temp_dir.h @@ -0,0 +1,47 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include +#include + +namespace doris::snii::writer { + +// Scratch directory for spill runs and section temp files. Uses Doris's configured +// tmp_file_dirs (ExecEnv::get_tmp_file_dirs) in production; when those are not +// initialized (unit tests / standalone) it falls back to SNII_TEMP_DIR -> TMPDIR -> +// /tmp. Defined in temp_dir.cpp to keep exec_env.h out of this header. +// +// The fallback (SNII_TEMP_DIR / TMPDIR) should point at a REAL disk (SSD/NVMe): +// /tmp is often tmpfs (RAM-backed), where spilling does NOT reduce RSS. +std::string resolve_temp_dir(); + +// Best-effort free bytes on the filesystem backing `dir`. Returns UINT64_MAX when +// statvfs fails, so a caller's space pre-check never false-positives on an +// unstattable path. CAVEATS: this is best-effort only -- it is subject to TOCTOU +// (free space can drop before/while the write runs), and on tmpfs it reports +// RAM-backed space (use the temp-dir config to avoid tmpfs in the first place). +inline uint64_t temp_dir_available_bytes(const std::string& dir) { + struct statvfs vfs; + if (::statvfs(dir.c_str(), &vfs) != 0) return UINT64_MAX; + return static_cast(vfs.f_bavail) * static_cast(vfs.f_frsize); +} + +} // namespace doris::snii::writer diff --git a/be/src/storage/index/snii/writer/term_posting_source.h b/be/src/storage/index/snii/writer/term_posting_source.h new file mode 100644 index 00000000000000..9de6b5405fee05 --- /dev/null +++ b/be/src/storage/index/snii/writer/term_posting_source.h @@ -0,0 +1,392 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/writer/memory_reporter.h" + +namespace doris::snii::writer { + +struct MutableTermPostingSpan { + std::span docids; + std::span freqs; + std::span positions_flat; +}; + +// Reusable, reservation-backed transfer storage for one source fill. A source +// may append multiple runs during one fill, but every run must agree on whether +// transient frequency statistics are present. clear_reuse() preserves capacity +// and its memory charge. +class TermPostingBuffer { +public: + explicit TermPostingBuffer(MemoryReporter* memory_reporter) + : memory_reporter_(memory_reporter), + capacity_reservation_(memory_reporter == nullptr + ? MemoryReporter::Reservation() + : memory_reporter->make_reservation()) {} + + TermPostingBuffer(const TermPostingBuffer&) = delete; + TermPostingBuffer& operator=(const TermPostingBuffer&) = delete; + TermPostingBuffer(TermPostingBuffer&&) = delete; + TermPostingBuffer& operator=(TermPostingBuffer&&) = delete; + + size_t document_count() const { return docids_.size(); } + bool empty() const { return docids_.empty(); } + + void clear_reuse() { + docids_.clear(); + freqs_.clear(); + positions_flat_.clear(); + has_freqs_.reset(); + } + + void clear_reuse_and_release_excess(size_t max_retained_capacity) { + clear_reuse(); + bool released = false; + if (docids_.capacity() > max_retained_capacity) { + std::vector().swap(docids_); + released = true; + } + if (freqs_.capacity() > max_retained_capacity) { + std::vector().swap(freqs_); + released = true; + } + if (positions_flat_.capacity() > max_retained_capacity) { + std::vector().swap(positions_flat_); + released = true; + } + if (released && memory_reporter_ != nullptr) { + uint64_t retained_bytes = 0; + DORIS_CHECK(capacity_bytes(docids_.capacity(), freqs_.capacity(), + positions_flat_.capacity(), &retained_bytes) + .ok()); + DORIS_CHECK(capacity_reservation_.set_bytes(retained_bytes).ok()); + } + } + + Status append(std::span docids, std::span freqs, + std::span positions_flat) { + const bool has_freqs = !freqs.empty(); + if (has_freqs && freqs.size() != docids.size()) { + return Status::Error( + "term posting buffer: freqs length must equal docids"); + } + if (!has_freqs && !positions_flat.empty()) { + return Status::Error( + "term posting buffer: positions require parallel freqs"); + } + uint64_t expected_positions = 0; + for (uint32_t freq : freqs) { + if (freq > std::numeric_limits::max() - expected_positions) { + return Status::Error( + "term posting buffer: position count overflow"); + } + expected_positions += freq; + } + if (!positions_flat.empty() && expected_positions != positions_flat.size()) { + return Status::Error( + "term posting buffer: positions count must equal sum(freqs)"); + } + if (has_freqs_.has_value() && *has_freqs_ != has_freqs) { + return Status::Error( + "term posting buffer: frequency shape changed within one fill"); + } + + MutableTermPostingSpan destination; + RETURN_IF_ERROR( + grow_uninitialized(docids.size(), has_freqs, positions_flat.size(), &destination)); + std::ranges::copy(docids, destination.docids.begin()); + std::ranges::copy(freqs, destination.freqs.begin()); + std::ranges::copy(positions_flat, destination.positions_flat.begin()); + return Status::OK(); + } + + // Extends the current fill once and exposes the new tail for direct decode. + // The caller must initialize every returned element before returning from + // TermPostingSource::fill. Shape and capacity changes are committed only + // after all reservations succeed. + Status grow_uninitialized(size_t document_count, bool has_freqs, size_t position_count, + MutableTermPostingSpan* destination) { + if (destination == nullptr) { + return Status::Error( + "term posting buffer: null writable span destination"); + } + if (!has_freqs && position_count != 0) { + return Status::Error( + "term posting buffer: positions require parallel freqs"); + } + if (has_freqs_.has_value() && *has_freqs_ != has_freqs) { + return Status::Error( + "term posting buffer: frequency shape changed within one fill"); + } + + const size_t doc_begin = docids_.size(); + const size_t freq_begin = freqs_.size(); + const size_t position_begin = positions_flat_.size(); + size_t target_docids = 0; + size_t target_freqs = 0; + size_t target_positions = 0; + RETURN_IF_ERROR(checked_size(doc_begin, document_count, &target_docids)); + RETURN_IF_ERROR(checked_size(freq_begin, has_freqs ? document_count : 0, &target_freqs)); + RETURN_IF_ERROR(checked_size(position_begin, position_count, &target_positions)); + RETURN_IF_ERROR(reserve_for_append(target_docids, target_freqs, target_positions)); + + docids_.resize(target_docids); + freqs_.resize(target_freqs); + positions_flat_.resize(target_positions); + destination->docids = std::span(docids_).subspan(doc_begin, document_count); + destination->freqs = std::span(freqs_).subspan(freq_begin, has_freqs ? document_count : 0); + destination->positions_flat = + std::span(positions_flat_).subspan(position_begin, position_count); + if (document_count != 0) { + has_freqs_ = has_freqs; + } + return Status::OK(); + } + + // Appends one position while a source decodes a frequency-bearing fill. + // The common path writes into retained capacity; growth keeps replacement + // reservation accounting atomic. + Status append_position(uint32_t position) { + if (!has_freqs_.value_or(false)) { + return Status::Error( + "term posting buffer: incremental positions require parallel freqs"); + } + if (positions_flat_.size() == positions_flat_.capacity()) { + size_t target_positions = 0; + RETURN_IF_ERROR(checked_size(positions_flat_.size(), 1, &target_positions)); + RETURN_IF_ERROR(reserve_for_append(docids_.size(), freqs_.size(), target_positions)); + } + positions_flat_.push_back(position); + return Status::OK(); + } + + std::span docids() const { return docids_; } + std::span freqs() const { return freqs_; } + std::span positions_flat() const { return positions_flat_; } + +private: + static Status checked_size(size_t current, size_t additional, size_t* target) { + if (additional > std::numeric_limits::max() - current) { + return Status::Error( + "term posting buffer: capacity overflow"); + } + *target = current + additional; + if (*target > std::numeric_limits::max() / sizeof(uint32_t)) { + return Status::Error( + "term posting buffer: byte capacity overflow"); + } + return Status::OK(); + } + + static Status growth_capacity(size_t required, size_t current, size_t* target) { + *target = required; + if (current != 0 && current <= std::numeric_limits::max() / 2) { + *target = std::max(required, current * 2); + } + if (*target > std::numeric_limits::max() / sizeof(uint32_t)) { + return Status::Error( + "term posting buffer: growth capacity overflow"); + } + return Status::OK(); + } + + static Status capacity_bytes(size_t docids_capacity, size_t freqs_capacity, + size_t positions_capacity, uint64_t* bytes) { + const uint64_t docids_bytes = docids_capacity * sizeof(uint32_t); + const uint64_t freqs_bytes = freqs_capacity * sizeof(uint32_t); + const uint64_t positions_bytes = positions_capacity * sizeof(uint32_t); + if (freqs_bytes > std::numeric_limits::max() - docids_bytes || + positions_bytes > std::numeric_limits::max() - docids_bytes - freqs_bytes) { + return Status::Error( + "term posting buffer: aggregate capacity overflow"); + } + *bytes = docids_bytes + freqs_bytes + positions_bytes; + return Status::OK(); + } + + Status reserve_for_append(size_t target_docids, size_t target_freqs, size_t target_positions) { + const bool grow_docids = target_docids > docids_.capacity(); + const bool grow_freqs = target_freqs > freqs_.capacity(); + const bool grow_positions = target_positions > positions_flat_.capacity(); + if (!grow_docids && !grow_freqs && !grow_positions) { + return Status::OK(); + } + size_t docids_capacity = docids_.capacity(); + size_t freqs_capacity = freqs_.capacity(); + size_t positions_capacity = positions_flat_.capacity(); + if (grow_docids) { + RETURN_IF_ERROR(growth_capacity(target_docids, docids_.capacity(), &docids_capacity)); + } + if (grow_freqs) { + RETURN_IF_ERROR(growth_capacity(target_freqs, freqs_.capacity(), &freqs_capacity)); + } + if (grow_positions) { + RETURN_IF_ERROR(growth_capacity(target_positions, positions_flat_.capacity(), + &positions_capacity)); + } + if (memory_reporter_ == nullptr) { + if (grow_docids) docids_.reserve(docids_capacity); + if (grow_freqs) freqs_.reserve(freqs_capacity); + if (grow_positions) positions_flat_.reserve(positions_capacity); + return Status::OK(); + } + + uint64_t previous_bytes = 0; + uint64_t final_bytes = 0; + RETURN_IF_ERROR(capacity_bytes(docids_.capacity(), freqs_.capacity(), + positions_flat_.capacity(), &previous_bytes)); + RETURN_IF_ERROR( + capacity_bytes(docids_capacity, freqs_capacity, positions_capacity, &final_bytes)); + RETURN_IF_ERROR(capacity_reservation_.set_bytes(final_bytes)); + + const uint64_t overlap_bytes = + std::max({grow_docids ? docids_.capacity() * sizeof(uint32_t) : 0, + grow_freqs ? freqs_.capacity() * sizeof(uint32_t) : 0, + grow_positions ? positions_flat_.capacity() * sizeof(uint32_t) : 0}); + MemoryReporter::Reservation overlap_reservation = memory_reporter_->make_reservation(); + Status overlap_status = overlap_reservation.set_bytes(overlap_bytes); + if (!overlap_status.ok()) { + DORIS_CHECK(capacity_reservation_.set_bytes(previous_bytes).ok()); + return overlap_status; + } + + if (grow_docids) { + docids_.reserve(docids_capacity); + DCHECK_EQ(docids_.capacity(), docids_capacity); + } + if (grow_freqs) { + freqs_.reserve(freqs_capacity); + DCHECK_EQ(freqs_.capacity(), freqs_capacity); + } + if (grow_positions) { + positions_flat_.reserve(positions_capacity); + DCHECK_EQ(positions_flat_.capacity(), positions_capacity); + } + return Status::OK(); + } + + MemoryReporter* memory_reporter_ = nullptr; + // The reservation precedes vectors so their allocations are destroyed first. + MemoryReporter::Reservation capacity_reservation_; + std::vector docids_; + std::vector freqs_; + std::vector positions_flat_; + std::optional has_freqs_; +}; + +class TermPostingSource { +public: + virtual ~TermPostingSource() = default; + + // out is empty on entry. Unless this call reaches the term end, it must + // return exactly target_docs postings. exhausted means no postings remain + // after this call. The source and output are borrowed synchronously. + virtual Status fill(uint32_t target_docs, TermPostingBuffer* out, bool* exhausted) = 0; +}; + +// Synchronous non-owning adapter for callers that already hold one materialized +// posting list. It slices the arrays into the writer's requested document +// windows without copying the whole term into an intermediate object. +class SpanTermPostingSource final : public TermPostingSource { +public: + SpanTermPostingSource(std::span docids, std::span freqs, + std::span positions_flat) + : docids_(docids), freqs_(freqs), positions_flat_(positions_flat) {} + + Status fill(uint32_t target_docs, TermPostingBuffer* out, bool* exhausted) override { + if (out == nullptr || exhausted == nullptr || target_docs == 0) { + return Status::Error( + "span posting source: invalid fill arguments"); + } + if (!out->empty()) { + return Status::Error( + "span posting source: output must be empty"); + } + if (!freqs_.empty() && freqs_.size() != docids_.size()) { + return Status::Error( + "span posting source: freqs length must equal docids"); + } + if (freqs_.empty() && !positions_flat_.empty()) { + return Status::Error( + "span posting source: positions require parallel freqs"); + } + + const size_t count = + std::min(static_cast(target_docs), docids_.size() - doc_offset_); + size_t position_count = 0; + if (!positions_flat_.empty()) { + for (size_t i = 0; i < count; ++i) { + RETURN_IF_ERROR( + checked_add(position_count, freqs_[doc_offset_ + i], &position_count)); + } + if (position_count > positions_flat_.size() - position_offset_) { + return Status::Error( + "span posting source: positions shorter than sum(freqs)"); + } + } + + RETURN_IF_ERROR(out->append( + docids_.subspan(doc_offset_, count), + freqs_.empty() ? std::span {} : freqs_.subspan(doc_offset_, count), + positions_flat_.subspan(position_offset_, position_count))); + doc_offset_ += count; + position_offset_ += position_count; + *exhausted = doc_offset_ == docids_.size(); + if (*exhausted && !positions_flat_.empty() && position_offset_ != positions_flat_.size()) { + return Status::Error( + "span posting source: positions longer than sum(freqs)"); + } + return Status::OK(); + } + + bool exhausted() const { return doc_offset_ == docids_.size(); } + +private: + static Status checked_add(size_t current, uint32_t additional, size_t* result) { + if (additional > std::numeric_limits::max() - current) { + return Status::Error( + "span posting source: position count overflow"); + } + *result = current + additional; + return Status::OK(); + } + + std::span docids_; + std::span freqs_; + std::span positions_flat_; + size_t doc_offset_ = 0; + size_t position_offset_ = 0; +}; + +struct StreamedTermPostings { + std::string term; + bool retain_positions = true; + TermPostingSource* source = nullptr; +}; + +} // namespace doris::snii::writer diff --git a/be/src/storage/olap_common.h b/be/src/storage/olap_common.h index 51f23f93cd1bd1..26706d3a5da525 100644 --- a/be/src/storage/olap_common.h +++ b/be/src/storage/olap_common.h @@ -40,6 +40,7 @@ #include "common/exception.h" #include "io/io_common.h" #include "storage/index/inverted/inverted_index_stats.h" +#include "storage/index/snii/snii_query_stats.h" #include "storage/olap_define.h" #include "storage/rowset/rowset_fwd.h" #include "util/hash_util.hpp" @@ -400,6 +401,8 @@ struct OlapReaderStatistics { int64_t inverted_index_query_timer = 0; int64_t inverted_index_query_cache_hit = 0; int64_t inverted_index_query_cache_miss = 0; + int64_t inverted_index_query_cache_lookup = 0; + int64_t inverted_index_query_cache_insert = 0; int64_t inverted_index_query_null_bitmap_timer = 0; int64_t inverted_index_query_bitmap_copy_timer = 0; int64_t inverted_index_searcher_open_timer = 0; @@ -411,6 +414,8 @@ struct OlapReaderStatistics { int64_t inverted_index_downgrade_count = 0; int64_t inverted_index_analyzer_timer = 0; int64_t inverted_index_lookup_timer = 0; + // See snii_query_stats.h: one field here instead of one per SNII counter. + snii::SniiQueryStats snii_stats; InvertedIndexStatistics inverted_index_stats; int64_t ann_index_load_ns = 0; diff --git a/be/src/storage/predicate_collector.cpp b/be/src/storage/predicate_collector.cpp deleted file mode 100644 index fa8fc0117ce34f..00000000000000 --- a/be/src/storage/predicate_collector.cpp +++ /dev/null @@ -1,314 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "storage/predicate_collector.h" - -#include - -#include - -#include "exec/common/variant_util.h" -#include "exprs/vexpr.h" -#include "exprs/vexpr_context.h" -#include "exprs/vliteral.h" -#include "exprs/vsearch.h" -#include "exprs/vslot_ref.h" -#include "gen_cpp/Exprs_types.h" -#include "storage/index/index_reader_helper.h" -#include "storage/index/inverted/analyzer/analyzer.h" -#include "storage/index/inverted/util/string_helper.h" -#include "storage/tablet/tablet_schema.h" - -namespace doris { - -using namespace segment_v2; - -VSlotRef* PredicateCollector::find_slot_ref(const VExprSPtr& expr) const { - if (!expr) { - return nullptr; - } - - auto cur = VExpr::expr_without_cast(expr); - if (cur->node_type() == TExprNodeType::SLOT_REF) { - return static_cast(cur.get()); - } - - for (const auto& ch : cur->children()) { - if (auto* s = find_slot_ref(ch)) { - return s; - } - } - - return nullptr; -} - -std::string PredicateCollector::build_field_name(int32_t col_unique_id, - const std::string& suffix_path) const { - std::string field_name = std::to_string(col_unique_id); - if (!suffix_path.empty()) { - field_name += "." + suffix_path; - } - return field_name; -} - -Status MatchPredicateCollector::collect(RuntimeState* state, const TabletSchemaSPtr& tablet_schema, - const VExprSPtr& expr, CollectInfoMap* collect_infos) { - DCHECK(collect_infos != nullptr); - - auto* left_slot_ref = find_slot_ref(expr->children()[0]); - if (left_slot_ref == nullptr) { - return Status::Error( - "Index statistics collection failed: Cannot find slot reference in match predicate " - "left expression"); - } - - auto* right_literal = static_cast(expr->children()[1].get()); - DCHECK(right_literal != nullptr); - - const auto* sd = state->desc_tbl().get_slot_descriptor(left_slot_ref->slot_id()); - if (sd == nullptr) { - return Status::Error( - "Index statistics collection failed: Cannot find slot descriptor for slot_id={}", - left_slot_ref->slot_id()); - } - - int32_t col_idx = tablet_schema->field_index(left_slot_ref->column_name()); - if (col_idx == -1) { - return Status::Error( - "Index statistics collection failed: Cannot find column index for column={}", - left_slot_ref->column_name()); - } - - const auto& column = tablet_schema->column(col_idx); - auto index_metas = tablet_schema->inverted_indexs(column); - std::vector> owned_index_metas; - std::string index_suffix_path = column.suffix_path(); - - // Schema-only fallback for variant sub-columns. Collector runs at tablet - // level without segment context, so we cannot do nested-group inference - // or inherit_index runtime-type dispatch. Two paths cover what is - // resolvable from schema alone: - // 1. field_pattern templates (MATCH_NAME / MATCH_NAME_GLOB) via - // generate_sub_column_info. - // 2. Plain parent inverted index when the schema column is the dynamic - // path's VARIANT placeholder produced by _init_variant_columns. In - // that state inverted_indexs(column) misses because - // _path_set_info_map.subcolumn_indexes is only populated for typed - // paths / field_pattern outputs, not for plain parent indexes added - // by ALTER. Clone the parent's non-field-pattern indexes with the - // variant path as suffix so segment-side BM25 statistics can be - // collected. - if (index_metas.empty() && column.is_extracted_column()) { - TabletSchema::SubColumnInfo sub_column_info; - const std::string relative_path = column.path_info_ptr()->copy_pop_front().get_path(); - if (variant_util::generate_sub_column_info(*tablet_schema, column.parent_unique_id(), - relative_path, &sub_column_info) && - !sub_column_info.indexes.empty()) { - index_suffix_path = sub_column_info.column.suffix_path(); - for (auto& idx : sub_column_info.indexes) { - index_metas.push_back(idx.get()); - owned_index_metas.emplace_back(std::move(idx)); - } - } else if (column.is_variant_type()) { - const auto parent_indexes = tablet_schema->inverted_indexs(column.parent_unique_id()); - for (const auto* index : parent_indexes) { - if (!index->field_pattern().empty()) { - continue; - } - auto index_ptr = std::make_shared(*index); - index_ptr->set_escaped_escaped_index_suffix_path( - column.path_info_ptr()->get_path()); - index_metas.push_back(index_ptr.get()); - owned_index_metas.emplace_back(std::move(index_ptr)); - } - } - } - -#ifndef BE_TEST - if (index_metas.empty()) { - return Status::Error( - "Index statistics collection failed: Score query is not supported without inverted " - "index for column={}", - left_slot_ref->column_name()); - } -#endif - - for (const auto* index_meta : index_metas) { - if (!InvertedIndexAnalyzer::should_analyzer(index_meta->properties())) { - continue; - } - - if (!IndexReaderHelper::is_need_similarity_score(expr->op(), index_meta)) { - continue; - } - - auto options = DataTypeSerDe::get_default_format_options(); - options.timezone = &state->timezone_obj(); - auto term_infos = InvertedIndexAnalyzer::get_analyse_result(right_literal->value(options), - index_meta->properties()); - - std::string field_name = - build_field_name(index_meta->col_unique_ids()[0], index_suffix_path); - std::wstring ws_field_name = StringHelper::to_wstring(field_name); - - auto iter = collect_infos->find(ws_field_name); - if (iter == collect_infos->end()) { - CollectInfo collect_info; - collect_info.term_infos.insert(term_infos.begin(), term_infos.end()); - collect_info.index_meta = index_meta; - for (const auto& owned_index_meta : owned_index_metas) { - if (owned_index_meta.get() == index_meta) { - collect_info.owned_index_meta = owned_index_meta; - break; - } - } - (*collect_infos)[ws_field_name] = std::move(collect_info); - } else { - iter->second.term_infos.insert(term_infos.begin(), term_infos.end()); - } - } - - return Status::OK(); -} - -Status SearchPredicateCollector::collect(RuntimeState* state, const TabletSchemaSPtr& tablet_schema, - const VExprSPtr& expr, CollectInfoMap* collect_infos) { - DCHECK(collect_infos != nullptr); - - auto* search_expr = dynamic_cast(expr.get()); - if (search_expr == nullptr) { - return Status::InternalError("SearchPredicateCollector: expr is not VSearchExpr type"); - } - - const TSearchParam& search_param = search_expr->get_search_param(); - - RETURN_IF_ERROR(collect_from_clause(search_param.root, state, tablet_schema, collect_infos)); - - return Status::OK(); -} - -Status SearchPredicateCollector::collect_from_clause(const TSearchClause& clause, - RuntimeState* state, - const TabletSchemaSPtr& tablet_schema, - CollectInfoMap* collect_infos) { - const std::string& clause_type = clause.clause_type; - ClauseTypeCategory category = get_clause_type_category(clause_type); - - if (category == ClauseTypeCategory::COMPOUND) { - if (clause.__isset.children) { - for (const auto& child_clause : clause.children) { - RETURN_IF_ERROR( - collect_from_clause(child_clause, state, tablet_schema, collect_infos)); - } - } - return Status::OK(); - } - - return collect_from_leaf(clause, state, tablet_schema, collect_infos); -} - -Status SearchPredicateCollector::collect_from_leaf(const TSearchClause& clause, RuntimeState* state, - const TabletSchemaSPtr& tablet_schema, - CollectInfoMap* collect_infos) { - if (!clause.__isset.field_name || !clause.__isset.value) { - return Status::InvalidArgument("Search clause missing field_name or value"); - } - - const std::string& field_name = clause.field_name; - const std::string& value = clause.value; - const std::string& clause_type = clause.clause_type; - - if (!is_score_query_type(clause_type)) { - return Status::OK(); - } - - int32_t col_idx = tablet_schema->field_index(field_name); - if (col_idx == -1) { - return Status::OK(); - } - - const auto& column = tablet_schema->column(col_idx); - - auto index_metas = tablet_schema->inverted_indexs(column.unique_id(), column.suffix_path()); - if (index_metas.empty()) { - return Status::OK(); - } - - ClauseTypeCategory category = get_clause_type_category(clause_type); - for (const auto* index_meta : index_metas) { - std::set term_infos; - - if (category == ClauseTypeCategory::TOKENIZED) { - if (InvertedIndexAnalyzer::should_analyzer(index_meta->properties())) { - auto analyzed_terms = - InvertedIndexAnalyzer::get_analyse_result(value, index_meta->properties()); - term_infos.insert(analyzed_terms.begin(), analyzed_terms.end()); - } else { - term_infos.insert(TermInfo(value)); - } - } else if (category == ClauseTypeCategory::NON_TOKENIZED) { - if (clause_type == "TERM" && - InvertedIndexAnalyzer::should_analyzer(index_meta->properties())) { - auto analyzed_terms = - InvertedIndexAnalyzer::get_analyse_result(value, index_meta->properties()); - term_infos.insert(analyzed_terms.begin(), analyzed_terms.end()); - } else { - term_infos.insert(TermInfo(value)); - } - } - - std::string lucene_field_name = - build_field_name(index_meta->col_unique_ids()[0], column.suffix_path()); - std::wstring ws_field_name = StringHelper::to_wstring(lucene_field_name); - - auto iter = collect_infos->find(ws_field_name); - if (iter == collect_infos->end()) { - CollectInfo collect_info; - collect_info.term_infos = std::move(term_infos); - collect_info.index_meta = index_meta; - (*collect_infos)[ws_field_name] = std::move(collect_info); - } else { - iter->second.term_infos.insert(term_infos.begin(), term_infos.end()); - } - } - - return Status::OK(); -} - -bool SearchPredicateCollector::is_score_query_type(const std::string& clause_type) const { - return clause_type == "TERM" || clause_type == "EXACT" || clause_type == "PHRASE" || - clause_type == "MATCH" || clause_type == "ANY" || clause_type == "ALL"; -} - -SearchPredicateCollector::ClauseTypeCategory SearchPredicateCollector::get_clause_type_category( - const std::string& clause_type) const { - if (clause_type == "AND" || clause_type == "OR" || clause_type == "NOT" || - clause_type == "OCCUR_BOOLEAN") { - return ClauseTypeCategory::COMPOUND; - } else if (clause_type == "TERM" || clause_type == "EXACT") { - return ClauseTypeCategory::NON_TOKENIZED; - } else if (clause_type == "PHRASE" || clause_type == "MATCH" || clause_type == "ANY" || - clause_type == "ALL") { - return ClauseTypeCategory::TOKENIZED; - } else { - LOG(WARNING) << "Unknown clause type '" << clause_type - << "', defaulting to NON_TOKENIZED category"; - return ClauseTypeCategory::NON_TOKENIZED; - } -} - -} // namespace doris diff --git a/be/src/storage/rowset/beta_rowset.cpp b/be/src/storage/rowset/beta_rowset.cpp index 4419fc3c0d470e..db284a8d76cb76 100644 --- a/be/src/storage/rowset/beta_rowset.cpp +++ b/be/src/storage/rowset/beta_rowset.cpp @@ -337,7 +337,7 @@ Status BetaRowset::remove() { } } } else { - if (_schema->has_inverted_index() || _schema->has_ann_index()) { + if (_schema->has_inverted_or_ann_index()) { std::string inverted_index_file = InvertedIndexDescriptor::get_index_file_path_v2( InvertedIndexDescriptor::get_index_file_path_prefix(seg_path)); st = fs->delete_file(inverted_index_file); @@ -449,7 +449,7 @@ Status BetaRowset::link_files_to(const std::string& dir, RowsetId new_rowset_id, } } } else { - if ((_schema->has_inverted_index() || _schema->has_ann_index()) && + if ((_schema->has_inverted_or_ann_index()) && (without_index_uids == nullptr || without_index_uids->empty())) { std::string inverted_index_file_src = InvertedIndexDescriptor::get_index_file_path_v2( @@ -516,7 +516,7 @@ Status BetaRowset::copy_files_to(const std::string& dir, const RowsetId& new_row } } } else { - if (_schema->has_inverted_index() || _schema->has_ann_index()) { + if (_schema->has_inverted_or_ann_index()) { std::string inverted_index_src_file = InvertedIndexDescriptor::get_index_file_path_v2( InvertedIndexDescriptor::get_index_file_path_prefix(src_path)); @@ -574,7 +574,7 @@ Status BetaRowset::upload_to(const StorageResource& dest_fs, const RowsetId& new } } } else { - if (_schema->has_inverted_index() || _schema->has_ann_index()) { + if (_schema->has_inverted_or_ann_index()) { std::string remote_inverted_index_file = InvertedIndexDescriptor::get_index_file_path_v2( InvertedIndexDescriptor::get_index_file_path_prefix( @@ -730,7 +730,7 @@ Status BetaRowset::add_to_binlog() { linked_success_files.push_back(binlog_index_file); } } else { - if (_schema->has_inverted_index() || _schema->has_ann_index()) { + if (_schema->has_inverted_or_ann_index()) { auto index_file = InvertedIndexDescriptor::get_index_file_path_v2( InvertedIndexDescriptor::get_index_file_path_prefix(seg_file)); auto binlog_index_file = (std::filesystem::path(binlog_dir) / @@ -777,7 +777,7 @@ Status BetaRowset::calc_file_crc(uint32_t* crc_value, int64_t* file_count) { } } } else { - if (_schema->has_inverted_index() || _schema->has_ann_index()) { + if (_schema->has_inverted_or_ann_index()) { std::string inverted_index_file = InvertedIndexDescriptor::get_index_file_path_v2( InvertedIndexDescriptor::get_index_file_path_prefix(seg_path)); file_paths.emplace_back(std::move(inverted_index_file)); @@ -831,6 +831,9 @@ Status BetaRowset::show_nested_index_file(rapidjson::Value* rowset_value, case InvertedIndexStorageFormatPB::V3: format_str = "V3"; break; + case InvertedIndexStorageFormatPB::SNII: + format_str = "SNII"; + break; default: return Status::InternalError("inverted index storage format error"); break; @@ -840,6 +843,19 @@ Status BetaRowset::show_nested_index_file(rapidjson::Value* rowset_value, rowset_value->AddMember("index_storage_format", rapidjson::Value(format_str.c_str(), allocator), allocator); rapidjson::Value segments(rapidjson::kArrayType); + auto add_file_info_to_json = [&](const std::string& path, + rapidjson::Value& json_value) -> Status { + json_value.AddMember("idx_file_path", rapidjson::Value(path.c_str(), allocator), allocator); + int64_t idx_file_size = 0; + auto st = fs->file_size(path, &idx_file_size); + if (st != Status::OK()) { + LOG(WARNING) << "show nested index file get file size error, file: " << path + << ", error: " << st.msg(); + return st; + } + json_value.AddMember("idx_file_size", rapidjson::Value(idx_file_size).Move(), allocator); + return Status::OK(); + }; for (int seg_id = 0; seg_id < num_segments(); ++seg_id) { rapidjson::Value segment(rapidjson::kObjectType); segment.AddMember("segment_id", rapidjson::Value(seg_id).Move(), allocator); @@ -850,24 +866,20 @@ Status BetaRowset::show_nested_index_file(rapidjson::Value* rowset_value, fs, std::string(index_file_path_prefix), storage_format, InvertedIndexFileInfo(), _rowset_meta->tablet_id()); RETURN_IF_ERROR(index_file_reader->init()); + if (storage_format == InvertedIndexStorageFormatPB::SNII) { + rapidjson::Value index_file(rapidjson::kObjectType); + auto index_file_path = + InvertedIndexDescriptor::get_index_file_path_v2(index_file_path_prefix); + RETURN_IF_ERROR(add_file_info_to_json(index_file_path, index_file)); + segment.AddMember("index_files", rapidjson::Value(rapidjson::kArrayType).Move(), + allocator); + auto& index_files = segment["index_files"]; + index_files.PushBack(index_file, allocator); + segments.PushBack(segment, allocator); + continue; + } auto dirs = index_file_reader->get_all_directories(); - auto add_file_info_to_json = [&](const std::string& path, - rapidjson::Value& json_value) -> Status { - json_value.AddMember("idx_file_path", rapidjson::Value(path.c_str(), allocator), - allocator); - int64_t idx_file_size = 0; - auto st = fs->file_size(path, &idx_file_size); - if (st != Status::OK()) { - LOG(WARNING) << "show nested index file get file size error, file: " << path - << ", error: " << st.msg(); - return st; - } - json_value.AddMember("idx_file_size", rapidjson::Value(idx_file_size).Move(), - allocator); - return Status::OK(); - }; - auto process_files = [&allocator, &index_file_reader](auto& index_meta, rapidjson::Value& indices, rapidjson::Value& index) -> Status { diff --git a/be/src/storage/rowset/beta_rowset_reader.cpp b/be/src/storage/rowset/beta_rowset_reader.cpp index 43fe44fd7a1851..adb46a86965cca 100644 --- a/be/src/storage/rowset/beta_rowset_reader.cpp +++ b/be/src/storage/rowset/beta_rowset_reader.cpp @@ -285,6 +285,9 @@ Status BetaRowsetReader::get_segment_iterators(RowsetReaderContext* read_context _read_options.io_ctx.remote_scan_cache_write_limiter = query_ctx->remote_scan_cache_write_limiter(); } + _read_options.io_ctx.inverted_index_snii_read_no_write_file_cache = + _read_context->runtime_state->query_options() + .inverted_index_snii_read_no_write_file_cache; } if (_read_context->condition_cache_digest) { diff --git a/be/src/storage/rowset/beta_rowset_writer.cpp b/be/src/storage/rowset/beta_rowset_writer.cpp index 3fa097554f5dae..4ab3fb6f8504f2 100644 --- a/be/src/storage/rowset/beta_rowset_writer.cpp +++ b/be/src/storage/rowset/beta_rowset_writer.cpp @@ -714,8 +714,7 @@ Status BetaRowsetWriter::_rename_compacted_indices(int64_t begin, int64_t end, u if (_context.tablet_schema->get_inverted_index_storage_format() >= InvertedIndexStorageFormatPB::V2) { - if (_context.tablet_schema->has_inverted_index() || - _context.tablet_schema->has_ann_index()) { + if (_context.tablet_schema->has_inverted_or_ann_index()) { auto src_idx_path = InvertedIndexDescriptor::get_index_file_path_v2(src_index_path_prefix); auto dst_idx_path = @@ -1009,8 +1008,7 @@ Status BetaRowsetWriter::build(RowsetSharedPtr& rowset) { _rowset_meta->set_tablet_schema(_context.tablet_schema); // If segment compaction occurs, the idx file info will become inaccurate. - if ((_context.tablet_schema->has_inverted_index() || _context.tablet_schema->has_ann_index()) && - _num_segcompacted == 0) { + if ((_context.tablet_schema->has_inverted_or_ann_index()) && _num_segcompacted == 0) { if (auto idx_files_info = _idx_files.inverted_index_file_info(_segment_start_id); !idx_files_info.has_value()) [[unlikely]] { LOG(ERROR) << "expected inverted index files info, but none presents: " @@ -1186,7 +1184,7 @@ Status BetaRowsetWriter::create_segment_writer_for_segcompaction( RETURN_IF_ERROR(_create_file_writer(path, file_writer, FileType::SEGMENT_FILE)); IndexFileWriterPtr index_file_writer; - if (_context.tablet_schema->has_inverted_index() || _context.tablet_schema->has_ann_index()) { + if (_context.tablet_schema->has_inverted_or_ann_index()) { io::FileWriterPtr idx_file_writer; std::string prefix(InvertedIndexDescriptor::get_index_file_path_prefix(path)); if (_context.tablet_schema->get_inverted_index_storage_format() != diff --git a/be/src/storage/rowset/rowset_writer_context.h b/be/src/storage/rowset/rowset_writer_context.h index 10b537522d7521..b539f1dbf7dad9 100644 --- a/be/src/storage/rowset/rowset_writer_context.h +++ b/be/src/storage/rowset/rowset_writer_context.h @@ -99,6 +99,12 @@ struct RowsetWriterContext { bool enable_unique_key_merge_on_write = false; // store column_unique_id to do index compaction std::set columns_to_do_index_compaction; + // SNII only: (column_unique_id, index_id) pairs whose postings are produced + // by index compaction. The segment writer raw-builds every OTHER SNII index + // of the column, so one eligible and one new index on the same column can + // coexist in a single pass. V2/V3 keep columns_to_do_index_compaction: + // their per-column CLucene directories cannot split an index off a column. + std::set> snii_indexes_to_do_compaction; DataWriteType write_type = DataWriteType::TYPE_DEFAULT; // need to figure out the sub type of compaction ReaderType compaction_type = ReaderType::UNKNOWN; diff --git a/be/src/storage/rowset/segcompaction.cpp b/be/src/storage/rowset/segcompaction.cpp index 0606bae3246204..7e9910006d40e6 100644 --- a/be/src/storage/rowset/segcompaction.cpp +++ b/be/src/storage/rowset/segcompaction.cpp @@ -168,7 +168,7 @@ Status SegcompactionWorker::_delete_original_segments(uint32_t begin, uint32_t e // message when we encounter an error. RETURN_NOT_OK_STATUS_WITH_WARN(fs->delete_file(seg_path), absl::Substitute("Failed to delete file=$0", seg_path)); - if ((schema->has_inverted_index() || schema->has_ann_index()) && + if ((schema->has_inverted_or_ann_index()) && schema->get_inverted_index_storage_format() >= InvertedIndexStorageFormatPB::V2) { auto idx_path = InvertedIndexDescriptor::get_index_file_path_v2( InvertedIndexDescriptor::get_index_file_path_prefix(seg_path)); diff --git a/be/src/storage/rowset/segment_creator.cpp b/be/src/storage/rowset/segment_creator.cpp index 90078eca355789..3f2a84b138b4f6 100644 --- a/be/src/storage/rowset/segment_creator.cpp +++ b/be/src/storage/rowset/segment_creator.cpp @@ -166,7 +166,7 @@ Status SegmentFlusher::_create_segment_writer(std::unique_ptrcreate(segment_id, segment_file_writer)); IndexFileWriterPtr index_file_writer; - if (_context.tablet_schema->has_inverted_index() || _context.tablet_schema->has_ann_index()) { + if (_context.tablet_schema->has_inverted_or_ann_index()) { RETURN_IF_ERROR(_context.file_writer_creator->create(segment_id, &index_file_writer)); } @@ -191,7 +191,7 @@ Status SegmentFlusher::_create_segment_writer(std::unique_ptrhas_inverted_index() || _context.tablet_schema->has_ann_index()) { + if (_context.tablet_schema->has_inverted_or_ann_index()) { RETURN_IF_ERROR(_idx_files.add(segment_id, std::move(index_file_writer))); } auto s = writer->init(); @@ -210,7 +210,7 @@ Status SegmentFlusher::_create_segment_writer( RETURN_IF_ERROR(_context.file_writer_creator->create(segment_id, segment_file_writer)); IndexFileWriterPtr index_file_writer; - if (_context.tablet_schema->has_inverted_index() || _context.tablet_schema->has_ann_index()) { + if (_context.tablet_schema->has_inverted_or_ann_index()) { RETURN_IF_ERROR(_context.file_writer_creator->create(segment_id, &index_file_writer)); } @@ -227,7 +227,7 @@ Status SegmentFlusher::_create_segment_writer( segment_file_writer.get(), segment_id, _context.tablet_schema, _context.tablet, _context.data_dir, writer_options, index_file_writer.get()); RETURN_IF_ERROR(_seg_files.add(segment_id, std::move(segment_file_writer))); - if (_context.tablet_schema->has_inverted_index() || _context.tablet_schema->has_ann_index()) { + if (_context.tablet_schema->has_inverted_or_ann_index()) { RETURN_IF_ERROR(_idx_files.add(segment_id, std::move(index_file_writer))); } auto s = writer->init(); diff --git a/be/src/storage/rowset/vertical_beta_rowset_writer.cpp b/be/src/storage/rowset/vertical_beta_rowset_writer.cpp index 154c1e546e47f2..f56c646ab2dc1d 100644 --- a/be/src/storage/rowset/vertical_beta_rowset_writer.cpp +++ b/be/src/storage/rowset/vertical_beta_rowset_writer.cpp @@ -173,7 +173,7 @@ Status VerticalBetaRowsetWriter::_create_segment_writer( DCHECK(segment_file_writer != nullptr); IndexFileWriterPtr index_file_writer; - if (context.tablet_schema->has_inverted_index() || context.tablet_schema->has_ann_index()) { + if (context.tablet_schema->has_inverted_or_ann_index()) { RETURN_IF_ERROR(this->create_index_file_writer(seg_id, &index_file_writer)); } @@ -188,7 +188,7 @@ Status VerticalBetaRowsetWriter::_create_segment_writer( context.data_dir, writer_options, index_file_writer.get()); RETURN_IF_ERROR(this->_seg_files.add(seg_id, std::move(segment_file_writer))); - if (context.tablet_schema->has_inverted_index() || context.tablet_schema->has_ann_index()) { + if (context.tablet_schema->has_inverted_or_ann_index()) { RETURN_IF_ERROR(this->_idx_files.add(seg_id, std::move(index_file_writer))); } diff --git a/be/src/storage/segment/column_reader.cpp b/be/src/storage/segment/column_reader.cpp index db8f2d6819b04d..125209a19cfb33 100644 --- a/be/src/storage/segment/column_reader.cpp +++ b/be/src/storage/segment/column_reader.cpp @@ -56,6 +56,8 @@ #include "storage/index/index_reader.h" #include "storage/index/inverted/analyzer/analyzer.h" #include "storage/index/inverted/inverted_index_reader.h" +#include "storage/index/snii/snii_bkd_index_reader.h" +#include "storage/index/snii/snii_index_reader.h" #include "storage/index/zone_map/zone_map_index.h" #include "storage/iterators.h" #include "storage/olap_common.h" @@ -722,6 +724,23 @@ Status ColumnReader::_load_index(const std::shared_ptr& index_f } IndexReaderPtr index_reader; + if (index_file_reader->get_storage_format() == InvertedIndexStorageFormatPB::SNII) { + // Mirrors the writer-side split in IndexColumnWriter::create: text is + // served by the SPIMI reader, numerics by the SNII-native BKD. + if (is_string_type(type)) { + auto reader_type = should_analyzer ? InvertedIndexReaderType::FULLTEXT + : InvertedIndexReaderType::STRING_TYPE; + index_reader = + SniiIndexReader::create_shared(index_meta, index_file_reader, reader_type); + } else if (field_is_numeric_type(type)) { + index_reader = SniiBkdIndexReader::create_shared(index_meta, index_file_reader); + } else { + return Status::Error( + "SNII inverted index storage format does not support index type {}", type); + } + _index_readers[index_meta->index_id()] = index_reader; + return Status::OK(); + } if (is_string_type(type)) { if (should_analyzer) { diff --git a/be/src/storage/segment/column_writer.cpp b/be/src/storage/segment/column_writer.cpp index c6af6dafb8fbe4..622153dd1f8396 100644 --- a/be/src/storage/segment/column_writer.cpp +++ b/be/src/storage/segment/column_writer.cpp @@ -565,6 +565,12 @@ Status ScalarColumnWriter::init() { RETURN_IF_ERROR(IndexColumnWriter::create( get_column(), &_inverted_index_builders[i], _opts.index_file_writer, _opts.inverted_indexes[i])); + // After create() (which runs the writer's init()) and before any + // value lands: forward the write type UNCONDITIONALLY so SNII + // can select its PRX zstd level -- and the + // documented "forwarded to every created IndexColumnWriter" + // contract holds for both values. + _inverted_index_builders[i]->set_direct_load(_opts.is_direct_load); } } while (false); } @@ -1056,6 +1062,9 @@ Status ArrayColumnWriter::init() { RETURN_IF_ERROR(IndexColumnWriter::create(get_column(), &_inverted_index_writer, _opts.index_file_writer, _opts.inverted_indexes[0])); + // Same unconditional forwarding as the scalar path: after + // create()/init(), before any array value is added. + _inverted_index_writer->set_direct_load(_opts.is_direct_load); } } if (_opts.need_ann_index) { diff --git a/be/src/storage/segment/column_writer.h b/be/src/storage/segment/column_writer.h index 0b8c827367abcb..b2f7a982088a5b 100644 --- a/be/src/storage/segment/column_writer.h +++ b/be/src/storage/segment/column_writer.h @@ -86,6 +86,13 @@ struct ColumnWriterOptions { BloomFilterOptions bf_options; std::vector inverted_indexes; IndexFileWriter* index_file_writer = nullptr; + // The owning segment serves a direct load (stream/broker load, + // DataWriteType::TYPE_DIRECT) rather than compaction / schema change. Set + // once by the segment writer and propagated to variant subcolumn writers; + // forwarded to every created IndexColumnWriter via set_direct_load() so + // SNII can select its direct-load PRX zstd level without plumbing + // DataWriteType itself down here. + bool is_direct_load = false; SegmentFooterPB* footer = nullptr; io::FileWriter* file_writer = nullptr; diff --git a/be/src/storage/segment/count_on_index_fastpath.h b/be/src/storage/segment/count_on_index_fastpath.h new file mode 100644 index 00000000000000..30b0ccf7218157 --- /dev/null +++ b/be/src/storage/segment/count_on_index_fastpath.h @@ -0,0 +1,196 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +// G02 count-only fast-path caller guard (functional core, unit-testable +// without a SegmentIterator). +// +// COUNT_ON_INDEX counts the rows of THIS segment that match the pushed-down +// predicates MINUS deleted rows: SegmentIterator seeds _row_bitmap with +// [0, num_rows), intersects the index result bitmap into it, then subtracts +// the MOW delete bitmap and applies delete predicates / row ranges, and the +// scan emits |_row_bitmap| default-valued rows that the agg counts. The SNII +// fast path replaces the index result with a FABRICATED [0, df) bitmap whose +// cardinality is exact but whose row ids are not real. That is only equal in +// observable behavior when, for this segment iterator: +// 1. only the COUNT matters (COUNT_ON_INDEX agg pushdown), +// 2. the single pushed-down MATCH predicate is the ONLY filter (nothing else +// may intersect _row_bitmap before or after the index apply), +// 3. no rows are deleted (mirror of the V3 handling: _lazy_init subtracts +// the per-segment delete bitmap AFTER the index apply, and delete +// predicates filter later -- a fabricated id range cannot participate in +// either subtraction), +// 4. nothing consumes REAL row ids (rowid recording, ANN topn, BM25 +// scoring, virtual columns), and +// 5. rows are emitted as defaults without reading column data at the +// fabricated ids (the COUNT_ON_INDEX no-read-data contract must be +// active: enable_no_need_read_data_opt + DUP_KEYS or MOW). +// +// SegmentIterator fills the facts from its state right before applying the +// index and only sets IndexQueryContext::count_on_index_fastpath when this +// predicate holds. +namespace doris::segment_v2 { + +struct CountOnIndexFastpathFacts { + // (1) count-only context. + bool is_count_on_index_agg = false; + // (2) single MATCH predicate, no other conjuncts. + bool has_column_predicates = false; + size_t common_expr_count = 0; + bool single_expr_is_match_pred = false; + bool has_virtual_column_exprs = false; + // (3) deletes must be absent for this segment. + bool has_delete_predicates = false; + bool segment_delete_bitmap_empty = false; + // (2 cont.) nothing else may prune the row space around the index apply. + bool has_col_id_predicates = false; + bool has_topn_filters = false; + bool has_external_row_ranges = false; + bool row_bitmap_is_full = false; + // (4) consumers of real row ids. + bool record_rowids = false; + bool has_ann_topn = false; + bool has_score_runtime = false; + // (5) rows must be emitted as defaults (no data read at fabricated ids). + bool no_need_read_data_opt_enabled = false; + bool keys_type_supported = false; +}; + +inline bool count_on_index_fastpath_safe(const CountOnIndexFastpathFacts& f) { + return f.is_count_on_index_agg && !f.has_column_predicates && f.common_expr_count == 1 && + f.single_expr_is_match_pred && !f.has_virtual_column_exprs && !f.has_delete_predicates && + f.segment_delete_bitmap_empty && !f.has_col_id_predicates && !f.has_topn_filters && + !f.has_external_row_ranges && f.row_bitmap_is_full && !f.record_rowids && + !f.has_ann_topn && !f.has_score_runtime && f.no_need_read_data_opt_enabled && + f.keys_type_supported; +} + +// G03 count-emission shortcut guard (functional core, unit-testable without a +// SegmentIterator). +// +// After the G02 fast path answered the single MATCH predicate with a +// count-shaped bitmap, the only remaining work of the scan is to emit +// |_row_bitmap| default-valued rows batch by batch: the per-batch rowid +// iteration over the fabricated bitmap and the per-column no-read checks are +// pure overhead. The shortcut replaces them with a countdown that fills the +// block columns with defaults directly, in VStatisticsIterator-sized batches. +// +// That replacement is byte-for-byte equal to today's emission only when, at +// the end of _lazy_init: +// 1. the reader ACTUALLY answered from df (count_fastpath_hit) -- a mere +// guard pass with a row-accurate decode keeps today's path untouched, +// 2. no evaluation stage survives (vec/short-circuit/expr eval, leftover +// column predicates or common exprs, delete predicates, lazy +// materialization), +// 3. nothing consumes real row ids or per-row values (virtual columns, +// rowid recording), +// 4. batch accounting is a pure countdown (no read limit, no reverse +// key-ordered read, no condition-cache writes), and +// 5. the block is exactly the read schema and EVERY column would take a +// defaults fill in _read_columns_by_index (no real column read, no +// storage->schema cast, no version/lsn/tso rewrite) -- checked +// per-column by the iterator and summarized in one fact. +// +// Every fact is re-verified from live iterator state even though the G02 +// facts guard already implies most of them: the shortcut independently +// refuses on any drift, falling through to today's emission (which is always +// count-exact). +struct CountEmitShortcutFacts { + // (1) reader answered with a fabricated count bitmap. + bool count_fastpath_hit = false; + // (2) nothing evaluates or filters rows after the index apply. + bool needs_vec_eval = false; + bool needs_short_eval = false; + bool needs_expr_eval = false; + bool has_remaining_col_predicates = false; + bool has_remaining_common_exprs = false; + bool has_delete_predicates = false; + bool lazy_materialization_read = false; + // (3) consumers of real row ids / per-row values. + bool has_virtual_columns = false; + bool record_rowids = false; + // (4) batch accounting must be a pure countdown. + bool has_read_limit = false; + bool read_orderby_key_reverse = false; + bool has_condition_cache_digest = false; + // (5) emitted block == read schema, all columns defaults-fillable. + bool block_shape_matches_schema = false; + bool all_columns_emit_defaults = false; +}; + +inline bool count_emit_shortcut_safe(const CountEmitShortcutFacts& f) { + return f.count_fastpath_hit && !f.needs_vec_eval && !f.needs_short_eval && !f.needs_expr_eval && + !f.has_remaining_col_predicates && !f.has_remaining_common_exprs && + !f.has_delete_predicates && !f.lazy_materialization_read && !f.has_virtual_columns && + !f.record_rowids && !f.has_read_limit && !f.read_orderby_key_reverse && + !f.has_condition_cache_digest && f.block_shape_matches_schema && + f.all_columns_emit_defaults; +} + +} // namespace doris::segment_v2 + +// Deterministic seam for the G03 count-emission shortcut, mirroring the SNII +// query seam (storage/index/snii/query/internal/query_test_counters.h): +// - count_emit_shortcut_hits : engage decisions that ADMITTED the +// shortcut (incremented inside +// _should_engage_count_emit_shortcut, which +// _lazy_init calls once per iterator). Guard +// fall-throughs leave it unchanged. +// - count_emit_shortcut_batches : default-rows batches emitted by the +// shortcut (== ceil(count / 65535) per +// engaged iterator with a non-zero count). +// +// Active only under BE_TEST (library-wide define of doris_be_test); in a +// release build the macro expands to ((void)0): zero overhead, no global +// mutable state on the production path. The singleton is intentionally +// unsynchronized -- single-threaded test-only seam; reset between cases with +// `count_emit_test_counters() = {}`. +#if defined(BE_TEST) && !defined(SNII_COUNT_EMIT_TEST_COUNTERS) +#define SNII_COUNT_EMIT_TEST_COUNTERS +#endif + +#ifdef SNII_COUNT_EMIT_TEST_COUNTERS + +namespace doris::segment_v2::internal { + +struct CountEmitTestCounters { + uint64_t count_emit_shortcut_hits = 0; + uint64_t count_emit_shortcut_batches = 0; +}; + +// `inline` gives a single shared instance across all TUs that include this +// header, so counter increments made in segment_iterator.cpp are visible to +// the test that reads them. +inline CountEmitTestCounters& count_emit_test_counters() { + static CountEmitTestCounters counters; + return counters; +} + +} // namespace doris::segment_v2::internal + +#define SNII_COUNT_EMIT_COUNT(field) \ + (++::doris::segment_v2::internal::count_emit_test_counters().field) + +#else + +#define SNII_COUNT_EMIT_COUNT(field) ((void)0) + +#endif diff --git a/be/src/storage/segment/row_ranges.h b/be/src/storage/segment/row_ranges.h index 0f44f599695b8c..9d98f02e245b15 100644 --- a/be/src/storage/segment/row_ranges.h +++ b/be/src/storage/segment/row_ranges.h @@ -247,7 +247,7 @@ class RowRanges { size_t count() { return _count; } - bool is_empty() { return _count == 0; } + bool is_empty() const { return _count == 0; } bool contain(rowid_t from, rowid_t to) { // binary search diff --git a/be/src/storage/segment/segment_iterator.cpp b/be/src/storage/segment/segment_iterator.cpp index 14a5fab34c582a..a31059030b6a5d 100644 --- a/be/src/storage/segment/segment_iterator.cpp +++ b/be/src/storage/segment/segment_iterator.cpp @@ -104,6 +104,7 @@ #include "storage/segment/column_reader.h" #include "storage/segment/column_reader_cache.h" #include "storage/segment/condition_cache.h" +#include "storage/segment/count_on_index_fastpath.h" #include "storage/segment/row_ranges.h" #include "storage/segment/segment.h" #include "storage/segment/segment_prefetcher.h" @@ -614,6 +615,16 @@ Status SegmentIterator::_lazy_init(Block* block) { _init_segment_prefetchers(); + // G03: engage the count-emission shortcut. All inputs are final here (the + // index apply ran, _row_bitmap saw every subtraction/intersection above, + // _vec_init_lazy_materialization fixed the eval flags), so the post-apply + // cardinality IS the exact row count today's batch loop would emit; the + // shortcut only changes how fast those default rows are produced. + _count_emit_shortcut = _should_engage_count_emit_shortcut(block); + if (_count_emit_shortcut) { + _count_emit_rows_remaining = _row_bitmap.cardinality(); + } + return Status::OK(); } @@ -825,6 +836,18 @@ Status SegmentIterator::_get_row_ranges_by_column_conditions() { (has_index_in_iterators() || !_common_expr_ctxs_push_down.empty())) { SCOPED_RAW_TIMER(&_opts.stats->inverted_index_filter_timer); size_t input_rows = _row_bitmap.cardinality(); + // G02 count-only fast path handshake: only while the single + // pushed-down MATCH predicate of a provably filter-free + // COUNT_ON_INDEX scan is evaluated may a reader answer with a + // count-shaped bitmap (see count_on_index_fastpath.h). The reply + // direction (did the reader actually fabricate one?) is captured + // into _count_fastpath_hit and both flags are reset on every exit + // path so no later read_from_index call can observe or forge them. + if (_index_query_context != nullptr) { + _index_query_context->count_on_index_fastpath = _count_on_index_fastpath_safe(); + _index_query_context->count_on_index_fastpath_hit = false; + } + DEFER({ _capture_count_fastpath_hit(); }); // Only apply column-level inverted index if we have iterators if (has_index_in_iterators()) { RETURN_IF_ERROR(_apply_inverted_index()); @@ -1337,6 +1360,146 @@ Status SegmentIterator::_apply_index_expr() { return Status::OK(); } +bool SegmentIterator::_count_on_index_fastpath_safe() const { + CountOnIndexFastpathFacts facts; + facts.is_count_on_index_agg = _opts.push_down_agg_type_opt == TPushAggOp::COUNT_ON_INDEX; + facts.has_column_predicates = !_col_predicates.empty(); + facts.common_expr_count = _common_expr_ctxs_push_down.size(); + facts.single_expr_is_match_pred = + _common_expr_ctxs_push_down.size() == 1 && + _common_expr_ctxs_push_down.front()->root() != nullptr && + _common_expr_ctxs_push_down.front()->root()->node_type() == TExprNodeType::MATCH_PRED; + facts.has_virtual_column_exprs = !_virtual_column_exprs.empty(); + facts.has_delete_predicates = _opts.delete_condition_predicates != nullptr && + _opts.delete_condition_predicates->num_of_column_predicate() > 0; + // Mirror of the _lazy_init delete-bitmap subtraction: the fast path is only + // sound when there is nothing to subtract for THIS segment. + const auto delete_bitmap_it = _opts.delete_bitmap.find(segment_id()); + facts.segment_delete_bitmap_empty = delete_bitmap_it == _opts.delete_bitmap.end() || + delete_bitmap_it->second == nullptr || + delete_bitmap_it->second->isEmpty(); + facts.has_col_id_predicates = !_opts.col_id_to_predicates.empty(); + facts.has_topn_filters = !_opts.topn_filter_source_node_ids.empty(); + facts.has_external_row_ranges = !_opts.row_ranges.is_empty(); + // Catches every earlier pruning source in one check (condition cache, key + // ranges): the fabricated [0, df) range only counts correctly against a + // full [0, num_rows) bitmap. + facts.row_bitmap_is_full = _row_bitmap.cardinality() == uint64_t(num_rows()); + facts.record_rowids = _opts.record_rowids; + facts.has_ann_topn = _opts.ann_topn_runtime != nullptr; + facts.has_score_runtime = _score_runtime != nullptr; + // Mirror of the _need_read_data preamble: rows must be emitted as defaults, + // never materialized from the fabricated row ids. + facts.no_need_read_data_opt_enabled = + _opts.runtime_state == nullptr || + _opts.runtime_state->query_options().enable_no_need_read_data_opt; + facts.keys_type_supported = _opts.tablet_schema->keys_type() == KeysType::DUP_KEYS || + (_opts.tablet_schema->keys_type() == KeysType::UNIQUE_KEYS && + _opts.enable_unique_key_merge_on_write); + return count_on_index_fastpath_safe(facts); +} + +void SegmentIterator::_capture_count_fastpath_hit() { + if (_index_query_context == nullptr) { + return; + } + _count_fastpath_hit = _index_query_context->count_on_index_fastpath_hit; + _index_query_context->count_on_index_fastpath = false; + _index_query_context->count_on_index_fastpath_hit = false; +} + +bool SegmentIterator::_column_emits_defaults_for_count(ColumnId cid) { + // Mirror of the per-batch fill in _read_columns_by_index: a column's batch + // content is reproducible by the emission shortcut iff the column takes + // the _no_need_read_key_data defaults fill or the _prune_column defaults + // fill. Anything else -- a real column read, a storage->schema cast in + // _init_current_block/_convert_to_expected_type (whose CAST(default) need + // not equal the schema-type default), or a version/lsn/tso rewrite source + // (those columns are never no-read, so the type/read checks below already + // veto them) -- must keep today's path. + if (_is_pred_column[cid]) { + return false; + } + const auto* column_desc = _schema->column(cid); + if (column_desc == nullptr) { + return false; + } + const auto& file_column_type = _storage_name_and_type[cid].second; + DataTypePtr expected_type = Schema::get_data_type_ptr(*column_desc); + if (file_column_type == nullptr || expected_type == nullptr || + !file_column_type->equals(*expected_type)) { + return false; + } + return _no_need_read_key_data_eligible(cid) || !_need_read_data(cid); +} + +bool SegmentIterator::_should_engage_count_emit_shortcut(const Block* block) { + if (!_count_fastpath_hit) { + return false; + } + CountEmitShortcutFacts facts; + facts.count_fastpath_hit = _count_fastpath_hit; + facts.needs_vec_eval = _is_need_vec_eval; + facts.needs_short_eval = _is_need_short_eval; + facts.needs_expr_eval = _is_need_expr_eval; + facts.has_remaining_col_predicates = !_col_predicates.empty(); + facts.has_remaining_common_exprs = !_common_expr_ctxs_push_down.empty(); + facts.has_delete_predicates = _opts.delete_condition_predicates != nullptr && + _opts.delete_condition_predicates->num_of_column_predicate() > 0; + facts.lazy_materialization_read = _lazy_materialization_read; + facts.has_virtual_columns = !_virtual_column_exprs.empty(); + facts.record_rowids = _opts.record_rowids || _record_rowids; + facts.has_read_limit = _opts.read_limit > 0; + facts.read_orderby_key_reverse = _opts.read_orderby_key_reverse; + facts.has_condition_cache_digest = _opts.condition_cache_digest != 0; + facts.block_shape_matches_schema = block->columns() == _schema->num_column_ids(); + facts.all_columns_emit_defaults = true; + for (size_t i = 0; i < _schema->num_column_ids() && facts.all_columns_emit_defaults; ++i) { + facts.all_columns_emit_defaults = _column_emits_defaults_for_count(_schema->column_id(i)); + } + const bool engage = count_emit_shortcut_safe(facts); + if (engage) { + SNII_COUNT_EMIT_COUNT(count_emit_shortcut_hits); + } + return engage; +} + +Status SegmentIterator::_emit_count_shortcut_batch(Block* block) { + if (_count_emit_rows_remaining == 0) { + // EOF twin of _process_eof: shortcut batches never move the block's + // columns into _current_return_columns, so there is nothing to + // restore; deliver the empty block and release iterator memory the + // same way. + block->clear_column_data(); + _column_iterators.clear(); + _index_iterators.clear(); + return Status::EndOfFile("no more data in segment"); + } + const auto rows = static_cast( + std::min(_count_emit_rows_remaining, kCountEmitBatchRows)); + block->clear_column_data(_schema->num_column_ids()); + for (size_t i = 0; i < _schema->num_column_ids(); ++i) { + MutableColumnPtr column = std::move(*block->get_by_position(i).column).mutate(); + // Same fill as the defaults branches of _read_columns_by_index + // (_no_need_read_key_data / _prune_column): NOT-NULL defaults. + // ColumnNullable::insert_many_defaults would insert NULLs and break + // count(col) parity. + if (is_column_nullable(*column)) { + auto* nullable_col_ptr = reinterpret_cast(column.get()); + nullable_col_ptr->get_null_map_column().insert_many_defaults(rows); + nullable_col_ptr->get_nested_column_ptr()->insert_many_defaults(rows); + } else { + column->insert_many_defaults(rows); + } + block->replace_by_position(i, std::move(column)); + } + _count_emit_rows_remaining -= rows; + _opts.stats->blocks_load += 1; + _opts.stats->raw_rows_read += rows; + SNII_COUNT_EMIT_COUNT(count_emit_shortcut_batches); + return Status::OK(); +} + bool SegmentIterator::_downgrade_without_index(Status res, bool need_remaining) { bool is_fallback = _opts.runtime_state->query_options().enable_fallback_on_missing_inverted_index; @@ -2889,6 +3052,14 @@ Status SegmentIterator::_next_batch_internal(Block* block) { SCOPED_RAW_TIMER(&_opts.stats->block_load_ns); + // G03: a count-fastpath scan emits the remaining count as default rows in + // statistics-sized batches; the row-bitmap iterator and the per-rowid + // machinery below are never touched. Engaged only when read_limit == 0, + // so the limit check below stays unreachable for shortcut scans. + if (_count_emit_shortcut) { + return _emit_count_shortcut_batch(block); + } + if (_opts.read_limit > 0 && _rows_returned >= _opts.read_limit) { return _process_eof(block); } @@ -3490,8 +3661,7 @@ void SegmentIterator::_calculate_common_expr_index_exec_status() { } } -bool SegmentIterator::_no_need_read_key_data(ColumnId cid, MutableColumnPtr& column, - size_t nrows_read) { +bool SegmentIterator::_no_need_read_key_data_eligible(ColumnId cid) { if (_opts.runtime_state && !_opts.runtime_state->query_options().enable_no_need_read_data_opt) { return false; } @@ -3518,6 +3688,14 @@ bool SegmentIterator::_no_need_read_key_data(ColumnId cid, MutableColumnPtr& col return false; } + return true; +} + +bool SegmentIterator::_no_need_read_key_data(ColumnId cid, MutableColumnPtr& column, + size_t nrows_read) { + if (!_no_need_read_key_data_eligible(cid)) { + return false; + } insert_many_not_null_defaults(column, nrows_read); return true; } diff --git a/be/src/storage/segment/segment_iterator.h b/be/src/storage/segment/segment_iterator.h index b67361d53ffa7a..61ed0afccfd428 100644 --- a/be/src/storage/segment/segment_iterator.h +++ b/be/src/storage/segment/segment_iterator.h @@ -198,6 +198,32 @@ class SegmentIterator : public RowwiseIterator { bool* continue_apply); [[nodiscard]] Status _apply_ann_topn_predicate(); [[nodiscard]] Status _apply_index_expr(); + // G02: true iff answering the single pushed-down MATCH predicate by its + // match COUNT alone is indistinguishable from the row-accurate bitmap for + // this COUNT_ON_INDEX scan (no deletes, no other filters, full row bitmap, + // no row-id consumers). Gates IndexQueryContext::count_on_index_fastpath; + // the decision predicate itself lives in count_on_index_fastpath.h. + bool _count_on_index_fastpath_safe() const; + // G03: teardown of the G02 handshake. Captures whether the reader answered + // with a fabricated count bitmap into _count_fastpath_hit and clears both + // context flags so no later read_from_index call can observe or forge + // them. Runs on every exit of the index-apply scope. + void _capture_count_fastpath_hit(); + // G03: true iff the per-batch defaults fill of _read_columns_by_index + // would apply to `cid` (the _no_need_read_key_data or _prune_column + // branch) AND the block column needs no storage->schema cast, i.e. the + // emission shortcut can reproduce the column's batch content exactly. + bool _column_emits_defaults_for_count(ColumnId cid); + // G03: fills CountEmitShortcutFacts from live iterator state at the end of + // _lazy_init and returns the pure-guard verdict; the decision predicate + // itself lives in count_on_index_fastpath.h. + bool _should_engage_count_emit_shortcut(const Block* block); + // G03: one emission-shortcut batch: min(remaining, kCountEmitBatchRows) + // default rows filled straight into the block (NOT-NULL defaults for + // nullable columns, mirroring _prune_column), then EOF once the countdown + // reaches zero. Replaces the whole per-rowid _next_batch_internal body for + // engaged scans. + Status _emit_count_shortcut_batch(Block* block); bool _column_has_fulltext_index(int32_t cid); bool _column_has_ann_index(int32_t cid); @@ -314,6 +340,10 @@ class SegmentIterator : public RowwiseIterator { Status _convert_to_expected_type(const std::vector& col_ids); bool _no_need_read_key_data(ColumnId cid, MutableColumnPtr& column, size_t nrows_read); + // Side-effect-free eligibility half of _no_need_read_key_data (no column + // fill); shared by the per-batch fill and the G03 engage-time per-column + // proof so the two can never drift. + bool _no_need_read_key_data_eligible(ColumnId cid); bool _has_delete_predicate(ColumnId cid); bool _can_skip_reading_extra_column(ColumnId cid); @@ -477,6 +507,23 @@ class SegmentIterator : public RowwiseIterator { IndexQueryContextPtr _index_query_context; + // G03 count-emission shortcut state (see count_on_index_fastpath.h). + // _count_fastpath_hit: the reader answered the single MATCH predicate with + // a fabricated count bitmap (captured from the G02 handshake reply). + // _count_emit_shortcut: engaged at the end of _lazy_init when + // count_emit_shortcut_safe holds; every subsequent batch is emitted by + // _emit_count_shortcut_batch from _count_emit_rows_remaining (initialized + // to the post-apply _row_bitmap cardinality) without touching the row + // bitmap iterator. + bool _count_fastpath_hit = false; + bool _count_emit_shortcut = false; + uint64_t _count_emit_rows_remaining = 0; + // Batch size for shortcut emission: VStatisticsIterator's + // MAX_ROW_SIZE_IN_COUNT, the largest default-rows block shape already + // proven through every consumer above the segment iterator by the plain + // COUNT pushdown (rowset reader, collect iterator, block reader, scanner). + static constexpr uint64_t kCountEmitBatchRows = 65535; + // key is column uid, value is the sparse column cache std::unordered_map _variant_sparse_column_cache; diff --git a/be/src/storage/segment/segment_writer.cpp b/be/src/storage/segment/segment_writer.cpp index 0f780986f1a119..2911a678194dde 100644 --- a/be/src/storage/segment/segment_writer.cpp +++ b/be/src/storage/segment/segment_writer.cpp @@ -193,9 +193,21 @@ Status SegmentWriter::_create_column_writer(uint32_t cid, const TabletColumn& co if (_opts.write_type == DataWriteType::TYPE_DIRECT && schema->skip_write_index_on_load()) { skip_inverted_index = true; } + // Let SNII select the direct-load PRX zstd level. + opts.is_direct_load = _opts.write_type == DataWriteType::TYPE_DIRECT; // indexes for this column if (!skip_inverted_index) { auto inverted_indexs = schema->inverted_indexs(column); + // SNII splits index compaction per (column, index): indexes in the set + // are produced by the postings merge, every sibling on the column still + // raw-builds here. V2/V3 skip whole columns above instead. + if (_opts.rowset_ctx != nullptr && + !_opts.rowset_ctx->snii_indexes_to_do_compaction.empty()) { + std::erase_if(inverted_indexs, [&](const TabletIndex* index_meta) { + return _opts.rowset_ctx->snii_indexes_to_do_compaction.contains( + {column.unique_id(), index_meta->index_id()}); + }); + } if (!inverted_indexs.empty()) { opts.inverted_indexes = inverted_indexs; opts.need_inverted_index = true; diff --git a/be/src/storage/segment/variant/variant_column_writer_impl.cpp b/be/src/storage/segment/variant/variant_column_writer_impl.cpp index 408bd9b4cd427c..f0ff9bffee27e9 100644 --- a/be/src/storage/segment/variant/variant_column_writer_impl.cpp +++ b/be/src/storage/segment/variant/variant_column_writer_impl.cpp @@ -380,6 +380,8 @@ Status prepare_materialized_subcolumn_writer( opts.rowset_ctx = base_opts.rowset_ctx; opts.file_writer = base_opts.file_writer; opts.storage_format = base_opts.storage_format; + // keep the segment writer's direct-load marking for subcolumn index writers + opts.is_direct_load = base_opts.is_direct_load; std::unique_ptr writer; variant_util::inherit_column_attributes(parent_column, tablet_column); diff --git a/be/src/storage/segment/vertical_segment_writer.cpp b/be/src/storage/segment/vertical_segment_writer.cpp index 5676e6b248bebb..c1a33f63f321fa 100644 --- a/be/src/storage/segment/vertical_segment_writer.cpp +++ b/be/src/storage/segment/vertical_segment_writer.cpp @@ -204,8 +204,20 @@ Status VerticalSegmentWriter::_create_column_writer(uint32_t cid, const TabletCo tablet_schema->skip_write_index_on_load()) { skip_inverted_index = true; } + // Let SNII select the direct-load PRX zstd level. + opts.is_direct_load = _opts.write_type == DataWriteType::TYPE_DIRECT; if (!skip_inverted_index) { auto inverted_indexs = tablet_schema->inverted_indexs(column); + // SNII splits index compaction per (column, index): indexes in the set + // are produced by the postings merge, every sibling on the column still + // raw-builds here. V2/V3 skip whole columns above instead. + if (_opts.rowset_ctx != nullptr && + !_opts.rowset_ctx->snii_indexes_to_do_compaction.empty()) { + std::erase_if(inverted_indexs, [&](const TabletIndex* index_meta) { + return _opts.rowset_ctx->snii_indexes_to_do_compaction.contains( + {column.unique_id(), index_meta->index_id()}); + }); + } if (!inverted_indexs.empty()) { opts.inverted_indexes = inverted_indexs; opts.need_inverted_index = true; diff --git a/be/src/storage/snapshot/snapshot_manager.cpp b/be/src/storage/snapshot/snapshot_manager.cpp index 7ccacb21d506ef..75c8867c4dfeea 100644 --- a/be/src/storage/snapshot/snapshot_manager.cpp +++ b/be/src/storage/snapshot/snapshot_manager.cpp @@ -816,7 +816,7 @@ Status SnapshotManager::_create_snapshot_files(const TabletSharedPtr& ref_tablet linked_success_files.push_back(snapshot_segment_index_file_path); } } else { - if (tablet_schema.has_inverted_index() || tablet_schema.has_ann_index()) { + if (tablet_schema.has_inverted_or_ann_index()) { auto index_file = InvertedIndexDescriptor::get_index_file_path_v2( InvertedIndexDescriptor::get_index_file_path_prefix( segment_file_path)); diff --git a/be/src/storage/tablet/tablet_meta.cpp b/be/src/storage/tablet/tablet_meta.cpp index 4fd34f1122ee7a..dfde6238ff0693 100644 --- a/be/src/storage/tablet/tablet_meta.cpp +++ b/be/src/storage/tablet/tablet_meta.cpp @@ -467,6 +467,9 @@ void TabletMeta::init_schema_from_thrift(const TTabletSchema& tablet_schema, case TInvertedIndexFileStorageFormat::V3: tablet_schema_pb->set_inverted_index_storage_format(InvertedIndexStorageFormatPB::V3); break; + case TInvertedIndexFileStorageFormat::SNII: + tablet_schema_pb->set_inverted_index_storage_format(InvertedIndexStorageFormatPB::SNII); + break; default: tablet_schema_pb->set_inverted_index_storage_format(InvertedIndexStorageFormatPB::V3); break; diff --git a/be/src/storage/tablet/tablet_schema.h b/be/src/storage/tablet/tablet_schema.h index 7b784d0b615201..89ecad433dedda 100644 --- a/be/src/storage/tablet/tablet_schema.h +++ b/be/src/storage/tablet/tablet_schema.h @@ -553,6 +553,31 @@ class TabletSchema : public MetadataAdder { } return inverted_indexes; } + // True when anything at all lives in this schema's inverted index FILE. + // + // Spelled out as `has_inverted_index() || has_ann_index()` at ~27 call sites + // before this existed -- rowset writers, segment creation, segcompaction, + // snapshot and migration, and most of the cloud paths. Every one of them is + // asking the same question ("is there an index file to carry, warm, link or + // rewrite?"), and each open-coded copy is a place a third index type would + // have to be remembered. + bool has_inverted_or_ann_index() const { return has_inverted_index() || has_ann_index(); } + + // Both index types that live in the inverted index FILE, in schema order. + // Every storage format stores them together: V1/V2/V3 as CLucene directories, + // SNII as text metadata groups plus an ANN blob logical index. A rewrite that + // enumerated only the inverted ones would seal a file missing every ANN index + // while the schema still claimed them. + const std::vector inverted_and_ann_indexes() const { + std::vector indexes; + for (const auto& index : _indexes) { + if (index->index_type() == IndexType::INVERTED || + index->index_type() == IndexType::ANN) { + indexes.emplace_back(index.get()); + } + } + return indexes; + } bool has_inverted_index() const { for (const auto& index : _indexes) { DBUG_EXECUTE_IF("tablet_schema::has_inverted_index", { diff --git a/be/src/storage/task/engine_storage_migration_task.cpp b/be/src/storage/task/engine_storage_migration_task.cpp index a2abd8b256cb72..38724d8dfdc2b5 100644 --- a/be/src/storage/task/engine_storage_migration_task.cpp +++ b/be/src/storage/task/engine_storage_migration_task.cpp @@ -428,7 +428,7 @@ Status EngineStorageMigrationTask::_copy_index_and_data_files( return status; } } - } else if (tablet_schema.has_inverted_index() || tablet_schema.has_ann_index()) { + } else if (tablet_schema.has_inverted_or_ann_index()) { auto index_file = InvertedIndexDescriptor::get_index_file_path_v2( InvertedIndexDescriptor::get_index_file_path_prefix(segment_file_path)); auto snapshot_segment_index_file_path = diff --git a/be/src/storage/task/index_builder.cpp b/be/src/storage/task/index_builder.cpp index f85fcee9ddf44e..aaab4783b37729 100644 --- a/be/src/storage/task/index_builder.cpp +++ b/be/src/storage/task/index_builder.cpp @@ -19,6 +19,7 @@ #include +#include "common/cast_set.h" #include "common/logging.h" #include "common/status.h" #include "storage/index/index_file_reader.h" @@ -61,6 +62,74 @@ Status IndexBuilder::init() { return Status::OK(); } +Status IndexBuilder::plan_snii_index_rewrite( + const TabletSchema& input_schema, const TabletSchema& output_schema, + const std::set& alter_index_ids, + const std::function& container_has, + bool source_container_has_blob, SniiIndexRewritePlan* plan) { + DORIS_CHECK(plan != nullptr); + plan->inherit_keys.clear(); + plan->build_columns.clear(); + // Keyed and ordered by column unique id, so one raw column read feeds every + // index on that column and the output layout is deterministic. + std::map> build_by_column; + std::set> seen_keys; + for (const TabletIndex* index : output_schema.inverted_and_ann_indexes()) { + const auto key = + std::make_pair(cast_set(index->index_id()), index->get_index_suffix()); + // The target schema holds each logical index exactly once; the final + // directory could not hold a duplicate key anyway. + DORIS_CHECK(seen_keys.insert(key).second); + bool in_container = false; + RETURN_IF_ERROR(container_has(*index, &in_container)); + const TabletIndex* input_index = nullptr; + for (const TabletIndex* candidate : input_schema.inverted_and_ann_indexes()) { + if (candidate->index_id() == index->index_id() && + candidate->get_index_suffix() == index->get_index_suffix()) { + input_index = candidate; + break; + } + } + const bool definition_unchanged = + input_index != nullptr && input_index->properties() == index->properties(); + // Inheritance is ONE snapshot of the source container per segment, and a + // container holding a blob logical index cannot be snapshotted at all: + // SniiSegmentReader rejects it by scanning every directory entry, not by + // what the snapshot was asked to keep. So leaving even one key here would + // fail the whole rewrite -- when the source holds a blob, everything is + // rebuilt instead, which costs a raw column read but is always correct. + if (in_container && definition_unchanged && !source_container_has_blob) { + plan->inherit_keys.push_back({.index_id = key.first, .index_suffix = key.second}); + continue; + } + if (!in_container && !alter_index_ids.contains(index->index_id())) { + // Not requested and nothing to inherit: the index stays absent for + // this rowset, exactly as the V2 path leaves it. + LOG(INFO) << "SNII index " << index->index_id() + << " is absent from the source container and was not requested; " + "it stays absent from the rewritten rowset"; + continue; + } + // Requested and buildable, or present under the same key with a CHANGED + // definition: the final directory must match the target schema, so the + // old metadata is dropped and the index is rebuilt from the raw column. + // + // Only THIS branch needs a column to read: inheriting copies raw bytes by + // key, and the "stays absent" branch above reads nothing. So an index + // that binds no column fails the rewrite only when it actually has to be + // rebuilt -- a malformed index elsewhere in the schema, neither requested + // nor present, must not block building the ones that are fine. + if (index->col_unique_ids().empty()) { + return Status::Error( + "SNII rewrite: index {} must be rebuilt but binds no column unique id", + index->index_id()); + } + build_by_column[index->col_unique_ids()[0]].push_back(index); + } + plan->build_columns.assign(build_by_column.begin(), build_by_column.end()); + return Status::OK(); +} + Status IndexBuilder::update_inverted_index_info() { // just do link files LOG(INFO) << "begin to update_inverted_index_info, tablet=" << _tablet->tablet_id() @@ -83,15 +152,20 @@ Status IndexBuilder::update_inverted_index_info() { TabletSchemaSPtr output_rs_tablet_schema = std::make_shared(); const auto& input_rs_tablet_schema = input_rowset->tablet_schema(); output_rs_tablet_schema->copy_from(*input_rs_tablet_schema); + const bool is_snii_drop = + _is_drop_op && input_rs_tablet_schema->get_inverted_index_storage_format() == + InvertedIndexStorageFormatPB::SNII; int64_t total_index_size = 0; - auto* beta_rowset = reinterpret_cast(input_rowset.get()); - auto size_st = beta_rowset->get_inverted_index_size(&total_index_size); - DBUG_EXECUTE_IF("IndexBuilder::update_inverted_index_info_size_st_not_ok", { - size_st = Status::Error("debug point: get fs failed"); - }) - if (!size_st.ok() && !size_st.is() && - !size_st.is()) { - return size_st; + if (!is_snii_drop) { + auto* beta_rowset = reinterpret_cast(input_rowset.get()); + auto size_st = beta_rowset->get_inverted_index_size(&total_index_size); + DBUG_EXECUTE_IF("IndexBuilder::update_inverted_index_info_size_st_not_ok", { + size_st = Status::Error("debug point: get fs failed"); + }) + if (!size_st.ok() && !size_st.is() && + !size_st.is()) { + return size_st; + } } auto num_segments = input_rowset->num_segments(); size_t drop_index_size = 0; @@ -240,6 +314,14 @@ Status IndexBuilder::update_inverted_index_info() { context.newest_write_timestamp = input_rs_reader->newest_write_timestamp(); auto output_rs_writer = DORIS_TRY(_tablet->create_rowset_writer(context, false)); _pending_rs_guards.push_back(_engine.add_pending_rowset(context)); + if (!_is_drop_op && output_rs_tablet_schema->get_inverted_index_storage_format() == + InvertedIndexStorageFormatPB::SNII) { + // The rewrite plan compares index definitions between the input and + // output schema (inherit vs rebuild); keep the input schema reachable + // from the output rowset id. + _input_rowset_schemas.emplace(output_rs_writer->rowset_id().to_string(), + input_rs_tablet_schema); + } // if without_index_uids is not empty, copy _alter_index_ids to it // else just use _alter_index_ids to avoid copy @@ -247,10 +329,21 @@ Status IndexBuilder::update_inverted_index_info() { without_index_uids.insert(_alter_index_ids.begin(), _alter_index_ids.end()); } + const bool preserve_snii_container = + is_snii_drop && (output_rs_tablet_schema->has_inverted_or_ann_index()); + std::set* excluded_index_ids = + without_index_uids.empty() ? &_alter_index_ids : &without_index_uids; + if (preserve_snii_container) { + // SNII logical indexes share one immutable container. Preserve that container in O(1) + // while the output schema hides the dropped logical index; compaction reclaims its + // physical bytes later. When no logical index survives, keep the exclusion set so no + // container is linked. + excluded_index_ids = nullptr; + } + // build output rowset RETURN_IF_ERROR(input_rowset->link_files_to( - _tablet->tablet_path(), output_rs_writer->rowset_id(), 0, - without_index_uids.empty() ? &_alter_index_ids : &without_index_uids)); + _tablet->tablet_path(), output_rs_writer->rowset_id(), 0, excluded_index_ids)); auto input_rowset_meta = input_rowset->rowset_meta(); RowsetMetaSharedPtr rowset_meta = std::make_shared(); @@ -272,6 +365,13 @@ Status IndexBuilder::update_inverted_index_info() { rowset_meta->set_data_disk_size(input_rowset_meta->data_disk_size()); rowset_meta->set_index_disk_size(input_rowset_meta->index_disk_size()); } + } else if (is_snii_drop) { + rowset_meta->set_total_disk_size(preserve_snii_container + ? input_rowset_meta->total_disk_size() + : input_rowset_meta->data_disk_size()); + rowset_meta->set_data_disk_size(input_rowset_meta->data_disk_size()); + rowset_meta->set_index_disk_size( + preserve_snii_container ? input_rowset_meta->index_disk_size() : 0); } else { for (int seg_id = 0; seg_id < num_segments; seg_id++) { auto seg_path = DORIS_TRY(input_rowset->segment_path(seg_id)); @@ -339,6 +439,13 @@ Status IndexBuilder::handle_single_rowset(RowsetMetaSharedPtr output_rowset_meta if (_is_drop_op) { const auto& output_rs_tablet_schema = output_rowset_meta->tablet_schema(); + if (output_rs_tablet_schema->get_inverted_index_storage_format() == + InvertedIndexStorageFormatPB::SNII) { + LOG(INFO) << "skip physical SNII inverted index rewrite for drop index. tablet_id=" + << _tablet->tablet_id() + << " rowset_id=" << output_rowset_meta->rowset_id().to_string(); + return Status::OK(); + } if (output_rs_tablet_schema->get_inverted_index_storage_format() != InvertedIndexStorageFormatPB::V1) { const auto& fs = output_rowset_meta->fs(); @@ -411,6 +518,10 @@ Status IndexBuilder::handle_single_rowset(RowsetMetaSharedPtr output_rowset_meta // create inverted or ann index writer const auto& fs = output_rowset_meta->fs(); auto output_rowset_schema = output_rowset_meta->tablet_schema(); + if (output_rowset_schema->get_inverted_index_storage_format() == + InvertedIndexStorageFormatPB::SNII) { + return _handle_single_rowset_snii(output_rowset_meta, segments); + } size_t inverted_index_size = 0; for (auto& seg_ptr : segments) { std::string index_path_prefix { @@ -517,8 +628,10 @@ Status IndexBuilder::handle_single_rowset(RowsetMetaSharedPtr output_rowset_meta if (inverted_index_builder) { auto writer_sign = std::make_pair(seg_ptr->id(), index_id); - _index_column_writers.insert( + auto [index_column_writer_it, inserted] = _index_column_writers.insert( std::make_pair(writer_sign, std::move(inverted_index_builder))); + DORIS_CHECK(inserted); + DORIS_CHECK(index_column_writer_it->second != nullptr); inverted_index_writer_signs.emplace_back(writer_sign); } } @@ -546,8 +659,10 @@ Status IndexBuilder::handle_single_rowset(RowsetMetaSharedPtr output_rowset_meta if (index_writer) { auto writer_sign = std::make_pair(seg_ptr->id(), index_id); - _index_column_writers.insert( + auto [index_column_writer_it, inserted] = _index_column_writers.insert( std::make_pair(writer_sign, std::move(index_writer))); + DORIS_CHECK(inserted); + DORIS_CHECK(index_column_writer_it->second != nullptr); inverted_index_writer_signs.emplace_back(writer_sign); } } @@ -555,7 +670,10 @@ Status IndexBuilder::handle_single_rowset(RowsetMetaSharedPtr output_rowset_meta } // DO NOT forget index_file_writer for the segment, otherwise, original inverted index will be deleted. - _index_file_writers.emplace(seg_ptr->id(), std::move(index_file_writer)); + auto [index_file_writer_it, inserted] = + _index_file_writers.emplace(seg_ptr->id(), std::move(index_file_writer)); + DORIS_CHECK(inserted); + DORIS_CHECK(index_file_writer_it->second != nullptr); if (return_columns.empty()) { // no columns to read continue; @@ -606,8 +724,7 @@ Status IndexBuilder::handle_single_rowset(RowsetMetaSharedPtr output_rowset_meta "handle_single_rowset_write_inverted_index_data_error"); }) if (!status.ok()) { - return Status::Error( - "failed to write block."); + return status; } block->clear_column_data(); } @@ -615,9 +732,10 @@ Status IndexBuilder::handle_single_rowset(RowsetMetaSharedPtr output_rowset_meta // finish write inverted index, flush data to compound file for (auto& writer_sign : inverted_index_writer_signs) { try { - if (_index_column_writers[writer_sign]) { - RETURN_IF_ERROR(_index_column_writers[writer_sign]->finish()); - } + auto index_column_writer_it = _index_column_writers.find(writer_sign); + DORIS_CHECK(index_column_writer_it != _index_column_writers.end()); + DORIS_CHECK(index_column_writer_it->second != nullptr); + RETURN_IF_ERROR(index_column_writer_it->second->finish()); DBUG_EXECUTE_IF("IndexBuilder::handle_single_rowset_index_build_finish_error", { _CLTHROWA(CL_ERR_IO, "debug point: handle_single_rowset_index_build_finish_error"); @@ -662,6 +780,211 @@ Status IndexBuilder::handle_single_rowset(RowsetMetaSharedPtr output_rowset_meta return Status::OK(); } +Status IndexBuilder::_handle_single_rowset_snii( + RowsetMetaSharedPtr output_rowset_meta, + std::vector& segments) { + const std::string rowset_id = output_rowset_meta->rowset_id().to_string(); + auto input_schema_it = _input_rowset_schemas.find(rowset_id); + DORIS_CHECK(input_schema_it != _input_rowset_schemas.end()); + + for (auto& seg_ptr : segments) { + RETURN_IF_ERROR(_rewrite_single_segment_snii(output_rowset_meta->fs(), + output_rowset_meta->tablet_schema(), + *input_schema_it->second, rowset_id, seg_ptr)); + } + size_t inverted_index_size = 0; + for (auto&& [seg_id, index_file_writer] : _index_file_writers) { + RETURN_IF_ERROR(index_file_writer->begin_close()); + inverted_index_size += index_file_writer->get_index_file_total_size(); + } + for (auto&& [seg_id, index_file_writer] : _index_file_writers) { + RETURN_IF_ERROR(index_file_writer->finish_close()); + } + _index_column_writers.clear(); + _index_file_writers.clear(); + output_rowset_meta->set_data_disk_size(output_rowset_meta->data_disk_size()); + output_rowset_meta->set_total_disk_size(output_rowset_meta->total_disk_size() + + inverted_index_size); + output_rowset_meta->set_index_disk_size(output_rowset_meta->index_disk_size() + + inverted_index_size); + LOG(INFO) << "all row nums. source_rows=" << output_rowset_meta->num_rows(); + return Status::OK(); +} + +Status IndexBuilder::_rewrite_single_segment_snii(const io::FileSystemSPtr& fs, + const TabletSchemaSPtr& output_rowset_schema, + const TabletSchema& input_schema, + const std::string& rowset_id, + const segment_v2::SegmentSharedPtr& seg_ptr) { + // The source reader was registered in update_inverted_index_info. A rowset + // written before any index existed has no container file at all; everything + // requested is then built fresh. + auto reader_it = _index_file_readers.find(std::make_pair(rowset_id, seg_ptr->id())); + DORIS_CHECK(reader_it != _index_file_readers.end()); + IndexFileReader* source_reader = reader_it->second.get(); + bool has_container = true; + { + Status init_status = source_reader->init(); + if (init_status.is()) { + has_container = false; + } else if (!init_status.ok()) { + return init_status; + } + } + // An ANN index needs no special case here any more. plan_snii_index_rewrite + // enumerates inverted_and_ann_indexes(), which covers ANN, and an ANN index is stored + // as a blob logical index -- so it is classified rebuild-always exactly like a + // BKD, and IndexColumnWriter::create routes it to the ANN writer from the same + // raw column read. + const auto container_has = [source_reader, has_container](const TabletIndex& index, + bool* exists) -> Status { + if (!has_container) { + *exists = false; + return Status::OK(); + } + return source_reader->index_file_exist(&index, exists); + }; + SniiIndexRewritePlan plan; + RETURN_IF_ERROR(plan_snii_index_rewrite( + input_schema, *output_rowset_schema, _alter_index_ids, container_has, + has_container && source_reader->snii_has_blob_index(), &plan)); + + std::string index_path_prefix {InvertedIndexDescriptor::get_index_file_path_prefix( + local_segment_path(_tablet->tablet_path(), rowset_id, seg_ptr->id()))}; + std::string index_path = InvertedIndexDescriptor::get_index_file_path_v2(index_path_prefix); + io::FileWriterPtr file_writer; + RETURN_IF_ERROR(fs->create_file(index_path, &file_writer)); + auto index_file_writer = std::make_unique( + fs, index_path_prefix, rowset_id, seg_ptr->id(), InvertedIndexStorageFormatPB::SNII, + std::move(file_writer), true /* can_use_ram_dir */, _tablet->tablet_id()); + + // ONE inheritance of the source's physical prefix per segment container: + // unchanged indexes cost no analyzer, no postings decode and no encode. + if (!plan.inherit_keys.empty()) { + DORIS_CHECK(has_container); + snii::reader::SniiRewriteSnapshot snapshot; + RETURN_IF_ERROR(source_reader->prepare_snii_rewrite_snapshot( + plan.inherit_keys, seg_ptr->num_rows(), &snapshot)); + RETURN_IF_ERROR(index_file_writer->inherit_snii(snapshot, source_reader->snii_io_reader())); + } + if (!plan.build_columns.empty()) { + RETURN_IF_ERROR(_build_snii_indexes_for_segment(output_rowset_schema, plan, + index_file_writer.get(), seg_ptr)); + } + auto [file_writer_it, inserted] = + _index_file_writers.emplace(seg_ptr->id(), std::move(index_file_writer)); + DORIS_CHECK(inserted); + DORIS_CHECK(file_writer_it->second != nullptr); + return Status::OK(); +} + +Status IndexBuilder::_build_snii_indexes_for_segment(const TabletSchemaSPtr& output_rowset_schema, + const SniiIndexRewritePlan& plan, + IndexFileWriter* index_file_writer, + const segment_v2::SegmentSharedPtr& seg_ptr) { + // One raw column read per column group; every writer on the column is fed + // from the same converted data. + std::vector>> group_writer_signs; + std::vector return_columns; + _olap_data_convertor->reserve(plan.build_columns.size()); + for (const auto& [col_unique_id, index_metas] : plan.build_columns) { + const int32_t column_idx = output_rowset_schema->field_index(col_unique_id); + DORIS_CHECK_GE(column_idx, 0); + const TabletColumn& column = output_rowset_schema->column(column_idx); + DORIS_CHECK(segment_v2::IndexColumnWriter::check_support_inverted_index(column)); + _olap_data_convertor->add_column_data_convertor(column); + return_columns.emplace_back(column_idx); + std::vector> signs; + for (const TabletIndex* index_meta : index_metas) { + std::unique_ptr index_column_writer; + try { + RETURN_IF_ERROR(segment_v2::IndexColumnWriter::create( + &column, &index_column_writer, index_file_writer, index_meta)); + } catch (const std::exception& e) { + return Status::Error( + "CLuceneError occurred: {}", e.what()); + } + auto writer_sign = + std::make_pair(seg_ptr->id(), index_meta->index_id()); + auto [writer_it, inserted] = _index_column_writers.insert( + std::make_pair(writer_sign, std::move(index_column_writer))); + DORIS_CHECK(inserted); + DORIS_CHECK(writer_it->second != nullptr); + signs.push_back(writer_sign); + } + group_writer_signs.push_back(std::move(signs)); + } + + StorageReadOptions read_options; + OlapReaderStatistics stats; + read_options.stats = &stats; + read_options.tablet_schema = output_rowset_schema; + std::shared_ptr schema = + std::make_shared(output_rowset_schema->columns(), return_columns); + std::unique_ptr iter; + RETURN_IF_ERROR(seg_ptr->new_iterator(schema, read_options, &iter)); + + auto block = Block::create_unique(output_rowset_schema->create_block(return_columns)); + while (true) { + Status status = iter->next_batch(block.get()); + if (!status.ok()) { + if (status.is()) { + break; + } + LOG(WARNING) << "failed to read next block when building SNII index." + << ", err=" << status.to_string(); + return status; + } + RETURN_IF_ERROR(_write_snii_index_data(output_rowset_schema, block.get(), plan, + group_writer_signs)); + block->clear_column_data(); + } + for (const auto& signs : group_writer_signs) { + for (const auto& writer_sign : signs) { + auto writer_it = _index_column_writers.find(writer_sign); + DORIS_CHECK(writer_it != _index_column_writers.end()); + RETURN_IF_ERROR(writer_it->second->finish()); + DBUG_EXECUTE_IF("IndexBuilder::handle_single_rowset_snii_index_build_finish_error", { + return Status::Error( + "debug point: handle_single_rowset_snii_index_build_finish_error"); + }) + } + } + _olap_data_convertor->reset(); + return Status::OK(); +} + +Status IndexBuilder::_write_snii_index_data( + const TabletSchemaSPtr& tablet_schema, Block* block, const SniiIndexRewritePlan& plan, + const std::vector>>& group_writer_signs) { + _olap_data_convertor->set_source_content(block, 0, block->rows()); + for (size_t group = 0; group < group_writer_signs.size(); ++group) { + auto converted_result = _olap_data_convertor->convert_column_data(group); + if (!converted_result.first.ok()) { + LOG(WARNING) << "failed to convert block, errcode: " << converted_result.first; + return converted_result.first; + } + const TabletColumn& column = tablet_schema->column_by_uid(plan.build_columns[group].first); + const auto* base_ptr = (const uint8_t*)converted_result.second->get_data(); + const auto* null_map = converted_result.second->get_nullmap(); + for (const auto& writer_sign : group_writer_signs[group]) { + // _add_nullable/_add_data advance the value pointer as they consume + // rows; every writer on the column starts from the SAME converted + // data, which is exactly the shared-column-read guarantee. + const uint8_t* ptr = base_ptr; + if (null_map) { + RETURN_IF_ERROR(_add_nullable(column.name(), writer_sign, &column, null_map, &ptr, + block->rows())); + } else { + RETURN_IF_ERROR( + _add_data(column.name(), writer_sign, &column, &ptr, block->rows())); + } + } + } + _olap_data_convertor->clear_source_content(); + return Status::OK(); +} + Status IndexBuilder::_write_inverted_index_data(TabletSchemaSPtr tablet_schema, int64_t segment_idx, Block* block) { VLOG_DEBUG << "begin to write inverted/ann index"; @@ -715,6 +1038,11 @@ Status IndexBuilder::_add_nullable(const std::string& column_name, const std::pair& index_writer_sign, const TabletColumn* column, const uint8_t* null_map, const uint8_t** ptr, size_t num_rows) { + auto index_column_writer_it = _index_column_writers.find(index_writer_sign); + DORIS_CHECK(index_column_writer_it != _index_column_writers.end()); + DORIS_CHECK(index_column_writer_it->second != nullptr); + auto* index_column_writer = index_column_writer_it->second.get(); + // TODO: need to process null data for inverted index if (column->type() == FieldType::OLAP_FIELD_TYPE_ARRAY) { DCHECK(column->get_subtype_count() == 1); @@ -726,15 +1054,14 @@ Status IndexBuilder::_add_nullable(const std::string& column_name, try { auto data = *(data_ptr + 2); auto nested_null_map = *(data_ptr + 3); - RETURN_IF_ERROR(_index_column_writers[index_writer_sign]->add_array_values( + RETURN_IF_ERROR(index_column_writer->add_array_values( field_type_size(column->get_sub_column(0).type()), reinterpret_cast(data), reinterpret_cast(nested_null_map), offsets_ptr, num_rows)); DBUG_EXECUTE_IF("IndexBuilder::_add_nullable_add_array_values_error", { _CLTHROWA(CL_ERR_IO, "debug point: _add_nullable_add_array_values_error"); }) - RETURN_IF_ERROR( - _index_column_writers[index_writer_sign]->add_array_nulls(null_map, num_rows)); + RETURN_IF_ERROR(index_column_writer->add_array_nulls(null_map, num_rows)); } catch (const std::exception& e) { return Status::Error( "CLuceneError occurred: {}", e.what()); @@ -758,11 +1085,9 @@ Status IndexBuilder::_add_nullable(const std::string& column_name, do { auto step = next_run_step(); if (null_map[offset]) { - RETURN_IF_ERROR(_index_column_writers[index_writer_sign]->add_nulls( - static_cast(step))); + RETURN_IF_ERROR(index_column_writer->add_nulls(static_cast(step))); } else { - RETURN_IF_ERROR(_index_column_writers[index_writer_sign]->add_values(column_name, - *ptr, step)); + RETURN_IF_ERROR(index_column_writer->add_values(column_name, *ptr, step)); } *ptr += field_type_size(column->type()) * step; offset += step; @@ -780,6 +1105,11 @@ Status IndexBuilder::_add_nullable(const std::string& column_name, Status IndexBuilder::_add_data(const std::string& column_name, const std::pair& index_writer_sign, const TabletColumn* column, const uint8_t** ptr, size_t num_rows) { + auto index_column_writer_it = _index_column_writers.find(index_writer_sign); + DORIS_CHECK(index_column_writer_it != _index_column_writers.end()); + DORIS_CHECK(index_column_writer_it->second != nullptr); + auto* index_column_writer = index_column_writer_it->second.get(); + try { if (column->type() == FieldType::OLAP_FIELD_TYPE_ARRAY) { DCHECK(column->get_subtype_count() == 1); @@ -792,14 +1122,13 @@ Status IndexBuilder::_add_data(const std::string& column_name, if (element_cnt > 0) { auto data = *(data_ptr + 2); auto nested_null_map = *(data_ptr + 3); - RETURN_IF_ERROR(_index_column_writers[index_writer_sign]->add_array_values( + RETURN_IF_ERROR(index_column_writer->add_array_values( field_type_size(column->get_sub_column(0).type()), reinterpret_cast(data), reinterpret_cast(nested_null_map), offsets_ptr, num_rows)); } } else { - RETURN_IF_ERROR(_index_column_writers[index_writer_sign]->add_values(column_name, *ptr, - num_rows)); + RETURN_IF_ERROR(index_column_writer->add_values(column_name, *ptr, num_rows)); } DBUG_EXECUTE_IF("IndexBuilder::_add_data_throw_exception", { _CLTHROWA(CL_ERR_IO, "debug point: _add_data_throw_exception"); }) diff --git a/be/src/storage/task/index_builder.h b/be/src/storage/task/index_builder.h index bf417182b7ff3c..e7cacd9051fadc 100644 --- a/be/src/storage/task/index_builder.h +++ b/be/src/storage/task/index_builder.h @@ -17,8 +17,11 @@ #pragma once +#include + #include "storage/index/index_file_writer.h" #include "storage/index/inverted/inverted_index_desc.h" +#include "storage/index/snii/reader/snii_segment_reader.h" #include "storage/iterator/olap_data_convertor.h" #include "storage/merger.h" #include "storage/olap_common.h" @@ -56,7 +59,59 @@ class IndexBuilder { virtual Status modify_rowsets(const Merger::Statistics* stats = nullptr); virtual void gc_output_rowset(); + // How one SNII segment rewrite treats the target schema's logical indexes: + // inherit_keys are carried over from the source container without decoding a + // posting; build_columns are built from raw column data, grouped by column + // unique id so every index on a column is fed from the SAME column read. + struct SniiIndexRewritePlan { + std::vector inherit_keys; + std::vector>> build_columns; + }; + + // Classifies the output schema's inverted indexes for one SNII segment + // rewrite (design: same key and definition -> inherit; requested or + // definition-changed -> build; keys the target schema dropped are simply not + // inherited). `container_has` reports whether the SOURCE container holds a + // given logical index; static and callback-based so the classification is + // directly unit-testable without files. + // + // `source_container_has_blob` suppresses inheritance for the WHOLE segment: + // inheritance snapshots the source container, and a container holding a blob + // logical index cannot be snapshotted at all (SniiSegmentReader rejects it, + // by directory content rather than by what is being kept). Rebuilding every + // index is then the only way through, and it is a correct one. + static Status plan_snii_index_rewrite( + const TabletSchema& input_schema, const TabletSchema& output_schema, + const std::set& alter_index_ids, + const std::function& container_has, + bool source_container_has_blob, SniiIndexRewritePlan* plan); + private: + // SNII counterpart of the V1/V2 build branch in handle_single_rowset: plans + // the rewrite per segment, inherits the source container's physical prefix + // once, and builds only the missing indexes -- one raw column read per + // column, shared by every writer on it. + Status _handle_single_rowset_snii(RowsetMetaSharedPtr output_rowset_meta, + std::vector& segments); + // One segment of the above: plan, inherit the prefix, build what is missing, + // and register the container writer for the shared close pass. + Status _rewrite_single_segment_snii(const io::FileSystemSPtr& fs, + const TabletSchemaSPtr& output_rowset_schema, + const TabletSchema& input_schema, + const std::string& rowset_id, + const segment_v2::SegmentSharedPtr& seg_ptr); + // The build half of one segment rewrite: creates the writers of every column + // group, scans each column once and feeds all of its writers. + Status _build_snii_indexes_for_segment(const TabletSchemaSPtr& output_rowset_schema, + const SniiIndexRewritePlan& plan, + IndexFileWriter* index_file_writer, + const segment_v2::SegmentSharedPtr& seg_ptr); + // Feeds one converted block into the SNII build writers. group_writer_signs + // parallels plan.build_columns: entry g holds the writer signs fed from + // convertor ordinal g. + Status _write_snii_index_data( + const TabletSchemaSPtr& tablet_schema, Block* block, const SniiIndexRewritePlan& plan, + const std::vector>>& group_writer_signs); Status _write_inverted_index_data(TabletSchemaSPtr tablet_schema, int64_t segment_idx, Block* block); Status _add_data(const std::string& column_name, @@ -87,6 +142,10 @@ class IndexBuilder { // std::unordered_map, std::unique_ptr> _index_file_readers; + // SNII only: output rowset id -> the INPUT rowset's schema. The rewrite plan + // compares an index's definition between input and output schema to decide + // inherit vs rebuild; the output rowset meta no longer carries the input's. + std::unordered_map _input_rowset_schemas; }; using IndexBuilderSharedPtr = std::shared_ptr; diff --git a/be/test/exec/scan/scanner_context_test.cpp b/be/test/exec/scan/scanner_context_test.cpp index 4eb76c27a219c9..0e2aa0a9bde9d4 100644 --- a/be/test/exec/scan/scanner_context_test.cpp +++ b/be/test/exec/scan/scanner_context_test.cpp @@ -39,6 +39,10 @@ #include "exec/scan/scanner_scheduler.h" #include "runtime/descriptors.h" #include "runtime/query_context.h" +#include "storage/options.h" +#include "storage/storage_engine.h" +#include "storage/tablet/tablet.h" +#include "storage/tablet/tablet_meta.h" #include "testutil/mock/mock_runtime_state.h" namespace doris { @@ -185,6 +189,165 @@ TEST_F(ScannerContextTest, test_init) { ASSERT_TRUE(st.ok()); } +TEST_F(ScannerContextTest, inverted_index_profile_collection_is_additive_and_idempotent) { + auto engine = std::make_unique(EngineOptions {}); + auto tablet_meta = std::make_shared(1, 2, 15673, 15674, 4, 5, TTabletSchema {}, 6, + std::unordered_map {{7, 8}}, + UniqueId(9, 10), TTabletType::TABLET_TYPE_DISK, + TCompressionType::LZ4F); + auto tablet = std::make_shared(*engine, std::move(tablet_meta), nullptr); + const int parallel_tasks = 1; + auto scan_operator = std::make_unique(obj_pool.get(), tnode, 0, *descs, + parallel_tasks, TQueryCacheParam {}); + auto local_state = OlapScanLocalState::create_unique(state.get(), scan_operator.get()); + const std::vector scan_ranges; + const std::map, + std::vector>>> + shared_state_map; + LocalStateInfo local_state_info {profile.get(), scan_ranges, nullptr, shared_state_map, 0}; + const Status init_status = local_state->init(state.get(), local_state_info); + ASSERT_TRUE(init_status.ok()) << init_status.to_string(); + + auto make_scanner = [&]() { + OlapScanner::Params params; + params.state = state.get(); + params.profile = profile.get(); + params.version = 0; + params.limit = -1; + params.aggregation = false; + return OlapScanner::create_shared(local_state.get(), std::move(params)); + }; + auto scanner1 = make_scanner(); + auto scanner2 = make_scanner(); + scanner1->_tablet_reader_params.tablet = tablet; + scanner2->_tablet_reader_params.tablet = tablet; + scanner1->_tablet_reader = std::make_unique(); + scanner2->_tablet_reader = std::make_unique(); + auto* stats1 = scanner1->_tablet_reader->mutable_stats(); + stats1->snii_stats.prx_raw_frames = 1; + stats1->snii_stats.prx_plaintext_bytes = 10; + stats1->snii_stats.prx_decode_ns = 100; + stats1->snii_stats.phrase_candidate_docs = 3; + stats1->snii_stats.common_grams_gram_plans = 1; + stats1->snii_stats.common_grams_fallback_kill_switch = 5; + stats1->snii_stats.common_grams_plain_posting_bytes = 10; + stats1->snii_stats.common_grams_gram_posting_bytes = 20; + stats1->snii_stats.common_grams_plain_estimated_candidate_df = 30; + stats1->snii_stats.common_grams_gram_estimated_candidate_df = 40; + stats1->snii_stats.common_grams_plain_estimated_cost = 50; + stats1->snii_stats.common_grams_gram_estimated_cost = 60; + stats1->snii_stats.common_grams_fallback_base_analyzer_mismatch = 61; + stats1->snii_stats.common_grams_fallback_prefix_tail_empty = 62; + stats1->snii_stats.common_grams_planning_ns = 65; + auto* stats2 = scanner2->_tablet_reader->mutable_stats(); + stats2->snii_stats.prx_raw_frames = 2; + stats2->snii_stats.prx_plaintext_bytes = 20; + stats2->snii_stats.prx_decode_ns = 200; + stats2->snii_stats.phrase_candidate_docs = 4; + stats2->snii_stats.common_grams_gram_plans = 2; + stats2->snii_stats.common_grams_fallback_kill_switch = 6; + stats2->snii_stats.common_grams_plain_posting_bytes = 1; + stats2->snii_stats.common_grams_gram_posting_bytes = 2; + stats2->snii_stats.common_grams_plain_estimated_candidate_df = 3; + stats2->snii_stats.common_grams_gram_estimated_candidate_df = 4; + stats2->snii_stats.common_grams_plain_estimated_cost = 5; + stats2->snii_stats.common_grams_gram_estimated_cost = 6; + stats2->snii_stats.common_grams_fallback_base_analyzer_mismatch = 7; + stats2->snii_stats.common_grams_fallback_prefix_tail_empty = 8; + stats2->snii_stats.common_grams_planning_ns = 11; + + RuntimeProfile* index_filter = local_state->_index_filter_profile.get(); + ASSERT_NE(index_filter, nullptr); + auto* raw_frames = index_filter->get_counter("SniiPrxRawFrames"); + auto* plaintext_bytes = index_filter->get_counter("SniiPrxPlaintextBytes"); + auto* decode_time = index_filter->get_counter("SniiPrxInclusiveDecodeTime"); + auto* phrase_candidate_docs = index_filter->get_counter("SniiPhraseCandidateDocs"); + auto* common_grams_gram_plans = index_filter->get_counter("SniiCommonGramsGramPlans"); + auto* common_grams_fallback_kill_switch = + index_filter->get_counter("SniiCommonGramsFallbackKillSwitch"); + struct ExpectedSniiCounter { + const char* name; + RuntimeProfile::Counter* counter; + int64_t scanner1_value; + int64_t combined_value; + }; + const ExpectedSniiCounter snii_counters[] = { + {"SniiCommonGramsPlainPostingBytes", + index_filter->get_counter("SniiCommonGramsPlainPostingBytes"), 10, 11}, + {"SniiCommonGramsGramPostingBytes", + index_filter->get_counter("SniiCommonGramsGramPostingBytes"), 20, 22}, + {"SniiCommonGramsPlainEstimatedCandidateDf", + index_filter->get_counter("SniiCommonGramsPlainEstimatedCandidateDf"), 30, 33}, + {"SniiCommonGramsGramEstimatedCandidateDf", + index_filter->get_counter("SniiCommonGramsGramEstimatedCandidateDf"), 40, 44}, + {"SniiCommonGramsPlainEstimatedCost", + index_filter->get_counter("SniiCommonGramsPlainEstimatedCost"), 50, 55}, + {"SniiCommonGramsGramEstimatedCost", + index_filter->get_counter("SniiCommonGramsGramEstimatedCost"), 60, 66}, + {"SniiCommonGramsFallbackBaseAnalyzerMismatch", + index_filter->get_counter("SniiCommonGramsFallbackBaseAnalyzerMismatch"), 61, 68}, + {"SniiCommonGramsFallbackPrefixTailEmpty", + index_filter->get_counter("SniiCommonGramsFallbackPrefixTailEmpty"), 62, 70}, + {"SniiCommonGramsPlanningTime", + index_filter->get_counter("SniiCommonGramsPlanningTime"), 65, 76}, + }; + + std::vector zero_nodes; + index_filter->to_thrift(&zero_nodes); + ASSERT_EQ(zero_nodes.size(), 1U); + for (const auto& expected : snii_counters) { + bool serialized = false; + for (const auto& thrift_counter : zero_nodes.front().counters) { + serialized |= thrift_counter.name == expected.name; + } + EXPECT_FALSE(serialized) << expected.name; + } + ASSERT_NE(raw_frames, nullptr); + ASSERT_NE(plaintext_bytes, nullptr); + ASSERT_NE(decode_time, nullptr); + ASSERT_NE(phrase_candidate_docs, nullptr); + ASSERT_NE(common_grams_gram_plans, nullptr); + ASSERT_NE(common_grams_fallback_kill_switch, nullptr); + for (const auto& expected : snii_counters) { + ASSERT_NE(expected.counter, nullptr) << expected.name; + EXPECT_NE(dynamic_cast(expected.counter), nullptr) + << expected.name; + } + + scanner1->_collect_profile_before_close(); + EXPECT_EQ(raw_frames->value(), 1); + EXPECT_EQ(plaintext_bytes->value(), 10); + EXPECT_EQ(decode_time->value(), 100); + EXPECT_EQ(phrase_candidate_docs->value(), 3); + EXPECT_EQ(common_grams_gram_plans->value(), 1); + EXPECT_EQ(common_grams_fallback_kill_switch->value(), 5); + for (const auto& expected : snii_counters) { + EXPECT_EQ(expected.counter->value(), expected.scanner1_value) << expected.name; + } + + scanner1->_collect_profile_before_close(); + EXPECT_EQ(raw_frames->value(), 1); + EXPECT_EQ(plaintext_bytes->value(), 10); + EXPECT_EQ(decode_time->value(), 100); + EXPECT_EQ(phrase_candidate_docs->value(), 3); + EXPECT_EQ(common_grams_gram_plans->value(), 1); + EXPECT_EQ(common_grams_fallback_kill_switch->value(), 5); + for (const auto& expected : snii_counters) { + EXPECT_EQ(expected.counter->value(), expected.scanner1_value) << expected.name; + } + + scanner2->_collect_profile_before_close(); + EXPECT_EQ(raw_frames->value(), 3); + EXPECT_EQ(plaintext_bytes->value(), 30); + EXPECT_EQ(decode_time->value(), 300); + EXPECT_EQ(phrase_candidate_docs->value(), 7); + EXPECT_EQ(common_grams_gram_plans->value(), 3); + EXPECT_EQ(common_grams_fallback_kill_switch->value(), 11); + for (const auto& expected : snii_counters) { + EXPECT_EQ(expected.counter->value(), expected.combined_value) << expected.name; + } +} + TEST_F(ScannerContextTest, test_serial_run) { const int parallel_tasks = 1; auto scan_operator = std::make_unique(obj_pool.get(), tnode, 0, *descs, diff --git a/be/test/exprs/function/function_is_null_test.cpp b/be/test/exprs/function/function_is_null_test.cpp index b10e7d76f31b05..fb5e59d391f507 100644 --- a/be/test/exprs/function/function_is_null_test.cpp +++ b/be/test/exprs/function/function_is_null_test.cpp @@ -77,11 +77,18 @@ class FunctionIsNullTest : public ::testing::Test { _inverted_index_query_cache = std::unique_ptr( InvertedIndexQueryCache::create_global_cache(inverted_index_cache_limit, 1)); + // Both caches are owned by this fixture, so the previous globals must come back in + // TearDown -- otherwise ExecEnv keeps pointing at them after the fixture is destroyed and + // the next test that reaches InvertedIndexQueryCache::instance() reads freed memory. + _previous_searcher_cache = ExecEnv::GetInstance()->get_inverted_index_searcher_cache(); + _previous_query_cache = ExecEnv::GetInstance()->get_inverted_index_query_cache(); ExecEnv::GetInstance()->set_inverted_index_searcher_cache( _inverted_index_searcher_cache.get()); - ExecEnv::GetInstance()->_inverted_index_query_cache = _inverted_index_query_cache.get(); + ExecEnv::GetInstance()->set_inverted_index_query_cache(_inverted_index_query_cache.get()); } void TearDown() override { + ExecEnv::GetInstance()->set_inverted_index_searcher_cache(_previous_searcher_cache); + ExecEnv::GetInstance()->set_inverted_index_query_cache(_previous_query_cache); EXPECT_TRUE(io::global_local_filesystem()->delete_directory(_tablet->tablet_path()).ok()); EXPECT_TRUE(io::global_local_filesystem()->delete_directory(_absolute_dir).ok()); EXPECT_TRUE(io::global_local_filesystem()->delete_directory(tmp_dir).ok()); @@ -135,6 +142,8 @@ class FunctionIsNullTest : public ::testing::Test { std::string _absolute_dir; std::string _curreent_dir; TabletSchemaPB _schema_pb; + InvertedIndexSearcherCache* _previous_searcher_cache = nullptr; + InvertedIndexQueryCache* _previous_query_cache = nullptr; std::unique_ptr _inverted_index_searcher_cache; std::unique_ptr _inverted_index_query_cache; }; diff --git a/be/test/exprs/function/function_match_test.cpp b/be/test/exprs/function/function_match_test.cpp index 738381a04ef576..ac9557d79f5bf5 100644 --- a/be/test/exprs/function/function_match_test.cpp +++ b/be/test/exprs/function/function_match_test.cpp @@ -29,6 +29,7 @@ #include "core/column/column_vector.h" #include "exprs/function/match.h" #include "storage/index/inverted/analyzer/analyzer.h" +#include "storage/index/inverted/analyzer/custom_analyzer.h" namespace doris { @@ -582,18 +583,32 @@ TEST(FunctionMatchTest, execute_impl_structure) { TEST(FunctionMatchTest, custom_analyzer_handling) { FunctionMatchAny match_any; - auto ctx = create_inverted_index_ctx(InvertedIndexParserType::PARSER_ENGLISH); + segment_v2::inverted_index::CustomAnalyzerConfig::Builder builder; + builder.with_tokenizer_config("keyword", {}); + builder.add_token_filter_config("lowercase", {}); + auto provider = + std::make_shared(builder.build()); + + InvertedIndexAnalyzerCtx analyzer_ctx; + analyzer_ctx.analyzer_name = "custom_keyword_lowercase"; + analyzer_ctx.parser_type = InvertedIndexParserType::PARSER_NONE; + analyzer_ctx.analyzer_provider = provider; + analyzer_ctx.analyzer = + provider->get_analyzer(segment_v2::inverted_index::AnalysisPurpose::kPlainQuery); - // Test without custom analyzer - ctx.ctx->analyzer_name = ""; - auto tokens1 = match_any.analyse_query_str_token(ctx.ctx.get(), "test query", "test_col"); - EXPECT_GT(tokens1.size(), 0); + auto query_tokens = match_any.analyse_query_str_token(&analyzer_ctx, "TEST QUERY", "test_col"); + ASSERT_EQ(query_tokens.size(), 1); + ASSERT_TRUE(query_tokens[0].is_single_term()); + EXPECT_EQ(query_tokens[0].get_single_term(), "test query"); - // Test with custom analyzer (should be handled appropriately) - ctx.ctx->analyzer_name = "custom_analyzer_name"; - auto tokens2 = match_any.analyse_query_str_token(ctx.ctx.get(), "test query", "test_col"); - // Custom analyzer handling would depend on implementation details - EXPECT_GE(tokens2.size(), 0); + auto string_col = ColumnString::create(); + string_col->insert_data("TEST QUERY", 10); + int32_t offset = 0; + auto data_tokens = match_any.analyse_data_token("test_col", &analyzer_ctx, string_col.get(), 0, + nullptr, offset); + ASSERT_EQ(data_tokens.size(), 1); + ASSERT_TRUE(data_tokens[0].is_single_term()); + EXPECT_EQ(data_tokens[0].get_single_term(), "test query"); } // Test column type validation @@ -841,4 +856,4 @@ TEST(FunctionMatchTest, function_registration) { EXPECT_TRUE(true); } -} // namespace doris \ No newline at end of file +} // namespace doris diff --git a/be/test/exprs/function/function_search_test.cpp b/be/test/exprs/function/function_search_test.cpp index ac57847f32e4de..4ac10cd510152c 100644 --- a/be/test/exprs/function/function_search_test.cpp +++ b/be/test/exprs/function/function_search_test.cpp @@ -21,25 +21,37 @@ #include #include +#include #include #include #include +#include #include #include +#include +#include "core/assert_cast.h" #include "core/block/block.h" +#include "core/column/column_nullable.h" +#include "core/column/column_vector.h" #include "core/data_type/data_type_array.h" #include "core/data_type/data_type_nullable.h" #include "core/data_type/data_type_string.h" #include "core/data_type/primitive_type.h" +#include "runtime/exec_env.h" +#include "runtime/index_policy/index_policy_mgr.h" #include "storage/index/index_file_reader.h" #include "storage/index/index_iterator.h" #include "storage/index/inverted/inverted_index_iterator.h" #include "storage/index/inverted/inverted_index_parser.h" +#include "storage/index/inverted/query_v2/collect/doc_set_collector.h" +#include "storage/index/inverted/query_v2/collect/top_k_collector.h" #include "storage/index/inverted/query_v2/phrase_query/multi_phrase_query.h" #include "storage/index/inverted/query_v2/phrase_query/multi_phrase_weight.h" #include "storage/index/inverted/query_v2/phrase_query/phrase_query.h" #include "storage/segment/variant/nested_group_provider.h" +#include "util/defer_op.h" +#include "util/thrift_util.h" namespace doris { @@ -115,6 +127,27 @@ class RecordingIndexIterator : public segment_v2::IndexIterator { Int32 last_int_value = 0; }; +class RecordingDirectInvertedIndexIterator final : public segment_v2::InvertedIndexIterator { +public: + Status read_from_index(const segment_v2::IndexParam& param) override { + ++read_calls; + auto* inverted_param = std::get_if(¶m); + DORIS_CHECK(inverted_param != nullptr); + DORIS_CHECK(*inverted_param != nullptr); + DORIS_CHECK((*inverted_param)->roaring != nullptr); + (*inverted_param)->roaring->add(3); + return Status::OK(); + } + + Status read_null_bitmap(segment_v2::InvertedIndexQueryCacheHandle* /*cache_handle*/) override { + return Status::OK(); + } + + Result has_null() override { return false; } + + int read_calls = 0; +}; + class DummyInvertedIndexReader final : public segment_v2::InvertedIndexReader { public: explicit DummyInvertedIndexReader(const TabletIndex* index_meta) @@ -151,6 +184,205 @@ class DummyInvertedIndexReader final : public segment_v2::InvertedIndexReader { segment_v2::InvertedIndexReaderType _reader_type = segment_v2::InvertedIndexReaderType::BKD; }; +class RejectingCluceneIndexFileReader final : public segment_v2::IndexFileReader { +public: + explicit RejectingCluceneIndexFileReader( + InvertedIndexStorageFormatPB storage_format = InvertedIndexStorageFormatPB::SNII, + const std::string& index_path = "/tmp/search_snii_native_idx") + : segment_v2::IndexFileReader(nullptr, index_path, storage_format) {} + + Status init(int32_t /*read_buffer_size*/, const io::IOContext* /*io_ctx*/) override { + ++init_calls; + return Status::OK(); + } + + Result> open( + const TabletIndex* /*index_meta*/, const io::IOContext* /*io_ctx*/) const override { + ++open_calls; + return ResultError(Status::InternalError("unexpected CLucene open for SNII search")); + } + + int init_calls = 0; + mutable int open_calls = 0; +}; + +class RecordingNativeInvertedIndexReader final : public segment_v2::InvertedIndexReader { +public: + RecordingNativeInvertedIndexReader( + const TabletIndex* index_meta, + const std::shared_ptr& index_file_reader, + segment_v2::InvertedIndexReaderType reader_type = + segment_v2::InvertedIndexReaderType::FULLTEXT) + : segment_v2::InvertedIndexReader(index_meta, index_file_reader), + _reader_type(reader_type), + _null_cache(1024 * 1024, 1), + _null_cache_key {"/tmp/search_snii_native_null", "", + segment_v2::InvertedIndexQueryType::UNKNOWN_QUERY, + std::to_string(index_meta->index_id())} { + set_has_null(false); + } + + Status new_iterator(std::unique_ptr* /*iterator*/) override { + return Status::OK(); + } + + Status query(const segment_v2::IndexQueryContextPtr& context, const std::string& column_name, + const Field& query_value, segment_v2::InvertedIndexQueryType query_type, + std::shared_ptr& bit_map, + const InvertedIndexAnalyzerCtx* analyzer_ctx = nullptr) override { + ++query_calls; + last_column_name = column_name; + last_query_type = query_type; + last_query_value_type = query_value.get_type(); + last_analyzer_ctx = analyzer_ctx; + if (last_query_value_type == TYPE_STRING) { + last_query_value = query_value.get(); + } + + bit_map = std::make_shared(); + auto result_it = query_results.find(last_query_value); + if (result_it != query_results.end()) { + *bit_map = result_it->second; + } + // SniiIndexReader publishes its per-document BM25 values through the collection + // similarity carried by the query context (score_plain_term_candidates / + // score_phrase_matches), never through a return value. Reproducing that handshake here + // is what lets the SEARCH scoring path be exercised without a physical SNII segment. + auto scores_it = query_scores.find(last_query_value); + if (scores_it != query_scores.end() && context != nullptr && + context->collection_similarity != nullptr) { + observed_similarity = context->collection_similarity.get(); + for (const auto& [doc, score] : scores_it->second) { + context->collection_similarity->collect(doc, score); + } + } + return Status::OK(); + } + + Status try_query(const segment_v2::IndexQueryContextPtr& /*context*/, + const std::string& /*column_name*/, const Field& /*query_value*/, + segment_v2::InvertedIndexQueryType /*query_type*/, + size_t* /*count*/) override { + return Status::OK(); + } + + Status read_null_bitmap(const segment_v2::IndexQueryContextPtr& /*context*/, + segment_v2::InvertedIndexQueryCacheHandle* cache_handle, + lucene::store::Directory* /*dir*/ = nullptr) override { + ++null_bitmap_calls; + _null_cache.insert(_null_cache_key, std::make_shared(_null_bitmap), + cache_handle); + return Status::OK(); + } + + segment_v2::InvertedIndexReaderType type() override { return _reader_type; } + + void set_query_result(const std::string& pattern, roaring::Roaring result) { + query_results[pattern] = std::move(result); + } + + void set_query_scores(const std::string& pattern, + std::vector> scores) { + query_scores[pattern] = std::move(scores); + } + + void set_null_bitmap(roaring::Roaring null_bitmap) { + _null_bitmap = std::move(null_bitmap); + set_has_null(!_null_bitmap.isEmpty()); + } + + int query_calls = 0; + int null_bitmap_calls = 0; + std::string last_column_name; + std::string last_query_value; + PrimitiveType last_query_value_type = PrimitiveType::TYPE_NULL; + segment_v2::InvertedIndexQueryType last_query_type = + segment_v2::InvertedIndexQueryType::UNKNOWN_QUERY; + const InvertedIndexAnalyzerCtx* last_analyzer_ctx = nullptr; + std::unordered_map query_results; + std::unordered_map>> query_scores; + // Identity of the similarity the reader was handed, so a test can prove the query's own + // collection similarity is not the one the reader writes into. + const CollectionSimilarity* observed_similarity = nullptr; + +private: + segment_v2::InvertedIndexReaderType _reader_type; + roaring::Roaring _null_bitmap; + segment_v2::InvertedIndexQueryCache _null_cache; + segment_v2::InvertedIndexQueryCache::CacheKey _null_cache_key; +}; + +class ScopedInvertedIndexQueryCache final { +public: + ScopedInvertedIndexQueryCache() + : _previous(ExecEnv::GetInstance()->get_inverted_index_query_cache()), + _cache(segment_v2::InvertedIndexQueryCache::create_global_cache(1024 * 1024, 1)) { + ExecEnv::GetInstance()->set_inverted_index_query_cache(_cache.get()); + } + + ~ScopedInvertedIndexQueryCache() { + ExecEnv::GetInstance()->set_inverted_index_query_cache(_previous); + } + + segment_v2::InvertedIndexQueryCache* get() const { return _cache.get(); } + +private: + segment_v2::InvertedIndexQueryCache* _previous; + std::unique_ptr _cache; +}; + +static roaring::Roaring make_bitmap(std::initializer_list docs) { + roaring::Roaring bitmap; + for (uint32_t doc : docs) { + bitmap.add(doc); + } + return bitmap; +} + +static void expect_bitmap_eq(const roaring::Roaring& actual, + std::initializer_list expected_docs) { + auto expected = make_bitmap(expected_docs); + EXPECT_EQ(expected.cardinality(), actual.cardinality()); + EXPECT_TRUE(actual == expected); +} + +static roaring::Roaring collect_docs( + const segment_v2::inverted_index::query_v2::ScorerPtr& scorer) { + roaring::Roaring docs; + for (uint32_t doc = scorer->doc(); doc != segment_v2::inverted_index::query_v2::TERMINATED; + doc = scorer->advance()) { + docs.add(doc); + } + return docs; +} + +static TSearchClause make_leaf_clause(const std::string& clause_type, const std::string& value) { + TSearchClause clause; + clause.clause_type = clause_type; + clause.field_name = "body"; + clause.value = value; + clause.__isset.field_name = true; + clause.__isset.value = true; + return clause; +} + +static Status insert_search_dsl_cache( + segment_v2::InvertedIndexQueryCache* cache, + const std::shared_ptr& index_file_reader, + const TSearchParam& search_param, roaring::Roaring bitmap) { + ThriftSerializer serializer(false, 1024); + TSearchParam copy = search_param; + std::string signature; + RETURN_IF_ERROR(serializer.serialize(©, &signature)); + + segment_v2::InvertedIndexQueryCache::CacheKey key { + index_file_reader->get_index_path_prefix(), "__search_dsl__", + segment_v2::InvertedIndexQueryType::SEARCH_DSL_QUERY, std::move(signature)}; + segment_v2::InvertedIndexQueryCacheHandle handle; + cache->insert(key, std::make_shared(std::move(bitmap)), &handle); + return Status::OK(); +} + static TabletIndex make_test_inverted_index( int64_t index_id, const std::map& properties = {}) { TabletIndex index_meta; @@ -166,6 +398,32 @@ static TabletIndex make_test_inverted_index( return index_meta; } +static Status resolve_non_variant_binding_with_mismatched_analyzer(const DataTypePtr& column_type) { + std::map index_properties; + index_properties[INVERTED_INDEX_PARSER_KEY] = INVERTED_INDEX_PARSER_STANDARD; + auto index_meta = make_test_inverted_index(13, index_properties); + auto reader = std::make_shared( + &index_meta, nullptr, segment_v2::InvertedIndexReaderType::FULLTEXT); + + segment_v2::InvertedIndexIterator iterator; + iterator.add_reader(segment_v2::InvertedIndexReaderType::FULLTEXT, reader); + + std::unordered_map data_type_with_names; + data_type_with_names.emplace("content", IndexFieldNameAndTypePair {"content", column_type}); + std::unordered_map iterators; + iterators["content"] = &iterator; + + TSearchFieldBinding field_binding; + field_binding.field_name = "content"; + field_binding.index_properties[INVERTED_INDEX_PARSER_KEY] = INVERTED_INDEX_PARSER_ENGLISH; + field_binding.__isset.index_properties = true; + + auto context = std::make_shared(); + FieldReaderResolver resolver(data_type_with_names, iterators, context, {field_binding}); + FieldReaderBinding binding; + return resolver.resolve("content", InvertedIndexQueryType::MATCH_ANY_QUERY, &binding); +} + TEST_F(FunctionSearchTest, TestGetName) { EXPECT_EQ("search", function_search->get_name()); } @@ -1716,6 +1974,7 @@ TEST_F(FunctionSearchTest, TestBuildLeafQueryPhrase) { binding.stored_field_wstr = L"content"; binding.index_properties["parser"] = "unicode"; binding.query_type = InvertedIndexQueryType::MATCH_PHRASE_QUERY; + binding.execution_mode = SearchFieldExecutionMode::CLUCENE; auto* dummy_reader = reinterpret_cast(0x1); binding.lucene_reader = std::shared_ptr( @@ -1736,6 +1995,80 @@ TEST_F(FunctionSearchTest, TestBuildLeafQueryPhrase) { EXPECT_NE(phrase_query, nullptr); } +TEST_F(FunctionSearchTest, TestBuildLeafQueryPhraseUsesPlainTerms) { + auto* exec_env = ExecEnv::GetInstance(); + auto* previous_policy_mgr = exec_env->index_policy_mgr(); + IndexPolicyMgr scoped_policy_mgr; + exec_env->_index_policy_mgr = &scoped_policy_mgr; + DEFER(exec_env->_index_policy_mgr = previous_policy_mgr); + + auto* policy_mgr = exec_env->index_policy_mgr(); + ASSERT_NE(policy_mgr, nullptr); + + TIndexPolicy tokenizer; + tokenizer.id = 910020; + tokenizer.name = "function_search_cg_tokenizer"; + tokenizer.type = TIndexPolicyType::TOKENIZER; + tokenizer.properties["type"] = "char_group"; + tokenizer.properties["tokenize_on_chars"] = "[whitespace]"; + + TIndexPolicy common_grams; + common_grams.id = 910021; + common_grams.name = "function_search_cg_filter"; + common_grams.type = TIndexPolicyType::TOKEN_FILTER; + common_grams.properties["type"] = "common_grams"; + + TIndexPolicy analyzer; + analyzer.id = 910022; + analyzer.name = "function_search_cg_analyzer"; + analyzer.type = TIndexPolicyType::ANALYZER; + analyzer.properties["tokenizer"] = tokenizer.name; + analyzer.properties["token_filter"] = "lowercase," + common_grams.name; + policy_mgr->apply_policy_changes({tokenizer, common_grams, analyzer}, {}); + + TSearchClause clause; + clause.clause_type = "PHRASE"; + clause.field_name = "content"; + clause.value = "man of the year"; + clause.__isset.field_name = true; + clause.__isset.value = true; + + auto context = std::make_shared(); + std::unordered_map data_type_with_names; + data_type_with_names.emplace("content", IndexFieldNameAndTypePair {"content", nullptr}); + std::unordered_map iterators; + FieldReaderResolver resolver(data_type_with_names, iterators, context); + + FieldReaderBinding binding; + binding.logical_field_name = "content"; + binding.stored_field_name = "content"; + binding.stored_field_wstr = L"content"; + binding.index_properties["analyzer"] = analyzer.name; + binding.query_type = InvertedIndexQueryType::MATCH_PHRASE_QUERY; + binding.execution_mode = SearchFieldExecutionMode::CLUCENE; + auto* dummy_reader = reinterpret_cast(0x1); + binding.lucene_reader = std::shared_ptr( + dummy_reader, [](lucene::index::IndexReader* /*ptr*/) {}); + binding.binding_key = + resolver.binding_key_for("content", InvertedIndexQueryType::MATCH_PHRASE_QUERY); + resolver._cache[binding.binding_key] = binding; + + inverted_index::query_v2::QueryPtr query; + std::string binding_key; + ASSERT_TRUE(function_search + ->build_leaf_query(clause, context, resolver, &query, &binding_key, "OR", 0) + .ok()); + + auto phrase = std::dynamic_pointer_cast(query); + ASSERT_NE(phrase, nullptr); + ASSERT_EQ(phrase->_term_infos.size(), 4); + EXPECT_EQ(phrase->_term_infos[0].get_single_term(), "man"); + EXPECT_EQ(phrase->_term_infos[1].get_single_term(), "of"); + EXPECT_EQ(phrase->_term_infos[2].get_single_term(), "the"); + EXPECT_EQ(phrase->_term_infos[3].get_single_term(), "year"); + policy_mgr->apply_policy_changes({}, {tokenizer.id, common_grams.id, analyzer.id}); +} + TEST_F(FunctionSearchTest, TestBuildLeafQueryVariantMissingFieldReturnsUnknown) { TSearchClause clause; clause.clause_type = "TERM"; @@ -1872,6 +2205,64 @@ TEST_F(FunctionSearchTest, EXPECT_EQ(ErrorCode::INVERTED_INDEX_FILE_NOT_FOUND, status.code()); } +TEST_F(FunctionSearchTest, + TestFieldReaderResolverNonVariantStringBindingRejectsMismatchedAnalyzer) { + auto status = resolve_non_variant_binding_with_mismatched_analyzer( + std::make_shared()); + + ASSERT_FALSE(status.ok()); + EXPECT_EQ(ErrorCode::INVERTED_INDEX_BYPASS, status.code()); +} + +TEST_F(FunctionSearchTest, + TestFieldReaderResolverNonVariantArrayStringBindingRejectsMismatchedAnalyzer) { + auto column_type = + std::make_shared(make_nullable(std::make_shared())); + auto status = resolve_non_variant_binding_with_mismatched_analyzer(column_type); + + ASSERT_FALSE(status.ok()); + EXPECT_EQ(ErrorCode::INVERTED_INDEX_BYPASS, status.code()); +} + +TEST_F(FunctionSearchTest, TestFieldReaderResolverExactIgnoresAnalyzedBindingHint) { + std::map analyzed_properties; + analyzed_properties[INVERTED_INDEX_PARSER_KEY] = INVERTED_INDEX_PARSER_STANDARD; + auto analyzed_index = make_test_inverted_index(13, analyzed_properties); + auto keyword_index = make_test_inverted_index(14); + auto index_file_reader = std::make_shared( + nullptr, "/tmp/search_exact_multi_index", InvertedIndexStorageFormatPB::SNII); + auto analyzed_reader = std::make_shared( + &analyzed_index, index_file_reader, segment_v2::InvertedIndexReaderType::FULLTEXT); + auto keyword_reader = std::make_shared( + &keyword_index, index_file_reader, segment_v2::InvertedIndexReaderType::STRING_TYPE); + + segment_v2::InvertedIndexIterator iterator; + iterator.add_reader(segment_v2::InvertedIndexReaderType::FULLTEXT, analyzed_reader); + iterator.add_reader(segment_v2::InvertedIndexReaderType::STRING_TYPE, keyword_reader); + + std::unordered_map data_type_with_names; + data_type_with_names.emplace( + "content", IndexFieldNameAndTypePair {"content", std::make_shared()}); + std::unordered_map iterators; + iterators["content"] = &iterator; + + TSearchFieldBinding field_binding; + field_binding.field_name = "content"; + field_binding.index_properties = analyzed_properties; + field_binding.__isset.index_properties = true; + + auto context = std::make_shared(); + FieldReaderResolver resolver(data_type_with_names, iterators, context, {field_binding}); + FieldReaderBinding binding; + auto status = resolver.resolve("content", InvertedIndexQueryType::EQUAL_QUERY, &binding); + + ASSERT_TRUE(status.ok()) << status; + ASSERT_NE(binding.inverted_reader, nullptr); + EXPECT_EQ(binding.inverted_reader->get_index_id(), 14); + EXPECT_EQ(binding.query_type, InvertedIndexQueryType::EQUAL_QUERY); + EXPECT_TRUE(binding.index_properties.empty()); +} + TEST_F(FunctionSearchTest, TestFieldReaderResolverVariantBkdDirectReader) { auto context = std::make_shared(); @@ -1913,6 +2304,783 @@ TEST_F(FunctionSearchTest, TestFieldReaderResolverVariantBkdDirectReader) { EXPECT_TRUE(cache.begin()->second.use_direct_index_reader()); } +TEST_F(FunctionSearchTest, TestFieldReaderResolverBindsSniiWithoutOpeningClucene) { + auto context = std::make_shared(); + auto index_meta = make_test_inverted_index( + 14, {{INVERTED_INDEX_PARSER_KEY, INVERTED_INDEX_PARSER_STANDARD}}); + auto index_file_reader = std::make_shared(); + auto reader = std::make_shared( + &index_meta, index_file_reader, segment_v2::InvertedIndexReaderType::FULLTEXT); + + segment_v2::InvertedIndexIterator iterator; + iterator.add_reader(segment_v2::InvertedIndexReaderType::FULLTEXT, reader); + + std::unordered_map data_type_with_names; + data_type_with_names.emplace( + "body", IndexFieldNameAndTypePair {"body", std::make_shared()}); + std::unordered_map iterators; + iterators["body"] = &iterator; + + TSearchFieldBinding field_binding; + field_binding.field_name = "body"; + field_binding.index_properties = index_meta.properties(); + field_binding.__isset.index_properties = true; + + FieldReaderResolver resolver(data_type_with_names, iterators, context, {field_binding}); + FieldReaderBinding binding; + auto status = resolver.resolve("body", InvertedIndexQueryType::MATCH_ANY_QUERY, &binding); + + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(0, index_file_reader->init_calls); + EXPECT_EQ(0, index_file_reader->open_calls); + EXPECT_EQ(reader, binding.inverted_reader); + EXPECT_EQ(nullptr, binding.lucene_reader); + EXPECT_TRUE(binding.use_snii_native_reader()); + EXPECT_FALSE(binding.use_direct_index_reader()); + EXPECT_EQ(SearchFieldExecutionMode::SNII_NATIVE, binding.execution_mode); +} + +TEST_F(FunctionSearchTest, TestBuildLeafQueryExecutesSelectedSniiWildcardReader) { + auto context = std::make_shared(); + std::map decoy_properties { + {INVERTED_INDEX_PARSER_KEY, INVERTED_INDEX_PARSER_ENGLISH}, + {INVERTED_INDEX_PARSER_LOWERCASE_KEY, INVERTED_INDEX_PARSER_TRUE}}; + std::map selected_properties { + {INVERTED_INDEX_PARSER_KEY, INVERTED_INDEX_PARSER_STANDARD}, + {INVERTED_INDEX_PARSER_LOWERCASE_KEY, INVERTED_INDEX_PARSER_TRUE}}; + auto decoy_meta = make_test_inverted_index(15, decoy_properties); + auto selected_meta = make_test_inverted_index(16, selected_properties); + auto decoy_file_reader = std::make_shared( + InvertedIndexStorageFormatPB::SNII, "/tmp/search_snii_decoy_idx"); + auto selected_file_reader = std::make_shared( + InvertedIndexStorageFormatPB::SNII, "/tmp/search_snii_selected_idx"); + auto decoy_reader = + std::make_shared(&decoy_meta, decoy_file_reader); + auto selected_reader = std::make_shared( + &selected_meta, selected_file_reader); + selected_reader->set_query_result("*lpha", make_bitmap({0, 2})); + selected_reader->set_null_bitmap(make_bitmap({3})); + + segment_v2::InvertedIndexIterator iterator; + iterator.add_reader(segment_v2::InvertedIndexReaderType::FULLTEXT, decoy_reader); + iterator.add_reader(segment_v2::InvertedIndexReaderType::FULLTEXT, selected_reader); + + std::unordered_map data_type_with_names; + data_type_with_names.emplace( + "body", IndexFieldNameAndTypePair {"stored_body", std::make_shared()}); + std::unordered_map iterators; + iterators["body"] = &iterator; + + TSearchFieldBinding field_binding; + field_binding.field_name = "body"; + field_binding.index_properties = selected_properties; + field_binding.__isset.index_properties = true; + + FieldReaderResolver resolver(data_type_with_names, iterators, context, {field_binding}); + auto clause = make_leaf_clause("WILDCARD", "*LPHA"); + inverted_index::query_v2::QueryPtr query; + std::string binding_key; + auto status = function_search->build_leaf_query(clause, context, resolver, &query, &binding_key, + "OR", 0, 4); + + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_NE(nullptr, query); + EXPECT_EQ(0, decoy_reader->query_calls); + EXPECT_EQ(1, selected_reader->query_calls); + EXPECT_EQ("stored_body", selected_reader->last_column_name); + EXPECT_EQ(TYPE_STRING, selected_reader->last_query_value_type); + EXPECT_EQ("*lpha", selected_reader->last_query_value); + EXPECT_EQ(InvertedIndexQueryType::WILDCARD_QUERY, selected_reader->last_query_type); + EXPECT_EQ(nullptr, selected_reader->last_analyzer_ctx); + EXPECT_EQ(0, decoy_file_reader->open_calls); + EXPECT_EQ(0, selected_file_reader->open_calls); + EXPECT_EQ(0, decoy_reader->null_bitmap_calls); + EXPECT_EQ(1, selected_reader->null_bitmap_calls); + const auto& bindings = resolver.binding_cache(); + ASSERT_EQ(1U, bindings.size()); + EXPECT_EQ(InvertedIndexQueryType::MATCH_ANY_QUERY, bindings.begin()->second.query_type); + + auto weight = query->weight(true); + ASSERT_NE(nullptr, weight); + inverted_index::query_v2::QueryExecutionContext exec_ctx; + exec_ctx.segment_num_rows = 4; + auto scorer = weight->scorer(exec_ctx, binding_key); + ASSERT_NE(nullptr, scorer); + EXPECT_EQ(0U, scorer->doc()); + EXPECT_FLOAT_EQ(1.0F, scorer->score()); + expect_bitmap_eq(collect_docs(scorer), {0, 2}); + ASSERT_TRUE(scorer->has_null_bitmap()); + const auto* null_bitmap = scorer->get_null_bitmap(); + ASSERT_NE(nullptr, null_bitmap); + expect_bitmap_eq(*null_bitmap, {3}); +} + +TEST_F(FunctionSearchTest, TestSniiWildcardPreservesThreeValuedBooleanAndFieldExists) { + auto context = std::make_shared(); + std::map properties { + {INVERTED_INDEX_PARSER_KEY, INVERTED_INDEX_PARSER_STANDARD}}; + auto index_meta = make_test_inverted_index(17, properties); + auto index_file_reader = std::make_shared(); + auto reader = + std::make_shared(&index_meta, index_file_reader); + reader->set_query_result("*lpha", make_bitmap({0})); + reader->set_query_result("beta*", make_bitmap({1})); + reader->set_null_bitmap(make_bitmap({3})); + + segment_v2::InvertedIndexIterator iterator; + iterator.add_reader(segment_v2::InvertedIndexReaderType::FULLTEXT, reader); + std::unordered_map data_type_with_names; + data_type_with_names.emplace( + "body", IndexFieldNameAndTypePair {"body", std::make_shared()}); + std::unordered_map iterators; + iterators["body"] = &iterator; + + TSearchFieldBinding field_binding; + field_binding.field_name = "body"; + field_binding.index_properties = properties; + field_binding.__isset.index_properties = true; + FieldReaderResolver resolver(data_type_with_names, iterators, context, {field_binding}); + + TSearchClause or_clause; + or_clause.clause_type = "OR"; + or_clause.children = {make_leaf_clause("WILDCARD", "*lpha"), + make_leaf_clause("WILDCARD", "beta*")}; + or_clause.__isset.children = true; + + TSearchClause not_clause; + not_clause.clause_type = "NOT"; + not_clause.children = {make_leaf_clause("WILDCARD", "*lpha")}; + not_clause.__isset.children = true; + auto exists_clause = make_leaf_clause("WILDCARD", "*"); + + auto verify_result = [&](const TSearchClause& root, + std::initializer_list expected_docs, + std::initializer_list expected_nulls) { + inverted_index::query_v2::QueryPtr query; + std::string binding_key; + auto status = function_search->build_query_recursive(root, context, resolver, &query, + &binding_key, "OR", 0, 4); + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_NE(nullptr, query); + auto weight = query->weight(false); + ASSERT_NE(nullptr, weight); + auto scorer = weight->scorer( + build_variant_search_query_execution_context(4, resolver, nullptr), binding_key); + ASSERT_NE(nullptr, scorer); + expect_bitmap_eq(collect_docs(scorer), expected_docs); + ASSERT_TRUE(scorer->has_null_bitmap()); + const auto* null_bitmap = scorer->get_null_bitmap(); + ASSERT_NE(nullptr, null_bitmap); + expect_bitmap_eq(*null_bitmap, expected_nulls); + }; + + verify_result(or_clause, {0, 1}, {3}); + verify_result(not_clause, {1, 2}, {3}); + verify_result(exists_clause, {0, 1, 2}, {3}); + EXPECT_EQ(3, reader->query_calls); +} + +// SNII native SEARCH forwards every clause type to the reader as a query type (see +// FunctionSearch::build_leaf_query's SNII branch); it no longer refuses non-WILDCARD clauses. +// A TERM clause maps to EQUAL_QUERY via clause_type_to_query_type and is forwarded unmodified +// (no normalize_wildcard_pattern -- that only applies to WILDCARD). +TEST_F(FunctionSearchTest, TestSniiNativeForwardsTermClauseAsEqualQuery) { + OlapReaderStatistics stats; + auto context = std::make_shared(); + context->stats = &stats; + auto index_meta = make_test_inverted_index( + 18, {{INVERTED_INDEX_PARSER_KEY, INVERTED_INDEX_PARSER_STANDARD}}); + auto index_file_reader = std::make_shared(); + auto reader = + std::make_shared(&index_meta, index_file_reader); + reader->set_query_result("alpha", make_bitmap({0, 2})); + segment_v2::InvertedIndexIterator iterator; + iterator.add_reader(segment_v2::InvertedIndexReaderType::FULLTEXT, reader); + + std::unordered_map data_type_with_names; + data_type_with_names.emplace( + "body", IndexFieldNameAndTypePair {"body", std::make_shared()}); + std::unordered_map iterators; + iterators["body"] = &iterator; + TSearchFieldBinding field_binding; + field_binding.field_name = "body"; + field_binding.index_properties = index_meta.properties(); + field_binding.__isset.index_properties = true; + FieldReaderResolver resolver(data_type_with_names, iterators, context, {field_binding}); + + auto clause = make_leaf_clause("TERM", "alpha"); + inverted_index::query_v2::QueryPtr query; + std::string binding_key; + auto status = function_search->build_leaf_query(clause, context, resolver, &query, &binding_key, + "OR", 0, 4); + + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_NE(nullptr, query); + EXPECT_EQ(1, reader->query_calls); + EXPECT_EQ(InvertedIndexQueryType::EQUAL_QUERY, reader->last_query_type); + EXPECT_EQ("alpha", reader->last_query_value); + EXPECT_EQ(0, index_file_reader->open_calls); + + auto weight = query->weight(false); + ASSERT_NE(nullptr, weight); + inverted_index::query_v2::QueryExecutionContext exec_ctx; + exec_ctx.segment_num_rows = 4; + auto scorer = weight->scorer(exec_ctx, binding_key); + ASSERT_NE(nullptr, scorer); + expect_bitmap_eq(collect_docs(scorer), {0, 2}); +} + +// default_operator "and" maps a multi-token TERM clause onto MATCH_ALL_QUERY instead of the +// default EQUAL_QUERY (which is an OR of terms) -- SNII has no boolean query tree to build, so +// this is expressed entirely as which query type gets forwarded to the reader. +TEST_F(FunctionSearchTest, TestSniiNativeTermDefaultOperatorAndMapsToMatchAllQuery) { + OlapReaderStatistics stats; + auto context = std::make_shared(); + context->stats = &stats; + auto index_meta = make_test_inverted_index( + 21, {{INVERTED_INDEX_PARSER_KEY, INVERTED_INDEX_PARSER_STANDARD}}); + auto index_file_reader = std::make_shared(); + auto reader = + std::make_shared(&index_meta, index_file_reader); + segment_v2::InvertedIndexIterator iterator; + iterator.add_reader(segment_v2::InvertedIndexReaderType::FULLTEXT, reader); + + std::unordered_map data_type_with_names; + data_type_with_names.emplace( + "body", IndexFieldNameAndTypePair {"body", std::make_shared()}); + std::unordered_map iterators; + iterators["body"] = &iterator; + TSearchFieldBinding field_binding; + field_binding.field_name = "body"; + field_binding.index_properties = index_meta.properties(); + field_binding.__isset.index_properties = true; + FieldReaderResolver resolver(data_type_with_names, iterators, context, {field_binding}); + + auto clause = make_leaf_clause("TERM", "alpha beta"); + inverted_index::query_v2::QueryPtr query; + std::string binding_key; + auto status = function_search->build_leaf_query(clause, context, resolver, &query, &binding_key, + "and", 0, 4); + + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(1, reader->query_calls); + EXPECT_EQ(InvertedIndexQueryType::MATCH_ALL_QUERY, reader->last_query_type); + EXPECT_EQ("alpha beta", reader->last_query_value); +} + +// minimum_should_match ("at least N of M terms") has no SNII query type -- EQUAL_QUERY is +// unconditionally ANY and MATCH_ALL_QUERY is unconditionally ALL, with nothing in between. SNII +// must refuse it explicitly for a TERM clause instead of silently answering a plain OR query. +TEST_F(FunctionSearchTest, TestSniiNativeTermRejectsMinimumShouldMatch) { + auto context = std::make_shared(); + auto index_meta = make_test_inverted_index( + 22, {{INVERTED_INDEX_PARSER_KEY, INVERTED_INDEX_PARSER_STANDARD}}); + auto index_file_reader = std::make_shared(); + auto reader = + std::make_shared(&index_meta, index_file_reader); + segment_v2::InvertedIndexIterator iterator; + iterator.add_reader(segment_v2::InvertedIndexReaderType::FULLTEXT, reader); + + std::unordered_map data_type_with_names; + data_type_with_names.emplace( + "body", IndexFieldNameAndTypePair {"body", std::make_shared()}); + std::unordered_map iterators; + iterators["body"] = &iterator; + TSearchFieldBinding field_binding; + field_binding.field_name = "body"; + field_binding.index_properties = index_meta.properties(); + field_binding.__isset.index_properties = true; + FieldReaderResolver resolver(data_type_with_names, iterators, context, {field_binding}); + + auto clause = make_leaf_clause("TERM", "alpha beta"); + inverted_index::query_v2::QueryPtr query; + std::string binding_key; + auto status = function_search->build_leaf_query(clause, context, resolver, &query, &binding_key, + "OR", 2, 4); + + ASSERT_FALSE(status.ok()); + EXPECT_EQ(ErrorCode::NOT_IMPLEMENTED_ERROR, status.code()); + EXPECT_NE(std::string::npos, status.to_string().find("minimum_should_match")); + EXPECT_EQ(0, reader->query_calls); + EXPECT_EQ(0, index_file_reader->open_calls); +} + +// A single-token TERM value has nothing for minimum_should_match to select "at least N of" +// among -- there is only one term. The CLucene path (function_search.cpp, `term_infos.size() == +// 1` branch) never even looks at minimum_should_match in that case and answers a plain +// TermQuery; SNII must do the same instead of hard-refusing every analysed TERM clause the +// instant msm is set, regardless of how many tokens the value actually produces. +TEST_F(FunctionSearchTest, TestSniiNativeTermSingleTokenAllowsMinimumShouldMatch) { + auto context = std::make_shared(); + auto index_meta = make_test_inverted_index( + 24, {{INVERTED_INDEX_PARSER_KEY, INVERTED_INDEX_PARSER_STANDARD}}); + auto index_file_reader = std::make_shared(); + auto reader = + std::make_shared(&index_meta, index_file_reader); + reader->set_query_result("alpha", make_bitmap({0, 2})); + segment_v2::InvertedIndexIterator iterator; + iterator.add_reader(segment_v2::InvertedIndexReaderType::FULLTEXT, reader); + + std::unordered_map data_type_with_names; + data_type_with_names.emplace( + "body", IndexFieldNameAndTypePair {"body", std::make_shared()}); + std::unordered_map iterators; + iterators["body"] = &iterator; + TSearchFieldBinding field_binding; + field_binding.field_name = "body"; + field_binding.index_properties = index_meta.properties(); + field_binding.__isset.index_properties = true; + FieldReaderResolver resolver(data_type_with_names, iterators, context, {field_binding}); + + auto clause = make_leaf_clause("TERM", "alpha"); + inverted_index::query_v2::QueryPtr query; + std::string binding_key; + auto status = function_search->build_leaf_query(clause, context, resolver, &query, &binding_key, + "OR", 1, 4); + + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_NE(nullptr, query); + EXPECT_EQ(1, reader->query_calls); + EXPECT_EQ(InvertedIndexQueryType::EQUAL_QUERY, reader->last_query_type); + EXPECT_EQ("alpha", reader->last_query_value); + + auto weight = query->weight(false); + ASSERT_NE(nullptr, weight); + inverted_index::query_v2::QueryExecutionContext exec_ctx; + exec_ctx.segment_num_rows = 4; + auto scorer = weight->scorer(exec_ctx, binding_key); + ASSERT_NE(nullptr, scorer); + expect_bitmap_eq(collect_docs(scorer), {0, 2}); +} + +// A value that tokenizes to zero terms (here, an empty string on an analysed field) must be +// handled the same way the CLucene path handles it -- an empty BitSetQuery -- instead of +// reaching SniiIndexReader::_query at all: that reader only short-circuits empty term_infos to +// an empty bitmap for proper MATCH_* query types (see is_match_query() in +// inverted_index_query_type.h), and a TERM clause maps to EQUAL_QUERY/MATCH_ALL_QUERY, neither +// of which qualifies, so it would otherwise surface INVERTED_INDEX_NO_TERMS instead of a match. +TEST_F(FunctionSearchTest, TestSniiNativeTermZeroTokenMinimumShouldMatchReturnsEmptyBitmap) { + auto context = std::make_shared(); + auto index_meta = make_test_inverted_index( + 25, {{INVERTED_INDEX_PARSER_KEY, INVERTED_INDEX_PARSER_STANDARD}}); + auto index_file_reader = std::make_shared(); + auto reader = + std::make_shared(&index_meta, index_file_reader); + segment_v2::InvertedIndexIterator iterator; + iterator.add_reader(segment_v2::InvertedIndexReaderType::FULLTEXT, reader); + + std::unordered_map data_type_with_names; + data_type_with_names.emplace( + "body", IndexFieldNameAndTypePair {"body", std::make_shared()}); + std::unordered_map iterators; + iterators["body"] = &iterator; + TSearchFieldBinding field_binding; + field_binding.field_name = "body"; + field_binding.index_properties = index_meta.properties(); + field_binding.__isset.index_properties = true; + FieldReaderResolver resolver(data_type_with_names, iterators, context, {field_binding}); + + auto clause = make_leaf_clause("TERM", ""); + inverted_index::query_v2::QueryPtr query; + std::string binding_key; + auto status = function_search->build_leaf_query(clause, context, resolver, &query, &binding_key, + "OR", 1, 4); + + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_NE(nullptr, query); + EXPECT_EQ(0, reader->query_calls); + + auto weight = query->weight(false); + ASSERT_NE(nullptr, weight); + inverted_index::query_v2::QueryExecutionContext exec_ctx; + exec_ctx.segment_num_rows = 4; + auto scorer = weight->scorer(exec_ctx, binding_key); + ASSERT_NE(nullptr, scorer); + expect_bitmap_eq(collect_docs(scorer), {}); +} + +// On a NON-analysed (keyword) field, PREFIX cannot map to MATCH_PHRASE_PREFIX_QUERY: FE keeps +// the trailing '*' in the value (SearchDslParser.java), and on a keyword field the whole string +// -- '*' included -- becomes one literal term (InvertedIndexAnalyzer::get_analyse_result), so +// MATCH_PHRASE_PREFIX_QUERY would search for a term that can never exist. build_leaf_query's SNII +// branch (function_search.cpp:819-832) special-cases this by checking +// !InvertedIndexAnalyzer::should_analyzer(binding.index_properties) and routing to WILDCARD_QUERY +// instead, exactly like the CLucene path's WildcardQuery(value) for PREFIX. index_meta below omits +// the parser property entirely, which should_analyzer() (analyzer.cpp:261-273) treats as +// PARSER_UNKNOWN -- not analysed -- the same as an explicit "none" parser. +TEST_F(FunctionSearchTest, TestSniiNativeKeywordPrefixRoutesToWildcardQuery) { + OlapReaderStatistics stats; + auto context = std::make_shared(); + context->stats = &stats; + auto index_meta = make_test_inverted_index(23); + auto index_file_reader = std::make_shared(); + auto reader = + std::make_shared(&index_meta, index_file_reader); + reader->set_query_result("al*", make_bitmap({0, 2})); + segment_v2::InvertedIndexIterator iterator; + iterator.add_reader(segment_v2::InvertedIndexReaderType::FULLTEXT, reader); + + std::unordered_map data_type_with_names; + data_type_with_names.emplace( + "body", IndexFieldNameAndTypePair {"body", std::make_shared()}); + std::unordered_map iterators; + iterators["body"] = &iterator; + TSearchFieldBinding field_binding; + field_binding.field_name = "body"; + field_binding.index_properties = index_meta.properties(); + field_binding.__isset.index_properties = true; + FieldReaderResolver resolver(data_type_with_names, iterators, context, {field_binding}); + + auto clause = make_leaf_clause("PREFIX", "al*"); + inverted_index::query_v2::QueryPtr query; + std::string binding_key; + auto status = function_search->build_leaf_query(clause, context, resolver, &query, &binding_key, + "OR", 0, 4); + + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_NE(nullptr, query); + EXPECT_EQ(1, reader->query_calls); + EXPECT_EQ(InvertedIndexQueryType::WILDCARD_QUERY, reader->last_query_type); + // The trailing '*' must survive unmodified: WILDCARD_QUERY on the reader interprets it as a + // wildcard, unlike the tokenizer path that would have stripped it. + EXPECT_EQ("al*", reader->last_query_value); + EXPECT_EQ(0, index_file_reader->open_calls); + + auto weight = query->weight(false); + ASSERT_NE(nullptr, weight); + inverted_index::query_v2::QueryExecutionContext exec_ctx; + exec_ctx.segment_num_rows = 4; + auto scorer = weight->scorer(exec_ctx, binding_key); + ASSERT_NE(nullptr, scorer); + expect_bitmap_eq(collect_docs(scorer), {0, 2}); +} + +// Shared wiring for the SNII native SEARCH scoring tests: one fake SNII reader bound to field +// "body" behind a standard analyzer, plus the resolver build_leaf_query needs. The resolver keeps +// references to the maps, so they must be owned by something that outlives it. +class SniiScoringFixture { +public: + SniiScoringFixture(int64_t index_id, uint32_t rows) : num_rows(rows) { + // support_phrase is what makes is_need_similarity_score accept a MATCH query type, which + // is the production gate the leaf builder consults before wiring up a score sink. + _properties = {{INVERTED_INDEX_PARSER_KEY, INVERTED_INDEX_PARSER_STANDARD}, + {INVERTED_INDEX_PARSER_PHRASE_SUPPORT_KEY, + INVERTED_INDEX_PARSER_PHRASE_SUPPORT_YES}}; + _index_meta = make_test_inverted_index(index_id, _properties); + _index_file_reader = std::make_shared(); + reader = std::make_shared(&_index_meta, + _index_file_reader); + context = std::make_shared(); + context->stats = &_stats; + context->collection_similarity = std::make_shared(); + _iterator.add_reader(segment_v2::InvertedIndexReaderType::FULLTEXT, reader); + _data_type_with_names.emplace( + "body", IndexFieldNameAndTypePair {"body", std::make_shared()}); + _iterators["body"] = &_iterator; + + TSearchFieldBinding field_binding; + field_binding.field_name = "body"; + field_binding.index_properties = _properties; + field_binding.__isset.index_properties = true; + resolver = std::make_unique( + _data_type_with_names, _iterators, context, + std::vector {field_binding}); + } + + SniiScoringFixture(const SniiScoringFixture&) = delete; + SniiScoringFixture& operator=(const SniiScoringFixture&) = delete; + + inverted_index::query_v2::QueryExecutionContext exec_context() const { + return build_variant_search_query_execution_context(num_rows, *resolver, nullptr); + } + + uint32_t num_rows; + std::shared_ptr reader; + std::shared_ptr context; + std::unique_ptr resolver; + +private: + OlapReaderStatistics _stats; + std::map _properties; + TabletIndex _index_meta; + std::shared_ptr _index_file_reader; + segment_v2::InvertedIndexIterator _iterator; + std::unordered_map _data_type_with_names; + std::unordered_map _iterators; +}; + +// Reads back what a CollectionSimilarity actually holds for the given documents, so a test can +// assert on the score values themselves rather than only on which rows survived. +static std::map read_collected_scores(const CollectionSimilarity& similarity, + const roaring::Roaring& docs) { + roaring::Roaring row_bitmap = docs; + IColumn::MutablePtr score_column; + auto row_ids = std::make_unique>(); + similarity.get_bm25_scores(&row_bitmap, score_column, row_ids); + const auto& nullable = assert_cast(*score_column); + const auto& values = assert_cast(nullable.get_nested_column()).get_data(); + std::map collected; + for (size_t i = 0; i < row_ids->size(); ++i) { + collected[static_cast((*row_ids)[i])] = values[i]; + } + return collected; +} + +// A SEARCH answered by the SNII native reader must rank by the reader's own BM25 values. The +// leaf used to be wrapped in a plain BitSetQuery, whose scorer returns a constant 1.0 for every +// document, so the early top-k collector saw an all-tie ranking and "ORDER BY score() DESC LIMIT +// k" returned an arbitrary k rows instead of the k best-scoring ones. +TEST_F(FunctionSearchTest, TestSniiNativeTopKRanksByReaderBm25Scores) { + // enable_inverted_index_wand_query defaults to true and function_search passes it straight + // through, so the wand variant is the one that actually ships; the non-wand variant is what + // an explicitly disabled session gets. Both must rank the same. + for (bool use_wand : {false, true}) { + SCOPED_TRACE(use_wand ? "use_wand=true" : "use_wand=false"); + SniiScoringFixture fixture(41, 5); + fixture.reader->set_query_result("alpha", make_bitmap({0, 1, 2, 3, 4})); + // Deliberately not monotonic in doc id: the two best documents are 1 and 3, which are not + // the two a doc-id-ordered tie-break would pick. + fixture.reader->set_query_scores("alpha", + {{0, 1.0F}, {1, 9.0F}, {2, 3.0F}, {3, 7.0F}, {4, 5.0F}}); + + inverted_index::query_v2::QueryPtr query; + std::string binding_key; + auto status = function_search->build_leaf_query(make_leaf_clause("MATCH", "alpha"), + fixture.context, *fixture.resolver, &query, + &binding_key, "OR", 0, fixture.num_rows); + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_NE(nullptr, query); + + auto weight = query->weight(true); + ASSERT_NE(nullptr, weight); + auto exec_ctx = fixture.exec_context(); + auto roaring = std::make_shared(); + inverted_index::query_v2::collect_multi_segment_top_k( + weight, exec_ctx, binding_key, 2, roaring, fixture.context->collection_similarity, + use_wand); + + expect_bitmap_eq(*roaring, {1, 3}); + auto collected = read_collected_scores(*fixture.context->collection_similarity, *roaring); + ASSERT_EQ(2U, collected.size()); + EXPECT_FLOAT_EQ(9.0F, collected[1]); + EXPECT_FLOAT_EQ(7.0F, collected[3]); + } +} + +// The reader used to publish its BM25 values straight into the query's collection similarity +// while the collector added the scorer's constant on top of them, so every document ended up +// with "BM25 + 1.0". Exactly one of the two channels may write. +TEST_F(FunctionSearchTest, TestSniiNativeDocSetCollectionScoresEachDocumentOnce) { + SniiScoringFixture fixture(42, 3); + fixture.reader->set_query_result("alpha", make_bitmap({0, 1, 2})); + fixture.reader->set_query_scores("alpha", {{0, 2.5F}, {1, 4.25F}, {2, 0.75F}}); + + inverted_index::query_v2::QueryPtr query; + std::string binding_key; + auto status = function_search->build_leaf_query(make_leaf_clause("MATCH", "alpha"), + fixture.context, *fixture.resolver, &query, + &binding_key, "OR", 0, fixture.num_rows); + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_NE(nullptr, query); + // The reader must have been handed a private sink, never the similarity the collector fills. + EXPECT_NE(fixture.context->collection_similarity.get(), fixture.reader->observed_similarity); + + auto weight = query->weight(true); + ASSERT_NE(nullptr, weight); + auto exec_ctx = fixture.exec_context(); + auto roaring = std::make_shared(); + inverted_index::query_v2::collect_multi_segment_doc_set(weight, exec_ctx, binding_key, roaring, + fixture.context->collection_similarity, + /*enable_scoring=*/true); + + expect_bitmap_eq(*roaring, {0, 1, 2}); + auto collected = read_collected_scores(*fixture.context->collection_similarity, *roaring); + ASSERT_EQ(3U, collected.size()); + EXPECT_FLOAT_EQ(2.5F, collected[0]); + EXPECT_FLOAT_EQ(4.25F, collected[1]); + EXPECT_FLOAT_EQ(0.75F, collected[2]); +} + +// A leaf for which the reader publishes no per-document score must keep the constant that the +// CLucene path also gives its own constant-score leaves, so that routing scored clauses through a +// new scorer does not quietly re-rank the unscored ones. The fake reader is what withholds the +// scores here, so this pins the "nothing published -> BitSetQuery -> 1.0" behaviour; it does not +// pin which query types the production is_need_similarity_score gate rejects. +TEST_F(FunctionSearchTest, TestSniiNativeLeafWithoutPublishedScoresKeepsConstantScore) { + SniiScoringFixture fixture(43, 4); + fixture.reader->set_query_result("alpha*", make_bitmap({0, 2})); + + inverted_index::query_v2::QueryPtr query; + std::string binding_key; + auto status = function_search->build_leaf_query(make_leaf_clause("WILDCARD", "alpha*"), + fixture.context, *fixture.resolver, &query, + &binding_key, "OR", 0, fixture.num_rows); + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_NE(nullptr, query); + + auto weight = query->weight(true); + ASSERT_NE(nullptr, weight); + auto exec_ctx = fixture.exec_context(); + auto scorer = weight->scorer(exec_ctx, binding_key); + ASSERT_NE(nullptr, scorer); + EXPECT_EQ(0U, scorer->doc()); + EXPECT_FLOAT_EQ(1.0F, scorer->score()); + expect_bitmap_eq(collect_docs(scorer), {0, 2}); +} + +// A non-scoring execution must not pay for the score plumbing: weight(false) has to hand back the +// same constant-score scorer the unscored path always used. +TEST_F(FunctionSearchTest, TestSniiNativeScoredQueryFallsBackToConstantScorerWithoutScoring) { + SniiScoringFixture fixture(44, 3); + fixture.reader->set_query_result("alpha", make_bitmap({0, 1, 2})); + fixture.reader->set_query_scores("alpha", {{0, 2.5F}, {1, 4.25F}, {2, 0.75F}}); + + inverted_index::query_v2::QueryPtr query; + std::string binding_key; + auto status = function_search->build_leaf_query(make_leaf_clause("MATCH", "alpha"), + fixture.context, *fixture.resolver, &query, + &binding_key, "OR", 0, fixture.num_rows); + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_NE(nullptr, query); + + auto weight = query->weight(false); + ASSERT_NE(nullptr, weight); + auto exec_ctx = fixture.exec_context(); + auto scorer = weight->scorer(exec_ctx, binding_key); + ASSERT_NE(nullptr, scorer); + EXPECT_EQ(0U, scorer->doc()); + EXPECT_FLOAT_EQ(1.0F, scorer->score()); + expect_bitmap_eq(collect_docs(scorer), {0, 1, 2}); +} + +TEST_F(FunctionSearchTest, TestSearchDslCacheIsDisabledForSniiNativeExecution) { + ScopedInvertedIndexQueryCache cache_guard; + auto index_meta = make_test_inverted_index( + 19, {{INVERTED_INDEX_PARSER_KEY, INVERTED_INDEX_PARSER_STANDARD}}); + auto index_file_reader = std::make_shared(); + auto reader = + std::make_shared(&index_meta, index_file_reader); + reader->set_query_result("*lpha", make_bitmap({0})); + segment_v2::InvertedIndexIterator iterator; + iterator.add_reader(segment_v2::InvertedIndexReaderType::FULLTEXT, reader); + + std::unordered_map data_type_with_names; + data_type_with_names.emplace( + "body", IndexFieldNameAndTypePair {"body", std::make_shared()}); + std::unordered_map iterators; + iterators["body"] = &iterator; + + TSearchFieldBinding field_binding; + field_binding.field_name = "body"; + field_binding.index_properties = index_meta.properties(); + field_binding.__isset.index_properties = true; + TSearchParam search_param; + search_param.original_dsl = "body:*lpha"; + search_param.root = make_leaf_clause("WILDCARD", "*lpha"); + search_param.field_bindings = {field_binding}; + ASSERT_TRUE(insert_search_dsl_cache(cache_guard.get(), index_file_reader, search_param, + make_bitmap({3})) + .ok()); + + InvertedIndexResultBitmap result; + auto status = function_search->evaluate_inverted_index_with_search_param( + search_param, data_type_with_names, iterators, 4, result, true); + + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_NE(nullptr, result.get_data_bitmap()); + expect_bitmap_eq(*result.get_data_bitmap(), {0}); + EXPECT_EQ(1, reader->query_calls); +} + +TEST_F(FunctionSearchTest, TestSearchDslCacheIsDisabledWhenScoring) { + ScopedInvertedIndexQueryCache cache_guard; + auto index_meta = make_test_inverted_index( + 20, {{INVERTED_INDEX_PARSER_KEY, INVERTED_INDEX_PARSER_STANDARD}}); + auto index_file_reader = std::make_shared( + InvertedIndexStorageFormatPB::V2, "/tmp/search_scoring_v2_idx"); + auto reader = std::make_shared( + &index_meta, index_file_reader, segment_v2::InvertedIndexReaderType::FULLTEXT); + segment_v2::InvertedIndexIterator iterator; + iterator.add_reader(segment_v2::InvertedIndexReaderType::FULLTEXT, reader); + + std::unordered_map data_type_with_names; + data_type_with_names.emplace( + "body", IndexFieldNameAndTypePair {"body", std::make_shared()}); + std::unordered_map iterators; + iterators["body"] = &iterator; + TSearchFieldBinding field_binding; + field_binding.field_name = "body"; + field_binding.index_properties = index_meta.properties(); + field_binding.__isset.index_properties = true; + TSearchParam search_param; + search_param.original_dsl = "body:alpha"; + search_param.root = make_leaf_clause("TERM", "alpha"); + search_param.field_bindings = {field_binding}; + ASSERT_TRUE(insert_search_dsl_cache(cache_guard.get(), index_file_reader, search_param, + make_bitmap({3})) + .ok()); + + auto scoring_context = std::make_shared(); + scoring_context->collection_similarity = std::make_shared(); + InvertedIndexResultBitmap result; + std::unordered_map field_name_to_column_id; + auto status = function_search->evaluate_inverted_index_with_search_param( + search_param, data_type_with_names, iterators, 4, result, true, nullptr, + field_name_to_column_id, scoring_context); + + EXPECT_FALSE(status.ok()); + EXPECT_EQ(1, index_file_reader->init_calls); + EXPECT_EQ(1, index_file_reader->open_calls); +} + +TEST_F(FunctionSearchTest, TestSearchDslCacheRemainsEnabledForUnreferencedSniiField) { + ScopedInvertedIndexQueryCache cache_guard; + auto text_index_meta = make_test_inverted_index( + 21, {{INVERTED_INDEX_PARSER_KEY, INVERTED_INDEX_PARSER_STANDARD}}); + auto number_index_meta = make_test_inverted_index(22); + auto text_file_reader = std::make_shared( + InvertedIndexStorageFormatPB::SNII, "/tmp/search_mixed_cache_idx"); + auto number_file_reader = std::make_shared( + InvertedIndexStorageFormatPB::V2, "/tmp/search_mixed_cache_idx"); + auto text_reader = std::make_shared( + &text_index_meta, text_file_reader, InvertedIndexReaderType::FULLTEXT); + auto number_reader = std::make_shared( + &number_index_meta, number_file_reader, InvertedIndexReaderType::BKD); + + InvertedIndexIterator text_iterator; + text_iterator.add_reader(InvertedIndexReaderType::FULLTEXT, text_reader); + RecordingDirectInvertedIndexIterator number_iterator; + number_iterator.add_reader(InvertedIndexReaderType::BKD, number_reader); + + std::unordered_map data_type_with_names; + data_type_with_names.emplace( + "body", IndexFieldNameAndTypePair {"body", std::make_shared()}); + data_type_with_names.emplace( + "age", IndexFieldNameAndTypePair {"age", std::make_shared()}); + std::unordered_map iterators; + iterators["body"] = &text_iterator; + iterators["age"] = &number_iterator; + + TSearchClause age_clause = make_leaf_clause("TERM", "42"); + age_clause.field_name = "age"; + TSearchParam search_param; + search_param.original_dsl = "age:42"; + search_param.root = age_clause; + ASSERT_TRUE(insert_search_dsl_cache(cache_guard.get(), number_file_reader, search_param, + make_bitmap({1})) + .ok()); + + InvertedIndexResultBitmap result; + auto status = function_search->evaluate_inverted_index_with_search_param( + search_param, data_type_with_names, iterators, 4, result, true); + + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_NE(nullptr, result.get_data_bitmap()); + expect_bitmap_eq(*result.get_data_bitmap(), {1}); + EXPECT_EQ(0, text_reader->query_calls); + EXPECT_EQ(0, number_iterator.read_calls); +} + TEST_F(FunctionSearchTest, TestBuildLeafQueryDirectUnknownClauseUsesLeafMapper) { TSearchClause clause; clause.clause_type = "PHRASE"; @@ -1942,6 +3110,7 @@ TEST_F(FunctionSearchTest, TestBuildLeafQueryDirectUnknownClauseUsesLeafMapper) binding.column_type = bool_type; binding.query_type = InvertedIndexQueryType::MATCH_PHRASE_QUERY; binding.state = SearchFieldBindingState::BOUND; + binding.execution_mode = SearchFieldExecutionMode::DIRECT_INDEX; TabletIndex index_meta; binding.inverted_reader = std::make_shared(&index_meta); @@ -2012,6 +3181,7 @@ TEST_F(FunctionSearchTest, TestBuildLeafQueryVariantBoolUsesDirectIndexReader) { binding.column_type = bool_type; binding.query_type = InvertedIndexQueryType::MATCH_ANY_QUERY; binding.state = SearchFieldBindingState::BOUND; + binding.execution_mode = SearchFieldExecutionMode::DIRECT_INDEX; TabletIndex index_meta; binding.inverted_reader = std::make_shared(&index_meta); @@ -2071,6 +3241,7 @@ TEST_F(FunctionSearchTest, TestBuildLeafQueryVariantNestedIntUsesDirectIndexRead binding.column_type = int_type; binding.query_type = InvertedIndexQueryType::MATCH_ANY_QUERY; binding.state = SearchFieldBindingState::BOUND; + binding.execution_mode = SearchFieldExecutionMode::DIRECT_INDEX; TabletIndex index_meta; binding.inverted_reader = std::make_shared(&index_meta); diff --git a/be/test/exprs/vmatch_predicate_test.cpp b/be/test/exprs/vmatch_predicate_test.cpp new file mode 100644 index 00000000000000..0fa08252090b45 --- /dev/null +++ b/be/test/exprs/vmatch_predicate_test.cpp @@ -0,0 +1,85 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "exprs/vmatch_predicate.h" + +#include + +#include +#include + +#include "core/data_type/primitive_type.h" + +namespace doris { +namespace { + +TExprNode make_match_node(const std::string& analyzer_name, const std::string& parser_type) { + TMatchPredicate match_predicate; + match_predicate.__set_analyzer_name(analyzer_name); + match_predicate.__set_parser_type(parser_type); + match_predicate.__set_parser_mode(""); + + TExprNode node; + node.__set_node_type(TExprNodeType::MATCH_PRED); + node.__set_type(create_type_desc(PrimitiveType::TYPE_BOOLEAN)); + node.__set_num_children(2); + node.__set_match_predicate(match_predicate); + return node; +} + +TEST(VMatchPredicateTest, ExplicitNoneKeepsSelectionKeyWithoutAnalyzer) { + auto predicate = VMatchPredicate::create_shared(make_match_node("none", "english")); + const auto* analyzer_ctx = predicate->query_analyzer_ctx(); + + EXPECT_EQ(predicate->get_analyzer_key(), "none"); + EXPECT_FALSE(analyzer_ctx->requires_analysis()); + EXPECT_EQ(analyzer_ctx->analyzer, nullptr); + EXPECT_EQ(analyzer_ctx->analyzer_provider, nullptr); +} + +TEST(VMatchPredicateTest, ResolvesBuiltinAndFallbackExecutionModes) { + struct TestCase { + std::string analyzer_name; + std::string parser_type; + std::string expected_key; + InvertedIndexParserType expected_parser; + bool requires_analysis; + }; + const std::array test_cases { + TestCase {"english", "chinese", "english", InvertedIndexParserType::PARSER_ENGLISH, + true}, + TestCase {"", "standard", "", InvertedIndexParserType::PARSER_STANDARD, true}, + TestCase {"", "unknown", "", InvertedIndexParserType::PARSER_UNKNOWN, true}, + TestCase {"", "", "", InvertedIndexParserType::PARSER_UNKNOWN, true}, + }; + + for (const auto& test_case : test_cases) { + auto predicate = VMatchPredicate::create_shared( + make_match_node(test_case.analyzer_name, test_case.parser_type)); + const auto* analyzer_ctx = predicate->query_analyzer_ctx(); + + EXPECT_EQ(predicate->get_analyzer_key(), test_case.expected_key); + EXPECT_TRUE(analyzer_ctx->analyzer_name.empty()); + EXPECT_EQ(analyzer_ctx->parser_type, test_case.expected_parser); + EXPECT_EQ(analyzer_ctx->requires_analysis(), test_case.requires_analysis); + EXPECT_EQ(analyzer_ctx->analyzer != nullptr, test_case.requires_analysis); + EXPECT_EQ(analyzer_ctx->analyzer_provider != nullptr, test_case.requires_analysis); + } +} + +} // namespace +} // namespace doris diff --git a/be/test/exprs/vsearch_expr_test.cpp b/be/test/exprs/vsearch_expr_test.cpp index 26316dbcbac4f5..9e19f6c1f626e0 100644 --- a/be/test/exprs/vsearch_expr_test.cpp +++ b/be/test/exprs/vsearch_expr_test.cpp @@ -33,7 +33,11 @@ #include "exprs/vexpr_context.h" #include "exprs/vliteral.h" #include "exprs/vsearch.h" +#include "storage/index/index_file_reader.h" #include "storage/index/index_iterator.h" +#include "storage/index/inverted/inverted_index_iterator.h" +#include "storage/index/inverted/inverted_index_parser.h" +#include "storage/index/inverted/inverted_index_reader.h" #include "storage/segment/variant/nested_group_provider.h" #include "storage/tablet/tablet_schema.h" @@ -46,6 +50,7 @@ #if defined(__clang__) #pragma clang diagnostic pop #endif +#include "exprs/vcompound_pred.h" namespace doris { @@ -541,7 +546,7 @@ TEST_F(VSearchExprTest, TestEvaluateInvertedIndexEmptyDSL) { VExprContext context(dummy_expr); auto status = vsearch_expr->evaluate_inverted_index(&context, 100); EXPECT_FALSE(status.ok()); - EXPECT_TRUE(status.code() == ErrorCode::INVALID_ARGUMENT); + EXPECT_EQ(ErrorCode::INVALID_ARGUMENT, status.code()); EXPECT_TRUE(status.to_string().find("search DSL is empty") != std::string::npos); } @@ -1293,6 +1298,7 @@ TEST_F(VSearchExprTest, TestEvaluateInvertedIndexWithEmptyDSL) { auto status = vsearch_expr->evaluate_inverted_index(&context, 100); EXPECT_FALSE(status.ok()); // Should return error due to empty DSL EXPECT_EQ(status.code(), ErrorCode::INVALID_ARGUMENT); + EXPECT_NE(status.to_string().find("search DSL is empty"), std::string::npos); } TEST_F(VSearchExprTest, FastExecuteReturnsPrecomputedColumn) { @@ -1335,6 +1341,7 @@ TEST_F(VSearchExprTest, EvaluateInvertedIndexFailsWithoutStorageType) { auto status = expr->evaluate_inverted_index(context.get(), 128); EXPECT_FALSE(status.ok()); EXPECT_EQ(ErrorCode::INTERNAL_ERROR, status.code()); + EXPECT_NE(status.to_string().find("storage_name_type not found"), std::string::npos); } TEST_F(VSearchExprTest, EvaluateInvertedIndexWithUnsupportedChildReturnsError) { @@ -1353,6 +1360,20 @@ TEST_F(VSearchExprTest, EvaluateInvertedIndexWithUnsupportedChildReturnsError) { auto status = expr->evaluate_inverted_index(context.get(), 64); EXPECT_FALSE(status.ok()); EXPECT_EQ(ErrorCode::INVALID_ARGUMENT, status.code()); + EXPECT_NE(status.to_string().find("Unsupported child node type"), std::string::npos); + + TExprNode compound_node; + compound_node.__set_type(test_node.type); + compound_node.__set_node_type(TExprNodeType::COMPOUND_PRED); + compound_node.__set_opcode(TExprOpcode::COMPOUND_AND); + compound_node.__set_num_children(1); + compound_node.__set_is_nullable(false); + auto compound = VCompoundPred::create_shared(compound_node); + compound->add_child(expr); + status = compound->evaluate_inverted_index(context.get(), 64); + EXPECT_FALSE(status.ok()); + EXPECT_EQ(ErrorCode::INVALID_ARGUMENT, status.code()); + EXPECT_NE(status.to_string().find("Unsupported child node type"), std::string::npos); } TEST_F(VSearchExprTest, EvaluateInvertedIndexHandlesMissingIterators) { @@ -1452,7 +1473,7 @@ TEST_F(VSearchExprTest, EvaluateInvertedIndexNestedFallbackReturnsNotSupportedIn ASSERT_TRUE(provider != nullptr); if (!provider->should_enable_nested_group_read_path()) { EXPECT_FALSE(status.ok()); - EXPECT_EQ(ErrorCode::NOT_IMPLEMENTED_ERROR, status.code()); + EXPECT_EQ(ErrorCode::INVERTED_INDEX_NOT_SUPPORTED, status.code()); EXPECT_TRUE(status.to_string().find("NestedGroup support") != std::string::npos); } else { EXPECT_FALSE(status.ok()); @@ -1478,10 +1499,90 @@ TEST_F(VSearchExprTest, EvaluateInvertedIndexPropagatesFunctionFailure) { auto status = expr->evaluate_inverted_index(context.get(), 256); EXPECT_FALSE(status.ok()); - EXPECT_EQ(ErrorCode::INVERTED_INDEX_FILE_NOT_FOUND, status.code()); + EXPECT_EQ(ErrorCode::INVERTED_INDEX_NOT_SUPPORTED, status.code()); EXPECT_FALSE(status_map[0][expr.get()]); } +TEST_F(VSearchExprTest, EvaluateInvertedIndexRejectsSearchFallback) { + test_node.search_param.root.clause_type = "WILDCARD"; + test_node.search_param.root.value = "hello*"; + test_node.search_param.field_bindings[0].index_properties[INVERTED_INDEX_PARSER_KEY] = + INVERTED_INDEX_PARSER_ENGLISH; + test_node.search_param.field_bindings[0].__isset.index_properties = true; + + auto expr = VSearchExpr::create_shared(test_node); + expr->add_child(create_slot_ref(0, "title")); + + TabletIndexPB index_pb; + index_pb.set_index_type(IndexType::INVERTED); + index_pb.set_index_id(1); + index_pb.set_index_name("search_fallback_index"); + index_pb.add_col_unique_id(1); + (*index_pb.mutable_properties())[INVERTED_INDEX_PARSER_KEY] = INVERTED_INDEX_PARSER_STANDARD; + TabletIndex index_meta; + index_meta.init_from_pb(index_pb); + auto index_file_reader = std::make_shared( + nullptr, "/tmp/search_fallback_idx", InvertedIndexStorageFormatPB::V2); + auto reader = std::make_shared(&index_meta, index_file_reader); + auto iterator = std::make_unique(); + iterator->add_reader(segment_v2::InvertedIndexReaderType::FULLTEXT, reader); + + std::vector col_ids = {0}; + std::vector> index_iterators; + index_iterators.emplace_back(std::move(iterator)); + std::vector storage_types; + storage_types.emplace_back("title", std::make_shared()); + std::unordered_map> status_map; + status_map[0][expr.get()] = false; + + auto inverted_ctx = make_inverted_context(col_ids, index_iterators, storage_types, status_map); + auto context = std::make_shared(expr); + context->set_index_context(inverted_ctx); + + auto status = expr->evaluate_inverted_index(context.get(), 256); + EXPECT_FALSE(status.ok()); + EXPECT_EQ(ErrorCode::INVERTED_INDEX_NOT_SUPPORTED, status.code()); + EXPECT_NE(status.to_string().find("cannot fall back"), std::string::npos); + EXPECT_NE(status.to_string().find("No inverted index found for analyzer"), std::string::npos); + EXPECT_FALSE(status_map[0][expr.get()]); + + for (TExprOpcode::type opcode : {TExprOpcode::COMPOUND_AND, TExprOpcode::COMPOUND_OR}) { + TExprNode compound_node; + compound_node.__set_type(test_node.type); + compound_node.__set_node_type(TExprNodeType::COMPOUND_PRED); + compound_node.__set_opcode(opcode); + compound_node.__set_num_children(1); + compound_node.__set_is_nullable(false); + auto compound = VCompoundPred::create_shared(compound_node); + compound->add_child(expr); + + status = compound->evaluate_inverted_index(context.get(), 256); + EXPECT_FALSE(status.ok()); + EXPECT_EQ(ErrorCode::INVERTED_INDEX_NOT_SUPPORTED, status.code()); + EXPECT_NE(status.to_string().find("No inverted index found for analyzer"), + std::string::npos); + } + + // A SNII sub-case used to live here: a TERM clause against a binding whose IndexFileReader + // reported SNII storage format, asserting the old hard refusal ("supports only WILDCARD"). + // That refusal was removed -- SNII SEARCH now forwards every clause type to the reader as a + // query type (see FunctionSearch::build_leaf_query's SNII branch) -- so the assertion no + // longer holds. It is not being replaced with a corrected assertion here, because the mock + // combination it used was never reachable in production to begin with: it paired a SNII- + // format IndexFileReader with a segment_v2::FullTextIndexReader (a CLucene reader). In real + // code the two are set atomically at the single production construction site + // (ColumnReader::_load_index, storage/segment/column_reader.cpp:727-743): that function + // returns as soon as it sees SNII storage format, having constructed only a SniiIndexReader + // or SniiBkdIndexReader; a FullTextIndexReader is built exclusively in the mutually + // exclusive non-SNII branch below it. So "use_snii_native_reader() true with a CLucene + // reader bound" cannot occur outside a hand-built test double. Forwarding coverage for SNII + // TERM clauses (EQUAL_QUERY, default_operator "and" -> MATCH_ALL_QUERY, and the explicit + // minimum_should_match refusal) lives in FunctionSearchTest + // (TestSniiNativeForwardsTermClauseAsEqualQuery and friends, + // be/test/exprs/function/function_search_test.cpp), which uses a reader double shaped like + // the real SNII reader instead of a mismatched CLucene one. +} + // Note: Full testing with actual IndexExecContext and real iterators // would require complex setup and is better suited for integration tests // The tests above cover the main execution paths in evaluate_inverted_index diff --git a/be/test/io/cache/block_file_cache_profile_reporter_test.cpp b/be/test/io/cache/block_file_cache_profile_reporter_test.cpp index 975f1e8eddd88b..246cc539952f35 100644 --- a/be/test/io/cache/block_file_cache_profile_reporter_test.cpp +++ b/be/test/io/cache/block_file_cache_profile_reporter_test.cpp @@ -64,6 +64,10 @@ io::FileCacheStatistics make_file_cache_stats(int64_t multiplier) { stats.num_peer_race_s3_win = multiplier * 38; stats.num_peer_lazy_fetch = multiplier * 39; stats.peer_lazy_fetch_timer = multiplier * 40; + stats.inverted_index_request_bytes = multiplier * 41; + stats.inverted_index_read_bytes = multiplier * 42; + stats.inverted_index_range_read_count = multiplier * 43; + stats.inverted_index_serial_read_rounds = multiplier * 44; return stats; } @@ -114,6 +118,10 @@ void expect_file_cache_stats_eq(const io::FileCacheStatistics& actual, EXPECT_EQ(actual.num_peer_race_s3_win, expected.num_peer_race_s3_win); EXPECT_EQ(actual.num_peer_lazy_fetch, expected.num_peer_lazy_fetch); EXPECT_EQ(actual.peer_lazy_fetch_timer, expected.peer_lazy_fetch_timer); + EXPECT_EQ(actual.inverted_index_request_bytes, expected.inverted_index_request_bytes); + EXPECT_EQ(actual.inverted_index_read_bytes, expected.inverted_index_read_bytes); + EXPECT_EQ(actual.inverted_index_range_read_count, expected.inverted_index_range_read_count); + EXPECT_EQ(actual.inverted_index_serial_read_rounds, expected.inverted_index_serial_read_rounds); } } // namespace @@ -163,6 +171,14 @@ TEST(FileCacheProfileReporterTest, ReporterAggregatesDeltaReportsToExactFinalTot after_second_report.cross_cg_peer_io_timer); EXPECT_EQ(profile->get_counter("PeerLazyFetchTime")->value(), after_second_report.peer_lazy_fetch_timer); + EXPECT_EQ(profile->get_counter("InvertedIndexRequestBytes")->value(), + after_second_report.inverted_index_request_bytes); + EXPECT_EQ(profile->get_counter("InvertedIndexReadBytes")->value(), + after_second_report.inverted_index_read_bytes); + EXPECT_EQ(profile->get_counter("InvertedIndexRangeReadCount")->value(), + after_second_report.inverted_index_range_read_count); + EXPECT_EQ(profile->get_counter("InvertedIndexSerialReadRounds")->value(), + after_second_report.inverted_index_serial_read_rounds); } } // namespace doris diff --git a/be/test/runtime/fragment_mgr_cross_cluster_cancel_test.cpp b/be/test/runtime/fragment_mgr_cross_cluster_cancel_test.cpp index 24c77261183f84..c178728712d6f2 100644 --- a/be/test/runtime/fragment_mgr_cross_cluster_cancel_test.cpp +++ b/be/test/runtime/fragment_mgr_cross_cluster_cancel_test.cpp @@ -18,11 +18,15 @@ #include #include +#include "common/config.h" #include "runtime/descriptor_helper.h" #include "runtime/exec_env.h" #include "runtime/fragment_mgr.h" #include "runtime/frontend_info.h" +#include "runtime/index_policy/index_policy_mgr.h" #include "runtime/workload_group/workload_group_manager.h" +#include "storage/id_manager.h" +#include "util/defer_op.h" namespace doris { diff --git a/be/test/runtime/index_policy/index_policy_mgr_test.cpp b/be/test/runtime/index_policy/index_policy_mgr_test.cpp index 923690ef3612c8..4485217652bb11 100644 --- a/be/test/runtime/index_policy/index_policy_mgr_test.cpp +++ b/be/test/runtime/index_policy/index_policy_mgr_test.cpp @@ -19,10 +19,34 @@ #include +#include +#include +#include +#include + +#include "common/config.h" #include "runtime/exec_env.h" #include "storage/index/inverted/analysis_factory_mgr.h" +#include "storage/index/inverted/analyzer/analyzer.h" +#include "storage/index/inverted/analyzer/segment_analyzer_context.h" +#include "storage/index/inverted/common_grams/common_grams_key_codec.h" +#include "storage/index/inverted/common_grams/common_grams_segment_metadata.h" +#include "storage/index/inverted/common_grams/common_word_set.h" +#include "util/defer_op.h" namespace doris { +namespace { + +TIndexPolicy common_grams_policy(int64_t id, std::string name) { + TIndexPolicy policy; + policy.id = id; + policy.name = std::move(name); + policy.type = TIndexPolicyType::TOKEN_FILTER; + policy.properties["type"] = "common_grams"; + return policy; +} + +} // namespace class IndexPolicyMgrTest : public testing::Test { protected: @@ -176,4 +200,220 @@ TEST_F(IndexPolicyMgrTest, TestTokenFilterProcessing) { ASSERT_NE(emptyAnalyzer, nullptr); } +TEST_F(IndexPolicyMgrTest, CommonGramsProviderRetainsImmutablePurposeConfiguration) { + TIndexPolicy tokenizer; + tokenizer.id = 20; + tokenizer.name = "cg_tokenizer"; + tokenizer.type = TIndexPolicyType::TOKENIZER; + tokenizer.properties["type"] = "char_group"; + tokenizer.properties["tokenize_on_chars"] = "[whitespace]"; + + auto common_grams = common_grams_policy(21, "cg_filter"); + + TIndexPolicy analyzer_policy; + analyzer_policy.id = 22; + analyzer_policy.name = "cg_analyzer"; + analyzer_policy.type = TIndexPolicyType::ANALYZER; + analyzer_policy.properties["tokenizer"] = "cg_tokenizer"; + analyzer_policy.properties["token_filter"] = "lowercase,cg_filter"; + mgr.apply_policy_changes({tokenizer, common_grams, analyzer_policy}, {}); + + auto provider = mgr.get_analyzer_provider_by_name("cg_analyzer"); + ASSERT_NE(provider, nullptr); + auto analyze = [](const segment_v2::inverted_index::AnalyzerProviderPtr& analyzer_provider, + segment_v2::inverted_index::AnalysisPurpose purpose, std::string_view text) { + auto analyzer = analyzer_provider->get_analyzer(purpose); + auto reader = std::make_shared>(); + reader->init(text.data(), static_cast(text.size()), true); + return segment_v2::inverted_index::InvertedIndexAnalyzer::get_analyse_result( + reader, analyzer.get()); + }; + + auto index = analyze(provider, segment_v2::inverted_index::AnalysisPurpose::kIndex, + "Man of the Year"); + auto plain = analyze(provider, segment_v2::inverted_index::AnalysisPurpose::kPlainQuery, + "Man of the Year"); + auto exact = analyze(provider, segment_v2::inverted_index::AnalysisPurpose::kExactPhraseQuery, + "Man of the Year"); + auto prefix = analyze(provider, segment_v2::inverted_index::AnalysisPurpose::kPhrasePrefixQuery, + "the wo"); + EXPECT_EQ(index.size(), 7); + EXPECT_EQ(plain.size(), 4); + EXPECT_EQ(exact.size(), 3); + EXPECT_EQ(prefix.size(), 1); + auto prefix_gram = segment_v2::inverted_index::encode_common_gram("the", "wo"); + ASSERT_TRUE(prefix_gram.has_value()) << prefix_gram.error(); + EXPECT_EQ(prefix.front().get_single_term(), prefix_gram.value()); + + auto single_index = mgr.get_analyzer_by_name( + "cg_analyzer", segment_v2::inverted_index::AnalysisPurpose::kIndex); + auto single_reader = std::make_shared>(); + const std::string single_input = "Man of the Year"; + single_reader->init(single_input.data(), static_cast(single_input.size()), true); + EXPECT_EQ(segment_v2::inverted_index::InvertedIndexAnalyzer::get_analyse_result( + single_reader, single_index.get()) + .size(), + 7); +} + +TEST_F(IndexPolicyMgrTest, AnalyzerProviderPreservesPurposeInsensitiveNormalizers) { + auto builtin = mgr.get_analyzer_provider_by_name("lowercase"); + auto builtin_analyzer = + builtin->get_analyzer(segment_v2::inverted_index::AnalysisPurpose::kPlainQuery); + EXPECT_EQ(builtin->get_analyzer(segment_v2::inverted_index::AnalysisPurpose::kIndex), + builtin_analyzer); + + TIndexPolicy normalizer; + normalizer.id = 23; + normalizer.name = "test_normalizer"; + normalizer.type = TIndexPolicyType::NORMALIZER; + normalizer.properties["token_filter"] = "lowercase"; + mgr.apply_policy_changes({normalizer}, {}); + + auto configured = mgr.get_analyzer_provider_by_name("test_normalizer"); + auto configured_analyzer = + configured->get_analyzer(segment_v2::inverted_index::AnalysisPurpose::kPlainQuery); + EXPECT_EQ(configured->get_analyzer(segment_v2::inverted_index::AnalysisPurpose::kIndex), + configured_analyzer); +} + +TEST_F(IndexPolicyMgrTest, FindsFreshAnalyzerProviderBySegmentBaseFingerprint) { + using segment_v2::inverted_index::AnalysisPurpose; + using segment_v2::inverted_index::InvertedIndexAnalyzer; + + const std::map slash_to_space = { + {INVERTED_INDEX_PARSER_CHAR_FILTER_TYPE, INVERTED_INDEX_CHAR_FILTER_CHAR_REPLACE}, + {INVERTED_INDEX_PARSER_CHAR_FILTER_PATTERN, "/"}, + {INVERTED_INDEX_PARSER_CHAR_FILTER_REPLACEMENT, " "}}; + auto configured = mgr.get_analyzer_provider_by_name("analyzer1", slash_to_space); + const std::string segment_fingerprint(configured->base_analyzer_fingerprint()); + + auto first = mgr.get_analyzer_provider_by_base_fingerprint(segment_fingerprint, slash_to_space); + ASSERT_NE(first, nullptr); + EXPECT_EQ(first->base_analyzer_fingerprint(), segment_fingerprint); + EXPECT_EQ(mgr.get_analyzer_provider_by_base_fingerprint(segment_fingerprint), nullptr); + EXPECT_EQ(mgr.get_analyzer_provider_by_base_fingerprint("unknown", slash_to_space), nullptr); + + auto second = + mgr.get_analyzer_provider_by_base_fingerprint(segment_fingerprint, slash_to_space); + ASSERT_NE(second, nullptr); + EXPECT_NE(first, second); + EXPECT_NE(first->get_analyzer(AnalysisPurpose::kPlainQuery), + second->get_analyzer(AnalysisPurpose::kPlainQuery)); + + mgr.apply_policy_changes({}, {5}); + auto reader = std::make_shared>(); + const std::string input = "ASCII TERM"; + reader->init(input.data(), static_cast(input.size()), true); + const auto terms = InvertedIndexAnalyzer::get_analyse_result( + reader, first->get_analyzer(AnalysisPurpose::kPlainQuery).get()); + ASSERT_EQ(terms.size(), 2U); + EXPECT_EQ(terms[0].get_single_term(), "ascii"); + EXPECT_EQ(terms[1].get_single_term(), "term"); +} + +TEST_F(IndexPolicyMgrTest, RebuildsQueryContextForPersistedSegmentAnalyzer) { + const std::map slash_to_space = { + {INVERTED_INDEX_PARSER_CHAR_FILTER_TYPE, INVERTED_INDEX_CHAR_FILTER_CHAR_REPLACE}, + {INVERTED_INDEX_PARSER_CHAR_FILTER_PATTERN, "/"}, + {INVERTED_INDEX_PARSER_CHAR_FILTER_REPLACEMENT, " "}}; + auto segment_provider = mgr.get_analyzer_provider_by_name("analyzer1", slash_to_space); + const std::string segment_fingerprint(segment_provider->base_analyzer_fingerprint()); + + segment_v2::inverted_index::Settings tokenizer_settings; + tokenizer_settings.set("tokenize_on_chars", "[whitespace]"); + segment_v2::inverted_index::CustomAnalyzerConfig::Builder request_builder; + request_builder.with_tokenizer_config("char_group", tokenizer_settings); + auto request_provider = std::make_shared( + request_builder.build()); + ASSERT_NE(request_provider->base_analyzer_fingerprint(), segment_fingerprint); + + InvertedIndexAnalyzerCtx request_context; + request_context.analyzer_name = "request_analyzer"; + request_context.parser_type = InvertedIndexParserType::PARSER_ENGLISH; + request_context.char_filter_map = {{"stale", "filter"}}; + request_context.analyzer = request_provider->get_analyzer( + segment_v2::inverted_index::AnalysisPurpose::kPlainQuery); + request_context.analyzer_provider = request_provider; + request_context.common_grams_identity = segment_v2::inverted_index::CommonGramsQueryIdentity { + .common_grams_dictionary_identity = "stale-dictionary", + .base_analyzer_fingerprint = std::string(request_provider->base_analyzer_fingerprint()), + .common_grams_fingerprint = "stale-common-grams"}; + + const std::map physical_properties = slash_to_space; + auto rebuilt = segment_v2::inverted_index::maybe_rebuild_segment_analyzer_context( + &request_context, segment_fingerprint, physical_properties, &mgr); + ASSERT_TRUE(rebuilt.has_value()) << rebuilt.error(); + ASSERT_TRUE(rebuilt->has_value()); + const auto& effective = rebuilt->value(); + EXPECT_EQ(effective.analyzer_name, request_context.analyzer_name); + EXPECT_EQ(effective.parser_type, request_context.parser_type); + EXPECT_EQ(effective.char_filter_map, slash_to_space); + EXPECT_EQ(effective.analyzer, nullptr); + ASSERT_NE(effective.analyzer_provider, nullptr); + EXPECT_EQ(effective.analyzer_provider->base_analyzer_fingerprint(), segment_fingerprint); + EXPECT_FALSE(effective.common_grams_identity.has_value()); + EXPECT_NE(effective.analyzer_provider, segment_provider); + + InvertedIndexAnalyzerCtx matching_context = request_context; + matching_context.analyzer_provider = segment_provider; + auto unchanged = segment_v2::inverted_index::maybe_rebuild_segment_analyzer_context( + &matching_context, segment_fingerprint, physical_properties, &mgr); + ASSERT_TRUE(unchanged.has_value()) << unchanged.error(); + EXPECT_FALSE(unchanged->has_value()); + + auto unavailable = segment_v2::inverted_index::maybe_rebuild_segment_analyzer_context( + &request_context, "missing-segment-fingerprint", physical_properties, &mgr); + ASSERT_FALSE(unavailable.has_value()); + EXPECT_EQ(unavailable.error().code(), ErrorCode::INVERTED_INDEX_BYPASS); +} + +TEST_F(IndexPolicyMgrTest, SegmentAnalyzerAdmissionKeepsLegacyRequestWithoutMetadata) { + auto admitted = segment_v2::inverted_index::maybe_rebuild_segment_analyzer_context( + nullptr, std::optional {}, {}, + nullptr); + + ASSERT_TRUE(admitted.has_value()) << admitted.error(); + EXPECT_FALSE(admitted->has_value()); +} + +TEST_F(IndexPolicyMgrTest, SegmentAnalyzerAdmissionBypassesTypedMetadataWithoutBaseFingerprint) { + segment_v2::inverted_index::CommonGramsSegmentMetadata metadata; + auto admitted = segment_v2::inverted_index::maybe_rebuild_segment_analyzer_context( + nullptr, std::optional {metadata}, {}, &mgr); + + ASSERT_FALSE(admitted.has_value()); + EXPECT_EQ(admitted.error().code(), ErrorCode::INVERTED_INDEX_BYPASS); +} + +TEST_F(IndexPolicyMgrTest, SegmentAnalyzerAdmissionRebuildsTypedMetadata) { + const std::map slash_to_space = { + {INVERTED_INDEX_PARSER_CHAR_FILTER_TYPE, INVERTED_INDEX_CHAR_FILTER_CHAR_REPLACE}, + {INVERTED_INDEX_PARSER_CHAR_FILTER_PATTERN, "/"}, + {INVERTED_INDEX_PARSER_CHAR_FILTER_REPLACEMENT, " "}}; + auto segment_provider = mgr.get_analyzer_provider_by_name("analyzer1", slash_to_space); + const std::string segment_fingerprint(segment_provider->base_analyzer_fingerprint()); + + segment_v2::inverted_index::Settings tokenizer_settings; + tokenizer_settings.set("tokenize_on_chars", "[whitespace]"); + segment_v2::inverted_index::CustomAnalyzerConfig::Builder request_builder; + request_builder.with_tokenizer_config("char_group", tokenizer_settings); + auto request_provider = std::make_shared( + request_builder.build()); + + InvertedIndexAnalyzerCtx request_context; + request_context.analyzer_provider = request_provider; + request_context.analyzer = request_provider->get_analyzer( + segment_v2::inverted_index::AnalysisPurpose::kPlainQuery); + segment_v2::inverted_index::CommonGramsSegmentMetadata metadata; + metadata.base_analyzer_fingerprint = segment_fingerprint; + + auto rebuilt = segment_v2::inverted_index::maybe_rebuild_segment_analyzer_context( + &request_context, std::optional {metadata}, slash_to_space, &mgr); + + ASSERT_TRUE(rebuilt.has_value()) << rebuilt.error(); + ASSERT_TRUE(rebuilt->has_value()); + EXPECT_EQ(rebuilt->value().analyzer_provider->base_analyzer_fingerprint(), segment_fingerprint); +} + } // namespace doris \ No newline at end of file diff --git a/be/test/storage/compaction/cloud_index_change_compaction_test.cpp b/be/test/storage/compaction/cloud_index_change_compaction_test.cpp index 663dd23aebb2f7..6788f317f8d791 100644 --- a/be/test/storage/compaction/cloud_index_change_compaction_test.cpp +++ b/be/test/storage/compaction/cloud_index_change_compaction_test.cpp @@ -19,6 +19,8 @@ #include +#include + #include "cloud/cloud_base_compaction.h" #include "cloud/cloud_cumulative_compaction.h" #include "cpp/sync_point.h" @@ -73,6 +75,95 @@ class CloudIndexChangeCompactionTest : public testing::Test { return origin.find(sub_str) != std::string::npos; } + CloudTabletSPtr create_tablet_for_index_compaction_gate( + InvertedIndexStorageFormatPB storage_format) { + TabletSchemaPB schema_pb; + schema_pb.set_keys_type(KeysType::DUP_KEYS); + schema_pb.set_inverted_index_storage_format(storage_format); + schema_pb.set_schema_version(1); + + ColumnPB* column_pb = schema_pb.add_column(); + column_pb->set_unique_id(1); + column_pb->set_name("body"); + column_pb->set_type("STRING"); + column_pb->set_is_nullable(true); + + auto add_index = [&schema_pb](int64_t index_id, std::string_view index_name) { + TabletIndexPB* index_pb = schema_pb.add_index(); + index_pb->set_index_id(index_id); + index_pb->set_index_name(std::string(index_name)); + index_pb->set_index_type(IndexType::INVERTED); + index_pb->add_col_unique_id(1); + (*index_pb->mutable_properties())[INVERTED_INDEX_PARSER_KEY] = + INVERTED_INDEX_PARSER_UNICODE; + }; + add_index(11001, "idx_to_drop"); + add_index(11002, "idx_to_keep"); + + auto input_schema = std::make_shared(); + input_schema->init_from_pb(schema_pb); + auto input_rowset_meta = std::make_shared(); + init_rs_meta(input_rowset_meta, 2, 2); + input_rowset_meta->set_num_segments(1); + input_rowset_meta->set_tablet_schema(input_schema); + RowsetSharedPtr input_rowset = + std::make_shared(input_schema, input_rowset_meta, ""); + + TabletMetaPB tablet_meta_pb; + tablet_meta_pb.set_tablet_id(1000); + tablet_meta_pb.set_schema_hash(123456); + tablet_meta_pb.set_tablet_state(PB_RUNNING); + *tablet_meta_pb.mutable_tablet_uid() = TabletUid::gen_uid().to_proto(); + auto tablet_meta = std::make_shared(); + tablet_meta->init_from_pb(tablet_meta_pb); + CloudTabletSPtr tablet = std::make_shared(*_engine, tablet_meta); + tablet->_rs_version_map[Version(2, 2)] = input_rowset; + return tablet; + } + + void verify_index_compaction_gate(InvertedIndexStorageFormatPB storage_format, + bool initially_enabled, bool expected_enabled) { + CloudTabletSPtr tablet = create_tablet_for_index_compaction_gate(storage_format); + + TColumn body_column; + body_column.__set_column_name("body"); + body_column.__set_col_unique_id(1); + TColumnType body_type; + body_type.__set_type(TPrimitiveType::STRING); + body_column.__set_column_type(body_type); + std::vector columns {body_column}; + + TOlapTableIndex surviving_index; + surviving_index.__set_index_id(11002); + surviving_index.__set_index_name("idx_to_keep"); + surviving_index.__set_index_type(TIndexType::INVERTED); + surviving_index.__set_columns({"body"}); + surviving_index.__set_column_unique_ids({1}); + surviving_index.__set_properties( + {{INVERTED_INDEX_PARSER_KEY, INVERTED_INDEX_PARSER_UNICODE}}); + std::vector final_indexes {surviving_index}; + + CloudIndexChangeCompaction compaction(*_engine, tablet, 2, final_indexes, columns); + compaction._enable_inverted_index_compaction = initially_enabled; + auto* sync_point = SyncPoint::get_instance(); + sync_point->set_call_back("CloudMetaMgr::sync_tablet_rowsets", [](auto&& outcome) { + auto* result = try_any_cast_ret(outcome); + result->second = true; + result->first = Status::OK(); + }); + Status prepare_status = compaction.prepare_compact(); + sync_point->clear_call_back("CloudMetaMgr::sync_tablet_rowsets"); + + ASSERT_TRUE(prepare_status.ok()) << prepare_status; + ASSERT_EQ(compaction._input_rowsets.size(), 1); + EXPECT_EQ(compaction._enable_inverted_index_compaction, expected_enabled); + ASSERT_TRUE(compaction.rebuild_tablet_schema().ok()); + ASSERT_EQ(compaction._final_tablet_schema->get_inverted_index_storage_format(), + storage_format); + ASSERT_EQ(compaction._final_tablet_schema->inverted_indexes().size(), 1); + EXPECT_EQ(compaction._final_tablet_schema->inverted_indexes().front()->index_id(), 11002); + } + public: std::unique_ptr _engine; }; @@ -413,6 +504,66 @@ TEST_F(CloudIndexChangeCompactionTest, basic_compaction_test) { ASSERT_TRUE(in_predicates[0].values().size() == 2); } +TEST_F(CloudIndexChangeCompactionTest, snii_drop_index_enables_native_index_compaction) { + verify_index_compaction_gate(InvertedIndexStorageFormatPB::SNII, true, true); + verify_index_compaction_gate(InvertedIndexStorageFormatPB::SNII, false, false); + verify_index_compaction_gate(InvertedIndexStorageFormatPB::V1, true, false); + verify_index_compaction_gate(InvertedIndexStorageFormatPB::V2, true, false); + verify_index_compaction_gate(InvertedIndexStorageFormatPB::V3, true, false); +} + +// Cloud index change whose sources cannot be read (the mock rowsets have no +// files behind them): the SNII preflight classifies every target index -- the +// pre-existing 11002 and the newly added 11003 alike -- as raw build BEFORE any +// write starts. Both compaction sets stay empty, so no index can silently +// switch paths mid-write; the raw build path covers everything. +TEST_F(CloudIndexChangeCompactionTest, snii_preflight_unreadable_source_classifies_raw_build) { + CloudTabletSPtr tablet = + create_tablet_for_index_compaction_gate(InvertedIndexStorageFormatPB::SNII); + + TColumn body_column; + body_column.__set_column_name("body"); + body_column.__set_col_unique_id(1); + TColumnType body_type; + body_type.__set_type(TPrimitiveType::STRING); + body_column.__set_column_type(body_type); + std::vector columns {body_column}; + + const auto make_index = [](int64_t index_id, std::string_view name) { + TOlapTableIndex index; + index.__set_index_id(index_id); + index.__set_index_name(std::string(name)); + index.__set_index_type(TIndexType::INVERTED); + index.__set_columns({"body"}); + index.__set_column_unique_ids({1}); + index.__set_properties({{INVERTED_INDEX_PARSER_KEY, INVERTED_INDEX_PARSER_UNICODE}}); + return index; + }; + // The target schema keeps 11002 and ADDS 11003 on the same column. + std::vector final_indexes {make_index(11002, "idx_to_keep"), + make_index(11003, "idx_added")}; + + CloudIndexChangeCompaction compaction(*_engine, tablet, 2, final_indexes, columns); + compaction._enable_inverted_index_compaction = true; + auto* sync_point = SyncPoint::get_instance(); + sync_point->set_call_back("CloudMetaMgr::sync_tablet_rowsets", [](auto&& outcome) { + auto* result = try_any_cast_ret(outcome); + result->second = true; + result->first = Status::OK(); + }); + Status prepare_status = compaction.prepare_compact(); + sync_point->clear_call_back("CloudMetaMgr::sync_tablet_rowsets"); + ASSERT_TRUE(prepare_status.ok()) << prepare_status; + ASSERT_EQ(compaction._input_rowsets.size(), 1); + ASSERT_TRUE(compaction.rebuild_tablet_schema().ok()); + ASSERT_EQ(compaction._cur_tablet_schema->inverted_indexes().size(), 2); + + RowsetWriterContext ctx; + compaction.construct_index_compaction_columns(ctx); + EXPECT_TRUE(ctx.columns_to_do_index_compaction.empty()); + EXPECT_TRUE(ctx.snii_indexes_to_do_compaction.empty()); +} + TEST_F(CloudIndexChangeCompactionTest, test_cloud_tablet) { TabletSchemaPB schema_pb; schema_pb.set_keys_type(KeysType::DUP_KEYS); @@ -524,4 +675,4 @@ TEST_F(CloudIndexChangeCompactionTest, test_cloud_tablet) { } } -} // namespace doris \ No newline at end of file +} // namespace doris diff --git a/be/test/storage/compaction/collection_statistics_test.cpp b/be/test/storage/compaction/collection_statistics_test.cpp deleted file mode 100644 index b78355b316ebdd..00000000000000 --- a/be/test/storage/compaction/collection_statistics_test.cpp +++ /dev/null @@ -1,1310 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "storage/compaction/collection_statistics.h" - -#include -#include -#include - -#include -#include - -#include "common/exception.h" -#include "core/data_type/data_type_string.h" -#include "exec/common/variant_util.h" -#include "exprs/vexpr.h" -#include "exprs/vexpr_context.h" -#include "exprs/vliteral.h" -#include "exprs/vslot_ref.h" -#include "io/fs/local_file_system.h" -#include "storage/compaction/collection_statistics.cpp" -#include "storage/rowset/rowset.h" -#include "storage/rowset/rowset_meta.h" -#include "storage/rowset/rowset_reader.h" -#include "storage/tablet/tablet_schema.h" -#include "testutil/mock/mock_runtime_state.h" - -namespace doris { - -namespace collection_statistics { - -class MockVExpr : public VExpr { -public: - MockVExpr(TExprNodeType::type node_type) : _mock_node_type(node_type) { - if (node_type == TExprNodeType::MATCH_PRED) { - _opcode = TExprOpcode::MATCH_PHRASE; - } - } - - TExprNodeType::type node_type() const override { return _mock_node_type; } - - Status execute(VExprContext* context, Block* block, int32_t* result_column_id) const override { - return Status::OK(); - } - - Status execute_column_impl(VExprContext* context, const Block* block, const Selector* selector, - size_t count, ColumnPtr& result_column) const override { - return Status::OK(); - } - - Status prepare(RuntimeState* state, const RowDescriptor& desc, VExprContext* context) override { - return Status::OK(); - } - - Status open(RuntimeState* state, VExprContext* context, - FunctionContext::FunctionStateScope scope) override { - return Status::OK(); - } - - void close(VExprContext* context, FunctionContext::FunctionStateScope scope) override {} - - const std::string& expr_name() const override { - static std::string name = "mock_expr"; - return name; - } - - std::string debug_string() const override { return "MockVExpr"; } - -private: - TExprNodeType::type _mock_node_type; -}; - -class MockVSlotRef : public VSlotRef { -public: - MockVSlotRef(const std::string& column_name, SlotId slot_id) - : _column_name(column_name), _slot_id(slot_id) { - _node_type = TExprNodeType::SLOT_REF; - } - - const std::string& column_name() const override { return _column_name; } - const std::string& expr_name() const override { return _column_name; } - std::string debug_string() const override { return "MockVSlotRef: " + _column_name; } - SlotId slot_id() const override { return _slot_id; } - -private: - std::string _column_name; - SlotId _slot_id; -}; - -class MockVLiteral : public VLiteral { -public: - MockVLiteral(const std::string& value) : _value(value) {} - - std::string value() const override { return _value; } - std::string value(const DataTypeSerDe::FormatOptions& options) const override { return _value; } - const std::string& expr_name() const override { return _value; } - std::string debug_string() const override { return "MockVLiteral: " + _value; } - -private: - std::string _value; -}; - -class MockRowsetMeta : public RowsetMeta { -public: - MockRowsetMeta() : RowsetMeta() { _fs = io::global_local_filesystem(); } - - io::FileSystemSPtr fs() override { return _fs; } - -private: - io::FileSystemSPtr _fs; -}; - -class MockRowset : public Rowset { -public: - MockRowset(TabletSchemaSPtr schema, RowsetMetaSharedPtr rowset_meta) - : Rowset(schema, rowset_meta, "/mock/tablet/path") { - _num_segments = 0; - } - - Status create_reader(std::shared_ptr* result) override { - return Status::NotSupported("MockRowset::create_reader not implemented"); - } - - Status remove() override { return Status::OK(); } - - Status link_files_to(const std::string& dir, RowsetId new_rowset_id, size_t start_seg_id, - std::set* without_index_uids) override { - return Status::OK(); - } - - Status copy_files_to(const std::string& dir, const RowsetId& new_rowset_id) override { - return Status::OK(); - } - - Status remove_old_files(std::vector* files_to_remove) override { - return Status::OK(); - } - - Status check_file_exist() override { return Status::OK(); } - - Status upload_to(const StorageResource& dest_fs, const RowsetId& new_rowset_id) override { - return Status::OK(); - } - - Status get_inverted_index_size(int64_t* index_size) override { - *index_size = 0; - return Status::OK(); - } - - void clear_inverted_index_cache() override {} - - Status init() override { return Status::OK(); } - - void do_close() override {} - - Status check_current_rowset_segment() override { return Status::OK(); } - - int64_t num_segments() const override { return _num_segments; } - - Result segment_path(int64_t seg_id) override { - if (_segment_paths.find(seg_id) != _segment_paths.end()) { - return _segment_paths.at(seg_id); - } - return ResultError(Status::InternalError("Segment path not found")); - } - - void set_segment_path(int64_t seg_id, const std::string& path) { - _segment_paths[seg_id] = path; - } - - void set_num_segments(int64_t num) { _num_segments = num; } - -private: - int64_t _num_segments; - std::map _segment_paths; -}; - -class MockRowsetReader : public RowsetReader { -public: - MockRowsetReader(std::shared_ptr rowset) : _rowset(rowset) {} - - Status init(RowsetReaderContext* read_context, const RowSetSplits& rs_splits) override { - return Status::OK(); - } - - Status get_segment_iterators(RowsetReaderContext* read_context, - std::vector* out_iters, - bool use_cache = false) override { - return Status::OK(); - } - - void reset_read_options() override {} - - Status next_batch(Block* block) override { - return Status::NotSupported("MockRowsetReader::next_batch not implemented"); - } - - Status next_batch(BlockView* block_view) override { - return Status::NotSupported("MockRowsetReader::next_batch not implemented"); - } - - Status next_batch(BlockWithSameBit* block_view) override { - return Status::NotSupported("MockRowsetReader::next_batch not implemented"); - } - - bool delete_flag() override { return false; } - - Version version() override { return Version(1, 1); } - - RowsetSharedPtr rowset() override { return _rowset; } - - int64_t filtered_rows() override { return 0; } - - uint64_t merged_rows() override { return 0; } - - RowsetTypePB type() const override { return BETA_ROWSET; } - - int64_t newest_write_timestamp() override { return 0; } - - void update_profile(RuntimeProfile* profile) override {} - - RowsetReaderSharedPtr clone() override { return std::make_shared(_rowset); } - - void set_topn_limit(size_t limit) override {} - -private: - std::shared_ptr _rowset; -}; - -} // namespace collection_statistics - -class CollectionStatisticsTest : public ::testing::Test { -protected: - void SetUp() override { - stats_ = std::make_unique(); - runtime_state_ = std::make_shared(); - runtime_state_->_mock_desc_tbl->add_slot_descriptor(SlotId(1), 1001); - test_dir_ = "./collection_statistics_test_" + - std::to_string(::testing::UnitTest::GetInstance()->random_seed()); - ASSERT_TRUE(io::global_local_filesystem()->create_directory(test_dir_).ok()); - } - - void TearDown() override { - stats_.reset(); - runtime_state_.reset(); - (void)io::global_local_filesystem()->delete_directory(test_dir_); - } - - TabletSchemaSPtr create_tablet_schema_with_inverted_index() { - auto tablet_schema = std::make_shared(); - - TabletColumn column; - column.set_unique_id(1); - column.set_name("content"); - column.set_type(FieldType::OLAP_FIELD_TYPE_STRING); - tablet_schema->append_column(column); - - TabletIndex index; - index._index_id = 1; - index._index_type = IndexType::INVERTED; - index._col_unique_ids.push_back(1); - std::map properties; - properties["parser"] = "standard"; - properties["support_phrase"] = "true"; - index._properties = properties; - - tablet_schema->append_index(std::move(index)); - - return tablet_schema; - } - - VExprContextSPtrs create_match_expr_contexts(const std::string& search_term = "search term") { - VExprContextSPtrs contexts; - - auto match_expr = - std::make_shared(TExprNodeType::MATCH_PRED); - auto slot_ref = std::make_shared("content", SlotId(1)); - auto literal = std::make_shared(search_term); - - match_expr->_children.push_back(slot_ref); - match_expr->_children.push_back(literal); - - auto context = std::make_shared(match_expr); - contexts.push_back(context); - - return contexts; - } - - std::vector create_mock_rowset_splits(int num_segments = 1) { - std::vector splits; - - auto rowset_meta = std::make_shared(); - auto rowset = std::make_shared( - create_tablet_schema_with_inverted_index(), rowset_meta); - rowset->set_num_segments(num_segments); - - for (int i = 0; i < num_segments; ++i) { - rowset->set_segment_path(i, test_dir_ + "/segment_" + std::to_string(i) + ".dat"); - } - - auto reader = std::make_shared(rowset); - - RowSetSplits split(reader); - splits.push_back(split); - - return splits; - } - - std::unique_ptr stats_; - std::shared_ptr runtime_state_; - std::string test_dir_; -}; - -TEST_F(CollectionStatisticsTest, CollectWithEmptyRowsetSplits) { - auto tablet_schema = create_tablet_schema_with_inverted_index(); - auto expr_contexts = create_match_expr_contexts(); - - std::vector empty_splits; - - auto status = stats_->collect(runtime_state_.get(), empty_splits, tablet_schema, expr_contexts, - nullptr); - EXPECT_TRUE(status.ok()) << status.msg(); -} - -TEST_F(CollectionStatisticsTest, CollectWithEmptyExpressions) { - auto tablet_schema = create_tablet_schema_with_inverted_index(); - VExprContextSPtrs empty_contexts; - - std::vector empty_splits; - - auto status = stats_->collect(runtime_state_.get(), empty_splits, tablet_schema, empty_contexts, - nullptr); - EXPECT_TRUE(status.ok()) << status.msg(); -} - -TEST_F(CollectionStatisticsTest, CollectWithNonMatchExpression) { - auto tablet_schema = create_tablet_schema_with_inverted_index(); - - VExprContextSPtrs contexts; - auto non_match_expr = - std::make_shared(TExprNodeType::BINARY_PRED); - auto context = std::make_shared(non_match_expr); - contexts.push_back(context); - - std::vector empty_splits; - - auto status = - stats_->collect(runtime_state_.get(), empty_splits, tablet_schema, contexts, nullptr); - EXPECT_TRUE(status.ok()) << status.msg(); -} - -TEST_F(CollectionStatisticsTest, CollectWithMultipleMatchExpressions) { - auto tablet_schema = create_tablet_schema_with_inverted_index(); - - VExprContextSPtrs contexts; - - auto match_expr1 = - std::make_shared(TExprNodeType::MATCH_PRED); - auto slot_ref1 = std::make_shared("content", SlotId(1)); - auto literal1 = std::make_shared("term1"); - match_expr1->_children.push_back(slot_ref1); - match_expr1->_children.push_back(literal1); - contexts.push_back(std::make_shared(match_expr1)); - - auto match_expr2 = - std::make_shared(TExprNodeType::MATCH_PRED); - auto slot_ref2 = std::make_shared("content", SlotId(1)); - auto literal2 = std::make_shared("term2"); - match_expr2->_children.push_back(slot_ref2); - match_expr2->_children.push_back(literal2); - contexts.push_back(std::make_shared(match_expr2)); - - std::vector empty_splits; - - auto status = - stats_->collect(runtime_state_.get(), empty_splits, tablet_schema, contexts, nullptr); - EXPECT_TRUE(status.ok()) << status.msg(); -} - -TEST_F(CollectionStatisticsTest, CollectWithNestedExpressions) { - auto tablet_schema = create_tablet_schema_with_inverted_index(); - - VExprContextSPtrs contexts; - - auto and_expr = std::make_shared(TExprNodeType::BINARY_PRED); - - auto match_expr = std::make_shared(TExprNodeType::MATCH_PRED); - auto slot_ref = std::make_shared("content", SlotId(1)); - auto literal = std::make_shared("nested term"); - match_expr->_children.push_back(slot_ref); - match_expr->_children.push_back(literal); - - auto other_expr = - std::make_shared(TExprNodeType::BINARY_PRED); - - and_expr->_children.push_back(match_expr); - and_expr->_children.push_back(other_expr); - - contexts.push_back(std::make_shared(and_expr)); - - std::vector empty_splits; - - auto status = - stats_->collect(runtime_state_.get(), empty_splits, tablet_schema, contexts, nullptr); - EXPECT_TRUE(status.ok()) << status.msg(); -} - -TEST_F(CollectionStatisticsTest, CollectWithMockRowsetSplits) { - auto tablet_schema = create_tablet_schema_with_inverted_index(); - auto expr_contexts = create_match_expr_contexts(); - - auto splits = create_mock_rowset_splits(2); - - auto status = - stats_->collect(runtime_state_.get(), splits, tablet_schema, expr_contexts, nullptr); - - EXPECT_TRUE(status.ok()); -} - -TEST_F(CollectionStatisticsTest, CollectWithEmptySegments) { - auto tablet_schema = create_tablet_schema_with_inverted_index(); - auto expr_contexts = create_match_expr_contexts(); - - auto splits = create_mock_rowset_splits(0); - - auto status = - stats_->collect(runtime_state_.get(), splits, tablet_schema, expr_contexts, nullptr); - EXPECT_TRUE(status.ok()) << status.msg(); -} - -TEST_F(CollectionStatisticsTest, CollectWithMultipleRowsetSplits) { - auto tablet_schema = create_tablet_schema_with_inverted_index(); - auto expr_contexts = create_match_expr_contexts(); - - std::vector splits; - - for (int i = 0; i < 3; ++i) { - auto rowset_meta = std::make_shared(); - auto rowset = - std::make_shared(tablet_schema, rowset_meta); - rowset->set_num_segments(0); - - auto reader = std::make_shared(rowset); - - RowSetSplits split(reader); - splits.push_back(split); - } - - auto status = - stats_->collect(runtime_state_.get(), splits, tablet_schema, expr_contexts, nullptr); - EXPECT_TRUE(status.ok()) << status.msg(); -} - -class TestableCollectionStatistics : public CollectionStatistics { -public: - void set_total_num_docs(uint64_t num_docs) { _total_num_docs = num_docs; } - - void set_total_num_tokens(const std::wstring& field_name, uint64_t num_tokens) { - _total_num_tokens[field_name] = num_tokens; - } - - void set_term_doc_freq(const std::wstring& field_name, const std::wstring& term, - uint64_t freq) { - _term_doc_freqs[field_name][term] = freq; - } -}; - -class CollectionStatisticsDetailedTest : public ::testing::Test { -protected: - void SetUp() override { stats_ = std::make_unique(); } - - void TearDown() override { stats_.reset(); } - - std::unique_ptr stats_; -}; - -TEST_F(CollectionStatisticsDetailedTest, GetStatisticsWithValidData) { - std::wstring field_name = L"test_field"; - std::wstring term = L"test_term"; - - stats_->set_total_num_docs(1000); - stats_->set_total_num_tokens(field_name, 5000); - stats_->set_term_doc_freq(field_name, term, 100); - - EXPECT_EQ(stats_->get_doc_num(), 1000); - EXPECT_EQ(stats_->get_total_term_cnt_by_col(field_name), 5000); - EXPECT_EQ(stats_->get_term_doc_freq_by_col(field_name, term), 100); - - float expected_avg_dl = 5000.0f / 1000.0f; - EXPECT_FLOAT_EQ(stats_->get_or_calculate_avg_dl(field_name), expected_avg_dl); - - float expected_idf = std::log(1 + (1000 - 100 + 0.5) / (100 + 0.5)); - EXPECT_FLOAT_EQ(stats_->get_or_calculate_idf(field_name, term), expected_idf); -} - -TEST_F(CollectionStatisticsDetailedTest, GetStatisticsThrowsWhenDataNotExists) { - std::wstring nonexistent_field = L"nonexistent"; - std::wstring nonexistent_term = L"nonexistent"; - - // Test exceptions for missing data - EXPECT_THROW(stats_->get_doc_num(), Exception); - EXPECT_THROW(stats_->get_total_term_cnt_by_col(nonexistent_field), Exception); - EXPECT_THROW(stats_->get_term_doc_freq_by_col(nonexistent_field, nonexistent_term), Exception); - EXPECT_THROW(stats_->get_or_calculate_avg_dl(nonexistent_field), Exception); - EXPECT_THROW(stats_->get_or_calculate_idf(nonexistent_field, nonexistent_term), Exception); -} - -TEST_F(CollectionStatisticsDetailedTest, CachingMechanismWorks) { - std::wstring field_name = L"test_field"; - std::wstring term = L"test_term"; - - stats_->set_total_num_docs(1000); - stats_->set_total_num_tokens(field_name, 5000); - stats_->set_term_doc_freq(field_name, term, 100); - - float first_avg_dl = stats_->get_or_calculate_avg_dl(field_name); - float first_idf = stats_->get_or_calculate_idf(field_name, term); - - stats_->set_total_num_docs(2000); - stats_->set_total_num_tokens(field_name, 10000); - stats_->set_term_doc_freq(field_name, term, 200); - - float second_avg_dl = stats_->get_or_calculate_avg_dl(field_name); - float second_idf = stats_->get_or_calculate_idf(field_name, term); - - EXPECT_FLOAT_EQ(first_avg_dl, second_avg_dl); - EXPECT_FLOAT_EQ(first_idf, second_idf); -} - -TEST_F(CollectionStatisticsDetailedTest, HandlesZeroValuesCorrectly) { - std::wstring field_name = L"test_field"; - std::wstring term = L"test_term"; - - stats_->set_total_num_docs(0); - EXPECT_THROW(stats_->get_doc_num(), Exception); - - stats_->set_total_num_docs(100); - stats_->set_total_num_tokens(field_name, 0); - stats_->set_term_doc_freq(field_name, term, 0); - - EXPECT_EQ(stats_->get_total_term_cnt_by_col(field_name), 0); - EXPECT_EQ(stats_->get_term_doc_freq_by_col(field_name, term), 0); - EXPECT_FLOAT_EQ(stats_->get_or_calculate_avg_dl(field_name), 0.0f); -} - -TEST_F(CollectionStatisticsDetailedTest, IdfCalculationWithDifferentFrequencies) { - std::wstring field_name = L"test_field"; - std::wstring common_term = L"common_term"; - std::wstring rare_term = L"rare_term"; - - stats_->set_total_num_docs(1000); - stats_->set_term_doc_freq(field_name, common_term, 500); - stats_->set_term_doc_freq(field_name, rare_term, 10); - - float common_idf = stats_->get_or_calculate_idf(field_name, common_term); - float rare_idf = stats_->get_or_calculate_idf(field_name, rare_term); - - EXPECT_GT(rare_idf, common_idf); - EXPECT_GT(common_idf, 0); - EXPECT_GT(rare_idf, 0); -} - -TEST_F(CollectionStatisticsTest, CollectWithCastWrappedSlotRef) { - auto tablet_schema = create_tablet_schema_with_inverted_index(); - - VExprContextSPtrs contexts; - - // match_pred(left: CAST(slot_ref), right: literal) - auto match_expr = std::make_shared(TExprNodeType::MATCH_PRED); - auto cast_expr = std::make_shared(TExprNodeType::CAST_EXPR); - auto slot_ref = std::make_shared("content", SlotId(1)); - auto literal = std::make_shared("cast term"); - - cast_expr->_children.push_back(slot_ref); - match_expr->_children.push_back(cast_expr); - match_expr->_children.push_back(literal); - - contexts.push_back(std::make_shared(match_expr)); - - std::vector empty_splits; - auto status = - stats_->collect(runtime_state_.get(), empty_splits, tablet_schema, contexts, nullptr); - EXPECT_TRUE(status.ok()) << status.msg(); -} - -TEST_F(CollectionStatisticsTest, CollectWithDoubleCastWrappedSlotRef) { - auto tablet_schema = create_tablet_schema_with_inverted_index(); - - VExprContextSPtrs contexts; - - // match_pred(left: CAST(CAST(slot_ref)), right: literal) - auto match_expr = std::make_shared(TExprNodeType::MATCH_PRED); - auto outer_cast = std::make_shared(TExprNodeType::CAST_EXPR); - auto inner_cast = std::make_shared(TExprNodeType::CAST_EXPR); - auto slot_ref = std::make_shared("content", SlotId(1)); - auto literal = std::make_shared("double cast term"); - - inner_cast->_children.push_back(slot_ref); - outer_cast->_children.push_back(inner_cast); - match_expr->_children.push_back(outer_cast); - match_expr->_children.push_back(literal); - - contexts.push_back(std::make_shared(match_expr)); - - std::vector empty_splits; - auto status = - stats_->collect(runtime_state_.get(), empty_splits, tablet_schema, contexts, nullptr); - EXPECT_TRUE(status.ok()) << status.msg(); -} - -// Regression for AIR-36: match score collection must resolve indexes for -// variant sub-columns whose indexes live in _path_set_info_map (typed paths or -// inherited sub-column indexes). The previous simple lookup using -// inverted_indexs(col_unique_id, suffix_path) missed those indexes. -TEST_F(CollectionStatisticsTest, ExtractCollectInfoForVariantSubcolumnIndex) { - auto tablet_schema = std::make_shared(); - - constexpr int32_t kVariantUid = 9001; - - TabletColumn variant_col; - variant_col.set_unique_id(kVariantUid); - variant_col.set_name("v"); - variant_col.set_type(FieldType::OLAP_FIELD_TYPE_VARIANT); - tablet_schema->append_column(variant_col); - - TabletColumn sub_col; - sub_col.set_unique_id(-1); - sub_col.set_name("v.host"); - sub_col.set_type(FieldType::OLAP_FIELD_TYPE_STRING); - sub_col.set_parent_unique_id(kVariantUid); - PathInData path("v.host"); - sub_col.set_path_info(path); - tablet_schema->append_column(sub_col); - - auto sub_index = std::make_shared(); - TabletIndexPB index_pb; - index_pb.set_index_id(2001); - index_pb.set_index_name("variant_subcolumn_idx"); - index_pb.set_index_type(IndexType::INVERTED); - index_pb.add_col_unique_id(kVariantUid); - auto* props = index_pb.mutable_properties(); - (*props)["parser"] = "standard"; - (*props)["support_phrase"] = "true"; - sub_index->init_from_pb(index_pb); - - TabletSchema::PathsSetInfo path_set_info; - TabletIndexes sub_indexes = {sub_index}; - path_set_info.subcolumn_indexes["host"] = sub_indexes; - std::unordered_map path_set_info_map; - path_set_info_map[kVariantUid] = std::move(path_set_info); - tablet_schema->set_path_set_info(std::move(path_set_info_map)); - - EXPECT_TRUE(tablet_schema->inverted_indexs(kVariantUid, "host").empty()); - - auto found = tablet_schema->inverted_indexs(tablet_schema->column(/*ordinal=*/1)); - ASSERT_EQ(found.size(), 1u); - EXPECT_EQ(found[0]->index_name(), "variant_subcolumn_idx"); - - constexpr int kSlotId = 42; - runtime_state_->_mock_desc_tbl->add_slot_descriptor(SlotId(kSlotId), kVariantUid); - - auto match_expr = std::make_shared(TExprNodeType::MATCH_PRED); - auto slot_ref = - std::make_shared("v.host", SlotId(kSlotId)); - auto literal = std::make_shared("foo"); - match_expr->_children.push_back(slot_ref); - match_expr->_children.push_back(literal); - - VExprContextSPtrs contexts; - contexts.push_back(std::make_shared(match_expr)); - - std::unordered_map collect_infos; - auto status = stats_->extract_collect_info(runtime_state_.get(), contexts, tablet_schema, - &collect_infos); - ASSERT_TRUE(status.ok()) << status.msg(); - ASSERT_EQ(collect_infos.size(), 1u); - auto it = collect_infos.find(StringHelper::to_wstring(std::to_string(kVariantUid) + ".v.host")); - ASSERT_NE(it, collect_infos.end()); - ASSERT_NE(it->second.index_meta, nullptr); - EXPECT_EQ(it->second.index_meta->index_name(), "variant_subcolumn_idx"); -} - -// Regression for score on a dynamic variant sub-column inherited from a plain -// parent variant inverted index (no field_pattern template). Matches the -// scan-time schema shape: _init_variant_columns materializes the accessed -// path as an extracted VARIANT placeholder, so neither inverted_indexs(column) -// nor generate_sub_column_info resolves the parent index. Collector clones -// the parent's non-field-pattern indexes with the variant path as suffix. -TEST_F(CollectionStatisticsTest, ExtractCollectInfoForVariantParentIndexWithoutTemplate) { - auto tablet_schema = std::make_shared(); - - constexpr int32_t kVariantUid = 9004; - - TabletColumn variant_col; - variant_col.set_unique_id(kVariantUid); - variant_col.set_name("v"); - variant_col.set_type(FieldType::OLAP_FIELD_TYPE_VARIANT); - tablet_schema->append_column(variant_col); - - TabletColumn sub_col; - sub_col.set_unique_id(-1); - sub_col.set_name("v.key"); - sub_col.set_type(FieldType::OLAP_FIELD_TYPE_VARIANT); - sub_col.set_parent_unique_id(kVariantUid); - PathInData path("v.key"); - sub_col.set_path_info(path); - tablet_schema->append_column(sub_col); - - TabletIndexPB index_pb; - index_pb.set_index_id(2004); - index_pb.set_index_name("variant_parent_idx"); - index_pb.set_index_type(IndexType::INVERTED); - index_pb.add_col_unique_id(kVariantUid); - auto* props = index_pb.mutable_properties(); - (*props)["parser"] = "english"; - (*props)["support_phrase"] = "true"; - - TabletIndex index; - index.init_from_pb(index_pb); - tablet_schema->append_index(std::move(index)); - - // Pre-conditions: column-aware lookup is empty (no inheritance pre-populated) - // and generate_sub_column_info returns false (no field_pattern template). - // The collector must still resolve through the VARIANT-placeholder branch. - ASSERT_TRUE(tablet_schema->inverted_indexs(tablet_schema->column(/*ordinal=*/1)).empty()); - ASSERT_EQ(tablet_schema->inverted_indexs(kVariantUid).size(), 1u); - TabletSchema::SubColumnInfo sub_column_info; - ASSERT_FALSE(variant_util::generate_sub_column_info(*tablet_schema, kVariantUid, "key", - &sub_column_info)); - - constexpr int kSlotId = 45; - runtime_state_->_mock_desc_tbl->add_slot_descriptor(SlotId(kSlotId), kVariantUid, "v.key", - {"key"}); - - auto match_expr = std::make_shared(TExprNodeType::MATCH_PRED); - auto cast_expr = std::make_shared(TExprNodeType::CAST_EXPR); - cast_expr->_data_type = std::make_shared(); - auto slot_ref = std::make_shared("v.key", SlotId(kSlotId)); - auto literal = std::make_shared("abc"); - cast_expr->_children.push_back(slot_ref); - match_expr->_children.push_back(cast_expr); - match_expr->_children.push_back(literal); - - VExprContextSPtrs contexts; - contexts.push_back(std::make_shared(match_expr)); - - std::unordered_map collect_infos; - auto status = stats_->extract_collect_info(runtime_state_.get(), contexts, tablet_schema, - &collect_infos); - ASSERT_TRUE(status.ok()) << status.msg(); - ASSERT_EQ(collect_infos.size(), 1u); - auto it = collect_infos.find(StringHelper::to_wstring(std::to_string(kVariantUid) + ".v.key")); - ASSERT_NE(it, collect_infos.end()); - ASSERT_NE(it->second.index_meta, nullptr); - ASSERT_NE(it->second.owned_index_meta, nullptr); - EXPECT_EQ(it->second.index_meta->index_name(), "variant_parent_idx"); -} - -namespace { - -// Build a sub-column template for the parent variant column. pattern_type has no -// public setter on TabletColumn, so construct through ColumnPB. -TabletColumn make_subcolumn_template(const std::string& pattern, PatternTypePB pattern_type) { - ColumnPB column_pb; - column_pb.set_unique_id(-1); - column_pb.set_name(pattern); - column_pb.set_type("STRING"); - column_pb.set_is_nullable(true); - column_pb.set_pattern_type(pattern_type); - - TabletColumn templ; - templ.init_from_pb(column_pb); - return templ; -} - -} // namespace - -TEST_F(CollectionStatisticsTest, ExtractCollectInfoForVariantFieldPatternIndex) { - auto tablet_schema = std::make_shared(); - - constexpr int32_t kVariantUid = 9002; - - TabletColumn variant_col; - variant_col.set_unique_id(kVariantUid); - variant_col.set_name("meta"); - variant_col.set_type(FieldType::OLAP_FIELD_TYPE_VARIANT); - TabletColumn host_template = make_subcolumn_template("host", PatternTypePB::MATCH_NAME); - variant_col.add_sub_column(host_template); - tablet_schema->append_column(variant_col); - - TabletColumn sub_col; - sub_col.set_unique_id(-1); - sub_col.set_name("meta.host"); - sub_col.set_type(FieldType::OLAP_FIELD_TYPE_STRING); - sub_col.set_parent_unique_id(kVariantUid); - PathInData path("meta.host"); - sub_col.set_path_info(path); - tablet_schema->append_column(sub_col); - - TabletIndexPB index_pb; - index_pb.set_index_id(2002); - index_pb.set_index_name("variant_field_pattern_idx"); - index_pb.set_index_type(IndexType::INVERTED); - index_pb.add_col_unique_id(kVariantUid); - auto* props = index_pb.mutable_properties(); - (*props)["parser"] = "standard"; - (*props)["support_phrase"] = "true"; - (*props)["field_pattern"] = "host"; - - TabletIndex index; - index.init_from_pb(index_pb); - tablet_schema->append_index(std::move(index)); - - ASSERT_TRUE(tablet_schema->inverted_indexs(tablet_schema->column(/*ordinal=*/1)).empty()); - ASSERT_EQ(tablet_schema->inverted_index_by_field_pattern(kVariantUid, "host").size(), 1u); - - constexpr int kSlotId = 43; - runtime_state_->_mock_desc_tbl->add_slot_descriptor(SlotId(kSlotId), kVariantUid, "meta.host", - {"host"}); - - auto match_expr = std::make_shared(TExprNodeType::MATCH_PRED); - auto slot_ref = - std::make_shared("meta.host", SlotId(kSlotId)); - auto literal = std::make_shared("alpha"); - match_expr->_children.push_back(slot_ref); - match_expr->_children.push_back(literal); - - VExprContextSPtrs contexts; - contexts.push_back(std::make_shared(match_expr)); - - std::unordered_map collect_infos; - auto status = stats_->extract_collect_info(runtime_state_.get(), contexts, tablet_schema, - &collect_infos); - ASSERT_TRUE(status.ok()) << status.msg(); - ASSERT_EQ(collect_infos.size(), 1u); - auto it = collect_infos.find( - StringHelper::to_wstring(std::to_string(kVariantUid) + ".meta.host")); - ASSERT_NE(it, collect_infos.end()); - ASSERT_NE(it->second.index_meta, nullptr); - ASSERT_NE(it->second.owned_index_meta, nullptr); - EXPECT_EQ(it->second.index_meta->index_name(), "variant_field_pattern_idx"); -} - -// Regression: field_pattern="user.*" is registered under the pattern string, -// while the query slot resolves to column_paths=["user", "name"]. The fallback -// must match the parent variant's sub-column template first, then use the -// matched pattern to fetch the index, and collect under the actual Lucene field. -TEST_F(CollectionStatisticsTest, ExtractCollectInfoForVariantFieldPatternGlobIndex) { - auto tablet_schema = std::make_shared(); - - constexpr int32_t kVariantUid = 9003; - - TabletColumn variant_col; - variant_col.set_unique_id(kVariantUid); - variant_col.set_name("meta"); - variant_col.set_type(FieldType::OLAP_FIELD_TYPE_VARIANT); - TabletColumn glob_template = make_subcolumn_template("user.*", PatternTypePB::MATCH_NAME_GLOB); - variant_col.add_sub_column(glob_template); - tablet_schema->append_column(variant_col); - - TabletColumn sub_col; - sub_col.set_unique_id(-1); - sub_col.set_name("meta.user.name"); - sub_col.set_type(FieldType::OLAP_FIELD_TYPE_STRING); - sub_col.set_parent_unique_id(kVariantUid); - PathInData path("meta.user.name"); - sub_col.set_path_info(path); - tablet_schema->append_column(sub_col); - - TabletIndexPB index_pb; - index_pb.set_index_id(2003); - index_pb.set_index_name("variant_field_pattern_glob_idx"); - index_pb.set_index_type(IndexType::INVERTED); - index_pb.add_col_unique_id(kVariantUid); - auto* props = index_pb.mutable_properties(); - (*props)["parser"] = "standard"; - (*props)["support_phrase"] = "true"; - (*props)["field_pattern"] = "user.*"; - - TabletIndex index; - index.init_from_pb(index_pb); - tablet_schema->append_index(std::move(index)); - - ASSERT_TRUE(tablet_schema->inverted_indexs(tablet_schema->column(/*ordinal=*/1)).empty()); - ASSERT_TRUE(tablet_schema->inverted_index_by_field_pattern(kVariantUid, "user.name").empty()); - ASSERT_EQ(tablet_schema->inverted_index_by_field_pattern(kVariantUid, "user.*").size(), 1u); - TabletSchema::SubColumnInfo sub_column_info; - ASSERT_TRUE(variant_util::generate_sub_column_info(*tablet_schema, kVariantUid, "user.name", - &sub_column_info)); - ASSERT_EQ(sub_column_info.indexes.size(), 1u); - EXPECT_EQ(sub_column_info.column.suffix_path(), "meta.user.name"); - EXPECT_EQ(sub_column_info.indexes[0]->index_name(), "variant_field_pattern_glob_idx"); - - constexpr int kSlotId = 44; - runtime_state_->_mock_desc_tbl->add_slot_descriptor(SlotId(kSlotId), kVariantUid, - "meta.user.name", {"user", "name"}); - - auto match_expr = std::make_shared(TExprNodeType::MATCH_PRED); - auto slot_ref = std::make_shared("meta.user.name", - SlotId(kSlotId)); - auto literal = std::make_shared("alice"); - match_expr->_children.push_back(slot_ref); - match_expr->_children.push_back(literal); - - VExprContextSPtrs contexts; - contexts.push_back(std::make_shared(match_expr)); - - std::unordered_map collect_infos; - auto status = stats_->extract_collect_info(runtime_state_.get(), contexts, tablet_schema, - &collect_infos); - ASSERT_TRUE(status.ok()) << status.msg(); - ASSERT_EQ(collect_infos.size(), 1u); - auto it = collect_infos.find( - StringHelper::to_wstring(std::to_string(kVariantUid) + ".meta.user.name")); - ASSERT_NE(it, collect_infos.end()); - ASSERT_NE(it->second.index_meta, nullptr); - ASSERT_NE(it->second.owned_index_meta, nullptr); - EXPECT_EQ(it->second.index_meta->index_name(), "variant_field_pattern_glob_idx"); -} - -// E1: Match predicate whose left subtree contains no VSlotRef. -// find_slot_ref recurses through children; when it returns nullptr the -// collector reports INVERTED_INDEX_NOT_SUPPORTED. -// Calls MatchPredicateCollector::collect() directly so coverage attribution -// is not muddied by extract_collect_info's virtual-dispatch indirection. -TEST_F(CollectionStatisticsTest, CollectMissingSlotRefReturnsError) { - auto tablet_schema = std::make_shared(); - TabletColumn col; - col.set_unique_id(1001); - col.set_name("c"); - col.set_type(FieldType::OLAP_FIELD_TYPE_STRING); - tablet_schema->append_column(col); - - auto match_expr = std::make_shared(TExprNodeType::MATCH_PRED); - auto literal_left = std::make_shared("foo"); - auto literal_right = std::make_shared("bar"); - match_expr->_children.push_back(literal_left); - match_expr->_children.push_back(literal_right); - - MatchPredicateCollector collector; - std::unordered_map collect_infos; - auto status = - collector.collect(runtime_state_.get(), tablet_schema, match_expr, &collect_infos); - ASSERT_FALSE(status.ok()); - EXPECT_EQ(status.code(), ErrorCode::INVERTED_INDEX_NOT_SUPPORTED); - EXPECT_TRUE(status.msg().find("Cannot find slot reference") != std::string::npos); -} - -// E2: SlotRef points to a slot_id absent from the runtime descriptor table. -TEST_F(CollectionStatisticsTest, CollectMissingSlotDescriptorReturnsError) { - auto tablet_schema = std::make_shared(); - TabletColumn col; - col.set_unique_id(1002); - col.set_name("c"); - col.set_type(FieldType::OLAP_FIELD_TYPE_STRING); - tablet_schema->append_column(col); - - constexpr int kAbsentSlotId = 99999; - - auto match_expr = std::make_shared(TExprNodeType::MATCH_PRED); - auto slot_ref = - std::make_shared("c", SlotId(kAbsentSlotId)); - auto literal = std::make_shared("v"); - match_expr->_children.push_back(slot_ref); - match_expr->_children.push_back(literal); - - MatchPredicateCollector collector; - std::unordered_map collect_infos; - auto status = - collector.collect(runtime_state_.get(), tablet_schema, match_expr, &collect_infos); - ASSERT_FALSE(status.ok()); - EXPECT_EQ(status.code(), ErrorCode::INVERTED_INDEX_NOT_SUPPORTED); - EXPECT_TRUE(status.msg().find("Cannot find slot descriptor") != std::string::npos); -} - -// E3: SlotRef name does not exist in tablet_schema (field_index returns -1). -TEST_F(CollectionStatisticsTest, CollectUnknownColumnNameReturnsError) { - auto tablet_schema = std::make_shared(); - TabletColumn col; - col.set_unique_id(1003); - col.set_name("declared"); - col.set_type(FieldType::OLAP_FIELD_TYPE_STRING); - tablet_schema->append_column(col); - - constexpr int kSlotId = 50; - runtime_state_->_mock_desc_tbl->add_slot_descriptor(SlotId(kSlotId), 1003, "missing", {}); - - auto match_expr = std::make_shared(TExprNodeType::MATCH_PRED); - auto slot_ref = - std::make_shared("missing", SlotId(kSlotId)); - auto literal = std::make_shared("v"); - match_expr->_children.push_back(slot_ref); - match_expr->_children.push_back(literal); - - MatchPredicateCollector collector; - std::unordered_map collect_infos; - auto status = - collector.collect(runtime_state_.get(), tablet_schema, match_expr, &collect_infos); - ASSERT_FALSE(status.ok()); - EXPECT_EQ(status.code(), ErrorCode::INVERTED_INDEX_NOT_SUPPORTED); - EXPECT_TRUE(status.msg().find("Cannot find column index") != std::string::npos); -} - -// I1 + L3 + O1: Plain string column with a direct inverted index. -// Direct hit produces a CollectInfo whose owned_index_meta is null -// (the meta lives in the schema and is not cloned). -TEST_F(CollectionStatisticsTest, CollectDirectIndexHitFromSchema) { - auto tablet_schema = std::make_shared(); - - constexpr int32_t kColUid = 1100; - TabletColumn col; - col.set_unique_id(kColUid); - col.set_name("note"); - col.set_type(FieldType::OLAP_FIELD_TYPE_STRING); - tablet_schema->append_column(col); - - TabletIndexPB index_pb; - index_pb.set_index_id(2100); - index_pb.set_index_name("note_idx"); - index_pb.set_index_type(IndexType::INVERTED); - index_pb.add_col_unique_id(kColUid); - auto* props = index_pb.mutable_properties(); - (*props)["parser"] = "english"; - (*props)["support_phrase"] = "true"; - TabletIndex index; - index.init_from_pb(index_pb); - tablet_schema->append_index(std::move(index)); - - constexpr int kSlotId = 60; - runtime_state_->_mock_desc_tbl->add_slot_descriptor(SlotId(kSlotId), kColUid, "note", {}); - - auto match_expr = std::make_shared(TExprNodeType::MATCH_PRED); - auto slot_ref = std::make_shared("note", SlotId(kSlotId)); - auto literal = std::make_shared("hello world"); - match_expr->_children.push_back(slot_ref); - match_expr->_children.push_back(literal); - - MatchPredicateCollector collector; - std::unordered_map collect_infos; - auto status = - collector.collect(runtime_state_.get(), tablet_schema, match_expr, &collect_infos); - ASSERT_TRUE(status.ok()) << status.msg(); - ASSERT_EQ(collect_infos.size(), 1u); - auto it = collect_infos.find(StringHelper::to_wstring(std::to_string(kColUid))); - ASSERT_NE(it, collect_infos.end()); - EXPECT_NE(it->second.index_meta, nullptr); - EXPECT_EQ(it->second.owned_index_meta, nullptr); // O1: schema-direct meta is not owned - EXPECT_FALSE(it->second.term_infos.empty()); -} - -// I2: Plain string column with no index and not an extracted variant -// sub-column. Fallback path does not apply (column.is_extracted_column() -// is false). In BE_TEST builds the empty-index check is skipped, so -// collect returns OK with no CollectInfo emitted. -TEST_F(CollectionStatisticsTest, CollectNotExtractedColumnSkipsFallback) { - auto tablet_schema = std::make_shared(); - - constexpr int32_t kColUid = 1200; - TabletColumn col; - col.set_unique_id(kColUid); - col.set_name("plain"); - col.set_type(FieldType::OLAP_FIELD_TYPE_STRING); - tablet_schema->append_column(col); - // no index appended - - constexpr int kSlotId = 70; - runtime_state_->_mock_desc_tbl->add_slot_descriptor(SlotId(kSlotId), kColUid, "plain", {}); - - auto match_expr = std::make_shared(TExprNodeType::MATCH_PRED); - auto slot_ref = std::make_shared("plain", SlotId(kSlotId)); - auto literal = std::make_shared("v"); - match_expr->_children.push_back(slot_ref); - match_expr->_children.push_back(literal); - - MatchPredicateCollector collector; - std::unordered_map collect_infos; - auto status = - collector.collect(runtime_state_.get(), tablet_schema, match_expr, &collect_infos); - ASSERT_TRUE(status.ok()) << status.msg(); - EXPECT_TRUE(collect_infos.empty()); -} - -// L1: Index whose properties do not request an analyzer -// (should_analyzer returns false). The matching index_meta is iterated -// but skipped before insertion. -TEST_F(CollectionStatisticsTest, CollectSkipsIndexWithoutAnalyzer) { - auto tablet_schema = std::make_shared(); - - constexpr int32_t kColUid = 1300; - TabletColumn col; - col.set_unique_id(kColUid); - col.set_name("kw"); - col.set_type(FieldType::OLAP_FIELD_TYPE_STRING); - tablet_schema->append_column(col); - - TabletIndexPB index_pb; - index_pb.set_index_id(2300); - index_pb.set_index_name("kw_idx"); - index_pb.set_index_type(IndexType::INVERTED); - index_pb.add_col_unique_id(kColUid); - // No "parser" property -> should_analyzer returns false - TabletIndex index; - index.init_from_pb(index_pb); - tablet_schema->append_index(std::move(index)); - - constexpr int kSlotId = 80; - runtime_state_->_mock_desc_tbl->add_slot_descriptor(SlotId(kSlotId), kColUid, "kw", {}); - - auto match_expr = std::make_shared(TExprNodeType::MATCH_PRED); - auto slot_ref = std::make_shared("kw", SlotId(kSlotId)); - auto literal = std::make_shared("v"); - match_expr->_children.push_back(slot_ref); - match_expr->_children.push_back(literal); - - MatchPredicateCollector collector; - std::unordered_map collect_infos; - auto status = - collector.collect(runtime_state_.get(), tablet_schema, match_expr, &collect_infos); - ASSERT_TRUE(status.ok()) << status.msg(); - EXPECT_TRUE(collect_infos.empty()); -} - -// L2: Index whose analyzer is set (should_analyzer returns true) but does -// not declare "support_phrase=true". MockVExpr drives MATCH_PHRASE opcode, -// so is_need_similarity_score returns false and the index is skipped. -TEST_F(CollectionStatisticsTest, CollectSkipsIndexWithoutSimilarityScore) { - auto tablet_schema = std::make_shared(); - - constexpr int32_t kColUid = 1350; - TabletColumn col; - col.set_unique_id(kColUid); - col.set_name("body"); - col.set_type(FieldType::OLAP_FIELD_TYPE_STRING); - tablet_schema->append_column(col); - - TabletIndexPB index_pb; - index_pb.set_index_id(2350); - index_pb.set_index_name("body_idx"); - index_pb.set_index_type(IndexType::INVERTED); - index_pb.add_col_unique_id(kColUid); - auto* props = index_pb.mutable_properties(); - (*props)["parser"] = "english"; // should_analyzer == true - // Intentionally omit "support_phrase" -> is_need_similarity_score == false - TabletIndex index; - index.init_from_pb(index_pb); - tablet_schema->append_index(std::move(index)); - - constexpr int kSlotId = 85; - runtime_state_->_mock_desc_tbl->add_slot_descriptor(SlotId(kSlotId), kColUid, "body", {}); - - auto match_expr = std::make_shared(TExprNodeType::MATCH_PRED); - auto slot_ref = std::make_shared("body", SlotId(kSlotId)); - auto literal = std::make_shared("hello"); - match_expr->_children.push_back(slot_ref); - match_expr->_children.push_back(literal); - - MatchPredicateCollector collector; - std::unordered_map collect_infos; - auto status = - collector.collect(runtime_state_.get(), tablet_schema, match_expr, &collect_infos); - ASSERT_TRUE(status.ok()) << status.msg(); - EXPECT_TRUE(collect_infos.empty()); -} - -// L4: Two MATCH predicates on the same column produce CollectInfo entries -// keyed on the same field_name; the second insertion merges term_infos -// into the first entry. -TEST_F(CollectionStatisticsTest, CollectMergesTermsForSameFieldName) { - auto tablet_schema = std::make_shared(); - - constexpr int32_t kColUid = 1400; - TabletColumn col; - col.set_unique_id(kColUid); - col.set_name("doc"); - col.set_type(FieldType::OLAP_FIELD_TYPE_STRING); - tablet_schema->append_column(col); - - TabletIndexPB index_pb; - index_pb.set_index_id(2400); - index_pb.set_index_name("doc_idx"); - index_pb.set_index_type(IndexType::INVERTED); - index_pb.add_col_unique_id(kColUid); - auto* props = index_pb.mutable_properties(); - (*props)["parser"] = "english"; - (*props)["support_phrase"] = "true"; - TabletIndex index; - index.init_from_pb(index_pb); - tablet_schema->append_index(std::move(index)); - - constexpr int kSlotId = 90; - runtime_state_->_mock_desc_tbl->add_slot_descriptor(SlotId(kSlotId), kColUid, "doc", {}); - - auto build_match = [&](const std::string& term) { - auto m = std::make_shared(TExprNodeType::MATCH_PRED); - auto s = std::make_shared("doc", SlotId(kSlotId)); - auto l = std::make_shared(term); - m->_children.push_back(s); - m->_children.push_back(l); - return m; - }; - - MatchPredicateCollector collector; - std::unordered_map collect_infos; - auto first = collector.collect(runtime_state_.get(), tablet_schema, build_match("alpha"), - &collect_infos); - ASSERT_TRUE(first.ok()) << first.msg(); - auto second = collector.collect(runtime_state_.get(), tablet_schema, build_match("beta"), - &collect_infos); - ASSERT_TRUE(second.ok()) << second.msg(); - ASSERT_EQ(collect_infos.size(), 1u); - auto it = collect_infos.find(StringHelper::to_wstring(std::to_string(kColUid))); - ASSERT_NE(it, collect_infos.end()); - EXPECT_GE(it->second.term_infos.size(), 2u); // both "alpha" and "beta" present -} - -// Test-only subclass that exposes the protected helpers of PredicateCollector. -class TestablePredicateCollector : public MatchPredicateCollector { -public: - using MatchPredicateCollector::build_field_name; - using MatchPredicateCollector::find_slot_ref; -}; - -// find_slot_ref: null shared_ptr returns nullptr (early-return branch). -TEST_F(CollectionStatisticsTest, FindSlotRefHandlesNullExpr) { - TestablePredicateCollector collector; - VExprSPtr null_expr; - EXPECT_EQ(collector.find_slot_ref(null_expr), nullptr); -} - -// find_slot_ref: when expr is a non-CAST wrapper containing a SLOT_REF in its -// children, the recursive descent finds the slot via the for-loop body. -TEST_F(CollectionStatisticsTest, FindSlotRefRecursesIntoChildren) { - TestablePredicateCollector collector; - auto wrapper = std::make_shared(TExprNodeType::FUNCTION_CALL); - auto slot_ref = std::make_shared("c", SlotId(99)); - wrapper->_children.push_back(slot_ref); - EXPECT_EQ(collector.find_slot_ref(wrapper), slot_ref.get()); -} - -// find_slot_ref: leaf non-slot (no children) returns nullptr after for-loop. -TEST_F(CollectionStatisticsTest, FindSlotRefReturnsNullForLeafNonSlot) { - TestablePredicateCollector collector; - auto literal = std::make_shared("x"); - EXPECT_EQ(collector.find_slot_ref(literal), nullptr); -} - -// build_field_name: non-empty suffix is appended with a dot separator. -TEST_F(CollectionStatisticsTest, BuildFieldNameWithSuffix) { - TestablePredicateCollector collector; - EXPECT_EQ(collector.build_field_name(42, "a.b"), "42.a.b"); -} - -// build_field_name: empty suffix returns just the unique id as string. -TEST_F(CollectionStatisticsTest, BuildFieldNameWithoutSuffix) { - TestablePredicateCollector collector; - EXPECT_EQ(collector.build_field_name(42, ""), "42"); -} - -TEST(TermInfoComparerTest, OrdersByTermAndDedups) { - using doris::TermInfoComparer; - using doris::segment_v2::TermInfo; - - std::set terms; - - TermInfo t1; - t1.term = std::string("banana"); - t1.position = 2; - - TermInfo t2; - t2.term = std::string("apple"); - t2.position = 10; - - TermInfo t3; - t3.term = std::string("cherry"); - t3.position = 1; - - TermInfo dup; - dup.term = std::string("banana"); - dup.position = 100; - - terms.insert(t1); - terms.insert(t2); - terms.insert(t3); - terms.insert(dup); - - std::vector ordered; - ordered.reserve(terms.size()); - for (const auto& t : terms) { - ordered.push_back(t.get_single_term()); - } - - EXPECT_EQ(terms.size(), 3u); - EXPECT_THAT(ordered, ::testing::ElementsAre("apple", "banana", "cherry")); -} - -} // namespace doris diff --git a/be/test/storage/compaction/segcompaction_test.cpp b/be/test/storage/compaction/segcompaction_test.cpp index d3b843c050da2e..43748f8638c698 100644 --- a/be/test/storage/compaction/segcompaction_test.cpp +++ b/be/test/storage/compaction/segcompaction_test.cpp @@ -19,12 +19,14 @@ #include #include +#include #include #include #include #include #include "common/config.h" +#include "cpp/sync_point.h" #include "io/fs/local_file_system.h" #include "runtime/exec_env.h" #include "runtime/memory/mem_tracker.h" @@ -41,6 +43,8 @@ #include "storage/tablet/tablet_meta.h" #include "storage/tablet/tablet_schema.h" #include "storage/utils.h" +#include "util/debug_points.h" +#include "util/defer_op.h" #include "util/slice.h" namespace doris { @@ -114,6 +118,10 @@ class SegCompactionTest : public testing::Test { } void TearDown() { + auto* sync_point = SyncPoint::get_instance(); + sync_point->disable_processing(); + sync_point->clear_all_call_backs(); + sync_point->clear_trace(); config::enable_segcompaction = false; ExecEnv* exec_env = doris::ExecEnv::GetInstance(); l_engine = nullptr; @@ -288,6 +296,55 @@ class SegCompactionTest : public testing::Test { std::unique_ptr _inverted_index_searcher_cache; }; +// A segcompaction failure that is not one of the retryable ones (out of memory, reader or +// writer init) terminates the write job, and build() must surface that instead of returning +// a rowset built on top of a failed compaction. The row-count check is the cheapest fatal +// failure to provoke: SegcompactionWorker::_check_correctness reports CHECK_LINES_ERROR, +// which falls through to the fatal branch of SegcompactionWorker::compact_segments. +// +// build() reports SEGCOMPACTION_FAILED rather than CHECK_LINES_ERROR because the worker +// hands the writer a bare error code, losing the original status. That is existing +// behavior; this test pins the failure reaching build(), not the code it arrives as. +TEST_F(SegCompactionTest, FatalSegcompactionFailsBuild) { + config::segcompaction_candidate_max_rows = 10; + config::segcompaction_batch_size = 2; + + const auto origin_enable_debug_points = config::enable_debug_points; + config::enable_debug_points = true; + DebugPoints::instance()->add("SegcompactionWorker._check_correctness_wrong_sum_src_row"); + Defer defer([origin_enable_debug_points]() { + DebugPoints::instance()->remove("SegcompactionWorker._check_correctness_wrong_sum_src_row"); + config::enable_debug_points = origin_enable_debug_points; + }); + + auto tablet_schema = std::make_shared(); + create_tablet_schema(tablet_schema, DUP_KEYS); + + RowsetWriterContext writer_context; + create_rowset_writer_context(10046, tablet_schema, &writer_context); + auto writer_result = RowsetFactory::create_rowset_writer(*l_engine, writer_context, false); + ASSERT_TRUE(writer_result.has_value()) << writer_result.error(); + auto rowset_writer = std::move(writer_result).value(); + + for (int segment_id = 0; segment_id < 3; ++segment_id) { + Block block = tablet_schema->create_block(); + auto columns = std::move(block).mutate_columns(); + for (uint32_t column_id = 0; column_id < columns.size(); ++column_id) { + const uint32_t value = segment_id + column_id; + columns[column_id]->insert_data(reinterpret_cast(&value), sizeof(value)); + } + ASSERT_TRUE(add_block_with_columns(rowset_writer.get(), &block, &columns).ok()); + ASSERT_TRUE(rowset_writer->flush().ok()); + } + + RowsetSharedPtr rowset; + // build() waits for the inflight segcompaction, so the failure is visible by the time it + // returns; no polling is needed. + const auto status = rowset_writer->build(rowset); + EXPECT_FALSE(status.ok()) << "expected the fatal segcompaction to fail the build"; + EXPECT_EQ(status.code(), SEGCOMPACTION_FAILED) << status; +} + TEST_F(SegCompactionTest, SegCompactionThenRead) { config::enable_segcompaction = true; Status s; diff --git a/be/test/storage/index/index_builder_test.cpp b/be/test/storage/index/index_builder_test.cpp index 4d5d890c887bb5..ee9542f57fc87e 100644 --- a/be/test/storage/index/index_builder_test.cpp +++ b/be/test/storage/index/index_builder_test.cpp @@ -20,7 +20,12 @@ #include #include +#include + +#include "common/config.h" +#include "storage/index/index_file_reader.h" #include "storage/index/index_writer.h" +#include "storage/index/snii/query/term_query.h" #include "storage/olap_common.h" #include "storage/rowset/beta_rowset.h" #include "storage/rowset/rowset_factory.h" @@ -28,10 +33,29 @@ #include "storage/storage_engine.h" #include "storage/tablet/tablet_fwd.h" #include "storage/tablet/tablet_schema.h" +#include "util/debug_points.h" namespace doris { using namespace testing; +class ScopedIndexBuilderDebugPoints { +public: + ScopedIndexBuilderDebugPoints() : _debug_points_enabled(config::enable_debug_points) { + config::enable_debug_points = true; + DebugPoints::instance()->clear(); + } + + ~ScopedIndexBuilderDebugPoints() { + DebugPoints::instance()->clear(); + config::enable_debug_points = _debug_points_enabled; + } + + void enable(const std::string& name) { DebugPoints::instance()->add(name); } + +private: + bool _debug_points_enabled; +}; + class IndexBuilderTest : public ::testing::Test { protected: void SetUp() override { @@ -170,6 +194,346 @@ class IndexBuilderTest : public ::testing::Test { rs_meta->set_tablet_schema(tablet_schema); } + void prepare_single_index_build(int64_t rowset_id) { + auto tablet_path = _absolute_dir + "/" + std::to_string(rowset_id); + _tablet->_tablet_path = tablet_path; + ASSERT_TRUE(io::global_local_filesystem()->delete_directory(tablet_path).ok()); + ASSERT_TRUE(io::global_local_filesystem()->create_directory(tablet_path).ok()); + + RowsetWriterContext writer_context; + writer_context.rowset_id.init(rowset_id); + writer_context.tablet_id = _tablet->tablet_id(); + writer_context.tablet_schema_hash = _tablet_meta->schema_hash(); + writer_context.partition_id = 10; + writer_context.rowset_type = BETA_ROWSET; + writer_context.tablet_path = tablet_path; + writer_context.rowset_state = VISIBLE; + writer_context.tablet_schema = _tablet_schema; + writer_context.version = Version(10, 10); + + auto result = RowsetFactory::create_rowset_writer(*_engine_ref, writer_context, false); + ASSERT_TRUE(result.has_value()) << result.error(); + auto rowset_writer = std::move(result).value(); + + Block block = _tablet_schema->create_block(); + auto columns = std::move(block).mutate_columns(); + for (int i = 0; i < 8; ++i) { + int32_t k1 = i * 10; + int32_t k2 = i; + columns[0]->insert_data(reinterpret_cast(&k1), sizeof(k1)); + columns[1]->insert_data(reinterpret_cast(&k2), sizeof(k2)); + } + block.set_columns(std::move(columns)); + ASSERT_TRUE(rowset_writer->add_block(&block).ok()); + ASSERT_TRUE(rowset_writer->flush().ok()); + + RowsetSharedPtr rowset; + ASSERT_TRUE(rowset_writer->build(rowset).ok()); + ASSERT_TRUE(_tablet->add_rowset(rowset).ok()); + + TOlapTableIndex index; + index.index_id = 101; + index.index_name = "k1_index"; + index.columns.emplace_back("k1"); + index.column_unique_ids.push_back(1); + index.index_type = TIndexType::INVERTED; + _alter_indexes.push_back(std::move(index)); + } + + Status build_single_index() { + IndexBuilder builder(*_engine_ref, _tablet, _columns, _alter_indexes, false); + RETURN_IF_ERROR(builder.init()); + return builder.do_build_inverted_index(); + } + + // One SNII inverted index for the schema/plan helpers below. + struct SniiIndexSpec { + int64_t index_id; + std::string_view index_name; + int32_t column_unique_id; + std::map properties = { + {"parser", "english"}, {"lower_case", "true"}, {"support_phrase", "true"}}; + // false leaves col_unique_ids empty, reproducing a malformed index meta + // that binds to no column. + bool bind_column = true; + }; + + // SNII schema with k1(uid 1, key) + body_a(uid 2) + body_b(uid 3) and the + // given inverted indexes. + static TabletSchemaSPtr create_snii_schema(const std::vector& indexes) { + TabletSchemaPB schema_pb; + schema_pb.set_keys_type(DUP_KEYS); + schema_pb.set_inverted_index_storage_format(InvertedIndexStorageFormatPB::SNII); + auto tablet_schema = std::make_shared(); + tablet_schema->init_from_pb(schema_pb); + + TabletColumn key_column; + key_column.set_unique_id(1); + key_column.set_name("k1"); + key_column.set_type(FieldType::OLAP_FIELD_TYPE_INT); + key_column.set_length(4); + key_column.set_index_length(4); + key_column.set_is_key(true); + key_column.set_is_nullable(false); + tablet_schema->append_column(key_column); + + for (const auto& [unique_id, name] : {std::pair {2, "body_a"}, + std::pair {3, "body_b"}}) { + TabletColumn column; + column.set_unique_id(unique_id); + column.set_name(std::string(name)); + column.set_type(FieldType::OLAP_FIELD_TYPE_VARCHAR); + column.set_length(65535); + column.set_is_nullable(false); + tablet_schema->append_column(column); + } + + for (const auto& spec : indexes) { + TabletIndex index; + index._index_id = spec.index_id; + index._index_name = spec.index_name; + index._index_type = IndexType::INVERTED; + if (spec.bind_column) { + index._col_unique_ids.push_back(spec.column_unique_id); + } + for (const auto& [key, value] : spec.properties) { + index._properties[key] = value; + } + tablet_schema->append_index(std::move(index)); + } + return tablet_schema; + } + + static TabletSchemaSPtr create_snii_drop_schema() { + return create_snii_schema( + {SniiIndexSpec {.index_id = 1, .index_name = "idx_a", .column_unique_id = 2}, + SniiIndexSpec {.index_id = 2, .index_name = "idx_b", .column_unique_id = 3}}); + } + + Status create_snii_drop_tablet(const TabletSchemaSPtr& tablet_schema, + const std::string& tablet_path, TabletSharedPtr* tablet) { + RETURN_IF_ERROR(io::global_local_filesystem()->delete_directory(tablet_path)); + RETURN_IF_ERROR(io::global_local_filesystem()->create_directory(tablet_path)); + auto tablet_meta = create_tablet_meta(); + tablet_meta->_schema = tablet_schema; + *tablet = std::make_shared(*_engine_ref, tablet_meta, _data_dir.get()); + (*tablet)->_tablet_path = tablet_path; + return (*tablet)->init(); + } + + Status create_snii_source_rowset(const TabletSharedPtr& tablet, + const TabletSchemaSPtr& tablet_schema, + const std::string& tablet_path, + RowsetSharedPtr* rowset) const { + RowsetWriterContext writer_context; + writer_context.rowset_id.init(15691); + writer_context.tablet_id = tablet->tablet_id(); + writer_context.tablet_schema_hash = tablet->schema_hash(); + writer_context.partition_id = 10; + writer_context.rowset_type = BETA_ROWSET; + writer_context.tablet_path = tablet_path; + writer_context.rowset_state = VISIBLE; + writer_context.tablet_schema = tablet_schema; + writer_context.version = Version(10, 10); + + auto rowset_writer = + DORIS_TRY(RowsetFactory::create_rowset_writer(*_engine_ref, writer_context, false)); + Block block = tablet_schema->create_block(); + auto columns = std::move(block).mutate_columns(); + const std::vector dropped_values = {"drop alpha", "drop beta"}; + const std::vector surviving_values = {"keep alpha", "keep beta"}; + for (int32_t i = 0; i < 2; ++i) { + columns[0]->insert_data(reinterpret_cast(&i), sizeof(i)); + columns[1]->insert_data(dropped_values[i].data(), dropped_values[i].size()); + columns[2]->insert_data(surviving_values[i].data(), surviving_values[i].size()); + } + block = tablet_schema->create_block(); + block.set_columns(std::move(columns)); + RETURN_IF_ERROR(rowset_writer->add_block(&block)); + RETURN_IF_ERROR(rowset_writer->flush()); + RETURN_IF_ERROR(rowset_writer->build(*rowset)); + return tablet->add_rowset(*rowset); + } + + static TOlapTableIndex create_drop_index(int64_t index_id, std::string index_name, + std::string column_name, int32_t column_unique_id) { + TOlapTableIndex index; + index.index_id = index_id; + index.index_name = std::move(index_name); + index.index_type = TIndexType::INVERTED; + index.columns.emplace_back(std::move(column_name)); + index.column_unique_ids.push_back(column_unique_id); + return index; + } + + Status drop_snii_index(const TabletSharedPtr& tablet, TOlapTableIndex index, + RowsetSharedPtr* output_rowset) const { + std::vector drop_indexes {std::move(index)}; + IndexBuilder builder(*_engine_ref, tablet, _columns, drop_indexes, true); + RETURN_IF_ERROR(builder.init()); + RETURN_IF_ERROR(builder.do_build_inverted_index()); + DORIS_CHECK_EQ(builder._output_rowsets.size(), 1); + *output_rowset = builder._output_rowsets.front(); + return Status::OK(); + } + + static TOlapTableIndex create_build_index(int64_t index_id, std::string index_name, + std::string column_name, int32_t column_unique_id, + std::map properties) { + TOlapTableIndex index = create_drop_index(index_id, std::move(index_name), + std::move(column_name), column_unique_id); + index.__set_properties(properties); + return index; + } + + // Runs a BUILD INDEX task; rowsets whose schema already carries every + // requested index are skipped upstream (pick_candidate_rowsets), so the + // output may legitimately be empty. + Status build_snii_index(const TabletSharedPtr& tablet, std::vector indexes, + std::vector* output_rowsets) const { + IndexBuilder builder(*_engine_ref, tablet, _columns, indexes, false); + RETURN_IF_ERROR(builder.init()); + RETURN_IF_ERROR(builder.do_build_inverted_index()); + *output_rowsets = builder._output_rowsets; + return Status::OK(); + } + + static std::string snii_index_path_of(const RowsetSharedPtr& rowset) { + auto segment_path = rowset->segment_path(0); + EXPECT_TRUE(segment_path.has_value()) << segment_path.error(); + return segment_v2::InvertedIndexDescriptor::get_index_file_path_v2( + segment_v2::InvertedIndexDescriptor::get_index_file_path_prefix( + segment_path.value())); + } + + static std::unique_ptr open_snii_reader( + const RowsetSharedPtr& rowset, int64_t tablet_id) { + auto segment_path = rowset->segment_path(0); + EXPECT_TRUE(segment_path.has_value()) << segment_path.error(); + const std::string index_path_prefix { + segment_v2::InvertedIndexDescriptor::get_index_file_path_prefix( + segment_path.value())}; + auto reader = std::make_unique( + io::global_local_filesystem(), index_path_prefix, + InvertedIndexStorageFormatPB::SNII, InvertedIndexFileInfo(), tablet_id); + EXPECT_TRUE(reader->init().ok()); + return reader; + } + + // Asserts term -> docids through the logical index (index_id) of rowset. + static void assert_snii_term(const RowsetSharedPtr& rowset, int64_t tablet_id, + int32_t column_unique_id, int64_t index_id, + const std::string& term, + const std::vector& expected_docids) { + auto reader = open_snii_reader(rowset, tablet_id); + const auto index_metas = rowset->tablet_schema()->inverted_indexs(column_unique_id); + const TabletIndex* index_meta = nullptr; + for (const auto* candidate : index_metas) { + if (candidate->index_id() == index_id) { + index_meta = candidate; + } + } + ASSERT_NE(index_meta, nullptr) << "index " << index_id << " missing from output schema"; + auto logical_index = reader->open_snii_index(index_meta); + ASSERT_TRUE(logical_index.has_value()) << logical_index.error(); + std::vector docids; + ASSERT_TRUE(snii::query::term_query(*logical_index.value(), term, &docids).ok()); + EXPECT_EQ(docids, expected_docids) << "term=" << term << " index_id=" << index_id; + } + + // Asserts the output container carries the source's valid physical prefix + // byte for byte (which also pins that the prefix was copied exactly once: + // a second copy would displace every inherited section reference). + static void assert_snii_inherited_prefix( + const RowsetSharedPtr& source_rowset, const RowsetSharedPtr& output_rowset, + int64_t tablet_id, const std::vector& inherit_keys, + uint32_t doc_count) { + auto source_reader = open_snii_reader(source_rowset, tablet_id); + snii::reader::SniiRewriteSnapshot snapshot; + ASSERT_TRUE(source_reader->prepare_snii_rewrite_snapshot(inherit_keys, doc_count, &snapshot) + .ok()); + ASSERT_GT(snapshot.physical_prefix_end(), 0U); + + const auto read_all = [](const std::string& path) { + io::FileReaderSPtr file_reader; + EXPECT_TRUE(io::global_local_filesystem()->open_file(path, &file_reader).ok()); + std::string content(file_reader->size(), '\0'); + size_t bytes_read = 0; + Slice slice(content); + EXPECT_TRUE(file_reader->read_at(0, slice, &bytes_read).ok()); + EXPECT_EQ(bytes_read, content.size()); + return content; + }; + const std::string source_bytes = read_all(snii_index_path_of(source_rowset)); + const std::string output_bytes = read_all(snii_index_path_of(output_rowset)); + ASSERT_GE(source_bytes.size(), snapshot.physical_prefix_end()); + ASSERT_GE(output_bytes.size(), snapshot.physical_prefix_end()); + EXPECT_EQ(source_bytes.substr(0, snapshot.physical_prefix_end()), + output_bytes.substr(0, snapshot.physical_prefix_end())) + << "inherited physical prefix must be byte-identical"; + } + + static void assert_snii_surviving_index(const RowsetSharedPtr& source_rowset, + const RowsetSharedPtr& output_rowset) { + const auto& output_schema = output_rowset->tablet_schema(); + EXPECT_FALSE(output_schema->has_inverted_index_with_index_id(1)); + ASSERT_TRUE(output_schema->has_inverted_index_with_index_id(2)); + EXPECT_EQ(output_rowset->index_disk_size(), source_rowset->index_disk_size()); + EXPECT_EQ(output_rowset->data_disk_size(), source_rowset->data_disk_size()); + EXPECT_EQ(output_rowset->total_disk_size(), source_rowset->total_disk_size()); + + auto source_segment_path = source_rowset->segment_path(0); + ASSERT_TRUE(source_segment_path.has_value()) << source_segment_path.error(); + auto output_segment_path = output_rowset->segment_path(0); + ASSERT_TRUE(output_segment_path.has_value()) << output_segment_path.error(); + const auto source_index_path = segment_v2::InvertedIndexDescriptor::get_index_file_path_v2( + segment_v2::InvertedIndexDescriptor::get_index_file_path_prefix( + source_segment_path.value())); + const auto output_index_path = segment_v2::InvertedIndexDescriptor::get_index_file_path_v2( + segment_v2::InvertedIndexDescriptor::get_index_file_path_prefix( + output_segment_path.value())); + std::error_code equivalent_error; + EXPECT_TRUE( + std::filesystem::equivalent(source_index_path, output_index_path, equivalent_error)) + << equivalent_error.message(); + } + + static void assert_snii_term_query(const RowsetSharedPtr& rowset, int64_t tablet_id) { + auto segment_path = rowset->segment_path(0); + ASSERT_TRUE(segment_path.has_value()) << segment_path.error(); + const std::string index_path_prefix { + segment_v2::InvertedIndexDescriptor::get_index_file_path_prefix( + segment_path.value())}; + segment_v2::IndexFileReader index_file_reader( + io::global_local_filesystem(), index_path_prefix, + InvertedIndexStorageFormatPB::SNII, InvertedIndexFileInfo(), tablet_id); + ASSERT_TRUE(index_file_reader.init().ok()); + + const auto& surviving_indexes = rowset->tablet_schema()->inverted_indexs(3); + ASSERT_EQ(surviving_indexes.size(), 1); + auto logical_index = index_file_reader.open_snii_index(surviving_indexes.front()); + ASSERT_TRUE(logical_index.has_value()) << logical_index.error(); + std::vector docids; + ASSERT_TRUE(snii::query::term_query(*logical_index.value(), "keep", &docids).ok()); + EXPECT_EQ(docids, (std::vector {0, 1})); + } + + static void assert_last_snii_index_dropped(const RowsetSharedPtr& source_rowset, + const RowsetSharedPtr& rowset) { + EXPECT_FALSE(rowset->tablet_schema()->has_inverted_index()); + EXPECT_EQ(rowset->index_disk_size(), 0); + EXPECT_EQ(rowset->data_disk_size(), source_rowset->data_disk_size()); + EXPECT_EQ(rowset->total_disk_size(), source_rowset->data_disk_size()); + auto segment_path = rowset->segment_path(0); + ASSERT_TRUE(segment_path.has_value()) << segment_path.error(); + const auto index_path = segment_v2::InvertedIndexDescriptor::get_index_file_path_v2( + segment_v2::InvertedIndexDescriptor::get_index_file_path_prefix( + segment_path.value())); + bool index_exists = true; + ASSERT_TRUE(io::global_local_filesystem()->exists(index_path, &index_exists).ok()); + EXPECT_FALSE(index_exists); + } + StorageEngine* _engine_ref = nullptr; TabletSharedPtr _tablet; TabletMetaSharedPtr _tablet_meta; @@ -203,6 +567,17 @@ TEST_F(IndexBuilderTest, BasicBuildTest) { EXPECT_EQ(builder._alter_index_ids.size(), 1); } +TEST_F(IndexBuilderTest, HandleSingleRowsetPreservesOrdinaryAppendFailure) { + prepare_single_index_build(16604); + ScopedIndexBuilderDebugPoints debug_points; + debug_points.enable("IndexBuilder::handle_single_rowset_write_inverted_index_data_error"); + + auto status = build_single_index(); + + EXPECT_TRUE(status.is()) << status; + EXPECT_EQ(status.msg(), "debug point: handle_single_rowset_write_inverted_index_data_error"); +} + TEST_F(IndexBuilderTest, DropInvertedIndexTest) { // 0. prepare tablet path auto tablet_path = _absolute_dir + "/" + std::to_string(15676); @@ -3132,4 +3507,295 @@ TEST_F(IndexBuilderTest, DropOneIndexNotAffectOtherIndexesOnSameColumnTest) { << "Should have exactly 1 inverted index remaining after drop"; } +TEST_F(IndexBuilderTest, DropOneSniiIndexPreservesSurvivingPhysicalIndex) { + const auto tablet_path = _absolute_dir + "/15691"; + auto tablet_schema = create_snii_drop_schema(); + TabletSharedPtr tablet; + ASSERT_TRUE(create_snii_drop_tablet(tablet_schema, tablet_path, &tablet).ok()); + RowsetSharedPtr source_rowset; + ASSERT_TRUE(create_snii_source_rowset(tablet, tablet_schema, tablet_path, &source_rowset).ok()); + ASSERT_GT(source_rowset->index_disk_size(), 0); + + RowsetSharedPtr output_rowset; + ASSERT_TRUE(drop_snii_index(tablet, create_drop_index(1, "idx_a", "body_a", 2), &output_rowset) + .ok()); + assert_snii_surviving_index(source_rowset, output_rowset); + assert_snii_term_query(output_rowset, tablet->tablet_id()); + + ScopedIndexBuilderDebugPoints debug_points; + debug_points.enable("IndexBuilder::update_inverted_index_info_index_file_reader_init_not_ok"); + RowsetSharedPtr final_rowset; + ASSERT_TRUE(drop_snii_index(tablet, create_drop_index(2, "idx_b", "body_b", 3), &final_rowset) + .ok()); + assert_last_snii_index_dropped(source_rowset, final_rowset); +} + +// Classification only: unchanged-and-present -> inherit; requested-and-absent -> +// build; same key with a changed definition -> rebuild, never inherit; a container +// key the target schema no longer holds is not inherited; two build indexes on one +// column share one column group (the "scan the column once" pin). +TEST_F(IndexBuilderTest, SniiBuildPlanClassifiesInheritBuildReplaceAndDrop) { + const auto input_schema = create_snii_schema( + {SniiIndexSpec {.index_id = 1, .index_name = "idx_a", .column_unique_id = 2}}); + auto output_schema = create_snii_schema( + {SniiIndexSpec {.index_id = 1, .index_name = "idx_a", .column_unique_id = 2}, + SniiIndexSpec {.index_id = 2, .index_name = "idx_b", .column_unique_id = 3}, + SniiIndexSpec {.index_id = 3, + .index_name = "idx_c", + .column_unique_id = 3, + .properties = {{"parser", "none"}}}}); + // The container carries idx_a plus a stale key (9) the target schema dropped. + const auto container_has = [](const TabletIndex& index, bool* exists) { + *exists = index.index_id() == 1; + return Status::OK(); + }; + + IndexBuilder::SniiIndexRewritePlan plan; + ASSERT_TRUE(IndexBuilder::plan_snii_index_rewrite(*input_schema, *output_schema, {2, 3}, + container_has, + /*source_container_has_blob=*/false, &plan) + .ok()); + ASSERT_EQ(plan.inherit_keys.size(), 1U); + EXPECT_EQ(plan.inherit_keys.front().index_id, 1U); + // idx_b and idx_c both target column 3: exactly ONE column group with both. + ASSERT_EQ(plan.build_columns.size(), 1U); + EXPECT_EQ(plan.build_columns.front().first, 3); + ASSERT_EQ(plan.build_columns.front().second.size(), 2U); +} + +// A malformed index that binds no column must not block the rewrite unless it +// actually has to be REBUILT. One corrupted index elsewhere in the schema -- +// neither requested nor present in the container -- would otherwise fail every +// segment of the tablet forever, blocking BUILD INDEX on healthy indexes. +TEST_F(IndexBuilderTest, SniiBuildPlanToleratesUnrequestedIndexWithoutColumnUniqueId) { + const auto input_schema = create_snii_schema( + {SniiIndexSpec {.index_id = 1, .index_name = "idx_a", .column_unique_id = 2}}); + auto output_schema = create_snii_schema( + {SniiIndexSpec {.index_id = 1, .index_name = "idx_a", .column_unique_id = 2}, + SniiIndexSpec {.index_id = 9, + .index_name = "idx_broken", + .column_unique_id = 3, + .bind_column = false}}); + // idx_a is in the container; idx_broken is not, and is not requested. + const auto container_has = [](const TabletIndex& index, bool* exists) { + *exists = index.index_id() == 1; + return Status::OK(); + }; + + IndexBuilder::SniiIndexRewritePlan plan; + ASSERT_TRUE(IndexBuilder::plan_snii_index_rewrite(*input_schema, *output_schema, {1}, + container_has, + /*source_container_has_blob=*/false, &plan) + .ok()); + // The healthy index still inherits; the malformed one just stays absent. + ASSERT_EQ(plan.inherit_keys.size(), 1U); + EXPECT_EQ(plan.inherit_keys.front().index_id, 1U); + EXPECT_TRUE(plan.build_columns.empty()); +} + +// An index the plan can neither inherit nor rebuild must fail the rewrite, not +// be skipped. Without a column unique id there is no raw column to read, so +// leaving it out would seal a container that does not match the target schema +// while reporting success. +TEST_F(IndexBuilderTest, SniiBuildPlanRejectsIndexWithoutColumnUniqueId) { + const auto input_schema = create_snii_schema({}); + auto output_schema = create_snii_schema({SniiIndexSpec { + .index_id = 1, .index_name = "idx_a", .column_unique_id = 2, .bind_column = false}}); + + const auto container_has = [](const TabletIndex&, bool* exists) { + *exists = false; + return Status::OK(); + }; + IndexBuilder::SniiIndexRewritePlan plan; + const Status status = + IndexBuilder::plan_snii_index_rewrite(*input_schema, *output_schema, {1}, container_has, + /*source_container_has_blob=*/false, &plan); + EXPECT_TRUE(status.is()) << status; + EXPECT_TRUE(plan.inherit_keys.empty()); + EXPECT_TRUE(plan.build_columns.empty()); +} + +// A source container holding a blob logical index (a numeric column, served by +// the native BKD) cannot be snapshotted for inheritance AT ALL: the snapshot is +// rejected by the container's directory content, not by the subset being kept. +// So an otherwise-inheritable text index must be reclassified to rebuild too -- +// leaving even one key in inherit_keys makes the whole segment rewrite fail. +TEST_F(IndexBuilderTest, SniiBuildPlanRebuildsEverythingWhenSourceContainerHoldsABlob) { + const auto input_schema = create_snii_schema( + {SniiIndexSpec {.index_id = 1, .index_name = "idx_a", .column_unique_id = 2}}); + // Same key, same definition, present in the container: inheritable by every + // rule except the container's blob. + const auto container_has = [](const TabletIndex& index, bool* exists) { + *exists = index.index_id() == 1; + return Status::OK(); + }; + + IndexBuilder::SniiIndexRewritePlan plan; + ASSERT_TRUE(IndexBuilder::plan_snii_index_rewrite(*input_schema, *input_schema, {1}, + container_has, + /*source_container_has_blob=*/true, &plan) + .ok()); + EXPECT_TRUE(plan.inherit_keys.empty()); + ASSERT_EQ(plan.build_columns.size(), 1U); + EXPECT_EQ(plan.build_columns.front().first, 2); +} + +TEST_F(IndexBuilderTest, SniiBuildPlanClassifiesReplaceAndRetry) { + const auto input_schema = create_snii_schema( + {SniiIndexSpec {.index_id = 1, .index_name = "idx_a", .column_unique_id = 2}}); + const auto container_has = [](const TabletIndex& index, bool* exists) { + *exists = index.index_id() == 1; + return Status::OK(); + }; + + // Same key, changed definition: the request replaces idx_a's parser, so the + // old metadata must NOT be inherited -- the index is rebuilt. + auto replaced_schema = create_snii_schema({SniiIndexSpec {.index_id = 1, + .index_name = "idx_a", + .column_unique_id = 2, + .properties = {{"parser", "none"}}}}); + IndexBuilder::SniiIndexRewritePlan replace_plan; + ASSERT_TRUE(IndexBuilder::plan_snii_index_rewrite( + *input_schema, *replaced_schema, {1}, container_has, + /*source_container_has_blob=*/false, &replace_plan) + .ok()); + EXPECT_TRUE(replace_plan.inherit_keys.empty()); + ASSERT_EQ(replace_plan.build_columns.size(), 1U); + EXPECT_EQ(replace_plan.build_columns.front().first, 2); + + // Retry: the requested index already exists in schema and container with the + // same definition -> inherit, no build work at all. + IndexBuilder::SniiIndexRewritePlan retry_plan; + ASSERT_TRUE( + IndexBuilder::plan_snii_index_rewrite(*input_schema, *input_schema, {1}, container_has, + /*source_container_has_blob=*/false, &retry_plan) + .ok()); + ASSERT_EQ(retry_plan.inherit_keys.size(), 1U); + EXPECT_TRUE(retry_plan.build_columns.empty()); +} + +TEST_F(IndexBuilderTest, SniiBuildAddsSecondIndexAndInheritsFirst) { + const auto tablet_path = _absolute_dir + "/15691"; + auto tablet_schema = create_snii_schema( + {SniiIndexSpec {.index_id = 1, .index_name = "idx_a", .column_unique_id = 2}}); + TabletSharedPtr tablet; + ASSERT_TRUE(create_snii_drop_tablet(tablet_schema, tablet_path, &tablet).ok()); + RowsetSharedPtr source_rowset; + ASSERT_TRUE(create_snii_source_rowset(tablet, tablet_schema, tablet_path, &source_rowset).ok()); + + std::vector output_rowsets; + ASSERT_TRUE(build_snii_index(tablet, + {create_build_index(2, "idx_b", "body_b", 3, + {{"parser", "english"}, + {"lower_case", "true"}, + {"support_phrase", "true"}})}, + &output_rowsets) + .ok()); + ASSERT_EQ(output_rowsets.size(), 1U); + const RowsetSharedPtr& output_rowset = output_rowsets.front(); + + ASSERT_TRUE(output_rowset->tablet_schema()->has_inverted_index_with_index_id(1)); + ASSERT_TRUE(output_rowset->tablet_schema()->has_inverted_index_with_index_id(2)); + // The inherited index answers as before; the built index answers over the + // historical rows. + assert_snii_term(output_rowset, tablet->tablet_id(), 2, 1, "drop", {0, 1}); + assert_snii_term(output_rowset, tablet->tablet_id(), 3, 2, "keep", {0, 1}); + assert_snii_inherited_prefix(source_rowset, output_rowset, tablet->tablet_id(), + {{.index_id = 1, .index_suffix = ""}}, /*doc_count=*/2); +} + +TEST_F(IndexBuilderTest, SniiBuildAllSharesOneColumnScanAndOnePrefixCopy) { + const auto tablet_path = _absolute_dir + "/15691"; + auto tablet_schema = create_snii_schema( + {SniiIndexSpec {.index_id = 1, .index_name = "idx_a", .column_unique_id = 2}}); + TabletSharedPtr tablet; + ASSERT_TRUE(create_snii_drop_tablet(tablet_schema, tablet_path, &tablet).ok()); + RowsetSharedPtr source_rowset; + ASSERT_TRUE(create_snii_source_rowset(tablet, tablet_schema, tablet_path, &source_rowset).ok()); + + // BUILD ALL: two new indexes on the SAME column plus the untouched idx_a. + std::vector output_rowsets; + ASSERT_TRUE( + build_snii_index(tablet, + {create_build_index(2, "idx_b", "body_b", 3, + {{"parser", "english"}, + {"lower_case", "true"}, + {"support_phrase", "true"}}), + create_build_index(3, "idx_c", "body_b", 3, {{"parser", "none"}})}, + &output_rowsets) + .ok()); + ASSERT_EQ(output_rowsets.size(), 1U); + const RowsetSharedPtr& output_rowset = output_rowsets.front(); + + assert_snii_term(output_rowset, tablet->tablet_id(), 2, 1, "drop", {0, 1}); + assert_snii_term(output_rowset, tablet->tablet_id(), 3, 2, "keep", {0, 1}); + // idx_c is untokenized: the whole cell value is one term. + assert_snii_term(output_rowset, tablet->tablet_id(), 3, 3, "keep alpha", {0}); + assert_snii_inherited_prefix(source_rowset, output_rowset, tablet->tablet_id(), + {{.index_id = 1, .index_suffix = ""}}, /*doc_count=*/2); +} + +// A retried build names an index the rowset schema already carries: the rowset +// is skipped upstream (pick_candidate_rowsets_to_build_inverted_index), so no +// analyzer, decode or encode runs and nothing is rewritten. The same-key-with- +// changed-definition case is unreachable here for the same reason; its +// classification is pinned by SniiBuildPlanClassifiesReplaceAndRetry. +TEST_F(IndexBuilderTest, SniiBuildRetrySkipsRowsetsAlreadyCoveringTheIndex) { + const auto tablet_path = _absolute_dir + "/15691"; + auto tablet_schema = create_snii_drop_schema(); // idx_a and idx_b both present + TabletSharedPtr tablet; + ASSERT_TRUE(create_snii_drop_tablet(tablet_schema, tablet_path, &tablet).ok()); + RowsetSharedPtr source_rowset; + ASSERT_TRUE(create_snii_source_rowset(tablet, tablet_schema, tablet_path, &source_rowset).ok()); + + std::vector output_rowsets; + ASSERT_TRUE(build_snii_index(tablet, + {create_build_index(2, "idx_b", "body_b", 3, + {{"parser", "english"}, + {"lower_case", "true"}, + {"support_phrase", "true"}})}, + &output_rowsets) + .ok()); + // Skipped, not rewritten: no output rowset, the tablet still serves the + // original one, and the container remains fully queryable. + EXPECT_TRUE(output_rowsets.empty()); + auto rowset = tablet->get_rowset_by_version(Version(10, 10)); + ASSERT_NE(rowset, nullptr); + EXPECT_EQ(rowset->rowset_id(), source_rowset->rowset_id()); + assert_snii_term(source_rowset, tablet->tablet_id(), 2, 1, "drop", {0, 1}); + assert_snii_term(source_rowset, tablet->tablet_id(), 3, 2, "keep", {0, 1}); +} + +TEST_F(IndexBuilderTest, SniiBuildFailureCommitsNoRowset) { + const auto tablet_path = _absolute_dir + "/15691"; + auto tablet_schema = create_snii_schema( + {SniiIndexSpec {.index_id = 1, .index_name = "idx_a", .column_unique_id = 2}}); + TabletSharedPtr tablet; + ASSERT_TRUE(create_snii_drop_tablet(tablet_schema, tablet_path, &tablet).ok()); + RowsetSharedPtr source_rowset; + ASSERT_TRUE(create_snii_source_rowset(tablet, tablet_schema, tablet_path, &source_rowset).ok()); + + ScopedIndexBuilderDebugPoints debug_points; + debug_points.enable("IndexBuilder::handle_single_rowset_snii_index_build_finish_error"); + std::vector output_rowsets; + const Status status = build_snii_index( + tablet, + {create_build_index( + 2, "idx_b", "body_b", 3, + {{"parser", "english"}, {"lower_case", "true"}, {"support_phrase", "true"}})}, + &output_rowsets); + // Specifically the INJECTED failure: the SNII build path must run far enough + // to hit the debug point and then fail the whole task. + ASSERT_TRUE(status.is()) + << "expected the injected index build failure, got: " << status; + + // The source rowset is untouched and still fully queryable ... + assert_snii_term(source_rowset, tablet->tablet_id(), 2, 1, "drop", {0, 1}); + // ... and the tablet still serves the ORIGINAL rowset for that version: the + // failed build committed nothing. + auto rowset = tablet->get_rowset_by_version(Version(10, 10)); + ASSERT_NE(rowset, nullptr); + EXPECT_EQ(rowset->rowset_id(), source_rowset->rowset_id()); +} + } // namespace doris diff --git a/be/test/storage/index/inverted/ananlyzer/analyzer_test.cpp b/be/test/storage/index/inverted/ananlyzer/analyzer_test.cpp index 4a51609a68806d..6d7d6e02221f6d 100644 --- a/be/test/storage/index/inverted/ananlyzer/analyzer_test.cpp +++ b/be/test/storage/index/inverted/ananlyzer/analyzer_test.cpp @@ -345,4 +345,23 @@ TEST_F(AnalyzerTest, TestAnalyzerFunctionality) { } } +TEST_F(AnalyzerTest, CommonGramsQueryPurposeSelection) { + EXPECT_EQ(select_analysis_purpose(InvertedIndexQueryType::MATCH_ANY_QUERY, 0, false), + AnalysisPurpose::kPlainQuery); + EXPECT_EQ(select_analysis_purpose(InvertedIndexQueryType::MATCH_ALL_QUERY, 0, false), + AnalysisPurpose::kPlainQuery); + EXPECT_EQ(select_analysis_purpose(InvertedIndexQueryType::MATCH_PHRASE_QUERY, 0, false), + AnalysisPurpose::kExactPhraseQuery); + EXPECT_EQ(select_analysis_purpose(InvertedIndexQueryType::MATCH_PHRASE_QUERY, 2, false), + AnalysisPurpose::kPlainQuery); + EXPECT_EQ(select_analysis_purpose(InvertedIndexQueryType::MATCH_PHRASE_QUERY, 0, true), + AnalysisPurpose::kPlainQuery); + EXPECT_EQ(select_analysis_purpose(InvertedIndexQueryType::MATCH_PHRASE_PREFIX_QUERY, 0, false), + AnalysisPurpose::kPhrasePrefixQuery); + EXPECT_EQ(select_analysis_purpose(InvertedIndexQueryType::MATCH_PHRASE_PREFIX_QUERY, 0, true), + AnalysisPurpose::kPlainQuery); + EXPECT_EQ(select_analysis_purpose(InvertedIndexQueryType::MATCH_REGEXP_QUERY, 0, false), + AnalysisPurpose::kPlainQuery); +} + } // namespace doris::segment_v2::inverted_index \ No newline at end of file diff --git a/be/test/storage/index/inverted/ananlyzer/custom_analyzer_test.cpp b/be/test/storage/index/inverted/ananlyzer/custom_analyzer_test.cpp index a96edc1f38b346..e6d0d73f5cf2a9 100644 --- a/be/test/storage/index/inverted/ananlyzer/custom_analyzer_test.cpp +++ b/be/test/storage/index/inverted/ananlyzer/custom_analyzer_test.cpp @@ -21,12 +21,16 @@ #include #include +#include #include "CLucene/store/Directory.h" #include "CLucene/store/FSDirectory.h" #include "roaring/roaring.hh" #include "runtime/exec_env.h" #include "storage/index/inverted/analysis_factory_mgr.h" +#include "storage/index/inverted/analyzer/analyzer.h" +#include "storage/index/inverted/common_grams/common_grams_key_codec.h" +#include "storage/index/inverted/common_grams/common_word_set.h" #include "storage/index/inverted/query/phrase_prefix_query.h" #include "storage/index/inverted/query/phrase_query.h" #include "storage/index/inverted/setting.h" @@ -255,6 +259,437 @@ TEST_F(CustomAnalyzerTest, TokenStreamWithReaderPtr) { delete token_stream; } +CustomAnalyzerConfigPtr common_grams_config( + const std::string& tokenizer, const Settings& tokenizer_settings, + const std::vector>& filters = {}, + const Settings& common_grams_settings = {}) { + CustomAnalyzerConfig::Builder builder; + builder.with_tokenizer_config(tokenizer, tokenizer_settings); + for (const auto& [name, settings] : filters) { + builder.add_token_filter_config(name, settings); + } + builder.add_token_filter_config("common_grams", common_grams_settings); + return builder.build(); +} + +Settings whitespace_tokenizer_settings() { + Settings settings; + settings.set("tokenize_on_chars", "[whitespace]"); + return settings; +} + +std::string expected_gram(std::string_view left, std::string_view right) { + auto result = encode_common_gram(left, right); + EXPECT_TRUE(result.has_value()) << result.error(); + return result.value(); +} + +void expect_common_grams_analyzer_error(const CustomAnalyzerConfigPtr& config, + AnalysisPurpose purpose) { + try { + CustomAnalyzer::build_custom_analyzer(config, purpose); + FAIL() << "expected CommonGrams analyzer error"; + } catch (const Exception& error) { + EXPECT_EQ(error.code(), ErrorCode::INVERTED_INDEX_ANALYZER_ERROR); + } +} + +TEST_F(CustomAnalyzerTest, CommonGramsBuildsIndependentPurposeStreams) { + auto config = + common_grams_config("char_group", whitespace_tokenizer_settings(), {{"lowercase", {}}}); + auto index = CustomAnalyzer::build_custom_analyzer(config, AnalysisPurpose::kIndex); + auto snii_index = + CustomAnalyzer::build_custom_analyzer(config, AnalysisPurpose::kSniiTransientIndex); + auto plain = CustomAnalyzer::build_custom_analyzer(config, AnalysisPurpose::kPlainQuery); + auto exact = CustomAnalyzer::build_custom_analyzer(config, AnalysisPurpose::kExactPhraseQuery); + auto prefix = + CustomAnalyzer::build_custom_analyzer(config, AnalysisPurpose::kPhrasePrefixQuery); + + EXPECT_EQ(tokenize1(index, "Man of the Year"), + (std::vector {{"man", 1}, + {expected_gram("man", "of"), 0}, + {"of", 1}, + {expected_gram("of", "the"), 0}, + {"the", 1}, + {expected_gram("the", "year"), 0}, + {"year", 1}})); + EXPECT_EQ(tokenize1(snii_index, "Man of the Year"), + (std::vector {{"man", 1}, + {expected_gram("man", "of"), 0}, + {"of", 1}, + {expected_gram("of", "the"), 0}, + {"the", 1}, + {expected_gram("the", "year"), 0}, + {"year", 1}})); + EXPECT_EQ(tokenize1(plain, "Man of the Year"), + (std::vector {{"man", 1}, {"of", 1}, {"the", 1}, {"year", 1}})); + EXPECT_EQ(tokenize1(exact, "Man of the Year"), + (std::vector {{expected_gram("man", "of"), 1}, + {expected_gram("of", "the"), 1}, + {expected_gram("the", "year"), 1}})); + EXPECT_EQ(tokenize1(prefix, "the wo"), + (std::vector {{expected_gram("the", "wo"), 1}})); + + EXPECT_EQ(tokenize1(exact, "plain terms"), + (std::vector {{"plain", 1}, {"terms", 1}})); + EXPECT_EQ(tokenize1(prefix, "of term"), + (std::vector {{expected_gram("of", "term"), 1}})); +} + +TEST_F(CustomAnalyzerTest, CommonGramsReusableIndexStreamDoesNotBridgeRows) { + auto analyzer = CustomAnalyzer::build_custom_analyzer( + common_grams_config("char_group", whitespace_tokenizer_settings(), {{"lowercase", {}}}), + AnalysisPurpose::kIndex); + auto reader = std::make_shared>(); + + auto tokenize_row = [&](std::string_view row) { + reader->init(row.data(), static_cast(row.size()), false); + auto* stream = analyzer->reusableTokenStream(L"", reader); + stream->reset(); + std::vector tokens; + Token token; + while (stream->next(&token)) { + tokens.emplace_back(std::string(token.termBuffer(), token.termLength()), + token.getPositionIncrement()); + } + return std::pair {stream, std::move(tokens)}; + }; + + auto [first_stream, first] = tokenize_row("foo of"); + auto [second_stream, second] = tokenize_row("the bar"); + + EXPECT_EQ(first_stream, second_stream); + EXPECT_EQ(first, (std::vector { + {"foo", 1}, {expected_gram("foo", "of"), 0}, {"of", 1}})); + EXPECT_EQ(second, (std::vector { + {"the", 1}, {expected_gram("the", "bar"), 0}, {"bar", 1}})); +} + +TEST_F(CustomAnalyzerTest, CommonGramsEscapesPlainKeysOnlyForIndexPurpose) { + Settings tokenizer_settings; + tokenizer_settings.set("tokenize_on_chars", "[\\u0020]"); + auto config = common_grams_config("char_group", tokenizer_settings); + auto index = CustomAnalyzer::build_custom_analyzer(config, AnalysisPurpose::kIndex); + auto plain = CustomAnalyzer::build_custom_analyzer(config, AnalysisPurpose::kPlainQuery); + auto exact = CustomAnalyzer::build_custom_analyzer(config, AnalysisPurpose::kExactPhraseQuery); + auto prefix = + CustomAnalyzer::build_custom_analyzer(config, AnalysisPurpose::kPhrasePrefixQuery); + + const std::string logical = std::string(1, '\x1f') + "literal"; + const std::string input = "the " + logical; + const std::string physical = std::string(1, PLAIN_ESCAPE_PREFIX) + "Gliteral"; + const std::string common_gram = expected_gram("the", logical); + + EXPECT_EQ(tokenize1(index, input), + (std::vector {{"the", 1}, {common_gram, 0}, {physical, 1}})); + EXPECT_EQ(tokenize1(plain, input), (std::vector {{"the", 1}, {logical, 1}})); + EXPECT_EQ(tokenize1(exact, input), (std::vector {{common_gram, 1}})); + EXPECT_EQ(tokenize1(prefix, input), (std::vector {{common_gram, 1}})); +} + +TEST_F(CustomAnalyzerTest, AnalyseResultPreservesCommonGramTermKind) { + auto config = + common_grams_config("char_group", whitespace_tokenizer_settings(), {{"lowercase", {}}}); + auto exact = CustomAnalyzer::build_custom_analyzer(config, AnalysisPurpose::kExactPhraseQuery); + auto reader = InvertedIndexAnalyzer::create_reader({}); + const std::string input = "Man of year"; + reader->init(input.data(), static_cast(input.size()), true); + + const auto terms = InvertedIndexAnalyzer::get_analyse_result(reader, exact.get()); + ASSERT_EQ(terms.size(), 2U); + EXPECT_EQ(terms[0].get_single_term(), expected_gram("man", "of")); + EXPECT_EQ(terms[0].key_kind, TermKeyKind::kCommonGram); + EXPECT_EQ(terms[1].get_single_term(), expected_gram("of", "year")); + EXPECT_EQ(terms[1].key_kind, TermKeyKind::kCommonGram); + + auto plain = CustomAnalyzer::build_custom_analyzer(config, AnalysisPurpose::kPlainQuery); + reader = InvertedIndexAnalyzer::create_reader({}); + reader->init(input.data(), static_cast(input.size()), true); + const auto plain_terms = InvertedIndexAnalyzer::get_analyse_result(reader, plain.get()); + ASSERT_EQ(plain_terms.size(), 3U); + for (const auto& term : plain_terms) { + EXPECT_EQ(term.key_kind, TermKeyKind::kPlain); + } +} + +TEST_F(CustomAnalyzerTest, AnalyseResultPreservesBothCommonGramTermKind) { + auto config = + common_grams_config("char_group", whitespace_tokenizer_settings(), {{"lowercase", {}}}); + auto index = CustomAnalyzer::build_custom_analyzer(config, AnalysisPurpose::kIndex); + auto reader = InvertedIndexAnalyzer::create_reader({}); + const std::string input = "of the"; + reader->init(input.data(), static_cast(input.size()), true); + + const auto terms = InvertedIndexAnalyzer::get_analyse_result(reader, index.get()); + ASSERT_EQ(terms.size(), 3U); + EXPECT_EQ(terms[1].get_single_term(), expected_gram("of", "the")); + EXPECT_EQ(terms[1].key_kind, TermKeyKind::kCommonGram); +} + +TEST_F(CustomAnalyzerTest, CommonGramsAllowsAbsentOrOneTerminalFilter) { + CustomAnalyzerConfig::Builder missing; + missing.with_tokenizer_config("char_group", whitespace_tokenizer_settings()); + auto missing_config = missing.build(); + for (AnalysisPurpose purpose : + {AnalysisPurpose::kIndex, AnalysisPurpose::kPlainQuery, AnalysisPurpose::kExactPhraseQuery, + AnalysisPurpose::kPhrasePrefixQuery}) { + EXPECT_NO_THROW(CustomAnalyzer::build_custom_analyzer(missing_config, purpose)); + } + + CustomAnalyzerConfig::Builder duplicate; + duplicate.with_tokenizer_config("char_group", whitespace_tokenizer_settings()); + duplicate.add_token_filter_config("common_grams", {}); + duplicate.add_token_filter_config("common_grams", {}); + expect_common_grams_analyzer_error(duplicate.build(), AnalysisPurpose::kIndex); + + CustomAnalyzerConfig::Builder not_last; + not_last.with_tokenizer_config("char_group", whitespace_tokenizer_settings()); + not_last.add_token_filter_config("common_grams", {}); + not_last.add_token_filter_config("lowercase", {}); + expect_common_grams_analyzer_error(not_last.build(), AnalysisPurpose::kIndex); +} + +TEST_F(CustomAnalyzerTest, CommonGramsValidatesPlainConfigurationButOmitsGrams) { + Settings unknown; + unknown.set("unknown_setting", "true"); + auto invalid = common_grams_config("char_group", whitespace_tokenizer_settings(), {}, unknown); + expect_common_grams_analyzer_error(invalid, AnalysisPurpose::kPlainQuery); + + auto valid = common_grams_config("char_group", whitespace_tokenizer_settings()); + auto plain = CustomAnalyzer::build_custom_analyzer(valid, AnalysisPurpose::kPlainQuery); + EXPECT_EQ(tokenize1(plain, "the term"), (std::vector {{"the", 1}, {"term", 1}})); +} + +TEST_F(CustomAnalyzerTest, CommonGramsIndexChainRejectsInvalidUtf8AfterAValidToken) { + auto analyzer = CustomAnalyzer::build_custom_analyzer( + common_grams_config("char_group", whitespace_tokenizer_settings(), {{"lowercase", {}}}), + AnalysisPurpose::kIndex); + const std::string input = std::string("valid b") + static_cast(0xFF) + std::string("ad"); + auto reader = std::make_shared>(); + reader->init(input.data(), static_cast(input.size()), false); + auto* stream = analyzer->reusableTokenStream(L"", reader); + stream->reset(); + + Token token; + ASSERT_NE(stream->next(&token), nullptr); + EXPECT_EQ(std::string(token.termBuffer(), token.termLength()), "valid"); + try { + stream->next(&token); + FAIL() << "expected malformed UTF-8 to fail the analyzer chain"; + } catch (const Exception& error) { + EXPECT_EQ(error.code(), ErrorCode::INVERTED_INDEX_ANALYZER_ERROR); + } +} + +TEST_F(CustomAnalyzerTest, CommonGramsDeniesUnsafePositionFactories) { + for (const std::string& tokenizer : {"standard", "pinyin", "basic", "icu", "keyword"}) { + expect_common_grams_analyzer_error(common_grams_config(tokenizer, {}), + AnalysisPurpose::kIndex); + } + + Settings preserve_original; + preserve_original.set("preserve_original", "true"); + expect_common_grams_analyzer_error( + common_grams_config("char_group", whitespace_tokenizer_settings(), + {{"asciifolding", preserve_original}}), + AnalysisPurpose::kIndex); + expect_common_grams_analyzer_error( + common_grams_config("char_group", whitespace_tokenizer_settings(), + {{"word_delimiter", {}}}), + AnalysisPurpose::kIndex); + expect_common_grams_analyzer_error( + common_grams_config("char_group", whitespace_tokenizer_settings(), {{"pinyin", {}}}), + AnalysisPurpose::kIndex); +} + +TEST_F(CustomAnalyzerTest, CommonGramsAcceptsReviewedUnitPositionFactories) { + for (const std::string& tokenizer : {"empty", "char_group", "ngram", "edge_ngram"}) { + EXPECT_NO_THROW(CustomAnalyzer::build_custom_analyzer( + common_grams_config(tokenizer, tokenizer == "char_group" + ? whitespace_tokenizer_settings() + : Settings {}), + AnalysisPurpose::kIndex)); + } + + for (const std::string& filter : {"empty", "lowercase", "icu_normalizer", "asciifolding"}) { + EXPECT_NO_THROW(CustomAnalyzer::build_custom_analyzer( + common_grams_config("char_group", whitespace_tokenizer_settings(), {{filter, {}}}), + AnalysisPurpose::kIndex)); + } +} + +TEST_F(CustomAnalyzerTest, CommonGramsRejectsTokensNormalizedToEmpty) { + const auto config = common_grams_config("char_group", whitespace_tokenizer_settings(), + {{"icu_normalizer", {}}}); + const std::string input = std::string("\xC2\xAD") + " the"; + for (AnalysisPurpose purpose : + {AnalysisPurpose::kIndex, AnalysisPurpose::kPlainQuery, AnalysisPurpose::kExactPhraseQuery, + AnalysisPurpose::kPhrasePrefixQuery}) { + auto analyzer = CustomAnalyzer::build_custom_analyzer(config, purpose); + try { + tokenize1(analyzer, input); + ADD_FAILURE() << "expected analyzer error for purpose " << static_cast(purpose); + } catch (const Exception& error) { + EXPECT_EQ(error.code(), ErrorCode::INVERTED_INDEX_ANALYZER_ERROR); + } + } +} + +TEST_F(CustomAnalyzerTest, CommonGramsInternalQueryFiltersAreNotRegistered) { + for (const std::string& internal : {"common_grams_query", "common_grams_phrase_prefix"}) { + CustomAnalyzerConfig::Builder builder; + builder.with_tokenizer_config("char_group", whitespace_tokenizer_settings()); + builder.add_token_filter_config(internal, {}); + builder.add_token_filter_config("common_grams", {}); + EXPECT_THROW( + CustomAnalyzer::build_custom_analyzer(builder.build(), AnalysisPurpose::kIndex), + Exception); + } +} + +// "the" is a member of the built-in stop-word list, which is what default_word_set() resolves to +// when no wordset file is installed -- the provider can no longer be handed a word list of its own. +TEST_F(CustomAnalyzerTest, CommonGramsProviderCachesPurposeAnalyzersWithOneWordSetSnapshot) { + auto provider = std::make_shared(common_grams_config( + "char_group", whitespace_tokenizer_settings(), {{"lowercase", {}}})); + + auto index = provider->get_analyzer(AnalysisPurpose::kIndex); + auto snii_index = provider->get_analyzer(AnalysisPurpose::kSniiTransientIndex); + auto plain = provider->get_analyzer(AnalysisPurpose::kPlainQuery); + auto exact = provider->get_analyzer(AnalysisPurpose::kExactPhraseQuery); + auto prefix = provider->get_analyzer(AnalysisPurpose::kPhrasePrefixQuery); + + EXPECT_EQ(tokenize1(std::dynamic_pointer_cast(index), "The year"), + (std::vector { + {"the", 1}, {expected_gram("the", "year"), 0}, {"year", 1}})); + EXPECT_EQ(tokenize1(std::dynamic_pointer_cast(snii_index), "The year"), + (std::vector { + {"the", 1}, {expected_gram("the", "year"), 0}, {"year", 1}})); + EXPECT_EQ(tokenize1(std::dynamic_pointer_cast(plain), "The year"), + (std::vector {{"the", 1}, {"year", 1}})); + EXPECT_EQ(tokenize1(std::dynamic_pointer_cast(exact), "The year"), + (std::vector {{expected_gram("the", "year"), 1}})); + EXPECT_EQ(tokenize1(std::dynamic_pointer_cast(prefix), "The ye"), + (std::vector {{expected_gram("the", "ye"), 1}})); + + // Every purpose analyzer shares the one process-wide word list. + EXPECT_EQ(provider->common_words(), CommonWordSet::default_word_set()); + EXPECT_EQ(provider->get_analyzer(AnalysisPurpose::kPlainQuery), plain); + EXPECT_NE(index, plain); + EXPECT_NE(index, snii_index); + EXPECT_NE(plain, exact); + EXPECT_NE(exact, prefix); +} + +TEST_F(CustomAnalyzerTest, CommonGramsProviderOwnsDeterministicBuiltinIdentity) { + Settings first_settings; + first_settings.set("max_token_length", "16383"); + first_settings.set("tokenize_on_chars", "[whitespace]"); + Settings second_settings; + second_settings.set("tokenize_on_chars", "[whitespace]"); + second_settings.set("max_token_length", "16383"); + + CustomAnalyzerProvider first( + common_grams_config("char_group", first_settings, {{"lowercase", {}}})); + CustomAnalyzerProvider second( + common_grams_config("char_group", second_settings, {{"lowercase", {}}})); + + ASSERT_NE(first.common_grams_identity(), nullptr); + ASSERT_NE(second.common_grams_identity(), nullptr); + EXPECT_EQ(first.common_grams_identity()->common_grams_dictionary_identity, + BUILTIN_COMMON_WORDS_RESOURCE); + EXPECT_EQ(*first.common_grams_identity(), *second.common_grams_identity()); + EXPECT_EQ(first.common_grams_identity()->base_analyzer_fingerprint.size(), 64U); + EXPECT_EQ(first.common_grams_identity()->common_grams_fingerprint.size(), 64U); +} + +TEST_F(CustomAnalyzerTest, CommonGramsProviderSeparatesBaseAndDictionaryIdentity) { + Settings first_settings = whitespace_tokenizer_settings(); + first_settings.set("max_token_length", "100"); + Settings second_settings = whitespace_tokenizer_settings(); + second_settings.set("max_token_length", "101"); + CustomAnalyzerProvider first(common_grams_config("char_group", first_settings)); + CustomAnalyzerProvider second(common_grams_config("char_group", second_settings)); + + ASSERT_NE(first.common_grams_identity(), nullptr); + ASSERT_NE(second.common_grams_identity(), nullptr); + EXPECT_NE(first.common_grams_identity()->base_analyzer_fingerprint, + second.common_grams_identity()->base_analyzer_fingerprint); + EXPECT_EQ(first.common_grams_identity()->common_grams_fingerprint, + second.common_grams_identity()->common_grams_fingerprint); + + // The dictionary half of the identity is the word list's own content identity. Neither the + // provider nor an index policy can supply one, so every provider in this process agrees on it. + EXPECT_EQ(first.common_grams_identity()->common_grams_dictionary_identity, + CommonWordSet::default_word_set()->identity()); + EXPECT_EQ(second.common_grams_identity()->common_grams_dictionary_identity, + first.common_grams_identity()->common_grams_dictionary_identity); +} + +TEST_F(CustomAnalyzerTest, CommonGramsIdentityIncludesOuterCharFilter) { + CustomAnalyzerProvider no_outer_filter( + common_grams_config("char_group", whitespace_tokenizer_settings())); + ASSERT_NE(no_outer_filter.common_grams_identity(), nullptr); + + const std::map slash_to_space = { + {INVERTED_INDEX_PARSER_CHAR_FILTER_TYPE, INVERTED_INDEX_CHAR_FILTER_CHAR_REPLACE}, + {INVERTED_INDEX_PARSER_CHAR_FILTER_PATTERN, "/"}, + {INVERTED_INDEX_PARSER_CHAR_FILTER_REPLACEMENT, " "}}; + const std::map dash_to_space = { + {INVERTED_INDEX_PARSER_CHAR_FILTER_TYPE, INVERTED_INDEX_CHAR_FILTER_CHAR_REPLACE}, + {INVERTED_INDEX_PARSER_CHAR_FILTER_PATTERN, "-"}, + {INVERTED_INDEX_PARSER_CHAR_FILTER_REPLACEMENT, " "}}; + CustomAnalyzerProvider slash_filter( + common_grams_config("char_group", whitespace_tokenizer_settings()), slash_to_space); + CustomAnalyzerProvider repeated_slash_filter( + common_grams_config("char_group", whitespace_tokenizer_settings()), slash_to_space); + CustomAnalyzerProvider dash_filter( + common_grams_config("char_group", whitespace_tokenizer_settings()), dash_to_space); + ASSERT_NE(slash_filter.common_grams_identity(), nullptr); + ASSERT_NE(repeated_slash_filter.common_grams_identity(), nullptr); + ASSERT_NE(dash_filter.common_grams_identity(), nullptr); + + EXPECT_EQ(*slash_filter.common_grams_identity(), + *repeated_slash_filter.common_grams_identity()); + EXPECT_EQ(slash_filter.common_grams_identity()->common_grams_dictionary_identity, + no_outer_filter.common_grams_identity()->common_grams_dictionary_identity); + EXPECT_EQ(slash_filter.common_grams_identity()->common_grams_fingerprint, + no_outer_filter.common_grams_identity()->common_grams_fingerprint); + EXPECT_NE(slash_filter.common_grams_identity()->base_analyzer_fingerprint, + no_outer_filter.common_grams_identity()->base_analyzer_fingerprint); + EXPECT_NE(slash_filter.common_grams_identity()->base_analyzer_fingerprint, + dash_filter.common_grams_identity()->base_analyzer_fingerprint); +} + +TEST_F(CustomAnalyzerTest, ProviderSharesOneLegacyAnalyzerWhenCommonGramsIsAbsent) { + CustomAnalyzerConfig::Builder builder; + builder.with_tokenizer_config("char_group", whitespace_tokenizer_settings()); + builder.add_token_filter_config("lowercase", {}); + auto provider = std::make_shared(builder.build()); + + auto analyzer = provider->get_analyzer(AnalysisPurpose::kIndex); + EXPECT_EQ(provider->get_analyzer(AnalysisPurpose::kPlainQuery), analyzer); + EXPECT_EQ(provider->get_analyzer(AnalysisPurpose::kExactPhraseQuery), analyzer); + EXPECT_EQ(provider->get_analyzer(AnalysisPurpose::kPhrasePrefixQuery), analyzer); + EXPECT_EQ(provider->common_grams_identity(), nullptr); +} + +TEST_F(CustomAnalyzerTest, ProviderExposesBaseFingerprintWithoutCommonGrams) { + CustomAnalyzerConfig::Builder plain_builder; + plain_builder.with_tokenizer_config("char_group", whitespace_tokenizer_settings()); + plain_builder.add_token_filter_config("lowercase", {}); + CustomAnalyzerProvider plain(plain_builder.build()); + + CustomAnalyzerProvider common_grams(common_grams_config( + "char_group", whitespace_tokenizer_settings(), {{"lowercase", {}}})); + + EXPECT_EQ(plain.base_analyzer_fingerprint().size(), 64U); + EXPECT_EQ(plain.base_analyzer_fingerprint(), common_grams.base_analyzer_fingerprint()); + ASSERT_NE(common_grams.common_grams_identity(), nullptr); + EXPECT_EQ(common_grams.base_analyzer_fingerprint(), + common_grams.common_grams_identity()->base_analyzer_fingerprint); +} + // TEST_F(CustomAnalyzerTest, test) { // std::string name = "name"; // std::string path = "/mnt/disk3/yangsiyu/clucene"; diff --git a/be/test/storage/index/inverted/common/single_flight_test.cpp b/be/test/storage/index/inverted/common/single_flight_test.cpp new file mode 100644 index 00000000000000..5f6527230552ae --- /dev/null +++ b/be/test/storage/index/inverted/common/single_flight_test.cpp @@ -0,0 +1,220 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/inverted/common/single_flight.h" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/check.h" + +namespace doris::segment_v2::inverted_index { + +struct BlockingMoveState { + std::mutex mutex; + std::condition_variable condition; + bool move_started = false; + bool release_move = false; +}; + +struct BlockingMoveResult { + explicit BlockingMoveResult(std::shared_ptr state_) + : state(std::move(state_)) {} + + BlockingMoveResult(const BlockingMoveResult&) = default; + BlockingMoveResult& operator=(const BlockingMoveResult&) = default; + + BlockingMoveResult(BlockingMoveResult&& other) noexcept : state(std::move(other.state)) { + std::unique_lock lock(state->mutex); + state->move_started = true; + state->condition.notify_all(); + state->condition.wait(lock, [&] { return state->release_move; }); + } + + BlockingMoveResult& operator=(BlockingMoveResult&&) = delete; + + std::shared_ptr state; +}; + +// A lone caller leads, a second concurrent caller follows, and the follower reuses the +// leader's published result. The in-flight entry is cleared on publish. +TEST(SingleFlight, LeadFollowReuse) { + SingleFlight sf; + + auto leader = sf.join_or_lead("k"); + EXPECT_FALSE(leader.has_value()); // first caller leads + EXPECT_EQ(sf.inflight_size(), 1U); + + auto follower = sf.join_or_lead("k"); + ASSERT_TRUE(follower.has_value()); // second caller follows the in-flight leader + EXPECT_EQ(sf.inflight_size(), 1U); // still one in-flight key + + sf.publish("k", 7); + EXPECT_EQ(follower->get(), 7); // follower reuses the leader's result + EXPECT_EQ(sf.inflight_size(), 0U); +} + +// Different keys execute independently -- each first caller leads. +TEST(SingleFlight, DistinctKeysLeadIndependently) { + SingleFlight sf; + EXPECT_FALSE(sf.join_or_lead("a").has_value()); + EXPECT_FALSE(sf.join_or_lead("b").has_value()); + EXPECT_EQ(sf.inflight_size(), 2U); + sf.publish("a", 1); + sf.publish("b", 2); + EXPECT_EQ(sf.inflight_size(), 0U); +} + +// After a key is published, the next caller of the same key leads a fresh execution. +TEST(SingleFlight, ReLeadAfterPublish) { + SingleFlight sf; + EXPECT_FALSE(sf.join_or_lead("k").has_value()); + sf.publish("k", 1); + EXPECT_EQ(sf.inflight_size(), 0U); + EXPECT_FALSE(sf.join_or_lead("k").has_value()); // leads again + EXPECT_EQ(sf.inflight_size(), 1U); + sf.publish("k", 2); +} + +TEST(SingleFlight, JoinDuringPublicationStillFollowsPublishedFlight) { + SingleFlight sf; + ASSERT_FALSE(sf.join_or_lead("k").has_value()); + + auto state = std::make_shared(); + BlockingMoveResult result(state); + std::thread publisher([&] { sf.publish("k", result); }); + { + std::unique_lock lock(state->mutex); + state->condition.wait(lock, [&] { return state->move_started; }); + } + + auto follower = sf.join_or_lead("k"); + EXPECT_TRUE(follower.has_value()); + { + std::lock_guard lock(state->mutex); + state->release_move = true; + } + state->condition.notify_all(); + publisher.join(); + + ASSERT_TRUE(follower.has_value()); + EXPECT_EQ(follower->get().state, state); + EXPECT_EQ(sf.inflight_size(), 0U); +} + +// The motivating scenario: many threads issue the same key concurrently. Exactly one leads; +// all followers receive the single shared result (one shared_ptr payload, shared read-only). +TEST(SingleFlight, ConcurrentCollapsesToOneLeader) { + constexpr int kThreads = 16; + SingleFlight> sf; + + std::atomic leader_count {0}; + std::latch joined(kThreads); + std::latch release_leader(1); + std::vector> results(kThreads); + auto payload = std::make_shared(42); + + std::vector threads; + threads.reserve(kThreads); + for (int i = 0; i < kThreads; ++i) { + threads.emplace_back([&, i] { + auto follow = sf.join_or_lead("hot-segment"); + const bool is_leader = !follow.has_value(); + joined.count_down(); + if (is_leader) { + leader_count.fetch_add(1, std::memory_order_relaxed); + joined.wait(); // ensure every thread has joined before we publish + release_leader.wait(); + sf.publish("hot-segment", payload); + } else { + results[i] = follow->get(); + } + }); + } + joined.wait(); + EXPECT_EQ(sf.inflight_size(), 1U); + release_leader.count_down(); + for (auto& t : threads) { + t.join(); + } + + EXPECT_EQ(leader_count.load(), 1); // only one open/execute happened + EXPECT_EQ(sf.inflight_size(), 0U); + for (int i = 0; i < kThreads; ++i) { + if (results[i] != nullptr) { + EXPECT_EQ(results[i], payload); // same shared object, not a recomputed copy + EXPECT_EQ(*results[i], 42); + } + } +} + +TEST(SingleFlight, ConcurrentDifferentRawQueriesNeverJoin) { + constexpr int kThreads = 16; + SingleFlight sf; + std::atomic leader_count {0}; + std::latch all_registered(kThreads); + std::latch release_publishers(1); + std::vector threads; + threads.reserve(kThreads); + for (int i = 0; i < kThreads; ++i) { + threads.emplace_back([&, i] { + const std::string key = "raw-query-" + std::to_string(i); + auto follower = sf.join_or_lead(key); + if (!follower.has_value()) { + leader_count.fetch_add(1, std::memory_order_relaxed); + } + all_registered.count_down(); + release_publishers.wait(); + DORIS_CHECK(!follower.has_value()); + sf.publish(key, i); + }); + } + all_registered.wait(); + EXPECT_EQ(sf.inflight_size(), kThreads); + release_publishers.count_down(); + for (auto& thread : threads) { + thread.join(); + } + + EXPECT_EQ(leader_count.load(std::memory_order_relaxed), kThreads); + EXPECT_EQ(sf.inflight_size(), 0U); +} + +// A leader that fails publishes its failure; followers observe it (and would then fall back +// to computing independently in the production path). +TEST(SingleFlight, ErrorResultPropagates) { + SingleFlight> sf; + auto leader = sf.join_or_lead("k"); + ASSERT_FALSE(leader.has_value()); + auto follower = sf.join_or_lead("k"); + ASSERT_TRUE(follower.has_value()); + + sf.publish("k", std::make_pair(false, 0)); // leader failed + auto [ok, value] = follower->get(); + EXPECT_FALSE(ok); + EXPECT_EQ(value, 0); +} + +} // namespace doris::segment_v2::inverted_index diff --git a/be/test/storage/index/inverted/common_grams/common_grams_key_codec_test.cpp b/be/test/storage/index/inverted/common_grams/common_grams_key_codec_test.cpp new file mode 100644 index 00000000000000..05ef2bb4e56453 --- /dev/null +++ b/be/test/storage/index/inverted/common_grams/common_grams_key_codec_test.cpp @@ -0,0 +1,553 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/inverted/common_grams/common_grams_key_codec.h" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "util/utf8_check.h" + +namespace doris::segment_v2::inverted_index { +namespace { + +TEST(CommonGramsKeyCodecTest, GramGoldenBytesAndMarkerRange) { + const std::string expected = + "\x1f" + "DORIS_COMMON_GRAM_V1" + "\x1f" + "00000001:ab"; + auto encoded = encode_common_gram("a", "b"); + ASSERT_TRUE(encoded.has_value()) << encoded.error(); + EXPECT_EQ(encoded.value(), expected); + + EXPECT_EQ(CG_V1_MARKER, std::string_view("\x1f" + "DORIS_COMMON_GRAM_V1" + "\x1f", + 22)); + EXPECT_EQ(CG_V1_MARKER_END, std::string_view("\x1f" + "DORIS_COMMON_GRAM_V1" + "\x20", + 22)); + EXPECT_GE(encoded.value(), CG_V1_MARKER); + EXPECT_LT(encoded.value(), CG_V1_MARKER_END); + EXPECT_EQ(encoded->find('\0'), std::string::npos); + EXPECT_TRUE(validate_utf8(encoded->data(), encoded->size())); + + auto hexadecimal_length = encode_common_gram("0123456789", "x"); + ASSERT_TRUE(hexadecimal_length.has_value()) << hexadecimal_length.error(); + EXPECT_EQ(hexadecimal_length.value(), + "\x1f" + "DORIS_COMMON_GRAM_V1" + "\x1f" + "0000000a:0123456789x"); + + auto different_boundary = encode_common_gram("ab", "c"); + ASSERT_TRUE(different_boundary.has_value()) << different_boundary.error(); + auto same_text = encode_common_gram("a", "bc"); + ASSERT_TRUE(same_text.has_value()) << same_text.error(); + EXPECT_NE(different_boundary.value(), same_text.value()); +} + +TEST(CommonGramsKeyCodecTest, ValidatesLogicalTermsWithoutEncoding) { + EXPECT_TRUE(validate_common_grams_logical_term("valid", "test term").ok()); + EXPECT_TRUE(validate_common_grams_logical_term("", "test term").ok()); + + const std::string nul_term("a\0b", 3); + const std::string invalid_utf8("\xc3\x28", 2); + const std::string overlong(COMMON_GRAM_MAX_ENCODED_BYTES + 1, 'x'); + for (const auto& term : {nul_term, invalid_utf8, overlong}) { + auto status = validate_common_grams_logical_term(term, "test term"); + EXPECT_EQ(status.code(), ErrorCode::INVERTED_INDEX_ANALYZER_ERROR); + } +} + +TEST(CommonGramsKeyCodecTest, TryEncodeDistinguishesTooLongAndReusesOutputCapacity) { + std::string output; + output.reserve(COMMON_GRAM_MAX_ENCODED_BYTES); + const char* reserved_data = output.data(); + + auto encoded = try_encode_common_gram("a", "b", &output); + ASSERT_TRUE(encoded.has_value()) << encoded.error(); + EXPECT_TRUE(encoded.value()); + EXPECT_EQ(output, encode_common_gram("a", "b").value()); + EXPECT_EQ(output.data(), reserved_data); + + const std::string huge(COMMON_GRAM_MAX_ENCODED_BYTES, 'x'); + auto too_long = try_encode_common_gram("the", huge, &output); + ASSERT_TRUE(too_long.has_value()) << too_long.error(); + EXPECT_FALSE(too_long.value()); + EXPECT_TRUE(output.empty()); + EXPECT_EQ(output.data(), reserved_data); + + const std::string invalid_utf8("\xc3\x28", 2); + auto invalid = try_encode_common_gram(invalid_utf8, "valid", &output); + EXPECT_FALSE(invalid.has_value()); + EXPECT_EQ(invalid.error().code(), ErrorCode::INVERTED_INDEX_ANALYZER_ERROR); + EXPECT_TRUE(output.empty()); +} + +TEST(CommonGramsKeyCodecTest, PrevalidatedEncoderMatchesCheckedEncoder) { + for (const auto& [left, right] : std::vector> { + {"a", "b"}, {"the", "term"}, {"", "right"}}) { + std::string checked; + auto checked_result = try_encode_common_gram(left, right, &checked); + ASSERT_TRUE(checked_result.has_value()) << checked_result.error(); + + std::string prevalidated; + EXPECT_EQ(try_encode_common_gram_prevalidated(left, right, prevalidated), + checked_result.value()); + EXPECT_EQ(prevalidated, checked); + } + + std::string too_long(COMMON_GRAM_MAX_ENCODED_BYTES, 'x'); + std::string output = "stale"; + EXPECT_FALSE(try_encode_common_gram_prevalidated("left", too_long, output)); + EXPECT_TRUE(output.empty()); +} + +TEST(CommonGramsKeyCodecTest, PrevalidatedPlainEncoderMatchesCheckedEncoderAndBoundaries) { + std::string output; + output.reserve(COMMON_GRAM_MAX_ENCODED_BYTES); + const char* reserved_data = output.data(); + + for (const char marker : {PLAIN_ESCAPE_PREFIX, '\x1f'}) { + const std::string logical = std::string(1, marker) + "plain"; + std::string checked; + auto checked_result = + try_encode_plain_term(logical, PlainTermKeyVersion::kEscapedV1, &checked); + ASSERT_TRUE(checked_result.has_value()) << checked_result.error(); + ASSERT_TRUE(checked_result.value()); + + EXPECT_TRUE(try_encode_escaped_plain_term_prevalidated(logical, output)); + EXPECT_EQ(output, checked); + EXPECT_EQ(output.data(), reserved_data); + } + + const std::string maximum_encodable = std::string(1, PLAIN_ESCAPE_PREFIX) + + std::string(COMMON_GRAM_MAX_ENCODED_BYTES - 2, 'x'); + EXPECT_TRUE(try_encode_escaped_plain_term_prevalidated(maximum_encodable, output)); + EXPECT_EQ(output.size(), COMMON_GRAM_MAX_ENCODED_BYTES); + EXPECT_EQ(output.data(), reserved_data); + + const std::string too_long = std::string(1, PLAIN_ESCAPE_PREFIX) + + std::string(COMMON_GRAM_MAX_ENCODED_BYTES - 1, 'x'); + EXPECT_FALSE(try_encode_escaped_plain_term_prevalidated(too_long, output)); + EXPECT_TRUE(output.empty()); + EXPECT_EQ(output.data(), reserved_data); +} + +TEST(CommonGramsKeyCodecTest, PlainTermViewBorrowsOrdinaryKeysAndUsesScratchOnlyForEscapes) { + const std::string ordinary = "ordinary"; + std::string scratch = "stale"; + auto borrowed = try_encode_plain_term_view(ordinary, PlainTermKeyVersion::kEscapedV1, &scratch); + ASSERT_TRUE(borrowed.has_value()) << borrowed.error(); + ASSERT_TRUE(borrowed->has_value()); + EXPECT_EQ(**borrowed, ordinary); + EXPECT_EQ(borrowed->value().data(), ordinary.data()); + EXPECT_TRUE(scratch.empty()); + + const std::string escaped = std::string(1, '\x1f') + "literal"; + auto encoded = try_encode_plain_term_view(escaped, PlainTermKeyVersion::kEscapedV1, &scratch); + ASSERT_TRUE(encoded.has_value()) << encoded.error(); + ASSERT_TRUE(encoded->has_value()); + EXPECT_EQ(**encoded, *encode_plain_term(escaped, PlainTermKeyVersion::kEscapedV1)); + EXPECT_EQ(encoded->value().data(), scratch.data()); + + std::string unrepresentable(COMMON_GRAM_MAX_ENCODED_BYTES, 'x'); + unrepresentable.front() = PLAIN_ESCAPE_PREFIX; + auto absent = + try_encode_plain_term_view(unrepresentable, PlainTermKeyVersion::kEscapedV1, &scratch); + ASSERT_TRUE(absent.has_value()) << absent.error(); + EXPECT_FALSE(absent->has_value()); + EXPECT_TRUE(scratch.empty()); +} + +TEST(CommonGramsKeyCodecTest, GramComponentsUseUnescapedLogicalBytes) { + const std::array cases { + std::tuple {std::string(1, '\x1e'), std::string(1, '\x1f'), + std::string("\x1f" + "DORIS_COMMON_GRAM_V1" + "\x1f" + "00000001:\x1e\x1f")}, + std::tuple {std::string(1, '\x1f'), std::string(1, '\x1e'), + std::string("\x1f" + "DORIS_COMMON_GRAM_V1" + "\x1f" + "00000001:\x1f\x1e")}, + std::tuple {std::string("\x1e" + "L"), + std::string("\x1f" + "R"), + std::string("\x1f" + "DORIS_COMMON_GRAM_V1" + "\x1f" + "00000002:\x1e" + "L\x1f" + "R")}, + std::tuple {std::string("\x1f" + "L"), + std::string("\x1e" + "R"), + std::string("\x1f" + "DORIS_COMMON_GRAM_V1" + "\x1f" + "00000002:\x1f" + "L\x1e" + "R")}, + }; + for (const auto& [left, right, expected] : cases) { + auto encoded = encode_common_gram(left, right); + ASSERT_TRUE(encoded.has_value()) << encoded.error(); + EXPECT_EQ(encoded.value(), expected); + } +} + +TEST(CommonGramsKeyCodecTest, PlainKeyVersionsHaveReversibleGoldenBytes) { + const std::array raw_versions {PlainTermKeyVersion::kLegacyRaw, + PlainTermKeyVersion::kRawNoInternal}; + const std::string escape_leading = + "\x1e" + "alpha"; + const std::string gram_leading = + "\x1f" + "alpha"; + + for (PlainTermKeyVersion version : raw_versions) { + auto escape_encoded = encode_plain_term(escape_leading, version); + ASSERT_TRUE(escape_encoded.has_value()) << escape_encoded.error(); + EXPECT_EQ(escape_encoded.value(), escape_leading); + auto escape_decoded = decode_plain_term(escape_encoded.value(), version); + ASSERT_TRUE(escape_decoded.has_value()) << escape_decoded.error(); + EXPECT_EQ(escape_decoded.value(), escape_leading); + + auto gram_encoded = encode_plain_term(gram_leading, version); + ASSERT_TRUE(gram_encoded.has_value()) << gram_encoded.error(); + EXPECT_EQ(gram_encoded.value(), gram_leading); + auto gram_decoded = decode_plain_term(gram_encoded.value(), version); + ASSERT_TRUE(gram_decoded.has_value()) << gram_decoded.error(); + EXPECT_EQ(gram_decoded.value(), gram_leading); + } + + auto escape_encoded = encode_plain_term(escape_leading, PlainTermKeyVersion::kEscapedV1); + ASSERT_TRUE(escape_encoded.has_value()) << escape_encoded.error(); + EXPECT_EQ(escape_encoded.value(), + "\x1e" + "Ealpha"); + auto escape_decoded = + decode_plain_term(escape_encoded.value(), PlainTermKeyVersion::kEscapedV1); + ASSERT_TRUE(escape_decoded.has_value()) << escape_decoded.error(); + EXPECT_EQ(escape_decoded.value(), escape_leading); + + auto gram_encoded = encode_plain_term(gram_leading, PlainTermKeyVersion::kEscapedV1); + ASSERT_TRUE(gram_encoded.has_value()) << gram_encoded.error(); + EXPECT_EQ(gram_encoded.value(), + "\x1e" + "Galpha"); + auto gram_decoded = decode_plain_term(gram_encoded.value(), PlainTermKeyVersion::kEscapedV1); + ASSERT_TRUE(gram_decoded.has_value()) << gram_decoded.error(); + EXPECT_EQ(gram_decoded.value(), gram_leading); + + EXPECT_FALSE(decode_plain_term("\x1e", PlainTermKeyVersion::kEscapedV1).has_value()); + EXPECT_FALSE(decode_plain_term("\x1e" + "Xalpha", + PlainTermKeyVersion::kEscapedV1) + .has_value()); +} + +TEST(CommonGramsKeyCodecTest, DecodePlainTermViewBorrowsOrdinaryInput) { + const std::array cases { + std::pair {PlainTermKeyVersion::kLegacyRaw, std::string("legacy")}, + std::pair {PlainTermKeyVersion::kEscapedV1, std::string("ordinary")}, + }; + for (const auto& [version, physical_term] : cases) { + std::string scratch; + auto decoded = decode_plain_term_view(physical_term, version, &scratch); + ASSERT_TRUE(decoded.has_value()) << decoded.error(); + EXPECT_EQ(decoded.value(), physical_term); + EXPECT_EQ(decoded->data(), physical_term.data()); + EXPECT_TRUE(scratch.empty()); + } +} + +TEST(CommonGramsKeyCodecTest, DecodePlainTermViewUsesScratchForEscapes) { + const std::array cases { + std::pair {std::string("\x1e" + "Ealpha"), + std::string("\x1e" + "alpha")}, + std::pair {std::string("\x1e" + "Galpha"), + std::string("\x1f" + "alpha")}, + }; + for (const auto& [physical_term, logical_term] : cases) { + std::string scratch = "stale"; + auto decoded = + decode_plain_term_view(physical_term, PlainTermKeyVersion::kEscapedV1, &scratch); + ASSERT_TRUE(decoded.has_value()) << decoded.error(); + EXPECT_EQ(decoded.value(), logical_term); + EXPECT_EQ(scratch, logical_term); + EXPECT_EQ(decoded->data(), scratch.data()); + } +} + +TEST(CommonGramsKeyCodecTest, EscapedPlainKeysCannotEnterGramMarkerRange) { + const std::array logical_terms {std::string("plain"), + std::string("\x1e" + "plain"), + std::string("\x1f" + "plain")}; + for (const auto& term : logical_terms) { + auto encoded = encode_plain_term(term, PlainTermKeyVersion::kEscapedV1); + ASSERT_TRUE(encoded.has_value()) << encoded.error(); + EXPECT_FALSE(encoded.value() >= CG_V1_MARKER && encoded.value() < CG_V1_MARKER_END); + auto decoded = decode_plain_term(encoded.value(), PlainTermKeyVersion::kEscapedV1); + ASSERT_TRUE(decoded.has_value()) << decoded.error(); + EXPECT_EQ(decoded.value(), term); + } +} + +TEST(CommonGramsKeyCodecTest, InternalNamespaceAndLegacyBypassAreSeparated) { + const std::string legacy_marker = + "\x1f" + "SNII_PHRASE_BIGRAM" + "\x1f"; + EXPECT_EQ(INTERNAL_TERM_NAMESPACE_BEGIN, std::string_view("\x1f", 1)); + EXPECT_EQ(INTERNAL_TERM_NAMESPACE_END, std::string_view("\x20", 1)); + + EXPECT_TRUE(is_internal_term_key(encode_common_gram("a", "b").value())); + EXPECT_TRUE(is_internal_term_key(legacy_marker + "payload")); + EXPECT_TRUE( + is_internal_term_key("\x1f" + "FUTURE_INTERNAL_TERM")); + EXPECT_FALSE( + is_internal_term_key("\x1e" + "Gescaped")); + EXPECT_FALSE(is_internal_term_key("plain")); + + EXPECT_TRUE(legacy_raw_exact_requires_bypass(CG_V1_MARKER)); + EXPECT_TRUE(legacy_raw_exact_requires_bypass(legacy_marker + "payload")); + EXPECT_FALSE(legacy_raw_exact_requires_bypass(std::string(1, '\x1f'))); + EXPECT_FALSE( + legacy_raw_exact_requires_bypass("\x1f" + "literal")); + EXPECT_FALSE(legacy_raw_exact_requires_bypass("plain")); + + EXPECT_TRUE(legacy_raw_prefix_requires_bypass("")); + EXPECT_TRUE(legacy_raw_prefix_requires_bypass(std::string(1, '\x1f'))); + EXPECT_TRUE( + legacy_raw_prefix_requires_bypass("\x1f" + "DORIS_COMMON")); + EXPECT_TRUE(legacy_raw_prefix_requires_bypass(CG_V1_MARKER)); + EXPECT_TRUE(legacy_raw_prefix_requires_bypass(legacy_marker + "payload")); + EXPECT_FALSE( + legacy_raw_prefix_requires_bypass("\x1f" + "literal")); + EXPECT_FALSE(legacy_raw_prefix_requires_bypass("plain")); +} + +TEST(CommonGramsKeyCodecTest, PlainEncodingPreservesLogicalPrefixRanges) { + const std::array logical_prefixes { + std::string("plain"), + std::string("\x1e" + "escape"), + std::string("\x1f" + "marker"), + }; + const std::array logical_terms { + logical_prefixes[0] + "-suffix", + logical_prefixes[1] + "-suffix", + logical_prefixes[2] + "-suffix", + }; + for (const std::string& logical_prefix : logical_prefixes) { + auto physical_prefix = encode_plain_term(logical_prefix, PlainTermKeyVersion::kEscapedV1); + ASSERT_TRUE(physical_prefix.has_value()) << physical_prefix.error(); + for (const std::string& logical_term : logical_terms) { + auto physical_term = encode_plain_term(logical_term, PlainTermKeyVersion::kEscapedV1); + ASSERT_TRUE(physical_term.has_value()) << physical_term.error(); + EXPECT_EQ(logical_term.starts_with(logical_prefix), + physical_term->starts_with(physical_prefix.value())); + } + } +} + +TEST(CommonGramsKeyCodecTest, EncodedLengthBoundariesAreShared) { + EXPECT_EQ(COMMON_GRAM_MAX_ENCODED_BYTES, 16383); + + const std::array all_versions {PlainTermKeyVersion::kLegacyRaw, PlainTermKeyVersion::kEscapedV1, + PlainTermKeyVersion::kRawNoInternal}; + for (PlainTermKeyVersion version : all_versions) { + for (size_t size : {size_t {16382}, size_t {16383}}) { + std::string term(size, 'p'); + auto encoded = encode_plain_term(term, version); + ASSERT_TRUE(encoded.has_value()) << encoded.error(); + EXPECT_EQ(encoded->size(), size); + auto decoded = decode_plain_term(encoded.value(), version); + ASSERT_TRUE(decoded.has_value()) << decoded.error(); + EXPECT_EQ(decoded.value(), term); + } + const std::string overlong(16384, 'p'); + EXPECT_FALSE(encode_plain_term(overlong, version).has_value()); + EXPECT_FALSE(decode_plain_term(overlong, version).has_value()); + } + + for (const auto& [logical_size, encoded_size] : + std::array {std::pair {size_t {16381}, size_t {16382}}, + std::pair {size_t {16382}, size_t {16383}}}) { + std::string term(logical_size, 'p'); + term.front() = '\x1e'; + auto encoded = encode_plain_term(term, PlainTermKeyVersion::kEscapedV1); + ASSERT_TRUE(encoded.has_value()) << encoded.error(); + EXPECT_EQ(encoded->size(), encoded_size); + } + + std::string escape_overflow(16383, 'p'); + escape_overflow.front() = '\x1f'; + EXPECT_FALSE(encode_plain_term(escape_overflow, PlainTermKeyVersion::kEscapedV1).has_value()); + std::string try_output = "stale"; + auto try_overflow = + try_encode_plain_term(escape_overflow, PlainTermKeyVersion::kEscapedV1, &try_output); + ASSERT_TRUE(try_overflow.has_value()) << try_overflow.error(); + EXPECT_FALSE(try_overflow.value()); + EXPECT_TRUE(try_output.empty()); + + const std::string invalid_utf8("\xc3\x28", 2); + auto try_invalid = + try_encode_plain_term(invalid_utf8, PlainTermKeyVersion::kEscapedV1, &try_output); + EXPECT_FALSE(try_invalid.has_value()); + EXPECT_EQ(try_invalid.error().code(), ErrorCode::INVERTED_INDEX_ANALYZER_ERROR); + EXPECT_TRUE(try_output.empty()); + + auto raw_fallback = encode_plain_term(escape_overflow, PlainTermKeyVersion::kRawNoInternal); + ASSERT_TRUE(raw_fallback.has_value()) << raw_fallback.error(); + EXPECT_EQ(raw_fallback->size(), 16383); + EXPECT_EQ(raw_fallback.value(), escape_overflow); + + for (const auto& [encoded_size, right_size, expected_encodable] : + std::array {std::tuple {size_t {16382}, size_t {16350}, true}, + std::tuple {size_t {16383}, size_t {16351}, true}, + std::tuple {size_t {16384}, size_t {16352}, false}}) { + std::string right(right_size, 'r'); + EXPECT_EQ(is_common_gram_encodable("a", right), expected_encodable); + auto encoded = encode_common_gram("a", right); + EXPECT_EQ(encoded.has_value(), expected_encodable); + if (encoded.has_value()) { + EXPECT_EQ(encoded->size(), encoded_size); + } + } +} + +TEST(CommonGramsKeyCodecTest, GramLengthBoundariesCanBeConcentratedInLeft) { + for (const auto& [encoded_size, left_size, expected_encodable] : + std::array {std::tuple {size_t {16382}, size_t {16350}, true}, + std::tuple {size_t {16383}, size_t {16351}, true}, + std::tuple {size_t {16384}, size_t {16352}, false}}) { + std::string left(left_size, 'l'); + EXPECT_EQ(is_common_gram_encodable(left, "r"), expected_encodable); + auto encoded = encode_common_gram(left, "r"); + EXPECT_EQ(encoded.has_value(), expected_encodable); + if (encoded.has_value()) { + EXPECT_EQ(encoded->size(), encoded_size); + } + } + + const std::string e_acute = "\xc3\xa9"; + std::string multibyte_left; + multibyte_left.reserve(16352); + for (size_t i = 0; i < 8175; ++i) { + multibyte_left.append(e_acute); + } + auto at_16382 = encode_common_gram(multibyte_left, "r"); + ASSERT_TRUE(at_16382.has_value()) << at_16382.error(); + EXPECT_EQ(at_16382->size(), 16382); + + multibyte_left.push_back('x'); + auto at_16383 = encode_common_gram(multibyte_left, "r"); + ASSERT_TRUE(at_16383.has_value()) << at_16383.error(); + EXPECT_EQ(at_16383->size(), 16383); + + multibyte_left.push_back('x'); + EXPECT_FALSE(is_common_gram_encodable(multibyte_left, "r")); + EXPECT_FALSE(encode_common_gram(multibyte_left, "r").has_value()); +} + +TEST(CommonGramsKeyCodecTest, LengthUsesUtf8BytesRatherThanCodePoints) { + const std::string chinese = "\xe4\xbd\xa0"; + const std::string e_acute = "\xc3\xa9"; + std::string right; + right.reserve(16350); + for (size_t i = 0; i < 8174; ++i) { + right.append(e_acute); + } + + auto at_16382 = encode_common_gram(chinese, right); + ASSERT_TRUE(at_16382.has_value()) << at_16382.error(); + EXPECT_EQ(at_16382->size(), 16382); + EXPECT_NE(at_16382->find("00000003:"), std::string::npos); + + right.push_back('x'); + auto at_16383 = encode_common_gram(chinese, right); + ASSERT_TRUE(at_16383.has_value()) << at_16383.error(); + EXPECT_EQ(at_16383->size(), 16383); + + right.push_back('x'); + EXPECT_FALSE(is_common_gram_encodable(chinese, right)); + EXPECT_FALSE(encode_common_gram(chinese, right).has_value()); +} + +TEST(CommonGramsKeyCodecTest, RejectsNulAndInvalidUtf8) { + const std::string nul_term("a\0b", 3); + const std::string invalid_utf8("\xc3\x28", 2); + for (const auto& term : {nul_term, invalid_utf8}) { + EXPECT_FALSE(encode_plain_term(term, PlainTermKeyVersion::kLegacyRaw).has_value()); + EXPECT_FALSE(encode_plain_term(term, PlainTermKeyVersion::kEscapedV1).has_value()); + EXPECT_FALSE(encode_plain_term(term, PlainTermKeyVersion::kRawNoInternal).has_value()); + EXPECT_FALSE(decode_plain_term(term, PlainTermKeyVersion::kLegacyRaw).has_value()); + EXPECT_FALSE(is_common_gram_encodable(term, "valid")); + EXPECT_FALSE(is_common_gram_encodable("valid", term)); + EXPECT_FALSE(encode_common_gram(term, "valid").has_value()); + EXPECT_FALSE(encode_common_gram("valid", term).has_value()); + } + + const std::string escaped_nul( + "\x1e" + "Ea\0b", + 5); + const std::string escaped_invalid( + "\x1e" + "G\xc3\x28", + 4); + EXPECT_FALSE(decode_plain_term(escaped_nul, PlainTermKeyVersion::kEscapedV1).has_value()); + EXPECT_FALSE(decode_plain_term(escaped_invalid, PlainTermKeyVersion::kEscapedV1).has_value()); + for (PlainTermKeyVersion version : + {PlainTermKeyVersion::kLegacyRaw, PlainTermKeyVersion::kRawNoInternal}) { + EXPECT_FALSE(decode_plain_term(nul_term, version).has_value()); + EXPECT_FALSE(decode_plain_term(invalid_utf8, version).has_value()); + } +} + +} // namespace +} // namespace doris::segment_v2::inverted_index diff --git a/be/test/storage/index/inverted/common_grams/common_word_set_test.cpp b/be/test/storage/index/inverted/common_grams/common_word_set_test.cpp new file mode 100644 index 00000000000000..2f1ce9fca4fe00 --- /dev/null +++ b/be/test/storage/index/inverted/common_grams/common_word_set_test.cpp @@ -0,0 +1,186 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/inverted/common_grams/common_word_set.h" + +#include + +#include +#include +#include + +#include "common/config.h" + +namespace doris::segment_v2::inverted_index { + +namespace common_grams_testing { +uint64_t common_word_hash_lookup_count(); +void reset_common_word_hash_lookup_count(); +} // namespace common_grams_testing + +namespace { + +TEST(CommonWordSetTest, DefaultWordSetLivesUnderTheSharedDictionaryRoot) { + // The word list follows the same layout as the icu, ik and pinyin dictionaries, so relocating + // inverted_index_dict_path moves all of them together instead of leaving this one behind. + const std::string saved = config::inverted_index_dict_path; + config::inverted_index_dict_path = "/somewhere/else"; + EXPECT_EQ(CommonWordSet::default_word_set_path(), + "/somewhere/else/common_grams/default_words.txt"); + config::inverted_index_dict_path = saved; +} + +TEST(CommonWordSetTest, BuiltinEnglishStopV1IsTheExactFrozen33WordSet) { + EXPECT_EQ(BUILTIN_COMMON_WORDS_RESOURCE, "builtin:lucene_english_stop:v1"); + const auto& words = CommonWordSet::builtin_english_stop_words_v1(); + constexpr std::array expected = { + "a", "an", "and", "are", "as", "at", "be", "but", "by", "for", "if", + "in", "into", "is", "it", "no", "not", "of", "on", "or", "such", "that", + "the", "their", "then", "there", "these", "they", "this", "to", "was", "will", "with"}; + + EXPECT_EQ(words.size(), expected.size()); + for (std::string_view word : expected) { + EXPECT_TRUE(words.contains(word)) << word; + } + EXPECT_FALSE(words.contains("english")); + EXPECT_FALSE(words.contains("The")); +} + +TEST(CommonWordSetTest, RejectsImpossibleShapesBeforeHashLookup) { + const auto& words = CommonWordSet::builtin_english_stop_words_v1(); + common_grams_testing::reset_common_word_hash_lookup_count(); + + EXPECT_FALSE(words.contains("encyclopedia")); // longer than every builtin word + EXPECT_FALSE(words.contains("zoo")); // impossible first byte + EXPECT_FALSE(words.contains("tea")); // possible shape, absent from the set + EXPECT_TRUE(words.contains("the")); + + EXPECT_EQ(common_grams_testing::common_word_hash_lookup_count(), 2); +} + +TEST(CommonWordSetTest, WordsetV1IgnoresEmptyAndCommentLinesAndDeduplicates) { + EXPECT_EQ(WORDSET_FORMAT_V1, "wordset:v1"); + const std::string content = + "# comment\n" + "alpha\n" + "\n" + "beta\n" + "alpha\n" + "# trailing comment\n" + "\xe4\xbd\xa0\xe5\xa5\xbd\n"; + auto parsed = CommonWordSet::parse_words(content); + ASSERT_TRUE(parsed.has_value()) << parsed.error(); + EXPECT_EQ(parsed->size(), 3); + EXPECT_TRUE(parsed->contains("alpha")); + EXPECT_TRUE(parsed->contains("beta")); + EXPECT_TRUE(parsed->contains("\xe4\xbd\xa0\xe5\xa5\xbd")); + EXPECT_FALSE(parsed->contains("# comment")); + EXPECT_FALSE(parsed->contains("")); +} + +TEST(CommonWordSetTest, WordsetV1PreservesTermBytesAndAcceptsCrLfAndFinalLine) { + const std::string content = + "alpha\r\n" + "beta\r\n" + "inline#hash\n" + " leading\n" + "trailing \n" + " #not-comment\n" + "final"; + auto parsed = CommonWordSet::parse_words(content); + ASSERT_TRUE(parsed.has_value()) << parsed.error(); + EXPECT_EQ(parsed->size(), 7); + for (std::string_view term : + {"alpha", "beta", "inline#hash", " leading", "trailing ", " #not-comment", "final"}) { + EXPECT_TRUE(parsed->contains(term)) << term; + } + EXPECT_FALSE(parsed->contains("alpha\r")); + EXPECT_FALSE(parsed->contains("beta\r")); +} + +TEST(CommonWordSetTest, UnterminatedFinalCarriageReturnIsTermData) { + auto final_with_carriage_return = CommonWordSet::parse_words("final\r"); + ASSERT_TRUE(final_with_carriage_return.has_value()) << final_with_carriage_return.error(); + EXPECT_EQ(final_with_carriage_return->size(), 1); + EXPECT_TRUE(final_with_carriage_return->contains("final\r")); + EXPECT_FALSE(final_with_carriage_return->contains("final")); + + auto lone_carriage_return = CommonWordSet::parse_words("\r"); + ASSERT_TRUE(lone_carriage_return.has_value()) << lone_carriage_return.error(); + EXPECT_EQ(lone_carriage_return->size(), 1); + EXPECT_TRUE(lone_carriage_return->contains("\r")); + EXPECT_FALSE(lone_carriage_return->contains("")); +} + +TEST(CommonWordSetTest, RejectsNulAndInvalidUtf8Terms) { + const std::string nul_content("alpha\na\0b\n", 10); + EXPECT_FALSE(CommonWordSet::parse_words(nul_content).has_value()); + + const std::string invalid_utf8("alpha\n\xc3\x28\n", 9); + EXPECT_FALSE(CommonWordSet::parse_words(invalid_utf8).has_value()); + + const std::string nul_comment("# bad\0comment\nalpha\n", 20); + EXPECT_FALSE(CommonWordSet::parse_words(nul_comment).has_value()); + const std::string invalid_comment("# bad\xc3\x28\nalpha\n", 14); + EXPECT_FALSE(CommonWordSet::parse_words(invalid_comment).has_value()); +} + +TEST(CommonWordSetTest, WordsetV1IsCaseSensitive) { + auto parsed = CommonWordSet::parse_words("Alpha\nalpha\n"); + ASSERT_TRUE(parsed.has_value()) << parsed.error(); + EXPECT_EQ(parsed->size(), 2); + EXPECT_TRUE(parsed->contains("Alpha")); + EXPECT_TRUE(parsed->contains("alpha")); +} + +TEST(CommonWordSetTest, BuiltinIdentityIsTheFrozenResourceName) { + EXPECT_EQ(CommonWordSet::builtin_english_stop_words_v1().identity(), + BUILTIN_COMMON_WORDS_RESOURCE); +} + +// A segment stamps this identity and the querying analyzer compares against it, so two BEs +// reading different word lists must not agree. Deriving it from the file bytes is what makes +// the BE-local word list safe; a fixed constant would let mismatched grams pass unnoticed. +TEST(CommonWordSetTest, ParsedIdentityIsDerivedFromContent) { + auto first = CommonWordSet::parse_words("alpha\nbeta\n"); + auto same = CommonWordSet::parse_words("alpha\nbeta\n"); + auto different = CommonWordSet::parse_words("alpha\ngamma\n"); + ASSERT_TRUE(first.has_value()) << first.error(); + ASSERT_TRUE(same.has_value()) << same.error(); + ASSERT_TRUE(different.has_value()) << different.error(); + + EXPECT_EQ(first->identity(), same->identity()); + EXPECT_NE(first->identity(), different->identity()); + EXPECT_TRUE(first->identity().starts_with("wordset:md5:")); + EXPECT_NE(first->identity(), BUILTIN_COMMON_WORDS_RESOURCE); +} + +// Comments and blank lines do not change the parsed word set, but they do change the identity. +// That direction is the safe one: a superfluous re-plan costs a cost comparison, whereas reusing +// grams across an edit that DID change the list would be wrong. +TEST(CommonWordSetTest, IdentityTracksRawBytesNotJustTheParsedWords) { + auto plain = CommonWordSet::parse_words("alpha\nbeta\n"); + auto commented = CommonWordSet::parse_words("# a note\nalpha\nbeta\n"); + ASSERT_TRUE(plain.has_value()) << plain.error(); + ASSERT_TRUE(commented.has_value()) << commented.error(); + + EXPECT_EQ(plain->size(), commented->size()); + EXPECT_NE(plain->identity(), commented->identity()); +} + +} // namespace +} // namespace doris::segment_v2::inverted_index diff --git a/be/test/storage/index/inverted/compaction/index_compaction_performance_test.cpp b/be/test/storage/index/inverted/compaction/index_compaction_performance_test.cpp index 24c8ca78127ff1..6626af5ff1dbc2 100644 --- a/be/test/storage/index/inverted/compaction/index_compaction_performance_test.cpp +++ b/be/test/storage/index/inverted/compaction/index_compaction_performance_test.cpp @@ -22,7 +22,7 @@ #include #include "storage/index/index_writer.h" -#include "storage/index/inverted/compaction/util/index_compaction_utils.cpp" +#include "storage/index/inverted/compaction/util/index_compaction_utils.h" #include "storage/utils.h" namespace doris { diff --git a/be/test/storage/index/inverted/compaction/index_compaction_test.cpp b/be/test/storage/index/inverted/compaction/index_compaction_test.cpp index 9f8eb7070dc81a..dd9ab02feca0ed 100644 --- a/be/test/storage/index/inverted/compaction/index_compaction_test.cpp +++ b/be/test/storage/index/inverted/compaction/index_compaction_test.cpp @@ -17,9 +17,13 @@ #include +#include + #include "storage/index/index_writer.h" -#include "storage/index/inverted/compaction/util/index_compaction_utils.cpp" +#include "storage/index/inverted/compaction/util/index_compaction_utils.h" #include "storage/utils.h" +#include "util/debug_points.h" +#include "util/defer_op.h" namespace doris { @@ -732,6 +736,62 @@ class IndexCompactionTest : public ::testing::Test { } } + void _build_snii_multi_index_tablet(bool second_supports_phrase = true) { + TabletSchemaPB schema_pb; + schema_pb.set_keys_type(KeysType::DUP_KEYS); + schema_pb.set_inverted_index_storage_format(InvertedIndexStorageFormatPB::SNII); + + IndexCompactionUtils::construct_column(schema_pb.add_column(), 0, "INT", "key"); + IndexCompactionUtils::construct_column(schema_pb.add_column(), 1, "STRING", "v1"); + IndexCompactionUtils::construct_column(schema_pb.add_column(), 2, "STRING", "v2"); + IndexCompactionUtils::construct_column(schema_pb.add_column(), 3, "INT", "v3"); + + auto add_index = [&schema_pb](int64_t index_id, std::string_view name, + bool supports_phrase) { + TabletIndexPB* index = schema_pb.add_index(); + index->set_index_id(index_id); + index->set_index_name(std::string(name)); + index->set_index_type(IndexType::INVERTED); + index->add_col_unique_id(1); + auto* properties = index->mutable_properties(); + (*properties)[INVERTED_INDEX_PARSER_KEY] = INVERTED_INDEX_PARSER_UNICODE; + (*properties)[INVERTED_INDEX_PARSER_PHRASE_SUPPORT_KEY] = + supports_phrase ? INVERTED_INDEX_PARSER_PHRASE_SUPPORT_YES + : INVERTED_INDEX_PARSER_PHRASE_SUPPORT_NO; + (*properties)[INVERTED_INDEX_PARSER_LOWERCASE_KEY] = INVERTED_INDEX_PARSER_TRUE; + }; + add_index(11001, "v1_phrase_a", true); + add_index(11002, "v1_phrase_b", second_supports_phrase); + + _tablet_schema = std::make_shared(); + _tablet_schema->init_from_pb(schema_pb); + TabletMetaSharedPtr tablet_meta(new TabletMeta(_tablet_schema)); + _tablet = std::make_shared(*_engine_ref, tablet_meta, _data_dir.get()); + EXPECT_TRUE(_tablet->init().ok()); + } + + std::vector _build_snii_source_rowsets() { + EXPECT_TRUE(io::global_local_filesystem()->delete_directory(_tablet->tablet_path()).ok()); + EXPECT_TRUE(io::global_local_filesystem()->create_directory(_tablet->tablet_path()).ok()); + const std::vector data_files = { + _current_dir + "/be/test/storage/index/inverted/data/data1.csv", + _current_dir + "/be/test/storage/index/inverted/data/data2.csv"}; + std::vector rowsets(data_files.size()); + auto check_indexes = [](const int32_t& size) { EXPECT_EQ(size, 2); }; + IndexCompactionUtils::build_rowsets( + _data_dir, _tablet_schema, _tablet, _engine_ref, rowsets, data_files, _inc_id, + check_indexes, false, 1000); + return rowsets; + } + + static std::string _read_index_file_bytes(const RowsetSharedPtr& rowset, uint32_t segment_id) { + const std::string path = fmt::format("{}/{}_{}.idx", rowset->tablet_path(), + rowset->rowset_id().to_string(), segment_id); + std::ifstream input(path, std::ios::binary); + EXPECT_TRUE(input.is_open()) << path; + return {std::istreambuf_iterator(input), std::istreambuf_iterator()}; + } + private: TabletSchemaSPtr _tablet_schema = nullptr; StorageEngine* _engine_ref = nullptr; @@ -1688,4 +1748,227 @@ TEST_F(IndexCompactionTest, test_inverted_index_ram_dir_disable_with_debug_point config::inverted_index_ram_dir_enable = original_ram_dir_enable; config::enable_debug_points = original_enable_debug_points; } + +TEST_F(IndexCompactionTest, snii_native_merge_validates_rowids_once_and_matches_raw_rebuild) { + const bool old_common_grams = config::enable_common_grams_index_build; + const bool old_debug_points = config::enable_debug_points; + const bool old_write_freq = config::snii_positions_index_write_freq; + config::enable_common_grams_index_build = false; + config::enable_debug_points = true; + config::snii_positions_index_write_freq = false; + constexpr std::string_view kValidationPoint = + "Compaction::snii_validated_rowid_conversion_created"; + constexpr std::string_view kReaderInitPoint = "Compaction::snii_eligibility_reader_initialized"; + DEFER({ + DebugPoints::instance()->remove(std::string(kValidationPoint)); + DebugPoints::instance()->remove(std::string(kReaderInitPoint)); + config::enable_common_grams_index_build = old_common_grams; + config::enable_debug_points = old_debug_points; + config::snii_positions_index_write_freq = old_write_freq; + }); + + _build_snii_multi_index_tablet(); + const std::vector rowsets = _build_snii_source_rowsets(); + size_t validation_count = 0; + size_t reader_init_count = 0; + std::function count_validation = [&validation_count]() { ++validation_count; }; + std::function count_reader_init = [&reader_init_count]() { ++reader_init_count; }; + DebugPoints::instance()->add_with_callback(std::string(kValidationPoint), count_validation); + DebugPoints::instance()->add_with_callback(std::string(kReaderInitPoint), count_reader_init); + + auto check_native_merge = [](const BaseCompaction& compaction, const RowsetWriterContext& ctx) { + // SNII classifies per (column, index): both logical indexes of column 1 + // merge natively, and the per-COLUMN set stays empty so the segment + // writer's V2/V3 skip logic never sees SNII columns. + EXPECT_TRUE(ctx.columns_to_do_index_compaction.empty()); + EXPECT_EQ(ctx.snii_indexes_to_do_compaction, + (std::set> {{1, 11001}, {1, 11002}})); + EXPECT_EQ(compaction._output_rowset->num_segments(), 2); + }; + RowsetSharedPtr native_merge; + Status status = IndexCompactionUtils::do_compaction(rowsets, _engine_ref, _tablet, true, + native_merge, check_native_merge, 1000); + ASSERT_TRUE(status.ok()) << status; + ASSERT_NE(native_merge, nullptr); + EXPECT_EQ(validation_count, 1); + EXPECT_EQ(reader_init_count, rowsets.size()); + DebugPoints::instance()->remove(std::string(kValidationPoint)); + DebugPoints::instance()->remove(std::string(kReaderInitPoint)); + + auto check_raw_rebuild = [](const BaseCompaction& compaction, const RowsetWriterContext& ctx) { + EXPECT_TRUE(ctx.columns_to_do_index_compaction.empty()); + EXPECT_EQ(compaction._output_rowset->num_segments(), 2); + }; + RowsetSharedPtr raw_rebuild; + status = IndexCompactionUtils::do_compaction(rowsets, _engine_ref, _tablet, false, raw_rebuild, + check_raw_rebuild, 1000); + ASSERT_TRUE(status.ok()) << status; + ASSERT_NE(raw_rebuild, nullptr); + ASSERT_EQ(native_merge->num_segments(), raw_rebuild->num_segments()); + for (uint32_t segment_id = 0; segment_id < native_merge->num_segments(); ++segment_id) { + EXPECT_EQ(_read_index_file_bytes(native_merge, segment_id), + _read_index_file_bytes(raw_rebuild, segment_id)); + } +} + +// The design's core split: two logical indexes share one column, the eligible +// one merges natively (no analyzer), the ineligible one (no phrase positions) +// raw-builds from the column -- in the SAME compaction pass. The old behavior +// AND-folded eligibility per column and fell back to raw for both. +TEST_F(IndexCompactionTest, snii_native_merge_compacts_eligible_index_and_raw_builds_sibling) { + const bool old_common_grams = config::enable_common_grams_index_build; + const bool old_write_freq = config::snii_positions_index_write_freq; + config::enable_common_grams_index_build = false; + config::snii_positions_index_write_freq = false; + DEFER({ + config::enable_common_grams_index_build = old_common_grams; + config::snii_positions_index_write_freq = old_write_freq; + }); + + _build_snii_multi_index_tablet(/*second_supports_phrase=*/false); + const std::vector rowsets = _build_snii_source_rowsets(); + auto check_split = [](const BaseCompaction& compaction, const RowsetWriterContext& ctx) { + // Only the phrase-capable index merges; its no-phrase sibling raw-builds. + EXPECT_TRUE(ctx.columns_to_do_index_compaction.empty()); + EXPECT_EQ(ctx.snii_indexes_to_do_compaction, + (std::set> {{1, 11001}})); + EXPECT_EQ(compaction._output_rowset->num_segments(), 2); + }; + RowsetSharedPtr output; + const Status status = IndexCompactionUtils::do_compaction(rowsets, _engine_ref, _tablet, true, + output, check_split, 1000); + ASSERT_TRUE(status.ok()) << status; + ASSERT_NE(output, nullptr); + std::vector segment_rows; + output->rowset_meta()->get_num_segment_rows(&segment_rows); + ASSERT_EQ(segment_rows.size(), output->num_segments()); + for (uint32_t segment_id = 0; segment_id < output->num_segments(); ++segment_id) { + const auto segment_path = output->segment_path(segment_id); + ASSERT_TRUE(segment_path.has_value()) << segment_path.error(); + auto file_reader = IndexCompactionUtils::init_index_file_reader( + output, segment_path.value(), InvertedIndexStorageFormatPB::SNII); + for (const TabletIndex* index : _tablet_schema->inverted_indexes()) { + const auto logical_index = file_reader->open_snii_index(index); + EXPECT_TRUE(logical_index.has_value()) << logical_index.error(); + // The merged and the raw-built index both cover every segment row. + EXPECT_EQ(logical_index.value()->stats().doc_count, + static_cast(segment_rows[segment_id])) + << "index " << index->index_id() << " segment " << segment_id; + } + } +} + +TEST_F(IndexCompactionTest, snii_native_merge_aborts_after_partial_destination_creation) { + const bool old_common_grams = config::enable_common_grams_index_build; + const bool old_debug_points = config::enable_debug_points; + const bool old_write_freq = config::snii_positions_index_write_freq; + config::enable_common_grams_index_build = false; + config::enable_debug_points = true; + config::snii_positions_index_write_freq = false; + constexpr std::string_view kFailurePoint = "Compaction::before_add_snii_destination_session"; + constexpr std::string_view kAbortPoint = "Compaction::snii_destination_session_aborted"; + DEFER({ + DebugPoints::instance()->remove(std::string(kFailurePoint)); + DebugPoints::instance()->remove(std::string(kAbortPoint)); + config::enable_common_grams_index_build = old_common_grams; + config::enable_debug_points = old_debug_points; + config::snii_positions_index_write_freq = old_write_freq; + }); + + _build_snii_multi_index_tablet(); + const std::vector rowsets = _build_snii_source_rowsets(); + std::function fail_second_destination = [](size_t destination_ordinal, + Status* status) { + if (destination_ordinal == 1) { + *status = Status::Error( + "injected SNII destination session failure"); + } + }; + DebugPoints::instance()->add_with_callback(std::string(kFailurePoint), fail_second_destination); + size_t abort_count = 0; + std::function count_abort = [&abort_count](size_t) { ++abort_count; }; + DebugPoints::instance()->add_with_callback(std::string(kAbortPoint), count_abort); + + RowsetSharedPtr output; + const Status status = IndexCompactionUtils::do_compaction(rowsets, _engine_ref, _tablet, true, + output, nullptr, 1000); + EXPECT_FALSE(status.ok()); + EXPECT_TRUE(status.is()); + EXPECT_THAT(status.to_string(), + testing::HasSubstr("injected SNII destination session failure")); + EXPECT_EQ(output, nullptr); + EXPECT_EQ(abort_count, 1); + for (const RowsetSharedPtr& rowset : rowsets) { + EXPECT_FALSE(rowset->is_skip_index_compaction(1)); + } +} + +TEST_F(IndexCompactionTest, snii_native_merge_mem_limit_arms_raw_rebuild_without_wrapping) { + const bool old_common_grams = config::enable_common_grams_index_build; + const bool old_debug_points = config::enable_debug_points; + const bool old_write_freq = config::snii_positions_index_write_freq; + config::enable_common_grams_index_build = false; + config::enable_debug_points = true; + config::snii_positions_index_write_freq = false; + constexpr std::string_view kFailurePoint = "Compaction::before_execute_snii_merge"; + DEFER({ + DebugPoints::instance()->remove(std::string(kFailurePoint)); + config::enable_common_grams_index_build = old_common_grams; + config::enable_debug_points = old_debug_points; + config::snii_positions_index_write_freq = old_write_freq; + }); + + _build_snii_multi_index_tablet(); + const std::vector rowsets = _build_snii_source_rowsets(); + std::function inject_mem_limit = [](Status* status) { + *status = Status::Error("injected SNII merge memory limit"); + }; + DebugPoints::instance()->add_with_callback(std::string(kFailurePoint), inject_mem_limit); + + RowsetSharedPtr output; + const Status status = IndexCompactionUtils::do_compaction(rowsets, _engine_ref, _tablet, true, + output, nullptr, 1000); + EXPECT_FALSE(status.ok()); + EXPECT_TRUE(status.is()); + EXPECT_THAT(status.to_string(), testing::HasSubstr("injected SNII merge memory limit")); + EXPECT_EQ(output, nullptr); + for (const RowsetSharedPtr& rowset : rowsets) { + EXPECT_TRUE(rowset->is_skip_index_compaction(1)); + } +} + +TEST_F(IndexCompactionTest, snii_native_merge_corruption_arms_raw_rebuild_fallback) { + const bool old_common_grams = config::enable_common_grams_index_build; + const bool old_debug_points = config::enable_debug_points; + const bool old_write_freq = config::snii_positions_index_write_freq; + config::enable_common_grams_index_build = false; + config::enable_debug_points = true; + config::snii_positions_index_write_freq = false; + constexpr std::string_view kFailurePoint = "Compaction::before_execute_snii_merge"; + DEFER({ + DebugPoints::instance()->remove(std::string(kFailurePoint)); + config::enable_common_grams_index_build = old_common_grams; + config::enable_debug_points = old_debug_points; + config::snii_positions_index_write_freq = old_write_freq; + }); + + _build_snii_multi_index_tablet(); + const std::vector rowsets = _build_snii_source_rowsets(); + std::function inject_corruption = [](Status* status) { + *status = Status::Error( + "injected SNII merge corruption"); + }; + DebugPoints::instance()->add_with_callback(std::string(kFailurePoint), inject_corruption); + + RowsetSharedPtr output; + const Status status = IndexCompactionUtils::do_compaction(rowsets, _engine_ref, _tablet, true, + output, nullptr, 1000); + EXPECT_FALSE(status.ok()); + EXPECT_TRUE(status.is()) << status; + EXPECT_THAT(status.to_string(), testing::HasSubstr("injected SNII merge corruption")); + EXPECT_EQ(output, nullptr); + for (const RowsetSharedPtr& rowset : rowsets) { + EXPECT_TRUE(rowset->is_skip_index_compaction(1)); + } +} } // namespace doris diff --git a/be/test/storage/index/inverted/compaction/index_compaction_write_index_test.cpp b/be/test/storage/index/inverted/compaction/index_compaction_write_index_test.cpp index a530be96da3775..48803988d876f6 100644 --- a/be/test/storage/index/inverted/compaction/index_compaction_write_index_test.cpp +++ b/be/test/storage/index/inverted/compaction/index_compaction_write_index_test.cpp @@ -28,7 +28,6 @@ #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wshadow-field" #include // IWYU pragma: keep -#include #include #include "CLucene/analysis/Analyzers.h" diff --git a/be/test/storage/index/inverted/compaction/util/index_compaction_utils.cpp b/be/test/storage/index/inverted/compaction/util/index_compaction_utils.h similarity index 92% rename from be/test/storage/index/inverted/compaction/util/index_compaction_utils.cpp rename to be/test/storage/index/inverted/compaction/util/index_compaction_utils.h index 6ca726770cfb5d..0a2c0de8a872e1 100644 --- a/be/test/storage/index/inverted/compaction/util/index_compaction_utils.cpp +++ b/be/test/storage/index/inverted/compaction/util/index_compaction_utils.h @@ -15,6 +15,8 @@ // specific language governing permissions and limitations // under the License. +#pragma once + #include #include @@ -427,7 +429,8 @@ class IndexCompactionUtils { const TabletSharedPtr& tablet, bool is_index_compaction, RowsetSharedPtr& rowset_ptr, const std::function custom_check = nullptr, - int64_t max_rows_per_segment = 100000) { + int64_t max_rows_per_segment = 100000, + const std::optional& output_storage_resource = std::nullopt) { config::inverted_index_compaction_enable = is_index_compaction; // control max rows in one block config::compaction_batch_size = max_rows_per_segment; @@ -441,6 +444,8 @@ class IndexCompactionUtils { RowsetWriterContext ctx; ctx.max_rows_per_segment = max_rows_per_segment; + // Empty for a local output rowset, which is what is_local_rowset() keys off. + ctx.storage_resource = output_storage_resource; RETURN_IF_ERROR(compaction.construct_output_rowset_writer(ctx)); compaction._stats.rowid_conversion = compaction._rowid_conversion.get(); @@ -585,7 +590,8 @@ class IndexCompactionUtils { InvertedIndexDescriptor::get_index_file_path_prefix(seg_path.value()); auto index_file_reader = std::make_shared( fs, std::string(index_file_path_prefix), - tablet_schema->get_inverted_index_storage_format(), index_info); + tablet_schema->get_inverted_index_storage_format(), index_info, + output_rowset->rowset_meta()->tablet_id()); EXPECT_TRUE(index_file_reader->init().ok()); const auto& dirs = index_file_reader->get_all_directories(); EXPECT_TRUE(dirs.has_value()); @@ -616,11 +622,11 @@ class IndexCompactionUtils { } } - static RowsetWriterContext rowset_writer_context(const std::unique_ptr& data_dir, - const TabletSchemaSPtr& schema, - const std::string& tablet_path, - int64_t& inc_id, - int64_t max_rows_per_segment = 200) { + static RowsetWriterContext rowset_writer_context( + const std::unique_ptr& data_dir, const TabletSchemaSPtr& schema, + const std::string& tablet_path, int64_t& inc_id, int64_t max_rows_per_segment = 200, + const std::optional& storage_resource = std::nullopt, + int64_t tablet_id = 0, bool write_file_cache = false) { RowsetWriterContext context; RowsetId rowset_id; rowset_id.init(inc_id); @@ -630,8 +636,16 @@ class IndexCompactionUtils { context.rowset_state = VISIBLE; context.tablet_schema = schema; context.tablet_path = tablet_path; + // Remote rowsets need this: CachedRemoteFileReader asserts tablet_id > 0 for Doris tables. + context.tablet_id = tablet_id; context.version = Version(inc_id, inc_id); context.max_rows_per_segment = max_rows_per_segment; + // Set only by callers that want the rowset to live on remote storage; a local rowset + // leaves it empty, which is what is_local_rowset() keys off. + context.storage_resource = storage_resource; + // Cloud sets this from the load request (cloud_rowset_builder.cpp), which is what leaves + // the block cache warm for compaction. Off by default, matching non-cloud writes. + context.write_file_cache = write_file_cache; inc_id++; return context; } @@ -643,7 +657,9 @@ class IndexCompactionUtils { const std::vector& data_files, int64_t& inc_id, const std::function custom_check = nullptr, const bool& is_performance = false, - int64_t max_rows_per_segment = 200) { + int64_t max_rows_per_segment = 200, + const std::optional& storage_resource = std::nullopt, + bool write_file_cache = false) { std::vector> data; for (const auto& file : data_files) { data.emplace_back(read_data(file)); @@ -652,7 +668,8 @@ class IndexCompactionUtils { const auto& res = RowsetFactory::create_rowset_writer( *engine_ref, rowset_writer_context(data_dir, schema, tablet->tablet_path(), inc_id, - max_rows_per_segment), + max_rows_per_segment, storage_resource, + tablet->tablet_id(), write_file_cache), false); EXPECT_TRUE(res.has_value()) << res.error(); const auto& rowset_writer = res.value(); @@ -715,8 +732,10 @@ class IndexCompactionUtils { (rowsets[i]->num_rows() / max_rows_per_segment)) << rowsets[i]->num_segments(); - // check rowset meta and file - for (int seg_id = 0; seg_id < rowsets[i]->num_segments(); seg_id++) { + // check rowset meta and file -- local only: the paths below are local, so a remote + // rowset would always report size 0 here. + for (int seg_id = 0; rowsets[i]->is_local() && seg_id < rowsets[i]->num_segments(); + seg_id++) { const auto& index_info = rowsets[i]->_rowset_meta->inverted_index_file_info(seg_id); EXPECT_TRUE(index_info.has_index_size()); const auto& fs = rowsets[i]->_rowset_meta->fs(); @@ -733,13 +752,26 @@ class IndexCompactionUtils { InvertedIndexDescriptor::get_index_file_path_prefix(seg_path.value()); auto index_file_reader = std::make_shared( fs, std::string(index_file_path_prefix), - schema->get_inverted_index_storage_format(), index_info); + schema->get_inverted_index_storage_format(), index_info, + rowsets[i]->rowset_meta()->tablet_id()); st = index_file_reader->init(); EXPECT_TRUE(st.ok()) << st.to_string(); - const auto& dirs = index_file_reader->get_all_directories(); - EXPECT_TRUE(dirs.has_value()); - if (custom_check) { - custom_check(dirs.value().size()); + if (schema->get_inverted_index_storage_format() == + InvertedIndexStorageFormatPB::SNII) { + const auto& indexes = schema->inverted_indexes(); + for (const TabletIndex* index : indexes) { + const auto logical_index = index_file_reader->open_snii_index(index); + EXPECT_TRUE(logical_index.has_value()) << logical_index.error(); + } + if (custom_check) { + custom_check(indexes.size()); + } + } else { + const auto& dirs = index_file_reader->get_all_directories(); + EXPECT_TRUE(dirs.has_value()); + if (custom_check) { + custom_check(dirs.value().size()); + } } } } diff --git a/be/test/storage/index/inverted/empty_index_file_test.cpp b/be/test/storage/index/inverted/empty_index_file_test.cpp index af1e7f0630f7eb..7acf6b13ef14a7 100644 --- a/be/test/storage/index/inverted/empty_index_file_test.cpp +++ b/be/test/storage/index/inverted/empty_index_file_test.cpp @@ -31,35 +31,56 @@ constexpr int64_t LOAD_ID_LO = 1; constexpr int64_t LOAD_ID_HI = 2; constexpr int64_t NUM_STREAM = 3; constexpr static std::string_view tmp_dir = "./ut_dir/tmp"; -class EmptyIndexFileTest : public testing::Test { +class EmptyIndexFileTest : public testing::TestWithParam { + struct WriteState { + size_t bytes_appended = 0; + int data_calls = 0; + int eos_calls = 0; + }; + class MockStreamStub : public LoadStreamStub { public: - MockStreamStub(PUniqueId load_id, int64_t src_id) + MockStreamStub(PUniqueId load_id, int64_t src_id, std::shared_ptr state) : LoadStreamStub(load_id, src_id, std::make_shared(), - std::make_shared()) {}; + std::make_shared()), + _state(std::move(state)) {}; - virtual ~MockStreamStub() = default; + ~MockStreamStub() override = default; // APPEND_DATA - virtual Status append_data(int64_t partition_id, int64_t index_id, int64_t tablet_id, - int32_t segment_id, uint64_t offset, std::span data, - bool segment_eos = false, - FileType file_type = FileType::SEGMENT_FILE) override { - EXPECT_TRUE(segment_eos); + Status append_data(int64_t partition_id, int64_t index_id, int64_t tablet_id, + int32_t segment_id, uint64_t offset, std::span data, + bool segment_eos = false, + FileType file_type = FileType::SEGMENT_FILE) override { + EXPECT_EQ(offset, _state->bytes_appended); + if (segment_eos) { + ++_state->eos_calls; + EXPECT_TRUE(data.empty()); + return Status::OK(); + } + ++_state->data_calls; + for (const auto& slice : data) { + _state->bytes_appended += slice.size; + } return Status::OK(); } + + private: + std::shared_ptr _state; }; public: EmptyIndexFileTest() = default; - ~EmptyIndexFileTest() = default; + ~EmptyIndexFileTest() override = default; protected: - virtual void SetUp() { + void SetUp() override { _load_id.set_hi(LOAD_ID_HI); _load_id.set_lo(LOAD_ID_LO); for (int src_id = 0; src_id < NUM_STREAM; src_id++) { - _streams.emplace_back(new MockStreamStub(_load_id, src_id)); + auto state = std::make_shared(); + _write_states.push_back(state); + _streams.emplace_back(new MockStreamStub(_load_id, src_id, std::move(state))); } EXPECT_TRUE(io::global_local_filesystem()->delete_directory(tmp_dir).ok()); EXPECT_TRUE(io::global_local_filesystem()->create_directory(tmp_dir).ok()); @@ -70,25 +91,34 @@ class EmptyIndexFileTest : public testing::Test { ExecEnv::GetInstance()->set_tmp_file_dir(std::move(tmp_file_dirs)); } - virtual void TearDown() { + void TearDown() override { EXPECT_TRUE(io::global_local_filesystem()->delete_directory(tmp_dir).ok()); } PUniqueId _load_id; std::vector> _streams; + std::vector> _write_states; }; -TEST_F(EmptyIndexFileTest, test_empty_index_file) { +TEST_P(EmptyIndexFileTest, PreservesZeroByteFileWhenNoLogicalIndexes) { io::FileWriterPtr file_writer = std::make_unique(_streams); auto fs = io::global_local_filesystem(); std::string index_path = "/tmp/empty_index_file_test"; std::string rowset_id = "1234567890"; int64_t seg_id = 1234567890; auto index_file_writer = std::make_unique( - fs, index_path, rowset_id, seg_id, InvertedIndexStorageFormatPB::V2, - std::move(file_writer), false); + fs, index_path, rowset_id, seg_id, GetParam(), std::move(file_writer), false); EXPECT_TRUE(index_file_writer->begin_close().ok()); EXPECT_TRUE(index_file_writer->finish_close().ok()); + for (const auto& state : _write_states) { + EXPECT_EQ(state->bytes_appended, 0); + EXPECT_EQ(state->data_calls, 0); + EXPECT_EQ(state->eos_calls, 1); + } } +INSTANTIATE_TEST_SUITE_P(LegacyCompoundFormats, EmptyIndexFileTest, + testing::Values(InvertedIndexStorageFormatPB::V2, + InvertedIndexStorageFormatPB::V3)); + } // namespace doris diff --git a/be/test/storage/index/inverted/inverted_index_reader_analysis_purpose_test.cpp b/be/test/storage/index/inverted/inverted_index_reader_analysis_purpose_test.cpp new file mode 100644 index 00000000000000..f7075c30601183 --- /dev/null +++ b/be/test/storage/index/inverted/inverted_index_reader_analysis_purpose_test.cpp @@ -0,0 +1,670 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "common/exception.h" +#include "io/fs/local_file_system.h" +#include "runtime/exec_env.h" +#include "runtime/runtime_state.h" +#include "storage/compaction/collection_similarity.h" +#include "storage/index/index_file_reader.h" +#include "storage/index/inverted/analyzer/analyzer.h" +#include "storage/index/inverted/analyzer/analyzer_provider.h" +#include "storage/index/inverted/analyzer/custom_analyzer.h" +#include "storage/index/inverted/common_grams/common_grams_key_codec.h" +#include "storage/index/inverted/inverted_index_cache.h" +#include "storage/index/inverted/inverted_index_reader.h" +#include "storage/index/inverted/token_filter/common_grams_filter.h" +#include "storage/index/snii/snii_index_reader.h" +#include "storage/tablet/tablet_schema.h" +#include "util/defer_op.h" +#include "util/time.h" + +namespace doris::segment_v2 { +namespace { + +using inverted_index::AnalysisPurpose; +using inverted_index::AnalyzerProvider; + +class RecordingFailingAnalyzerProvider final : public AnalyzerProvider { +public: + std::shared_ptr get_analyzer( + AnalysisPurpose purpose) const override { + purposes.push_back(purpose); + throw Exception(ErrorCode::INVERTED_INDEX_ANALYZER_ERROR, + "forced analyzer provider failure"); + } + + mutable std::vector purposes; +}; + +class IdentityFailingAnalyzerProvider final : public AnalyzerProvider { +public: + explicit IdentityFailingAnalyzerProvider(inverted_index::CommonGramsQueryIdentity identity) + : _identity(std::move(identity)) {} + + std::shared_ptr get_analyzer( + AnalysisPurpose purpose) const override { + purposes.push_back(purpose); + throw Exception(ErrorCode::INVERTED_INDEX_ANALYZER_ERROR, + "forced analyzer provider failure"); + } + + bool uses_common_grams() const override { return true; } + + const inverted_index::CommonGramsQueryIdentity* common_grams_identity() const override { + return &_identity; + } + + mutable std::vector purposes; + +private: + inverted_index::CommonGramsQueryIdentity _identity; +}; + +class PartialFailureTokenStream final : public lucene::analysis::TokenStream { +public: + explicit PartialFailureTokenStream(std::shared_ptr> emitted_tokens) + : _emitted_tokens(std::move(emitted_tokens)) {} + + lucene::analysis::Token* next(lucene::analysis::Token* token) override { + if (!_emitted) { + _emitted = true; + _term = "partial"; + token->clear(); + token->setTextNoCopy(_term.data(), static_cast(_term.size())); + token->positionIncrement = 1; + _emitted_tokens->fetch_add(1, std::memory_order_relaxed); + return token; + } + throw Exception(ErrorCode::INVERTED_INDEX_ANALYZER_ERROR, + "forced failure after first token"); + } + + void close() override {} + void reset() override { _emitted = false; } + +private: + std::shared_ptr> _emitted_tokens; + bool _emitted = false; + std::string _term; +}; + +class PartialFailureAnalyzer final : public lucene::analysis::Analyzer { +public: + explicit PartialFailureAnalyzer(std::shared_ptr> emitted_tokens) + : _emitted_tokens(std::move(emitted_tokens)) {} + + bool isSDocOpt() override { return true; } + + lucene::analysis::TokenStream* tokenStream(const TCHAR*, lucene::util::Reader*) override { + return new PartialFailureTokenStream(_emitted_tokens); + } + + lucene::analysis::TokenStream* reusableTokenStream(const TCHAR*, + lucene::util::Reader*) override { + _reusable = std::make_unique(_emitted_tokens); + return _reusable.get(); + } + + lucene::analysis::TokenStream* tokenStream(const TCHAR*, + const inverted_index::ReaderPtr&) override { + return new PartialFailureTokenStream(_emitted_tokens); + } + + lucene::analysis::TokenStream* reusableTokenStream(const TCHAR*, + const inverted_index::ReaderPtr&) override { + _reusable = std::make_unique(_emitted_tokens); + return _reusable.get(); + } + +private: + std::shared_ptr> _emitted_tokens; + std::unique_ptr _reusable; +}; + +class RecordingPartialFailureAnalyzerProvider final : public AnalyzerProvider { +public: + std::shared_ptr get_analyzer( + AnalysisPurpose purpose) const override { + purposes.push_back(purpose); + return std::make_shared(emitted_tokens); + } + + mutable std::vector purposes; + std::shared_ptr> emitted_tokens = + std::make_shared>(0); +}; + +class GeneratedTokenStream final : public lucene::analysis::TokenStream { +public: + GeneratedTokenStream(std::string term, bool mark_common_gram) + : _term(std::move(term)), _mark_common_gram(mark_common_gram) {} + + lucene::analysis::Token* next(lucene::analysis::Token* token) override { + if (_emitted) { + return nullptr; + } + _emitted = true; + token->clear(); + token->setTextNoCopy(_term.data(), static_cast(_term.size())); + token->setPositionIncrement(1); + if (_mark_common_gram) { + token->setType(inverted_index::COMMON_GRAM_TOKEN_TYPE); + } + return token; + } + + void close() override {} + void reset() override { _emitted = false; } + +private: + std::string _term; + bool _mark_common_gram = false; + bool _emitted = false; +}; + +class GeneratedTokenAnalyzer final : public lucene::analysis::Analyzer { +public: + GeneratedTokenAnalyzer(std::string term, bool mark_common_gram) + : _term(std::move(term)), _mark_common_gram(mark_common_gram) {} + + bool isSDocOpt() override { return true; } + + lucene::analysis::TokenStream* tokenStream(const TCHAR*, lucene::util::Reader*) override { + return new GeneratedTokenStream(_term, _mark_common_gram); + } + + lucene::analysis::TokenStream* reusableTokenStream(const TCHAR*, + lucene::util::Reader*) override { + _reusable = std::make_unique(_term, _mark_common_gram); + return _reusable.get(); + } + + lucene::analysis::TokenStream* tokenStream(const TCHAR*, + const inverted_index::ReaderPtr&) override { + return new GeneratedTokenStream(_term, _mark_common_gram); + } + + lucene::analysis::TokenStream* reusableTokenStream(const TCHAR*, + const inverted_index::ReaderPtr&) override { + _reusable = std::make_unique(_term, _mark_common_gram); + return _reusable.get(); + } + +private: + std::string _term; + bool _mark_common_gram = false; + std::unique_ptr _reusable; +}; + +class GeneratedGramAnalyzerProvider final : public AnalyzerProvider { +public: + GeneratedGramAnalyzerProvider() + : _gram(*inverted_index::encode_common_gram("the", "history")), + _identity {.common_grams_dictionary_identity = "builtin-stopwords:v1", + .base_analyzer_fingerprint = "base:v1", + .common_grams_fingerprint = "grams:v1"} {} + + std::shared_ptr get_analyzer( + AnalysisPurpose purpose) const override { + purposes.push_back(purpose); + return std::make_shared(_gram, true); + } + + bool uses_common_grams() const override { return true; } + const inverted_index::CommonGramsQueryIdentity* common_grams_identity() const override { + return &_identity; + } + + mutable std::vector purposes; + +private: + std::string _gram; + inverted_index::CommonGramsQueryIdentity _identity; +}; + +struct QueryExecutionContext { + explicit QueryExecutionContext(bool scoring) { + TQueryOptions query_options; + query_options.enable_inverted_index_query_cache = true; + query_options.enable_inverted_index_searcher_cache = true; + runtime_state.set_query_options(query_options); + context->io_ctx = &io_ctx; + context->stats = &stats; + context->runtime_state = &runtime_state; + if (scoring) { + context->collection_similarity = std::make_shared(); + } + } + + OlapReaderStatistics stats; + io::IOContext io_ctx; + RuntimeState runtime_state; + IndexQueryContextPtr context = std::make_shared(); +}; + +class InvertedIndexReaderAnalysisPurposeTest : public testing::Test { +protected: + void SetUp() override { + _previous_searcher_cache = ExecEnv::GetInstance()->get_inverted_index_searcher_cache(); + _previous_query_cache = ExecEnv::GetInstance()->get_inverted_index_query_cache(); + _searcher_cache.reset(InvertedIndexSearcherCache::create_global_instance(1024 * 1024, 1)); + _query_cache.reset(InvertedIndexQueryCache::create_global_cache(1024 * 1024, 1)); + ExecEnv::GetInstance()->set_inverted_index_searcher_cache(_searcher_cache.get()); + ExecEnv::GetInstance()->set_inverted_index_query_cache(_query_cache.get()); + + TabletIndexPB pb; + pb.set_index_type(IndexType::INVERTED); + pb.set_index_id(73); + pb.set_index_name("analysis_purpose_idx"); + pb.add_col_unique_id(0); + pb.mutable_properties()->insert({"parser", "english"}); + pb.mutable_properties()->insert({"lower_case", "true"}); + pb.mutable_properties()->insert({"support_phrase", "true"}); + _meta.init_from_pb(pb); + + _snii_file_reader = std::make_shared( + io::global_local_filesystem(), "./ut_dir/missing_snii_analysis_purpose", + InvertedIndexStorageFormatPB::SNII); + _snii_reader = SniiIndexReader::create_shared(&_meta, _snii_file_reader, + InvertedIndexReaderType::FULLTEXT); + } + + void TearDown() override { + _snii_reader.reset(); + _snii_file_reader.reset(); + ExecEnv::GetInstance()->set_inverted_index_searcher_cache(_previous_searcher_cache); + ExecEnv::GetInstance()->set_inverted_index_query_cache(_previous_query_cache); + _searcher_cache.reset(); + _query_cache.reset(); + } + + // This entry is admission-only: tests must exit during analysis before dereferencing the + // SNII postings reader. + void preload_legacy_searcher_cache_entries() { + const InvertedIndexSearcherCache::CacheKey snii_key( + _snii_file_reader->get_index_file_cache_key(&_meta)); + _searcher_cache->insert( + snii_key, new InvertedIndexSearcherCache::CacheValue( + std::make_unique(), 1, + UnixMillis(), _snii_file_reader)); + } + + template + void expect_analysis_failure_after_segment_admission(const std::shared_ptr& reader, + InvertedIndexQueryType query_type, + std::string query, bool scoring, + AnalysisPurpose expected_purpose, + const std::shared_ptr& provider, + int64_t expected_query_cache_lookups = 0, + int64_t expected_searcher_cache_hits = 1) { + QueryExecutionContext execution(scoring); + InvertedIndexAnalyzerCtx analyzer_ctx; + analyzer_ctx.parser_type = InvertedIndexParserType::PARSER_ENGLISH; + analyzer_ctx.analyzer_provider = provider; + + auto original_bitmap = std::make_shared(); + original_bitmap->add(999); + std::shared_ptr bitmap = original_bitmap; + const Field query_value = Field::create_field(std::move(query)); + + Status status; + EXPECT_NO_THROW(status = reader->query(execution.context, "content", query_value, + query_type, bitmap, &analyzer_ctx)); + EXPECT_EQ(status.code(), ErrorCode::INVERTED_INDEX_ANALYZER_ERROR) << status; + EXPECT_EQ(provider->purposes, (std::vector {expected_purpose})); + EXPECT_EQ(bitmap, original_bitmap); + EXPECT_EQ(bitmap->cardinality(), 1); + EXPECT_TRUE(bitmap->contains(999)); + EXPECT_EQ(execution.stats.inverted_index_query_cache_hit, 0); + EXPECT_EQ(execution.stats.inverted_index_query_cache_miss, expected_query_cache_lookups); + EXPECT_EQ(execution.stats.inverted_index_query_cache_lookup, expected_query_cache_lookups); + EXPECT_EQ(execution.stats.inverted_index_query_cache_insert, 0); + EXPECT_EQ(execution.stats.inverted_index_searcher_cache_hit, expected_searcher_cache_hits); + EXPECT_EQ(execution.stats.inverted_index_searcher_cache_miss, 0); + } + + template + void expect_provider_failure_after_segment_admission(const std::shared_ptr& reader, + InvertedIndexQueryType query_type, + std::string query, bool scoring, + AnalysisPurpose expected_purpose, + int64_t expected_query_cache_lookups = 0, + int64_t expected_searcher_cache_hits = 1) { + expect_analysis_failure_after_segment_admission( + reader, query_type, std::move(query), scoring, expected_purpose, + std::make_shared(), expected_query_cache_lookups, + expected_searcher_cache_hits); + } + + template + void expect_generated_gram_bypass_after_segment_admission( + const std::shared_ptr& reader) { + QueryExecutionContext execution(/*scoring=*/false); + auto provider = std::make_shared(); + InvertedIndexAnalyzerCtx analyzer_ctx; + analyzer_ctx.parser_type = InvertedIndexParserType::PARSER_ENGLISH; + analyzer_ctx.analyzer_provider = provider; + auto original_bitmap = std::make_shared(); + original_bitmap->add(999); + std::shared_ptr bitmap = original_bitmap; + const Field query_value = Field::create_field("the history"); + + const Status status = + reader->query(execution.context, "content", query_value, + InvertedIndexQueryType::MATCH_PHRASE_QUERY, bitmap, &analyzer_ctx); + EXPECT_EQ(status.code(), ErrorCode::INVERTED_INDEX_BYPASS) << status; + EXPECT_EQ(provider->purposes, + (std::vector {AnalysisPurpose::kPlainQuery})); + EXPECT_EQ(bitmap, original_bitmap); + EXPECT_EQ(execution.stats.inverted_index_query_cache_hit, 0); + EXPECT_EQ(execution.stats.inverted_index_query_cache_miss, 1); + EXPECT_EQ(execution.stats.inverted_index_query_cache_lookup, 1); + EXPECT_EQ(execution.stats.inverted_index_query_cache_insert, 0); + EXPECT_EQ(execution.stats.inverted_index_searcher_cache_hit, 1); + EXPECT_EQ(execution.stats.inverted_index_searcher_cache_miss, 0); + } + + template + void expect_raw_query_bypasses_analyzer(const std::shared_ptr& reader, + InvertedIndexQueryType query_type, std::string query) { + QueryExecutionContext execution(/*scoring=*/false); + auto provider = std::make_shared(); + InvertedIndexAnalyzerCtx analyzer_ctx; + analyzer_ctx.parser_type = InvertedIndexParserType::PARSER_ENGLISH; + analyzer_ctx.analyzer_provider = provider; + std::shared_ptr bitmap; + const Field query_value = Field::create_field(std::move(query)); + + const Status status = reader->query(execution.context, "content", query_value, query_type, + bitmap, &analyzer_ctx); + EXPECT_NE(status.code(), ErrorCode::INVERTED_INDEX_ANALYZER_ERROR) << status; + EXPECT_TRUE(provider->purposes.empty()); + } + + template + void expect_raw_cache_hit_before_analysis(const std::shared_ptr& reader, + const std::shared_ptr& file_reader, + const std::shared_ptr& provider, + bool common_grams_query_plan_enabled) { + QueryExecutionContext execution(/*scoring=*/false); + InvertedIndexAnalyzerCtx analyzer_ctx; + analyzer_ctx.parser_type = InvertedIndexParserType::PARSER_ENGLISH; + analyzer_ctx.analyzer_provider = provider; + + const std::string raw_query = "the history"; + const InvertedIndexRawQuerySemantic semantic { + .raw_query_bytes = raw_query, + .query_type = InvertedIndexQueryType::MATCH_PHRASE_QUERY, + .slop = 0, + .ordered = false, + .max_expansions = + execution.runtime_state.query_options().inverted_index_max_expansions, + .common_grams_query_plan_enabled = common_grams_query_plan_enabled}; + const InvertedIndexQueryCache::CacheKey key { + file_reader->get_index_file_cache_key(&_meta), "content", + InvertedIndexQueryType::MATCH_PHRASE_QUERY, semantic.encode()}; + auto cached = std::make_shared(); + cached->add(7); + InvertedIndexQueryCacheHandle insert_handle; + _query_cache->insert(key, cached, &insert_handle); + + std::shared_ptr bitmap; + const Field query_value = Field::create_field(raw_query); + const Status status = + reader->query(execution.context, "content", query_value, + InvertedIndexQueryType::MATCH_PHRASE_QUERY, bitmap, &analyzer_ctx); + EXPECT_TRUE(status.ok()) << status; + EXPECT_TRUE(provider->purposes.empty()); + ASSERT_NE(bitmap, nullptr); + EXPECT_EQ(bitmap->cardinality(), 1); + EXPECT_TRUE(bitmap->contains(7)); + EXPECT_EQ(execution.stats.inverted_index_query_cache_hit, 1); + EXPECT_EQ(execution.stats.inverted_index_query_cache_miss, 0); + EXPECT_EQ(execution.stats.inverted_index_query_cache_lookup, 1); + EXPECT_EQ(execution.stats.inverted_index_query_cache_insert, 0); + EXPECT_EQ(execution.stats.inverted_index_searcher_cache_hit, 0); + EXPECT_EQ(execution.stats.inverted_index_searcher_cache_miss, 0); + } + + InvertedIndexSearcherCache* _previous_searcher_cache = nullptr; + InvertedIndexQueryCache* _previous_query_cache = nullptr; + std::unique_ptr _searcher_cache; + std::unique_ptr _query_cache; + TabletIndex _meta; + std::shared_ptr _snii_file_reader; + std::shared_ptr _snii_reader; +}; + +TEST(InvertedIndexRawQuerySemanticTest, EncodesOnlyRawSemanticDimensionsWithoutDelimiters) { + const std::string raw_query("a/b\0c", 5); + InvertedIndexRawQuerySemantic base {.raw_query_bytes = raw_query, + .query_type = InvertedIndexQueryType::MATCH_PHRASE_QUERY, + .slop = 2, + .ordered = true, + .max_expansions = 50, + .cache_semantics_version = 3, + .common_grams_query_plan_enabled = true}; + const std::string encoded = base.encode(); + constexpr size_t kFixedEncodedBytes = sizeof(uint32_t) + sizeof(uint64_t) + sizeof(uint32_t) + + sizeof(uint32_t) + sizeof(uint8_t) + sizeof(uint32_t) + + sizeof(uint8_t); + EXPECT_EQ(encoded.size(), kFixedEncodedBytes + raw_query.size()); + + auto changed = base; + changed.raw_query_bytes = std::string_view(raw_query).substr(0, 3); + EXPECT_NE(changed.encode(), encoded); + changed = base; + changed.query_type = InvertedIndexQueryType::MATCH_PHRASE_PREFIX_QUERY; + EXPECT_NE(changed.encode(), encoded); + changed = base; + changed.slop = 3; + EXPECT_NE(changed.encode(), encoded); + changed = base; + changed.ordered = false; + EXPECT_NE(changed.encode(), encoded); + changed = base; + changed.max_expansions = 51; + EXPECT_NE(changed.encode(), encoded); + changed = base; + changed.cache_semantics_version = 4; + EXPECT_NE(changed.encode(), encoded); + changed = base; + changed.common_grams_query_plan_enabled = false; + EXPECT_NE(changed.encode(), encoded); +} + +TEST(InvertedIndexRawQuerySemanticTest, CacheEnvelopeSeparatesSlashAndNulBoundaries) { + const InvertedIndexQueryCache::CacheKey slash_left { + io::Path("a/b"), "c", InvertedIndexQueryType::MATCH_PHRASE_QUERY, "d"}; + const InvertedIndexQueryCache::CacheKey slash_right { + io::Path("a"), "b/c", InvertedIndexQueryType::MATCH_PHRASE_QUERY, "d"}; + EXPECT_NE(slash_left.encode(), slash_right.encode()); + + const InvertedIndexQueryCache::CacheKey nul_left { + io::Path("a"), std::string("b\0c", 3), InvertedIndexQueryType::MATCH_PHRASE_QUERY, "d"}; + const InvertedIndexQueryCache::CacheKey nul_right { + io::Path(std::string("a\0b", 3)), "c", InvertedIndexQueryType::MATCH_PHRASE_QUERY, "d"}; + EXPECT_NE(nul_left.encode(), nul_right.encode()); +} + +TEST(InvertedIndexRawQuerySemanticTest, CommonGramsQueryPlanIsDisabledByDefault) { + EXPECT_FALSE(config::enable_common_grams_query_plan); +} + +TEST(InvertedIndexRawQuerySemanticTest, CostModelConfigUpdatesValues) { + const int32_t original_ratio = config::common_grams_plan_cost_ratio_percent; + const int32_t original_factor = config::common_grams_position_verify_factor; + const int32_t changed_ratio = original_ratio == 84 ? 85 : 84; + const int32_t changed_factor = original_factor == 7 ? 8 : 7; + + ASSERT_TRUE(config::set_config("common_grams_plan_cost_ratio_percent", + std::to_string(changed_ratio)) + .ok()); + ASSERT_TRUE(config::set_config("common_grams_position_verify_factor", + std::to_string(changed_factor)) + .ok()); + EXPECT_EQ(config::common_grams_plan_cost_ratio_percent, changed_ratio); + EXPECT_EQ(config::common_grams_position_verify_factor, changed_factor); + + ASSERT_TRUE(config::set_config("common_grams_plan_cost_ratio_percent", + std::to_string(original_ratio)) + .ok()); + ASSERT_TRUE(config::set_config("common_grams_position_verify_factor", + std::to_string(original_factor)) + .ok()); +} + +TEST(InvertedIndexRawQuerySemanticTest, InvalidCostModelConfigDoesNotMutateValues) { + const int32_t before_ratio = config::common_grams_plan_cost_ratio_percent; + const int32_t before_factor = config::common_grams_position_verify_factor; + + for (const auto& [field, value] : std::vector> { + {"common_grams_plan_cost_ratio_percent", "-1"}, + {"common_grams_plan_cost_ratio_percent", "101"}, + {"common_grams_position_verify_factor", "-1"}}) { + EXPECT_FALSE(config::set_config(field, value).ok()); + EXPECT_EQ(config::common_grams_plan_cost_ratio_percent, before_ratio); + EXPECT_EQ(config::common_grams_position_verify_factor, before_factor); + } +} + +TEST(InvertedIndexAnalyzerCtxTest, UsesProviderCommonGramsIdentityWithoutCopyingIt) { + auto provider = std::make_shared(); + InvertedIndexAnalyzerCtx analyzer_ctx; + analyzer_ctx.analyzer_provider = provider; + EXPECT_EQ(analyzer_ctx.get_common_grams_identity(), provider->common_grams_identity()); +} + +TEST_F(InvertedIndexReaderAnalysisPurposeTest, + SniiRawCacheLookupIsIndependentOfRequestAnalyzerIdentity) { + const bool original = config::enable_common_grams_query_plan; + Defer restore([original] { + EXPECT_TRUE(config::set_config("enable_common_grams_query_plan", + original ? "true" : "false", /*need_persist=*/false) + .ok()); + }); + ASSERT_TRUE(config::set_config("enable_common_grams_query_plan", "true", + /*need_persist=*/false) + .ok()); + const inverted_index::CommonGramsQueryIdentity complete_identity { + .common_grams_dictionary_identity = "dictionary:complete", + .base_analyzer_fingerprint = "base:complete", + .common_grams_fingerprint = "grams:complete"}; + const inverted_index::CommonGramsQueryIdentity empty_identity; + for (const auto& identity : {complete_identity, empty_identity}) { + expect_raw_cache_hit_before_analysis( + _snii_reader, _snii_file_reader, + std::make_shared(identity), + config::enable_common_grams_query_plan); + } + expect_raw_cache_hit_before_analysis(_snii_reader, _snii_file_reader, + std::make_shared(), + config::enable_common_grams_query_plan); +} + +TEST_F(InvertedIndexReaderAnalysisPurposeTest, DisabledResultCacheDoesNotLookupCountOrInsert) { + QueryExecutionContext execution(/*scoring=*/false); + TQueryOptions disabled_options; + disabled_options.enable_inverted_index_query_cache = false; + disabled_options.enable_inverted_index_searcher_cache = true; + execution.runtime_state.set_query_options(disabled_options); + + const InvertedIndexQueryCache::CacheKey key {io::Path("disabled-cache"), "content", + InvertedIndexQueryType::MATCH_PHRASE_QUERY, + "raw-semantic"}; + auto bitmap = std::make_shared(); + bitmap->add(3); + InvertedIndexQueryCacheHandle handle; + _snii_reader->insert_query_cache(execution.context, _query_cache.get(), key, bitmap, &handle); + std::shared_ptr lookup_bitmap; + EXPECT_FALSE(_snii_reader->handle_query_cache(execution.context, _query_cache.get(), key, + &handle, lookup_bitmap)); + EXPECT_EQ(execution.stats.inverted_index_query_cache_hit, 0); + EXPECT_EQ(execution.stats.inverted_index_query_cache_miss, 0); + EXPECT_EQ(execution.stats.inverted_index_query_cache_lookup, 0); + EXPECT_EQ(execution.stats.inverted_index_query_cache_insert, 0); + + TQueryOptions enabled_options; + enabled_options.enable_inverted_index_query_cache = true; + enabled_options.enable_inverted_index_searcher_cache = true; + execution.runtime_state.set_query_options(enabled_options); + EXPECT_FALSE(_snii_reader->handle_query_cache(execution.context, _query_cache.get(), key, + &handle, lookup_bitmap)); + EXPECT_EQ(execution.stats.inverted_index_query_cache_miss, 1); + EXPECT_EQ(execution.stats.inverted_index_query_cache_lookup, 1); +} + +TEST_F(InvertedIndexReaderAnalysisPurposeTest, SniiSelectsPurposeAfterSegmentAdmission) { + preload_legacy_searcher_cache_entries(); + expect_provider_failure_after_segment_admission( + _snii_reader, InvertedIndexQueryType::MATCH_PHRASE_QUERY, "the history", false, + AnalysisPurpose::kPlainQuery, /*expected_query_cache_lookups=*/1); + expect_provider_failure_after_segment_admission( + _snii_reader, InvertedIndexQueryType::MATCH_PHRASE_QUERY, "the history ~2", false, + AnalysisPurpose::kPlainQuery, /*expected_query_cache_lookups=*/1); + expect_provider_failure_after_segment_admission( + _snii_reader, InvertedIndexQueryType::MATCH_PHRASE_PREFIX_QUERY, "the hist", false, + AnalysisPurpose::kPlainQuery, /*expected_query_cache_lookups=*/1); + expect_provider_failure_after_segment_admission( + _snii_reader, InvertedIndexQueryType::MATCH_PHRASE_QUERY, "the history", true, + AnalysisPurpose::kPlainQuery); +} + +TEST_F(InvertedIndexReaderAnalysisPurposeTest, PartialAnalysisFailureDoesNotPublishState) { + preload_legacy_searcher_cache_entries(); + auto snii_provider = std::make_shared(); + expect_analysis_failure_after_segment_admission( + _snii_reader, InvertedIndexQueryType::MATCH_PHRASE_QUERY, "the history", false, + AnalysisPurpose::kPlainQuery, snii_provider, /*expected_query_cache_lookups=*/1); + EXPECT_EQ(snii_provider->emitted_tokens->load(std::memory_order_relaxed), 1); +} + +TEST_F(InvertedIndexReaderAnalysisPurposeTest, + SniiGeneratedGramBypassesAfterSegmentAdmissionBeforeSearch) { + preload_legacy_searcher_cache_entries(); + const bool original = config::enable_common_grams_query_plan; + Defer restore([original] { + EXPECT_TRUE(config::set_config("enable_common_grams_query_plan", + original ? "true" : "false", /*need_persist=*/false) + .ok()); + }); + ASSERT_TRUE(config::set_config("enable_common_grams_query_plan", "true", + /*need_persist=*/false) + .ok()); + expect_generated_gram_bypass_after_segment_admission(_snii_reader); +} + +TEST_F(InvertedIndexReaderAnalysisPurposeTest, RegexpAndWildcardBypassAnalyzer) { + for (const auto query_type : + {InvertedIndexQueryType::MATCH_REGEXP_QUERY, InvertedIndexQueryType::WILDCARD_QUERY}) { + expect_raw_query_bypasses_analyzer(_snii_reader, query_type, "hist.*"); + } +} + +} // namespace +} // namespace doris::segment_v2 diff --git a/be/test/storage/index/inverted/query/phrase_edge_query_test.cpp b/be/test/storage/index/inverted/query/phrase_edge_query_test.cpp index 2da72758470a27..975ae80051678f 100644 --- a/be/test/storage/index/inverted/query/phrase_edge_query_test.cpp +++ b/be/test/storage/index/inverted/query/phrase_edge_query_test.cpp @@ -62,12 +62,19 @@ class PhraseEdgeQueryTest : public testing::Test { _inverted_index_query_cache = std::unique_ptr( InvertedIndexQueryCache::create_global_cache(inverted_index_cache_limit, 1)); + // Both caches are owned by this fixture, so the previous globals must come back in + // TearDown -- otherwise ExecEnv keeps pointing at them after the fixture is destroyed and + // the next test that reaches InvertedIndexQueryCache::instance() reads freed memory. + _previous_searcher_cache = ExecEnv::GetInstance()->get_inverted_index_searcher_cache(); + _previous_query_cache = ExecEnv::GetInstance()->get_inverted_index_query_cache(); ExecEnv::GetInstance()->set_inverted_index_searcher_cache( _inverted_index_searcher_cache.get()); - ExecEnv::GetInstance()->_inverted_index_query_cache = _inverted_index_query_cache.get(); + ExecEnv::GetInstance()->set_inverted_index_query_cache(_inverted_index_query_cache.get()); } void TearDown() override { + ExecEnv::GetInstance()->set_inverted_index_searcher_cache(_previous_searcher_cache); + ExecEnv::GetInstance()->set_inverted_index_query_cache(_previous_query_cache); EXPECT_TRUE(io::global_local_filesystem()->delete_directory(kTestDir).ok()); } @@ -180,6 +187,8 @@ class PhraseEdgeQueryTest : public testing::Test { ~PhraseEdgeQueryTest() override = default; private: + InvertedIndexSearcherCache* _previous_searcher_cache = nullptr; + InvertedIndexQueryCache* _previous_query_cache = nullptr; std::unique_ptr _inverted_index_searcher_cache; std::unique_ptr _inverted_index_query_cache; }; diff --git a/be/test/storage/index/inverted/similarity/bm25_similarity_test.cpp b/be/test/storage/index/inverted/similarity/bm25_similarity_test.cpp index 24fdfbd2e62709..9ceba92421e87f 100644 --- a/be/test/storage/index/inverted/similarity/bm25_similarity_test.cpp +++ b/be/test/storage/index/inverted/similarity/bm25_similarity_test.cpp @@ -22,8 +22,8 @@ #include #include "common/be_mock_util.h" -#include "storage/compaction/collection_statistics.h" #include "storage/index/index_query_context.h" +#include "storage/index/inverted/similarity/collection_statistics.h" using namespace doris; using namespace doris::segment_v2; diff --git a/be/test/storage/index/inverted/similarity/collection_statistics_test.cpp b/be/test/storage/index/inverted/similarity/collection_statistics_test.cpp new file mode 100644 index 00000000000000..3aedbf2594e04d --- /dev/null +++ b/be/test/storage/index/inverted/similarity/collection_statistics_test.cpp @@ -0,0 +1,2949 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/inverted/similarity/collection_statistics.h" + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "common/exception.h" +#include "core/data_type/data_type_string.h" +#include "exec/common/variant_util.h" +#include "exprs/vexpr.h" +#include "exprs/vexpr_context.h" +#include "exprs/vliteral.h" +#include "exprs/vsearch.h" +#include "exprs/vslot_ref.h" +#include "io/fs/local_file_system.h" +#include "runtime/exec_env.h" +#include "storage/index/index_file_reader.h" +#include "storage/index/index_file_writer.h" +#include "storage/index/index_writer.h" +#include "storage/index/inverted/analyzer/analyzer.h" +#include "storage/index/inverted/common_grams/common_grams_key_codec.h" +#include "storage/index/inverted/inverted_index_desc.h" +#include "storage/index/inverted/util/string_helper.h" +#include "storage/index/snii/query/bm25_scorer.h" +#include "storage/index/snii/snii_doris_adapter.h" +#include "storage/index/snii/stats/snii_stats_provider.h" +#include "storage/index/snii/writer/snii_compound_writer.h" +#include "storage/index/snii_query_test_util.h" +#include "storage/rowset/rowset.h" +#include "storage/rowset/rowset_meta.h" +#include "storage/rowset/rowset_reader.h" +#include "storage/tablet/tablet_schema.h" +#include "testutil/mock/mock_runtime_state.h" +#include "util/slice.h" + +namespace doris { + +using collection_statistics_detail::add_term_doc_frequency; +using collection_statistics_detail::resolve_snii_scoring_segment; +using collection_statistics_detail::SniiScoringSegmentStats; + +namespace collection_statistics { + +class MockVExpr : public VExpr { +public: + MockVExpr(TExprNodeType::type node_type) : _mock_node_type(node_type) { + if (node_type == TExprNodeType::MATCH_PRED) { + _opcode = TExprOpcode::MATCH_PHRASE; + InvertedIndexAnalyzerConfig config; + config.parser_type = InvertedIndexParserType::PARSER_STANDARD; + config.stop_words = "none"; + _analyzer_ctx = std::make_shared(); + _analyzer_ctx->analyzer_provider = + segment_v2::inverted_index::InvertedIndexAnalyzer::create_analyzer_provider( + &config); + } + } + + TExprNodeType::type node_type() const override { return _mock_node_type; } + + Status execute(VExprContext* context, Block* block, int32_t* result_column_id) const override { + return Status::OK(); + } + + Status execute_column_impl(VExprContext* context, const Block* block, const Selector* selector, + size_t count, ColumnPtr& result_column) const override { + return Status::OK(); + } + + Status prepare(RuntimeState* state, const RowDescriptor& desc, VExprContext* context) override { + return Status::OK(); + } + + Status open(RuntimeState* state, VExprContext* context, + FunctionContext::FunctionStateScope scope) override { + return Status::OK(); + } + + void close(VExprContext* context, FunctionContext::FunctionStateScope scope) override {} + + const std::string& expr_name() const override { + static std::string name = "mock_expr"; + return name; + } + + std::string debug_string() const override { return "MockVExpr"; } + + const InvertedIndexAnalyzerCtx* query_analyzer_ctx() const override { + return _analyzer_ctx.get(); + } + + void set_analyzer_ctx(InvertedIndexAnalyzerCtxSPtr analyzer_ctx) { + _analyzer_ctx = std::move(analyzer_ctx); + } + + void set_opcode(TExprOpcode::type opcode) { _opcode = opcode; } + +private: + TExprNodeType::type _mock_node_type; + InvertedIndexAnalyzerCtxSPtr _analyzer_ctx; +}; + +class FixedFingerprintAnalyzerProvider final : public segment_v2::inverted_index::AnalyzerProvider { +public: + FixedFingerprintAnalyzerProvider(std::shared_ptr analyzer, + std::string fingerprint) + : _analyzer(std::move(analyzer)), _fingerprint(std::move(fingerprint)) {} + + std::shared_ptr get_analyzer( + segment_v2::inverted_index::AnalysisPurpose) const override { + return _analyzer; + } + + std::string_view base_analyzer_fingerprint() const override { return _fingerprint; } + +private: + std::shared_ptr _analyzer; + std::string _fingerprint; +}; + +class MockVSlotRef : public VSlotRef { +public: + MockVSlotRef(const std::string& column_name, SlotId slot_id) + : _column_name(column_name), _slot_id(slot_id) { + _node_type = TExprNodeType::SLOT_REF; + } + + const std::string& column_name() const override { return _column_name; } + const std::string& expr_name() const override { return _column_name; } + std::string debug_string() const override { return "MockVSlotRef: " + _column_name; } + SlotId slot_id() const override { return _slot_id; } + +private: + std::string _column_name; + SlotId _slot_id; +}; + +class MockVLiteral : public VLiteral { +public: + MockVLiteral(const std::string& value) : _value(value) {} + + std::string value() const override { return _value; } + std::string value(const DataTypeSerDe::FormatOptions& options) const override { return _value; } + const std::string& expr_name() const override { return _value; } + std::string debug_string() const override { return "MockVLiteral: " + _value; } + +private: + std::string _value; +}; + +class MockRowsetMeta : public RowsetMeta { +public: + MockRowsetMeta() : RowsetMeta() { _fs = io::global_local_filesystem(); } + + io::FileSystemSPtr fs() override { return _fs; } + +private: + io::FileSystemSPtr _fs; +}; + +class MockRowset : public Rowset { +public: + MockRowset(TabletSchemaSPtr schema, RowsetMetaSharedPtr rowset_meta) + : Rowset(schema, rowset_meta, "/mock/tablet/path") { + _num_segments = 0; + } + + Status create_reader(std::shared_ptr* result) override { + return Status::NotSupported("MockRowset::create_reader not implemented"); + } + + Status remove() override { return Status::OK(); } + + Status link_files_to(const std::string& dir, RowsetId new_rowset_id, size_t start_seg_id, + std::set* without_index_uids) override { + return Status::OK(); + } + + Status copy_files_to(const std::string& dir, const RowsetId& new_rowset_id) override { + return Status::OK(); + } + + Status remove_old_files(std::vector* files_to_remove) override { + return Status::OK(); + } + + Status check_file_exist() override { return Status::OK(); } + + Status upload_to(const StorageResource& dest_fs, const RowsetId& new_rowset_id) override { + return Status::OK(); + } + + Status get_inverted_index_size(int64_t* index_size) override { + *index_size = 0; + return Status::OK(); + } + + void clear_inverted_index_cache() override {} + + Status init() override { return Status::OK(); } + + void do_close() override {} + + Status check_current_rowset_segment() override { return Status::OK(); } + + int64_t num_segments() const override { return _num_segments; } + + Result segment_path(int64_t seg_id) override { + _segment_path_requests.push_back(seg_id); + if (_segment_paths.find(seg_id) != _segment_paths.end()) { + return _segment_paths.at(seg_id); + } + return ResultError(Status::InternalError("Segment path not found")); + } + + void set_segment_path(int64_t seg_id, const std::string& path) { + _segment_paths[seg_id] = path; + } + + void set_num_segments(int64_t num) { _num_segments = num; } + + const std::vector& segment_path_requests() const { return _segment_path_requests; } + +private: + int64_t _num_segments; + std::map _segment_paths; + std::vector _segment_path_requests; +}; + +class MockRowsetReader : public RowsetReader { +public: + MockRowsetReader(std::shared_ptr rowset) : _rowset(rowset) {} + + Status init(RowsetReaderContext* read_context, const RowSetSplits& rs_splits) override { + return Status::OK(); + } + + Status get_segment_iterators(RowsetReaderContext* read_context, + std::vector* out_iters, + bool use_cache = false) override { + return Status::OK(); + } + + void reset_read_options() override {} + + Status next_batch(Block* block) override { + return Status::NotSupported("MockRowsetReader::next_batch not implemented"); + } + + Status next_batch(BlockView* block_view) override { + return Status::NotSupported("MockRowsetReader::next_batch not implemented"); + } + + Status next_batch(BlockWithSameBit* block_view) override { + return Status::NotSupported("MockRowsetReader::next_batch not implemented"); + } + + bool delete_flag() override { return false; } + + Version version() override { return Version(1, 1); } + + RowsetSharedPtr rowset() override { return _rowset; } + + int64_t filtered_rows() override { return 0; } + + uint64_t merged_rows() override { return 0; } + + RowsetTypePB type() const override { return BETA_ROWSET; } + + int64_t newest_write_timestamp() override { return 0; } + + void update_profile(RuntimeProfile* profile) override {} + + RowsetReaderSharedPtr clone() override { return std::make_shared(_rowset); } + + void set_topn_limit(size_t limit) override {} + +private: + std::shared_ptr _rowset; +}; + +} // namespace collection_statistics + +class CollectionStatisticsTest : public ::testing::Test { +protected: + void SetUp() override { + stats_ = std::make_unique(); + runtime_state_ = std::make_shared(); + runtime_state_->_mock_desc_tbl->add_slot_descriptor(SlotId(1), 1001); + test_dir_ = "./collection_statistics_test_" + + std::to_string(::testing::UnitTest::GetInstance()->random_seed()); + ASSERT_TRUE(io::global_local_filesystem()->create_directory(test_dir_).ok()); + } + + void TearDown() override { + stats_.reset(); + runtime_state_.reset(); + (void)io::global_local_filesystem()->delete_directory(test_dir_); + } + + TabletSchemaSPtr create_tablet_schema_with_inverted_index() { + auto tablet_schema = std::make_shared(); + + TabletColumn column; + column.set_unique_id(1); + column.set_name("content"); + column.set_type(FieldType::OLAP_FIELD_TYPE_STRING); + tablet_schema->append_column(column); + + TabletIndex index; + index._index_id = 1; + index._index_type = IndexType::INVERTED; + index._col_unique_ids.push_back(1); + std::map properties; + properties["parser"] = "standard"; + properties["support_phrase"] = "true"; + index._properties = properties; + + tablet_schema->append_index(std::move(index)); + + return tablet_schema; + } + + VExprContextSPtrs create_match_expr_contexts( + const std::string& search_term = "search term", + const std::string& base_analyzer_fingerprint = "") { + VExprContextSPtrs contexts; + + auto match_expr = + std::make_shared(TExprNodeType::MATCH_PRED); + if (!base_analyzer_fingerprint.empty()) { + auto analyzer = match_expr->query_analyzer_ctx()->analyzer_provider->get_analyzer( + segment_v2::inverted_index::AnalysisPurpose::kPlainQuery); + auto provider = + std::make_shared( + std::move(analyzer), base_analyzer_fingerprint); + auto analyzer_ctx = std::make_shared(); + analyzer_ctx->analyzer_provider = std::move(provider); + match_expr->set_analyzer_ctx(std::move(analyzer_ctx)); + } + auto slot_ref = std::make_shared("content", SlotId(1)); + auto literal = std::make_shared(search_term); + + match_expr->_children.push_back(slot_ref); + match_expr->_children.push_back(literal); + + auto context = std::make_shared(match_expr); + contexts.push_back(context); + + return contexts; + } + + std::vector create_mock_rowset_splits(int num_segments = 1) { + std::vector splits; + + auto rowset_meta = std::make_shared(); + auto rowset = std::make_shared( + create_tablet_schema_with_inverted_index(), rowset_meta); + rowset->set_num_segments(num_segments); + + for (int i = 0; i < num_segments; ++i) { + rowset->set_segment_path(i, test_dir_ + "/segment_" + std::to_string(i) + ".dat"); + } + + auto reader = std::make_shared(rowset); + + RowSetSplits split(reader); + splits.push_back(split); + + return splits; + } + + TabletSchemaSPtr create_legacy_v3_schema() { + TabletSchemaPB schema_pb; + schema_pb.set_keys_type(DUP_KEYS); + schema_pb.set_inverted_index_storage_format(InvertedIndexStorageFormatPB::V3); + auto tablet_schema = std::make_shared(); + tablet_schema->init_from_pb(schema_pb); + + TabletColumn column; + column.set_unique_id(1); + column.set_name("content"); + column.set_type(FieldType::OLAP_FIELD_TYPE_STRING); + tablet_schema->append_column(column); + + TabletIndex index; + index._index_id = 1; + index._index_type = IndexType::INVERTED; + index._col_unique_ids.push_back(1); + index._properties["parser"] = "standard"; + index._properties["support_phrase"] = "true"; + tablet_schema->append_index(std::move(index)); + return tablet_schema; + } + + TabletSchemaSPtr create_snii_schema(int64_t index_id = 1) { + TabletSchemaPB schema_pb; + schema_pb.set_keys_type(DUP_KEYS); + schema_pb.set_inverted_index_storage_format(InvertedIndexStorageFormatPB::SNII); + auto tablet_schema = std::make_shared(); + tablet_schema->init_from_pb(schema_pb); + + TabletColumn column; + column.set_unique_id(1); + column.set_name("content"); + column.set_type(FieldType::OLAP_FIELD_TYPE_STRING); + tablet_schema->append_column(column); + + TabletIndex index; + index._index_id = index_id; + index._index_type = IndexType::INVERTED; + index._col_unique_ids.push_back(1); + index._properties["parser"] = "standard"; + index._properties["support_phrase"] = "true"; + tablet_schema->append_index(std::move(index)); + return tablet_schema; + } + + Status write_legacy_v3_segment(const TabletSchemaSPtr& tablet_schema, + const std::string& segment_path) { + std::vector paths; + paths.emplace_back(test_dir_, 1024); + auto tmp_file_dirs = std::make_unique(paths); + RETURN_IF_ERROR(tmp_file_dirs->init()); + ExecEnv::GetInstance()->set_tmp_file_dir(std::move(tmp_file_dirs)); + + const std::string index_path_prefix { + segment_v2::InvertedIndexDescriptor::get_index_file_path_prefix(segment_path)}; + io::FileWriterPtr compound_file; + io::FileWriterOptions options; + auto fs = io::global_local_filesystem(); + RETURN_IF_ERROR(fs->create_file( + segment_v2::InvertedIndexDescriptor::get_index_file_path_v2(index_path_prefix), + &compound_file, &options)); + + segment_v2::IndexFileWriter file_writer(fs, index_path_prefix, "legacy_v3", 0, + InvertedIndexStorageFormatPB::V3, + std::move(compound_file)); + const auto index_metas = tablet_schema->inverted_indexs(1); + DORIS_CHECK(index_metas.size() == 1); + std::unique_ptr column_writer; + RETURN_IF_ERROR(segment_v2::IndexColumnWriter::create( + &tablet_schema->column(0), &column_writer, &file_writer, index_metas[0])); + std::vector values {Slice("alpha beta")}; + RETURN_IF_ERROR(column_writer->add_values("content", values.data(), values.size())); + RETURN_IF_ERROR(column_writer->finish()); + RETURN_IF_ERROR(file_writer.begin_close()); + return file_writer.finish_close(); + } + + Status write_snii_common_grams_segment(const std::string& segment_path, + std::string base_analyzer_fingerprint, + uint64_t scoring_token_count = 3) { + const std::string index_path_prefix { + segment_v2::InvertedIndexDescriptor::get_index_file_path_prefix(segment_path)}; + io::FileWriterPtr file_writer; + io::FileWriterOptions options; + auto fs = io::global_local_filesystem(); + RETURN_IF_ERROR(fs->create_file( + segment_v2::InvertedIndexDescriptor::get_index_file_path_v2(index_path_prefix), + &file_writer, &options)); + + segment_v2::snii_doris::DorisSniiFileWriter adapter(file_writer.get()); + snii::writer::SniiCompoundWriter writer(&adapter); + auto metadata = segment_v2::inverted_index::make_common_grams_segment_metadata( + {.common_grams_dictionary_identity = "test-stopwords-v1", + .base_analyzer_fingerprint = std::move(base_analyzer_fingerprint), + .common_grams_fingerprint = "test-common-grams-v1"}); + metadata.scoring_doc_count = 2; + metadata.scoring_token_count = scoring_token_count; + + snii::writer::TermPostings alpha; + alpha.term = "alpha"; + alpha.docids = {0, 1}; + alpha.freqs = {1, 1}; + alpha.positions_flat = {0, 0}; + + snii::writer::TermPostings beta; + beta.term = "beta"; + beta.docids = {0}; + beta.freqs = {1}; + beta.positions_flat = {1}; + + snii::writer::TermPostings gram; + gram.term = DORIS_TRY(segment_v2::inverted_index::encode_common_gram("alpha", "beta")); + gram.docids = {0}; + gram.freqs = {1}; + gram.positions_flat = {0}; + + snii::writer::SniiIndexInput input; + input.index_id = 1; + input.config = snii::format::IndexConfig::kDocsPositionsScoring; + input.doc_count = 2; + input.encoded_norms = {snii::query::encode_norm(2), snii::query::encode_norm(1)}; + input.terms = {std::move(gram), std::move(alpha), std::move(beta)}; + std::ranges::sort(input.terms, {}, &snii::writer::TermPostings::term); + input.common_grams_metadata = std::move(metadata); + + RETURN_IF_ERROR(writer.add_logical_index(input)); + RETURN_IF_ERROR(writer.finish()); + return file_writer->close(false); + } + + Status write_plain_snii_scoring_segment(const std::string& segment_path) { + const std::string index_path_prefix { + segment_v2::InvertedIndexDescriptor::get_index_file_path_prefix(segment_path)}; + io::FileWriterPtr file_writer; + io::FileWriterOptions options; + auto fs = io::global_local_filesystem(); + RETURN_IF_ERROR(fs->create_file( + segment_v2::InvertedIndexDescriptor::get_index_file_path_v2(index_path_prefix), + &file_writer, &options)); + + segment_v2::snii_doris::DorisSniiFileWriter adapter(file_writer.get()); + snii::writer::SniiCompoundWriter writer(&adapter); + + snii::writer::TermPostings alpha; + alpha.term = "alpha"; + alpha.docids = {0, 1}; + alpha.freqs = {1, 1}; + alpha.positions_flat = {0, 0}; + + snii::writer::TermPostings beta; + beta.term = "beta"; + beta.docids = {0}; + beta.freqs = {1}; + beta.positions_flat = {1}; + + snii::writer::SniiIndexInput input; + input.index_id = 1; + input.config = snii::format::IndexConfig::kDocsPositionsScoring; + input.doc_count = 2; + input.encoded_norms = {snii::query::encode_norm(2), snii::query::encode_norm(1)}; + input.terms = {std::move(alpha), std::move(beta)}; + + RETURN_IF_ERROR(writer.add_logical_index(input)); + RETURN_IF_ERROR(writer.finish()); + return file_writer->close(false); + } + + VExprContextSPtrs create_search_contexts(const std::string& clause_type, + const std::string& value) { + TSearchClause clause; + clause.clause_type = clause_type; + clause.field_name = "content"; + clause.value = value; + clause.__isset.field_name = true; + clause.__isset.value = true; + + return create_search_contexts(std::move(clause)); + } + + VExprContextSPtrs create_search_contexts(TSearchClause root, + std::vector field_bindings = {}) { + TSearchParam search_param; + search_param.root = std::move(root); + search_param.field_bindings = std::move(field_bindings); + + TExprNode node; + node.node_type = TExprNodeType::SEARCH_EXPR; + TTypeNode type_node; + type_node.type = TTypeNodeType::SCALAR; + TScalarType scalar_type; + scalar_type.__set_type(TPrimitiveType::BOOLEAN); + type_node.__set_scalar_type(scalar_type); + TTypeDesc type_desc; + type_desc.types.push_back(type_node); + node.__set_type(type_desc); + node.search_param = std::move(search_param); + node.__isset.search_param = true; + + return {std::make_shared(VSearchExpr::create_shared(node))}; + } + + TabletSchemaSPtr create_tablet_schema_with_keyword_and_fulltext_indexes() { + auto tablet_schema = std::make_shared(); + + TabletColumn column; + column.set_unique_id(1); + column.set_name("content"); + column.set_type(FieldType::OLAP_FIELD_TYPE_STRING); + tablet_schema->append_column(column); + + TabletIndex keyword_index; + keyword_index._index_id = 10; + keyword_index._index_type = IndexType::INVERTED; + keyword_index._col_unique_ids.push_back(1); + tablet_schema->append_index(std::move(keyword_index)); + + TabletIndex fulltext_index; + fulltext_index._index_id = 20; + fulltext_index._index_type = IndexType::INVERTED; + fulltext_index._col_unique_ids.push_back(1); + fulltext_index._properties["parser"] = "standard"; + fulltext_index._properties["support_phrase"] = "true"; + tablet_schema->append_index(std::move(fulltext_index)); + + return tablet_schema; + } + + TabletSchemaSPtr create_array_tablet_schema_with_keyword_and_fulltext_indexes() { + auto tablet_schema = std::make_shared(); + + TabletColumn item; + item.set_type(FieldType::OLAP_FIELD_TYPE_STRING); + TabletColumn column; + column.set_unique_id(1); + column.set_name("content"); + column.set_type(FieldType::OLAP_FIELD_TYPE_ARRAY); + column.add_sub_column(item); + tablet_schema->append_column(column); + + TabletIndex keyword_index; + keyword_index._index_id = 10; + keyword_index._index_type = IndexType::INVERTED; + keyword_index._col_unique_ids.push_back(1); + tablet_schema->append_index(std::move(keyword_index)); + + TabletIndex fulltext_index; + fulltext_index._index_id = 20; + fulltext_index._index_type = IndexType::INVERTED; + fulltext_index._col_unique_ids.push_back(1); + fulltext_index._properties["parser"] = "standard"; + fulltext_index._properties["support_phrase"] = "true"; + tablet_schema->append_index(std::move(fulltext_index)); + + return tablet_schema; + } + + TabletSchemaSPtr create_tablet_schema_with_two_fulltext_indexes() { + auto tablet_schema = std::make_shared(); + + TabletColumn column; + column.set_unique_id(1); + column.set_name("content"); + column.set_type(FieldType::OLAP_FIELD_TYPE_STRING); + tablet_schema->append_column(column); + + for (const auto& [index_id, parser] : {std::pair {10, "standard"}, + std::pair {20, "english"}}) { + TabletIndex index; + index._index_id = index_id; + index._index_type = IndexType::INVERTED; + index._col_unique_ids.push_back(1); + index._properties["parser"] = parser; + index._properties["support_phrase"] = "true"; + tablet_schema->append_index(std::move(index)); + } + + return tablet_schema; + } + + VExprContextSPtrs create_reserved_exact_search_contexts() { + return create_search_contexts( + "EXACT", std::string(segment_v2::inverted_index::CG_V1_MARKER) + "user"); + } + + void expect_no_collected_tokens(const std::wstring& field_name) { + EXPECT_THROW(stats_->get_total_term_cnt_by_col(field_name), Exception); + } + + void expect_collected_tokens(const std::wstring& field_name, uint64_t token_count) { + EXPECT_EQ(stats_->get_total_term_cnt_by_col(field_name), token_count); + } + + void expect_no_collected_term(const std::wstring& field_name, const std::wstring& term) { + EXPECT_THROW(stats_->get_term_doc_freq_by_col(field_name, term), Exception); + } + + void expect_collected_term(const std::wstring& field_name, const std::wstring& term, + uint64_t doc_frequency) { + EXPECT_EQ(stats_->get_term_doc_freq_by_col(field_name, term), doc_frequency); + } + + struct SniiScoringFieldInput { + SniiScoringFieldInput( + std::wstring field_name, + std::optional metadata, + uint64_t index_doc_count, bool has_semantic_norms, + std::optional expected_base_analyzer_fingerprint = std::nullopt) + : field_name(std::move(field_name)), + metadata(std::move(metadata)), + index_doc_count(index_doc_count), + physical_sum_total_term_freq(this->metadata ? this->metadata->scoring_token_count + : 0), + has_semantic_norms(has_semantic_norms), + expected_base_analyzer_fingerprint( + std::move(expected_base_analyzer_fingerprint)) {} + + std::wstring field_name; + std::optional metadata; + uint64_t index_doc_count = 0; + uint64_t physical_sum_total_term_freq = 0; + bool has_scoring_tier = true; + bool has_positions = true; + bool has_semantic_norms = false; + std::optional expected_base_analyzer_fingerprint; + }; + + Status stage_snii_fields_for_test( + CollectionStatistics* statistics, const std::vector& fields, + CollectionStatistics::SniiScoringSegmentAccumulator* segment_accumulator) { + for (const auto& field : fields) { + segment_v2::inverted_index::PlainTermKeyVersion key_version; + const std::string_view expected_base_analyzer_fingerprint = + field.expected_base_analyzer_fingerprint.has_value() + ? *field.expected_base_analyzer_fingerprint + : field.metadata.has_value() + ? std::string_view(field.metadata->base_analyzer_fingerprint) + : std::string_view(); + RETURN_IF_ERROR(statistics->admit_snii_scoring_segment( + field.field_name, field.metadata, expected_base_analyzer_fingerprint, + field.index_doc_count, field.physical_sum_total_term_freq, + field.has_scoring_tier, field.has_positions, field.has_semantic_norms, + &key_version, segment_accumulator)); + } + return Status::OK(); + } + + Status admit_snii_fields_for_test(CollectionStatistics* statistics, + const std::vector& fields) { + CollectionStatistics::SniiScoringSegmentAccumulator segment_accumulator; + RETURN_IF_ERROR(stage_snii_fields_for_test(statistics, fields, &segment_accumulator)); + statistics->commit_snii_scoring_segment(std::move(segment_accumulator)); + return Status::OK(); + } + + Status admit_snii_segment_for_test( + CollectionStatistics* statistics, const std::wstring& field_name, + const std::optional& metadata, + uint64_t index_doc_count, bool has_semantic_norms) { + return admit_snii_fields_for_test( + statistics, {{field_name, metadata, index_doc_count, has_semantic_norms}}); + } + + Status stage_snii_fields_then_file_not_found_for_test( + CollectionStatistics* statistics, + const std::vector& fields_before_failure) { + CollectionStatistics::SniiScoringSegmentAccumulator segment_accumulator; + RETURN_IF_ERROR(stage_snii_fields_for_test(statistics, fields_before_failure, + &segment_accumulator)); + for (const auto& field : fields_before_failure) { + add_term_doc_frequency(&segment_accumulator.term_doc_freqs, field.field_name, L"staged", + 1); + } + return Status::Error( + "simulated later field failure"); + } + + void expect_collected_stats(const std::wstring& field_name, uint64_t doc_count, + uint64_t token_count) { + EXPECT_EQ(stats_->get_doc_num(), doc_count); + expect_collected_tokens(field_name, token_count); + } + + std::unique_ptr stats_; + std::shared_ptr runtime_state_; + std::string test_dir_; +}; + +TEST_F(CollectionStatisticsTest, CollectWithEmptyRowsetSplits) { + auto tablet_schema = create_tablet_schema_with_inverted_index(); + auto expr_contexts = create_match_expr_contexts(); + + std::vector empty_splits; + + auto status = stats_->collect(runtime_state_.get(), empty_splits, tablet_schema, expr_contexts, + nullptr); + EXPECT_TRUE(status.ok()) << status.msg(); +} + +TEST_F(CollectionStatisticsTest, CollectWithEmptyExpressions) { + auto tablet_schema = create_tablet_schema_with_inverted_index(); + VExprContextSPtrs empty_contexts; + + std::vector empty_splits; + + auto status = stats_->collect(runtime_state_.get(), empty_splits, tablet_schema, empty_contexts, + nullptr); + EXPECT_TRUE(status.ok()) << status.msg(); +} + +TEST_F(CollectionStatisticsTest, CollectWithNonMatchExpression) { + auto tablet_schema = create_tablet_schema_with_inverted_index(); + + VExprContextSPtrs contexts; + auto non_match_expr = + std::make_shared(TExprNodeType::BINARY_PRED); + auto context = std::make_shared(non_match_expr); + contexts.push_back(context); + + std::vector empty_splits; + + auto status = + stats_->collect(runtime_state_.get(), empty_splits, tablet_schema, contexts, nullptr); + EXPECT_TRUE(status.ok()) << status.msg(); +} + +TEST_F(CollectionStatisticsTest, CollectWithMultipleMatchExpressions) { + auto tablet_schema = create_tablet_schema_with_inverted_index(); + + VExprContextSPtrs contexts; + + auto match_expr1 = + std::make_shared(TExprNodeType::MATCH_PRED); + auto slot_ref1 = std::make_shared("content", SlotId(1)); + auto literal1 = std::make_shared("term1"); + match_expr1->_children.push_back(slot_ref1); + match_expr1->_children.push_back(literal1); + contexts.push_back(std::make_shared(match_expr1)); + + auto match_expr2 = + std::make_shared(TExprNodeType::MATCH_PRED); + auto slot_ref2 = std::make_shared("content", SlotId(1)); + auto literal2 = std::make_shared("term2"); + match_expr2->_children.push_back(slot_ref2); + match_expr2->_children.push_back(literal2); + contexts.push_back(std::make_shared(match_expr2)); + + std::vector empty_splits; + + auto status = + stats_->collect(runtime_state_.get(), empty_splits, tablet_schema, contexts, nullptr); + EXPECT_TRUE(status.ok()) << status.msg(); +} + +TEST_F(CollectionStatisticsTest, CollectWithNestedExpressions) { + auto tablet_schema = create_tablet_schema_with_inverted_index(); + + VExprContextSPtrs contexts; + + auto and_expr = std::make_shared(TExprNodeType::BINARY_PRED); + + auto match_expr = std::make_shared(TExprNodeType::MATCH_PRED); + auto slot_ref = std::make_shared("content", SlotId(1)); + auto literal = std::make_shared("nested term"); + match_expr->_children.push_back(slot_ref); + match_expr->_children.push_back(literal); + + auto other_expr = + std::make_shared(TExprNodeType::BINARY_PRED); + + and_expr->_children.push_back(match_expr); + and_expr->_children.push_back(other_expr); + + contexts.push_back(std::make_shared(and_expr)); + + std::vector empty_splits; + + auto status = + stats_->collect(runtime_state_.get(), empty_splits, tablet_schema, contexts, nullptr); + EXPECT_TRUE(status.ok()) << status.msg(); +} + +TEST_F(CollectionStatisticsTest, CollectWithMockRowsetSplits) { + auto tablet_schema = create_tablet_schema_with_inverted_index(); + auto expr_contexts = create_match_expr_contexts(); + + auto splits = create_mock_rowset_splits(2); + + auto status = + stats_->collect(runtime_state_.get(), splits, tablet_schema, expr_contexts, nullptr); + + EXPECT_TRUE(status.ok()) << status; + expect_no_collected_tokens(L"1"); +} + +TEST_F(CollectionStatisticsTest, CollectWithEmptySegments) { + auto tablet_schema = create_tablet_schema_with_inverted_index(); + auto expr_contexts = create_match_expr_contexts(); + + auto splits = create_mock_rowset_splits(0); + + auto status = + stats_->collect(runtime_state_.get(), splits, tablet_schema, expr_contexts, nullptr); + EXPECT_TRUE(status.ok()) << status.msg(); +} + +TEST_F(CollectionStatisticsTest, LegacyReservedTermUsesRawV3Namespace) { + auto tablet_schema = create_legacy_v3_schema(); + const std::string segment_path = test_dir_ + "/legacy_v3_0.dat"; + ASSERT_TRUE(write_legacy_v3_segment(tablet_schema, segment_path).ok()); + + auto rowset_meta = std::make_shared(); + auto rowset = std::make_shared(tablet_schema, rowset_meta); + rowset->set_num_segments(1); + rowset->set_segment_path(0, segment_path); + auto reader = std::make_shared(rowset); + std::vector splits {RowSetSplits(reader)}; + + auto status = stats_->collect(runtime_state_.get(), splits, tablet_schema, + create_reserved_exact_search_contexts(), nullptr); + + ASSERT_TRUE(status.ok()) << status; + expect_collected_stats(L"1", 1, 2); + expect_collected_term(L"1", + segment_v2::inverted_index::StringHelper::to_wstring( + std::string(segment_v2::inverted_index::CG_V1_MARKER) + "user"), + 0); +} + +TEST_F(CollectionStatisticsTest, LegacyV3SegmentsUsePhysicalScoringStatistics) { + auto tablet_schema = create_legacy_v3_schema(); + const std::string first_segment_path = test_dir_ + "/legacy_v3_0.dat"; + const std::string second_segment_path = test_dir_ + "/legacy_v3_1.dat"; + ASSERT_TRUE(write_legacy_v3_segment(tablet_schema, first_segment_path).ok()); + ASSERT_TRUE(write_legacy_v3_segment(tablet_schema, second_segment_path).ok()); + + auto rowset_meta = std::make_shared(); + auto rowset = std::make_shared(tablet_schema, rowset_meta); + rowset->set_num_segments(2); + rowset->set_segment_path(0, first_segment_path); + rowset->set_segment_path(1, second_segment_path); + auto reader = std::make_shared(rowset); + std::vector splits {RowSetSplits(reader)}; + + auto status = stats_->collect(runtime_state_.get(), splits, tablet_schema, + create_match_expr_contexts("alpha"), nullptr); + + ASSERT_TRUE(status.ok()) << status; + expect_collected_stats(L"1", 2, 4); + expect_collected_term(L"1", L"alpha", 2); +} + +TEST_F(CollectionStatisticsTest, LegacyV3SkipsMissingSegmentAfterCollectingAvailableStatistics) { + auto tablet_schema = create_legacy_v3_schema(); + const std::string first_segment_path = test_dir_ + "/legacy_v3_0.dat"; + ASSERT_TRUE(write_legacy_v3_segment(tablet_schema, first_segment_path).ok()); + + auto rowset_meta = std::make_shared(); + auto rowset = std::make_shared(tablet_schema, rowset_meta); + rowset->set_num_segments(2); + rowset->set_segment_path(0, first_segment_path); + rowset->set_segment_path(1, test_dir_ + "/missing_v3_1.dat"); + auto reader = std::make_shared(rowset); + std::vector splits {RowSetSplits(reader)}; + + auto status = stats_->collect(runtime_state_.get(), splits, tablet_schema, + create_match_expr_contexts("alpha"), nullptr); + + ASSERT_TRUE(status.ok()) << status; + expect_collected_stats(L"1", 1, 2); + expect_collected_term(L"1", L"alpha", 1); +} + +TEST_F(CollectionStatisticsTest, LegacyV3SkipsEmptySegmentAfterCollectingAvailableStatistics) { + auto tablet_schema = create_legacy_v3_schema(); + const std::string first_segment_path = test_dir_ + "/legacy_v3_0.dat"; + const std::string empty_segment_path = test_dir_ + "/legacy_v3_empty_1.dat"; + ASSERT_TRUE(write_legacy_v3_segment(tablet_schema, first_segment_path).ok()); + + const std::string empty_index_path = + segment_v2::InvertedIndexDescriptor::get_index_file_path_v2( + segment_v2::InvertedIndexDescriptor::get_index_file_path_prefix( + empty_segment_path)); + io::FileWriterPtr empty_file; + io::FileWriterOptions options; + ASSERT_TRUE(io::global_local_filesystem() + ->create_file(empty_index_path, &empty_file, &options) + .ok()); + ASSERT_TRUE(empty_file->close(false).ok()); + + auto rowset_meta = std::make_shared(); + auto rowset = std::make_shared(tablet_schema, rowset_meta); + rowset->set_num_segments(2); + rowset->set_segment_path(0, first_segment_path); + rowset->set_segment_path(1, empty_segment_path); + auto reader = std::make_shared(rowset); + std::vector splits {RowSetSplits(reader)}; + + const Status status = stats_->collect(runtime_state_.get(), splits, tablet_schema, + create_match_expr_contexts("alpha"), nullptr); + + ASSERT_TRUE(status.ok()) << status; + expect_collected_stats(L"1", 1, 2); + expect_collected_term(L"1", L"alpha", 1); +} + +TEST_F(CollectionStatisticsTest, SniiCommonGramsUsesSemanticScoringStatistics) { + auto tablet_schema = create_snii_schema(); + auto expr_contexts = create_match_expr_contexts("alpha", "test-base-v1"); + const auto* analyzer_ctx = expr_contexts.front()->root()->query_analyzer_ctx(); + ASSERT_NE(analyzer_ctx, nullptr); + ASSERT_NE(analyzer_ctx->analyzer_provider, nullptr); + + const std::string segment_path = test_dir_ + "/snii_common_grams_0.dat"; + auto write_status = write_snii_common_grams_segment( + segment_path, + std::string(analyzer_ctx->analyzer_provider->base_analyzer_fingerprint())); + ASSERT_TRUE(write_status.ok()) << write_status; + + auto rowset_meta = std::make_shared(); + auto rowset = std::make_shared(tablet_schema, rowset_meta); + rowset->set_num_segments(1); + rowset->set_segment_path(0, segment_path); + auto reader = std::make_shared(rowset); + std::vector splits {RowSetSplits(reader)}; + + auto status = + stats_->collect(runtime_state_.get(), splits, tablet_schema, expr_contexts, nullptr); + + ASSERT_TRUE(status.ok()) << status; + expect_collected_stats(L"1", 2, 3); + expect_collected_term(L"1", L"alpha", 2); +} + +TEST_F(CollectionStatisticsTest, SniiScoringLookupUsesCallerIoContext) { + snii::snii_test::ScopedEnv force_nonresident_dict("SNII_DICT_RESIDENT_MAX", "0"); + auto tablet_schema = create_snii_schema(); + auto expr_contexts = create_match_expr_contexts("alpha", "test-base-v1"); + const auto* analyzer_ctx = expr_contexts.front()->root()->query_analyzer_ctx(); + ASSERT_NE(analyzer_ctx, nullptr); + ASSERT_NE(analyzer_ctx->analyzer_provider, nullptr); + + const std::string segment_path = test_dir_ + "/snii_io_context_0.dat"; + ASSERT_TRUE(write_snii_common_grams_segment( + segment_path, + std::string(analyzer_ctx->analyzer_provider->base_analyzer_fingerprint())) + .ok()); + + auto rowset_meta = std::make_shared(); + auto rowset = std::make_shared(tablet_schema, rowset_meta); + rowset->set_num_segments(1); + rowset->set_segment_path(0, segment_path); + auto reader = std::make_shared(rowset); + std::vector splits {RowSetSplits(reader)}; + + const std::string index_path_prefix { + segment_v2::InvertedIndexDescriptor::get_index_file_path_prefix(segment_path)}; + io::FileCacheStatistics open_stats; + io::IOContext open_io_ctx; + open_io_ctx.file_cache_stats = &open_stats; + segment_v2::IndexFileReader file_reader(io::global_local_filesystem(), index_path_prefix, + InvertedIndexStorageFormatPB::SNII); + ASSERT_TRUE(file_reader.init(config::inverted_index_read_buffer_size, &open_io_ctx).ok()); + const auto index_metas = tablet_schema->inverted_indexs(1); + ASSERT_EQ(index_metas.size(), 1); + auto logical_reader = file_reader.open_snii_index(index_metas.front(), &open_io_ctx); + ASSERT_TRUE(logical_reader.has_value()) << logical_reader.error(); + ASSERT_GT(open_stats.inverted_index_range_read_count, 0); + + io::FileCacheStatistics collect_stats; + io::IOContext collect_io_ctx; + collect_io_ctx.file_cache_stats = &collect_stats; + const auto status = stats_->collect(runtime_state_.get(), splits, tablet_schema, expr_contexts, + &collect_io_ctx); + + ASSERT_TRUE(status.ok()) << status; + EXPECT_GT(collect_stats.inverted_index_range_read_count, + open_stats.inverted_index_range_read_count); +} + +TEST_F(CollectionStatisticsTest, SniiStatsProviderUsesSemanticCommonGramsTokenCount) { + auto tablet_schema = create_snii_schema(); + auto expr_contexts = create_match_expr_contexts("alpha", "test-base-v1"); + const auto* analyzer_ctx = expr_contexts.front()->root()->query_analyzer_ctx(); + ASSERT_NE(analyzer_ctx, nullptr); + ASSERT_NE(analyzer_ctx->analyzer_provider, nullptr); + + const std::string segment_path = test_dir_ + "/snii_semantic_stats_0.dat"; + auto write_status = write_snii_common_grams_segment( + segment_path, + std::string(analyzer_ctx->analyzer_provider->base_analyzer_fingerprint())); + ASSERT_TRUE(write_status.ok()) << write_status; + + const std::string index_path_prefix { + segment_v2::InvertedIndexDescriptor::get_index_file_path_prefix(segment_path)}; + segment_v2::IndexFileReader file_reader(io::global_local_filesystem(), index_path_prefix, + InvertedIndexStorageFormatPB::SNII); + ASSERT_TRUE(file_reader.init().ok()); + const auto index_metas = tablet_schema->inverted_indexs(1); + ASSERT_EQ(index_metas.size(), 1); + auto logical_reader = file_reader.open_snii_index(index_metas.front()); + ASSERT_TRUE(logical_reader.has_value()) << logical_reader.error(); + + snii::stats::SniiStatsProvider provider; + ASSERT_TRUE(snii::stats::SniiStatsProvider::open(logical_reader->get(), &provider).ok()); + EXPECT_EQ(provider.doc_count(), 2); + EXPECT_EQ(provider.indexed_doc_count(), 2); + EXPECT_EQ(provider.sum_total_term_freq(), 3); + EXPECT_DOUBLE_EQ(provider.avgdl(), 1.5); + EXPECT_TRUE(provider.has_norms()); +} + +TEST_F(CollectionStatisticsTest, SniiWriterRejectsMissingSemanticScoringMetadata) { + const std::string segment_path = test_dir_ + "/snii_missing_scoring_metadata_0.dat"; + auto write_status = write_plain_snii_scoring_segment(segment_path); + EXPECT_EQ(write_status.code(), ErrorCode::INVERTED_INDEX_NOT_SUPPORTED); +} + +TEST_F(CollectionStatisticsTest, SniiScoringRejectsMissingSegmentForWholeCollection) { + auto tablet_schema = create_snii_schema(); + auto expr_contexts = create_match_expr_contexts("alpha", "test-base-v1"); + const auto* analyzer_ctx = expr_contexts.front()->root()->query_analyzer_ctx(); + ASSERT_NE(analyzer_ctx, nullptr); + ASSERT_NE(analyzer_ctx->analyzer_provider, nullptr); + + const std::string first_segment_path = test_dir_ + "/snii_complete_0.dat"; + auto write_status = write_snii_common_grams_segment( + first_segment_path, + std::string(analyzer_ctx->analyzer_provider->base_analyzer_fingerprint())); + ASSERT_TRUE(write_status.ok()) << write_status; + + auto rowset_meta = std::make_shared(); + auto rowset = std::make_shared(tablet_schema, rowset_meta); + rowset->set_num_segments(2); + rowset->set_segment_path(0, first_segment_path); + rowset->set_segment_path(1, test_dir_ + "/missing_snii_1.dat"); + auto reader = std::make_shared(rowset); + std::vector splits {RowSetSplits(reader)}; + + auto status = + stats_->collect(runtime_state_.get(), splits, tablet_schema, expr_contexts, nullptr); + + EXPECT_EQ(status.code(), ErrorCode::INVERTED_INDEX_NOT_SUPPORTED); +} + +TEST_F(CollectionStatisticsTest, SniiScoringRejectsMissingLogicalIndexForWholeCollection) { + auto tablet_schema = create_snii_schema(/*index_id=*/2); + auto expr_contexts = create_match_expr_contexts("alpha", "test-base-v1"); + const auto* analyzer_ctx = expr_contexts.front()->root()->query_analyzer_ctx(); + ASSERT_NE(analyzer_ctx, nullptr); + ASSERT_NE(analyzer_ctx->analyzer_provider, nullptr); + + const std::string segment_path = test_dir_ + "/snii_missing_logical_index_0.dat"; + auto write_status = write_snii_common_grams_segment( + segment_path, + std::string(analyzer_ctx->analyzer_provider->base_analyzer_fingerprint())); + ASSERT_TRUE(write_status.ok()) << write_status; + + auto rowset_meta = std::make_shared(); + auto rowset = std::make_shared(tablet_schema, rowset_meta); + rowset->set_num_segments(1); + rowset->set_segment_path(0, segment_path); + auto reader = std::make_shared(rowset); + std::vector splits {RowSetSplits(reader)}; + + auto status = + stats_->collect(runtime_state_.get(), splits, tablet_schema, expr_contexts, nullptr); + + EXPECT_EQ(status.code(), ErrorCode::INVERTED_INDEX_NOT_SUPPORTED); + expect_no_collected_tokens(L"1"); +} + +TEST_F(CollectionStatisticsTest, SniiWriterRejectsZeroSemanticTokensForNonemptyPostings) { + auto status = write_snii_common_grams_segment(test_dir_ + "/snii_zero_semantic_tokens_0.dat", + "test-base-v1", + /*scoring_token_count=*/0); + + EXPECT_EQ(status.code(), ErrorCode::INVALID_ARGUMENT); + EXPECT_THAT(status.msg(), ::testing::HasSubstr("zero semantic scoring tokens")); +} + +TEST_F(CollectionStatisticsTest, CollectWithMultipleRowsetSplits) { + auto tablet_schema = create_tablet_schema_with_inverted_index(); + auto expr_contexts = create_match_expr_contexts(); + + std::vector splits; + + for (int i = 0; i < 3; ++i) { + auto rowset_meta = std::make_shared(); + auto rowset = + std::make_shared(tablet_schema, rowset_meta); + rowset->set_num_segments(0); + + auto reader = std::make_shared(rowset); + + RowSetSplits split(reader); + splits.push_back(split); + } + + auto status = + stats_->collect(runtime_state_.get(), splits, tablet_schema, expr_contexts, nullptr); + EXPECT_TRUE(status.ok()) << status.msg(); +} + +class TestableCollectionStatistics : public CollectionStatistics { +public: + void set_total_num_docs(uint64_t num_docs) { _total_num_docs = num_docs; } + + void set_total_num_tokens(const std::wstring& field_name, uint64_t num_tokens) { + _total_num_tokens[field_name] = num_tokens; + } + + void set_term_doc_freq(const std::wstring& field_name, const std::wstring& term, + uint64_t freq) { + _term_doc_freqs[field_name][term] = freq; + } +}; + +class CollectionStatisticsDetailedTest : public ::testing::Test { +protected: + void SetUp() override { stats_ = std::make_unique(); } + + void TearDown() override { stats_.reset(); } + + std::unique_ptr stats_; +}; + +segment_v2::inverted_index::CommonGramsSegmentMetadata complete_snii_scoring_metadata( + std::string base_analyzer_fingerprint = "base-v1", uint64_t doc_count = 3, + uint64_t token_count = 7) { + using namespace segment_v2::inverted_index; + segment_v2::inverted_index::CommonGramsSegmentMetadata metadata; + metadata.plain_term_key_version = PlainTermKeyVersion::kEscapedV1; + metadata.common_grams_coverage = CommonGramsCoverage::kComplete; + metadata.common_grams_semantics_version = COMMON_GRAMS_SEMANTICS_VERSION_V1; + metadata.common_grams_key_version = COMMON_GRAMS_KEY_VERSION_V1; + metadata.common_grams_dictionary_identity = "builtin-stopwords:v1"; + metadata.base_analyzer_fingerprint = std::move(base_analyzer_fingerprint); + metadata.common_grams_fingerprint = "common-grams-v1"; + metadata.scoring_coverage = ScoringCoverage::kComplete; + metadata.scoring_stats_version = COMMON_GRAMS_SCORING_STATS_VERSION_V1; + metadata.norm_semantics_version = COMMON_GRAMS_NORM_SEMANTICS_VERSION_V1; + metadata.scoring_doc_count = doc_count; + metadata.scoring_token_count = token_count; + return metadata; +} + +Result resolve_snii_scoring_segment_for_test( + std::optional metadata, + uint64_t physical_doc_count, bool has_norms) { + const uint64_t physical_sum_total_term_freq = metadata ? metadata->scoring_token_count : 0; + return resolve_snii_scoring_segment(metadata, physical_doc_count, physical_sum_total_term_freq, + /*has_scoring_tier=*/true, + /*has_positions=*/true, has_norms); +} + +TEST(CollectionStatisticsCommonGramsTest, MissingMetadataWithoutPersistedProofRejectsScoring) { + auto result = resolve_snii_scoring_segment_for_test(std::nullopt, 3, true); + + ASSERT_FALSE(result.has_value()); + EXPECT_EQ(result.error().code(), ErrorCode::INVERTED_INDEX_NOT_SUPPORTED); +} + +TEST(CollectionStatisticsCommonGramsTest, RawNoInternalMetadataWithoutScoringProofIsRejected) { + segment_v2::inverted_index::CommonGramsSegmentMetadata metadata; + metadata.plain_term_key_version = + segment_v2::inverted_index::PlainTermKeyVersion::kRawNoInternal; + metadata.common_grams_coverage = segment_v2::inverted_index::CommonGramsCoverage::kNone; + metadata.base_analyzer_fingerprint = "base-v1"; + + auto result = resolve_snii_scoring_segment_for_test(metadata, 3, true); + + ASSERT_FALSE(result.has_value()); + EXPECT_EQ(result.error().code(), ErrorCode::INVERTED_INDEX_NOT_SUPPORTED); +} + +TEST(CollectionStatisticsCommonGramsTest, ScoringCoverageNoneRejectsScoringAdmission) { + auto metadata = complete_snii_scoring_metadata(); + metadata.scoring_coverage = segment_v2::inverted_index::ScoringCoverage::kNone; + metadata.scoring_stats_version = 0; + metadata.norm_semantics_version = 0; + + auto result = resolve_snii_scoring_segment_for_test(metadata, 3, true); + + ASSERT_FALSE(result.has_value()); + EXPECT_EQ(result.error().code(), ErrorCode::INVERTED_INDEX_NOT_SUPPORTED); +} + +TEST(CollectionStatisticsCommonGramsTest, IncompatibleScoringVersionsRejectScoringAdmission) { + auto metadata = complete_snii_scoring_metadata(); + + ++metadata.scoring_stats_version; + auto scoring_version_result = resolve_snii_scoring_segment_for_test(metadata, 3, true); + ASSERT_FALSE(scoring_version_result.has_value()); + EXPECT_EQ(scoring_version_result.error().code(), ErrorCode::INVERTED_INDEX_NOT_SUPPORTED); + + --metadata.scoring_stats_version; + ++metadata.norm_semantics_version; + auto norm_version_result = resolve_snii_scoring_segment_for_test(metadata, 3, true); + ASSERT_FALSE(norm_version_result.has_value()); + EXPECT_EQ(norm_version_result.error().code(), ErrorCode::INVERTED_INDEX_NOT_SUPPORTED); +} + +TEST(CollectionStatisticsCommonGramsTest, ScoringDocCountMismatchRejectsScoringAdmission) { + auto result = resolve_snii_scoring_segment_for_test(complete_snii_scoring_metadata(), 4, true); + + ASSERT_FALSE(result.has_value()); + EXPECT_EQ(result.error().code(), ErrorCode::INVERTED_INDEX_FILE_CORRUPTED); +} + +TEST(CollectionStatisticsCommonGramsTest, MissingSemanticNormsRejectScoringAdmission) { + auto result = resolve_snii_scoring_segment_for_test(complete_snii_scoring_metadata(), 3, false); + + ASSERT_FALSE(result.has_value()); + EXPECT_EQ(result.error().code(), ErrorCode::INVERTED_INDEX_FILE_CORRUPTED); +} + +TEST(CollectionStatisticsCommonGramsTest, CompleteMetadataOnNonScoringTierIsCorruption) { + auto metadata = complete_snii_scoring_metadata(); + auto result = resolve_snii_scoring_segment(metadata, 3, 7, + /*has_scoring_tier=*/false, + /*has_positions=*/true, + /*has_semantic_norms=*/true); + + ASSERT_FALSE(result.has_value()); + EXPECT_EQ(result.error().code(), ErrorCode::INVERTED_INDEX_FILE_CORRUPTED); +} + +TEST(CollectionStatisticsCommonGramsTest, ZeroSemanticTokensWithPhysicalTermsIsCorruption) { + auto metadata = complete_snii_scoring_metadata("base-v1", 3, 0); + auto result = resolve_snii_scoring_segment(metadata, 3, 1, + /*has_scoring_tier=*/true, + /*has_positions=*/true, + /*has_semantic_norms=*/true); + + ASSERT_FALSE(result.has_value()); + EXPECT_EQ(result.error().code(), ErrorCode::INVERTED_INDEX_FILE_CORRUPTED); +} + +TEST(CollectionStatisticsCommonGramsTest, EmptyPhysicalAndSemanticTokenCountsAreValid) { + auto metadata = complete_snii_scoring_metadata("base-v1", 3, 0); + auto result = resolve_snii_scoring_segment(metadata, 3, 0, + /*has_scoring_tier=*/true, + /*has_positions=*/true, + /*has_semantic_norms=*/true); + + ASSERT_TRUE(result.has_value()) << result.error(); + EXPECT_EQ(result->token_count, 0); +} + +TEST(CollectionStatisticsCommonGramsTest, LegacyPhysicalScoringRequiresDocumentLengthNorms) { + auto result = resolve_snii_scoring_segment_for_test(std::nullopt, 3, false); + + ASSERT_FALSE(result.has_value()); + EXPECT_EQ(result.error().code(), ErrorCode::INVERTED_INDEX_NOT_SUPPORTED); +} + +TEST(CollectionStatisticsCommonGramsTest, CompleteMetadataUsesSemanticDocAndTokenCounts) { + auto result = resolve_snii_scoring_segment_for_test(complete_snii_scoring_metadata(), 3, true); + + ASSERT_TRUE(result.has_value()) << result.error(); + EXPECT_EQ(result->doc_count, 3); + EXPECT_EQ(result->token_count, 7); + EXPECT_EQ(result->plain_term_key_version, + segment_v2::inverted_index::PlainTermKeyVersion::kEscapedV1); + EXPECT_EQ(result->base_analyzer_fingerprint, "base-v1"); +} + +TEST(CollectionStatisticsCommonGramsTest, CompletePlainMetadataIsExplicitSemanticProof) { + auto metadata = complete_snii_scoring_metadata(); + metadata.plain_term_key_version = + segment_v2::inverted_index::PlainTermKeyVersion::kRawNoInternal; + metadata.common_grams_coverage = segment_v2::inverted_index::CommonGramsCoverage::kNone; + metadata.common_grams_semantics_version = 0; + metadata.common_grams_key_version = 0; + metadata.common_grams_dictionary_identity.clear(); + metadata.common_grams_fingerprint.clear(); + + auto result = resolve_snii_scoring_segment_for_test(metadata, 3, true); + + ASSERT_TRUE(result.has_value()) << result.error(); + EXPECT_EQ(result->doc_count, 3); + EXPECT_EQ(result->token_count, 7); +} + +TEST(CollectionStatisticsCommonGramsTest, PlainSemanticTokenCountMustEqualPhysicalCount) { + auto metadata = complete_snii_scoring_metadata(); + metadata.plain_term_key_version = + segment_v2::inverted_index::PlainTermKeyVersion::kRawNoInternal; + metadata.common_grams_coverage = segment_v2::inverted_index::CommonGramsCoverage::kNone; + metadata.common_grams_semantics_version = 0; + metadata.common_grams_key_version = 0; + metadata.common_grams_dictionary_identity.clear(); + metadata.common_grams_fingerprint.clear(); + + auto result = resolve_snii_scoring_segment(metadata, 3, 8, + /*has_scoring_tier=*/true, + /*has_positions=*/true, + /*has_semantic_norms=*/true); + + ASSERT_FALSE(result.has_value()); + EXPECT_EQ(result.error().code(), ErrorCode::INVERTED_INDEX_FILE_CORRUPTED); +} + +TEST(CollectionStatisticsCommonGramsTest, EmptySemanticFingerprintIsNotScoringProof) { + auto metadata = complete_snii_scoring_metadata(); + metadata.base_analyzer_fingerprint.clear(); + + auto result = resolve_snii_scoring_segment_for_test(metadata, 3, true); + + ASSERT_FALSE(result.has_value()); + EXPECT_EQ(result.error().code(), ErrorCode::INVERTED_INDEX_FILE_CORRUPTED); +} + +TEST_F(CollectionStatisticsTest, MixedBaseFingerprintRejectsAndClearsWholeCollection) { + auto first = complete_snii_scoring_metadata("base-v1", 3, 7); + ASSERT_TRUE(admit_snii_segment_for_test(stats_.get(), L"1", first, 3, true).ok()); + EXPECT_FLOAT_EQ(stats_->get_or_calculate_avg_dl(L"1"), 7.0F / 3.0F); + + auto second = complete_snii_scoring_metadata("base-v2", 3, 5); + auto status = admit_snii_segment_for_test(stats_.get(), L"1", second, 3, true); + + EXPECT_EQ(status.code(), ErrorCode::INVERTED_INDEX_NOT_SUPPORTED); + expect_no_collected_tokens(L"1"); + EXPECT_THROW(stats_->get_or_calculate_avg_dl(L"1"), Exception); +} + +TEST_F(CollectionStatisticsTest, PersistedFingerprintMustMatchRequestAnalyzer) { + auto metadata = complete_snii_scoring_metadata("persisted-base", 3, 7); + auto status = admit_snii_fields_for_test( + stats_.get(), {{L"1", metadata, 3, true, std::string("request-base")}}); + + EXPECT_EQ(status.code(), ErrorCode::INVERTED_INDEX_NOT_SUPPORTED); + expect_no_collected_tokens(L"1"); + EXPECT_THROW(stats_->get_doc_num(), Exception); +} + +TEST_F(CollectionStatisticsTest, CollectionStatisticsInstancesKeepAdmissionStateIsolated) { + CollectionStatistics first; + CollectionStatistics second; + + ASSERT_TRUE(admit_snii_segment_for_test(&first, L"1", + complete_snii_scoring_metadata("base-a", 2, 6), 2, true) + .ok()); + ASSERT_TRUE(admit_snii_segment_for_test( + &second, L"1", complete_snii_scoring_metadata("base-b", 5, 25), 5, true) + .ok()); + + EXPECT_FLOAT_EQ(first.get_or_calculate_avg_dl(L"1"), 3.0F); + EXPECT_FLOAT_EQ(second.get_or_calculate_avg_dl(L"1"), 5.0F); +} + +TEST_F(CollectionStatisticsTest, LegacyAndUnprovedRawNoInternalRejectWholeCollection) { + ASSERT_TRUE(admit_snii_segment_for_test(stats_.get(), L"1", + complete_snii_scoring_metadata("base-v1", 2, 6), 2, + true) + .ok()); + + segment_v2::inverted_index::CommonGramsSegmentMetadata plain_metadata; + plain_metadata.plain_term_key_version = + segment_v2::inverted_index::PlainTermKeyVersion::kRawNoInternal; + plain_metadata.common_grams_coverage = segment_v2::inverted_index::CommonGramsCoverage::kNone; + plain_metadata.base_analyzer_fingerprint = "base-v1"; + auto status = admit_snii_segment_for_test(stats_.get(), L"1", plain_metadata, 3, true); + + EXPECT_EQ(status.code(), ErrorCode::INVERTED_INDEX_NOT_SUPPORTED); + expect_no_collected_tokens(L"1"); + EXPECT_THROW(stats_->get_doc_num(), Exception); +} + +TEST_F(CollectionStatisticsTest, LegacyAndCommonGramsMixRejectsWholeCollection) { + ASSERT_TRUE(admit_snii_segment_for_test(stats_.get(), L"1", + complete_snii_scoring_metadata("base-v1", 2, 6), 2, + true) + .ok()); + + auto status = admit_snii_segment_for_test(stats_.get(), L"1", std::nullopt, 3, true); + + EXPECT_EQ(status.code(), ErrorCode::INVERTED_INDEX_NOT_SUPPORTED); + expect_no_collected_tokens(L"1"); + EXPECT_THROW(stats_->get_or_calculate_avg_dl(L"1"), Exception); +} + +TEST_F(CollectionStatisticsTest, ExplicitSemanticPlainAndCommonGramsSegmentsAccumulate) { + auto plain_metadata = complete_snii_scoring_metadata("base-v1", 2, 6); + plain_metadata.plain_term_key_version = + segment_v2::inverted_index::PlainTermKeyVersion::kRawNoInternal; + plain_metadata.common_grams_coverage = segment_v2::inverted_index::CommonGramsCoverage::kNone; + plain_metadata.common_grams_semantics_version = 0; + plain_metadata.common_grams_key_version = 0; + plain_metadata.common_grams_dictionary_identity.clear(); + plain_metadata.common_grams_fingerprint.clear(); + + ASSERT_TRUE(admit_snii_segment_for_test(stats_.get(), L"1", plain_metadata, 2, true).ok()); + ASSERT_TRUE(admit_snii_segment_for_test(stats_.get(), L"1", + complete_snii_scoring_metadata("base-v1", 3, 7), 3, + true) + .ok()); + + expect_collected_stats(L"1", 5, 13); + EXPECT_FLOAT_EQ(stats_->get_or_calculate_avg_dl(L"1"), 13.0F / 5.0F); +} + +TEST_F(CollectionStatisticsTest, CommonGramsSegmentsAccumulateSemanticStatistics) { + ASSERT_TRUE(admit_snii_segment_for_test(stats_.get(), L"1", + complete_snii_scoring_metadata("base-v1", 2, 6), 2, + true) + .ok()); + ASSERT_TRUE(admit_snii_segment_for_test(stats_.get(), L"1", + complete_snii_scoring_metadata("base-v1", 3, 9), 3, + true) + .ok()); + + expect_collected_stats(L"1", 5, 15); + EXPECT_FLOAT_EQ(stats_->get_or_calculate_avg_dl(L"1"), 3.0F); +} + +TEST_F(CollectionStatisticsTest, MultiFieldSegmentsCommitAndAccumulateAtomically) { + ASSERT_TRUE(admit_snii_fields_for_test( + stats_.get(), + {{L"1", complete_snii_scoring_metadata("field-1", 3, 7), 3, true}, + {L"2", complete_snii_scoring_metadata("field-2", 3, 12), 3, true}}) + .ok()); + ASSERT_TRUE(admit_snii_fields_for_test( + stats_.get(), + {{L"1", complete_snii_scoring_metadata("field-1", 2, 5), 2, true}, + {L"2", complete_snii_scoring_metadata("field-2", 2, 8), 2, true}}) + .ok()); + + expect_collected_stats(L"1", 5, 12); + expect_collected_tokens(L"2", 20); + EXPECT_FLOAT_EQ(stats_->get_or_calculate_avg_dl(L"1"), 12.0F / 5.0F); + EXPECT_FLOAT_EQ(stats_->get_or_calculate_avg_dl(L"2"), 4.0F); +} + +TEST_F(CollectionStatisticsTest, MultiFieldSegmentDocCountsMustAgree) { + auto status = admit_snii_fields_for_test( + stats_.get(), {{L"1", complete_snii_scoring_metadata("field-1", 3, 7), 3, true}, + {L"2", complete_snii_scoring_metadata("field-2", 4, 12), 4, true}}); + + EXPECT_EQ(status.code(), ErrorCode::INVERTED_INDEX_NOT_SUPPORTED); + expect_no_collected_tokens(L"1"); + expect_no_collected_tokens(L"2"); + EXPECT_THROW(stats_->get_doc_num(), Exception); +} + +TEST_F(CollectionStatisticsTest, LaterFieldFileNotFoundDoesNotPublishPartialSegment) { + ASSERT_TRUE(admit_snii_segment_for_test(stats_.get(), L"1", + complete_snii_scoring_metadata("field-1", 2, 6), 2, + true) + .ok()); + + auto status = stage_snii_fields_then_file_not_found_for_test( + stats_.get(), + {{L"2", complete_snii_scoring_metadata("staged-field-2", 3, 12), 3, true}}); + + EXPECT_EQ(status.code(), ErrorCode::INVERTED_INDEX_FILE_NOT_FOUND); + expect_collected_stats(L"1", 2, 6); + expect_no_collected_tokens(L"2"); + expect_no_collected_term(L"2", L"staged"); + + ASSERT_TRUE(admit_snii_segment_for_test( + stats_.get(), L"2", + complete_snii_scoring_metadata("different-field-2", 1, 4), 1, true) + .ok()); + expect_collected_tokens(L"2", 4); +} + +TEST(CollectionStatisticsCommonGramsTest, DocFrequencySupportsLogicalAndPhysicalTermKeys) { + std::unordered_map> + logical_frequencies; + + add_term_doc_frequency(&logical_frequencies, L"field", L"logical", 3); + EXPECT_EQ(logical_frequencies[L"field"][L"logical"], 3); + + add_term_doc_frequency(&logical_frequencies, L"field", L"same", 5); + EXPECT_EQ(logical_frequencies[L"field"][L"same"], 5); +} + +TEST(CollectionStatisticsCommonGramsTest, PhysicalAliasesCannotMergeLogicalDocFrequencies) { + using Frequencies = + std::unordered_map>; + Frequencies logical_frequencies; + + const std::wstring alias = std::wstring(1, wchar_t {0x1e}) + L"G00000001:"; + add_term_doc_frequency(&logical_frequencies, L"field", L"\x1f", 3); + add_term_doc_frequency(&logical_frequencies, L"field", alias, 5); + + EXPECT_EQ(logical_frequencies[L"field"][alias], 5); +} + +TEST(CollectionStatisticsCommonGramsTest, UnrepresentablePlainTermRegistersZeroDocFrequency) { + using Frequencies = + std::unordered_map>; + Frequencies logical_frequencies; + + add_term_doc_frequency(&logical_frequencies, L"field", L"unrepresentable", 0); + + ASSERT_TRUE(logical_frequencies.contains(L"field")); + EXPECT_TRUE(logical_frequencies.at(L"field").contains(L"unrepresentable")); + EXPECT_EQ(logical_frequencies.at(L"field").at(L"unrepresentable"), 0); +} + +TEST_F(CollectionStatisticsDetailedTest, GetStatisticsWithValidData) { + std::wstring field_name = L"test_field"; + std::wstring term = L"test_term"; + + stats_->set_total_num_docs(1000); + stats_->set_total_num_tokens(field_name, 5000); + stats_->set_term_doc_freq(field_name, term, 100); + + EXPECT_EQ(stats_->get_doc_num(), 1000); + EXPECT_EQ(stats_->get_total_term_cnt_by_col(field_name), 5000); + EXPECT_EQ(stats_->get_term_doc_freq_by_col(field_name, term), 100); + + float expected_avg_dl = 5000.0f / 1000.0f; + EXPECT_FLOAT_EQ(stats_->get_or_calculate_avg_dl(field_name), expected_avg_dl); + + float expected_idf = std::log(1 + (1000 - 100 + 0.5) / (100 + 0.5)); + EXPECT_FLOAT_EQ(stats_->get_or_calculate_idf(field_name, term), expected_idf); +} + +TEST_F(CollectionStatisticsDetailedTest, GetStatisticsThrowsWhenDataNotExists) { + std::wstring nonexistent_field = L"nonexistent"; + std::wstring nonexistent_term = L"nonexistent"; + + // Test exceptions for missing data + EXPECT_THROW(stats_->get_doc_num(), Exception); + EXPECT_THROW(stats_->get_total_term_cnt_by_col(nonexistent_field), Exception); + EXPECT_THROW(stats_->get_term_doc_freq_by_col(nonexistent_field, nonexistent_term), Exception); + EXPECT_THROW(stats_->get_or_calculate_avg_dl(nonexistent_field), Exception); + EXPECT_THROW(stats_->get_or_calculate_idf(nonexistent_field, nonexistent_term), Exception); +} + +TEST_F(CollectionStatisticsDetailedTest, CachingMechanismWorks) { + std::wstring field_name = L"test_field"; + std::wstring term = L"test_term"; + + stats_->set_total_num_docs(1000); + stats_->set_total_num_tokens(field_name, 5000); + stats_->set_term_doc_freq(field_name, term, 100); + + float first_avg_dl = stats_->get_or_calculate_avg_dl(field_name); + float first_idf = stats_->get_or_calculate_idf(field_name, term); + + stats_->set_total_num_docs(2000); + stats_->set_total_num_tokens(field_name, 10000); + stats_->set_term_doc_freq(field_name, term, 200); + + float second_avg_dl = stats_->get_or_calculate_avg_dl(field_name); + float second_idf = stats_->get_or_calculate_idf(field_name, term); + + EXPECT_FLOAT_EQ(first_avg_dl, second_avg_dl); + EXPECT_FLOAT_EQ(first_idf, second_idf); +} + +TEST_F(CollectionStatisticsDetailedTest, HandlesZeroValuesCorrectly) { + std::wstring field_name = L"test_field"; + std::wstring term = L"test_term"; + + stats_->set_total_num_docs(0); + EXPECT_THROW(stats_->get_doc_num(), Exception); + + stats_->set_total_num_docs(100); + stats_->set_total_num_tokens(field_name, 0); + stats_->set_term_doc_freq(field_name, term, 0); + + EXPECT_EQ(stats_->get_total_term_cnt_by_col(field_name), 0); + EXPECT_EQ(stats_->get_term_doc_freq_by_col(field_name, term), 0); + EXPECT_FLOAT_EQ(stats_->get_or_calculate_avg_dl(field_name), 0.0f); +} + +TEST_F(CollectionStatisticsDetailedTest, IdfCalculationWithDifferentFrequencies) { + std::wstring field_name = L"test_field"; + std::wstring common_term = L"common_term"; + std::wstring rare_term = L"rare_term"; + + stats_->set_total_num_docs(1000); + stats_->set_term_doc_freq(field_name, common_term, 500); + stats_->set_term_doc_freq(field_name, rare_term, 10); + + float common_idf = stats_->get_or_calculate_idf(field_name, common_term); + float rare_idf = stats_->get_or_calculate_idf(field_name, rare_term); + + EXPECT_GT(rare_idf, common_idf); + EXPECT_GT(common_idf, 0); + EXPECT_GT(rare_idf, 0); +} + +TEST_F(CollectionStatisticsTest, CollectWithCastWrappedSlotRef) { + auto tablet_schema = create_tablet_schema_with_inverted_index(); + + VExprContextSPtrs contexts; + + // match_pred(left: CAST(slot_ref), right: literal) + auto match_expr = std::make_shared(TExprNodeType::MATCH_PRED); + auto cast_expr = std::make_shared(TExprNodeType::CAST_EXPR); + auto slot_ref = std::make_shared("content", SlotId(1)); + auto literal = std::make_shared("cast term"); + + cast_expr->_children.push_back(slot_ref); + match_expr->_children.push_back(cast_expr); + match_expr->_children.push_back(literal); + + contexts.push_back(std::make_shared(match_expr)); + + std::vector empty_splits; + auto status = + stats_->collect(runtime_state_.get(), empty_splits, tablet_schema, contexts, nullptr); + EXPECT_TRUE(status.ok()) << status.msg(); +} + +TEST_F(CollectionStatisticsTest, CollectWithDoubleCastWrappedSlotRef) { + auto tablet_schema = create_tablet_schema_with_inverted_index(); + + VExprContextSPtrs contexts; + + // match_pred(left: CAST(CAST(slot_ref)), right: literal) + auto match_expr = std::make_shared(TExprNodeType::MATCH_PRED); + auto outer_cast = std::make_shared(TExprNodeType::CAST_EXPR); + auto inner_cast = std::make_shared(TExprNodeType::CAST_EXPR); + auto slot_ref = std::make_shared("content", SlotId(1)); + auto literal = std::make_shared("double cast term"); + + inner_cast->_children.push_back(slot_ref); + outer_cast->_children.push_back(inner_cast); + match_expr->_children.push_back(outer_cast); + match_expr->_children.push_back(literal); + + contexts.push_back(std::make_shared(match_expr)); + + std::vector empty_splits; + auto status = + stats_->collect(runtime_state_.get(), empty_splits, tablet_schema, contexts, nullptr); + EXPECT_TRUE(status.ok()) << status.msg(); +} + +// Regression for AIR-36: match score collection must resolve indexes for +// variant sub-columns whose indexes live in _path_set_info_map (typed paths or +// inherited sub-column indexes). The previous simple lookup using +// inverted_indexs(col_unique_id, suffix_path) missed those indexes. +TEST_F(CollectionStatisticsTest, ExtractCollectInfoForVariantSubcolumnIndex) { + auto tablet_schema = std::make_shared(); + + constexpr int32_t kVariantUid = 9001; + + TabletColumn variant_col; + variant_col.set_unique_id(kVariantUid); + variant_col.set_name("v"); + variant_col.set_type(FieldType::OLAP_FIELD_TYPE_VARIANT); + tablet_schema->append_column(variant_col); + + TabletColumn sub_col; + sub_col.set_unique_id(-1); + sub_col.set_name("v.host"); + sub_col.set_type(FieldType::OLAP_FIELD_TYPE_STRING); + sub_col.set_parent_unique_id(kVariantUid); + PathInData path("v.host"); + sub_col.set_path_info(path); + tablet_schema->append_column(sub_col); + + auto sub_index = std::make_shared(); + TabletIndexPB index_pb; + index_pb.set_index_id(2001); + index_pb.set_index_name("variant_subcolumn_idx"); + index_pb.set_index_type(IndexType::INVERTED); + index_pb.add_col_unique_id(kVariantUid); + auto* props = index_pb.mutable_properties(); + (*props)["parser"] = "standard"; + (*props)["support_phrase"] = "true"; + sub_index->init_from_pb(index_pb); + + TabletSchema::PathsSetInfo path_set_info; + TabletIndexes sub_indexes = {sub_index}; + path_set_info.subcolumn_indexes["host"] = sub_indexes; + std::unordered_map path_set_info_map; + path_set_info_map[kVariantUid] = std::move(path_set_info); + tablet_schema->set_path_set_info(std::move(path_set_info_map)); + + EXPECT_TRUE(tablet_schema->inverted_indexs(kVariantUid, "host").empty()); + + auto found = tablet_schema->inverted_indexs(tablet_schema->column(/*ordinal=*/1)); + ASSERT_EQ(found.size(), 1u); + EXPECT_EQ(found[0]->index_name(), "variant_subcolumn_idx"); + + constexpr int kSlotId = 42; + runtime_state_->_mock_desc_tbl->add_slot_descriptor(SlotId(kSlotId), kVariantUid); + + auto match_expr = std::make_shared(TExprNodeType::MATCH_PRED); + auto slot_ref = + std::make_shared("v.host", SlotId(kSlotId)); + auto literal = std::make_shared("foo"); + match_expr->_children.push_back(slot_ref); + match_expr->_children.push_back(literal); + + VExprContextSPtrs contexts; + contexts.push_back(std::make_shared(match_expr)); + + std::unordered_map collect_infos; + auto status = stats_->extract_collect_info(runtime_state_.get(), contexts, tablet_schema, + &collect_infos); + ASSERT_TRUE(status.ok()) << status.msg(); + ASSERT_EQ(collect_infos.size(), 1u); + auto it = collect_infos.find(StringHelper::to_wstring(std::to_string(kVariantUid) + ".v.host")); + ASSERT_NE(it, collect_infos.end()); + ASSERT_NE(it->second.index_meta, nullptr); + EXPECT_EQ(it->second.index_meta->index_name(), "variant_subcolumn_idx"); +} + +TEST_F(CollectionStatisticsTest, MatchScoringUsesTextSemanticsForVariantParentIndexFallback) { + auto tablet_schema = std::make_shared(); + + constexpr int32_t kVariantUid = 9004; + + TabletColumn variant_col; + variant_col.set_unique_id(kVariantUid); + variant_col.set_name("v"); + variant_col.set_type(FieldType::OLAP_FIELD_TYPE_VARIANT); + tablet_schema->append_column(variant_col); + + TabletColumn sub_col; + sub_col.set_unique_id(-1); + sub_col.set_name("v.key"); + sub_col.set_type(FieldType::OLAP_FIELD_TYPE_VARIANT); + sub_col.set_parent_unique_id(kVariantUid); + PathInData path("v.key"); + sub_col.set_path_info(path); + tablet_schema->append_column(sub_col); + + TabletIndexPB index_pb; + index_pb.set_index_id(2004); + index_pb.set_index_name("variant_parent_idx"); + index_pb.set_index_type(IndexType::INVERTED); + index_pb.add_col_unique_id(kVariantUid); + auto* props = index_pb.mutable_properties(); + (*props)["parser"] = "english"; + (*props)["support_phrase"] = "true"; + + TabletIndex index; + index.init_from_pb(index_pb); + tablet_schema->append_index(std::move(index)); + + // Pre-conditions: column-aware lookup is empty (no inheritance pre-populated) + // and generate_sub_column_info returns false (no field_pattern template). + // The collector must still resolve through the VARIANT-placeholder branch. + ASSERT_TRUE(tablet_schema->inverted_indexs(tablet_schema->column(/*ordinal=*/1)).empty()); + ASSERT_EQ(tablet_schema->inverted_indexs(kVariantUid).size(), 1u); + TabletSchema::SubColumnInfo sub_column_info; + ASSERT_FALSE(variant_util::generate_sub_column_info(*tablet_schema, kVariantUid, "key", + &sub_column_info)); + + constexpr int kSlotId = 45; + runtime_state_->_mock_desc_tbl->add_slot_descriptor(SlotId(kSlotId), kVariantUid, "v.key", + {"key"}); + + auto match_expr = std::make_shared(TExprNodeType::MATCH_PRED); + auto cast_expr = std::make_shared(TExprNodeType::CAST_EXPR); + cast_expr->_data_type = std::make_shared(); + auto slot_ref = std::make_shared("v.key", SlotId(kSlotId)); + auto literal = std::make_shared("abc"); + cast_expr->_children.push_back(slot_ref); + match_expr->_children.push_back(cast_expr); + match_expr->_children.push_back(literal); + + VExprContextSPtrs contexts; + contexts.push_back(std::make_shared(match_expr)); + + std::unordered_map collect_infos; + auto status = stats_->extract_collect_info(runtime_state_.get(), contexts, tablet_schema, + &collect_infos); + ASSERT_TRUE(status.ok()) << status; + ASSERT_EQ(collect_infos.size(), 1U); + auto it = collect_infos.find(StringHelper::to_wstring(std::to_string(kVariantUid) + ".v.key")); + ASSERT_NE(it, collect_infos.end()); + ASSERT_NE(it->second.index_meta, nullptr); + ASSERT_NE(it->second.owned_index_meta, nullptr); + EXPECT_EQ(it->second.index_meta->index_id(), 2004); + EXPECT_EQ(it->second.unique_terms, std::vector({"abc"})); +} + +namespace { + +// Build a sub-column template for the parent variant column. pattern_type has no +// public setter on TabletColumn, so construct through ColumnPB. +TabletColumn make_subcolumn_template(const std::string& pattern, PatternTypePB pattern_type) { + ColumnPB column_pb; + column_pb.set_unique_id(-1); + column_pb.set_name(pattern); + column_pb.set_type("STRING"); + column_pb.set_is_nullable(true); + column_pb.set_pattern_type(pattern_type); + + TabletColumn templ; + templ.init_from_pb(column_pb); + return templ; +} + +} // namespace + +TEST_F(CollectionStatisticsTest, ExtractCollectInfoForVariantFieldPatternIndex) { + auto tablet_schema = std::make_shared(); + + constexpr int32_t kVariantUid = 9002; + + TabletColumn variant_col; + variant_col.set_unique_id(kVariantUid); + variant_col.set_name("meta"); + variant_col.set_type(FieldType::OLAP_FIELD_TYPE_VARIANT); + TabletColumn host_template = make_subcolumn_template("host", PatternTypePB::MATCH_NAME); + variant_col.add_sub_column(host_template); + tablet_schema->append_column(variant_col); + + TabletColumn sub_col; + sub_col.set_unique_id(-1); + sub_col.set_name("meta.host"); + sub_col.set_type(FieldType::OLAP_FIELD_TYPE_STRING); + sub_col.set_parent_unique_id(kVariantUid); + PathInData path("meta.host"); + sub_col.set_path_info(path); + tablet_schema->append_column(sub_col); + + TabletIndexPB index_pb; + index_pb.set_index_id(2002); + index_pb.set_index_name("variant_field_pattern_idx"); + index_pb.set_index_type(IndexType::INVERTED); + index_pb.add_col_unique_id(kVariantUid); + auto* props = index_pb.mutable_properties(); + (*props)["parser"] = "standard"; + (*props)["support_phrase"] = "true"; + (*props)["field_pattern"] = "host"; + + TabletIndex index; + index.init_from_pb(index_pb); + tablet_schema->append_index(std::move(index)); + + ASSERT_TRUE(tablet_schema->inverted_indexs(tablet_schema->column(/*ordinal=*/1)).empty()); + ASSERT_EQ(tablet_schema->inverted_index_by_field_pattern(kVariantUid, "host").size(), 1u); + + constexpr int kSlotId = 43; + runtime_state_->_mock_desc_tbl->add_slot_descriptor(SlotId(kSlotId), kVariantUid, "meta.host", + {"host"}); + + auto match_expr = std::make_shared(TExprNodeType::MATCH_PRED); + auto slot_ref = + std::make_shared("meta.host", SlotId(kSlotId)); + auto literal = std::make_shared("alpha"); + match_expr->_children.push_back(slot_ref); + match_expr->_children.push_back(literal); + + VExprContextSPtrs contexts; + contexts.push_back(std::make_shared(match_expr)); + + std::unordered_map collect_infos; + auto status = stats_->extract_collect_info(runtime_state_.get(), contexts, tablet_schema, + &collect_infos); + ASSERT_TRUE(status.ok()) << status.msg(); + ASSERT_EQ(collect_infos.size(), 1u); + auto it = collect_infos.find( + StringHelper::to_wstring(std::to_string(kVariantUid) + ".meta.host")); + ASSERT_NE(it, collect_infos.end()); + ASSERT_NE(it->second.index_meta, nullptr); + ASSERT_NE(it->second.owned_index_meta, nullptr); + EXPECT_EQ(it->second.index_meta->index_name(), "variant_field_pattern_idx"); +} + +// Regression: field_pattern="user.*" is registered under the pattern string, +// while the query slot resolves to column_paths=["user", "name"]. The fallback +// must match the parent variant's sub-column template first, then use the +// matched pattern to fetch the index, and collect under the actual Lucene field. +TEST_F(CollectionStatisticsTest, ExtractCollectInfoForVariantFieldPatternGlobIndex) { + auto tablet_schema = std::make_shared(); + + constexpr int32_t kVariantUid = 9003; + + TabletColumn variant_col; + variant_col.set_unique_id(kVariantUid); + variant_col.set_name("meta"); + variant_col.set_type(FieldType::OLAP_FIELD_TYPE_VARIANT); + TabletColumn glob_template = make_subcolumn_template("user.*", PatternTypePB::MATCH_NAME_GLOB); + variant_col.add_sub_column(glob_template); + tablet_schema->append_column(variant_col); + + TabletColumn sub_col; + sub_col.set_unique_id(-1); + sub_col.set_name("meta.user.name"); + sub_col.set_type(FieldType::OLAP_FIELD_TYPE_STRING); + sub_col.set_parent_unique_id(kVariantUid); + PathInData path("meta.user.name"); + sub_col.set_path_info(path); + tablet_schema->append_column(sub_col); + + TabletIndexPB index_pb; + index_pb.set_index_id(2003); + index_pb.set_index_name("variant_field_pattern_glob_idx"); + index_pb.set_index_type(IndexType::INVERTED); + index_pb.add_col_unique_id(kVariantUid); + auto* props = index_pb.mutable_properties(); + (*props)["parser"] = "standard"; + (*props)["support_phrase"] = "true"; + (*props)["field_pattern"] = "user.*"; + + TabletIndex index; + index.init_from_pb(index_pb); + tablet_schema->append_index(std::move(index)); + + ASSERT_TRUE(tablet_schema->inverted_indexs(tablet_schema->column(/*ordinal=*/1)).empty()); + ASSERT_TRUE(tablet_schema->inverted_index_by_field_pattern(kVariantUid, "user.name").empty()); + ASSERT_EQ(tablet_schema->inverted_index_by_field_pattern(kVariantUid, "user.*").size(), 1u); + TabletSchema::SubColumnInfo sub_column_info; + ASSERT_TRUE(variant_util::generate_sub_column_info(*tablet_schema, kVariantUid, "user.name", + &sub_column_info)); + ASSERT_EQ(sub_column_info.indexes.size(), 1u); + EXPECT_EQ(sub_column_info.column.suffix_path(), "meta.user.name"); + EXPECT_EQ(sub_column_info.indexes[0]->index_name(), "variant_field_pattern_glob_idx"); + + constexpr int kSlotId = 44; + runtime_state_->_mock_desc_tbl->add_slot_descriptor(SlotId(kSlotId), kVariantUid, + "meta.user.name", {"user", "name"}); + + auto match_expr = std::make_shared(TExprNodeType::MATCH_PRED); + auto slot_ref = std::make_shared("meta.user.name", + SlotId(kSlotId)); + auto literal = std::make_shared("alice"); + match_expr->_children.push_back(slot_ref); + match_expr->_children.push_back(literal); + + VExprContextSPtrs contexts; + contexts.push_back(std::make_shared(match_expr)); + + std::unordered_map collect_infos; + auto status = stats_->extract_collect_info(runtime_state_.get(), contexts, tablet_schema, + &collect_infos); + ASSERT_TRUE(status.ok()) << status.msg(); + ASSERT_EQ(collect_infos.size(), 1u); + auto it = collect_infos.find( + StringHelper::to_wstring(std::to_string(kVariantUid) + ".meta.user.name")); + ASSERT_NE(it, collect_infos.end()); + ASSERT_NE(it->second.index_meta, nullptr); + ASSERT_NE(it->second.owned_index_meta, nullptr); + EXPECT_EQ(it->second.index_meta->index_name(), "variant_field_pattern_glob_idx"); +} + +// E1: Match predicate whose left subtree contains no VSlotRef. +// find_slot_ref recurses through children; when it returns nullptr the +// collector reports INVERTED_INDEX_NOT_SUPPORTED. +// Calls MatchPredicateCollector::collect() directly so coverage attribution +// is not muddied by extract_collect_info's virtual-dispatch indirection. +TEST_F(CollectionStatisticsTest, CollectMissingSlotRefReturnsError) { + auto tablet_schema = std::make_shared(); + TabletColumn col; + col.set_unique_id(1001); + col.set_name("c"); + col.set_type(FieldType::OLAP_FIELD_TYPE_STRING); + tablet_schema->append_column(col); + + auto match_expr = std::make_shared(TExprNodeType::MATCH_PRED); + auto literal_left = std::make_shared("foo"); + auto literal_right = std::make_shared("bar"); + match_expr->_children.push_back(literal_left); + match_expr->_children.push_back(literal_right); + + MatchPredicateCollector collector; + std::unordered_map collect_infos; + auto status = + collector.collect(runtime_state_.get(), tablet_schema, match_expr, &collect_infos); + ASSERT_FALSE(status.ok()); + EXPECT_EQ(status.code(), ErrorCode::INVERTED_INDEX_NOT_SUPPORTED); + EXPECT_TRUE(status.msg().find("Cannot find slot reference") != std::string::npos); +} + +// E2: SlotRef points to a slot_id absent from the runtime descriptor table. +TEST_F(CollectionStatisticsTest, CollectMissingSlotDescriptorReturnsError) { + auto tablet_schema = std::make_shared(); + TabletColumn col; + col.set_unique_id(1002); + col.set_name("c"); + col.set_type(FieldType::OLAP_FIELD_TYPE_STRING); + tablet_schema->append_column(col); + + constexpr int kAbsentSlotId = 99999; + + auto match_expr = std::make_shared(TExprNodeType::MATCH_PRED); + auto slot_ref = + std::make_shared("c", SlotId(kAbsentSlotId)); + auto literal = std::make_shared("v"); + match_expr->_children.push_back(slot_ref); + match_expr->_children.push_back(literal); + + MatchPredicateCollector collector; + std::unordered_map collect_infos; + auto status = + collector.collect(runtime_state_.get(), tablet_schema, match_expr, &collect_infos); + ASSERT_FALSE(status.ok()); + EXPECT_EQ(status.code(), ErrorCode::INVERTED_INDEX_NOT_SUPPORTED); + EXPECT_TRUE(status.msg().find("Cannot find slot descriptor") != std::string::npos); +} + +// E3: SlotRef name does not exist in tablet_schema (field_index returns -1). +TEST_F(CollectionStatisticsTest, CollectUnknownColumnNameReturnsError) { + auto tablet_schema = std::make_shared(); + TabletColumn col; + col.set_unique_id(1003); + col.set_name("declared"); + col.set_type(FieldType::OLAP_FIELD_TYPE_STRING); + tablet_schema->append_column(col); + + constexpr int kSlotId = 50; + runtime_state_->_mock_desc_tbl->add_slot_descriptor(SlotId(kSlotId), 1003, "missing", {}); + + auto match_expr = std::make_shared(TExprNodeType::MATCH_PRED); + auto slot_ref = + std::make_shared("missing", SlotId(kSlotId)); + auto literal = std::make_shared("v"); + match_expr->_children.push_back(slot_ref); + match_expr->_children.push_back(literal); + + MatchPredicateCollector collector; + std::unordered_map collect_infos; + auto status = + collector.collect(runtime_state_.get(), tablet_schema, match_expr, &collect_infos); + ASSERT_FALSE(status.ok()); + EXPECT_EQ(status.code(), ErrorCode::INVERTED_INDEX_NOT_SUPPORTED); + EXPECT_TRUE(status.msg().find("Cannot find column index") != std::string::npos); +} + +// I1 + L3 + O1: Plain string column with a direct inverted index. +// Direct hit produces a CollectInfo whose owned_index_meta is null +// (the meta lives in the schema and is not cloned). +TEST_F(CollectionStatisticsTest, CollectDirectIndexHitFromSchema) { + auto tablet_schema = std::make_shared(); + + constexpr int32_t kColUid = 1100; + TabletColumn col; + col.set_unique_id(kColUid); + col.set_name("note"); + col.set_type(FieldType::OLAP_FIELD_TYPE_STRING); + tablet_schema->append_column(col); + + TabletIndexPB index_pb; + index_pb.set_index_id(2100); + index_pb.set_index_name("note_idx"); + index_pb.set_index_type(IndexType::INVERTED); + index_pb.add_col_unique_id(kColUid); + auto* props = index_pb.mutable_properties(); + (*props)["parser"] = "english"; + (*props)["support_phrase"] = "true"; + TabletIndex index; + index.init_from_pb(index_pb); + tablet_schema->append_index(std::move(index)); + + constexpr int kSlotId = 60; + runtime_state_->_mock_desc_tbl->add_slot_descriptor(SlotId(kSlotId), kColUid, "note", {}); + + auto match_expr = std::make_shared(TExprNodeType::MATCH_PRED); + auto slot_ref = std::make_shared("note", SlotId(kSlotId)); + auto literal = std::make_shared("hello world"); + match_expr->_children.push_back(slot_ref); + match_expr->_children.push_back(literal); + + MatchPredicateCollector collector; + std::unordered_map collect_infos; + auto status = + collector.collect(runtime_state_.get(), tablet_schema, match_expr, &collect_infos); + ASSERT_TRUE(status.ok()) << status.msg(); + ASSERT_EQ(collect_infos.size(), 1u); + auto it = collect_infos.find(StringHelper::to_wstring(std::to_string(kColUid))); + ASSERT_NE(it, collect_infos.end()); + EXPECT_NE(it->second.index_meta, nullptr); + EXPECT_EQ(it->second.owned_index_meta, nullptr); // O1: schema-direct meta is not owned + EXPECT_FALSE(it->second.unique_terms.empty()); +} + +// I2: Plain string column with no index and not an extracted variant +// sub-column. Fallback path does not apply (column.is_extracted_column() +// is false). In BE_TEST builds the empty-index check is skipped, so +// collect returns OK with no CollectInfo emitted. +TEST_F(CollectionStatisticsTest, CollectNotExtractedColumnSkipsFallback) { + auto tablet_schema = std::make_shared(); + + constexpr int32_t kColUid = 1200; + TabletColumn col; + col.set_unique_id(kColUid); + col.set_name("plain"); + col.set_type(FieldType::OLAP_FIELD_TYPE_STRING); + tablet_schema->append_column(col); + // no index appended + + constexpr int kSlotId = 70; + runtime_state_->_mock_desc_tbl->add_slot_descriptor(SlotId(kSlotId), kColUid, "plain", {}); + + auto match_expr = std::make_shared(TExprNodeType::MATCH_PRED); + auto slot_ref = std::make_shared("plain", SlotId(kSlotId)); + auto literal = std::make_shared("v"); + match_expr->_children.push_back(slot_ref); + match_expr->_children.push_back(literal); + + MatchPredicateCollector collector; + std::unordered_map collect_infos; + auto status = + collector.collect(runtime_state_.get(), tablet_schema, match_expr, &collect_infos); + ASSERT_TRUE(status.ok()) << status.msg(); + EXPECT_TRUE(collect_infos.empty()); +} + +// L1: Index whose properties do not request an analyzer +// (should_analyzer returns false). The matching index_meta is iterated +// but skipped before insertion. +TEST_F(CollectionStatisticsTest, CollectSkipsIndexWithoutAnalyzer) { + auto tablet_schema = std::make_shared(); + + constexpr int32_t kColUid = 1300; + TabletColumn col; + col.set_unique_id(kColUid); + col.set_name("kw"); + col.set_type(FieldType::OLAP_FIELD_TYPE_STRING); + tablet_schema->append_column(col); + + TabletIndexPB index_pb; + index_pb.set_index_id(2300); + index_pb.set_index_name("kw_idx"); + index_pb.set_index_type(IndexType::INVERTED); + index_pb.add_col_unique_id(kColUid); + // No "parser" property -> should_analyzer returns false + TabletIndex index; + index.init_from_pb(index_pb); + tablet_schema->append_index(std::move(index)); + + constexpr int kSlotId = 80; + runtime_state_->_mock_desc_tbl->add_slot_descriptor(SlotId(kSlotId), kColUid, "kw", {}); + + auto match_expr = std::make_shared(TExprNodeType::MATCH_PRED); + auto slot_ref = std::make_shared("kw", SlotId(kSlotId)); + auto literal = std::make_shared("v"); + match_expr->_children.push_back(slot_ref); + match_expr->_children.push_back(literal); + auto analyzer_ctx = std::make_shared(); + analyzer_ctx->analyzer_key = "none"; + analyzer_ctx->parser_type = InvertedIndexParserType::PARSER_NONE; + match_expr->set_analyzer_ctx(std::move(analyzer_ctx)); + + MatchPredicateCollector collector; + std::unordered_map collect_infos; + auto status = + collector.collect(runtime_state_.get(), tablet_schema, match_expr, &collect_infos); + ASSERT_TRUE(status.ok()) << status.msg(); + EXPECT_TRUE(collect_infos.empty()); +} + +TEST_F(CollectionStatisticsTest, ExplicitNoneDoesNotSelectNormalizerIndexForScoring) { + auto tablet_schema = std::make_shared(); + + constexpr int32_t kColUid = 1325; + TabletColumn col; + col.set_unique_id(kColUid); + col.set_name("normalized"); + col.set_type(FieldType::OLAP_FIELD_TYPE_STRING); + tablet_schema->append_column(col); + + TabletIndexPB index_pb; + index_pb.set_index_id(2325); + index_pb.set_index_name("normalized_idx"); + index_pb.set_index_type(IndexType::INVERTED); + index_pb.add_col_unique_id(kColUid); + auto* props = index_pb.mutable_properties(); + (*props)[INVERTED_INDEX_NORMALIZER_NAME_KEY] = "lowercase"; + (*props)[INVERTED_INDEX_PARSER_PHRASE_SUPPORT_KEY] = "true"; + TabletIndex index; + index.init_from_pb(index_pb); + tablet_schema->append_index(std::move(index)); + + constexpr int kSlotId = 82; + runtime_state_->_mock_desc_tbl->add_slot_descriptor(SlotId(kSlotId), kColUid, "normalized", {}); + + auto match_expr = std::make_shared(TExprNodeType::MATCH_PRED); + match_expr->_children.push_back( + std::make_shared("normalized", SlotId(kSlotId))); + match_expr->_children.push_back(std::make_shared("ABC")); + auto analyzer_ctx = std::make_shared(); + analyzer_ctx->analyzer_key = "none"; + analyzer_ctx->parser_type = InvertedIndexParserType::PARSER_NONE; + match_expr->set_analyzer_ctx(std::move(analyzer_ctx)); + + MatchPredicateCollector collector; + std::unordered_map collect_infos; + auto status = + collector.collect(runtime_state_.get(), tablet_schema, match_expr, &collect_infos); + + EXPECT_FALSE(status.ok()); + EXPECT_NE(status.msg().find("No inverted index found for analyzer 'none'"), std::string::npos) + << status; + EXPECT_TRUE(collect_infos.empty()); +} + +// L2: Index whose analyzer is set (should_analyzer returns true) but does +// not declare "support_phrase=true". MockVExpr drives MATCH_PHRASE opcode, +// so is_need_similarity_score returns false and the index is skipped. +TEST_F(CollectionStatisticsTest, CollectSkipsIndexWithoutSimilarityScore) { + auto tablet_schema = std::make_shared(); + + constexpr int32_t kColUid = 1350; + TabletColumn col; + col.set_unique_id(kColUid); + col.set_name("body"); + col.set_type(FieldType::OLAP_FIELD_TYPE_STRING); + tablet_schema->append_column(col); + + TabletIndexPB index_pb; + index_pb.set_index_id(2350); + index_pb.set_index_name("body_idx"); + index_pb.set_index_type(IndexType::INVERTED); + index_pb.add_col_unique_id(kColUid); + auto* props = index_pb.mutable_properties(); + (*props)["parser"] = "english"; // should_analyzer == true + // Intentionally omit "support_phrase" -> is_need_similarity_score == false + TabletIndex index; + index.init_from_pb(index_pb); + tablet_schema->append_index(std::move(index)); + + constexpr int kSlotId = 85; + runtime_state_->_mock_desc_tbl->add_slot_descriptor(SlotId(kSlotId), kColUid, "body", {}); + + auto match_expr = std::make_shared(TExprNodeType::MATCH_PRED); + auto slot_ref = std::make_shared("body", SlotId(kSlotId)); + auto literal = std::make_shared("hello"); + match_expr->_children.push_back(slot_ref); + match_expr->_children.push_back(literal); + + MatchPredicateCollector collector; + std::unordered_map collect_infos; + auto status = + collector.collect(runtime_state_.get(), tablet_schema, match_expr, &collect_infos); + ASSERT_TRUE(status.ok()) << status.msg(); + EXPECT_TRUE(collect_infos.empty()); +} + +TEST_F(CollectionStatisticsTest, CollectPreservesLogicalClauseShapesForSameFieldName) { + auto tablet_schema = std::make_shared(); + + constexpr int32_t kColUid = 1400; + TabletColumn col; + col.set_unique_id(kColUid); + col.set_name("doc"); + col.set_type(FieldType::OLAP_FIELD_TYPE_STRING); + tablet_schema->append_column(col); + + TabletIndexPB index_pb; + index_pb.set_index_id(2400); + index_pb.set_index_name("doc_idx"); + index_pb.set_index_type(IndexType::INVERTED); + index_pb.add_col_unique_id(kColUid); + auto* props = index_pb.mutable_properties(); + (*props)["parser"] = "english"; + (*props)["support_phrase"] = "true"; + TabletIndex index; + index.init_from_pb(index_pb); + tablet_schema->append_index(std::move(index)); + + constexpr int kSlotId = 90; + runtime_state_->_mock_desc_tbl->add_slot_descriptor(SlotId(kSlotId), kColUid, "doc", {}); + + auto build_match = [&](const std::string& term) { + auto m = std::make_shared(TExprNodeType::MATCH_PRED); + auto s = std::make_shared("doc", SlotId(kSlotId)); + auto l = std::make_shared(term); + m->_children.push_back(s); + m->_children.push_back(l); + return m; + }; + + MatchPredicateCollector collector; + std::unordered_map collect_infos; + auto first = collector.collect(runtime_state_.get(), tablet_schema, build_match("alpha beta"), + &collect_infos); + ASSERT_TRUE(first.ok()) << first.msg(); + auto second = collector.collect(runtime_state_.get(), tablet_schema, + build_match("alpha alpha beta"), &collect_infos); + ASSERT_TRUE(second.ok()) << second.msg(); + ASSERT_EQ(collect_infos.size(), 1u); + auto it = collect_infos.find(StringHelper::to_wstring(std::to_string(kColUid))); + ASSERT_NE(it, collect_infos.end()); + ASSERT_EQ(it->second.unique_terms, std::vector({"alpha", "beta"})); + ASSERT_EQ(it->second.unique_term_slots.size(), 2u); + EXPECT_EQ(it->second.unique_term_slots.at("alpha"), 0u); + EXPECT_EQ(it->second.unique_term_slots.at("beta"), 1u); + ASSERT_EQ(it->second.logical_scoring_leaves.size(), 2u); + ASSERT_EQ(it->second.logical_scoring_leaves[0].clauses.size(), 2u); + EXPECT_EQ(it->second.logical_scoring_leaves[0].clauses[0].df_slot, 0u); + EXPECT_EQ(it->second.logical_scoring_leaves[0].clauses[0].position, 1); + EXPECT_EQ(it->second.logical_scoring_leaves[0].clauses[1].df_slot, 1u); + EXPECT_EQ(it->second.logical_scoring_leaves[0].clauses[1].position, 2); + ASSERT_EQ(it->second.logical_scoring_leaves[1].clauses.size(), 3u); + EXPECT_EQ(it->second.logical_scoring_leaves[1].clauses[0].df_slot, 0u); + EXPECT_EQ(it->second.logical_scoring_leaves[1].clauses[0].position, 1); + EXPECT_EQ(it->second.logical_scoring_leaves[1].clauses[1].df_slot, 0u); + EXPECT_EQ(it->second.logical_scoring_leaves[1].clauses[1].position, 2); + EXPECT_EQ(it->second.logical_scoring_leaves[1].clauses[2].df_slot, 1u); + EXPECT_EQ(it->second.logical_scoring_leaves[1].clauses[2].position, 3); +} + +TEST_F(CollectionStatisticsTest, CollectUsesMatchRequestAnalyzerProviderAndFingerprint) { + auto tablet_schema = create_tablet_schema_with_inverted_index(); + + auto analyzer = segment_v2::inverted_index::InvertedIndexAnalyzer::create_builtin_analyzer( + InvertedIndexParserType::PARSER_ENGLISH, "", INVERTED_INDEX_PARSER_FALSE, "none"); + auto provider = std::make_shared( + std::move(analyzer), "request-base-v1"); + auto analyzer_ctx = std::make_shared(); + analyzer_ctx->analyzer_provider = provider; + + auto match_expr = std::make_shared(TExprNodeType::MATCH_PRED); + match_expr->set_analyzer_ctx(std::move(analyzer_ctx)); + auto slot_ref = std::make_shared("content", SlotId(1)); + auto literal = std::make_shared("Alpha ALPHA"); + match_expr->_children.push_back(slot_ref); + match_expr->_children.push_back(literal); + + MatchPredicateCollector collector; + CollectInfoMap collect_infos; + auto status = + collector.collect(runtime_state_.get(), tablet_schema, match_expr, &collect_infos); + + ASSERT_TRUE(status.ok()) << status.msg(); + ASSERT_EQ(collect_infos.size(), 1u); + const auto& collect_info = collect_infos.begin()->second; + EXPECT_EQ(collect_info.expected_base_analyzer_fingerprint, "request-base-v1"); + ASSERT_EQ(collect_info.unique_terms, std::vector({"Alpha", "ALPHA"})); + ASSERT_EQ(collect_info.logical_scoring_leaves.size(), 1u); + ASSERT_EQ(collect_info.logical_scoring_leaves[0].clauses.size(), 2u); + EXPECT_EQ(collect_info.logical_scoring_leaves[0].clauses[0].df_slot, 0u); + EXPECT_EQ(collect_info.logical_scoring_leaves[0].clauses[1].df_slot, 1u); +} + +TEST_F(CollectionStatisticsTest, CollectPhrasePrefixExcludesScoringTail) { + auto tablet_schema = create_tablet_schema_with_inverted_index(); + auto match_expr = std::make_shared(TExprNodeType::MATCH_PRED); + match_expr->set_opcode(TExprOpcode::MATCH_PHRASE_PREFIX); + match_expr->_children.push_back( + std::make_shared("content", SlotId(1))); + match_expr->_children.push_back( + std::make_shared("alpha beta gamma")); + + MatchPredicateCollector collector; + CollectInfoMap collect_infos; + auto status = + collector.collect(runtime_state_.get(), tablet_schema, match_expr, &collect_infos); + + ASSERT_TRUE(status.ok()) << status.msg(); + ASSERT_EQ(collect_infos.size(), 1u); + const auto& collect_info = collect_infos.begin()->second; + EXPECT_EQ(collect_info.unique_terms, std::vector({"alpha", "beta"})); + ASSERT_EQ(collect_info.logical_scoring_leaves.size(), 1u); + ASSERT_EQ(collect_info.logical_scoring_leaves[0].clauses.size(), 2u); + EXPECT_EQ(collect_info.logical_scoring_leaves[0].clauses[0].position, 1); + EXPECT_EQ(collect_info.logical_scoring_leaves[0].clauses[1].position, 2); +} + +TEST_F(CollectionStatisticsTest, CollectSingleTermPhrasePrefixHasEmptyScoringLeaf) { + auto tablet_schema = create_tablet_schema_with_inverted_index(); + auto match_expr = std::make_shared(TExprNodeType::MATCH_PRED); + match_expr->set_opcode(TExprOpcode::MATCH_PHRASE_PREFIX); + match_expr->_children.push_back( + std::make_shared("content", SlotId(1))); + match_expr->_children.push_back(std::make_shared("alpha")); + + MatchPredicateCollector collector; + CollectInfoMap collect_infos; + auto status = + collector.collect(runtime_state_.get(), tablet_schema, match_expr, &collect_infos); + + ASSERT_TRUE(status.ok()) << status.msg(); + ASSERT_EQ(collect_infos.size(), 1u); + const auto& collect_info = collect_infos.begin()->second; + EXPECT_TRUE(collect_info.unique_terms.empty()); + ASSERT_EQ(collect_info.logical_scoring_leaves.size(), 1u); + EXPECT_TRUE(collect_info.logical_scoring_leaves[0].clauses.empty()); +} + +TEST_F(CollectionStatisticsTest, SearchMatchCollectsRawExecutionTerm) { + auto tablet_schema = create_tablet_schema_with_inverted_index(); + auto contexts = create_search_contexts("MATCH", "alpha beta"); + CollectInfoMap collect_infos; + + auto status = stats_->extract_collect_info(runtime_state_.get(), contexts, tablet_schema, + &collect_infos); + + ASSERT_TRUE(status.ok()) << status.msg(); + ASSERT_EQ(collect_infos.size(), 1u); + const auto& collect_info = collect_infos.begin()->second; + EXPECT_EQ(collect_info.unique_terms, std::vector({"alpha beta"})); + ASSERT_EQ(collect_info.logical_scoring_leaves.size(), 1u); + ASSERT_EQ(collect_info.logical_scoring_leaves[0].clauses.size(), 1u); + EXPECT_EQ(collect_info.logical_scoring_leaves[0].clauses[0].df_slot, 0u); +} + +TEST_F(CollectionStatisticsTest, MatchSelectsOnlyTheRuntimeAnalyzerIndex) { + auto tablet_schema = create_tablet_schema_with_two_fulltext_indexes(); + auto match_expr = std::make_shared(TExprNodeType::MATCH_PRED); + match_expr->_children.push_back( + std::make_shared("content", SlotId(1))); + match_expr->_children.push_back( + std::make_shared("running quickly")); + + InvertedIndexAnalyzerConfig config; + config.analyzer_name = "english"; + config.parser_type = InvertedIndexParserType::PARSER_ENGLISH; + config.stop_words = "none"; + auto analyzer_ctx = std::make_shared(); + analyzer_ctx->analyzer_key = "english"; + analyzer_ctx->parser_type = InvertedIndexParserType::PARSER_ENGLISH; + analyzer_ctx->analyzer_provider = + segment_v2::inverted_index::InvertedIndexAnalyzer::create_analyzer_provider(&config); + match_expr->set_analyzer_ctx(std::move(analyzer_ctx)); + + MatchPredicateCollector collector; + CollectInfoMap collect_infos; + auto status = + collector.collect(runtime_state_.get(), tablet_schema, match_expr, &collect_infos); + + ASSERT_TRUE(status.ok()) << status.msg(); + ASSERT_EQ(collect_infos.size(), 1u); + const auto& collect_info = collect_infos.begin()->second; + ASSERT_NE(collect_info.index_meta, nullptr); + EXPECT_EQ(collect_info.index_meta->index_id(), 20); + EXPECT_EQ(collect_info.logical_scoring_leaves.size(), 1u); +} + +TEST_F(CollectionStatisticsTest, MatchArrayStringSelectsFulltextLeafIndex) { + auto tablet_schema = create_array_tablet_schema_with_keyword_and_fulltext_indexes(); + auto match_expr = std::make_shared(TExprNodeType::MATCH_PRED); + match_expr->_children.push_back( + std::make_shared("content", SlotId(1))); + match_expr->_children.push_back( + std::make_shared("alpha beta")); + + MatchPredicateCollector collector; + CollectInfoMap collect_infos; + auto status = + collector.collect(runtime_state_.get(), tablet_schema, match_expr, &collect_infos); + + ASSERT_TRUE(status.ok()) << status.msg(); + ASSERT_EQ(collect_infos.size(), 1u); + ASSERT_NE(collect_infos.begin()->second.index_meta, nullptr); + EXPECT_EQ(collect_infos.begin()->second.index_meta->index_id(), 20); +} + +TEST_F(CollectionStatisticsTest, SearchTermSelectsOnlyTheRuntimeFullTextIndex) { + auto tablet_schema = create_tablet_schema_with_keyword_and_fulltext_indexes(); + auto contexts = create_search_contexts("TERM", "alpha beta"); + CollectInfoMap collect_infos; + + auto status = stats_->extract_collect_info(runtime_state_.get(), contexts, tablet_schema, + &collect_infos); + + ASSERT_TRUE(status.ok()) << status.msg(); + ASSERT_EQ(collect_infos.size(), 1u); + const auto& collect_info = collect_infos.begin()->second; + ASSERT_NE(collect_info.index_meta, nullptr); + EXPECT_EQ(collect_info.index_meta->index_id(), 20); + EXPECT_EQ(collect_info.logical_scoring_leaves.size(), 1u); +} + +TEST_F(CollectionStatisticsTest, SearchArrayStringSelectsFulltextLeafIndex) { + auto tablet_schema = create_array_tablet_schema_with_keyword_and_fulltext_indexes(); + auto contexts = create_search_contexts("TERM", "alpha beta"); + CollectInfoMap collect_infos; + + auto status = stats_->extract_collect_info(runtime_state_.get(), contexts, tablet_schema, + &collect_infos); + + ASSERT_TRUE(status.ok()) << status.msg(); + ASSERT_EQ(collect_infos.size(), 1u); + ASSERT_NE(collect_infos.begin()->second.index_meta, nullptr); + EXPECT_EQ(collect_infos.begin()->second.index_meta->index_id(), 20); +} + +TEST_F(CollectionStatisticsTest, SearchExactIgnoresAnalyzedBindingHint) { + auto tablet_schema = create_tablet_schema_with_keyword_and_fulltext_indexes(); + TSearchClause clause; + clause.clause_type = "EXACT"; + clause.field_name = "content"; + clause.value = "running quickly"; + clause.__isset.field_name = true; + clause.__isset.value = true; + TSearchFieldBinding binding; + binding.field_name = "content"; + binding.slot_index = 0; + binding.index_properties["parser"] = "english"; + binding.index_properties["support_phrase"] = "true"; + binding.__isset.index_properties = true; + CollectInfoMap collect_infos; + + auto status = stats_->extract_collect_info( + runtime_state_.get(), create_search_contexts(std::move(clause), {std::move(binding)}), + tablet_schema, &collect_infos); + + ASSERT_TRUE(status.ok()) << status.msg(); + ASSERT_EQ(collect_infos.size(), 1u); + ASSERT_NE(collect_infos.begin()->second.index_meta, nullptr); + EXPECT_EQ(collect_infos.begin()->second.index_meta->index_id(), 10); + EXPECT_EQ(collect_infos.begin()->second.unique_terms, + std::vector({"running quickly"})); +} + +TEST_F(CollectionStatisticsTest, SearchScoringRejectsMissingField) { + auto tablet_schema = create_tablet_schema_with_inverted_index(); + TSearchClause clause; + clause.clause_type = "TERM"; + clause.field_name = "missing"; + clause.value = "alpha"; + clause.__isset.field_name = true; + clause.__isset.value = true; + CollectInfoMap collect_infos; + + auto status = stats_->extract_collect_info(runtime_state_.get(), + create_search_contexts(std::move(clause)), + tablet_schema, &collect_infos); + + EXPECT_EQ(status.code(), ErrorCode::INVERTED_INDEX_NOT_SUPPORTED); + EXPECT_TRUE(collect_infos.empty()); +} + +TEST_F(CollectionStatisticsTest, SearchScoringRejectsMissingIndex) { + auto tablet_schema = std::make_shared(); + TabletColumn column; + column.set_unique_id(1); + column.set_name("content"); + column.set_type(FieldType::OLAP_FIELD_TYPE_STRING); + tablet_schema->append_column(column); + CollectInfoMap collect_infos; + + auto status = stats_->extract_collect_info(runtime_state_.get(), + create_search_contexts("TERM", "alpha"), + tablet_schema, &collect_infos); + + EXPECT_EQ(status.code(), ErrorCode::INVERTED_INDEX_NOT_SUPPORTED); + EXPECT_TRUE(collect_infos.empty()); +} + +TEST_F(CollectionStatisticsTest, SearchTypedVariantBindingSelectsItsAnalyzerIndex) { + auto tablet_schema = std::make_shared(); + constexpr int32_t kVariantUid = 9010; + + TabletColumn variant_column; + variant_column.set_unique_id(kVariantUid); + variant_column.set_name("v"); + variant_column.set_type(FieldType::OLAP_FIELD_TYPE_VARIANT); + tablet_schema->append_column(variant_column); + + TabletColumn subcolumn; + subcolumn.set_unique_id(-1); + subcolumn.set_name("v.host"); + subcolumn.set_type(FieldType::OLAP_FIELD_TYPE_STRING); + subcolumn.set_parent_unique_id(kVariantUid); + subcolumn.set_path_info(PathInData("v.host", true)); + tablet_schema->append_column(subcolumn); + + TabletSchema::PathsSetInfo path_set_info; + TabletSchema::SubColumnInfo typed_path_info; + typed_path_info.column = subcolumn; + for (const auto& [index_id, parser] : {std::pair {3010, "standard"}, + std::pair {3020, "english"}}) { + auto index = std::make_shared(); + TabletIndexPB index_pb; + index_pb.set_index_id(index_id); + index_pb.set_index_name(parser + "_variant_idx"); + index_pb.set_index_type(IndexType::INVERTED); + index_pb.add_col_unique_id(kVariantUid); + (*index_pb.mutable_properties())["parser"] = parser; + (*index_pb.mutable_properties())["support_phrase"] = "true"; + index->init_from_pb(index_pb); + typed_path_info.indexes.push_back(std::move(index)); + } + path_set_info.typed_path_set.emplace("host", std::move(typed_path_info)); + std::unordered_map path_set_info_map; + path_set_info_map.emplace(kVariantUid, std::move(path_set_info)); + tablet_schema->set_path_set_info(std::move(path_set_info_map)); + + TSearchClause clause; + clause.clause_type = "TERM"; + clause.field_name = "v.host"; + clause.value = "running"; + clause.__isset.field_name = true; + clause.__isset.value = true; + TSearchFieldBinding binding; + binding.field_name = "v.host"; + binding.slot_index = 0; + binding.is_variant_subcolumn = true; + binding.__isset.is_variant_subcolumn = true; + binding.parent_field_name = "v"; + binding.__isset.parent_field_name = true; + binding.subcolumn_path = "host"; + binding.__isset.subcolumn_path = true; + binding.index_properties["parser"] = "english"; + binding.index_properties["support_phrase"] = "true"; + binding.__isset.index_properties = true; + CollectInfoMap collect_infos; + + auto status = stats_->extract_collect_info( + runtime_state_.get(), create_search_contexts(std::move(clause), {std::move(binding)}), + tablet_schema, &collect_infos); + + ASSERT_TRUE(status.ok()) << status.msg(); + ASSERT_EQ(collect_infos.size(), 1u); + const auto& collect_info = collect_infos.begin()->second; + ASSERT_NE(collect_info.index_meta, nullptr); + EXPECT_EQ(collect_info.index_meta->index_id(), 3020); + EXPECT_EQ(collect_info.unique_terms, std::vector({"running"})); +} + +TEST_F(CollectionStatisticsTest, SearchScoringUsesTextSemanticsForVariantParentIndexFallback) { + auto tablet_schema = std::make_shared(); + constexpr int32_t kVariantUid = 9015; + + TabletColumn variant_column; + variant_column.set_unique_id(kVariantUid); + variant_column.set_name("v"); + variant_column.set_type(FieldType::OLAP_FIELD_TYPE_VARIANT); + tablet_schema->append_column(variant_column); + + TabletIndex parent_index; + parent_index._index_id = 3025; + parent_index._index_type = IndexType::INVERTED; + parent_index._col_unique_ids.push_back(kVariantUid); + parent_index._properties["parser"] = "standard"; + parent_index._properties["support_phrase"] = "true"; + tablet_schema->append_index(std::move(parent_index)); + + TSearchClause clause; + clause.clause_type = "PHRASE"; + clause.field_name = "v.dynamic"; + clause.value = "alpha beta"; + clause.__isset.field_name = true; + clause.__isset.value = true; + TSearchFieldBinding binding; + binding.field_name = "v.dynamic"; + binding.slot_index = 0; + binding.is_variant_subcolumn = true; + binding.__isset.is_variant_subcolumn = true; + binding.parent_field_name = "v"; + binding.__isset.parent_field_name = true; + binding.subcolumn_path = "dynamic"; + binding.__isset.subcolumn_path = true; + binding.index_properties["parser"] = "standard"; + binding.index_properties["support_phrase"] = "true"; + binding.__isset.index_properties = true; + CollectInfoMap collect_infos; + + auto status = stats_->extract_collect_info( + runtime_state_.get(), create_search_contexts(std::move(clause), {std::move(binding)}), + tablet_schema, &collect_infos); + + ASSERT_TRUE(status.ok()) << status; + ASSERT_EQ(collect_infos.size(), 1U); + auto it = collect_infos.find( + StringHelper::to_wstring(std::to_string(kVariantUid) + ".v.dynamic")); + ASSERT_NE(it, collect_infos.end()); + ASSERT_NE(it->second.index_meta, nullptr); + ASSERT_NE(it->second.owned_index_meta, nullptr); + EXPECT_EQ(it->second.index_meta->index_id(), 3025); + EXPECT_EQ(it->second.unique_terms, std::vector({"alpha", "beta"})); +} + +TEST_F(CollectionStatisticsTest, SearchVariantFieldPatternKeepsSelectedMetadataAlive) { + auto tablet_schema = std::make_shared(); + constexpr int32_t kVariantUid = 9020; + + TabletColumn variant_column; + variant_column.set_unique_id(kVariantUid); + variant_column.set_name("meta"); + variant_column.set_type(FieldType::OLAP_FIELD_TYPE_VARIANT); + auto subcolumn_template = make_subcolumn_template("user.*", PatternTypePB::MATCH_NAME_GLOB); + variant_column.add_sub_column(subcolumn_template); + tablet_schema->append_column(variant_column); + + TabletIndexPB index_pb; + index_pb.set_index_id(3030); + index_pb.set_index_name("variant_search_field_pattern_idx"); + index_pb.set_index_type(IndexType::INVERTED); + index_pb.add_col_unique_id(kVariantUid); + (*index_pb.mutable_properties())["parser"] = "standard"; + (*index_pb.mutable_properties())["support_phrase"] = "true"; + (*index_pb.mutable_properties())["field_pattern"] = "user.*"; + TabletIndex index; + index.init_from_pb(index_pb); + tablet_schema->append_index(std::move(index)); + + TSearchClause clause; + clause.clause_type = "PHRASE"; + clause.field_name = "meta.user.name"; + clause.value = "alice smith"; + clause.__isset.field_name = true; + clause.__isset.value = true; + TSearchFieldBinding binding; + binding.field_name = "meta.user.name"; + binding.slot_index = 0; + binding.is_variant_subcolumn = true; + binding.__isset.is_variant_subcolumn = true; + binding.parent_field_name = "meta"; + binding.__isset.parent_field_name = true; + binding.subcolumn_path = "user.name"; + binding.__isset.subcolumn_path = true; + binding.index_properties["parser"] = "standard"; + binding.index_properties["support_phrase"] = "true"; + binding.__isset.index_properties = true; + CollectInfoMap collect_infos; + + auto status = stats_->extract_collect_info( + runtime_state_.get(), create_search_contexts(std::move(clause), {std::move(binding)}), + tablet_schema, &collect_infos); + + ASSERT_TRUE(status.ok()) << status.msg(); + ASSERT_EQ(collect_infos.size(), 1u); + auto iter = collect_infos.find( + StringHelper::to_wstring(std::to_string(kVariantUid) + ".meta.user.name")); + ASSERT_NE(iter, collect_infos.end()); + ASSERT_NE(iter->second.index_meta, nullptr); + ASSERT_NE(iter->second.owned_index_meta, nullptr); + EXPECT_EQ(iter->second.index_meta->index_id(), 3030); +} + +TEST_F(CollectionStatisticsTest, SearchScoringRejectsNumericBkdLeaf) { + auto tablet_schema = std::make_shared(); + TabletColumn column; + column.set_unique_id(2); + column.set_name("number"); + column.set_type(FieldType::OLAP_FIELD_TYPE_INT); + tablet_schema->append_column(column); + TabletIndex index; + index._index_id = 3040; + index._index_type = IndexType::INVERTED; + index._col_unique_ids.push_back(2); + tablet_schema->append_index(std::move(index)); + TSearchClause clause; + clause.clause_type = "TERM"; + clause.field_name = "number"; + clause.value = "42"; + clause.__isset.field_name = true; + clause.__isset.value = true; + CollectInfoMap collect_infos; + + auto status = stats_->extract_collect_info(runtime_state_.get(), + create_search_contexts(std::move(clause)), + tablet_schema, &collect_infos); + + EXPECT_EQ(status.code(), ErrorCode::INVERTED_INDEX_NOT_SUPPORTED); + EXPECT_TRUE(collect_infos.empty()); +} + +TEST_F(CollectionStatisticsTest, NestedSearchScoringIsRejected) { + TSearchClause phrase; + phrase.clause_type = "PHRASE"; + phrase.field_name = "content"; + phrase.value = "alpha beta"; + phrase.__isset.field_name = true; + phrase.__isset.value = true; + + TSearchClause nested; + nested.clause_type = "NESTED"; + nested.children.push_back(std::move(phrase)); + nested.__isset.children = true; + + CollectInfoMap collect_infos; + auto status = stats_->extract_collect_info( + runtime_state_.get(), create_search_contexts(std::move(nested)), + create_tablet_schema_with_inverted_index(), &collect_infos); + + EXPECT_EQ(status.code(), ErrorCode::INVERTED_INDEX_NOT_SUPPORTED); + EXPECT_TRUE(collect_infos.empty()); +} + +TEST_F(CollectionStatisticsTest, OneScoringFieldCannotSelectDifferentPhysicalIndexes) { + auto tablet_schema = create_tablet_schema_with_two_fulltext_indexes(); + auto build_match = [](const std::string& analyzer_name, InvertedIndexParserType parser_type) { + auto match_expr = + std::make_shared(TExprNodeType::MATCH_PRED); + match_expr->_children.push_back( + std::make_shared("content", SlotId(1))); + match_expr->_children.push_back( + std::make_shared("alpha beta")); + + InvertedIndexAnalyzerConfig config; + config.analyzer_name = analyzer_name; + config.parser_type = parser_type; + config.stop_words = "none"; + auto analyzer_ctx = std::make_shared(); + analyzer_ctx->analyzer_key = analyzer_name; + analyzer_ctx->parser_type = parser_type; + analyzer_ctx->analyzer_provider = + segment_v2::inverted_index::InvertedIndexAnalyzer::create_analyzer_provider( + &config); + match_expr->set_analyzer_ctx(std::move(analyzer_ctx)); + return match_expr; + }; + + MatchPredicateCollector collector; + CollectInfoMap collect_infos; + auto first = collector.collect( + runtime_state_.get(), tablet_schema, + build_match("standard", InvertedIndexParserType::PARSER_STANDARD), &collect_infos); + ASSERT_TRUE(first.ok()) << first.msg(); + + auto second = collector.collect(runtime_state_.get(), tablet_schema, + build_match("english", InvertedIndexParserType::PARSER_ENGLISH), + &collect_infos); + + EXPECT_EQ(second.code(), ErrorCode::INVERTED_INDEX_NOT_SUPPORTED); +} + +// Test-only subclass that exposes the protected helpers of PredicateCollector. +class TestablePredicateCollector : public MatchPredicateCollector { +public: + using MatchPredicateCollector::build_field_name; + using MatchPredicateCollector::find_slot_ref; +}; + +// find_slot_ref: null shared_ptr returns nullptr (early-return branch). +TEST_F(CollectionStatisticsTest, FindSlotRefHandlesNullExpr) { + TestablePredicateCollector collector; + VExprSPtr null_expr; + EXPECT_EQ(collector.find_slot_ref(null_expr), nullptr); +} + +// find_slot_ref: when expr is a non-CAST wrapper containing a SLOT_REF in its +// children, the recursive descent finds the slot via the for-loop body. +TEST_F(CollectionStatisticsTest, FindSlotRefRecursesIntoChildren) { + TestablePredicateCollector collector; + auto wrapper = std::make_shared(TExprNodeType::FUNCTION_CALL); + auto slot_ref = std::make_shared("c", SlotId(99)); + wrapper->_children.push_back(slot_ref); + EXPECT_EQ(collector.find_slot_ref(wrapper), slot_ref.get()); +} + +// find_slot_ref: leaf non-slot (no children) returns nullptr after for-loop. +TEST_F(CollectionStatisticsTest, FindSlotRefReturnsNullForLeafNonSlot) { + TestablePredicateCollector collector; + auto literal = std::make_shared("x"); + EXPECT_EQ(collector.find_slot_ref(literal), nullptr); +} + +// build_field_name: non-empty suffix is appended with a dot separator. +TEST_F(CollectionStatisticsTest, BuildFieldNameWithSuffix) { + TestablePredicateCollector collector; + EXPECT_EQ(collector.build_field_name(42, "a.b"), "42.a.b"); +} + +// build_field_name: empty suffix returns just the unique id as string. +TEST_F(CollectionStatisticsTest, BuildFieldNameWithoutSuffix) { + TestablePredicateCollector collector; + EXPECT_EQ(collector.build_field_name(42, ""), "42"); +} + +} // namespace doris diff --git a/be/test/storage/index/inverted/token_filter/common_grams_filter_test.cpp b/be/test/storage/index/inverted/token_filter/common_grams_filter_test.cpp new file mode 100644 index 00000000000000..9d4feb731bb150 --- /dev/null +++ b/be/test/storage/index/inverted/token_filter/common_grams_filter_test.cpp @@ -0,0 +1,644 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/inverted/token_filter/common_grams_filter.h" + +#include + +#include +#include +#include +#include +#include + +#include "common/exception.h" +#include "storage/index/inverted/common_grams/common_grams_key_codec.h" +#include "storage/index/inverted/common_grams/common_word_set.h" + +namespace doris::segment_v2::inverted_index { +namespace { + +struct ScriptedToken { + std::string term; + int32_t position_increment = 1; + int32_t start_offset = 0; + int32_t end_offset = 0; + const TCHAR* type = Token::getDefaultType(); +}; + +class ScriptedTokenStream final : public TokenStream { +public: + explicit ScriptedTokenStream(std::vector tokens) : _tokens(std::move(tokens)) {} + + Token* next(Token* token) override { + if (_next == _tokens.size()) { + return nullptr; + } + const auto& scripted = _tokens[_next++]; + _scratch = scripted.term; + token->clear(); + token->setTextNoCopy(_scratch.data(), static_cast(_scratch.size())); + token->positionIncrement = scripted.position_increment; + token->setStartOffset(scripted.start_offset); + token->setEndOffset(scripted.end_offset); + token->setType(scripted.type); + return token; + } + + void close() override {} + void reset() override { _next = 0; } + + void set_tokens(std::vector tokens) { + _tokens = std::move(tokens); + _next = 0; + } + + size_t consumed() const { return _next; } + +private: + std::vector _tokens; + size_t _next = 0; + std::string _scratch; +}; + +struct ActualToken { + std::string term; + int32_t position_increment; + int32_t start_offset; + int32_t end_offset; + std::wstring type; + + bool operator==(const ActualToken&) const = default; +}; + +struct TokenSemantics { + std::string term; + int32_t position; + std::wstring type; + + bool operator==(const TokenSemantics&) const = default; +}; + +template +concept HasAnalyzerCommonGramClassification = requires(Event event) { + event.has_preceding_gram; + event.preceding_gram_both_common; +}; + +static_assert(!HasAnalyzerCommonGramClassification); + +std::string gram(std::string_view left, std::string_view right) { + auto encoded = encode_common_gram(left, right); + EXPECT_TRUE(encoded.has_value()) << encoded.error(); + return encoded.value(); +} + +std::vector collect(const TokenStreamPtr& stream) { + std::vector result; + Token token; + while (stream->next(&token) != nullptr) { + result.push_back({std::string(token.termBuffer(), token.termLength()), + token.getPositionIncrement(), token.startOffset(), token.endOffset(), + token.type()}); + } + return result; +} + +std::vector collect_semantics(const TokenStreamPtr& stream) { + std::vector result; + Token token; + int32_t position = 0; + while (stream->next(&token) != nullptr) { + position += token.getPositionIncrement(); + result.push_back({std::string(token.termBuffer(), token.termLength()), position, + token.type()}); + } + return result; +} + +std::vector expand_snii_index_events(CommonGramsFilter* stream, size_t* event_count, + size_t* both_common_gram_count) { + std::vector result; + std::optional previous_logical_term; + bool previous_is_common = false; + int32_t position = 0; + SniiCommonGramsIndexEvent event; + while (stream->next_snii_index_event(&event)) { + ++*event_count; + const std::string physical_plain_term(event.plain_term); + const std::string logical_term = + decode_plain_term(physical_plain_term, PlainTermKeyVersion::kEscapedV1).value(); + EXPECT_EQ(event.logical_term, logical_term); + const bool current_is_common = + CommonWordSet::builtin_english_stop_words_v1().contains(logical_term); + const bool has_preceding_gram = previous_logical_term.has_value() && + (previous_is_common || current_is_common) && + common_gram_component_sizes_encodable( + previous_logical_term->size(), logical_term.size()); + if (has_preceding_gram) { + EXPECT_TRUE(previous_logical_term.has_value()); + *both_common_gram_count += previous_is_common && current_is_common; + result.push_back({gram(previous_logical_term.value(), logical_term), position, + COMMON_GRAM_TOKEN_TYPE}); + } + ++position; + result.push_back({physical_plain_term, position, Token::getDefaultType()}); + previous_logical_term = logical_term; + previous_is_common = current_is_common; + } + return result; +} + +std::vector words(std::initializer_list terms) { + std::vector result; + int32_t offset = 0; + for (auto term : terms) { + result.push_back( + {std::string(term), 1, offset, offset + static_cast(term.size())}); + offset += static_cast(term.size()) + 1; + } + return result; +} + +std::vector terms(const std::vector& tokens) { + std::vector result; + for (const auto& token : tokens) { + EXPECT_EQ(token.position_increment, 1); + result.push_back(token.term); + } + return result; +} + +std::shared_ptr builtin_common_words() { + static const auto common_words = std::shared_ptr( + &CommonWordSet::builtin_english_stop_words_v1(), [](const CommonWordSet*) {}); + return common_words; +} + +TokenStreamPtr index_stream(const std::shared_ptr& input) { + return std::make_shared(input, builtin_common_words()); +} + +TokenStreamPtr escaped_index_stream(const std::shared_ptr& input) { + return std::make_shared(input, builtin_common_words(), + CommonGramsOutputMode::kEscapedV1Index); +} + +TokenStreamPtr spimi_index_stream(const std::shared_ptr& input) { + return std::make_shared(input, builtin_common_words(), + CommonGramsOutputMode::kEscapedV1SpimiIndex); +} + +TokenStreamPtr plain_stream(const std::shared_ptr& input) { + return std::make_shared(input); +} + +TokenStreamPtr exact_stream(const std::shared_ptr& input) { + return std::make_shared(index_stream(input), builtin_common_words()); +} + +TokenStreamPtr prefix_stream(const std::shared_ptr& input) { + return std::make_shared(index_stream(input), + builtin_common_words()); +} + +TEST(CommonGramsFilterTest, IndexPreservesUnigramsAndAddsEligibleOwnedGrams) { + auto input = std::make_shared(words({"man", "of", "the", "year"})); + auto stream = index_stream(input); + + EXPECT_EQ(collect(stream), (std::vector { + {"man", 1, 0, 3, Token::getDefaultType()}, + {gram("man", "of"), 0, 0, 6, COMMON_GRAM_TOKEN_TYPE}, + {"of", 1, 4, 6, Token::getDefaultType()}, + {gram("of", "the"), 0, 4, 10, COMMON_GRAM_TOKEN_TYPE}, + {"the", 1, 7, 10, Token::getDefaultType()}, + {gram("the", "year"), 0, 7, 15, COMMON_GRAM_TOKEN_TYPE}, + {"year", 1, 11, 15, Token::getDefaultType()}, + })); +} + +TEST(CommonGramsFilterTest, QueryModesNormalizeBothCommonGramType) { + auto input = std::make_shared(words({"of", "the"})); + EXPECT_EQ(collect(exact_stream(input)), + (std::vector { + {gram("of", "the"), 1, 0, 6, COMMON_GRAM_TOKEN_TYPE}, + })); + + input = std::make_shared(words({"of", "the"})); + EXPECT_EQ(collect(prefix_stream(input)), + (std::vector { + {gram("of", "the"), 1, 0, 6, COMMON_GRAM_TOKEN_TYPE}, + })); +} + +TEST(CommonGramsFilterTest, QueryGramEligibilityMatchesPurposeSpecificFilters) { + const std::vector> cases { + {}, + {"alpha"}, + {"alpha", "beta"}, + {"alpha", "the"}, + {"the", "alpha"}, + {"alpha", "beta", "the"}, + {"alpha", "the", "beta"}, + {"alpha", "beta", "gamma", "delta"}, + }; + for (const auto& query_terms : cases) { + SCOPED_TRACE(testing::PrintToString(query_terms)); + std::vector scripted; + scripted.reserve(query_terms.size()); + for (const auto& term : query_terms) { + scripted.push_back({.term = term}); + } + + for (const auto mode : + {CommonGramsQueryMode::kExact, CommonGramsQueryMode::kPhrasePrefix}) { + auto input = std::make_shared(scripted); + const auto output = + collect(mode == CommonGramsQueryMode::kExact ? exact_stream(input) + : prefix_stream(input)); + bool filter_used_gram = false; + for (const auto& token : output) { + filter_used_gram = + filter_used_gram || token.type == std::wstring(COMMON_GRAM_TOKEN_TYPE); + } + EXPECT_EQ(common_grams_query_may_use_gram(query_terms, mode, *builtin_common_words()), + filter_used_gram); + } + } +} + +TEST(CommonGramsFilterTest, PhysicalIndexModeEscapesPlainTermsButGramsUseLogicalBytes) { + const std::string internal_plain = std::string(1, '\x1f') + "literal"; + auto input = std::make_shared(words({internal_plain, "of"})); + auto stream = escaped_index_stream(input); + + EXPECT_EQ(collect(stream), + (std::vector { + {std::string(1, PLAIN_ESCAPE_PREFIX) + "Gliteral", 1, 0, + static_cast(internal_plain.size()), Token::getDefaultType()}, + {gram(internal_plain, "of"), 0, 0, + static_cast(internal_plain.size() + 3), COMMON_GRAM_TOKEN_TYPE}, + {"of", 1, static_cast(internal_plain.size() + 1), + static_cast(internal_plain.size() + 3), Token::getDefaultType()}, + })); + + const std::string escape_plain = std::string(1, PLAIN_ESCAPE_PREFIX) + "literal"; + input->set_tokens(words({escape_plain})); + stream->reset(); + EXPECT_EQ(terms(collect(stream)), + (std::vector {std::string(1, PLAIN_ESCAPE_PREFIX) + "Eliteral"})); +} + +TEST(CommonGramsFilterTest, SpimiIndexModeEmitsPhysicalGramsAndEscapedPlainTerms) { + const std::string internal_plain = std::string(1, '\x1f') + "literal"; + auto input = std::make_shared(words({internal_plain, "of"})); + + const std::string physical = gram(internal_plain, "of"); + EXPECT_EQ(collect(spimi_index_stream(input)), + (std::vector { + {std::string(1, PLAIN_ESCAPE_PREFIX) + "Gliteral", 1, 0, + static_cast(internal_plain.size()), Token::getDefaultType()}, + {physical, 0, 0, static_cast(internal_plain.size() + 3), + COMMON_GRAM_TOKEN_TYPE}, + {"of", 1, static_cast(internal_plain.size() + 1), + static_cast(internal_plain.size() + 3), Token::getDefaultType()}, + })); +} + +TEST(CommonGramsFilterTest, SniiIndexEventsExpandToExistingSpimiTokenSemantics) { + const std::string internal_plain = std::string(1, '\x1f') + "literal"; + const std::string escaped_plain = std::string(1, PLAIN_ESCAPE_PREFIX) + "literal"; + const std::vector input_tokens = + words({internal_plain, "of", "the", escaped_plain, "中文词"}); + + auto legacy_input = std::make_shared(input_tokens); + const std::vector expected = + collect_semantics(spimi_index_stream(legacy_input)); + + auto event_input = std::make_shared(input_tokens); + CommonGramsFilter event_stream(event_input, builtin_common_words(), + CommonGramsOutputMode::kEscapedV1SpimiIndex); + size_t event_count = 0; + size_t both_common_gram_count = 0; + const std::vector actual = + expand_snii_index_events(&event_stream, &event_count, &both_common_gram_count); + + EXPECT_EQ(event_count, input_tokens.size()); + EXPECT_EQ(both_common_gram_count, 1U); + EXPECT_EQ(actual, expected); +} + +TEST(CommonGramsFilterTest, SniiIndexEventsDeferCommonWordClassificationToWriter) { + auto input = std::make_shared( + words({"the", "database", "of", "the", "world", "and", "search"})); + CommonGramsFilter stream(input, builtin_common_words(), + CommonGramsOutputMode::kEscapedV1SpimiIndex); + + common_grams_testing::reset_common_word_membership_lookup_count(); + SniiCommonGramsIndexEvent event; + size_t event_count = 0; + while (stream.next_snii_index_event(&event)) { + EXPECT_FALSE(event.logical_term.empty()); + EXPECT_FALSE(event.plain_term.empty()); + ++event_count; + } + + EXPECT_EQ(event_count, 7U); + EXPECT_EQ(common_grams_testing::common_word_membership_lookup_count(), 0U); +} + +TEST(CommonGramsFilterTest, IndexOmitsNonCommonPair) { + auto input = std::make_shared(words({"man", "year"})); + EXPECT_EQ(terms(collect(index_stream(input))), (std::vector {"man", "year"})); +} + +TEST(CommonGramsFilterTest, CachesCommonWordMembershipWithoutDefeatingShortCircuit) { + auto input = std::make_shared(words({"single"})); + common_grams_testing::reset_common_word_membership_lookup_count(); + collect(index_stream(input)); + EXPECT_EQ(common_grams_testing::common_word_membership_lookup_count(), 0); + + input = std::make_shared(words({"of", "dog"})); + common_grams_testing::reset_common_word_membership_lookup_count(); + collect(index_stream(input)); + EXPECT_EQ(common_grams_testing::common_word_membership_lookup_count(), 1); + + input = std::make_shared(words({"of", "the", "and", "to"})); + common_grams_testing::reset_common_word_membership_lookup_count(); + collect(exact_stream(input)); + EXPECT_EQ(common_grams_testing::common_word_membership_lookup_count(), 6); + + input = std::make_shared(words({"man", "dog"})); + common_grams_testing::reset_common_word_membership_lookup_count(); + collect(prefix_stream(input)); + EXPECT_EQ(common_grams_testing::common_word_membership_lookup_count(), 3); + + input = std::make_shared(words({"man", "dog", "year", "thing"})); + common_grams_testing::reset_common_word_membership_lookup_count(); + EXPECT_EQ(terms(collect(index_stream(input))), + (std::vector {"man", "dog", "year", "thing"})); + EXPECT_EQ(common_grams_testing::common_word_membership_lookup_count(), 4); + + input = std::make_shared(words({"man", "dog", "year", "thing"})); + common_grams_testing::reset_common_word_membership_lookup_count(); + EXPECT_EQ(terms(collect(exact_stream(input))), + (std::vector {"man", "dog", "year", "thing"})); + EXPECT_EQ(common_grams_testing::common_word_membership_lookup_count(), 8); +} + +TEST(CommonGramsFilterTest, ExactRewriteTruthTable) { + struct Case { + std::vector input; + std::vector expected; + }; + const std::vector cases = { + {words({"man", "dog", "year"}), {"man", "dog", "year"}}, + {words({"man", "dog", "the"}), {"man", gram("dog", "the")}}, + {words({"man", "of", "year"}), {gram("man", "of"), gram("of", "year")}}, + {words({"man", "of", "the"}), {gram("man", "of"), gram("of", "the")}}, + {words({"of", "dog", "year"}), {gram("of", "dog"), "dog", "year"}}, + {words({"of", "dog", "the"}), {gram("of", "dog"), gram("dog", "the")}}, + {words({"of", "the", "year"}), {gram("of", "the"), gram("the", "year")}}, + {words({"of", "the", "and"}), {gram("of", "the"), gram("the", "and")}}, + {words({"the", "the", "the"}), {gram("the", "the"), gram("the", "the")}}, + }; + + for (const auto& test_case : cases) { + auto input = std::make_shared(test_case.input); + EXPECT_EQ(terms(collect(exact_stream(input))), test_case.expected); + } +} + +TEST(CommonGramsFilterTest, PhrasePrefixRewritesOnlyCommonLeftBoundary) { + struct Case { + std::vector input; + std::vector expected; + }; + const std::vector cases = { + {words({"the", "wo"}), {gram("the", "wo")}}, + {words({"foo", "the"}), {"foo", "the"}}, + {words({"foo", "of", "th"}), {gram("foo", "of"), gram("of", "th")}}, + {words({"the", "bar", "ba"}), {gram("the", "bar"), "bar", "ba"}}, + {words({"the"}), {"the"}}, + }; + + for (const auto& test_case : cases) { + auto input = std::make_shared(test_case.input); + EXPECT_EQ(terms(collect(prefix_stream(input))), test_case.expected); + } +} + +TEST(CommonGramsFilterTest, ResetAndReuseRestoresUnigramMetadata) { + auto input = std::make_shared(words({"man", "of"})); + auto stream = index_stream(input); + ASSERT_EQ(collect(stream).size(), 3); + + input->set_tokens({{"plain", 1, 17, 22, Token::getDefaultType()}}); + stream->reset(); + EXPECT_EQ(collect(stream), (std::vector { + {"plain", 1, 17, 22, Token::getDefaultType()}, + })); +} + +TEST(CommonGramsFilterTest, PreservesUpstreamUnigramTypes) { + auto input = std::make_shared(std::vector { + {"man", 1, 3, 6, L"left_type"}, {"of", 1, 7, 9, L"right_type"}}); + EXPECT_EQ(collect(index_stream(input)), + (std::vector { + {"man", 1, 3, 6, L"left_type"}, + {gram("man", "of"), 0, 3, 9, COMMON_GRAM_TOKEN_TYPE}, + {"of", 1, 7, 9, L"right_type"}, + })); + + input->set_tokens({{"man", 1, 3, 6, L"left_type"}, {"year", 1, 7, 11, L"right_type"}}); + auto exact = exact_stream(input); + EXPECT_EQ(collect(exact), (std::vector { + {"man", 1, 3, 6, L"left_type"}, + {"year", 1, 7, 11, L"right_type"}, + })); +} + +TEST(CommonGramsFilterTest, ExactAndPrefixResetReuseIndependentInput) { + auto exact_input = std::make_shared(words({"man", "of"})); + auto exact = exact_stream(exact_input); + EXPECT_EQ(terms(collect(exact)), (std::vector {gram("man", "of")})); + exact_input->set_tokens(words({"plain", "terms"})); + exact->reset(); + EXPECT_EQ(terms(collect(exact)), (std::vector {"plain", "terms"})); + + auto prefix_input = std::make_shared(words({"the", "term"})); + auto prefix = prefix_stream(prefix_input); + EXPECT_EQ(terms(collect(prefix)), (std::vector {gram("the", "term")})); + prefix_input->set_tokens(words({"plain", "terms"})); + prefix->reset(); + EXPECT_EQ(terms(collect(prefix)), (std::vector {"plain", "terms"})); +} + +TEST(CommonGramsFilterTest, EmptyAndSingleTokenStreamsAreStable) { + auto input = std::make_shared(std::vector {}); + auto stream = index_stream(input); + EXPECT_TRUE(collect(stream).empty()); + + input->set_tokens(words({"of"})); + stream->reset(); + EXPECT_EQ(terms(collect(stream)), (std::vector {"of"})); +} + +TEST(CommonGramsFilterTest, RejectsNonUnitInputForEveryStreamPurpose) { + using Factory = TokenStreamPtr (*)(const std::shared_ptr&); + for (Factory factory : {index_stream, plain_stream, exact_stream, prefix_stream}) { + for (int32_t bad_increment : {-1, 0, 2}) { + auto input = std::make_shared( + std::vector {{"of", bad_increment, 0, 2}}); + try { + collect(factory(input)); + FAIL() << "expected analyzer error for increment " << bad_increment; + } catch (const Exception& error) { + EXPECT_EQ(error.code(), ErrorCode::INVERTED_INDEX_ANALYZER_ERROR); + } + } + + for (int32_t bad_increment : {-1, 0, 2}) { + auto empty_input = std::make_shared( + std::vector {{"", bad_increment, 0, 0}}); + try { + collect(factory(empty_input)); + FAIL() << "expected analyzer error for empty token increment " << bad_increment; + } catch (const Exception& error) { + EXPECT_EQ(error.code(), ErrorCode::INVERTED_INDEX_ANALYZER_ERROR); + } + } + + auto input = std::make_shared( + std::vector {{"man", 1, 0, 3}, {"of", 2, 4, 6}}); + auto stream = factory(input); + try { + collect(stream); + FAIL() << "expected analyzer error after a valid token"; + } catch (const Exception& error) { + EXPECT_EQ(error.code(), ErrorCode::INVERTED_INDEX_ANALYZER_ERROR); + EXPECT_EQ(input->consumed(), 2); + } + } +} + +TEST(CommonGramsFilterTest, RejectsEmptyInputForEveryStreamPurpose) { + using Factory = TokenStreamPtr (*)(const std::shared_ptr&); + for (Factory factory : {index_stream, plain_stream, exact_stream, prefix_stream}) { + auto input = + std::make_shared(std::vector {{"", 1, 0, 0}}); + try { + collect(factory(input)); + FAIL() << "expected analyzer error for an empty token"; + } catch (const Exception& error) { + EXPECT_EQ(error.code(), ErrorCode::INVERTED_INDEX_ANALYZER_ERROR); + } + } +} + +TEST(CommonGramsFilterTest, QueryPreparationLatchesFailureUntilReset) { + for (auto factory : {exact_stream, prefix_stream}) { + auto input = std::make_shared( + std::vector {{"the", 1, 0, 3}, {"term", 2, 4, 8}}); + auto stream = factory(input); + for (int attempt = 0; attempt < 2; ++attempt) { + try { + collect(stream); + FAIL() << "expected latched analyzer error on attempt " << attempt; + } catch (const Exception& error) { + EXPECT_EQ(error.code(), ErrorCode::INVERTED_INDEX_ANALYZER_ERROR); + } + } + + input->set_tokens(words({"the", "term"})); + stream->reset(); + EXPECT_EQ(terms(collect(stream)), (std::vector {gram("the", "term")})); + } +} + +TEST(CommonGramsFilterTest, UnencodableRequiredGramFallsBackToWholePlainSequence) { + const std::string huge(COMMON_GRAM_MAX_ENCODED_BYTES, 'x'); + + for (auto factory : {exact_stream, prefix_stream}) { + auto input = std::make_shared(words({"the", huge})); + EXPECT_EQ(terms(collect(factory(input))), (std::vector {"the", huge})); + + input = std::make_shared(words({"the", "of", huge})); + EXPECT_EQ(terms(collect(factory(input))), (std::vector {"the", "of", huge})); + } + + auto input = std::make_shared(words({"the", huge})); + EXPECT_EQ(terms(collect(index_stream(input))), (std::vector {"the", huge})); +} + +TEST(CommonGramsFilterTest, MaximumMarkerLeadingUnigramFailsWithBuildSwitchRecovery) { + for (const char marker : {PLAIN_ESCAPE_PREFIX, '\x1f'}) { + std::string term(COMMON_GRAM_MAX_ENCODED_BYTES, 'x'); + term.front() = marker; + auto input = std::make_shared( + std::vector {{term, 1, 0, static_cast(term.size())}}); + try { + static_cast(collect(escaped_index_stream(input))); + FAIL() << "expected CommonGrams escaped-term overflow"; + } catch (const Exception& e) { + EXPECT_EQ(e.code(), ErrorCode::INVERTED_INDEX_ANALYZER_ERROR); + EXPECT_NE(std::string(e.what()).find("enable_common_grams_index_build=false"), + std::string::npos); + EXPECT_NE(std::string(e.what()).find("new transaction"), std::string::npos); + } + } +} + +TEST(CommonGramsFilterTest, LargestEscapableMarkerLeadingUnigramUsesPhysicalKeyLimit) { + for (const char marker : {PLAIN_ESCAPE_PREFIX, '\x1f'}) { + std::string term(COMMON_GRAM_MAX_ENCODED_BYTES - 1, 'x'); + term.front() = marker; + auto input = std::make_shared( + std::vector {{term, 1, 0, static_cast(term.size())}}); + const auto tokens = collect(escaped_index_stream(input)); + ASSERT_EQ(tokens.size(), 1); + EXPECT_EQ(tokens[0].term.size(), COMMON_GRAM_MAX_ENCODED_BYTES); + } +} + +TEST(CommonGramsFilterTest, InvalidLogicalTermsRemainHardAnalyzerErrors) { + for (const std::string& term : {std::string("bad\0term", 8), std::string("\xc3", 1)}) { + auto input = std::make_shared( + std::vector {{term, 1, 0, static_cast(term.size())}}); + try { + collect(escaped_index_stream(input)); + FAIL() << "expected analyzer error"; + } catch (const Exception& error) { + EXPECT_EQ(error.code(), ErrorCode::INVERTED_INDEX_ANALYZER_ERROR); + } + } +} + +TEST(CommonGramsFilterTest, RejectsOverlongLogicalToken) { + auto input = std::make_shared( + std::vector {{std::string(COMMON_GRAM_MAX_ENCODED_BYTES + 1, 'x'), 1, 0, + static_cast(COMMON_GRAM_MAX_ENCODED_BYTES + 1)}}); + try { + collect(index_stream(input)); + FAIL() << "expected analyzer error"; + } catch (const Exception& error) { + EXPECT_EQ(error.code(), ErrorCode::INVERTED_INDEX_ANALYZER_ERROR); + } +} + +} // namespace +} // namespace doris::segment_v2::inverted_index diff --git a/be/test/storage/index/inverted/token_filter/lower_case_filter_factory_test.cpp b/be/test/storage/index/inverted/token_filter/lower_case_filter_factory_test.cpp index cd8c72755e16ce..6c19d437c9da59 100644 --- a/be/test/storage/index/inverted/token_filter/lower_case_filter_factory_test.cpp +++ b/be/test/storage/index/inverted/token_filter/lower_case_filter_factory_test.cpp @@ -20,10 +20,17 @@ #include +#include + #include "storage/index/inverted/tokenizer/keyword/keyword_tokenizer_factory.h" namespace doris::segment_v2::inverted_index { +namespace lower_case_testing { +uint64_t unicode_path_count(); +void reset_unicode_path_count(); +} // namespace lower_case_testing + TokenStreamPtr create_lowercase_filter(const std::string& text, Settings settings = Settings()) { ReaderPtr reader = std::make_shared>(); reader->init(text.data(), text.size(), false); @@ -45,6 +52,32 @@ struct ExpectedToken { int pos_inc; }; +class ScriptedLowercaseInput final : public TokenStream { +public: + explicit ScriptedLowercaseInput(std::vector terms) : _terms(std::move(terms)) {} + + Token* next(Token* token) override { + if (_next == _terms.size()) { + return nullptr; + } + const auto& term = _terms[_next++]; + token->clear(); + token->setTextNoCopy(term.data(), static_cast(term.size())); + token->setPositionIncrement(3); + token->setStartOffset(7); + token->setEndOffset(19); + token->setType(_T("scripted")); + return token; + } + + void close() override {} + void reset() override { _next = 0; } + +private: + std::vector _terms; + size_t _next = 0; +}; + class LowerCaseFilterTest : public ::testing::Test { protected: void assert_filter_output(const std::string& text, const std::vector& expected) { @@ -75,10 +108,73 @@ TEST_F(LowerCaseFilterTest, HandlesMixedCase) { assert_filter_output("HeLLo WoRLd", {{"hello world", 1}}); } +TEST_F(LowerCaseFilterTest, ASCIIBypassesUnicodeConversion) { + lower_case_testing::reset_unicode_path_count(); + assert_filter_output("already lowercase", {{"already lowercase", 1}}); + assert_filter_output("ASCII UPPER", {{"ascii upper", 1}}); + EXPECT_EQ(lower_case_testing::unicode_path_count(), 0); + + assert_filter_output( + "\xC3\x9C" + "BER", + {{"\xC3\xBC" + "ber", + 1}}); + EXPECT_EQ(lower_case_testing::unicode_path_count(), 1); +} + +TEST_F(LowerCaseFilterTest, ASCIIPreservesMetadataAndEmbeddedNul) { + auto input = std::make_shared( + std::vector {std::string("A\0B", 3), "already lower"}); + LowerCaseFilterFactory factory; + factory.initialize({}); + auto filter = factory.create(input); + filter->reset(); + + Token token; + ASSERT_NE(filter->next(&token), nullptr); + EXPECT_EQ(std::string(token.termBuffer(), token.termLength()), + std::string("a\0b", 3)); + EXPECT_EQ(token.getPositionIncrement(), 3); + EXPECT_EQ(token.startOffset(), 7); + EXPECT_EQ(token.endOffset(), 19); + EXPECT_EQ(std::wstring(token.type()), L"scripted"); + + ASSERT_NE(filter->next(&token), nullptr); + EXPECT_EQ(std::string(token.termBuffer(), token.termLength()), "already lower"); + EXPECT_EQ(token.getPositionIncrement(), 3); + EXPECT_EQ(token.startOffset(), 7); + EXPECT_EQ(token.endOffset(), 19); + EXPECT_EQ(std::wstring(token.type()), L"scripted"); +} + TEST_F(LowerCaseFilterTest, ConvertsUnicodeCharacters) { assert_filter_output("ÜBER ΜΈΓΑ", {{"über μέγα", 1}}); } +TEST_F(LowerCaseFilterTest, RetriesUnicodeExpansionWithRequiredBufferSize) { + assert_filter_output("\xC4\xB0", {{"i\xCC\x87", 1}}); +} + +TEST_F(LowerCaseFilterTest, RejectsInvalidUtf8WithAnalyzerError) { + auto input = std::make_shared( + std::vector {"VALID", std::string(1, static_cast(0xFF))}); + LowerCaseFilterFactory factory; + factory.initialize({}); + auto filter = factory.create(input); + filter->reset(); + + Token token; + ASSERT_NE(filter->next(&token), nullptr); + EXPECT_EQ(std::string(token.termBuffer(), token.termLength()), "valid"); + try { + filter->next(&token); + FAIL() << "expected malformed UTF-8 to fail analysis"; + } catch (const Exception& error) { + EXPECT_EQ(error.code(), ErrorCode::INVERTED_INDEX_ANALYZER_ERROR); + } +} + TEST_F(LowerCaseFilterTest, HandlesNumbersAndSymbols) { assert_filter_output("123!@# ABC", {{"123!@# abc", 1}}); } diff --git a/be/test/storage/index/inverted/tokenizer/char_group_tokenizer_factory_test.cpp b/be/test/storage/index/inverted/tokenizer/char_group_tokenizer_factory_test.cpp index 986074b052ce7e..a124c1793eb2ba 100644 --- a/be/test/storage/index/inverted/tokenizer/char_group_tokenizer_factory_test.cpp +++ b/be/test/storage/index/inverted/tokenizer/char_group_tokenizer_factory_test.cpp @@ -23,6 +23,11 @@ namespace doris::segment_v2::inverted_index { +namespace char_tokenizer_testing { +uint64_t non_ascii_decode_count(); +void reset_non_ascii_decode_count(); +} // namespace char_tokenizer_testing + class CharGroupTokenizerTest : public ::testing::Test { protected: std::vector tokenize(CharGroupTokenizerFactory& factory, const std::string& text) { @@ -65,6 +70,22 @@ TEST_F(CharGroupTokenizerTest, TokenizeOnSpace) { ASSERT_EQ(tokens, expected); } +TEST_F(CharGroupTokenizerTest, ASCIIUsesPrecomputedClassification) { + CharGroupTokenizerFactory factory; + Settings settings; + settings.set("tokenize_on_chars", "[whitespace], [punctuation]"); + factory.initialize(settings); + + char_tokenizer_testing::reset_non_ascii_decode_count(); + EXPECT_EQ(tokenize(factory, "Hello, ASCII world!"), + (std::vector {"Hello", "ASCII", "world"})); + EXPECT_EQ(char_tokenizer_testing::non_ascii_decode_count(), 0); + + EXPECT_EQ(tokenize(factory, "Hello \xE4\xB8\x96\xE7\x95\x8C"), + (std::vector {"Hello", "\xE4\xB8\x96\xE7\x95\x8C"})); + EXPECT_EQ(char_tokenizer_testing::non_ascii_decode_count(), 2); +} + TEST_F(CharGroupTokenizerTest, TokenizeOnLetter) { CharGroupTokenizerFactory factory; Settings settings; diff --git a/be/test/storage/index/inverted_index_parser_test.cpp b/be/test/storage/index/inverted_index_parser_test.cpp index 719f8fb170f21e..0fd2e46dc3107f 100644 --- a/be/test/storage/index/inverted_index_parser_test.cpp +++ b/be/test/storage/index/inverted_index_parser_test.cpp @@ -263,29 +263,28 @@ TEST_F(InvertedIndexParserTest, TestGetAnalyzerNameFromProperties) { EXPECT_EQ(get_analyzer_name_from_properties(properties), "another_analyzer"); } -TEST_F(InvertedIndexParserTest, TestInvertedIndexAnalyzerCtxShouldTokenize) { +TEST_F(InvertedIndexParserTest, TestInvertedIndexAnalyzerCtxRequiresAnalysis) { InvertedIndexAnalyzerCtx ctx; - // New design: should_tokenize() only depends on parser_type - // PARSER_NONE means no tokenization (keyword index) + // PARSER_NONE without a custom analyzer uses raw string matching. ctx.parser_type = InvertedIndexParserType::PARSER_NONE; ctx.analyzer_name.clear(); - EXPECT_FALSE(ctx.should_tokenize()); + EXPECT_FALSE(ctx.requires_analysis()); // Any parser other than NONE means tokenization ctx.parser_type = InvertedIndexParserType::PARSER_ENGLISH; - EXPECT_TRUE(ctx.should_tokenize()); + EXPECT_TRUE(ctx.requires_analysis()); ctx.parser_type = InvertedIndexParserType::PARSER_CHINESE; - EXPECT_TRUE(ctx.should_tokenize()); + EXPECT_TRUE(ctx.requires_analysis()); ctx.parser_type = InvertedIndexParserType::PARSER_STANDARD; - EXPECT_TRUE(ctx.should_tokenize()); + EXPECT_TRUE(ctx.requires_analysis()); - // Even with custom_analyzer name, PARSER_NONE means no tokenization + // A custom analyzer must execute even when its legacy parser type is NONE. ctx.parser_type = InvertedIndexParserType::PARSER_NONE; ctx.analyzer_name = "custom_analyzer"; - EXPECT_FALSE(ctx.should_tokenize()); + EXPECT_TRUE(ctx.requires_analysis()); } // Test constants @@ -369,13 +368,12 @@ TEST_F(InvertedIndexParserTest, NormalizeAnalyzerKey_AlreadyLowercase) { // ============================================================================ // build_analyzer_key_from_properties Tests -// New design: returns actual parser/analyzer name, empty means no properties +// New design: returns the physical analyzer key; raw indexes use "none". // ============================================================================ TEST_F(InvertedIndexParserTest, BuildAnalyzerKeyFromProperties_EmptyProperties) { std::map properties; - // Empty properties = empty key (no explicit configuration) - EXPECT_EQ(build_analyzer_key_from_properties(properties), ""); + EXPECT_EQ(build_analyzer_key_from_properties(properties), "none"); } TEST_F(InvertedIndexParserTest, BuildAnalyzerKeyFromProperties_CustomAnalyzer) { @@ -390,6 +388,12 @@ TEST_F(InvertedIndexParserTest, BuildAnalyzerKeyFromProperties_CustomAnalyzerUpp EXPECT_EQ(build_analyzer_key_from_properties(properties), "my_custom"); } +TEST_F(InvertedIndexParserTest, BuildAnalyzerKeyFromProperties_Normalizer) { + std::map properties; + properties[INVERTED_INDEX_NORMALIZER_NAME_KEY] = "MY_NORMALIZER"; + EXPECT_EQ(build_analyzer_key_from_properties(properties), "my_normalizer"); +} + TEST_F(InvertedIndexParserTest, BuildAnalyzerKeyFromProperties_ParserKey) { std::map properties; properties[INVERTED_INDEX_PARSER_KEY] = "chinese"; @@ -418,64 +422,104 @@ TEST_F(InvertedIndexParserTest, BuildAnalyzerKeyFromProperties_CustomOverridesPa EXPECT_EQ(build_analyzer_key_from_properties(properties), "my_custom"); } +TEST_F(InvertedIndexParserTest, BuildAnalyzerKeyFromProperties_NormalizerOverridesParser) { + std::map properties; + properties[INVERTED_INDEX_NORMALIZER_NAME_KEY] = "MY_NORMALIZER"; + properties[INVERTED_INDEX_PARSER_KEY] = "chinese"; + + EXPECT_EQ(build_analyzer_key_from_properties(properties), "my_normalizer"); +} + +TEST_F(InvertedIndexParserTest, BuildAnalyzerKeyFromProperties_AnalyzerOverridesNormalizer) { + std::map properties; + properties[INVERTED_INDEX_ANALYZER_NAME_KEY] = "MY_ANALYZER"; + properties[INVERTED_INDEX_NORMALIZER_NAME_KEY] = "MY_NORMALIZER"; + properties[INVERTED_INDEX_PARSER_KEY] = "chinese"; + + EXPECT_EQ(build_analyzer_key_from_properties(properties), "my_analyzer"); +} + // ============================================================================ // AnalyzerConfigParser Tests // ============================================================================ TEST_F(InvertedIndexParserTest, AnalyzerConfigParser_EmptyInput) { auto config = AnalyzerConfigParser::parse("", ""); - // New design: empty input gives empty analyzer_key (means "user did not specify") + // Empty selection keeps the legacy default analyzer execution semantics. EXPECT_EQ(config.analyzer_key, ""); - EXPECT_EQ(config.parser_type, InvertedIndexParserType::PARSER_NONE); - EXPECT_TRUE(config.custom_analyzer.empty()); - EXPECT_FALSE(config.is_custom()); + EXPECT_EQ(config.parser_type, InvertedIndexParserType::PARSER_UNKNOWN); + EXPECT_TRUE(config.provider_name.empty()); + EXPECT_FALSE(config.uses_provider()); } TEST_F(InvertedIndexParserTest, AnalyzerConfigParser_OnlyAnalyzerCustom) { auto config = AnalyzerConfigParser::parse("my_custom_analyzer", ""); - EXPECT_EQ(config.custom_analyzer, "my_custom_analyzer"); + EXPECT_EQ(config.provider_name, "my_custom_analyzer"); EXPECT_EQ(config.analyzer_key, "my_custom_analyzer"); EXPECT_EQ(config.parser_type, InvertedIndexParserType::PARSER_NONE); - EXPECT_TRUE(config.is_custom()); + EXPECT_TRUE(config.uses_provider()); } TEST_F(InvertedIndexParserTest, AnalyzerConfigParser_OnlyAnalyzerBuiltin) { auto config = AnalyzerConfigParser::parse("chinese", ""); - EXPECT_TRUE(config.custom_analyzer.empty()); + EXPECT_TRUE(config.provider_name.empty()); EXPECT_EQ(config.analyzer_key, "chinese"); EXPECT_EQ(config.parser_type, InvertedIndexParserType::PARSER_CHINESE); - EXPECT_FALSE(config.is_custom()); + EXPECT_FALSE(config.uses_provider()); } TEST_F(InvertedIndexParserTest, AnalyzerConfigParser_OnlyParserTypeStr) { auto config = AnalyzerConfigParser::parse("", "standard"); - EXPECT_TRUE(config.custom_analyzer.empty()); - EXPECT_EQ(config.analyzer_key, "standard"); + EXPECT_TRUE(config.provider_name.empty()); + EXPECT_TRUE(config.analyzer_key.empty()); EXPECT_EQ(config.parser_type, InvertedIndexParserType::PARSER_STANDARD); - EXPECT_FALSE(config.is_custom()); + EXPECT_FALSE(config.uses_provider()); + + config = AnalyzerConfigParser::parse("", "none"); + EXPECT_TRUE(config.provider_name.empty()); + EXPECT_TRUE(config.analyzer_key.empty()); + EXPECT_EQ(config.parser_type, InvertedIndexParserType::PARSER_NONE); + EXPECT_FALSE(config.uses_provider()); } TEST_F(InvertedIndexParserTest, AnalyzerConfigParser_BothAnalyzerAndParser) { - // parser_type_str takes precedence for determining parser_type + // A non-empty analyzer name takes precedence over the parser fallback. auto config = AnalyzerConfigParser::parse("ik", "chinese"); - EXPECT_TRUE(config.custom_analyzer.empty()); - EXPECT_EQ(config.parser_type, InvertedIndexParserType::PARSER_CHINESE); - EXPECT_EQ(config.analyzer_key, "ik"); // analyzer_name used for key + EXPECT_TRUE(config.provider_name.empty()); + EXPECT_EQ(config.parser_type, InvertedIndexParserType::PARSER_IK); + EXPECT_EQ(config.analyzer_key, "ik"); +} + +TEST_F(InvertedIndexParserTest, AnalyzerConfigParser_AnalyzerNameOverridesParserFallback) { + auto config = AnalyzerConfigParser::parse("none", "english"); + EXPECT_TRUE(config.provider_name.empty()); + EXPECT_EQ(config.parser_type, InvertedIndexParserType::PARSER_NONE); + EXPECT_EQ(config.analyzer_key, "none"); + + config = AnalyzerConfigParser::parse("ik", "chinese"); + EXPECT_TRUE(config.provider_name.empty()); + EXPECT_EQ(config.parser_type, InvertedIndexParserType::PARSER_IK); + EXPECT_EQ(config.analyzer_key, "ik"); + + config = AnalyzerConfigParser::parse("customer_analyzer", "english"); + EXPECT_EQ(config.provider_name, "customer_analyzer"); + EXPECT_EQ(config.parser_type, InvertedIndexParserType::PARSER_NONE); + EXPECT_EQ(config.analyzer_key, "customer_analyzer"); } TEST_F(InvertedIndexParserTest, AnalyzerConfigParser_CaseInsensitive) { auto config = AnalyzerConfigParser::parse("CHINESE", ""); - EXPECT_TRUE(config.custom_analyzer.empty()); + EXPECT_TRUE(config.provider_name.empty()); EXPECT_EQ(config.analyzer_key, "chinese"); EXPECT_EQ(config.parser_type, InvertedIndexParserType::PARSER_CHINESE); } TEST_F(InvertedIndexParserTest, AnalyzerConfigParser_UnknownAnalyzerAsCustom) { auto config = AnalyzerConfigParser::parse("unknown_xyz", ""); - EXPECT_EQ(config.custom_analyzer, "unknown_xyz"); + EXPECT_EQ(config.provider_name, "unknown_xyz"); EXPECT_EQ(config.analyzer_key, "unknown_xyz"); EXPECT_EQ(config.parser_type, InvertedIndexParserType::PARSER_NONE); - EXPECT_TRUE(config.is_custom()); + EXPECT_TRUE(config.uses_provider()); } TEST_F(InvertedIndexParserTest, AnalyzerConfigParser_AllBuiltinTypes) { @@ -494,8 +538,8 @@ TEST_F(InvertedIndexParserTest, AnalyzerConfigParser_AllBuiltinTypes) { for (const auto& [name, expected_type] : builtin_types) { auto config = AnalyzerConfigParser::parse(name, ""); EXPECT_EQ(config.parser_type, expected_type) << "Failed for: " << name; - EXPECT_TRUE(config.custom_analyzer.empty()) << "Failed for: " << name; - EXPECT_FALSE(config.is_custom()) << "Failed for: " << name; + EXPECT_TRUE(config.provider_name.empty()) << "Failed for: " << name; + EXPECT_FALSE(config.uses_provider()) << "Failed for: " << name; } } diff --git a/be/test/storage/index/snii/bench/bkd_native_vs_clucene_bench_test.cpp b/be/test/storage/index/snii/bench/bkd_native_vs_clucene_bench_test.cpp new file mode 100644 index 00000000000000..6f7d06cba1211f --- /dev/null +++ b/be/test/storage/index/snii/bench/bkd_native_vs_clucene_bench_test.cpp @@ -0,0 +1,785 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// The SNII-native BKD against the CLucene BKD it replaces (design 12 / task P4-1). +// +// WHY THIS EXISTS: design 11's comparison table is a STRUCTURAL argument, not a +// measurement, and the design says so -- no performance claim from it belongs in +// a PR description until this runs. One entry in that table is explicitly a +// REVERSIBLE decision awaiting evidence: the leaf layout puts values first and +// records a docid_block_offset, betting that one extra offset parse on a +// whole-leaf hit costs less than the old layout's skip-read on a boundary leaf. +// If the bet is wrong the layout should be flipped back. The two range cases +// below are shaped to answer exactly that: +// +// range_wide - spans many leaves, so almost every leaf is a WHOLE-leaf hit +// and the offset-parse cost dominates. +// range_narrow - touches one or two leaves, both BOUNDARY leaves, so the +// value-scan-without-skip path dominates. +// +// A native win on both vindicates the layout. A native loss on range_wide with a +// win on range_narrow is the signal to flip it. +// +// Why CPU time is the headline: this machine is shared and hybrid-core, so wall +// clock moves with whatever else runs. Process CPU time barely does. Wall is +// still reported -- a large wall/CPU gap means the run was descheduled and +// should be repeated. +// +// Pin to a performance core; on a hybrid CPU an E-core sample is not comparable +// to a P-core one and mixing them silently widens every percentile: +// +// taskset -c 4 env SNII_BKD_BENCH_POINTS=2000000 SNII_BKD_BENCH_ITERATIONS=30 \ +// ./run-be-ut.sh --run --filter='*BkdNativeVsClucene*' -j 28 +// +// Both indexes are built from the SAME encoded points in the SAME process, so +// the comparison isolates the implementation and not the data or the machine. +// +// DISABLED_ so CI never runs it; the filter above opts in. + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/check.h" +#include "common/config.h" +#include "storage/index/inverted/inverted_index_fs_directory.h" +#include "storage/index/snii/bkd/bkd_builder.h" +#include "storage/index/snii/bkd/bkd_index_block.h" +#include "storage/index/snii/bkd/bkd_reader.h" +#include "storage/index/snii/bkd/leaf_codec.h" +#include "storage/index/snii/encoding/byte_sink.h" +#include "storage/index/snii/io/file_writer.h" +#include "storage/key_coder.h" +#include "storage/olap_common.h" +#include "util/time.h" + +namespace doris::snii::bkd { +namespace { + +constexpr FieldType kFieldType = FieldType::OLAP_FIELD_TYPE_BIGINT; +constexpr uint32_t kBytesPerDim = sizeof(int64_t); +constexpr uint32_t kDefaultPoints = 2000000; +constexpr int kDefaultIterations = 30; + +struct Measurement { + double wall_s = 0; + double cpu_s = 0; +}; + +Measurement measure(const std::function& body) { + timespec cpu_start {}; + timespec cpu_end {}; + clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &cpu_start); + const int64_t wall_start = MonotonicNanos(); + body(); + const int64_t wall_end = MonotonicNanos(); + clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &cpu_end); + Measurement m; + m.wall_s = static_cast(wall_end - wall_start) / 1e9; + m.cpu_s = static_cast(cpu_end.tv_sec - cpu_start.tv_sec) + + static_cast(cpu_end.tv_nsec - cpu_start.tv_nsec) / 1e9; + return m; +} + +// Nearest-rank, matching the SNII/V3 benchmark so the two report the same statistic. +double nearest_rank_percentile(const std::vector& sorted, size_t percentile) { + DORIS_CHECK(!sorted.empty()); + DORIS_CHECK(percentile > 0 && percentile <= 100); + const size_t whole_hundreds = sorted.size() / 100; + const size_t remainder = sorted.size() % 100; + const size_t rank = whole_hundreds * percentile + (remainder * percentile + 99) / 100; + return sorted[rank - 1]; +} + +double mean_of(const std::vector& xs) { + return std::accumulate(xs.begin(), xs.end(), 0.0) / static_cast(xs.size()); +} + +double stddev_of(const std::vector& xs) { + if (xs.size() < 2) { + return 0.0; + } + const double m = mean_of(xs); + double acc = 0; + for (const double x : xs) { + acc += (x - m) * (x - m); + } + return std::sqrt(acc / static_cast(xs.size() - 1)); +} + +int env_int(const char* name, int fallback) { + const char* const raw = std::getenv(name); + if (raw == nullptr) { + return fallback; + } + const int value = std::atoi(raw); + return value > 0 ? value : fallback; +} + +std::string encode(int64_t value) { + std::string out; + get_key_coder(kFieldType)->full_encode_ascending(&value, &out); + return out; +} + +Slice slice_of(const std::string& bytes) { + return Slice(reinterpret_cast(bytes.data()), bytes.size()); +} + +struct EncodedPoint { + std::string value; + int64_t raw = 0; + uint32_t doc_id = 0; +}; + +// A skewed-but-not-degenerate key distribution: a wide numeric domain with +// duplicates, which is what a real id / timestamp / metric column looks like. +// A uniform permutation would make every leaf equally selective and hide the +// boundary-leaf behaviour the layout question is about. +std::vector make_points(uint32_t count, int64_t span) { + std::vector points; + points.reserve(count); + uint64_t state = 0x9E3779B97F4A7C15ULL; + for (uint32_t i = 0; i < count; ++i) { + state = state * 6364136223846793005ULL + 1442695040888963407ULL; + const int64_t raw = static_cast(state % static_cast(2 * span)) - span; + points.push_back(EncodedPoint {encode(raw), raw, i}); + } + return points; +} + +// Collects bkd_data in memory so the measurement is CPU, not page cache. +class MemoryFileWriter final : public io::FileWriter { +public: + Status append(Slice data) override { + bytes_.insert(bytes_.end(), data.data(), data.data() + data.size()); + return Status::OK(); + } + Status finalize() override { return Status::OK(); } + uint64_t bytes_written() const override { return bytes_.size(); } + const std::vector& bytes() const { return bytes_; } + +private: + std::vector bytes_; +}; + +// --------------------------------------------------------------------------- +// The CLucene baseline, driven exactly as InvertedIndexColumnWriter drives it +// (DIMS 1, MAX_LEAF_COUNT 1024, total_point_count INT32_MAX, +// single_value_per_doc true, docs_seen_/max_doc_ pushed in before finish). +// Reproducing the production call sequence is the whole point of a baseline. +// --------------------------------------------------------------------------- +using RamDirPtr = std::unique_ptr; + +class CluceneBkd { +public: + ~CluceneBkd() { + reader_.reset(); + if (dir_) { + dir_->close(); + } + } + + void build(const std::vector& points) { + uint32_t max_doc = 0; + std::set distinct; + for (const EncodedPoint& p : points) { + max_doc = std::max(max_doc, p.doc_id + 1); + distinct.insert(p.doc_id); + } + dir_ = RamDirPtr(_CLNEW lucene::store::RAMDirectory()); + auto writer = std::make_shared( + static_cast(max_doc), 1, 1, static_cast(kBytesPerDim), + /*maxPointsInLeafNode=*/1024, /*maxMBSortInHeap=*/512.0, + /*totalPointCount=*/std::numeric_limits::max(), + /*singleValuePerDoc=*/true, config::max_depth_in_bkd_tree); + for (const EncodedPoint& p : points) { + writer->add(reinterpret_cast(p.value.data()), kBytesPerDim, + static_cast(p.doc_id)); + } + writer->max_doc_ = static_cast(max_doc); + writer->docs_seen_ = static_cast(distinct.size()); + + std::unique_ptr data_out(dir_->createOutput("bkd")); + std::unique_ptr index_out(dir_->createOutput("bkd_index")); + std::unique_ptr meta_out(dir_->createOutput("bkd_meta")); + const int64_t index_fp = writer->finish(data_out.get(), index_out.get()); + writer->meta_finish(meta_out.get(), index_fp, 0); + bytes_ = data_out->getFilePointer() + index_out->getFilePointer() + + meta_out->getFilePointer(); + data_out->close(); + index_out->close(); + meta_out->close(); + + reader_ = std::make_shared(dir_.get(), false); + DORIS_CHECK(reader_->open()); + } + + uint64_t bytes() const { return static_cast(bytes_); } + lucene::util::bkd::bkd_reader* reader() const { return reader_.get(); } + +private: + RamDirPtr dir_; + std::shared_ptr reader_; + int64_t bytes_ = 0; +}; + +// The visitor shape InvertedIndexVisitor uses: a closed [min, max] box with the +// strictness folded into the bounds, which is the only interval the old reader +// can express. +class RangeVisitor : public lucene::util::bkd::bkd_reader::intersect_visitor { +public: + RangeVisitor(std::string low, std::string high, roaring::Roaring* hits) + : low_(std::move(low)), high_(std::move(high)), hits_(hits) {} + + void visit(int docid) override { hits_->add(static_cast(docid)); } + void visit(roaring::Roaring& docids) override { *hits_ |= docids; } + void visit(roaring::Roaring&& docids) override { visit(docids); } + int visit(int docid, std::vector& packed) override { + if (accepts(packed)) { + hits_->add(static_cast(docid)); + } + return 0; + } + void visit(std::vector& docids, std::vector& packed) override { + if (!accepts(packed)) { + return; + } + auto bitmap = roaring::Roaring::read(docids.data(), false); + visit(bitmap); + } + void visit(roaring::Roaring* docids, std::vector& packed) override { + if (accepts(packed)) { + visit(*docids); + } + } + void visit(lucene::util::bkd::bkd_docid_set_iterator* iter, + std::vector& packed) override { + if (!accepts(packed)) { + return; + } + int32_t docid = iter->docid_set->nextDoc(); + while (docid != lucene::util::bkd::bkd_docid_set::NO_MORE_DOCS) { + hits_->add(static_cast(docid)); + docid = iter->docid_set->nextDoc(); + } + } + + lucene::util::bkd::relation compare(std::vector& min_packed, + std::vector& max_packed) override { + if (cmp(max_packed, low_) < 0 || cmp(min_packed, high_) > 0) { + return lucene::util::bkd::relation::CELL_OUTSIDE_QUERY; + } + if (cmp(min_packed, low_) >= 0 && cmp(max_packed, high_) <= 0) { + return lucene::util::bkd::relation::CELL_INSIDE_QUERY; + } + return lucene::util::bkd::relation::CELL_CROSSES_QUERY; + } + + lucene::util::bkd::relation compare_prefix(std::vector&) override { + return lucene::util::bkd::relation::CELL_CROSSES_QUERY; + } + +private: + static int cmp(const std::vector& packed, const std::string& bound) { + return std::memcmp(packed.data(), bound.data(), kBytesPerDim); + } + bool accepts(const std::vector& packed) const { + return cmp(packed, low_) >= 0 && cmp(packed, high_) <= 0; + } + std::string low_; + std::string high_; + roaring::Roaring* hits_; +}; + +struct QueryCase { + const char* label; + int64_t low; + int64_t high; +}; + +// Reports one metric's spread. stddev is printed alongside the percentiles +// because a p50 with a stddev of its own magnitude is not a measurement. +void report(const char* format, const char* label, std::vector cpu, + std::vector wall) { + std::sort(cpu.begin(), cpu.end()); + std::sort(wall.begin(), wall.end()); + printf(" %-8s %-14s cpu p50=%9.3f ms p99=%9.3f ms mean=%9.3f ms sd=%8.3f ms | " + "wall p50=%9.3f ms\n", + format, label, nearest_rank_percentile(cpu, 50) * 1e3, + nearest_rank_percentile(cpu, 99) * 1e3, mean_of(cpu) * 1e3, stddev_of(cpu) * 1e3, + nearest_rank_percentile(wall, 50) * 1e3); +} + +// Serves the two sub-files from one contiguous buffer, exactly as the container +// lays them out, so query cost is CPU and not storage. +class ConcatReader final : public io::FileReader { +public: + explicit ConcatReader(const std::vector* bytes) : bytes_(bytes) {} + Status read_at(uint64_t offset, size_t len, std::vector* out) override { + out->resize(len); + return read_into(offset, out->data(), len); + } + Status read_into(uint64_t offset, uint8_t* out, size_t len) override { + DORIS_CHECK(offset + len <= bytes_->size()); + std::memcpy(out, bytes_->data() + offset, len); + return Status::OK(); + } + uint64_t size() const override { return bytes_->size(); } + +private: + const std::vector* bytes_; +}; + +class BkdNativeVsCluceneBench : public ::testing::Test {}; + +// --------------------------------------------------------------------------- + +TEST_F(BkdNativeVsCluceneBench, DISABLED_BuildAndQuery) { + const uint32_t point_count = + static_cast(env_int("SNII_BKD_BENCH_POINTS", kDefaultPoints)); + const int iterations = env_int("SNII_BKD_BENCH_ITERATIONS", kDefaultIterations); + const int64_t span = 1 << 20; + + printf("\n=== SNII-native BKD vs CLucene BKD ===\n"); + printf("points=%u span=+/-%ld iterations=%d points_per_leaf=%u\n", point_count, span, + iterations, kDefaultPointsPerLeaf); + + const std::vector points = make_points(point_count, span); + + // ---- build ---- + std::vector native_index_bytes; + MemoryFileWriter native_data; + BkdStats stats; + const Measurement native_build = measure([&] { + BkdBuilderOptions options; + options.bytes_per_dim = kBytesPerDim; + options.field_type = kFieldType; + std::unique_ptr builder; + DORIS_CHECK(BkdBuilder::create(options, &builder).ok()); + for (const EncodedPoint& p : points) { + DORIS_CHECK(builder->add(p.doc_id, slice_of(p.value)).ok()); + } + ByteSink index; + DORIS_CHECK(builder->finish(&native_data, &index, &stats).ok()); + native_index_bytes = index.take(); + }); + + CluceneBkd clucene; + const Measurement clucene_build = measure([&] { clucene.build(points); }); + + const uint64_t native_bytes = native_index_bytes.size() + native_data.bytes().size(); + printf("\nbuild native cpu=%8.3f s bytes=%10lu leaves=%u\n", native_build.cpu_s, + native_bytes, stats.leaf_count); + printf("build clucene cpu=%8.3f s bytes=%10lu\n", clucene_build.cpu_s, clucene.bytes()); + printf("build ratio cpu=%8.3fx bytes=%8.3fx (>1 means native is worse)\n", + native_build.cpu_s / clucene_build.cpu_s, + static_cast(native_bytes) / static_cast(clucene.bytes())); + + // ---- query ---- + BkdSections sections; + sections.index_offset = 0; + sections.index_length = native_index_bytes.size(); + sections.data_offset = 0; + sections.data_length = native_data.bytes().size(); + + // The native reader addresses index and data by absolute offset in ONE + // stream, so the two sub-files are concatenated exactly as the container + // lays them out. + std::vector concatenated = native_index_bytes; + const uint64_t data_offset = concatenated.size(); + concatenated.insert(concatenated.end(), native_data.bytes().begin(), native_data.bytes().end()); + sections.data_offset = data_offset; + + ConcatReader reader_source(&concatenated); + std::unique_ptr native; + DORIS_CHECK(BkdReader::open(&reader_source, sections, &native).ok()); + + // range_wide spans many leaves (whole-leaf hits dominate); range_narrow + // touches one or two (boundary leaves dominate). eq is the degenerate + // boundary case. + // eq must probe a value the dataset actually CONTAINS. Probing an absent + // value times the early-exit path -- the reader proves emptiness from the + // leaf directory and never reads a leaf -- which is a different question + // from what a point lookup costs. + const int64_t present = points[points.size() / 2].raw; + const std::vector cases = { + {"eq", present, present}, + {"range_narrow", -16, 16}, + {"range_mid", -span / 64, span / 64}, + {"range_wide", -span / 2, span / 2}, + // One-sided unbounded: the native reader leaves the open side genuinely + // unbounded while the baseline must encode a type-limit sentinel and + // compare against it at every leaf. This is the largest win end to end + // and had no unit-level counterpart until now. + {"lt", std::numeric_limits::min(), -1}, + }; + + printf("\n"); + for (const QueryCase& c : cases) { + const std::string low = encode(c.low); + const std::string high = encode(c.high); + + std::vector native_cpu; + std::vector native_wall; + std::vector clucene_cpu; + std::vector clucene_wall; + uint64_t native_hits = 0; + uint64_t clucene_hits = 0; + + for (int i = 0; i < iterations; ++i) { + roaring::Roaring hits; + const Measurement m = measure([&] { + DORIS_CHECK(native->range(slice_of(low), true, slice_of(high), true, &hits).ok()); + }); + native_cpu.push_back(m.cpu_s); + native_wall.push_back(m.wall_s); + native_hits = hits.cardinality(); + } + for (int i = 0; i < iterations; ++i) { + roaring::Roaring hits; + RangeVisitor visitor(low, high, &hits); + const Measurement m = measure([&] { clucene.reader()->intersect(&visitor); }); + clucene_cpu.push_back(m.cpu_s); + clucene_wall.push_back(m.wall_s); + clucene_hits = hits.cardinality(); + } + + // A performance comparison between two implementations that disagree on + // the ANSWER is meaningless; assert equality before reporting. + ASSERT_EQ(native_hits, clucene_hits) << "case " << c.label << " disagrees on the result"; + + printf("%s (hits=%lu)\n", c.label, native_hits); + report("native", c.label, native_cpu, native_wall); + report("clucene", c.label, clucene_cpu, clucene_wall); + std::sort(native_cpu.begin(), native_cpu.end()); + std::sort(clucene_cpu.begin(), clucene_cpu.end()); + printf(" ratio %-14s cpu p50=%8.3fx (>1 means native is slower)\n\n", c.label, + nearest_rank_percentile(native_cpu, 50) / nearest_rank_percentile(clucene_cpu, 50)); + } +} + +// IN (...) as it will actually arrive: one lookup_many pass over N ascending, +// deduplicated values against N independent traversals on the baseline side. +// N is large on purpose -- the 5-value case used end to end lands in noise and +// cannot show the ">= min(N, leaves) leaf reads" claim at all. +TEST_F(BkdNativeVsCluceneBench, DISABLED_InListManyValues) { + const uint32_t point_count = + static_cast(env_int("SNII_BKD_BENCH_POINTS", kDefaultPoints)); + const int iterations = env_int("SNII_BKD_BENCH_ITERATIONS", kDefaultIterations); + const int n_values = env_int("SNII_BKD_BENCH_IN_VALUES", 256); + const int64_t span = 1 << 20; + const std::vector points = make_points(point_count, span); + + printf("\n=== IN (%d values), points=%u, iterations=%d ===\n", n_values, point_count, + iterations); + + BkdBuilderOptions options; + options.bytes_per_dim = kBytesPerDim; + options.field_type = kFieldType; + MemoryFileWriter data; + std::vector index_bytes; + BkdStats stats; + { + std::unique_ptr builder; + DORIS_CHECK(BkdBuilder::create(options, &builder).ok()); + for (const EncodedPoint& p : points) { + DORIS_CHECK(builder->add(p.doc_id, slice_of(p.value)).ok()); + } + ByteSink index; + DORIS_CHECK(builder->finish(&data, &index, &stats).ok()); + index_bytes = index.take(); + } + std::vector concatenated = index_bytes; + BkdSections sections; + sections.index_offset = 0; + sections.index_length = index_bytes.size(); + sections.data_offset = concatenated.size(); + sections.data_length = data.bytes().size(); + concatenated.insert(concatenated.end(), data.bytes().begin(), data.bytes().end()); + ConcatReader source(&concatenated); + std::unique_ptr reader; + DORIS_CHECK(BkdReader::open(&source, sections, &reader).ok()); + + CluceneBkd clucene; + clucene.build(points); + + // Ascending and deduplicated, as lookup_many requires. Drawn from values the + // dataset holds so the probes are real hits. + std::set wanted; + for (int i = 0; i < n_values; ++i) { + wanted.insert(points[(static_cast(i) * 7919) % points.size()].raw); + } + std::vector encoded; + for (const int64_t v : wanted) { + encoded.push_back(encode(v)); + } + std::vector probes; + for (const std::string& e : encoded) { + probes.push_back(slice_of(e)); + } + + std::vector native_cpu; + std::vector native_wall; + uint64_t native_hits = 0; + for (int i = 0; i < iterations; ++i) { + roaring::Roaring hits; + const Measurement m = + measure([&] { DORIS_CHECK(reader->lookup_many(probes, &hits).ok()); }); + native_cpu.push_back(m.cpu_s); + native_wall.push_back(m.wall_s); + native_hits = hits.cardinality(); + } + + std::vector clucene_cpu; + std::vector clucene_wall; + uint64_t clucene_hits = 0; + for (int i = 0; i < iterations; ++i) { + roaring::Roaring hits; + const Measurement m = measure([&] { + // The baseline shape: one full traversal per value, unioned. + for (const std::string& e : encoded) { + RangeVisitor visitor(e, e, &hits); + clucene.reader()->intersect(&visitor); + } + }); + clucene_cpu.push_back(m.cpu_s); + clucene_wall.push_back(m.wall_s); + clucene_hits = hits.cardinality(); + } + + ASSERT_EQ(native_hits, clucene_hits) << "in_list disagrees on the result"; + printf("in_list (values=%zu, hits=%lu)\n", encoded.size(), native_hits); + report("native", "in_list", native_cpu, native_wall); + report("clucene", "in_list", clucene_cpu, clucene_wall); + std::sort(native_cpu.begin(), native_cpu.end()); + std::sort(clucene_cpu.begin(), clucene_cpu.end()); + printf(" ratio %-14s cpu p50=%8.3fx (>1 means native is slower)\n\n", "in_list", + nearest_rank_percentile(native_cpu, 50) / nearest_rank_percentile(clucene_cpu, 50)); +} + +// How much of a large-result query is IRREDUCIBLE? +// +// Both implementations must materialize the same doc ids into the same roaring +// bitmap. That construction is a floor neither can optimize away, so it bounds +// how fast either can possibly get. This measures the floor directly instead of +// inferring it: the bitmap is rebuilt from an already-decoded doc id array, with +// no index work at all in the timed region. +// +// If the floor is a large fraction of the measured query time, then a target +// expressed as "N% faster than the baseline" is arithmetically out of reach for +// this operator regardless of how good the index is, and the honest response is +// to say so rather than keep optimizing. +TEST_F(BkdNativeVsCluceneBench, DISABLED_ResultMaterializationFloor) { + const uint32_t point_count = + static_cast(env_int("SNII_BKD_BENCH_POINTS", kDefaultPoints)); + const int iterations = env_int("SNII_BKD_BENCH_ITERATIONS", kDefaultIterations); + const int64_t span = 1 << 20; + const std::vector points = make_points(point_count, span); + + BkdBuilderOptions options; + options.bytes_per_dim = kBytesPerDim; + options.field_type = kFieldType; + MemoryFileWriter data; + std::vector index_bytes; + BkdStats stats; + { + std::unique_ptr builder; + DORIS_CHECK(BkdBuilder::create(options, &builder).ok()); + for (const EncodedPoint& p : points) { + DORIS_CHECK(builder->add(p.doc_id, slice_of(p.value)).ok()); + } + ByteSink index; + DORIS_CHECK(builder->finish(&data, &index, &stats).ok()); + index_bytes = index.take(); + } + std::vector concatenated = index_bytes; + BkdSections sections; + sections.index_offset = 0; + sections.index_length = index_bytes.size(); + sections.data_offset = concatenated.size(); + sections.data_length = data.bytes().size(); + concatenated.insert(concatenated.end(), data.bytes().begin(), data.bytes().end()); + ConcatReader source(&concatenated); + std::unique_ptr reader; + DORIS_CHECK(BkdReader::open(&source, sections, &reader).ok()); + + const std::string low = encode(-span / 2); + const std::string high = encode(span / 2); + + // The full query, for reference. + std::vector full; + roaring::Roaring answer; + for (int i = 0; i < iterations; ++i) { + roaring::Roaring hits; + full.push_back( + measure([&] { + DORIS_CHECK( + reader->range(slice_of(low), true, slice_of(high), true, &hits).ok()); + }).cpu_s); + answer = hits; + } + + // The same doc ids, already decoded, inserted into a fresh bitmap. No index, + // no decode -- only the materialization every implementation has to pay. + std::vector docids; + docids.reserve(answer.cardinality()); + for (const uint32_t d : answer) { + docids.push_back(d); + } + std::vector floor_only; + for (int i = 0; i < iterations; ++i) { + roaring::Roaring rebuilt; + floor_only.push_back(measure([&] { rebuilt.addMany(docids.size(), docids.data()); }).cpu_s); + DORIS_CHECK(rebuilt.cardinality() == answer.cardinality()); + } + + // Component isolation: the same leaves, read and decoded, with NOTHING + // inserted into a bitmap. Subtracting this from the full query separates + // "getting the doc ids out of the format" from "putting them in the answer". + // Guessing at this split produced three failed experiments; measuring it + // takes one run. Done entirely through public API -- the benchmark opens its + // own index block for the leaf extents rather than adding a bench-only + // accessor to the reader. + std::vector decode_only; + { + BkdIndexBlockReader block; + DORIS_CHECK( + BkdIndexBlockReader::open(Slice(index_bytes), sections.data_length, &block).ok()); + const uint32_t leaf_count = block.leaf_count(); + const uint8_t* data_base = concatenated.data() + sections.data_offset; + for (int i = 0; i < iterations; ++i) { + uint64_t sink = 0; + std::vector ids; + decode_only.push_back( + measure([&] { + for (uint32_t leaf = 0; leaf < leaf_count; ++leaf) { + const uint64_t off = block.leaf(leaf).offset; + const uint64_t end_off = (leaf + 1 < leaf_count) + ? block.leaf(leaf + 1).offset + : sections.data_length; + const Slice blk(data_base + off, static_cast(end_off - off)); + if (decode_leaf_doc_ids(blk, kBytesPerDim, block.leaf(leaf).count, &ids) + .ok()) { + sink += ids.size(); + } + } + }).cpu_s); + DORIS_CHECK(sink > 0); + } + std::sort(decode_only.begin(), decode_only.end()); + } + + std::sort(full.begin(), full.end()); + std::sort(floor_only.begin(), floor_only.end()); + const double q = nearest_rank_percentile(full, 50) * 1e3; + const double f = nearest_rank_percentile(floor_only, 50) * 1e3; + printf("\n=== result materialization floor (range_wide, %lu hits) ===\n", answer.cardinality()); + printf(" full query p50 = %8.3f ms\n", q); + printf(" bitmap build only p50 = %8.3f ms (%.1f%% of the query)\n", f, 100.0 * f / q); + printf(" index work p50 = %8.3f ms\n", q - f); + printf(" => even a FREE index could not go below %.3f ms on this operator.\n", f); + printf(" read+decode ALL %u leaves p50 = %8.3f ms (no bitmap at all)\n", reader->leaf_count(), + nearest_rank_percentile(decode_only, 50) * 1e3); +} + +// points_per_leaf calibration. The default is 1024 because that is what the +// CLucene writer used, not because anything measured it here. A larger leaf +// reads more bytes for a boundary hit but cuts the split array and the number of +// leaf reads a wide range performs; the crossover is what this sweep locates. +TEST_F(BkdNativeVsCluceneBench, DISABLED_PointsPerLeafSweep) { + const uint32_t point_count = + static_cast(env_int("SNII_BKD_BENCH_POINTS", kDefaultPoints)); + const int iterations = env_int("SNII_BKD_BENCH_ITERATIONS", kDefaultIterations); + const int64_t span = 1 << 20; + const std::vector points = make_points(point_count, span); + + printf("\n=== points_per_leaf sweep (points=%u, iterations=%d) ===\n", point_count, iterations); + printf("%8s %10s %10s %12s %12s %12s\n", "ppl", "leaves", "bytes", "build_cpu_s", + "narrow_p50ms", "wide_p50ms"); + + for (const uint32_t ppl : {128U, 256U, 512U, 1024U, 2048U, 4096U}) { + BkdBuilderOptions options; + options.bytes_per_dim = kBytesPerDim; + options.field_type = kFieldType; + options.points_per_leaf = ppl; + + MemoryFileWriter data; + std::vector index_bytes; + BkdStats stats; + const Measurement build = measure([&] { + std::unique_ptr builder; + DORIS_CHECK(BkdBuilder::create(options, &builder).ok()); + for (const EncodedPoint& p : points) { + DORIS_CHECK(builder->add(p.doc_id, slice_of(p.value)).ok()); + } + ByteSink index; + DORIS_CHECK(builder->finish(&data, &index, &stats).ok()); + index_bytes = index.take(); + }); + + std::vector concatenated = index_bytes; + BkdSections sections; + sections.index_offset = 0; + sections.index_length = index_bytes.size(); + sections.data_offset = concatenated.size(); + sections.data_length = data.bytes().size(); + concatenated.insert(concatenated.end(), data.bytes().begin(), data.bytes().end()); + + ConcatReader source(&concatenated); + std::unique_ptr reader; + DORIS_CHECK(BkdReader::open(&source, sections, &reader).ok()); + + const auto time_case = [&](int64_t low, int64_t high) { + const std::string l = encode(low); + const std::string h = encode(high); + std::vector cpu; + for (int i = 0; i < iterations; ++i) { + roaring::Roaring hits; + cpu.push_back(measure([&] { + DORIS_CHECK( + reader->range(slice_of(l), true, slice_of(h), true, &hits) + .ok()); + }).cpu_s); + } + std::sort(cpu.begin(), cpu.end()); + return nearest_rank_percentile(cpu, 50) * 1e3; + }; + + printf("%8u %10u %10lu %12.3f %12.3f %12.3f\n", ppl, stats.leaf_count, + index_bytes.size() + data.bytes().size(), build.cpu_s, time_case(-16, 16), + time_case(-span / 2, span / 2)); + } +} + +} // namespace +} // namespace doris::snii::bkd diff --git a/be/test/storage/index/snii/bench/snii_vs_v3_benchmark_test.cpp b/be/test/storage/index/snii/bench/snii_vs_v3_benchmark_test.cpp new file mode 100644 index 00000000000000..cc0340b8369bab --- /dev/null +++ b/be/test/storage/index/snii/bench/snii_vs_v3_benchmark_test.cpp @@ -0,0 +1,2010 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// +// SNII vs V3 baseline over the same wikipedia corpus, covering the three phases that matter for the +// format: building the index while loading, compacting it, and querying it across MATCH_ANY, +// MATCH_ALL, MATCH_PHRASE and MATCH_PHRASE_PREFIX. +// +// Why this lives in a unit test rather than a cluster benchmark: a UT iterates in minutes instead +// of a deploy cycle, and it measures one process we control end to end. +// +// Why CPU time is the headline number: this machine is shared, so wall clock moves with whatever +// else is running. Process CPU time barely does. Wall time is still reported -- a large wall/CPU +// gap means the run was IO bound or descheduled and the comparison should be rerun -- but the +// SNII/V3 verdict is taken from CPU. +// +// Corpus is not committed (~41 MB). Point SNII_BENCH_CORPUS_DIR at a directory of wikipedia_*.json +// with {"title","content"} per line: +// +// SNII_BENCH_CORPUS_DIR=/path/to/corpus \ +// SNII_BENCH_QUERY_ITERATIONS=30 \ +// ./run-be-ut.sh --run --filter='*SniiVsV3Benchmark*' -j +// +// SNII_BENCH_QUERY_ITERATIONS defaults to 30. The benchmark reports nearest-rank +// p50/p99 query CPU and wall time from the sorted per-iteration samples. +// +// SNII_BENCH_QUERY_INPUT_ROWSETS=1 points the query phase at the input rowsets instead of the +// compacted output. With one corpus file per rowset that is the shape of a tablet still ingesting: +// N indexes open at once and N rounds of remote reads per cold query, rather than one merged index. +// It also skips compaction, whose only purpose here is to produce the rowset the query reads -- +// compaction is benchmarked by the default compacted-output mode, where it is on the measured path. +// +// SNII_BENCH_ONLY_CASE=