From c597b0e45369ac7c9f278c196b520764f101d99b Mon Sep 17 00:00:00 2001 From: airborne12 Date: Tue, 28 Jul 2026 18:39:58 +0800 Subject: [PATCH 01/43] [feature](be) Add SNII inverted index storage format Issue Number: None Related PR: None Problem Summary: Add SNII as a complete inverted-index storage format for Doris. The implementation provides streamed index writing and reading, protobuf v1 metadata, CommonGrams acceleration, exact and analyzed query operators, phrase and sloppy-phrase execution, scoring, native index compaction, local and cloud index-change handling, remote IO and cache profiling, and an opt-in SNII-versus-V3 benchmark. This commit also includes the FE validation, protobuf and thrift definitions, BE unit tests, and regression suites required to exercise the format. The complete feature history is squashed into one atomic commit so the implementation can be reviewed and rebased as a single change. Add the SNII inverted-index storage format with protobuf metadata, query execution, scoring, and compaction support. - Test: Unit Test and Regression test - ASAN BE build passed - Targeted SNII, inverted-index, collection-statistics, and empty-index BE unit tests passed - SNII storage-format, exact-search, BM25, and empty-variant-index regression suites passed - clang-format 16 check passed - Squash verification confirmed the resulting tree is byte-for-byte identical to the pre-squash HEAD - clang-tidy 16 was attempted; analysis is blocked by pre-existing template and benchmark diagnostics plus a missing sanitizer header in the clang-tidy toolchain - Behavior changed: Yes. Doris gains the SNII inverted-index storage format and its associated query, scoring, compaction, and index-change behavior. - Does this need documentation: No --- .../cloud/cloud_index_change_compaction.cpp | 7 +- be/src/common/config.cpp | 189 ++ be/src/common/config.h | 74 + be/src/common/status.h | 1 + be/src/exec/operator/olap_scan_operator.cpp | 6 + be/src/exec/operator/olap_scan_operator.h | 5 + be/src/exec/scan/olap_scanner.cpp | 12 +- be/src/exprs/function/function_search.cpp | 116 +- be/src/exprs/function/function_tokenize.cpp | 2 +- be/src/exprs/function/match.cpp | 12 +- .../variant_inverted_index_search.cpp | 46 +- .../function/variant_inverted_index_search.h | 13 +- be/src/exprs/vcompound_pred.h | 15 + be/src/exprs/vexpr.h | 4 + be/src/exprs/vmatch_predicate.cpp | 4 +- be/src/exprs/vmatch_predicate.h | 4 + be/src/exprs/vsearch.cpp | 26 +- be/src/io/cache/block_file_cache_profile.cpp | 33 +- be/src/io/cache/block_file_cache_profile.h | 9 +- be/src/io/cache/cached_remote_file_reader.cpp | 22 +- be/src/io/cache/file_cache_common.h | 1 + be/src/io/io_common.h | 16 +- .../runtime/index_policy/index_policy_mgr.cpp | 126 +- .../runtime/index_policy/index_policy_mgr.h | 20 + .../compaction/collection_statistics.cpp | 290 --- be/src/storage/compaction/compaction.cpp | 327 ++- be/src/storage/index/analyzer_key_matcher.cpp | 2 - be/src/storage/index/analyzer_key_matcher.h | 8 +- be/src/storage/index/index_file_reader.cpp | 116 +- be/src/storage/index/index_file_reader.h | 20 +- be/src/storage/index/index_file_writer.cpp | 191 +- be/src/storage/index/index_file_writer.h | 68 + be/src/storage/index/index_query_context.h | 20 +- be/src/storage/index/index_writer.cpp | 16 + be/src/storage/index/index_writer.h | 9 + .../inverted/abstract_analysis_factory.h | 13 + .../index/inverted/analysis_factory_mgr.cpp | 3 + .../index/inverted/analyzer/analyzer.cpp | 86 +- .../index/inverted/analyzer/analyzer.h | 10 + .../inverted/analyzer/analyzer_provider.h | 46 + .../inverted/analyzer/custom_analyzer.cpp | 235 +- .../index/inverted/analyzer/custom_analyzer.h | 48 +- .../analyzer/custom_analyzer_config.cpp | 6 +- .../analyzer/custom_analyzer_config.h | 7 +- .../analyzer/segment_analyzer_context.cpp | 110 + .../analyzer/segment_analyzer_context.h | 57 + .../index/inverted/common/single_flight.h | 109 + .../common_grams/common_grams_key_codec.cpp | 279 ++ .../common_grams/common_grams_key_codec.h | 72 + .../common_grams/common_grams_query_cost.h | 53 + .../common_grams_segment_metadata.cpp | 158 ++ .../common_grams_segment_metadata.h | 87 + .../inverted/common_grams/common_word_set.cpp | 197 ++ .../inverted/common_grams/common_word_set.h | 94 + .../index/inverted/inverted_index_cache.cpp | 40 + .../index/inverted/inverted_index_cache.h | 52 +- .../inverted/inverted_index_fs_directory.cpp | 43 +- .../inverted/inverted_index_fs_directory.h | 5 + .../inverted/inverted_index_iterator.cpp | 220 +- .../index/inverted/inverted_index_iterator.h | 37 +- .../index/inverted/inverted_index_parser.cpp | 16 +- .../index/inverted/inverted_index_parser.h | 35 +- .../index/inverted/inverted_index_reader.cpp | 40 +- .../index/inverted/inverted_index_reader.h | 19 +- .../inverted/inverted_index_selector.cpp | 145 ++ .../index/inverted/inverted_index_selector.h | 57 + .../inverted/query/conjunction_query.cpp | 2 +- .../storage/index/inverted/query/query_info.h | 17 + be/src/storage/index/inverted/setting.h | 8 + .../similarity/collection_statistics.cpp | 529 ++++ .../similarity}/collection_statistics.h | 34 +- .../similarity/predicate_collector.cpp | 594 +++++ .../similarity}/predicate_collector.h | 26 +- .../ascii_folding_filter_factory.h | 5 + .../token_filter/common_grams_filter.cpp | 384 +++ .../token_filter/common_grams_filter.h | 152 ++ .../common_grams_filter_factory.h | 107 + .../token_filter/empty_token_filter_factory.h | 4 + .../icu_normalizer_filter_factory.h | 4 + .../inverted/token_filter/lower_case_filter.h | 86 +- .../token_filter/lower_case_filter_factory.h | 4 + .../token_filter/token_filter_factory.h | 1 + .../char/char_group_tokenizer_factory.h | 4 + .../tokenizer/char/char_tokenizer.cpp | 58 +- .../inverted/tokenizer/char/char_tokenizer.h | 8 + .../tokenizer/empty/empty_tokenizer_factory.h | 4 + .../ngram/edge_ngram_tokenizer_factory.h | 4 + .../tokenizer/ngram/ngram_tokenizer_factory.h | 4 + .../inverted/tokenizer/tokenizer_factory.h | 1 + be/src/storage/index/snii/common/slice.h | 56 + .../index/snii/common/uninitialized_buffer.h | 82 + .../index/snii/compaction/eligibility.cpp | 386 +++ .../index/snii/compaction/eligibility.h | 114 + .../snii/compaction/indexed_winner_tree.h | 103 + .../index/snii/compaction/posting_cursor.cpp | 1025 ++++++++ .../index/snii/compaction/posting_cursor.h | 258 ++ .../snii/compaction/posting_run_merger.cpp | 463 ++++ .../snii/compaction/posting_run_merger.h | 116 + .../index/snii/compaction/region_reader.cpp | 274 ++ .../index/snii/compaction/region_reader.h | 156 ++ .../snii/compaction/rowid_conversion.cpp | 240 ++ .../index/snii/compaction/rowid_conversion.h | 85 + .../snii/compaction/snii_index_compaction.cpp | 557 ++++ .../snii/compaction/snii_index_compaction.h | 146 ++ .../index/snii/compaction/term_cursor.cpp | 164 ++ .../index/snii/compaction/term_cursor.h | 124 + .../snii/compaction/term_merge_frontier.cpp | 103 + .../snii/compaction/term_merge_frontier.h | 56 + .../storage/index/snii/encoding/byte_sink.cpp | 60 + .../storage/index/snii/encoding/byte_sink.h | 71 + .../index/snii/encoding/byte_source.cpp | 179 ++ .../storage/index/snii/encoding/byte_source.h | 91 + be/src/storage/index/snii/encoding/crc32c.cpp | 278 ++ be/src/storage/index/snii/encoding/crc32c.h | 72 + be/src/storage/index/snii/encoding/pfor.cpp | 452 ++++ be/src/storage/index/snii/encoding/pfor.h | 53 + .../index/snii/encoding/section_framer.cpp | 59 + .../index/snii/encoding/section_framer.h | 44 + be/src/storage/index/snii/encoding/varint.cpp | 73 + be/src/storage/index/snii/encoding/varint.h | 43 + .../index/snii/encoding/zstd_codec.cpp | 55 + .../storage/index/snii/encoding/zstd_codec.h | 33 + .../index/snii/format/bootstrap_header.cpp | 113 + .../index/snii/format/bootstrap_header.h | 71 + be/src/storage/index/snii/format/bsbf.cpp | 257 ++ be/src/storage/index/snii/format/bsbf.h | 135 + .../index/snii/format/core_metadata.cpp | 285 +++ .../storage/index/snii/format/core_metadata.h | 62 + .../storage/index/snii/format/dict_block.cpp | 604 +++++ be/src/storage/index/snii/format/dict_block.h | 232 ++ .../snii/format/dict_block_directory.cpp | 131 + .../index/snii/format/dict_block_directory.h | 95 + .../storage/index/snii/format/dict_entry.cpp | 383 +++ be/src/storage/index/snii/format/dict_entry.h | 184 ++ .../index/snii/format/format_constants.h | 137 + be/src/storage/index/snii/format/frq_pod.cpp | 439 ++++ be/src/storage/index/snii/format/frq_pod.h | 150 ++ .../storage/index/snii/format/frq_prelude.cpp | 642 +++++ .../storage/index/snii/format/frq_prelude.h | 256 ++ .../index/snii/format/metadata_blob.cpp | 121 + .../storage/index/snii/format/metadata_blob.h | 40 + .../index/snii/format/metadata_directory.cpp | 150 ++ .../index/snii/format/metadata_directory.h | 60 + .../storage/index/snii/format/norms_pod.cpp | 85 + be/src/storage/index/snii/format/norms_pod.h | 89 + .../storage/index/snii/format/null_bitmap.cpp | 295 +++ .../storage/index/snii/format/null_bitmap.h | 131 + .../storage/index/snii/format/phrase_bigram.h | 33 + .../index/snii/format/prx_decode_stats.h | 106 + be/src/storage/index/snii/format/prx_pod.cpp | 1459 +++++++++++ be/src/storage/index/snii/format/prx_pod.h | 189 ++ .../index/snii/format/sampled_term_index.cpp | 189 ++ .../index/snii/format/sampled_term_index.h | 103 + .../storage/index/snii/format/stats_block.h | 34 + .../index/snii/format/tail_pointer.cpp | 122 + .../storage/index/snii/format/tail_pointer.h | 67 + .../index/snii/io/batch_range_fetcher.cpp | 102 + .../index/snii/io/batch_range_fetcher.h | 70 + be/src/storage/index/snii/io/file_reader.h | 67 + be/src/storage/index/snii/io/file_writer.h | 40 + be/src/storage/index/snii/io/io_metrics.h | 43 + .../storage/index/snii/query/bm25_scorer.cpp | 64 + be/src/storage/index/snii/query/bm25_scorer.h | 85 + .../index/snii/query/boolean_query.cpp | 124 + .../storage/index/snii/query/boolean_query.h | 50 + .../storage/index/snii/query/count_query.cpp | 80 + be/src/storage/index/snii/query/count_query.h | 69 + .../index/snii/query/docid_conjunction.cpp | 929 +++++++ .../index/snii/query/docid_posting_reader.cpp | 380 +++ .../index/snii/query/docid_set_ops.cpp | 127 + be/src/storage/index/snii/query/docid_sink.h | 90 + .../storage/index/snii/query/docid_union.cpp | 58 + .../snii/query/internal/docid_conjunction.h | 113 + .../query/internal/docid_posting_reader.h | 62 + .../index/snii/query/internal/docid_set_ops.h | 39 + .../index/snii/query/internal/docid_union.h | 38 + .../snii/query/internal/phrase_query_split.h | 598 +++++ .../snii/query/internal/plain_term_routing.h | 194 ++ .../index/snii/query/internal/position_math.h | 47 + .../snii/query/internal/query_test_counters.h | 110 + .../index/snii/query/internal/regex_prefix.h | 37 + .../query/internal/resolved_phrase_plan.h | 76 + .../query/internal/sloppy_phrase_matcher.h | 76 + .../snii/query/internal/term_expansion.h | 47 + .../snii/query/internal/wildcard_matcher.h | 81 + .../storage/index/snii/query/phrase_cost.cpp | 276 ++ .../storage/index/snii/query/phrase_emit.cpp | 718 ++++++ .../storage/index/snii/query/phrase_plan.cpp | 600 +++++ .../index/snii/query/phrase_planned_query.cpp | 743 ++++++ .../snii/query/phrase_position_source.cpp | 395 +++ .../index/snii/query/phrase_prefix_exec.cpp | 719 ++++++ .../index/snii/query/phrase_prx_validation.h | 55 + .../storage/index/snii/query/phrase_query.cpp | 326 +++ .../storage/index/snii/query/phrase_query.h | 141 + .../index/snii/query/phrase_verify_timer.h | 51 + .../storage/index/snii/query/prefix_query.cpp | 56 + .../storage/index/snii/query/prefix_query.h | 42 + .../index/snii/query/query_profile.cpp | 94 + .../storage/index/snii/query/query_profile.h | 67 + .../storage/index/snii/query/regexp_query.cpp | 192 ++ .../storage/index/snii/query/regexp_query.h | 43 + .../index/snii/query/scoring_query.cpp | 913 +++++++ .../storage/index/snii/query/scoring_query.h | 98 + .../snii/query/sloppy_phrase_matcher.cpp | 266 ++ .../index/snii/query/term_expansion.cpp | 168 ++ .../storage/index/snii/query/term_query.cpp | 58 + be/src/storage/index/snii/query/term_query.h | 41 + .../index/snii/query/wildcard_query.cpp | 77 + .../storage/index/snii/query/wildcard_query.h | 42 + .../index/snii/reader/dict_block_cache.h | 124 + .../snii/reader/logical_index_reader.cpp | 1107 ++++++++ .../index/snii/reader/logical_index_reader.h | 285 +++ .../index/snii/reader/snii_segment_reader.cpp | 193 ++ .../index/snii/reader/snii_segment_reader.h | 79 + .../index/snii/reader/windowed_posting.cpp | 308 +++ .../index/snii/reader/windowed_posting.h | 121 + .../storage/index/snii/snii_doris_adapter.cpp | 405 +++ .../storage/index/snii/snii_doris_adapter.h | 114 + .../storage/index/snii/snii_index_reader.cpp | 1183 +++++++++ be/src/storage/index/snii/snii_index_reader.h | 180 ++ .../storage/index/snii/snii_index_writer.cpp | 508 ++++ be/src/storage/index/snii/snii_index_writer.h | 144 ++ be/src/storage/index/snii/snii_prx_profile.h | 308 +++ be/src/storage/index/snii/snii_query_stats.h | 66 + .../index/snii/stats/snii_stats_provider.cpp | 145 ++ .../index/snii/stats/snii_stats_provider.h | 92 + .../snii/writer/compact_posting_pool.cpp | 120 + .../index/snii/writer/compact_posting_pool.h | 333 +++ .../snii/writer/global_memory_limiter.cpp | 166 ++ .../index/snii/writer/global_memory_limiter.h | 201 ++ .../snii/writer/logical_index_writer.cpp | 952 +++++++ .../index/snii/writer/logical_index_writer.h | 459 ++++ .../index/snii/writer/memory_reporter.h | 227 ++ .../snii/writer/posting_window_emitter.cpp | 579 +++++ .../snii/writer/posting_window_emitter.h | 112 + .../snii/writer/snii_compound_writer.cpp | 398 +++ .../index/snii/writer/snii_compound_writer.h | 206 ++ .../index/snii/writer/spill_run_codec.cpp | 937 +++++++ .../index/snii/writer/spill_run_codec.h | 236 ++ .../index/snii/writer/spillable_byte_buffer.h | 305 +++ .../index/snii/writer/spimi_term_buffer.cpp | 2097 +++++++++++++++ .../index/snii/writer/spimi_term_buffer.h | 730 ++++++ be/src/storage/index/snii/writer/temp_dir.cpp | 41 + be/src/storage/index/snii/writer/temp_dir.h | 47 + .../index/snii/writer/term_posting_source.h | 392 +++ be/src/storage/olap_common.h | 5 + be/src/storage/predicate_collector.cpp | 314 --- be/src/storage/rowset/beta_rowset.cpp | 44 +- be/src/storage/segment/column_reader.cpp | 12 + be/src/storage/segment/column_writer.cpp | 9 + be/src/storage/segment/column_writer.h | 7 + .../storage/segment/count_on_index_fastpath.h | 196 ++ be/src/storage/segment/row_ranges.h | 2 +- be/src/storage/segment/segment_iterator.cpp | 182 +- be/src/storage/segment/segment_iterator.h | 47 + be/src/storage/segment/segment_writer.cpp | 2 + .../variant/variant_column_writer_impl.cpp | 2 + .../segment/vertical_segment_writer.cpp | 2 + be/src/storage/tablet/tablet_meta.cpp | 3 + be/src/storage/task/index_builder.cpp | 105 +- be/test/exec/scan/scanner_context_test.cpp | 163 ++ .../exprs/function/function_is_null_test.cpp | 11 +- .../exprs/function/function_search_test.cpp | 708 ++++++ be/test/exprs/vsearch_expr_test.cpp | 116 +- ...block_file_cache_profile_reporter_test.cpp | 16 + ...fragment_mgr_cross_cluster_cancel_test.cpp | 4 + .../index_policy/index_policy_mgr_test.cpp | 240 ++ .../cloud_index_change_compaction_test.cpp | 101 +- .../compaction/collection_statistics_test.cpp | 1703 ++++++++++++- .../storage/compaction/segcompaction_test.cpp | 56 + be/test/storage/index/index_builder_test.cpp | 282 ++ .../inverted/ananlyzer/analyzer_test.cpp | 19 + .../ananlyzer/common_grams_filter_test.cpp | 644 +++++ .../ananlyzer/common_grams_key_codec_test.cpp | 553 ++++ .../ananlyzer/common_word_set_test.cpp | 186 ++ .../ananlyzer/custom_analyzer_test.cpp | 435 ++++ .../inverted/common/single_flight_test.cpp | 220 ++ .../compaction/index_compaction_test.cpp | 264 ++ .../index_compaction_write_index_test.cpp | 1 - .../util/index_compaction_utils.cpp | 62 +- .../index/inverted/empty_index_file_test.cpp | 62 +- ...ted_index_reader_analysis_purpose_test.cpp | 697 +++++ .../inverted/query/phrase_edge_query_test.cpp | 11 +- .../similarity/bm25_similarity_test.cpp | 2 +- .../lower_case_filter_factory_test.cpp | 96 + .../char_group_tokenizer_factory_test.cpp | 21 + .../index/inverted_index_parser_test.cpp | 17 + .../snii/bench/snii_vs_v3_benchmark_test.cpp | 1996 +++++++++++++++ .../storage/index/snii/common/slice_test.cpp | 42 + .../snii/common/uninitialized_buffer_test.cpp | 154 ++ .../snii_compaction_eligibility_test.cpp | 553 ++++ .../compaction/snii_index_compaction_test.cpp | 1284 ++++++++++ .../snii_logical_null_reader_test.cpp | 289 +++ .../compaction/snii_memory_reporter_test.cpp | 180 ++ .../compaction/snii_posting_cursor_test.cpp | 1036 ++++++++ .../compaction/snii_rowid_conversion_test.cpp | 155 ++ .../compaction/snii_streamed_session_test.cpp | 1580 ++++++++++++ .../snii/compaction/snii_term_cursor_test.cpp | 942 +++++++ .../index/snii/encoding/byte_sink_test.cpp | 103 + .../index/snii/encoding/byte_source_test.cpp | 262 ++ .../index/snii/encoding/crc32c_test.cpp | 336 +++ .../storage/index/snii/encoding/pfor_test.cpp | 488 ++++ .../snii/encoding/section_framer_test.cpp | 76 + .../index/snii/encoding/varint_test.cpp | 73 + .../index/snii/encoding/zstd_codec_test.cpp | 51 + .../snii/format/bootstrap_header_test.cpp | 184 ++ .../storage/index/snii/format/bsbf_test.cpp | 211 ++ .../index/snii/format/core_metadata_test.cpp | 319 +++ .../snii/format/dict_block_directory_test.cpp | 352 +++ .../format/dict_block_owned_finish_test.cpp | 98 + .../index/snii/format/dict_block_test.cpp | 643 +++++ .../index/snii/format/dict_entry_test.cpp | 377 +++ .../snii/format/format_constants_test.cpp | 52 + .../index/snii/format/frq_pod_test.cpp | 621 +++++ .../index/snii/format/frq_prelude_test.cpp | 428 ++++ .../index/snii/format/metadata_blob_test.cpp | 293 +++ .../snii/format/metadata_directory_test.cpp | 226 ++ .../index/snii/format/norms_pod_test.cpp | 218 ++ .../index/snii/format/null_bitmap_test.cpp | 286 +++ .../index/snii/format/prx_pod_test.cpp | 1857 ++++++++++++++ .../snii/format/sampled_term_index_test.cpp | 265 ++ .../index/snii/format/tail_pointer_test.cpp | 172 ++ .../snii/io/batch_range_fetcher_test.cpp | 127 + be/test/storage/index/snii/io/local_file.cpp | 134 + be/test/storage/index/snii/io/local_file.h | 84 + .../storage/index/snii/io/local_file_test.cpp | 143 ++ .../index/snii/io/metered_file_reader.cpp | 139 + .../index/snii/io/metered_file_reader.h | 67 + .../snii/io/metered_file_reader_test.cpp | 181 ++ .../index/snii/query/boolean_query_test.cpp | 337 +++ .../index/snii/query/byte_skip_test.cpp | 473 ++++ .../query/common_grams_dict_plan_test.cpp | 489 ++++ .../common_grams_namespace_query_test.cpp | 1563 ++++++++++++ .../index/snii/query/count_query_test.cpp | 290 +++ .../index/snii/query/docid_set_ops_test.cpp | 61 + .../index/snii/query/pattern_query_test.cpp | 285 +++ .../snii/query/phrase_prefix_query_test.cpp | 960 +++++++ .../index/snii/query/phrase_skip_test.cpp | 659 +++++ .../index/snii/query/position_math_test.cpp | 57 + .../snii/query/posting_grouping_test.cpp | 514 ++++ .../index/snii/query/prefix_query_test.cpp | 185 ++ .../snii/query/query_operator_error_test.cpp | 288 +++ .../index/snii/query/query_profile_test.cpp | 986 +++++++ .../query_term_resolution_batch_test.cpp | 304 +++ .../index/snii/query/scoring_query_test.cpp | 756 ++++++ .../query/scoring_wand_selective_test.cpp | 520 ++++ .../snii/query/sloppy_phrase_matcher_test.cpp | 236 ++ .../snii/reader/snii_end_to_end_test.cpp | 295 +++ .../reader/snii_segment_reader_open_test.cpp | 383 +++ .../snii_index_reader_count_fallback_test.cpp | 1714 +++++++++++++ be/test/storage/index/snii/snii_test_env.cpp | 57 + .../spimi_common_gram_chain_encoding_test.cpp | 137 + .../snii/spimi_dense_rank_order_test.cpp | 69 + .../index/snii/spimi_spill_rank_test.cpp | 221 ++ .../snii/writer/compact_posting_pool_test.cpp | 602 +++++ .../logical_index_writer_freq_stats_test.cpp | 562 ++++ .../snii/writer/memory_reporter_test.cpp | 85 + .../writer/memory_reporter_wiring_test.cpp | 296 +++ .../snii/writer/phase_a_readback_test.cpp | 303 +++ .../snii/writer/posting_interleave_test.cpp | 560 ++++ .../writer/posting_window_emitter_test.cpp | 215 ++ .../snii/writer/snii_compound_writer_test.cpp | 1264 +++++++++ .../writer/snii_no_bigram_writer_test.cpp | 379 +++ .../snii/writer/snii_prx_tier_writer_test.cpp | 290 +++ .../writer/snii_writer_golden_bytes_test.cpp | 406 +++ .../snii/writer/spill_run_codec_test.cpp | 1254 +++++++++ .../writer/spillable_byte_buffer_test.cpp | 128 + .../spimi_global_memory_limiter_test.cpp | 542 ++++ .../snii/writer/spimi_locality_bench_test.cpp | 280 ++ .../writer/spimi_resident_accounting_test.cpp | 161 ++ .../index/snii/writer/spimi_run_cap_test.cpp | 198 ++ .../snii/writer/spimi_spill_writer_test.cpp | 324 +++ .../writer/spimi_streaming_writer_test.cpp | 249 ++ .../snii/writer/spimi_term_buffer_test.cpp | 860 +++++++ .../index/snii/writer/temp_dir_test.cpp | 46 + .../snii/writer/term_posting_source_test.cpp | 245 ++ .../snii/writer/term_posting_test_utils.h | 63 + .../storage/index/snii_b2_or_read_test.cpp | 346 +++ .../storage/index/snii_b2_t07_dict_test.cpp | 634 +++++ .../index/snii_b2_t08_wildcard_test.cpp | 359 +++ .../storage/index/snii_b2_t10_cover_test.cpp | 376 +++ .../storage/index/snii_clucene_guard_test.cpp | 176 ++ .../storage/index/snii_crc32c_equiv_test.cpp | 145 ++ .../storage/index/snii_doris_adapter_test.cpp | 897 +++++++ .../index/snii_frq_direct_encode_test.cpp | 411 +++ .../storage/index/snii_frq_prelude_test.cpp | 520 ++++ be/test/storage/index/snii_query_test.cpp | 2260 +++++++++++++++++ be/test/storage/index/snii_query_test_util.h | 258 ++ be/test/storage/index/snii_spill_io_test.cpp | 226 ++ .../storage/index/snii_spimi_intern_test.cpp | 1364 ++++++++++ be/test/storage/index/snii_writer_test.cpp | 529 ++++ .../segment/count_on_index_fastpath_test.cpp | 313 +++ .../inverted_index_file_reader_test.cpp | 78 + .../inverted_index_file_writer_test.cpp | 42 + .../inverted_index_fs_directory_test.cpp | 52 + .../segment/inverted_index_iterator_test.cpp | 86 +- .../segment/inverted_index_reader_test.cpp | 75 +- .../segment/inverted_index_writer_test.cpp | 511 +++- ...ment_iterator_count_emit_shortcut_test.cpp | 419 +++ build-support/check-header-deps.py | 1 + build.sh | 1 + .../doris/analysis/InvertedIndexUtil.java | 41 +- .../datasource/CloudInternalCatalog.java | 4 + .../doris/common/util/PropertyAnalyzer.java | 6 + .../AsciiFoldingTokenFilterValidator.java | 3 +- .../apache/doris/indexpolicy/IndexPolicy.java | 9 + .../doris/indexpolicy/IndexPolicyMgr.java | 165 +- .../rules/rewrite/CheckScoreUsage.java | 88 + .../PushDownScoreTopNIntoOlapScan.java | 4 + .../plans/commands/info/BuildIndexOp.java | 5 + .../plans/commands/info/CreateTableInfo.java | 6 +- .../plans/commands/info/IndexDefinition.java | 35 +- .../doris/alter/IndexChangeJobTest.java | 42 + .../analysis/InvertedIndexPropertiesTest.java | 75 + .../CommonGramsDdlValidationTest.java | 154 ++ .../indexpolicy/PolicyValidatorTests.java | 17 + .../rules/rewrite/CheckScoreUsageTest.java | 471 ++++ .../plans/commands/IndexDefinitionTest.java | 64 + gensrc/proto/olap_file.proto | 1 + gensrc/proto/snii.proto | 82 + gensrc/thrift/Types.thrift | 3 +- .../storage_format/common_grams_docs.csv | 20 + .../storage_format/test_common_grams_snii.out | 666 +++++ .../test_storage_format_snii.out | 40 + .../test_common_grams_snii.groovy | 1043 ++++++++ .../test_storage_format_snii.groovy | 305 +++ 426 files changed, 101855 insertions(+), 1169 deletions(-) delete mode 100644 be/src/storage/compaction/collection_statistics.cpp create mode 100644 be/src/storage/index/inverted/analyzer/analyzer_provider.h create mode 100644 be/src/storage/index/inverted/analyzer/segment_analyzer_context.cpp create mode 100644 be/src/storage/index/inverted/analyzer/segment_analyzer_context.h create mode 100644 be/src/storage/index/inverted/common/single_flight.h create mode 100644 be/src/storage/index/inverted/common_grams/common_grams_key_codec.cpp create mode 100644 be/src/storage/index/inverted/common_grams/common_grams_key_codec.h create mode 100644 be/src/storage/index/inverted/common_grams/common_grams_query_cost.h create mode 100644 be/src/storage/index/inverted/common_grams/common_grams_segment_metadata.cpp create mode 100644 be/src/storage/index/inverted/common_grams/common_grams_segment_metadata.h create mode 100644 be/src/storage/index/inverted/common_grams/common_word_set.cpp create mode 100644 be/src/storage/index/inverted/common_grams/common_word_set.h create mode 100644 be/src/storage/index/inverted/inverted_index_selector.cpp create mode 100644 be/src/storage/index/inverted/inverted_index_selector.h create mode 100644 be/src/storage/index/inverted/similarity/collection_statistics.cpp rename be/src/storage/{compaction => index/inverted/similarity}/collection_statistics.h (65%) create mode 100644 be/src/storage/index/inverted/similarity/predicate_collector.cpp rename be/src/storage/{ => index/inverted/similarity}/predicate_collector.h (80%) create mode 100644 be/src/storage/index/inverted/token_filter/common_grams_filter.cpp create mode 100644 be/src/storage/index/inverted/token_filter/common_grams_filter.h create mode 100644 be/src/storage/index/inverted/token_filter/common_grams_filter_factory.h create mode 100644 be/src/storage/index/snii/common/slice.h create mode 100644 be/src/storage/index/snii/common/uninitialized_buffer.h create mode 100644 be/src/storage/index/snii/compaction/eligibility.cpp create mode 100644 be/src/storage/index/snii/compaction/eligibility.h create mode 100644 be/src/storage/index/snii/compaction/indexed_winner_tree.h create mode 100644 be/src/storage/index/snii/compaction/posting_cursor.cpp create mode 100644 be/src/storage/index/snii/compaction/posting_cursor.h create mode 100644 be/src/storage/index/snii/compaction/posting_run_merger.cpp create mode 100644 be/src/storage/index/snii/compaction/posting_run_merger.h create mode 100644 be/src/storage/index/snii/compaction/region_reader.cpp create mode 100644 be/src/storage/index/snii/compaction/region_reader.h create mode 100644 be/src/storage/index/snii/compaction/rowid_conversion.cpp create mode 100644 be/src/storage/index/snii/compaction/rowid_conversion.h create mode 100644 be/src/storage/index/snii/compaction/snii_index_compaction.cpp create mode 100644 be/src/storage/index/snii/compaction/snii_index_compaction.h create mode 100644 be/src/storage/index/snii/compaction/term_cursor.cpp create mode 100644 be/src/storage/index/snii/compaction/term_cursor.h create mode 100644 be/src/storage/index/snii/compaction/term_merge_frontier.cpp create mode 100644 be/src/storage/index/snii/compaction/term_merge_frontier.h create mode 100644 be/src/storage/index/snii/encoding/byte_sink.cpp create mode 100644 be/src/storage/index/snii/encoding/byte_sink.h create mode 100644 be/src/storage/index/snii/encoding/byte_source.cpp create mode 100644 be/src/storage/index/snii/encoding/byte_source.h create mode 100644 be/src/storage/index/snii/encoding/crc32c.cpp create mode 100644 be/src/storage/index/snii/encoding/crc32c.h create mode 100644 be/src/storage/index/snii/encoding/pfor.cpp create mode 100644 be/src/storage/index/snii/encoding/pfor.h create mode 100644 be/src/storage/index/snii/encoding/section_framer.cpp create mode 100644 be/src/storage/index/snii/encoding/section_framer.h create mode 100644 be/src/storage/index/snii/encoding/varint.cpp create mode 100644 be/src/storage/index/snii/encoding/varint.h create mode 100644 be/src/storage/index/snii/encoding/zstd_codec.cpp create mode 100644 be/src/storage/index/snii/encoding/zstd_codec.h create mode 100644 be/src/storage/index/snii/format/bootstrap_header.cpp create mode 100644 be/src/storage/index/snii/format/bootstrap_header.h create mode 100644 be/src/storage/index/snii/format/bsbf.cpp create mode 100644 be/src/storage/index/snii/format/bsbf.h create mode 100644 be/src/storage/index/snii/format/core_metadata.cpp create mode 100644 be/src/storage/index/snii/format/core_metadata.h create mode 100644 be/src/storage/index/snii/format/dict_block.cpp create mode 100644 be/src/storage/index/snii/format/dict_block.h create mode 100644 be/src/storage/index/snii/format/dict_block_directory.cpp create mode 100644 be/src/storage/index/snii/format/dict_block_directory.h create mode 100644 be/src/storage/index/snii/format/dict_entry.cpp create mode 100644 be/src/storage/index/snii/format/dict_entry.h create mode 100644 be/src/storage/index/snii/format/format_constants.h create mode 100644 be/src/storage/index/snii/format/frq_pod.cpp create mode 100644 be/src/storage/index/snii/format/frq_pod.h create mode 100644 be/src/storage/index/snii/format/frq_prelude.cpp create mode 100644 be/src/storage/index/snii/format/frq_prelude.h create mode 100644 be/src/storage/index/snii/format/metadata_blob.cpp create mode 100644 be/src/storage/index/snii/format/metadata_blob.h create mode 100644 be/src/storage/index/snii/format/metadata_directory.cpp create mode 100644 be/src/storage/index/snii/format/metadata_directory.h create mode 100644 be/src/storage/index/snii/format/norms_pod.cpp create mode 100644 be/src/storage/index/snii/format/norms_pod.h create mode 100644 be/src/storage/index/snii/format/null_bitmap.cpp create mode 100644 be/src/storage/index/snii/format/null_bitmap.h create mode 100644 be/src/storage/index/snii/format/phrase_bigram.h create mode 100644 be/src/storage/index/snii/format/prx_decode_stats.h create mode 100644 be/src/storage/index/snii/format/prx_pod.cpp create mode 100644 be/src/storage/index/snii/format/prx_pod.h create mode 100644 be/src/storage/index/snii/format/sampled_term_index.cpp create mode 100644 be/src/storage/index/snii/format/sampled_term_index.h create mode 100644 be/src/storage/index/snii/format/stats_block.h create mode 100644 be/src/storage/index/snii/format/tail_pointer.cpp create mode 100644 be/src/storage/index/snii/format/tail_pointer.h create mode 100644 be/src/storage/index/snii/io/batch_range_fetcher.cpp create mode 100644 be/src/storage/index/snii/io/batch_range_fetcher.h create mode 100644 be/src/storage/index/snii/io/file_reader.h create mode 100644 be/src/storage/index/snii/io/file_writer.h create mode 100644 be/src/storage/index/snii/io/io_metrics.h create mode 100644 be/src/storage/index/snii/query/bm25_scorer.cpp create mode 100644 be/src/storage/index/snii/query/bm25_scorer.h create mode 100644 be/src/storage/index/snii/query/boolean_query.cpp create mode 100644 be/src/storage/index/snii/query/boolean_query.h create mode 100644 be/src/storage/index/snii/query/count_query.cpp create mode 100644 be/src/storage/index/snii/query/count_query.h create mode 100644 be/src/storage/index/snii/query/docid_conjunction.cpp create mode 100644 be/src/storage/index/snii/query/docid_posting_reader.cpp create mode 100644 be/src/storage/index/snii/query/docid_set_ops.cpp create mode 100644 be/src/storage/index/snii/query/docid_sink.h create mode 100644 be/src/storage/index/snii/query/docid_union.cpp create mode 100644 be/src/storage/index/snii/query/internal/docid_conjunction.h create mode 100644 be/src/storage/index/snii/query/internal/docid_posting_reader.h create mode 100644 be/src/storage/index/snii/query/internal/docid_set_ops.h create mode 100644 be/src/storage/index/snii/query/internal/docid_union.h create mode 100644 be/src/storage/index/snii/query/internal/phrase_query_split.h create mode 100644 be/src/storage/index/snii/query/internal/plain_term_routing.h create mode 100644 be/src/storage/index/snii/query/internal/position_math.h create mode 100644 be/src/storage/index/snii/query/internal/query_test_counters.h create mode 100644 be/src/storage/index/snii/query/internal/regex_prefix.h create mode 100644 be/src/storage/index/snii/query/internal/resolved_phrase_plan.h create mode 100644 be/src/storage/index/snii/query/internal/sloppy_phrase_matcher.h create mode 100644 be/src/storage/index/snii/query/internal/term_expansion.h create mode 100644 be/src/storage/index/snii/query/internal/wildcard_matcher.h create mode 100644 be/src/storage/index/snii/query/phrase_cost.cpp create mode 100644 be/src/storage/index/snii/query/phrase_emit.cpp create mode 100644 be/src/storage/index/snii/query/phrase_plan.cpp create mode 100644 be/src/storage/index/snii/query/phrase_planned_query.cpp create mode 100644 be/src/storage/index/snii/query/phrase_position_source.cpp create mode 100644 be/src/storage/index/snii/query/phrase_prefix_exec.cpp create mode 100644 be/src/storage/index/snii/query/phrase_prx_validation.h create mode 100644 be/src/storage/index/snii/query/phrase_query.cpp create mode 100644 be/src/storage/index/snii/query/phrase_query.h create mode 100644 be/src/storage/index/snii/query/phrase_verify_timer.h create mode 100644 be/src/storage/index/snii/query/prefix_query.cpp create mode 100644 be/src/storage/index/snii/query/prefix_query.h create mode 100644 be/src/storage/index/snii/query/query_profile.cpp create mode 100644 be/src/storage/index/snii/query/query_profile.h create mode 100644 be/src/storage/index/snii/query/regexp_query.cpp create mode 100644 be/src/storage/index/snii/query/regexp_query.h create mode 100644 be/src/storage/index/snii/query/scoring_query.cpp create mode 100644 be/src/storage/index/snii/query/scoring_query.h create mode 100644 be/src/storage/index/snii/query/sloppy_phrase_matcher.cpp create mode 100644 be/src/storage/index/snii/query/term_expansion.cpp create mode 100644 be/src/storage/index/snii/query/term_query.cpp create mode 100644 be/src/storage/index/snii/query/term_query.h create mode 100644 be/src/storage/index/snii/query/wildcard_query.cpp create mode 100644 be/src/storage/index/snii/query/wildcard_query.h create mode 100644 be/src/storage/index/snii/reader/dict_block_cache.h create mode 100644 be/src/storage/index/snii/reader/logical_index_reader.cpp create mode 100644 be/src/storage/index/snii/reader/logical_index_reader.h create mode 100644 be/src/storage/index/snii/reader/snii_segment_reader.cpp create mode 100644 be/src/storage/index/snii/reader/snii_segment_reader.h create mode 100644 be/src/storage/index/snii/reader/windowed_posting.cpp create mode 100644 be/src/storage/index/snii/reader/windowed_posting.h create mode 100644 be/src/storage/index/snii/snii_doris_adapter.cpp create mode 100644 be/src/storage/index/snii/snii_doris_adapter.h create mode 100644 be/src/storage/index/snii/snii_index_reader.cpp create mode 100644 be/src/storage/index/snii/snii_index_reader.h create mode 100644 be/src/storage/index/snii/snii_index_writer.cpp create mode 100644 be/src/storage/index/snii/snii_index_writer.h create mode 100644 be/src/storage/index/snii/snii_prx_profile.h create mode 100644 be/src/storage/index/snii/snii_query_stats.h create mode 100644 be/src/storage/index/snii/stats/snii_stats_provider.cpp create mode 100644 be/src/storage/index/snii/stats/snii_stats_provider.h create mode 100644 be/src/storage/index/snii/writer/compact_posting_pool.cpp create mode 100644 be/src/storage/index/snii/writer/compact_posting_pool.h create mode 100644 be/src/storage/index/snii/writer/global_memory_limiter.cpp create mode 100644 be/src/storage/index/snii/writer/global_memory_limiter.h create mode 100644 be/src/storage/index/snii/writer/logical_index_writer.cpp create mode 100644 be/src/storage/index/snii/writer/logical_index_writer.h create mode 100644 be/src/storage/index/snii/writer/memory_reporter.h create mode 100644 be/src/storage/index/snii/writer/posting_window_emitter.cpp create mode 100644 be/src/storage/index/snii/writer/posting_window_emitter.h create mode 100644 be/src/storage/index/snii/writer/snii_compound_writer.cpp create mode 100644 be/src/storage/index/snii/writer/snii_compound_writer.h create mode 100644 be/src/storage/index/snii/writer/spill_run_codec.cpp create mode 100644 be/src/storage/index/snii/writer/spill_run_codec.h create mode 100644 be/src/storage/index/snii/writer/spillable_byte_buffer.h create mode 100644 be/src/storage/index/snii/writer/spimi_term_buffer.cpp create mode 100644 be/src/storage/index/snii/writer/spimi_term_buffer.h create mode 100644 be/src/storage/index/snii/writer/temp_dir.cpp create mode 100644 be/src/storage/index/snii/writer/temp_dir.h create mode 100644 be/src/storage/index/snii/writer/term_posting_source.h delete mode 100644 be/src/storage/predicate_collector.cpp create mode 100644 be/src/storage/segment/count_on_index_fastpath.h create mode 100644 be/test/storage/index/inverted/ananlyzer/common_grams_filter_test.cpp create mode 100644 be/test/storage/index/inverted/ananlyzer/common_grams_key_codec_test.cpp create mode 100644 be/test/storage/index/inverted/ananlyzer/common_word_set_test.cpp create mode 100644 be/test/storage/index/inverted/common/single_flight_test.cpp create mode 100644 be/test/storage/index/inverted/inverted_index_reader_analysis_purpose_test.cpp create mode 100644 be/test/storage/index/snii/bench/snii_vs_v3_benchmark_test.cpp create mode 100644 be/test/storage/index/snii/common/slice_test.cpp create mode 100644 be/test/storage/index/snii/common/uninitialized_buffer_test.cpp create mode 100644 be/test/storage/index/snii/compaction/snii_compaction_eligibility_test.cpp create mode 100644 be/test/storage/index/snii/compaction/snii_index_compaction_test.cpp create mode 100644 be/test/storage/index/snii/compaction/snii_logical_null_reader_test.cpp create mode 100644 be/test/storage/index/snii/compaction/snii_memory_reporter_test.cpp create mode 100644 be/test/storage/index/snii/compaction/snii_posting_cursor_test.cpp create mode 100644 be/test/storage/index/snii/compaction/snii_rowid_conversion_test.cpp create mode 100644 be/test/storage/index/snii/compaction/snii_streamed_session_test.cpp create mode 100644 be/test/storage/index/snii/compaction/snii_term_cursor_test.cpp create mode 100644 be/test/storage/index/snii/encoding/byte_sink_test.cpp create mode 100644 be/test/storage/index/snii/encoding/byte_source_test.cpp create mode 100644 be/test/storage/index/snii/encoding/crc32c_test.cpp create mode 100644 be/test/storage/index/snii/encoding/pfor_test.cpp create mode 100644 be/test/storage/index/snii/encoding/section_framer_test.cpp create mode 100644 be/test/storage/index/snii/encoding/varint_test.cpp create mode 100644 be/test/storage/index/snii/encoding/zstd_codec_test.cpp create mode 100644 be/test/storage/index/snii/format/bootstrap_header_test.cpp create mode 100644 be/test/storage/index/snii/format/bsbf_test.cpp create mode 100644 be/test/storage/index/snii/format/core_metadata_test.cpp create mode 100644 be/test/storage/index/snii/format/dict_block_directory_test.cpp create mode 100644 be/test/storage/index/snii/format/dict_block_owned_finish_test.cpp create mode 100644 be/test/storage/index/snii/format/dict_block_test.cpp create mode 100644 be/test/storage/index/snii/format/dict_entry_test.cpp create mode 100644 be/test/storage/index/snii/format/format_constants_test.cpp create mode 100644 be/test/storage/index/snii/format/frq_pod_test.cpp create mode 100644 be/test/storage/index/snii/format/frq_prelude_test.cpp create mode 100644 be/test/storage/index/snii/format/metadata_blob_test.cpp create mode 100644 be/test/storage/index/snii/format/metadata_directory_test.cpp create mode 100644 be/test/storage/index/snii/format/norms_pod_test.cpp create mode 100644 be/test/storage/index/snii/format/null_bitmap_test.cpp create mode 100644 be/test/storage/index/snii/format/prx_pod_test.cpp create mode 100644 be/test/storage/index/snii/format/sampled_term_index_test.cpp create mode 100644 be/test/storage/index/snii/format/tail_pointer_test.cpp create mode 100644 be/test/storage/index/snii/io/batch_range_fetcher_test.cpp create mode 100644 be/test/storage/index/snii/io/local_file.cpp create mode 100644 be/test/storage/index/snii/io/local_file.h create mode 100644 be/test/storage/index/snii/io/local_file_test.cpp create mode 100644 be/test/storage/index/snii/io/metered_file_reader.cpp create mode 100644 be/test/storage/index/snii/io/metered_file_reader.h create mode 100644 be/test/storage/index/snii/io/metered_file_reader_test.cpp create mode 100644 be/test/storage/index/snii/query/boolean_query_test.cpp create mode 100644 be/test/storage/index/snii/query/byte_skip_test.cpp create mode 100644 be/test/storage/index/snii/query/common_grams_dict_plan_test.cpp create mode 100644 be/test/storage/index/snii/query/common_grams_namespace_query_test.cpp create mode 100644 be/test/storage/index/snii/query/count_query_test.cpp create mode 100644 be/test/storage/index/snii/query/docid_set_ops_test.cpp create mode 100644 be/test/storage/index/snii/query/pattern_query_test.cpp create mode 100644 be/test/storage/index/snii/query/phrase_prefix_query_test.cpp create mode 100644 be/test/storage/index/snii/query/phrase_skip_test.cpp create mode 100644 be/test/storage/index/snii/query/position_math_test.cpp create mode 100644 be/test/storage/index/snii/query/posting_grouping_test.cpp create mode 100644 be/test/storage/index/snii/query/prefix_query_test.cpp create mode 100644 be/test/storage/index/snii/query/query_operator_error_test.cpp create mode 100644 be/test/storage/index/snii/query/query_profile_test.cpp create mode 100644 be/test/storage/index/snii/query/query_term_resolution_batch_test.cpp create mode 100644 be/test/storage/index/snii/query/scoring_query_test.cpp create mode 100644 be/test/storage/index/snii/query/scoring_wand_selective_test.cpp create mode 100644 be/test/storage/index/snii/query/sloppy_phrase_matcher_test.cpp create mode 100644 be/test/storage/index/snii/reader/snii_end_to_end_test.cpp create mode 100644 be/test/storage/index/snii/reader/snii_segment_reader_open_test.cpp create mode 100644 be/test/storage/index/snii/snii_index_reader_count_fallback_test.cpp create mode 100644 be/test/storage/index/snii/snii_test_env.cpp create mode 100644 be/test/storage/index/snii/spimi_common_gram_chain_encoding_test.cpp create mode 100644 be/test/storage/index/snii/spimi_dense_rank_order_test.cpp create mode 100644 be/test/storage/index/snii/spimi_spill_rank_test.cpp create mode 100644 be/test/storage/index/snii/writer/compact_posting_pool_test.cpp create mode 100644 be/test/storage/index/snii/writer/logical_index_writer_freq_stats_test.cpp create mode 100644 be/test/storage/index/snii/writer/memory_reporter_test.cpp create mode 100644 be/test/storage/index/snii/writer/memory_reporter_wiring_test.cpp create mode 100644 be/test/storage/index/snii/writer/phase_a_readback_test.cpp create mode 100644 be/test/storage/index/snii/writer/posting_interleave_test.cpp create mode 100644 be/test/storage/index/snii/writer/posting_window_emitter_test.cpp create mode 100644 be/test/storage/index/snii/writer/snii_compound_writer_test.cpp create mode 100644 be/test/storage/index/snii/writer/snii_no_bigram_writer_test.cpp create mode 100644 be/test/storage/index/snii/writer/snii_prx_tier_writer_test.cpp create mode 100644 be/test/storage/index/snii/writer/snii_writer_golden_bytes_test.cpp create mode 100644 be/test/storage/index/snii/writer/spill_run_codec_test.cpp create mode 100644 be/test/storage/index/snii/writer/spillable_byte_buffer_test.cpp create mode 100644 be/test/storage/index/snii/writer/spimi_global_memory_limiter_test.cpp create mode 100644 be/test/storage/index/snii/writer/spimi_locality_bench_test.cpp create mode 100644 be/test/storage/index/snii/writer/spimi_resident_accounting_test.cpp create mode 100644 be/test/storage/index/snii/writer/spimi_run_cap_test.cpp create mode 100644 be/test/storage/index/snii/writer/spimi_spill_writer_test.cpp create mode 100644 be/test/storage/index/snii/writer/spimi_streaming_writer_test.cpp create mode 100644 be/test/storage/index/snii/writer/spimi_term_buffer_test.cpp create mode 100644 be/test/storage/index/snii/writer/temp_dir_test.cpp create mode 100644 be/test/storage/index/snii/writer/term_posting_source_test.cpp create mode 100644 be/test/storage/index/snii/writer/term_posting_test_utils.h create mode 100644 be/test/storage/index/snii_b2_or_read_test.cpp create mode 100644 be/test/storage/index/snii_b2_t07_dict_test.cpp create mode 100644 be/test/storage/index/snii_b2_t08_wildcard_test.cpp create mode 100644 be/test/storage/index/snii_b2_t10_cover_test.cpp create mode 100644 be/test/storage/index/snii_clucene_guard_test.cpp create mode 100644 be/test/storage/index/snii_crc32c_equiv_test.cpp create mode 100644 be/test/storage/index/snii_doris_adapter_test.cpp create mode 100644 be/test/storage/index/snii_frq_direct_encode_test.cpp create mode 100644 be/test/storage/index/snii_frq_prelude_test.cpp create mode 100644 be/test/storage/index/snii_query_test.cpp create mode 100644 be/test/storage/index/snii_query_test_util.h create mode 100644 be/test/storage/index/snii_spill_io_test.cpp create mode 100644 be/test/storage/index/snii_spimi_intern_test.cpp create mode 100644 be/test/storage/index/snii_writer_test.cpp create mode 100644 be/test/storage/segment/count_on_index_fastpath_test.cpp create mode 100644 be/test/storage/segment/segment_iterator_count_emit_shortcut_test.cpp create mode 100644 fe/fe-core/src/test/java/org/apache/doris/indexpolicy/CommonGramsDdlValidationTest.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/CheckScoreUsageTest.java create mode 100644 gensrc/proto/snii.proto create mode 100644 regression-test/data/inverted_index_p0/storage_format/common_grams_docs.csv create mode 100644 regression-test/data/inverted_index_p0/storage_format/test_common_grams_snii.out create mode 100644 regression-test/data/inverted_index_p0/storage_format/test_storage_format_snii.out create mode 100644 regression-test/suites/inverted_index_p0/storage_format/test_common_grams_snii.groovy create mode 100644 regression-test/suites/inverted_index_p0/storage_format/test_storage_format_snii.groovy diff --git a/be/src/cloud/cloud_index_change_compaction.cpp b/be/src/cloud/cloud_index_change_compaction.cpp index 55654ccc958824..03a0e02aec6647 100644 --- a/be/src/cloud/cloud_index_change_compaction.cpp +++ b/be/src/cloud/cloud_index_change_compaction.cpp @@ -83,7 +83,12 @@ Status CloudIndexChangeCompaction::prepare_compact() { _input_rowsets_total_size += rs->total_disk_size(); } - _enable_inverted_index_compaction = false; + // Non-SNII compaction cannot reuse index files when the index schema changes. SNII validates + // each destination logical index independently and rebuilds ineligible columns from raw data. + _enable_inverted_index_compaction = + _enable_inverted_index_compaction && + input_rowset->tablet_schema()->get_inverted_index_storage_format() == + InvertedIndexStorageFormatPB::SNII; LOG_INFO("[index_change]prepare CloudIndexChangeCompaction, tablet_id={}, range=[{}-{}]", _tablet->tablet_id(), _input_rowsets.front()->start_version(), _input_rowsets.back()->end_version()) diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index d27cf607a36b24..2f2264511e34de 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -20,6 +20,7 @@ #include #include +#include #include // IWYU pragma: no_include #include @@ -30,6 +31,7 @@ #include // IWYU pragma: keep #include #include +#include #include #include #include @@ -1296,6 +1298,103 @@ DEFINE_Int32(inverted_index_query_cache_shards, "256"); // inverted index match bitmap cache size DEFINE_String(inverted_index_query_cache_limit, "10%"); +namespace { + +constexpr uint64_t COMMON_GRAMS_QUERY_PLAN_ENABLED_MASK = 1; +constexpr uint64_t COMMON_GRAMS_QUERY_PLAN_GENERATION_SHIFT = 1; +std::atomic common_grams_query_plan_state {COMMON_GRAMS_QUERY_PLAN_ENABLED_MASK}; +constexpr uint64_t COMMON_GRAMS_COST_RATIO_MASK = std::numeric_limits::max(); +constexpr uint64_t COMMON_GRAMS_VERIFY_FACTOR_SHIFT = 32; +std::atomic common_grams_cost_model_state {85}; + +bool valid_common_grams_cost_ratio(int32_t value) { + return value >= 0 && value <= 100; +} + +bool valid_common_grams_verify_factor(int32_t value) { + return value >= 0; +} + +void publish_common_grams_query_plan_state(bool enabled) { + uint64_t current = common_grams_query_plan_state.load(std::memory_order_acquire); + while ((current & COMMON_GRAMS_QUERY_PLAN_ENABLED_MASK) != static_cast(enabled)) { + const uint64_t generation = (current >> COMMON_GRAMS_QUERY_PLAN_GENERATION_SHIFT) + 1; + const uint64_t desired = (generation << COMMON_GRAMS_QUERY_PLAN_GENERATION_SHIFT) | + static_cast(enabled); + if (common_grams_query_plan_state.compare_exchange_weak( + current, desired, std::memory_order_acq_rel, std::memory_order_acquire)) { + return; + } + } +} + +void publish_common_grams_cost_ratio(int32_t value) { + uint64_t current = common_grams_cost_model_state.load(std::memory_order_acquire); + if ((current & COMMON_GRAMS_COST_RATIO_MASK) == static_cast(value)) { + return; + } + uint64_t desired = 0; + do { + desired = (current & ~COMMON_GRAMS_COST_RATIO_MASK) | static_cast(value); + } while (!common_grams_cost_model_state.compare_exchange_weak( + current, desired, std::memory_order_acq_rel, std::memory_order_acquire)); +} + +void publish_common_grams_verify_factor(int32_t value) { + uint64_t current = common_grams_cost_model_state.load(std::memory_order_acquire); + if (current >> COMMON_GRAMS_VERIFY_FACTOR_SHIFT == static_cast(value)) { + return; + } + uint64_t desired = 0; + do { + desired = (current & COMMON_GRAMS_COST_RATIO_MASK) | + (static_cast(static_cast(value)) + << COMMON_GRAMS_VERIFY_FACTOR_SHIFT); + } while (!common_grams_cost_model_state.compare_exchange_weak( + current, desired, std::memory_order_acq_rel, std::memory_order_acquire)); +} + +} // namespace + +DEFINE_mBool(enable_common_grams_query_plan, "false"); +DEFINE_ON_UPDATE(enable_common_grams_query_plan, [](bool /*old_value*/, bool new_value) { + publish_common_grams_query_plan_state(new_value); +}); +DEFINE_mBool(enable_common_grams_index_build, "true"); +DEFINE_mInt32(common_grams_plan_cost_ratio_percent, "85"); +DEFINE_Validator(common_grams_plan_cost_ratio_percent, valid_common_grams_cost_ratio); +DEFINE_ON_UPDATE(common_grams_plan_cost_ratio_percent, + [](int32_t /*old_value*/, int32_t new_value) { + publish_common_grams_cost_ratio(new_value); + }); +DEFINE_mInt32(common_grams_position_verify_factor, "0"); +DEFINE_Validator(common_grams_position_verify_factor, valid_common_grams_verify_factor); +DEFINE_ON_UPDATE(common_grams_position_verify_factor, [](int32_t /*old_value*/, int32_t new_value) { + publish_common_grams_verify_factor(new_value); +}); + +namespace { + +void publish_common_grams_query_plan_config() { + publish_common_grams_query_plan_state(enable_common_grams_query_plan); + publish_common_grams_cost_ratio(common_grams_plan_cost_ratio_percent); + publish_common_grams_verify_factor(common_grams_position_verify_factor); +} + +} // namespace + +CommonGramsQueryPlanConfigSnapshot common_grams_query_plan_config_snapshot() { + const uint64_t state = common_grams_query_plan_state.load(std::memory_order_acquire); + const uint64_t cost_model_state = common_grams_cost_model_state.load(std::memory_order_acquire); + return {.enabled = (state & COMMON_GRAMS_QUERY_PLAN_ENABLED_MASK) != 0, + .cache_generation = state >> COMMON_GRAMS_QUERY_PLAN_GENERATION_SHIFT, + .plan_cost_ratio_percent = + static_cast(cost_model_state & COMMON_GRAMS_COST_RATIO_MASK), + .position_verify_factor = + static_cast(cost_model_state >> COMMON_GRAMS_VERIFY_FACTOR_SHIFT), + .cost_model_generation = cost_model_state}; +} + // condition cache limit DEFINE_Int16(condition_cache_limit, "512"); @@ -1309,6 +1408,63 @@ DEFINE_mDouble(inverted_index_ram_buffer_size, "512"); // -1 indicates not working. // Normally we should not change this, it's useful for testing. DEFINE_mInt32(inverted_index_max_buffered_docs, "-1"); +// DIAGNOSTIC (default off): force SNII inverted-index reads onto NO_CACHE +// (precise S3 range GETs) instead of the 1MiB FILE_BLOCK_CACHE. Per-open on the +// SNII reader only -- does NOT touch global enable_file_cache, so cloud mode +// still boots. Measures the block-cache read amplification's true cost. Warm +// queries lose the local cache under this flag, so it is a measurement knob, not +// a production setting. +DEFINE_mBool(inverted_index_read_bypass_file_cache, "false"); +// G16-c: whether plain positions-tier (non-scoring) SNII indexes lay out freq +// regions. Freq bytes serve ONLY BM25 scoring, which the Doris integration +// does not reach yet (scoring_query has no production caller), so the default +// drops them (textbench: -2.2 GB index). Scoring-config indexes always write +// freq regardless. Applies at segment build (write side only); existing +// segments keep whatever layout they were written with (self-describing). +DEFINE_mBool(snii_positions_index_write_freq, "false"); +// G16-h: zstd levels for the SNII dict-block compression and the .prx window +// auto mode. Level 9 (vs the historical 3) shrinks the two largest compressed +// sections -- textbench: index -457 MB (0.918x -> 0.891x V3) -- for an import +// CPU cost inside the run-to-run variance band; zstd decode speed does not +// depend on the level, and warm/cold benches measured no query change. +// Write side only; segments self-describe their compression. +// Default 3 since the all-level-3 evaluation (2026-07-11, 4 corpora): vs +// level 9 the settled index grows only +0.6%..+6.3% (whole table +// +0.3%..+1.9%) while import index CPU drops 17-24% and full-compaction CPU +// 8-24%; settled cold-query latency is unchanged (interleaved A/B). The +// delta+varint-encoded payloads are high-entropy, so level 9's extra search +// buys almost no ratio. Raise only for size-critical deployments. +DEFINE_mInt32(snii_dict_block_zstd_level, "3"); +DEFINE_mInt32(snii_prx_zstd_level, "3"); +// Patch C prx tiering: zstd level for the prx region of DIRECT-LOAD segments +// only (stream/broker load, see IndexColumnWriter::set_direct_load). Inert at +// the defaults (both levels 3); it exists for size-critical deployments that +// RAISE snii_prx_zstd_level (e.g. 9) and still want cheap loads: compaction +// rewrites every segment at snii_prx_zstd_level, so SETTLED data (and the +// cold-query path over it) is unaffected by the load tier -- measured -290s +// (httplogs) / -204s (agentlogs) of import index CPU at 3 vs 9. Same clamp +// [3, 19]. Read at index flush like snii_prx_zstd_level (a mid-load change +// lands on in-flight segments); the direct-load BIT itself is captured once. +DEFINE_mInt32(snii_prx_zstd_level_direct_load, "3"); +// G16-d: target SNII dict block size in bytes; 0 uses the format default +// (64 KiB). Larger blocks compress better under the per-block zstd (the dict +// is the dominant physical section on high-cardinality corpora) at the cost +// of a larger fetch+decompress unit per cold dict-block miss. Write side +// only; the block size is self-described by the on-disk directory. +DEFINE_mInt32(snii_target_dict_block_bytes, "0"); +// Process-wide SNII index-build RAM budget across ALL live segment writers +// (G09); the largest writers are asked to spill early once the sum crosses it. +// 0 disables. Default 8 GiB. +DEFINE_mInt64(snii_index_writer_global_memory_bytes, "0"); +// Minimum reclaimable posting-arena bytes before a G09 forced spill is honored +// (and before a writer is eligible as a spill victim): forced spills reclaim +// ONLY the arena, so smaller triggers cut tiny runs for near-zero relief. +// Default 64 MiB. +DEFINE_mInt64(snii_forced_spill_min_arena_bytes, "67108864"); +// Max spill-run files one SNII writer accumulates before its runs are +// merge-compacted into one (bounds the k-way merge fan-in and its open fds; +// every run is held open for the whole merge). 0 = uncapped. Default 64. +DEFINE_mInt32(snii_spill_max_run_files_per_buffer, "64"); // dict path for chinese analyzer DEFINE_String(inverted_index_dict_path, "${DORIS_HOME}/dict"); DEFINE_Int32(inverted_index_read_buffer_size, "4096"); @@ -2204,6 +2360,10 @@ bool init(const char* conf_file, bool fill_conf_map, bool must_exist, bool set_t SET_FIELD(it.second, std::vector, fill_conf_map, set_to_default); } + // Startup loading does not invoke mutable-config callbacks. Publish after all three fields are + // validated so query threads never seed the atomics by reading mutable config globals. + publish_common_grams_query_plan_config(); + // Emit a warning for every key present in the conf file that does not correspond to a // registered BE config field. Such keys (typos or configs removed in a newer version) // are silently ignored above, so without this warning operators would have no feedback @@ -2270,6 +2430,33 @@ bool init(const char* conf_file, bool fill_conf_map, bool must_exist, bool set_t return Status::OK(); \ } +namespace { + +// UPDATE_FIELD invokes registered validators before assigning the candidate value. Validate the two +// mutable planner coefficients explicitly so their startup and runtime constraints stay identical. +Status validate_common_grams_runtime_config(const std::string& field, const std::string& value) { + bool (*validator)(int32_t) = nullptr; + if (field == "common_grams_plan_cost_ratio_percent") { + validator = valid_common_grams_cost_ratio; + } else if (field == "common_grams_position_verify_factor") { + validator = valid_common_grams_verify_factor; + } else { + return Status::OK(); + } + + int32_t candidate = 0; + if (!convert(value, candidate)) { + return Status::OK(); + } + if (!validator(candidate)) { + return Status::Error("validate {}={} failed", field, + candidate); + } + return Status::OK(); +} + +} // namespace + // write config to be_custom.conf // the caller need to make sure that the given config is valid Status persist_config(const std::string& field, const std::string& value) { @@ -2300,6 +2487,8 @@ Status set_config(const std::string& field, const std::string& value, bool need_ "'{}' is not support to modify", field); } + RETURN_IF_ERROR(validate_common_grams_runtime_config(field, value)); + UPDATE_FIELD(it->second, value, bool, need_persist); UPDATE_FIELD(it->second, value, int16_t, need_persist); UPDATE_FIELD(it->second, value, int32_t, need_persist); diff --git a/be/src/common/config.h b/be/src/common/config.h index 470d88819703de..91a65c5b3755e6 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -1363,6 +1363,24 @@ DECLARE_Int32(inverted_index_query_cache_shards); // inverted index match bitmap cache size DECLARE_String(inverted_index_query_cache_limit); +// Process-wide emergency switch for CommonGrams query plans. The generation changes on every +// effective transition so result-cache and single-flight entries from an older state are unreachable. +DECLARE_mBool(enable_common_grams_query_plan); +// Build-only CommonGrams kill switch. Logical index writers snapshot it at construction; changing +// it affects only writers created after the transition and never changes query/cache semantics. +DECLARE_mBool(enable_common_grams_index_build); +// Release-calibrated query-planner coefficients. Both remain mutable for controlled recalibration. +DECLARE_mInt32(common_grams_plan_cost_ratio_percent); +DECLARE_mInt32(common_grams_position_verify_factor); +struct CommonGramsQueryPlanConfigSnapshot { + bool enabled = true; + uint64_t cache_generation = 0; + uint32_t plan_cost_ratio_percent = 85; + uint32_t position_verify_factor = 0; + uint64_t cost_model_generation = 0; +}; +CommonGramsQueryPlanConfigSnapshot common_grams_query_plan_config_snapshot(); + // condition cache limit DECLARE_Int16(condition_cache_limit); @@ -1374,6 +1392,62 @@ DECLARE_Int32(ann_index_result_cache_stale_sweep_time_sec); // inverted index DECLARE_mDouble(inverted_index_ram_buffer_size); DECLARE_mInt32(inverted_index_max_buffered_docs); +// DIAGNOSTIC: force SNII inverted-index reads to bypass the 1MiB FILE_BLOCK_CACHE +// and issue precise S3 range GETs (NO_CACHE) instead. Applies ONLY to the SNII +// index file reader (per-open cache_type), NOT the global enable_file_cache, so +// cloud mode does not FATAL. Default false (keep block cache). Used to measure +// whether the block-cache read amplification hurts cold reads at the cost of +// re-reading S3 on every (warm) query. Not for production: warm loses the local +// cache. Read-side only. +DECLARE_mBool(inverted_index_read_bypass_file_cache); +// G16-c: whether plain positions-tier (non-scoring) SNII indexes lay out freq +// regions. Freq serves ONLY BM25 scoring (no production caller yet), so the +// default (false) drops the layout; scoring-config indexes always keep freq. +// Write-side only; segments are self-describing either way. +DECLARE_mBool(snii_positions_index_write_freq); +// G16-h: zstd levels for SNII dict blocks / prx windows. Default 3 (the +// all-level-3 evaluation showed level 9 buys <=6.3% index size for 17-24% +// import CPU; see the DEFINEs in config.cpp). +DECLARE_mInt32(snii_dict_block_zstd_level); +DECLARE_mInt32(snii_prx_zstd_level); +// Patch C: prx zstd level for DIRECT-LOAD segments only (default 3, cheaper +// import); compaction rewrites at snii_prx_zstd_level so settled segments are +// unaffected. Full contract at the DEFINE in config.cpp. +DECLARE_mInt32(snii_prx_zstd_level_direct_load); +// G16-d: target SNII dict block size in bytes; 0 = format default (64 KiB). +// Bigger blocks -> better per-block zstd on the dict region, larger cold +// fetch+decompress unit per dict-block miss. Write side only. +DECLARE_mInt32(snii_target_dict_block_bytes); +// PROCESS-WIDE budget in bytes for SNII index-build RAM, summed across every +// live SNII segment writer of the BE (all tablets x all concurrent loads; G09). +// The per-writer inverted_index_ram_buffer_size is a reclaimable-buffer spill +// threshold, not a hard cap on persistent vocabulary bytes. A concurrent load +// keeps (tablets x concurrency) writers alive at once, none of which may reach +// that threshold, while their SUM can still be large. When the registered total +// exceeds this process-wide budget, the LARGEST writers are asked to spill their +// posting buffers to disk early (async-safe advisory requests, honored on each +// writer's own thread; output stays byte-identical). 0 disables the global +// limiter (per-writer spilling only). Checked at index-writer creation: a change +// applies to writers created afterwards. +DECLARE_mInt64(snii_index_writer_global_memory_bytes); +// G09 forced-spill floor: minimum reclaimable posting-arena bytes a SNII +// writer must hold before a process-wide forced-spill request is honored, and +// before the global limiter selects it as a spill victim. A forced spill +// reclaims ONLY the posting arena -- the persistent vocab / pair-map +// structures survive it -- so honoring below a real floor degenerates into a +// storm of tiny runs whenever the budget is dominated by persistent bytes +// (each run then costs a file, a sort and a merge-fd for near-zero memory +// relief). The global budget therefore bounds SPILLABLE memory, not +// persistent memory. Default 64 MiB. +DECLARE_mInt64(snii_forced_spill_min_arena_bytes); +// G09 run-file cap: maximum spill-run files one SNII writer may accumulate; +// on the next spill past the cap, the existing runs are merge-compacted into +// a single run first (term stream unchanged). Bounds the final k-way merge's +// fan-in and, decisively, its simultaneously-open file descriptors -- every +// run of a buffer is reopened and held open for the whole merge, so unbounded +// run counts across ~100 concurrent writers can exhaust the BE nofile rlimit +// ("Too many open files" at run reopen). 0 disables the cap. Default 64. +DECLARE_mInt32(snii_spill_max_run_files_per_buffer); // dict path for chinese analyzer DECLARE_String(inverted_index_dict_path); DECLARE_Int32(inverted_index_read_buffer_size); diff --git a/be/src/common/status.h b/be/src/common/status.h index d29b66459a3832..37ff6917b4f567 100644 --- a/be/src/common/status.h +++ b/be/src/common/status.h @@ -299,6 +299,7 @@ namespace ErrorCode { E(INVERTED_INDEX_COMPACTION_ERROR, -6010, false); \ E(INVERTED_INDEX_ANALYZER_ERROR, -6011, false); \ E(INVERTED_INDEX_FILE_CORRUPTED, -6012, false); \ + E(INVERTED_INDEX_SNII_NOT_FOUND, -6013, false); \ E(KEY_NOT_FOUND, -7000, false); \ E(KEY_ALREADY_EXISTS, -7001, false); \ E(ENTRY_NOT_FOUND, -7002, false); \ diff --git a/be/src/exec/operator/olap_scan_operator.cpp b/be/src/exec/operator/olap_scan_operator.cpp index dda268de829252..3c8462c923dd3f 100644 --- a/be/src/exec/operator/olap_scan_operator.cpp +++ b/be/src/exec/operator/olap_scan_operator.cpp @@ -264,6 +264,10 @@ Status OlapScanLocalState::_init_profile() { ADD_COUNTER_WITH_LEVEL(_segment_profile, "InvertedIndexQueryCacheHit", TUnit::UNIT, 1); _inverted_index_query_cache_miss_counter = ADD_COUNTER_WITH_LEVEL(_segment_profile, "InvertedIndexQueryCacheMiss", TUnit::UNIT, 1); + _inverted_index_query_cache_lookup_counter = ADD_COUNTER_WITH_LEVEL( + _segment_profile, "InvertedIndexQueryCacheLookup", TUnit::UNIT, 1); + _inverted_index_query_cache_insert_counter = ADD_COUNTER_WITH_LEVEL( + _segment_profile, "InvertedIndexQueryCacheInsert", TUnit::UNIT, 1); _inverted_index_query_timer = ADD_TIMER_WITH_LEVEL(_segment_profile, "InvertedIndexQueryTime", 1); _inverted_index_query_null_bitmap_timer = @@ -348,6 +352,8 @@ Status OlapScanLocalState::_init_profile() { _index_filter_profile = std::make_unique("IndexFilter"); _scanner_profile->add_child(_index_filter_profile.get(), true, nullptr); + _snii_prx_profile_counters.initialize(_index_filter_profile.get()); + _snii_phrase_profile_counters.initialize(_index_filter_profile.get()); /* SegmentIterator: - AnnIndexLoadCosts: 102.262us diff --git a/be/src/exec/operator/olap_scan_operator.h b/be/src/exec/operator/olap_scan_operator.h index 9f7ddca05274a0..38aae4c0ce0d91 100644 --- a/be/src/exec/operator/olap_scan_operator.h +++ b/be/src/exec/operator/olap_scan_operator.h @@ -28,6 +28,7 @@ #include "exec/operator/operator.h" #include "exec/operator/scan_operator.h" #include "runtime/runtime_profile.h" +#include "storage/index/snii/snii_prx_profile.h" #include "storage/olap_scan_common.h" #include "storage/tablet/tablet_reader.h" @@ -149,6 +150,8 @@ class OlapScanLocalState final : public ScanLocalState { std::unique_ptr _segment_profile; std::unique_ptr _index_filter_profile; + snii::SniiPrxRuntimeProfileCounters _snii_prx_profile_counters; + snii::SniiPhraseRuntimeProfileCounters _snii_phrase_profile_counters; RuntimeProfile::Counter* _tablet_counter = nullptr; RuntimeProfile::Counter* _key_range_counter = nullptr; @@ -243,6 +246,8 @@ class OlapScanLocalState final : public ScanLocalState { RuntimeProfile::Counter* _inverted_index_query_null_bitmap_timer = nullptr; RuntimeProfile::Counter* _inverted_index_query_cache_hit_counter = nullptr; RuntimeProfile::Counter* _inverted_index_query_cache_miss_counter = nullptr; + RuntimeProfile::Counter* _inverted_index_query_cache_lookup_counter = nullptr; + RuntimeProfile::Counter* _inverted_index_query_cache_insert_counter = nullptr; RuntimeProfile::Counter* _inverted_index_query_timer = nullptr; RuntimeProfile::Counter* _inverted_index_query_bitmap_copy_timer = nullptr; RuntimeProfile::Counter* _inverted_index_searcher_open_timer = nullptr; diff --git a/be/src/exec/scan/olap_scanner.cpp b/be/src/exec/scan/olap_scanner.cpp index 9ffdb94e149c91..91aa9685f0c824 100644 --- a/be/src/exec/scan/olap_scanner.cpp +++ b/be/src/exec/scan/olap_scanner.cpp @@ -57,6 +57,7 @@ #include "storage/binlog.h" #include "storage/id_manager.h" #include "storage/index/inverted/inverted_index_profile.h" +#include "storage/index/inverted/similarity/collection_statistics.h" #include "storage/iterator/block_reader.h" #include "storage/olap_common.h" #include "storage/olap_tuple.h" @@ -156,7 +157,10 @@ static bool has_file_cache_statistics(const io::FileCacheStatistics& stats) { stats.inverted_index_bytes_read_from_remote != 0 || stats.inverted_index_bytes_read_from_peer != 0 || stats.inverted_index_local_io_timer != 0 || stats.inverted_index_remote_io_timer != 0 || - stats.inverted_index_peer_io_timer != 0 || stats.inverted_index_io_timer != 0; + stats.inverted_index_peer_io_timer != 0 || stats.inverted_index_io_timer != 0 || + stats.inverted_index_request_bytes != 0 || stats.inverted_index_read_bytes != 0 || + stats.inverted_index_range_read_count != 0 || + stats.inverted_index_serial_read_rounds != 0; } io::IOContext build_score_runtime_collection_io_context(RuntimeState* state, ReaderType reader_type, @@ -1001,6 +1005,10 @@ void OlapScanner::_collect_profile_before_close() { stats.inverted_index_query_cache_hit); COUNTER_UPDATE(local_state->_inverted_index_query_cache_miss_counter, stats.inverted_index_query_cache_miss); + COUNTER_UPDATE(local_state->_inverted_index_query_cache_lookup_counter, + stats.inverted_index_query_cache_lookup); + COUNTER_UPDATE(local_state->_inverted_index_query_cache_insert_counter, + stats.inverted_index_query_cache_insert); COUNTER_UPDATE(local_state->_inverted_index_query_timer, stats.inverted_index_query_timer); COUNTER_UPDATE(local_state->_inverted_index_query_null_bitmap_timer, stats.inverted_index_query_null_bitmap_timer); @@ -1023,6 +1031,8 @@ void OlapScanner::_collect_profile_before_close() { COUNTER_UPDATE(local_state->_inverted_index_analyzer_timer, stats.inverted_index_analyzer_timer); COUNTER_UPDATE(local_state->_inverted_index_lookup_timer, stats.inverted_index_lookup_timer); + local_state->_snii_prx_profile_counters.update(stats); + local_state->_snii_phrase_profile_counters.update(stats); COUNTER_UPDATE(local_state->_variant_scan_sparse_column_timer, stats.variant_scan_sparse_column_timer_ns); COUNTER_UPDATE(local_state->_variant_scan_sparse_column_bytes, diff --git a/be/src/exprs/function/function_search.cpp b/be/src/exprs/function/function_search.cpp index e87aff95ee075c..eb9c3e253d2179 100644 --- a/be/src/exprs/function/function_search.cpp +++ b/be/src/exprs/function/function_search.cpp @@ -115,6 +115,50 @@ static std::string extract_segment_prefix( return ""; } +static void collect_referenced_fields(const TSearchClause& clause, + std::unordered_set* fields) { + DORIS_CHECK(fields != nullptr); + if (clause.__isset.field_name && !clause.field_name.empty()) { + fields->insert(clause.field_name); + } + for (const auto& child : clause.children) { + collect_referenced_fields(child, fields); + } +} + +static bool referenced_fields_contain_snii_reader( + const TSearchClause& root, + const std::unordered_map& iterators) { + std::unordered_set referenced_fields; + collect_referenced_fields(root, &referenced_fields); + for (const auto& field_name : referenced_fields) { + auto iterator_it = iterators.find(field_name); + if (iterator_it == iterators.end()) { + continue; + } + auto* inv_iter = dynamic_cast(iterator_it->second); + if (inv_iter == nullptr) { + continue; + } + for (auto type : {InvertedIndexReaderType::FULLTEXT, InvertedIndexReaderType::STRING_TYPE, + InvertedIndexReaderType::BKD}) { + IndexReaderType reader_type = type; + auto reader = inv_iter->get_reader(reader_type); + if (reader == nullptr) { + continue; + } + auto inv_reader = std::dynamic_pointer_cast(reader); + DORIS_CHECK(inv_reader != nullptr); + auto file_reader = inv_reader->get_index_file_reader(); + DORIS_CHECK(file_reader != nullptr); + if (file_reader->get_storage_format() == InvertedIndexStorageFormatPB::SNII) { + return true; + } + } + } + return false; +} + namespace { bool is_nested_group_search_supported() { @@ -216,6 +260,14 @@ InvertedIndexQueryType direct_index_query_type_for_clause(const std::string& cla return InvertedIndexQueryType::UNKNOWN_QUERY; } +std::string normalize_wildcard_pattern(const std::string& value, + const std::map& index_properties) { + const bool has_parser = + inverted_index::InvertedIndexAnalyzer::should_analyzer(index_properties); + const std::string lowercase_setting = get_parser_lowercase_from_properties(index_properties); + return has_parser && lowercase_setting == INVERTED_INDEX_PARSER_TRUE ? to_lower(value) : value; +} + } // namespace Status FunctionSearch::execute_impl(FunctionContext* /*context*/, Block& /*block*/, @@ -274,13 +326,16 @@ Status FunctionSearch::evaluate_inverted_index_with_search_param( OlapReaderStatistics* outer_stats = index_query_context ? index_query_context->stats : nullptr; SCOPED_RAW_TIMER(outer_stats ? &outer_stats->inverted_index_query_timer : &query_timer_dummy); - const bool need_similarity_score = - index_query_context && index_query_context->collection_similarity; - // DSL result cache only stores bitmap/null bitmap. It does not store BM25 scores, // so score() queries must execute scorers to populate CollectionSimilarity. - auto* dsl_cache = (enable_cache && !need_similarity_score) ? InvertedIndexQueryCache::instance() - : nullptr; + const bool enable_scoring = + index_query_context != nullptr && index_query_context->collection_similarity != nullptr; + // Also bypass the DSL cache when any referenced field is served by an SNII reader. + auto* dsl_cache = + enable_cache && !enable_scoring && + !referenced_fields_contain_snii_reader(search_param.root, iterators) + ? InvertedIndexQueryCache::instance() + : nullptr; std::string seg_prefix; std::string dsl_sig; InvertedIndexQueryCache::CacheKey dsl_cache_key; @@ -394,11 +449,9 @@ Status FunctionSearch::evaluate_inverted_index_with_search_param( query_v2::QueryExecutionContext exec_ctx = build_variant_search_query_execution_context(num_rows, resolver, &null_resolver); - bool enable_scoring = false; bool is_asc = false; size_t top_k = 0; if (index_query_context) { - enable_scoring = index_query_context->collection_similarity != nullptr; is_asc = index_query_context->is_asc; top_k = index_query_context->query_limit; } @@ -734,6 +787,40 @@ Status FunctionSearch::build_leaf_query(const TSearchClause& clause, *binding_key = binding.binding_key; } + if (binding.use_snii_native_reader()) { + DORIS_CHECK(binding.inverted_reader != nullptr); + if (clause_type != "WILDCARD") { + return Status::NotSupported( + "SNII native SEARCH supports only WILDCARD clauses; got '{}'", clause_type); + } + + auto data_bitmap = std::make_shared(); + if (value == "*") { + data_bitmap->addRange(0, num_rows); + } else { + std::string pattern = normalize_wildcard_pattern(value, binding.index_properties); + Field query_value = Field::create_field(pattern); + RETURN_IF_ERROR(binding.inverted_reader->query( + context, binding.stored_field_name, query_value, + InvertedIndexQueryType::WILDCARD_QUERY, data_bitmap, nullptr)); + VLOG_DEBUG << "search: SNII WILDCARD clause processed, field=" << field_name + << ", pattern='" << pattern << "' (original='" << value << "')"; + } + + auto null_bitmap = std::make_shared(); + if (binding.inverted_reader->has_null()) { + segment_v2::InvertedIndexQueryCacheHandle null_bitmap_cache_handle; + RETURN_IF_ERROR(binding.inverted_reader->read_null_bitmap( + context, &null_bitmap_cache_handle, nullptr)); + auto cached_null_bitmap = null_bitmap_cache_handle.get_bitmap(); + DORIS_CHECK(cached_null_bitmap != nullptr); + null_bitmap = std::move(cached_null_bitmap); + } + *data_bitmap -= *null_bitmap; + return finish_leaf_query(std::make_shared(std::move(data_bitmap), + std::move(null_bitmap))); + } + if (binding.use_direct_index_reader()) { auto direct_query_type = direct_index_query_type_for_clause(clause_type); if (direct_query_type == InvertedIndexQueryType::UNKNOWN_QUERY) { @@ -800,7 +887,8 @@ Status FunctionSearch::build_leaf_query(const TSearchClause& clause, std::vector term_infos = inverted_index::InvertedIndexAnalyzer::get_analyse_result( - value, binding.index_properties); + value, binding.index_properties, + inverted_index::AnalysisPurpose::kPlainQuery); if (term_infos.empty()) { LOG(WARNING) << "search: No terms found after tokenization for TERM query, field=" << field_name << ", value='" << value @@ -864,7 +952,8 @@ Status FunctionSearch::build_leaf_query(const TSearchClause& clause, std::vector term_infos = inverted_index::InvertedIndexAnalyzer::get_analyse_result( - value, binding.index_properties); + value, binding.index_properties, + inverted_index::AnalysisPurpose::kPlainQuery); if (term_infos.empty()) { LOG(WARNING) << "search: No terms found after tokenization for PHRASE query, field=" << field_name << ", value='" << value @@ -879,8 +968,7 @@ Status FunctionSearch::build_leaf_query(const TSearchClause& clause, const auto& term_info = phrase_term_infos[0]; if (term_info.is_single_term()) { std::wstring term_wstr = StringHelper::to_wstring(term_info.get_single_term()); - return finish_leaf_query( - std::make_shared(context, field_wstr, term_wstr)); + return finish_leaf_query(make_term_query(term_wstr)); } else { auto builder = create_operator_boolean_query_builder(query_v2::OperatorType::OP_OR); @@ -922,7 +1010,8 @@ Status FunctionSearch::build_leaf_query(const TSearchClause& clause, std::vector term_infos = inverted_index::InvertedIndexAnalyzer::get_analyse_result( - value, binding.index_properties); + value, binding.index_properties, + inverted_index::AnalysisPurpose::kPlainQuery); if (term_infos.empty()) { LOG(WARNING) << "search: tokenization yielded no terms for clause '" << clause_type << "', field=" << field_name << ", returning empty BitSetQuery"; @@ -994,8 +1083,7 @@ Status FunctionSearch::build_leaf_query(const TSearchClause& clause, binding.index_properties); std::string lowercase_setting = get_parser_lowercase_from_properties(binding.index_properties); - bool should_lowercase = has_parser && (lowercase_setting == INVERTED_INDEX_PARSER_TRUE); - std::string pattern = should_lowercase ? to_lower(value) : value; + std::string pattern = normalize_wildcard_pattern(value, binding.index_properties); VLOG_DEBUG << "search: WILDCARD clause processed, field=" << field_name << ", pattern='" << pattern << "' (original='" << value << "', has_parser=" << has_parser << ", lower_case=" << lowercase_setting << ")"; diff --git a/be/src/exprs/function/function_tokenize.cpp b/be/src/exprs/function/function_tokenize.cpp index a3d616d635763c..b7aae728cfe504 100644 --- a/be/src/exprs/function/function_tokenize.cpp +++ b/be/src/exprs/function/function_tokenize.cpp @@ -187,7 +187,7 @@ Status FunctionTokenize::execute_impl(FunctionContext* /*context*/, Block& block try { analyzer_holder = doris::segment_v2::inverted_index::InvertedIndexAnalyzer::create_analyzer( - &config); + &config, AnalysisPurpose::kPlainQuery); } catch (CLuceneError& e) { return Status::Error( "inverted index create analyzer failed: {}", e.what()); diff --git a/be/src/exprs/function/match.cpp b/be/src/exprs/function/match.cpp index a280dd035e25b4..b507793785f78c 100644 --- a/be/src/exprs/function/match.cpp +++ b/be/src/exprs/function/match.cpp @@ -87,6 +87,8 @@ Status FunctionMatchBase::evaluate_inverted_index( param.query_type = get_query_type_from_fn_name(); param.num_rows = num_rows; param.roaring = std::make_shared(); + segment_v2::InvertedIndexQueryCacheHandle null_bitmap_cache_handle; + param.null_bitmap_cache_handle = &null_bitmap_cache_handle; param.analyzer_ctx = analyzer_ctx; if (is_string_type(param_type)) { RETURN_IF_ERROR(iter->read_from_index(¶m)); @@ -95,11 +97,11 @@ Status FunctionMatchBase::evaluate_inverted_index( "invalid params type for FunctionMatchBase::evaluate_inverted_index {}", param_type); } - std::shared_ptr null_bitmap = std::make_shared(); - if (iter->has_null()) { - segment_v2::InvertedIndexQueryCacheHandle null_bitmap_cache_handle; - RETURN_IF_ERROR(iter->read_null_bitmap(&null_bitmap_cache_handle)); - null_bitmap = null_bitmap_cache_handle.get_bitmap(); + std::shared_ptr null_bitmap = null_bitmap_cache_handle.get_bitmap(); + if (null_bitmap == nullptr) { + // query_with_null_bitmap leaves the handle empty only when the selected reader proves that + // the index has no null rows. + null_bitmap = std::make_shared(); } segment_v2::InvertedIndexResultBitmap result(param.roaring, null_bitmap); bitmap_result = result; diff --git a/be/src/exprs/function/variant_inverted_index_search.cpp b/be/src/exprs/function/variant_inverted_index_search.cpp index cf3fc0505188c6..1e5d8f244898d4 100644 --- a/be/src/exprs/function/variant_inverted_index_search.cpp +++ b/be/src/exprs/function/variant_inverted_index_search.cpp @@ -36,6 +36,7 @@ #include "storage/index/inverted/inverted_index_compound_reader.h" #include "storage/index/inverted/inverted_index_parser.h" #include "storage/index/inverted/inverted_index_searcher.h" +#include "storage/index/inverted/inverted_index_selector.h" #include "storage/index/inverted/query_v2/bit_set_query/bit_set_scorer.h" #include "storage/index/inverted/query_v2/doc_set.h" #include "storage/index/inverted/query_v2/scorer.h" @@ -138,18 +139,18 @@ Status FieldReaderResolver::resolve(const std::string& field_name, InvertedIndexQueryType effective_query_type = query_type; const auto& column_type = data_it->second.second; - const bool is_text_field = - column_type != nullptr && is_string_type(column_type->get_storage_field_type()); + const bool is_text_field = column_type != nullptr && + is_string_type(get_inverted_index_leaf_field_type(column_type)); auto fb_it = _field_binding_map.find(field_name); std::string analyzer_key; - if (is_text_field && is_variant_sub && fb_it != _field_binding_map.end() && - fb_it->second->__isset.index_properties && !fb_it->second->index_properties.empty()) { + if (is_text_field && effective_query_type != InvertedIndexQueryType::EQUAL_QUERY && + fb_it != _field_binding_map.end() && fb_it->second->__isset.index_properties && + !fb_it->second->index_properties.empty()) { analyzer_key = normalize_analyzer_key( build_analyzer_key_from_properties(fb_it->second->index_properties)); if (inverted_index::InvertedIndexAnalyzer::should_analyzer( fb_it->second->index_properties) && - (effective_query_type == InvertedIndexQueryType::EQUAL_QUERY || - effective_query_type == InvertedIndexQueryType::WILDCARD_QUERY)) { + effective_query_type == InvertedIndexQueryType::WILDCARD_QUERY) { effective_query_type = InvertedIndexQueryType::MATCH_ANY_QUERY; } } @@ -200,12 +201,7 @@ Status FieldReaderResolver::resolve(const std::string& field_name, resolved.inverted_reader = inverted_reader; resolved.binding_key = binding_key; resolved.state = SearchFieldBindingState::BOUND; - if (fb_it != _field_binding_map.end() && fb_it->second->__isset.index_properties && - !fb_it->second->index_properties.empty()) { - resolved.index_properties = fb_it->second->index_properties; - } else { - resolved.index_properties = inverted_reader->get_index_properties(); - } + resolved.index_properties = inverted_reader->get_index_properties(); resolved.analyzer_key = normalize_analyzer_key(build_analyzer_key_from_properties(resolved.index_properties)); @@ -225,6 +221,7 @@ Status FieldReaderResolver::resolve(const std::string& field_name, } if (inverted_reader->type() == InvertedIndexReaderType::BKD) { + resolved.execution_mode = SearchFieldExecutionMode::DIRECT_INDEX; _cache.emplace(binding_key, resolved); if (is_variant_sub) { bool index_file_exists = false; @@ -249,6 +246,29 @@ Status FieldReaderResolver::resolve(const std::string& field_name, return Status::OK(); } + if (index_file_reader->get_storage_format() == InvertedIndexStorageFormatPB::SNII) { + resolved.execution_mode = SearchFieldExecutionMode::SNII_NATIVE; + _cache.emplace(binding_key, resolved); + if (is_variant_sub) { + add_search_binding_diagnostic( + _context, + fmt::format("[VariantSearchBinding] phase=field_resolve " + "result=selected_snii_native logical_field={} stored_field={} " + "query_type={} effective_query_type={} index_id={} suffix={} " + "reader_type={} analyzer_key={} index_file={}", + field_name, stored_field_name, query_type_to_string(query_type), + query_type_to_string(effective_query_type), + inverted_reader->get_index_id(), + inverted_reader->get_index_meta().get_index_suffix(), + reader_type_to_string(inverted_reader->type()), + resolved.analyzer_key, + index_file_reader->get_index_file_path( + &inverted_reader->get_index_meta()))); + } + *binding = resolved; + return Status::OK(); + } + auto index_file_key = index_file_reader->get_index_file_cache_key(&inverted_reader->get_index_meta()); InvertedIndexSearcherCache::CacheKey searcher_cache_key(index_file_key); @@ -291,7 +311,6 @@ Status FieldReaderResolver::resolve(const std::string& field_name, index_file_reader->init(config::inverted_index_read_buffer_size, _context->io_ctx)); auto directory = DORIS_TRY( index_file_reader->open(&inverted_reader->get_index_meta(), _context->io_ctx)); - auto index_searcher_builder = DORIS_TRY( IndexSearcherBuilder::create_index_searcher_builder(inverted_reader->type())); auto searcher_result = @@ -334,6 +353,7 @@ Status FieldReaderResolver::resolve(const std::string& field_name, _searcher_cache_handles.push_back(std::move(searcher_cache_handle)); resolved.lucene_reader = reader_holder; + resolved.execution_mode = SearchFieldExecutionMode::CLUCENE; _binding_readers[binding_key] = reader_holder; _field_readers[resolved.stored_field_wstr] = reader_holder; _readers.emplace_back(reader_holder); diff --git a/be/src/exprs/function/variant_inverted_index_search.h b/be/src/exprs/function/variant_inverted_index_search.h index 973c9c8c826c55..d03dbc486d8e18 100644 --- a/be/src/exprs/function/variant_inverted_index_search.h +++ b/be/src/exprs/function/variant_inverted_index_search.h @@ -68,6 +68,13 @@ enum class SearchFieldBindingState { MISSING_IN_SEGMENT, }; +enum class SearchFieldExecutionMode { + UNBOUND, + CLUCENE, + DIRECT_INDEX, + SNII_NATIVE, +}; + struct FieldReaderBinding { std::string logical_field_name; std::string stored_field_name; @@ -80,13 +87,17 @@ struct FieldReaderBinding { std::string binding_key; std::string analyzer_key; SearchFieldBindingState state = SearchFieldBindingState::MISSING_IN_SEGMENT; + SearchFieldExecutionMode execution_mode = SearchFieldExecutionMode::UNBOUND; bool is_bound() const { return state == SearchFieldBindingState::BOUND || inverted_reader != nullptr || lucene_reader != nullptr; } bool use_direct_index_reader() const { - return is_bound() && inverted_reader != nullptr && lucene_reader == nullptr; + return is_bound() && execution_mode == SearchFieldExecutionMode::DIRECT_INDEX; + } + bool use_snii_native_reader() const { + return is_bound() && execution_mode == SearchFieldExecutionMode::SNII_NATIVE; } }; diff --git a/be/src/exprs/vcompound_pred.h b/be/src/exprs/vcompound_pred.h index 8a153ef8e88615..d44346ee114cbd 100644 --- a/be/src/exprs/vcompound_pred.h +++ b/be/src/exprs/vcompound_pred.h @@ -44,6 +44,15 @@ inline std::string compound_operator_to_string(TExprOpcode::type op) { } } +inline bool inverted_index_status_allows_row_fallback(const Status& status) { + DORIS_CHECK(!status.ok()); + return status.is() || + status.is() || + status.is() || + status.is() || + status.is(); +} + class VCompoundPred : public VectorizedFnCall { ENABLE_FACTORY_CREATOR(VCompoundPred); @@ -272,6 +281,9 @@ class VCompoundPred : public VectorizedFnCall { !st.ok()) { LOG(ERROR) << "expr:" << child->expr_name() << " evaluate_inverted_index error:" << st.to_string(); + if (!inverted_index_status_allows_row_fallback(st)) { + return st; + } all_pass = false; continue; } @@ -301,6 +313,9 @@ class VCompoundPred : public VectorizedFnCall { !st.ok()) { LOG(ERROR) << "expr:" << child->expr_name() << " evaluate_inverted_index error:" << st.to_string(); + if (!inverted_index_status_allows_row_fallback(st)) { + return st; + } all_pass = false; continue; } diff --git a/be/src/exprs/vexpr.h b/be/src/exprs/vexpr.h index c0b99ab7a8f4dc..de5ad018d3be09 100644 --- a/be/src/exprs/vexpr.h +++ b/be/src/exprs/vexpr.h @@ -267,6 +267,10 @@ class VExpr { return empty; } + [[nodiscard]] virtual const InvertedIndexAnalyzerCtx* query_analyzer_ctx() const { + return nullptr; + } + Status _evaluate_inverted_index(VExprContext* context, const FunctionBasePtr& function, uint32_t segment_num_rows); diff --git a/be/src/exprs/vmatch_predicate.cpp b/be/src/exprs/vmatch_predicate.cpp index cf4f87e45d3b7e..fb9f9f23a651be 100644 --- a/be/src/exprs/vmatch_predicate.cpp +++ b/be/src/exprs/vmatch_predicate.cpp @@ -77,7 +77,8 @@ VMatchPredicate::VMatchPredicate(const TExprNode& node) : VExpr(node) { // Always create analyzer based on parser_type for slow path (tables without index). // For index path, FullTextIndexReader will check analyzer_name to decide whether // to use this analyzer or fallback to index's own analyzer. - _analyzer = inverted_index::InvertedIndexAnalyzer::create_analyzer(&config); + _analyzer_provider = inverted_index::InvertedIndexAnalyzer::create_analyzer_provider(&config); + _analyzer = _analyzer_provider->get_analyzer(inverted_index::AnalysisPurpose::kPlainQuery); // Step 3: Create runtime context (only extract runtime-needed info) _analyzer_ctx = std::make_shared(); @@ -85,6 +86,7 @@ VMatchPredicate::VMatchPredicate(const TExprNode& node) : VExpr(node) { _analyzer_ctx->parser_type = config.parser_type; _analyzer_ctx->char_filter_map = std::move(config.char_filter_map); _analyzer_ctx->analyzer = _analyzer; + _analyzer_ctx->analyzer_provider = _analyzer_provider; } VMatchPredicate::~VMatchPredicate() = default; diff --git a/be/src/exprs/vmatch_predicate.h b/be/src/exprs/vmatch_predicate.h index e36b695e3cfb1a..4b6792c44ed88d 100644 --- a/be/src/exprs/vmatch_predicate.h +++ b/be/src/exprs/vmatch_predicate.h @@ -57,6 +57,9 @@ class VMatchPredicate final : public VExpr { const std::string& expr_name() const override; const std::string& function_name() const; [[nodiscard]] const std::string& get_analyzer_key() const override; + [[nodiscard]] const InvertedIndexAnalyzerCtx* query_analyzer_ctx() const override { + return _analyzer_ctx.get(); + } std::string debug_string() const override; @@ -69,6 +72,7 @@ class VMatchPredicate final : public VExpr { // Lifecycle management: holds ownership of the analyzer std::shared_ptr _analyzer; + segment_v2::inverted_index::AnalyzerProviderPtr _analyzer_provider; // Runtime context: holds raw pointer to analyzer and necessary runtime info InvertedIndexAnalyzerCtxSPtr _analyzer_ctx; diff --git a/be/src/exprs/vsearch.cpp b/be/src/exprs/vsearch.cpp index ecaef392db87b2..75a3b48c67fad2 100644 --- a/be/src/exprs/vsearch.cpp +++ b/be/src/exprs/vsearch.cpp @@ -166,6 +166,24 @@ Status collect_search_inputs(const VSearchExpr& expr, VExprContext* context, return Status::OK(); } +bool search_status_allows_row_fallback(const Status& status) { + DORIS_CHECK(!status.ok()); + return status.is() || + status.is() || + status.is() || + status.is() || + status.is(); +} + +Status prevent_search_row_fallback(Status status) { + DORIS_CHECK(!status.ok()); + if (!search_status_allows_row_fallback(status)) { + return status; + } + return Status::Error( + "SEARCH cannot fall back to row execution: {}", status.to_string()); +} + } // namespace VSearchExpr::VSearchExpr(const TExprNode& node) : VExpr(node) { @@ -202,7 +220,7 @@ Status VSearchExpr::execute_column_impl(VExprContext* context, const Block* bloc Status VSearchExpr::evaluate_inverted_index(VExprContext* context, uint32_t segment_num_rows) { if (_search_param.original_dsl.empty()) { - return Status::InvalidArgument("search DSL is empty"); + return prevent_search_row_fallback(Status::InvalidArgument("search DSL is empty")); } auto index_context = context->get_index_context(); @@ -212,7 +230,9 @@ Status VSearchExpr::evaluate_inverted_index(VExprContext* context, uint32_t segm } SearchInputBundle bundle; - RETURN_IF_ERROR(collect_search_inputs(*this, context, &bundle)); + if (auto status = collect_search_inputs(*this, context, &bundle); !status.ok()) { + return prevent_search_row_fallback(std::move(status)); + } VLOG_DEBUG << "VSearchExpr: bundle.iterators.size()=" << bundle.iterators.size(); @@ -242,7 +262,7 @@ Status VSearchExpr::evaluate_inverted_index(VExprContext* context, uint32_t segm if (!status.ok()) { LOG(WARNING) << "VSearchExpr: Function evaluation failed: " << status.to_string(); - return status; + return prevent_search_row_fallback(std::move(status)); } index_context->set_index_result_for_expr(this, result_bitmap); diff --git a/be/src/io/cache/block_file_cache_profile.cpp b/be/src/io/cache/block_file_cache_profile.cpp index 7aed33ccaa6e07..40a031f7bcb658 100644 --- a/be/src/io/cache/block_file_cache_profile.cpp +++ b/be/src/io/cache/block_file_cache_profile.cpp @@ -17,6 +17,7 @@ #include "io/cache/block_file_cache_profile.h" +#include #include #include #include @@ -106,12 +107,17 @@ FileCacheStatistics diff_file_cache_statistics(const FileCacheStatistics& curren SUBTRACT_FIELD(inverted_index_bytes_read_from_local); SUBTRACT_FIELD(inverted_index_bytes_read_from_remote); SUBTRACT_FIELD(inverted_index_bytes_read_from_peer); + SUBTRACT_FIELD(inverted_index_remote_physical_read_bytes); + SUBTRACT_FIELD(inverted_index_bytes_write_into_cache); SUBTRACT_FIELD(inverted_index_local_io_timer); SUBTRACT_FIELD(inverted_index_remote_io_timer); SUBTRACT_FIELD(inverted_index_peer_io_timer); SUBTRACT_FIELD(inverted_index_io_timer); SUBTRACT_FIELD(inverted_index_write_cache_io_timer); - SUBTRACT_FIELD(inverted_index_bytes_write_into_cache); + SUBTRACT_FIELD(inverted_index_request_bytes); + SUBTRACT_FIELD(inverted_index_read_bytes); + SUBTRACT_FIELD(inverted_index_range_read_count); + SUBTRACT_FIELD(inverted_index_serial_read_rounds); SUBTRACT_FIELD(segment_footer_index_num_local_io_total); SUBTRACT_FIELD(segment_footer_index_num_remote_io_total); @@ -193,6 +199,10 @@ FileCacheProfileReporter::FileCacheProfileReporter(RuntimeProfile* profile, profile, "InvertedIndexBytesScannedFromRemote", TUnit::BYTES, cache_profile, 1); inverted_index_bytes_scanned_from_peer = ADD_CHILD_COUNTER_WITH_LEVEL( profile, "InvertedIndexBytesScannedFromPeer", TUnit::BYTES, cache_profile, 1); + inverted_index_remote_physical_read_bytes = ADD_CHILD_COUNTER_WITH_LEVEL( + profile, "InvertedIndexRemotePhysicalReadBytes", TUnit::BYTES, cache_profile, 1); + inverted_index_bytes_write_into_cache = ADD_CHILD_COUNTER_WITH_LEVEL( + profile, "InvertedIndexBytesWriteIntoCache", TUnit::BYTES, cache_profile, 1); inverted_index_local_io_timer = ADD_CHILD_TIMER_WITH_LEVEL(profile, "InvertedIndexLocalIOUseTimer", cache_profile, 1); inverted_index_remote_io_timer = @@ -203,8 +213,14 @@ FileCacheProfileReporter::FileCacheProfileReporter(RuntimeProfile* profile, ADD_CHILD_TIMER_WITH_LEVEL(profile, "InvertedIndexIOTimer", cache_profile, 1); inverted_index_write_cache_io_timer = ADD_CHILD_TIMER_WITH_LEVEL( profile, "InvertedIndexWriteCacheIOUseTimer", cache_profile, 1); - inverted_index_bytes_write_into_cache = ADD_CHILD_COUNTER_WITH_LEVEL( - profile, "InvertedIndexBytesWriteIntoCache", TUnit::BYTES, cache_profile, 1); + inverted_index_request_bytes = ADD_CHILD_COUNTER_WITH_LEVEL( + profile, "InvertedIndexRequestBytes", TUnit::BYTES, cache_profile, 1); + inverted_index_read_bytes = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "InvertedIndexReadBytes", + TUnit::BYTES, cache_profile, 1); + inverted_index_range_read_count = ADD_CHILD_COUNTER_WITH_LEVEL( + profile, "InvertedIndexRangeReadCount", TUnit::UNIT, cache_profile, 1); + inverted_index_serial_read_rounds = ADD_CHILD_COUNTER_WITH_LEVEL( + profile, "InvertedIndexSerialReadRounds", TUnit::UNIT, cache_profile, 1); segment_footer_index_num_local_io_total = ADD_CHILD_COUNTER_WITH_LEVEL( profile, "SegmentFooterIndexNumLocalIOTotal", TUnit::UNIT, cache_profile, 1); @@ -290,14 +306,16 @@ void FileCacheProfileReporter::update(const FileCacheStatistics* statistics) con statistics->inverted_index_bytes_read_from_remote); COUNTER_UPDATE(inverted_index_bytes_scanned_from_peer, statistics->inverted_index_bytes_read_from_peer); + COUNTER_UPDATE(inverted_index_remote_physical_read_bytes, + statistics->inverted_index_remote_physical_read_bytes); + COUNTER_UPDATE(inverted_index_bytes_write_into_cache, + statistics->inverted_index_bytes_write_into_cache); COUNTER_UPDATE(inverted_index_local_io_timer, statistics->inverted_index_local_io_timer); COUNTER_UPDATE(inverted_index_remote_io_timer, statistics->inverted_index_remote_io_timer); COUNTER_UPDATE(inverted_index_peer_io_timer, statistics->inverted_index_peer_io_timer); COUNTER_UPDATE(inverted_index_io_timer, statistics->inverted_index_io_timer); COUNTER_UPDATE(inverted_index_write_cache_io_timer, statistics->inverted_index_write_cache_io_timer); - COUNTER_UPDATE(inverted_index_bytes_write_into_cache, - statistics->inverted_index_bytes_write_into_cache); COUNTER_UPDATE(segment_footer_index_num_local_io_total, statistics->segment_footer_index_num_local_io_total); @@ -343,6 +361,11 @@ void FileCacheProfileReporter::update(const FileCacheStatistics* statistics) con } _profile->add_info_string("PeerCacheNodes", peer_nodes); } + COUNTER_UPDATE(inverted_index_request_bytes, statistics->inverted_index_request_bytes); + COUNTER_UPDATE(inverted_index_read_bytes, statistics->inverted_index_read_bytes); + COUNTER_UPDATE(inverted_index_range_read_count, statistics->inverted_index_range_read_count); + COUNTER_UPDATE(inverted_index_serial_read_rounds, + statistics->inverted_index_serial_read_rounds); } } // namespace doris::io diff --git a/be/src/io/cache/block_file_cache_profile.h b/be/src/io/cache/block_file_cache_profile.h index d3fd31033649c8..5e7a9e2a267f1a 100644 --- a/be/src/io/cache/block_file_cache_profile.h +++ b/be/src/io/cache/block_file_cache_profile.h @@ -19,6 +19,7 @@ #include +#include #include #include #include @@ -60,7 +61,6 @@ class FileCacheMetrics { void register_entity(); void update_metrics_callback(); -private: std::mutex _mtx; // use shared_ptr for concurrent std::shared_ptr _statistics; @@ -99,12 +99,17 @@ struct FileCacheProfileReporter { RuntimeProfile::Counter* inverted_index_bytes_scanned_from_cache = nullptr; RuntimeProfile::Counter* inverted_index_bytes_scanned_from_remote = nullptr; RuntimeProfile::Counter* inverted_index_bytes_scanned_from_peer = nullptr; + RuntimeProfile::Counter* inverted_index_remote_physical_read_bytes = nullptr; + RuntimeProfile::Counter* inverted_index_bytes_write_into_cache = nullptr; RuntimeProfile::Counter* inverted_index_local_io_timer = nullptr; RuntimeProfile::Counter* inverted_index_remote_io_timer = nullptr; RuntimeProfile::Counter* inverted_index_peer_io_timer = nullptr; RuntimeProfile::Counter* inverted_index_io_timer = nullptr; RuntimeProfile::Counter* inverted_index_write_cache_io_timer = nullptr; - RuntimeProfile::Counter* inverted_index_bytes_write_into_cache = nullptr; + RuntimeProfile::Counter* inverted_index_request_bytes = nullptr; + RuntimeProfile::Counter* inverted_index_read_bytes = nullptr; + RuntimeProfile::Counter* inverted_index_range_read_count = nullptr; + RuntimeProfile::Counter* inverted_index_serial_read_rounds = nullptr; RuntimeProfile::Counter* segment_footer_index_num_local_io_total = nullptr; RuntimeProfile::Counter* segment_footer_index_num_remote_io_total = nullptr; diff --git a/be/src/io/cache/cached_remote_file_reader.cpp b/be/src/io/cache/cached_remote_file_reader.cpp index 47fe02ee2c3e92..dcd8dde92946b3 100644 --- a/be/src/io/cache/cached_remote_file_reader.cpp +++ b/be/src/io/cache/cached_remote_file_reader.cpp @@ -325,7 +325,11 @@ Status execute_s3_read(size_t empty_start, size_t& size, std::unique_ptr s3_read_counter << 1; SCOPED_RAW_TIMER(&stats.remote_read_timer); stats.from_peer_cache = false; - return remote_file_reader->read_at(empty_start, Slice(buffer.get(), size), &size, io_ctx); + auto st = remote_file_reader->read_at(empty_start, Slice(buffer.get(), size), &size, io_ctx); + if (st.ok()) { + stats.remote_physical_read_bytes += size; + } + return st; } CloudWarmUpManager& get_warm_up_manager() { @@ -343,6 +347,10 @@ struct RaceState { Status peer_status; Status s3_status; std::unique_ptr s3_buf; + // Actual bytes fetched by the winning S3 leg; merged into the caller's + // ReadStatistics only in the winner==1 branch of collect_race_result (the + // losing S3 leg may outlive the caller's stack, so it must not touch stats). + size_t s3_read_size = 0; PeerFetchResult peer_res; std::string peer_winner_cg_id; // compute_group_id of the winning peer candidate std::string peer_winner_host; // host of the winning peer candidate @@ -466,6 +474,7 @@ void launch_s3_race(std::shared_ptr race, size_t empty_start, size_t if (st.ok() && race->winner < 0) { race->winner = 1; race->s3_buf = std::move(s3_buf); + race->s3_read_size = read_size; } race->cv.notify_all(); }; @@ -548,9 +557,11 @@ Status collect_race_result(std::shared_ptr race, size_t span_size, } return Status::OK(); } else if (race->winner == 1) { - // S3 won. + // S3 won: this was a real storage GET, so account it as physical remote IO + // exactly like the non-race download path does. buffer = std::move(race->s3_buf); stats.from_peer_cache = false; + stats.remote_physical_read_bytes += race->s3_read_size; g_peer_race_s3_win << 1; if (io_ctx != nullptr && io_ctx->file_cache_stats != nullptr) { io_ctx->file_cache_stats->num_peer_race_s3_win++; @@ -1038,6 +1049,8 @@ Status CachedRemoteFileReader::_read_remaining_blocks_from_cache( &remote_bytes_read, io_ctx)); indirect_read_bytes += read_size; source_read_breakdown.remote_bytes += remote_bytes_read; + // Self-heal fell back to a real storage GET; count it as physical remote IO. + stats.remote_physical_read_bytes += remote_bytes_read; DCHECK(remote_bytes_read == read_size); } @@ -1112,6 +1125,10 @@ Status CachedRemoteFileReader::_read_remote_only_on_cache_miss( *bytes_read = remote_bytes_read; DCHECK_EQ(*bytes_read, bytes_req); source_read_breakdown.remote_bytes += remote_bytes_read; + // This is a real storage GET, so it must count as physical remote IO just like + // the block-download path; otherwise profiles show scanned-from-remote bytes + // with zero physical reads whenever remote-only-on-miss is active. + stats.remote_physical_read_bytes += remote_bytes_read; g_read_cache_indirect_bytes << remote_bytes_read; g_read_cache_indirect_total_bytes << remote_bytes_read; return Status::OK(); @@ -1413,6 +1430,7 @@ void CachedRemoteFileReader::_update_stats(const ReadStatistics& read_stats, statis->inverted_index_remote_io_timer, statis->inverted_index_peer_io_timer, statis->inverted_index_write_cache_io_timer, statis->inverted_index_bytes_write_into_cache); + statis->inverted_index_remote_physical_read_bytes += read_stats.remote_physical_read_bytes; break; case FileCacheReadType::SEGMENT_FOOTER_INDEX: update_index_stats(statis->segment_footer_index_num_local_io_total, diff --git a/be/src/io/cache/file_cache_common.h b/be/src/io/cache/file_cache_common.h index 8b52af8d161b9a..7357f1a431acc2 100644 --- a/be/src/io/cache/file_cache_common.h +++ b/be/src/io/cache/file_cache_common.h @@ -75,6 +75,7 @@ struct ReadStatistics { int64_t bytes_read_from_local = 0; int64_t bytes_read_from_remote = 0; int64_t bytes_read_from_peer = 0; + int64_t remote_physical_read_bytes = 0; int64_t bytes_write_into_file_cache = 0; int64_t remote_read_timer = 0; int64_t peer_read_timer = 0; diff --git a/be/src/io/io_common.h b/be/src/io/io_common.h index c085f93347850f..83799485b2935f 100644 --- a/be/src/io/io_common.h +++ b/be/src/io/io_common.h @@ -19,6 +19,9 @@ #include +#include +#include +#include #include #include @@ -78,12 +81,17 @@ struct FileCacheStatistics { int64_t inverted_index_bytes_read_from_local = 0; int64_t inverted_index_bytes_read_from_remote = 0; int64_t inverted_index_bytes_read_from_peer = 0; + int64_t inverted_index_remote_physical_read_bytes = 0; + int64_t inverted_index_bytes_write_into_cache = 0; int64_t inverted_index_local_io_timer = 0; int64_t inverted_index_remote_io_timer = 0; int64_t inverted_index_peer_io_timer = 0; int64_t inverted_index_io_timer = 0; int64_t inverted_index_write_cache_io_timer = 0; - int64_t inverted_index_bytes_write_into_cache = 0; + int64_t inverted_index_request_bytes = 0; + int64_t inverted_index_read_bytes = 0; + int64_t inverted_index_range_read_count = 0; + int64_t inverted_index_serial_read_rounds = 0; int64_t segment_footer_index_num_local_io_total = 0; int64_t segment_footer_index_num_remote_io_total = 0; @@ -139,12 +147,18 @@ struct FileCacheStatistics { inverted_index_bytes_read_from_local += other.inverted_index_bytes_read_from_local; inverted_index_bytes_read_from_remote += other.inverted_index_bytes_read_from_remote; inverted_index_bytes_read_from_peer += other.inverted_index_bytes_read_from_peer; + inverted_index_remote_physical_read_bytes += + other.inverted_index_remote_physical_read_bytes; inverted_index_local_io_timer += other.inverted_index_local_io_timer; inverted_index_remote_io_timer += other.inverted_index_remote_io_timer; inverted_index_peer_io_timer += other.inverted_index_peer_io_timer; inverted_index_io_timer += other.inverted_index_io_timer; inverted_index_write_cache_io_timer += other.inverted_index_write_cache_io_timer; inverted_index_bytes_write_into_cache += other.inverted_index_bytes_write_into_cache; + inverted_index_request_bytes += other.inverted_index_request_bytes; + inverted_index_read_bytes += other.inverted_index_read_bytes; + inverted_index_range_read_count += other.inverted_index_range_read_count; + inverted_index_serial_read_rounds += other.inverted_index_serial_read_rounds; segment_footer_index_num_local_io_total += other.segment_footer_index_num_local_io_total; segment_footer_index_num_remote_io_total += other.segment_footer_index_num_remote_io_total; diff --git a/be/src/runtime/index_policy/index_policy_mgr.cpp b/be/src/runtime/index_policy/index_policy_mgr.cpp index 5ab41a409ba921..e7ec8dfd2b9d0e 100644 --- a/be/src/runtime/index_policy/index_policy_mgr.cpp +++ b/be/src/runtime/index_policy/index_policy_mgr.cpp @@ -17,6 +17,7 @@ #include "runtime/index_policy/index_policy_mgr.h" +#include #include #include #include @@ -24,6 +25,24 @@ #include namespace doris { +namespace { + +class PurposeInsensitiveAnalyzerProvider final + : public segment_v2::inverted_index::AnalyzerProvider { +public: + explicit PurposeInsensitiveAnalyzerProvider(AnalyzerPtr analyzer) + : _analyzer(std::move(analyzer)) {} + + AnalyzerPtr get_analyzer( + segment_v2::inverted_index::AnalysisPurpose /*purpose*/) const override { + return _analyzer; + } + +private: + const AnalyzerPtr _analyzer; +}; + +} // namespace const std::unordered_set IndexPolicyMgr::BUILTIN_NORMALIZERS = {"lowercase"}; @@ -49,11 +68,9 @@ void IndexPolicyMgr::apply_policy_changes(const std::vector& polic LOG(INFO) << "Deleting policy - " << "ID: " << id << ", " << "Name: " << it->second.name; - - // Use normalized name for deletion _name_to_id.erase(normalize_name(it->second.name)); _policys.erase(it); - success_deletes++; + ++success_deletes; } else { LOG(WARNING) << "Delete failed - Policy ID not found: " << id; } @@ -66,8 +83,6 @@ void IndexPolicyMgr::apply_policy_changes(const std::vector& polic << " | New name: " << policy.name; continue; } - - // Use normalized name for case-insensitive lookup std::string normalized_name = normalize_name(policy.name); if (_name_to_id.contains(normalized_name)) { LOG(ERROR) << "Reject update - Duplicate policy name: " << policy.name @@ -77,10 +92,8 @@ void IndexPolicyMgr::apply_policy_changes(const std::vector& polic } _policys.emplace(policy.id, policy); - // Store with normalized key for case-insensitive lookup _name_to_id.emplace(normalized_name, policy.id); - success_updates++; - + ++success_updates; LOG(INFO) << "Successfully applied policy - " << "ID: " << policy.id << ", " << "Name: " << policy.name << ", " @@ -131,7 +144,79 @@ AnalyzerPtr IndexPolicyMgr::get_policy_by_name(const std::string& name) { throw Exception(ErrorCode::INVALID_ARGUMENT, "Policy not found with type: " + name); } -AnalyzerPtr IndexPolicyMgr::build_analyzer_from_policy(const TIndexPolicy& index_policy_analyzer) { +AnalyzerPtr IndexPolicyMgr::get_analyzer_by_name( + const std::string& name, segment_v2::inverted_index::AnalysisPurpose purpose) { + std::shared_lock lock(_mutex); + const std::string normalized_name = normalize_name(name); + auto name_it = _name_to_id.find(normalized_name); + if (name_it == _name_to_id.end()) { + if (is_builtin_normalizer(normalized_name)) { + return build_builtin_normalizer(name); + } + throw Exception(ErrorCode::INVALID_ARGUMENT, "Policy not found with name: " + name); + } + auto policy_it = _policys.find(name_it->second); + if (policy_it == _policys.end()) { + throw Exception(ErrorCode::INVALID_ARGUMENT, "Policy not found with id: " + name); + } + if (policy_it->second.type == TIndexPolicyType::ANALYZER) { + return build_analyzer_provider_from_config( + build_analyzer_config_from_policy(policy_it->second), {}) + ->get_analyzer(purpose); + } + if (policy_it->second.type == TIndexPolicyType::NORMALIZER) { + return build_normalizer_from_policy(policy_it->second); + } + throw Exception(ErrorCode::INVALID_ARGUMENT, "Analyzer policy not found: " + name); +} + +AnalyzerProviderPtr IndexPolicyMgr::get_analyzer_provider_by_name( + const std::string& name, const std::map& outer_char_filter_map) { + std::shared_lock lock(_mutex); + const std::string normalized_name = normalize_name(name); + auto name_it = _name_to_id.find(normalized_name); + if (name_it == _name_to_id.end()) { + if (is_builtin_normalizer(normalized_name)) { + return std::make_shared( + build_builtin_normalizer(name)); + } + throw Exception(ErrorCode::INVALID_ARGUMENT, "Policy not found with name: " + name); + } + auto policy_it = _policys.find(name_it->second); + if (policy_it == _policys.end()) { + throw Exception(ErrorCode::INVALID_ARGUMENT, "Policy not found with id: " + name); + } + if (policy_it->second.type == TIndexPolicyType::ANALYZER) { + return build_analyzer_provider_from_config( + build_analyzer_config_from_policy(policy_it->second), outer_char_filter_map); + } + if (policy_it->second.type == TIndexPolicyType::NORMALIZER) { + return std::make_shared( + build_normalizer_from_policy(policy_it->second)); + } + throw Exception(ErrorCode::INVALID_ARGUMENT, "Analyzer policy not found: " + name); +} + +AnalyzerProviderPtr IndexPolicyMgr::get_analyzer_provider_by_base_fingerprint( + std::string_view base_analyzer_fingerprint, + const std::map& outer_char_filter_map) { + std::shared_lock lock(_mutex); + for (const auto& [_, policy] : _policys) { + if (policy.type != TIndexPolicyType::ANALYZER) { + continue; + } + auto config = build_analyzer_config_from_policy(policy); + if (segment_v2::inverted_index::CustomAnalyzerProvider::calculate_base_analyzer_fingerprint( + config, outer_char_filter_map) != base_analyzer_fingerprint) { + continue; + } + return build_analyzer_provider_from_config(std::move(config), outer_char_filter_map); + } + return nullptr; +} + +segment_v2::inverted_index::CustomAnalyzerConfigPtr +IndexPolicyMgr::build_analyzer_config_from_policy(const TIndexPolicy& index_policy_analyzer) { segment_v2::inverted_index::CustomAnalyzerConfig::Builder builder; auto tokenizer_it = index_policy_analyzer.properties.find(PROP_TOKENIZER); @@ -175,9 +260,23 @@ AnalyzerPtr IndexPolicyMgr::build_analyzer_from_policy(const TIndexPolicy& index builder.add_token_filter_config(name, settings); }); - auto custom_analyzer_config = builder.build(); - return segment_v2::inverted_index::CustomAnalyzer::build_custom_analyzer( - custom_analyzer_config); + return builder.build(); +} + +AnalyzerProviderPtr IndexPolicyMgr::build_analyzer_provider_from_config( + segment_v2::inverted_index::CustomAnalyzerConfigPtr config, + const std::map& outer_char_filter_map) { + // One shape for every policy: the provider sources its CommonGrams word list from the + // BE-local default, so there is no per-policy word set to look up and no "not yet prepared" + // state to represent. + return std::make_shared( + std::move(config), outer_char_filter_map); +} + +AnalyzerPtr IndexPolicyMgr::build_analyzer_from_policy(const TIndexPolicy& index_policy_analyzer) { + return build_analyzer_provider_from_config( + build_analyzer_config_from_policy(index_policy_analyzer), {}) + ->get_analyzer(segment_v2::inverted_index::AnalysisPurpose::kIndex); } AnalyzerPtr IndexPolicyMgr::build_normalizer_from_policy( @@ -224,7 +323,8 @@ void IndexPolicyMgr::process_filter_configs( std::string normalized_filter_name = normalize_name(filter_name); if (_name_to_id.contains(normalized_filter_name)) { // Nested filter policy - const auto& filter_policy = _policys[_name_to_id[normalized_filter_name]]; + const int64_t filter_policy_id = _name_to_id.at(normalized_filter_name); + const auto& filter_policy = _policys.at(filter_policy_id); auto type_it = filter_policy.properties.find(PROP_TYPE); if (type_it == filter_policy.properties.end()) { throw Exception( diff --git a/be/src/runtime/index_policy/index_policy_mgr.h b/be/src/runtime/index_policy/index_policy_mgr.h index edbdee938b86ad..0fb1dc3b3a87c1 100644 --- a/be/src/runtime/index_policy/index_policy_mgr.h +++ b/be/src/runtime/index_policy/index_policy_mgr.h @@ -19,7 +19,13 @@ #include +#include +#include +#include #include +#include +#include +#include #include #include "storage/index/inverted/analyzer/custom_analyzer.h" @@ -29,6 +35,7 @@ namespace doris { using Policys = std::unordered_map; using AnalyzerPtr = std::shared_ptr; +using AnalyzerProviderPtr = segment_v2::inverted_index::AnalyzerProviderPtr; class IndexPolicyMgr { public: @@ -40,8 +47,21 @@ class IndexPolicyMgr { Policys get_index_policys(); AnalyzerPtr get_policy_by_name(const std::string& name); + AnalyzerPtr get_analyzer_by_name(const std::string& name, + segment_v2::inverted_index::AnalysisPurpose purpose); + AnalyzerProviderPtr get_analyzer_provider_by_name( + const std::string& name, + const std::map& outer_char_filter_map = {}); + AnalyzerProviderPtr get_analyzer_provider_by_base_fingerprint( + std::string_view base_analyzer_fingerprint, + const std::map& outer_char_filter_map = {}); private: + segment_v2::inverted_index::CustomAnalyzerConfigPtr build_analyzer_config_from_policy( + const TIndexPolicy& index_policy_analyzer); + AnalyzerProviderPtr build_analyzer_provider_from_config( + segment_v2::inverted_index::CustomAnalyzerConfigPtr config, + const std::map& outer_char_filter_map); AnalyzerPtr build_analyzer_from_policy(const TIndexPolicy& index_policy_analyzer); AnalyzerPtr build_normalizer_from_policy(const TIndexPolicy& index_policy_normalizer); diff --git a/be/src/storage/compaction/collection_statistics.cpp b/be/src/storage/compaction/collection_statistics.cpp deleted file mode 100644 index 9afbe5f0944d43..00000000000000 --- a/be/src/storage/compaction/collection_statistics.cpp +++ /dev/null @@ -1,290 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "storage/compaction/collection_statistics.h" - -#include -#include - -#include "common/exception.h" -#include "exprs/vexpr.h" -#include "exprs/vexpr_context.h" -#include "exprs/vliteral.h" -#include "exprs/vslot_ref.h" -#include "storage/index/index_file_reader.h" -#include "storage/index/index_reader_helper.h" -#include "storage/index/inverted/analyzer/analyzer.h" -#include "storage/index/inverted/util/string_helper.h" -#include "storage/index/inverted/util/term_iterator.h" -#include "storage/rowset/rowset.h" -#include "storage/rowset/rowset_reader.h" -#include "util/uid_util.h" - -namespace doris { - -Status CollectionStatistics::collect(RuntimeState* state, - const std::vector& rs_splits, - const TabletSchemaSPtr& tablet_schema, - const VExprContextSPtrs& common_expr_ctxs_push_down, - io::IOContext* io_ctx) { - std::unordered_map collect_infos; - RETURN_IF_ERROR( - extract_collect_info(state, common_expr_ctxs_push_down, tablet_schema, &collect_infos)); - if (collect_infos.empty()) { - LOG(WARNING) << "Index statistics collection: no collect info extracted."; - return Status::OK(); - } - - for (const auto& rs_split : rs_splits) { - const auto& rs_reader = rs_split.rs_reader; - auto rowset = rs_reader->rowset(); - auto num_segments = rowset->num_segments(); - for (int32_t seg_id = 0; seg_id < num_segments; ++seg_id) { - auto status = - process_segment(rowset, seg_id, tablet_schema.get(), collect_infos, io_ctx); - if (!status.ok()) { - if (status.code() == ErrorCode::INVERTED_INDEX_FILE_NOT_FOUND || - status.code() == ErrorCode::INVERTED_INDEX_BYPASS) { - LOG(ERROR) << "Index statistics collection failed: " << status.to_string(); - } else { - return status; - } - } - } - } - - // Build a single-line log with query_id, tablet_ids, and per-field term statistics - if (VLOG_IS_ON(1)) { - std::set tablet_ids; - for (const auto& rs_split : rs_splits) { - if (rs_split.rs_reader && rs_split.rs_reader->rowset()) { - tablet_ids.insert(rs_split.rs_reader->rowset()->rowset_meta()->tablet_id()); - } - } - - std::ostringstream oss; - oss << "CollectionStatistics: query_id=" << print_id(state->query_id()); - - oss << ", tablet_ids=["; - bool first_tablet = true; - for (int64_t tid : tablet_ids) { - if (!first_tablet) oss << ","; - oss << tid; - first_tablet = false; - } - oss << "]"; - - oss << ", total_num_docs=" << _total_num_docs; - - for (const auto& [ws_field_name, num_tokens] : _total_num_tokens) { - oss << ", {field=" << StringHelper::to_string(ws_field_name) - << ", num_tokens=" << num_tokens << ", terms=["; - - bool first_term = true; - for (const auto& [term, doc_freq] : _term_doc_freqs.at(ws_field_name)) { - if (!first_term) oss << ", "; - oss << "(" << StringHelper::to_string(term) << ":" << doc_freq << ")"; - first_term = false; - } - oss << "]}"; - } - - VLOG(1) << oss.str(); - } - - return Status::OK(); -} - -Status CollectionStatistics::extract_collect_info( - RuntimeState* state, const VExprContextSPtrs& common_expr_ctxs_push_down, - const TabletSchemaSPtr& tablet_schema, CollectInfoMap* collect_infos) { - DCHECK(collect_infos != nullptr); - - std::unordered_map collectors; - collectors[TExprNodeType::MATCH_PRED] = std::make_unique(); - collectors[TExprNodeType::SEARCH_EXPR] = std::make_unique(); - - for (const auto& root_expr_ctx : common_expr_ctxs_push_down) { - const auto& root_expr = root_expr_ctx->root(); - if (root_expr == nullptr) { - continue; - } - - std::stack stack; - stack.emplace(root_expr); - - while (!stack.empty()) { - auto expr = stack.top(); - stack.pop(); - - if (!expr) { - continue; - } - - auto collector_it = collectors.find(expr->node_type()); - if (collector_it != collectors.end()) { - RETURN_IF_ERROR( - collector_it->second->collect(state, tablet_schema, expr, collect_infos)); - } - - const auto& children = expr->children(); - for (const auto& child : children) { - stack.push(child); - } - } - } - - LOG(INFO) << "Extracted collect info for " << collect_infos->size() << " fields"; - - return Status::OK(); -} - -Status CollectionStatistics::process_segment(const RowsetSharedPtr& rowset, int32_t seg_id, - const TabletSchema* tablet_schema, - const CollectInfoMap& collect_infos, - io::IOContext* io_ctx) { - auto seg_path = DORIS_TRY(rowset->segment_path(seg_id)); - auto rowset_meta = rowset->rowset_meta(); - - auto idx_file_reader = std::make_unique( - rowset_meta->fs(), - std::string {InvertedIndexDescriptor::get_index_file_path_prefix(seg_path)}, - tablet_schema->get_inverted_index_storage_format(), - rowset_meta->inverted_index_file_info(seg_id), rowset_meta->tablet_id()); - RETURN_IF_ERROR(idx_file_reader->init(config::inverted_index_read_buffer_size, io_ctx)); - - int32_t total_seg_num_docs = 0; - - for (const auto& [ws_field_name, collect_info] : collect_infos) { - lucene::search::IndexSearcher* index_searcher = nullptr; - lucene::index::IndexReader* index_reader = nullptr; - -#ifdef BE_TEST - auto compound_reader = DORIS_TRY(idx_file_reader->open(collect_info.index_meta, io_ctx)); - auto* reader = lucene::index::IndexReader::open(compound_reader.get()); - auto searcher_ptr = std::make_shared(reader, true); - index_searcher = searcher_ptr.get(); - index_reader = index_searcher->getReader(); -#else - InvertedIndexCacheHandle inverted_index_cache_handle; - auto index_file_key = idx_file_reader->get_index_file_cache_key(collect_info.index_meta); - InvertedIndexSearcherCache::CacheKey searcher_cache_key(index_file_key); - - if (!InvertedIndexSearcherCache::instance()->lookup(searcher_cache_key, - &inverted_index_cache_handle)) { - auto compound_reader = - DORIS_TRY(idx_file_reader->open(collect_info.index_meta, io_ctx)); - auto* reader = lucene::index::IndexReader::open(compound_reader.get()); - size_t reader_size = reader->getTermInfosRAMUsed(); - auto searcher_ptr = std::make_shared(reader, true); - auto* cache_value = new InvertedIndexSearcherCache::CacheValue( - std::move(searcher_ptr), reader_size, UnixMillis()); - InvertedIndexSearcherCache::instance()->insert(searcher_cache_key, cache_value, - &inverted_index_cache_handle); - } - - auto searcher_variant = inverted_index_cache_handle.get_index_searcher(); - auto index_searcher_ptr = std::get(searcher_variant); - index_searcher = index_searcher_ptr.get(); - index_reader = index_searcher->getReader(); -#endif - total_seg_num_docs = std::max(total_seg_num_docs, index_reader->maxDoc()); - - _total_num_tokens[ws_field_name] += - index_reader->sumTotalTermFreq(ws_field_name.c_str()).value_or(0); - - for (const auto& term_info : collect_info.term_infos) { - auto iter = TermIterator::create(io_ctx, false, index_reader, ws_field_name, - term_info.get_single_term()); - _term_doc_freqs[ws_field_name][iter->term()] += iter->doc_freq(); - } - } - - _total_num_docs += total_seg_num_docs; - - return Status::OK(); -} - -uint64_t CollectionStatistics::get_term_doc_freq_by_col(const std::wstring& lucene_col_name, - const std::wstring& term) { - if (!_term_doc_freqs.contains(lucene_col_name)) { - throw Exception(ErrorCode::INVERTED_INDEX_CLUCENE_ERROR, - "Index statistics collection failed: Not such column {}", - StringHelper::to_string(lucene_col_name)); - } - - if (!_term_doc_freqs[lucene_col_name].contains(term)) { - throw Exception(ErrorCode::INVERTED_INDEX_CLUCENE_ERROR, - "Index statistics collection failed: Not such term {}", - StringHelper::to_string(term)); - } - - return _term_doc_freqs[lucene_col_name][term]; -} - -uint64_t CollectionStatistics::get_total_term_cnt_by_col(const std::wstring& lucene_col_name) { - if (!_total_num_tokens.contains(lucene_col_name)) { - throw Exception(ErrorCode::INVERTED_INDEX_CLUCENE_ERROR, - "Index statistics collection failed: Not such column {}", - StringHelper::to_string(lucene_col_name)); - } - - return _total_num_tokens[lucene_col_name]; -} - -uint64_t CollectionStatistics::get_doc_num() const { - if (_total_num_docs == 0) { - throw Exception( - ErrorCode::INVERTED_INDEX_CLUCENE_ERROR, - "Index statistics collection failed: No data available for SimilarityCollector"); - } - - return _total_num_docs; -} - -float CollectionStatistics::get_or_calculate_avg_dl(const std::wstring& lucene_col_name) { - auto iter = _avg_dl_by_col.find(lucene_col_name); - if (iter != _avg_dl_by_col.end()) { - return iter->second; - } - - const uint64_t total_term_cnt = get_total_term_cnt_by_col(lucene_col_name); - const uint64_t total_doc_cnt = get_doc_num(); - float avg_dl = total_doc_cnt > 0 ? float((double)total_term_cnt / (double)total_doc_cnt) : 0.0F; - _avg_dl_by_col[lucene_col_name] = avg_dl; - return avg_dl; -} - -float CollectionStatistics::get_or_calculate_idf(const std::wstring& lucene_col_name, - const std::wstring& term) { - auto iter = _idf_by_col_term.find(lucene_col_name); - if (iter != _idf_by_col_term.end()) { - auto term_iter = iter->second.find(term); - if (term_iter != iter->second.end()) { - return term_iter->second; - } - } - - const uint64_t doc_num = get_doc_num(); - const uint64_t doc_freq = get_term_doc_freq_by_col(lucene_col_name, term); - auto idf = (float)std::log(1 + ((double)doc_num - (double)doc_freq + (double)0.5) / - ((double)doc_freq + (double)0.5)); - _idf_by_col_term[lucene_col_name][term] = idf; - return idf; -} - -} // namespace doris \ No newline at end of file diff --git a/be/src/storage/compaction/compaction.cpp b/be/src/storage/compaction/compaction.cpp index 46dc32d2bcd31f..cf5d4d74173d7d 100644 --- a/be/src/storage/compaction/compaction.cpp +++ b/be/src/storage/compaction/compaction.cpp @@ -22,9 +22,12 @@ #include #include +#include #include #include #include +#include +#include #include #include #include @@ -41,6 +44,7 @@ #include "cloud/cloud_tablet.h" #include "cloud/config.h" #include "cloud/pb_convert.h" +#include "common/check.h" #include "common/config.h" #include "common/metrics/doris_metrics.h" #include "common/status.h" @@ -53,7 +57,6 @@ #include "io/io_common.h" #include "runtime/memory/mem_tracker_limiter.h" #include "runtime/thread_context.h" -#include "storage/compaction/collection_statistics.h" #include "storage/compaction/cumulative_compaction.h" #include "storage/compaction/cumulative_compaction_binlog_policy.h" #include "storage/compaction/cumulative_compaction_policy.h" @@ -65,6 +68,10 @@ #include "storage/index/inverted/inverted_index_compaction.h" #include "storage/index/inverted/inverted_index_desc.h" #include "storage/index/inverted/inverted_index_fs_directory.h" +#include "storage/index/inverted/similarity/collection_statistics.h" +#include "storage/index/snii/compaction/eligibility.h" +#include "storage/index/snii/compaction/snii_index_compaction.h" +#include "storage/index/snii/writer/memory_reporter.h" #include "storage/olap_common.h" #include "storage/olap_define.h" #include "storage/rowset/beta_rowset.h" @@ -84,6 +91,7 @@ #include "storage/txn/txn_manager.h" #include "storage/utils.h" #include "util/pretty_printer.h" +#include "util/stopwatch.hpp" #include "util/time.h" #include "util/trace.h" @@ -119,6 +127,8 @@ bool should_enable_compaction_cache_index_only(bool write_file_cache, ReaderType namespace { +constexpr size_t kSniiCompactionReadAheadBudgetBytes = 64ULL << 20; + bool is_rowset_tidy(std::string& pre_max_key, bool& pre_rs_key_bounds_truncated, const RowsetSharedPtr& rhs) { size_t min_tidy_size = config::ordered_data_compaction_min_segment_size; @@ -833,6 +843,7 @@ Status Compaction::do_inverted_index_compaction() { } OlapStopWatch inverted_watch; + ThreadCpuStopWatch inverted_cpu_watch(true); // translation vec // <> @@ -858,7 +869,8 @@ Status Compaction::do_inverted_index_compaction() { if (dest_segment_num <= 0) { LOG(INFO) << "skip doing index compaction due to no output segments" << ". tablet=" << _tablet->tablet_id() << ", input row number=" << _input_row_num - << ". elapsed time=" << inverted_watch.get_elapse_second() << "s."; + << ". elapsed time=" << inverted_watch.get_elapse_second() + << "s. thread cpu time=" << inverted_cpu_watch.elapsed_time() / 1e9 << "s."; return Status::OK(); } @@ -924,6 +936,7 @@ Status Compaction::do_inverted_index_compaction() { // src index dirs std::vector> index_file_readers(src_segment_num); + std::vector source_rowsets(src_segment_num, nullptr); for (const auto& m : src_seg_to_id_map) { const auto& [rowset_id, seg_id] = m.first; @@ -988,6 +1001,7 @@ Status Compaction::do_inverted_index_compaction() { _tablet->tablet_id(), rowset_id.to_string(), seg_id); } index_file_readers[m.second] = std::move(index_file_reader); + source_rowsets[m.second] = rowset; } // dest index files @@ -1017,6 +1031,15 @@ Status Compaction::do_inverted_index_compaction() { << ", destination index size=" << dest_segment_num << "."; Status status = Status::OK(); + std::shared_ptr snii_merge_memory_reporter; + std::unique_ptr validated_snii_rowid_conversion; + if (_cur_tablet_schema->get_inverted_index_storage_format() == + InvertedIndexStorageFormatPB::SNII) { + const size_t spill_threshold = + static_cast(config::inverted_index_ram_buffer_size * 1024 * 1024); + snii_merge_memory_reporter = std::make_shared( + nullptr, spill_threshold, snii::writer::MemoryReporter::CapPolicy::kHardLimit); + } for (auto&& column_uniq_id : ctx.columns_to_do_index_compaction) { auto col = _cur_tablet_schema->column_by_uid(column_uniq_id); auto index_metas = _cur_tablet_schema->inverted_indexs(col); @@ -1032,6 +1055,152 @@ Status Compaction::do_inverted_index_compaction() { break; } for (const auto& index_meta : index_metas) { + if (_cur_tablet_schema->get_inverted_index_storage_format() == + InvertedIndexStorageFormatPB::SNII) { + std::vector> source_indexes; + std::vector plan_sources; + std::vector eligibility_sources; + source_indexes.reserve(src_segment_num); + plan_sources.reserve(src_segment_num); + eligibility_sources.reserve(src_segment_num); + Status merge_status = Status::OK(); + for (size_t source_ordinal = 0; source_ordinal < src_segment_num; + ++source_ordinal) { + DORIS_CHECK(source_rowsets[source_ordinal] != nullptr); + const auto source_index_metas = + source_rowsets[source_ordinal]->tablet_schema()->inverted_indexs( + column_uniq_id); + const auto source_index_it = std::find_if( + source_index_metas.begin(), source_index_metas.end(), + [&index_meta](const TabletIndex* source_index) { + return source_index->index_id() == index_meta->index_id(); + }); + if (source_index_it == source_index_metas.end()) { + merge_status = Status::Error( + "source SNII index metadata disappeared after eligibility"); + break; + } + const TabletIndex* source_index_meta = *source_index_it; + auto source_index = index_file_readers[source_ordinal]->open_snii_index( + source_index_meta, nullptr, + snii::reader::LogicalIndexOpenMode::kCompaction); + if (!source_index.has_value()) { + merge_status = source_index.error(); + break; + } + source_indexes.push_back(std::move(source_index.value())); + plan_sources.push_back(source_indexes.back().get()); + eligibility_sources.push_back({.reader = std::cref(*source_indexes.back()), + .index_meta = std::cref(*source_index_meta)}); + } + + snii::compaction::SniiCompactionEligibility merge_eligibility; + if (merge_status.ok()) { + merge_status = snii::compaction::validate_snii_compaction_eligibility( + eligibility_sources, *index_meta, &merge_eligibility); + } + + if (merge_status.ok() && validated_snii_rowid_conversion == nullptr) { + std::vector source_segment_doc_counts; + source_segment_doc_counts.reserve(plan_sources.size()); + for (const snii::reader::LogicalIndexReader* source : plan_sources) { + DORIS_CHECK(source != nullptr); + if (source->stats().doc_count > std::numeric_limits::max()) { + merge_status = Status::Error( + "source doc count exceeds the SNII uint32 docid domain"); + break; + } + source_segment_doc_counts.push_back( + static_cast(source->stats().doc_count)); + } + if (merge_status.ok()) { + merge_status = snii::compaction::ValidatedRowIdConversion::create( + &trans_vec, source_segment_doc_counts, dest_segment_num_rows, + &validated_snii_rowid_conversion); + if (merge_status.ok()) { + DBUG_EXECUTE_IF("Compaction::snii_validated_rowid_conversion_created", + DBUG_RUN_CALLBACK()); + } + } + } + + std::unique_ptr merge_plan; + if (merge_status.ok()) { + DORIS_CHECK(validated_snii_rowid_conversion != nullptr); + merge_status = snii::compaction::SniiPlainT2MergePlan::prepare( + std::move(plan_sources), *validated_snii_rowid_conversion, + merge_eligibility, kSniiCompactionReadAheadBudgetBytes, + snii_merge_memory_reporter, &merge_plan); + } + + std::vector destination_sessions( + dest_segment_num, nullptr); + if (merge_status.ok()) { + DORIS_CHECK(snii_merge_memory_reporter != nullptr); + for (size_t destination_ordinal = 0; destination_ordinal < dest_segment_num; + ++destination_ordinal) { + DBUG_EXECUTE_IF("Compaction::before_add_snii_destination_session", + DBUG_RUN_CALLBACK(destination_ordinal, &merge_status)); + if (!merge_status.ok()) { + break; + } + auto* destination_writer = + inverted_index_file_writers[cast_set(destination_ordinal)] + .get(); + if (merge_eligibility.kind == + snii::compaction::SniiStreamedMergeKind::kCommonGramsT3) { + merge_status = destination_writer->add_snii_index_streamed( + index_meta, dest_segment_num_rows[destination_ordinal], + merge_plan->take_destination_null_docids(destination_ordinal), + merge_plan->take_destination_encoded_norms(destination_ordinal), + merge_plan->destination_common_grams_metadata( + destination_ordinal), + merge_plan->destination_common_grams_posting_policy(), + merge_plan->destination_index_config(), + snii_merge_memory_reporter, + &destination_sessions[destination_ordinal]); + } else { + merge_status = destination_writer->add_snii_index_streamed( + index_meta, dest_segment_num_rows[destination_ordinal], + merge_plan->take_destination_null_docids(destination_ordinal), + merge_plan->destination_index_config(), + snii_merge_memory_reporter, + &destination_sessions[destination_ordinal]); + } + if (!merge_status.ok()) { + break; + } + } + } + if (merge_status.ok()) { + DBUG_EXECUTE_IF("Compaction::before_execute_snii_merge", + DBUG_RUN_CALLBACK(&merge_status)); + } + if (!merge_status.ok()) { + for (size_t destination_ordinal = 0; + destination_ordinal < destination_sessions.size(); ++destination_ordinal) { + snii::writer::SniiStreamedIndexSession* session = + destination_sessions[destination_ordinal]; + if (session != nullptr) { + session->abort(merge_status); + DBUG_EXECUTE_IF("Compaction::snii_destination_session_aborted", + DBUG_RUN_CALLBACK(destination_ordinal)); + } + } + } else { + merge_status = merge_plan->execute(destination_sessions); + } + if (!merge_status.ok()) { + if (merge_status.is() || + merge_status.is() || + merge_status.is()) { + error_handler(index_meta->index_id(), column_uniq_id); + } + return merge_status; + } + continue; + } + std::vector dest_index_dirs(dest_segment_num); try { std::vector> src_idx_dirs( @@ -1095,7 +1264,8 @@ Status Compaction::do_inverted_index_compaction() { } LOG(INFO) << "succeed to do index compaction" << ". tablet=" << _tablet->tablet_id() - << ". elapsed time=" << inverted_watch.get_elapse_second() << "s."; + << ". elapsed time=" << inverted_watch.get_elapse_second() + << "s. thread cpu time=" << inverted_cpu_watch.elapsed_time() / 1e9 << "s."; return Status::OK(); } @@ -1233,6 +1403,157 @@ static bool check_rowset_has_inverted_index(const RowsetSharedPtr& src_rs, int32 } void Compaction::construct_index_compaction_columns(RowsetWriterContext& ctx) { + if (_cur_tablet_schema->get_inverted_index_storage_format() == + InvertedIndexStorageFormatPB::SNII) { + std::map column_eligibility; + std::map> source_file_readers; + for (const auto& destination_index : _cur_tablet_schema->inverted_indexes()) { + const auto& col_unique_ids = destination_index->col_unique_ids(); + if (col_unique_ids.empty()) { + LOG(WARNING) << "tablet[" << _tablet->tablet_id() << "] index[" + << destination_index->index_id() + << "] has no column unique id, will rebuild its SNII index"; + continue; + } + const int32_t col_unique_id = col_unique_ids[0]; + if (!_cur_tablet_schema->has_column_unique_id(col_unique_id) || + !field_is_slice_type(_cur_tablet_schema->column_by_uid(col_unique_id).type())) { + continue; + } + + bool eligible = true; + size_t source_segment_count = 0; + for (const auto& rowset : _input_rowsets) { + source_segment_count += rowset->num_segments(); + } + if (source_segment_count == 0 || + source_segment_count > kSniiCompactionReadAheadBudgetBytes / + snii::compaction::SniiPlainT2MergePlan:: + kMinReadAheadBudgetPerSource) { + eligible = false; + } + + Status eligibility_status = + eligible ? Status::OK() + : Status::Error( + "source index unavailable or read-ahead budget gate failed"); + std::optional merge_eligibility; + size_t source_ordinal = 0; + + for (const auto& rowset : _input_rowsets) { + if (!eligible) { + break; + } + auto* beta_rowset = static_cast(rowset.get()); + if (beta_rowset->is_skip_index_compaction(col_unique_id)) { + eligible = false; + break; + } + const auto source_index_metas = + rowset->tablet_schema()->inverted_indexs(col_unique_id); + const auto source_index_it = std::find_if( + source_index_metas.begin(), source_index_metas.end(), + [&destination_index](const TabletIndex* source_index) { + return source_index->index_id() == destination_index->index_id(); + }); + if (source_index_it == source_index_metas.end()) { + eligible = false; + break; + } + const TabletIndex* source_index_meta = *source_index_it; + const auto fs = rowset->rowset_meta()->fs(); + if (fs == nullptr) { + eligible = false; + break; + } + + for (uint32_t segment_id = 0; segment_id < rowset->num_segments(); ++segment_id) { + auto segment_path = rowset->segment_path(segment_id); + if (!segment_path.has_value()) { + eligible = false; + break; + } + const std::string index_file_path_prefix = + std::string {InvertedIndexDescriptor::get_index_file_path_prefix( + segment_path.value())}; + auto source_file_reader_it = source_file_readers.find(index_file_path_prefix); + if (source_file_reader_it == source_file_readers.end()) { + auto source_file_reader = std::make_unique( + fs, index_file_path_prefix, InvertedIndexStorageFormatPB::SNII, + rowset->rowset_meta()->inverted_index_file_info(segment_id), + _tablet->tablet_id()); + const Status init_status = + source_file_reader->init(config::inverted_index_read_buffer_size); + if (!init_status.ok()) { + eligible = false; + break; + } + DBUG_EXECUTE_IF("Compaction::snii_eligibility_reader_initialized", + DBUG_RUN_CALLBACK()); + source_file_reader_it = source_file_readers + .emplace(index_file_path_prefix, + std::move(source_file_reader)) + .first; + } + auto source_index = source_file_reader_it->second->open_snii_index( + source_index_meta, nullptr, + snii::reader::LogicalIndexOpenMode::kCompaction); + if (!source_index.has_value()) { + eligible = false; + break; + } + if (!merge_eligibility.has_value()) { + const std::array source = { + snii::compaction::PlainT2CompactionSource { + .reader = std::cref(*source_index.value()), + .index_meta = std::cref(*source_index_meta)}}; + snii::compaction::SniiCompactionEligibility eligibility; + eligibility_status = snii::compaction::validate_snii_compaction_eligibility( + source, *destination_index, &eligibility); + if (eligibility_status.ok()) { + merge_eligibility = std::move(eligibility); + } + } else if (source_index_meta->index_id() != destination_index->index_id() || + source_index_meta->get_index_suffix() != + destination_index->get_index_suffix() || + source_index_meta->properties() != destination_index->properties()) { + eligibility_status = Status::Error( + "source SNII index identity or properties differ from destination"); + } else { + eligibility_status = snii::compaction::validate_snii_source_eligibility( + *source_index.value(), source_ordinal, *merge_eligibility); + } + if (!eligibility_status.ok()) { + eligible = false; + break; + } + ++source_ordinal; + } + } + + if (!eligible && eligibility_status.ok()) { + eligibility_status = Status::Error( + "source SNII index file or metadata is unavailable"); + } + auto [column_it, inserted] = column_eligibility.emplace(col_unique_id, eligible); + if (!inserted) { + column_it->second = column_it->second && eligible; + } + if (!eligible) { + LOG(INFO) << "tablet[" << _tablet->tablet_id() << "] index[" + << destination_index->index_id() + << "] is not eligible for SNII postings compaction; rebuild from raw " + "column. reason=" + << eligibility_status; + } + } + for (const auto& [col_unique_id, eligible] : column_eligibility) { + if (eligible) { + ctx.columns_to_do_index_compaction.insert(col_unique_id); + } + } + return; + } for (const auto& index : _cur_tablet_schema->inverted_indexes()) { auto col_unique_ids = index->col_unique_ids(); // check if column unique ids is empty to avoid crash diff --git a/be/src/storage/index/analyzer_key_matcher.cpp b/be/src/storage/index/analyzer_key_matcher.cpp index ef0bdddf75ddb1..81373fa529327f 100644 --- a/be/src/storage/index/analyzer_key_matcher.cpp +++ b/be/src/storage/index/analyzer_key_matcher.cpp @@ -17,8 +17,6 @@ #include "storage/index/analyzer_key_matcher.h" -#include "storage/index/inverted/inverted_index_iterator.h" - namespace doris::segment_v2 { AnalyzerMatchResult AnalyzerKeyMatcher::match( diff --git a/be/src/storage/index/analyzer_key_matcher.h b/be/src/storage/index/analyzer_key_matcher.h index 833193b259d592..e683c2d4dbb480 100644 --- a/be/src/storage/index/analyzer_key_matcher.h +++ b/be/src/storage/index/analyzer_key_matcher.h @@ -23,11 +23,15 @@ #include #include "storage/index/inverted/inverted_index_parser.h" +#include "storage/index/inverted/inverted_index_reader.h" namespace doris::segment_v2 { -// Forward declaration -struct ReaderEntry; +struct ReaderEntry { + InvertedIndexReaderType type; + std::string analyzer_key; + InvertedIndexReaderPtr reader; +}; // Result of analyzer key matching operation. // Contains candidate readers that match the requested analyzer key. diff --git a/be/src/storage/index/index_file_reader.cpp b/be/src/storage/index/index_file_reader.cpp index 348e1399421e5a..95dbee73397107 100644 --- a/be/src/storage/index/index_file_reader.cpp +++ b/be/src/storage/index/index_file_reader.cpp @@ -20,6 +20,8 @@ #include #include +#include "common/cast_set.h" +#include "common/config.h" #include "storage/index/inverted/inverted_index_compound_reader.h" #include "storage/index/inverted/inverted_index_fs_directory.h" #include "storage/tablet/tablet_schema.h" @@ -31,7 +33,9 @@ Status IndexFileReader::init(int32_t read_buffer_size, const io::IOContext* io_c std::unique_lock lock(_mutex); // Lock for writing if (!_inited) { _read_buffer_size = read_buffer_size; - if (_storage_format >= InvertedIndexStorageFormatPB::V2) { + if (_storage_format == InvertedIndexStorageFormatPB::SNII) { + RETURN_IF_ERROR(_init_snii(io_ctx)); + } else if (_storage_format >= InvertedIndexStorageFormatPB::V2) { RETURN_IF_ERROR(_init_from(read_buffer_size, io_ctx)); } _inited = true; @@ -136,7 +140,51 @@ Status IndexFileReader::_init_from(int32_t read_buffer_size, const io::IOContext return Status::OK(); } +Status IndexFileReader::_init_snii(const io::IOContext* io_ctx) { + auto index_file_full_path = InvertedIndexDescriptor::get_index_file_path_v2(_index_path_prefix); + int64_t file_size = -1; + if (_idx_file_info.has_index_size()) { + file_size = _idx_file_info.index_size(); + } + file_size = file_size == 0 ? -1 : file_size; + + io::FileReaderOptions opts; + // DIAGNOSTIC: inverted_index_read_bypass_file_cache forces NO_CACHE (precise S3 + // range GETs) for the SNII index reader only, without touching the global + // enable_file_cache (cloud mode requires the latter). Default path keeps the + // block cache. + opts.cache_type = (config::enable_file_cache && !config::inverted_index_read_bypass_file_cache) + ? io::FileCachePolicy::FILE_BLOCK_CACHE + : io::FileCachePolicy::NO_CACHE; + opts.is_doris_table = true; + opts.file_size = file_size; + opts.tablet_id = _tablet_id; + io::FileReaderSPtr reader; + RETURN_IF_ERROR(_fs->open_file(index_file_full_path, &reader, &opts)); + // With NO_CACHE on a remote filesystem there is no CachedRemoteFileReader to + // account physical remote bytes, so the adapter must count its own reads. + const bool direct_remote_io = opts.cache_type == io::FileCachePolicy::NO_CACHE && + _fs->type() != io::FileSystemType::LOCAL; + _snii_file_reader = std::make_shared( + std::move(reader), /*io_ctx=*/nullptr, direct_remote_io); + _snii_segment_reader = std::make_unique(); + io::IOContext meta_io_ctx; + if (io_ctx != nullptr) { + meta_io_ctx = *io_ctx; + } + meta_io_ctx.is_inverted_index = true; + meta_io_ctx.is_index_data = true; + snii_doris::DorisSniiFileReader::ScopedIOContext io_context_scope(&meta_io_ctx); + RETURN_IF_ERROR(doris::snii::reader::SniiSegmentReader::open(_snii_file_reader.get(), + _snii_segment_reader.get())); + return Status::OK(); +} + Result IndexFileReader::get_all_directories() { + if (_storage_format == InvertedIndexStorageFormatPB::SNII) { + return ResultError(Status::Error( + "SNII format does not expose CLucene directories")); + } InvertedIndexDirectoryMap res; std::shared_lock lock(_mutex); // Lock for reading for (auto& [index, _] : _indices_entries) { @@ -155,6 +203,11 @@ Result> IndexFileReader:: int64_t index_id, const std::string& index_suffix, const io::IOContext* io_ctx) const { std::unique_ptr compound_reader; + if (_storage_format == InvertedIndexStorageFormatPB::SNII) { + return ResultError(Status::Error( + "SNII format does not open CLucene compound readers")); + } + if (_storage_format == InvertedIndexStorageFormatPB::V1) { auto index_file_path = InvertedIndexDescriptor::get_index_file_path_v1( _index_path_prefix, index_id, index_suffix); @@ -229,6 +282,35 @@ Result> IndexFileReader:: return compound_reader; } +Result> IndexFileReader::open_snii_index( + const TabletIndex* index_meta, const io::IOContext* io_ctx, + doris::snii::reader::LogicalIndexOpenMode open_mode) const { + DCHECK(_storage_format == InvertedIndexStorageFormatPB::SNII); + std::shared_lock lock(_mutex); + if (_snii_segment_reader == nullptr) { + return ResultError(Status::Error( + "SNII index file {} is not opened", + InvertedIndexDescriptor::get_index_file_path_v2(_index_path_prefix))); + } + io::IOContext meta_io_ctx; + if (io_ctx != nullptr) { + meta_io_ctx = *io_ctx; + } + meta_io_ctx.is_inverted_index = true; + meta_io_ctx.is_index_data = true; + snii_doris::DorisSniiFileReader::ScopedIOContext io_context_scope(&meta_io_ctx); + + auto logical_reader = std::make_unique(); + auto status = _snii_segment_reader->open_index(cast_set(index_meta->index_id()), + index_meta->get_index_suffix(), + logical_reader.get(), open_mode); + auto doris_status = status; + if (!doris_status.ok()) { + return ResultError(doris_status); + } + return logical_reader; +} + Result> IndexFileReader::open( const TabletIndex* index_meta, const io::IOContext* io_ctx) const { auto index_id = index_meta->index_id(); @@ -254,6 +336,19 @@ Status IndexFileReader::index_file_exist(const TabletIndex* index_meta, bool* re auto index_file_path = InvertedIndexDescriptor::get_index_file_path_v1( _index_path_prefix, index_meta->index_id(), index_meta->get_index_suffix()); return _fs->exists(index_file_path, res); + } else if (_storage_format == InvertedIndexStorageFormatPB::SNII) { + auto index_file_path = InvertedIndexDescriptor::get_index_file_path_v2(_index_path_prefix); + RETURN_IF_ERROR(_fs->exists(index_file_path, res)); + if (!*res) { + return Status::OK(); + } + std::shared_lock lock(_mutex); + if (_snii_segment_reader == nullptr) { + *res = false; + return Status::OK(); + } + return _snii_segment_reader->index_exists(cast_set(index_meta->index_id()), + index_meta->get_index_suffix(), res); } else { std::shared_lock lock(_mutex); // Lock for reading if (_stream == nullptr) { @@ -279,6 +374,25 @@ Status IndexFileReader::has_null(const TabletIndex* index_meta, bool* res) const *res = true; return Status::OK(); } + if (_storage_format == InvertedIndexStorageFormatPB::SNII) { + std::shared_lock lock(_mutex); + if (_snii_segment_reader == nullptr) { + return Status::Error( + "SNII index file {} is not opened", + InvertedIndexDescriptor::get_index_file_path_v2(_index_path_prefix)); + } + io::IOContext meta_io_ctx; + meta_io_ctx.is_inverted_index = true; + meta_io_ctx.is_index_data = true; + snii_doris::DorisSniiFileReader::ScopedIOContext io_context_scope(&meta_io_ctx); + + doris::snii::format::SectionRefs section_refs; + RETURN_IF_ERROR(_snii_segment_reader->section_refs_for_index( + cast_set(index_meta->index_id()), index_meta->get_index_suffix(), + §ion_refs)); + *res = section_refs.null_bitmap.length > 0; + return Status::OK(); + } std::shared_lock lock(_mutex); // Lock for reading if (_stream == nullptr) { return Status::Error( diff --git a/be/src/storage/index/index_file_reader.h b/be/src/storage/index/index_file_reader.h index fb4ec2b9a62fe3..f09aa9f8476503 100644 --- a/be/src/storage/index/index_file_reader.h +++ b/be/src/storage/index/index_file_reader.h @@ -35,6 +35,9 @@ #include "io/fs/file_system.h" #include "storage/index/index_file_writer.h" #include "storage/index/inverted/inverted_index_desc.h" +#include "storage/index/snii/reader/logical_index_reader.h" +#include "storage/index/snii/reader/snii_segment_reader.h" +#include "storage/index/snii/snii_doris_adapter.h" namespace doris { class TabletIndex; @@ -60,7 +63,7 @@ class IndexFileReader { : _fs(std::move(fs)), _index_path_prefix(std::move(index_path_prefix)), _storage_format(storage_format), - _idx_file_info(idx_file_info), + _idx_file_info(std::move(idx_file_info)), _tablet_id(tablet_id) {} virtual ~IndexFileReader() = default; @@ -68,6 +71,10 @@ class IndexFileReader { const io::IOContext* io_ctx = nullptr); MOCK_FUNCTION Result> open( const TabletIndex* index_meta, const io::IOContext* io_ctx = nullptr) const; + Result> open_snii_index( + const TabletIndex* index_meta, const io::IOContext* io_ctx = nullptr, + doris::snii::reader::LogicalIndexOpenMode open_mode = + doris::snii::reader::LogicalIndexOpenMode::kQuery) const; void debug_file_entries(); std::string get_index_file_cache_key(const TabletIndex* index_meta) const; std::string get_index_file_path(const TabletIndex* index_meta) const; @@ -75,12 +82,19 @@ class IndexFileReader { Status has_null(const TabletIndex* index_meta, bool* res) const; Result get_all_directories(); // open file v2, init _stream - int64_t get_inverted_file_size() const { return _stream == nullptr ? 0 : _stream->length(); } + int64_t get_inverted_file_size() const { + if (_storage_format == InvertedIndexStorageFormatPB::SNII) { + return _snii_file_reader == nullptr ? 0 : _snii_file_reader->size(); + } + return _stream == nullptr ? 0 : _stream->length(); + } const std::string& get_index_path_prefix() const { return _index_path_prefix; } + InvertedIndexStorageFormatPB get_storage_format() const { return _storage_format; } friend IndexFileWriter; protected: Status _init_from(int32_t read_buffer_size, const io::IOContext* io_ctx); + Status _init_snii(const io::IOContext* io_ctx); Result> _open( int64_t index_id, const std::string& index_suffix, const io::IOContext* io_ctx = nullptr) const; @@ -88,6 +102,8 @@ class IndexFileReader { private: IndicesEntriesMap _indices_entries; std::unique_ptr _stream = nullptr; + std::shared_ptr _snii_file_reader; + std::unique_ptr _snii_segment_reader; const io::FileSystemSPtr _fs; std::string _index_path_prefix; int32_t _read_buffer_size = -1; diff --git a/be/src/storage/index/index_file_writer.cpp b/be/src/storage/index/index_file_writer.cpp index fcaded4bd7a4a9..41618afe6a5e0d 100644 --- a/be/src/storage/index/index_file_writer.cpp +++ b/be/src/storage/index/index_file_writer.cpp @@ -19,9 +19,12 @@ #include +#include #include #include +#include "common/cast_set.h" +#include "common/config.h" #include "common/status.h" #include "io/fs/packed_file_writer.h" #include "io/fs/s3_file_writer.h" @@ -35,10 +38,53 @@ #include "storage/index/inverted/inverted_index_desc.h" #include "storage/index/inverted/inverted_index_fs_directory.h" #include "storage/index/inverted/inverted_index_reader.h" +#include "storage/index/snii/snii_doris_adapter.h" #include "storage/tablet/tablet_schema.h" namespace doris::segment_v2 { +// Resolves whether one segment index lays out freq regions (G16-c). Freq +// serves ONLY BM25 scoring: a scoring config always keeps it; a plain +// positions config keeps it only when the escape-hatch config asks for the +// full T2 layout. NOT in the anonymous namespace on purpose -- the UT covers +// this production policy line directly (a flipped operator or inverted flag +// here would otherwise stay green: no BE test drives add_snii_index). +bool snii_effective_write_freq(doris::snii::format::IndexConfig index_config) { + return doris::snii::format::has_scoring(index_config) || + config::snii_positions_index_write_freq; +} + +// Shared write-parameter resolution for one SNII index flush; `input->config` +// must already be set. BOTH the build path (add_snii_index) and the T2.2 +// compaction-merge streamed session resolve through this single helper, so the +// merge fast path can never drift from the rebuild contract (the T2 semantic +// golden invariant depends on parameter parity). NOT in the anonymous +// namespace on purpose -- the UT pins the resolved values directly. +void snii_resolve_index_write_params(bool is_direct_load, + doris::snii::writer::SniiIndexInput* input) { + // G16-c: freq regions serve only BM25 scoring; a plain positions index + // drops them unless the escape hatch asks for the full T2 layout. + input->write_freq = snii_effective_write_freq(input->config); + // G16-h: zstd levels. dict blocks accept zstd's full sane range; the prx + // level floor is 3 because the writer passes -level into the prx builders + // and -1 is the historic "auto at default level 3" sentinel -- a + // configured level 1 would silently resolve to 3 anyway (levels 1-2 buy + // nothing over 3 on these payloads). + input->dict_block_zstd_level = std::clamp(config::snii_dict_block_zstd_level, 1, 19); + // Patch C prx tiering: a direct load compresses prx at the cheaper load + // level; compaction / schema change / ADD INDEX keep snii_prx_zstd_level + // and compaction rewrites every segment with it, so settled segments (and + // the cold-query path over them) are byte-for-byte unaffected. + input->prx_zstd_level = std::clamp( + is_direct_load ? config::snii_prx_zstd_level_direct_load : config::snii_prx_zstd_level, + 3, 19); + // G16-d: dict block size experiment knob; <= 0 keeps the format default. + if (config::snii_target_dict_block_bytes > 0) { + input->target_dict_block_bytes = + static_cast(config::snii_target_dict_block_bytes); + } +} + IndexFileWriter::IndexFileWriter(io::FileSystemSPtr fs, std::string index_path_prefix, std::string rowset_id, int64_t seg_id, InvertedIndexStorageFormatPB storage_format, @@ -57,7 +103,7 @@ IndexFileWriter::IndexFileWriter(io::FileSystemSPtr fs, std::string index_path_p _tmp_dir = tmp_file_dir.native(); if (_storage_format == InvertedIndexStorageFormatPB::V1) { _index_storage_format = std::make_unique(this); - } else { + } else if (_storage_format != InvertedIndexStorageFormatPB::SNII) { _index_storage_format = std::make_unique(this); } } @@ -85,6 +131,10 @@ Status IndexFileWriter::_insert_directory_into_map(int64_t index_id, } Result> IndexFileWriter::open(const TabletIndex* index_meta) { + if (_storage_format == InvertedIndexStorageFormatPB::SNII) { + return ResultError(Status::Error( + "SNII format does not open CLucene directories")); + } auto local_fs_index_path = InvertedIndexDescriptor::get_temporary_index_path( _tmp_dir, _rowset_id, _seg_id, index_meta->index_id(), index_meta->get_index_suffix()); auto dir = std::shared_ptr(DorisFSDirectoryFactory::getDirectory( @@ -98,6 +148,121 @@ Result> IndexFileWriter::open(const TabletInde return dir; } +Status IndexFileWriter::add_snii_index(const TabletIndex* index_meta, uint32_t doc_count, + std::vector null_docids, + doris::snii::writer::SpimiTermBuffer* const term_buffer, + doris::snii::format::IndexConfig index_config, + SniiAddIndexOptions options, + doris::snii::writer::MemoryReporter* const mem_reporter) { + DCHECK(_storage_format == InvertedIndexStorageFormatPB::SNII); + DCHECK(index_meta != nullptr); + DCHECK(term_buffer != nullptr); + if (_idx_v2_writer == nullptr) { + return Status::Error( + "SNII index file writer is null for {}", _index_path_prefix); + } + if (_snii_file_writer == nullptr) { + _snii_file_writer = std::make_unique(_idx_v2_writer.get()); + _snii_compound_writer = + std::make_unique(_snii_file_writer.get()); + } + + doris::snii::writer::SniiIndexInput input; + input.index_id = cast_set(index_meta->index_id()); + input.index_suffix = index_meta->get_index_suffix(); + input.config = index_config; + input.doc_count = doc_count; + input.null_docids = std::move(null_docids); + input.encoded_norms = std::move(options.encoded_norms); + input.common_grams_metadata = std::move(options.common_grams_metadata); + input.common_grams_posting_policy = options.common_grams_posting_policy; + input.term_source = term_buffer; + input.mem_reporter = mem_reporter; + snii_resolve_index_write_params(options.is_direct_load, &input); + RETURN_IF_ERROR(_snii_compound_writer->add_logical_index(input)); + ++_snii_index_count; + return Status::OK(); +} + +Status IndexFileWriter::add_snii_index_streamed( + const TabletIndex* index_meta, uint32_t doc_count, + doris::snii::writer::TrackedNullDocids null_docids, + doris::snii::format::IndexConfig index_config, + std::shared_ptr mem_reporter, + doris::snii::writer::SniiStreamedIndexSession** session) { + return add_snii_index_streamed(index_meta, doc_count, std::move(null_docids), + doris::snii::writer::TrackedEncodedNorms(std::vector()), + std::nullopt, + doris::snii::format::CommonGramsPostingPolicy::kNone, + index_config, std::move(mem_reporter), session); +} + +Status IndexFileWriter::add_snii_index_streamed( + const TabletIndex* index_meta, uint32_t doc_count, + doris::snii::writer::TrackedNullDocids null_docids, + doris::snii::writer::TrackedEncodedNorms encoded_norms, + std::optional common_grams_metadata, + doris::snii::format::CommonGramsPostingPolicy common_grams_posting_policy, + doris::snii::format::IndexConfig index_config, + std::shared_ptr mem_reporter, + doris::snii::writer::SniiStreamedIndexSession** session) { + DCHECK(_storage_format == InvertedIndexStorageFormatPB::SNII); + DCHECK(index_meta != nullptr); + if (session == nullptr) { + return Status::Error( + "SNII streamed session out parameter is null for {}", _index_path_prefix); + } + *session = nullptr; + if (_idx_v2_writer == nullptr) { + return Status::Error( + "SNII index file writer is null for {}", _index_path_prefix); + } + const bool has_scoring = doris::snii::format::has_scoring(index_config); + const bool valid_scoring_shape = + has_scoring ? common_grams_metadata.has_value() && encoded_norms.size() == doc_count + : !common_grams_metadata.has_value() && encoded_norms.empty(); + if (!valid_scoring_shape) { + return Status::InternalError( + "SNII streamed merge scoring shape disagrees with eligibility for {}", + _index_path_prefix); + } + if (_snii_file_writer == nullptr) { + _snii_file_writer = std::make_unique(_idx_v2_writer.get()); + _snii_compound_writer = + std::make_unique(_snii_file_writer.get()); + } + + doris::snii::writer::SniiIndexInput input; + input.index_id = cast_set(index_meta->index_id()); + input.index_suffix = index_meta->get_index_suffix(); + input.config = index_config; + input.doc_count = doc_count; + input.mem_reporter = mem_reporter.get(); + input.common_grams_metadata = std::move(common_grams_metadata); + input.common_grams_posting_policy = common_grams_posting_policy; + // Merge output is always the settled-segment shape: COMPACTION prx level. + snii_resolve_index_write_params(/*is_direct_load=*/false, &input); + if (mem_reporter != nullptr) { + constexpr uint64_t kMaxStreamedDictResidentBytes = 64ULL << 20; + DORIS_CHECK_GE(mem_reporter->cap_bytes(), 8); + input.dict_resident_cap_bytes = + std::min(kMaxStreamedDictResidentBytes, mem_reporter->cap_bytes() / 8); + } + RETURN_IF_ERROR(_snii_compound_writer->begin_streamed_index( + std::move(input), std::move(null_docids), std::move(encoded_norms), session)); + if (mem_reporter != nullptr) { + _snii_memory_reporters.push_back(std::move(mem_reporter)); + } + ++_snii_index_count; + return Status::OK(); +} + +void IndexFileWriter::retain_snii_memory_reporter( + std::unique_ptr mem_reporter) { + DCHECK(mem_reporter != nullptr); + _snii_memory_reporters.emplace_back(std::move(mem_reporter)); +} + Status IndexFileWriter::delete_index(const TabletIndex* index_meta) { DBUG_EXECUTE_IF("IndexFileWriter::delete_index_index_meta_nullptr", { index_meta = nullptr; }); if (!index_meta) { @@ -124,6 +289,9 @@ Status IndexFileWriter::delete_index(const TabletIndex* index_meta) { } Status IndexFileWriter::add_into_searcher_cache() { + if (_storage_format == InvertedIndexStorageFormatPB::SNII) { + return Status::OK(); + } auto index_file_reader = std::make_unique( _fs, _index_path_prefix, _storage_format, InvertedIndexFileInfo(), _tablet_id); auto st = index_file_reader->init(); @@ -197,6 +365,21 @@ Result> IndexFileWriter::_construct_index_ Status IndexFileWriter::begin_close() { DCHECK(!_closed) << debug_string(); _closed = true; + if (_storage_format == InvertedIndexStorageFormatPB::SNII) { + if (_snii_compound_writer == nullptr) { + if (_idx_v2_writer == nullptr) { + return Status::OK(); + } + _snii_file_writer = + std::make_unique(_idx_v2_writer.get()); + _snii_compound_writer = std::make_unique( + _snii_file_writer.get()); + } + RETURN_IF_ERROR(_snii_compound_writer->finish()); + _total_file_size = _idx_v2_writer->bytes_appended(); + _file_info.set_index_size(_total_file_size); + return _idx_v2_writer->close(true); + } if (_indices_dirs.empty()) { // An empty file must still be created even if there are no indexes to write if (dynamic_cast(_idx_v2_writer.get()) != nullptr || @@ -239,6 +422,12 @@ Status IndexFileWriter::begin_close() { Status IndexFileWriter::finish_close() { DCHECK(_closed) << debug_string(); + if (_storage_format == InvertedIndexStorageFormatPB::SNII) { + if (_idx_v2_writer != nullptr && _idx_v2_writer->state() != io::FileWriter::State::CLOSED) { + RETURN_IF_ERROR(_idx_v2_writer->close(false)); + } + return Status::OK(); + } if (_indices_dirs.empty()) { // An empty file must still be created even if there are no indexes to write if (dynamic_cast(_idx_v2_writer.get()) != nullptr || diff --git a/be/src/storage/index/index_file_writer.h b/be/src/storage/index/index_file_writer.h index a303de8b68c156..0c2e8a1734917b 100644 --- a/be/src/storage/index/index_file_writer.h +++ b/be/src/storage/index/index_file_writer.h @@ -22,23 +22,38 @@ #include #include +#include #include #include +#include #include "common/be_mock_util.h" #include "io/fs/file_system.h" #include "io/fs/file_writer.h" #include "io/fs/local_file_system.h" #include "storage/index/index_storage_format.h" +#include "storage/index/inverted/common_grams/common_grams_segment_metadata.h" #include "storage/index/inverted/inverted_index_common.h" #include "storage/index/inverted/inverted_index_compound_reader.h" #include "storage/index/inverted/inverted_index_searcher.h" +#include "storage/index/snii/format/format_constants.h" +#include "storage/index/snii/snii_doris_adapter.h" +#include "storage/index/snii/writer/snii_compound_writer.h" + +namespace doris::snii::writer { +class MemoryReporter; +class SpimiTermBuffer; +class SniiCompoundWriter; +} // namespace doris::snii::writer namespace doris { class TabletIndex; namespace segment_v2 { class DorisFSDirectory; +namespace snii_doris { +class DorisSniiFileWriter; +} // namespace snii_doris using InvertedIndexDirectoryMap = std::map, std::shared_ptr>; @@ -55,6 +70,55 @@ class IndexFileWriter { virtual ~IndexFileWriter() = default; MOCK_FUNCTION Result> open(const TabletIndex* index_meta); + // Write-path facts for one SNII index flush. + struct SniiAddIndexOptions { + // This flush serves a stream/broker load (DataWriteType::TYPE_DIRECT): + // the prx region compresses at snii_prx_zstd_level_direct_load; + // compaction / schema change / ADD INDEX keep snii_prx_zstd_level. + bool is_direct_load = false; + // Present only for a CommonGrams writer that has a complete immutable + // capability identity. These are semantic BM25 inputs; physical TTF is + // still derived from every emitted unigram and gram posting. + std::vector encoded_norms; + std::optional common_grams_metadata; + snii::format::CommonGramsPostingPolicy common_grams_posting_policy = + snii::format::CommonGramsPostingPolicy::kNone; + }; + Status add_snii_index(const TabletIndex* index_meta, uint32_t doc_count, + std::vector null_docids, + doris::snii::writer::SpimiTermBuffer* const term_buffer, + doris::snii::format::IndexConfig index_config, + SniiAddIndexOptions options, + doris::snii::writer::MemoryReporter* const mem_reporter); + // T2.2 compaction index merge fast path: begins a STREAMED SNII index + // session on this compound. Unlike add_snii_index (which drains a SPIMI + // term buffer), the caller pushes pre-merged, lexicographically sorted + // terms through *session and seals the index with (*session)->finish(). + // Write parameters resolve through the SAME helper as add_snii_index + // (write_freq / zstd levels / dict block size), always at the COMPACTION + // prx tier (a merge is never a direct load). CommonGrams T3 callers transfer + // a precharged destination norm vector and a validated static metadata seed; + // the streamed session late-binds semantic token_count before finish. Only ONE + // session may be active per compound at a time, and begin_close() with an + // unfinished session fails instead of sealing a half-fed container. The + // handle is owned by this writer and valid until it is destroyed. + Status add_snii_index_streamed( + const TabletIndex* index_meta, uint32_t doc_count, + doris::snii::writer::TrackedNullDocids null_docids, + doris::snii::format::IndexConfig index_config, + std::shared_ptr mem_reporter, + doris::snii::writer::SniiStreamedIndexSession** session); + Status add_snii_index_streamed( + const TabletIndex* index_meta, uint32_t doc_count, + doris::snii::writer::TrackedNullDocids null_docids, + doris::snii::writer::TrackedEncodedNorms encoded_norms, + std::optional common_grams_metadata, + doris::snii::format::CommonGramsPostingPolicy common_grams_posting_policy, + doris::snii::format::IndexConfig index_config, + std::shared_ptr mem_reporter, + doris::snii::writer::SniiStreamedIndexSession** session); + void retain_snii_memory_reporter( + std::unique_ptr mem_reporter); Status delete_index(const TabletIndex* index_meta); Status initialize(InvertedIndexDirectoryMap& indices_dirs); Status add_into_searcher_cache(); @@ -113,6 +177,10 @@ class IndexFileWriter { IndexStorageFormatPtr _index_storage_format; int64_t _tablet_id = -1; + std::unique_ptr _snii_file_writer; + std::vector> _snii_memory_reporters; + std::unique_ptr _snii_compound_writer; + size_t _snii_index_count = 0; friend class IndexStorageFormatV1; friend class IndexStorageFormatV2; diff --git a/be/src/storage/index/index_query_context.h b/be/src/storage/index/index_query_context.h index 92fb698f4d3767..14106db4a5b7b7 100644 --- a/be/src/storage/index/index_query_context.h +++ b/be/src/storage/index/index_query_context.h @@ -18,7 +18,7 @@ #pragma once #include "storage/compaction/collection_similarity.h" -#include "storage/compaction/collection_statistics.h" +#include "storage/index/inverted/similarity/collection_statistics.h" namespace doris::segment_v2 { @@ -32,6 +32,24 @@ struct IndexQueryContext { size_t query_limit = 0; bool is_asc = false; + + // G02 count-only fast-path handshake. Set by SegmentIterator ONLY while it + // evaluates the single pushed-down MATCH predicate of a COUNT_ON_INDEX scan + // whose row space is provably unfiltered (no deletes, no other conjuncts, + // full row bitmap, no row-id consumers -- see count_on_index_fastpath.h), + // and reset immediately after. When set, an index reader MAY answer the + // query with a bitmap whose CARDINALITY equals the match count without the + // row ids being real (SNII returns [0, df) straight from dict-entry df, + // skipping the posting decode). Readers must never cache such a bitmap + // under a key a row-accurate query could hit. + bool count_on_index_fastpath = false; + // G03 reply direction of the same handshake. Set by a reader iff it DID + // answer with such a fabricated count bitmap (never on a query-cache hit, + // a single-flight shared result, or any row-accurate decode). Read and + // reset by SegmentIterator right after the index apply; a true value is + // the precondition for the count-emission shortcut that materializes the + // remaining count as default rows without iterating the row bitmap. + bool count_on_index_fastpath_hit = false; }; using IndexQueryContextPtr = std::shared_ptr; diff --git a/be/src/storage/index/index_writer.cpp b/be/src/storage/index/index_writer.cpp index 2325d280471337..80aac9ae9a071f 100644 --- a/be/src/storage/index/index_writer.cpp +++ b/be/src/storage/index/index_writer.cpp @@ -18,6 +18,7 @@ #include "common/exception.h" #include "storage/index/ann/ann_index_writer.h" #include "storage/index/inverted/inverted_index_writer.h" +#include "storage/index/snii/snii_index_writer.h" #include "storage/tablet/tablet_schema.h" #include "storage/types.h" @@ -80,6 +81,21 @@ Status IndexColumnWriter::create(const TabletColumn* column, } } + if (storage_format == InvertedIndexStorageFormatPB::SNII) { + if (!is_string_type(type)) { + return Status::Error( + "SNII inverted index storage format does not support BKD index type {}", + type); + } + *res = std::make_unique(index_file_writer, index_meta, type); + auto st = (*res)->init(); + if (!st.ok()) { + (*res)->close_on_error(); + return st; + } + return Status::OK(); + } + DBUG_EXECUTE_IF("InvertedIndexColumnWriter::create_unsupported_type_for_inverted_index", { type = FieldType::OLAP_FIELD_TYPE_JSONB; }) switch (type) { diff --git a/be/src/storage/index/index_writer.h b/be/src/storage/index/index_writer.h index a0760f99000fcb..def7a8fc4c72cc 100644 --- a/be/src/storage/index/index_writer.h +++ b/be/src/storage/index/index_writer.h @@ -59,6 +59,15 @@ class IndexColumnWriter { virtual Status add_nulls(uint32_t count) = 0; virtual Status add_array_nulls(const uint8_t* null_map, size_t num_rows) = 0; + // Write-path hint from the segment writer: this writer serves a direct load + // (stream/broker load, DataWriteType::TYPE_DIRECT) as opposed to compaction + // / schema change / index build. The column writer forwards it + // unconditionally right after create() succeeds (i.e. after init(), before + // any value is added); creation paths outside the column writer (e.g. ADD + // INDEX in index_builder.cpp) never call it and keep the non-direct default. + // Default no-op: SNII uses it to select the direct-load PRX zstd level. + virtual void set_direct_load(bool /*is_direct_load*/) {} + virtual Status finish() = 0; virtual int64_t size() const = 0; diff --git a/be/src/storage/index/inverted/abstract_analysis_factory.h b/be/src/storage/index/inverted/abstract_analysis_factory.h index f9afc42f19db93..3452f93fef7cdf 100644 --- a/be/src/storage/index/inverted/abstract_analysis_factory.h +++ b/be/src/storage/index/inverted/abstract_analysis_factory.h @@ -21,6 +21,19 @@ namespace doris::segment_v2::inverted_index { +enum class AnalysisPurpose { + kIndex, + kSniiTransientIndex, + kPlainQuery, + kExactPhraseQuery, + kPhrasePrefixQuery, +}; + +enum class PositionCapability { + kUnknown, + kAlwaysUnitIncrement, +}; + class AbstractAnalysisFactory { public: virtual ~AbstractAnalysisFactory() = default; diff --git a/be/src/storage/index/inverted/analysis_factory_mgr.cpp b/be/src/storage/index/inverted/analysis_factory_mgr.cpp index a208a275bda640..d41960fad206a9 100644 --- a/be/src/storage/index/inverted/analysis_factory_mgr.cpp +++ b/be/src/storage/index/inverted/analysis_factory_mgr.cpp @@ -21,6 +21,7 @@ #include "storage/index/inverted/char_filter/empty_char_filter_factory.h" #include "storage/index/inverted/char_filter/icu_normalizer_char_filter_factory.h" #include "storage/index/inverted/token_filter/ascii_folding_filter_factory.h" +#include "storage/index/inverted/token_filter/common_grams_filter_factory.h" #include "storage/index/inverted/token_filter/empty_token_filter_factory.h" #include "storage/index/inverted/token_filter/icu_normalizer_filter_factory.h" #include "storage/index/inverted/token_filter/lower_case_filter_factory.h" @@ -82,6 +83,8 @@ void AnalysisFactoryMgr::initialise() { "pinyin", []() { return std::make_shared(); }); registerFactory( "icu_normalizer", []() { return std::make_shared(); }); + registerFactory( + "common_grams", []() { return std::make_shared(); }); }); } diff --git a/be/src/storage/index/inverted/analyzer/analyzer.cpp b/be/src/storage/index/inverted/analyzer/analyzer.cpp index 6d16613bcd9b00..c9735ee032b4f7 100644 --- a/be/src/storage/index/inverted/analyzer/analyzer.cpp +++ b/be/src/storage/index/inverted/analyzer/analyzer.cpp @@ -40,8 +40,27 @@ #include "storage/index/inverted/analyzer/icu/icu_analyzer.h" #include "storage/index/inverted/analyzer/ik/IKAnalyzer.h" #include "storage/index/inverted/char_filter/char_replace_char_filter_factory.h" +#include "storage/index/inverted/token_filter/common_grams_filter.h" namespace doris::segment_v2::inverted_index { +namespace { + +class BuiltinAnalyzerProvider final : public AnalyzerProvider { +public: + explicit BuiltinAnalyzerProvider(InvertedIndexAnalyzerConfig config) + : _analyzer(InvertedIndexAnalyzer::create_builtin_analyzer( + config.analyzer_name.empty() + ? config.parser_type + : get_inverted_index_parser_type_from_string(config.analyzer_name), + config.parser_mode, config.lower_case, config.stop_words)) {} + + AnalyzerPtr get_analyzer(AnalysisPurpose) const override { return _analyzer; } + +private: + const AnalyzerPtr _analyzer; +}; + +} // namespace ReaderPtr InvertedIndexAnalyzer::create_reader(const CharFilterMap& char_filter_map) { ReaderPtr reader = std::make_shared>(); @@ -130,32 +149,43 @@ AnalyzerPtr InvertedIndexAnalyzer::create_builtin_analyzer(InvertedIndexParserTy } AnalyzerPtr InvertedIndexAnalyzer::create_analyzer(const InvertedIndexAnalyzerConfig* config) { - DCHECK(config != nullptr); - const std::string& analyzer_name = config->analyzer_name; - - // Handle empty analyzer name - use builtin analyzer based on parser_type. - // This is the common case when user does not specify USING ANALYZER. - if (analyzer_name.empty()) { - return create_builtin_analyzer(config->parser_type, config->parser_mode, config->lower_case, - config->stop_words); - } + return create_analyzer(config, AnalysisPurpose::kPlainQuery); +} - // Check if it's a builtin analyzer name (english, chinese, standard, etc.) - if (is_builtin_analyzer(analyzer_name)) { - InvertedIndexParserType parser_type = - get_inverted_index_parser_type_from_string(analyzer_name); +AnalyzerPtr InvertedIndexAnalyzer::create_analyzer(const InvertedIndexAnalyzerConfig* config, + AnalysisPurpose purpose) { + DCHECK(config != nullptr); + if (config->analyzer_name.empty() || is_builtin_analyzer(config->analyzer_name)) { + const InvertedIndexParserType parser_type = + config->analyzer_name.empty() + ? config->parser_type + : get_inverted_index_parser_type_from_string(config->analyzer_name); return create_builtin_analyzer(parser_type, config->parser_mode, config->lower_case, config->stop_words); } - // Custom analyzer - look up in policy manager auto* index_policy_mgr = doris::ExecEnv::GetInstance()->index_policy_mgr(); - if (!index_policy_mgr) { + if (index_policy_mgr == nullptr) { throw Exception(ErrorCode::INVERTED_INDEX_ANALYZER_ERROR, "Index policy manager is not initialized"); } + return index_policy_mgr->get_analyzer_by_name(config->analyzer_name, purpose); +} + +AnalyzerProviderPtr InvertedIndexAnalyzer::create_analyzer_provider( + const InvertedIndexAnalyzerConfig* config) { + DCHECK(config != nullptr); + if (config->analyzer_name.empty() || is_builtin_analyzer(config->analyzer_name)) { + return std::make_shared(*config); + } - return index_policy_mgr->get_policy_by_name(analyzer_name); + auto* index_policy_mgr = doris::ExecEnv::GetInstance()->index_policy_mgr(); + if (index_policy_mgr == nullptr) { + throw Exception(ErrorCode::INVERTED_INDEX_ANALYZER_ERROR, + "Index policy manager is not initialized"); + } + return index_policy_mgr->get_analyzer_provider_by_name(config->analyzer_name, + config->char_filter_map); } std::vector InvertedIndexAnalyzer::get_analyse_result( @@ -172,6 +202,8 @@ std::vector InvertedIndexAnalyzer::get_analyse_result( t.term = std::string(token.termBuffer(), token.termLength()); position += token.getPositionIncrement(); t.position = position; + t.key_kind = is_common_gram_token_type(token.type()) ? TermKeyKind::kCommonGram + : TermKeyKind::kPlain; analyse_result.emplace_back(std::move(t)); } } @@ -185,6 +217,12 @@ std::vector InvertedIndexAnalyzer::get_analyse_result( std::vector InvertedIndexAnalyzer::get_analyse_result( const std::string& search_str, const std::map& properties) { + return get_analyse_result(search_str, properties, AnalysisPurpose::kPlainQuery); +} + +std::vector InvertedIndexAnalyzer::get_analyse_result( + const std::string& search_str, const std::map& properties, + AnalysisPurpose purpose) { if (!should_analyzer(properties)) { // Keyword index: all strings (including empty) are valid tokens for exact match. // Empty string is a valid value in keyword index and should be matchable. @@ -200,12 +238,26 @@ std::vector InvertedIndexAnalyzer::get_analyse_result( config.lower_case = get_parser_lowercase_from_properties(properties); config.stop_words = get_parser_stopwords_from_properties(properties); config.char_filter_map = get_parser_char_filter_map_from_properties(properties); - auto analyzer = create_analyzer(&config); + auto analyzer = create_analyzer(&config, purpose); auto reader = create_reader(config.char_filter_map); reader->init(search_str.data(), static_cast(search_str.size()), true); return get_analyse_result(reader, analyzer.get()); } +AnalysisPurpose select_analysis_purpose(InvertedIndexQueryType query_type, int32_t slop, + bool is_similarity) { + if (is_similarity) { + return AnalysisPurpose::kPlainQuery; + } + if (query_type == InvertedIndexQueryType::MATCH_PHRASE_QUERY && slop == 0) { + return AnalysisPurpose::kExactPhraseQuery; + } + if (query_type == InvertedIndexQueryType::MATCH_PHRASE_PREFIX_QUERY) { + return AnalysisPurpose::kPhrasePrefixQuery; + } + return AnalysisPurpose::kPlainQuery; +} + bool InvertedIndexAnalyzer::should_analyzer(const std::map& properties) { auto parser_type = get_inverted_index_parser_type_from_string( get_parser_string_from_properties(properties)); diff --git a/be/src/storage/index/inverted/analyzer/analyzer.h b/be/src/storage/index/inverted/analyzer/analyzer.h index 98588a251bf047..4378b2e6c24b94 100644 --- a/be/src/storage/index/inverted/analyzer/analyzer.h +++ b/be/src/storage/index/inverted/analyzer/analyzer.h @@ -20,6 +20,7 @@ #include #include +#include "storage/index/inverted/analyzer/analyzer_provider.h" #include "storage/index/inverted/inverted_index_parser.h" #include "storage/index/inverted/inverted_index_query_type.h" #include "storage/index/inverted/query/query.h" @@ -49,14 +50,23 @@ class InvertedIndexAnalyzer { const std::string& lower_case, const std::string& stop_words); static AnalyzerPtr create_analyzer(const InvertedIndexAnalyzerConfig* config); + static AnalyzerPtr create_analyzer(const InvertedIndexAnalyzerConfig* config, + AnalysisPurpose purpose); + static AnalyzerProviderPtr create_analyzer_provider(const InvertedIndexAnalyzerConfig* config); static std::vector get_analyse_result(ReaderPtr reader, lucene::analysis::Analyzer* analyzer); static std::vector get_analyse_result( const std::string& search_str, const std::map& properties); + static std::vector get_analyse_result( + const std::string& search_str, const std::map& properties, + AnalysisPurpose purpose); static bool should_analyzer(const std::map& properties); }; +AnalysisPurpose select_analysis_purpose(InvertedIndexQueryType query_type, int32_t slop, + bool is_similarity); + } // namespace doris::segment_v2::inverted_index \ No newline at end of file diff --git a/be/src/storage/index/inverted/analyzer/analyzer_provider.h b/be/src/storage/index/inverted/analyzer/analyzer_provider.h new file mode 100644 index 00000000000000..60fe8250b00a85 --- /dev/null +++ b/be/src/storage/index/inverted/analyzer/analyzer_provider.h @@ -0,0 +1,46 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include "storage/index/inverted/abstract_analysis_factory.h" +#include "storage/index/inverted/common_grams/common_grams_segment_metadata.h" + +namespace lucene::analysis { +class Analyzer; +} + +namespace doris::segment_v2::inverted_index { + +class CommonWordSet; + +class AnalyzerProvider { +public: + virtual ~AnalyzerProvider() = default; + virtual std::shared_ptr get_analyzer( + AnalysisPurpose purpose) const = 0; + virtual std::string_view base_analyzer_fingerprint() const { return {}; } + virtual bool uses_common_grams() const { return false; } + virtual const CommonGramsQueryIdentity* common_grams_identity() const { return nullptr; } + virtual const CommonWordSet* common_grams_word_set() const { return nullptr; } +}; +using AnalyzerProviderPtr = std::shared_ptr; + +} // namespace doris::segment_v2::inverted_index diff --git a/be/src/storage/index/inverted/analyzer/custom_analyzer.cpp b/be/src/storage/index/inverted/analyzer/custom_analyzer.cpp index 3970d2e3cac74d..469f5ee2d6921a 100644 --- a/be/src/storage/index/inverted/analyzer/custom_analyzer.cpp +++ b/be/src/storage/index/inverted/analyzer/custom_analyzer.cpp @@ -17,12 +17,116 @@ #include "storage/index/inverted/analyzer/custom_analyzer.h" +#include +#include + #include "common/status.h" #include "runtime/exec_env.h" #include "storage/index/inverted/analysis_factory_mgr.h" +#include "storage/index/inverted/common_grams/common_word_set.h" +#include "storage/index/inverted/token_filter/common_grams_filter_factory.h" #include "storage/index/inverted/token_stream.h" +#include "util/sha.h" namespace doris::segment_v2::inverted_index { +namespace { + +bool config_uses_common_grams(const ImmutableCustomAnalyzerConfigPtr& config) { + DORIS_CHECK(config != nullptr); + const auto& filter_configs = config->get_token_filter_configs(); + return std::any_of(filter_configs.begin(), filter_configs.end(), + [](const auto& entry) { return entry->get_name() == "common_grams"; }); +} + +void append_canonical_value(std::string_view value, std::string* output) { + output->append(std::to_string(value.size())); + output->push_back(':'); + output->append(value); +} + +void append_component(std::string_view role, const ComponentConfigPtr& component, + std::string* output) { + append_canonical_value(role, output); + append_canonical_value(component->get_name(), output); + const auto entries = component->get_params().sorted_entries(); + append_canonical_value(std::to_string(entries.size()), output); + for (const auto& [key, value] : entries) { + append_canonical_value(key, output); + append_canonical_value(value, output); + } +} + +std::string sha256(std::string_view value) { + SHA256Digest digest; + digest.reset(value.data(), value.size()); + return std::string(digest.digest()); +} + +std::string calculate_base_analyzer_fingerprint_impl( + const ImmutableCustomAnalyzerConfigPtr& config, + const std::map& outer_char_filter_map) { + DORIS_CHECK(config != nullptr); + std::string base; + append_canonical_value("doris-common-grams-base-analyzer:v1", &base); + append_canonical_value("outer_char_filter", &base); + append_canonical_value(std::to_string(outer_char_filter_map.size()), &base); + for (const auto& [key, value] : outer_char_filter_map) { + append_canonical_value(key, &base); + append_canonical_value(value, &base); + } + append_component("tokenizer", config->get_tokenizer_config(), &base); + const auto char_filters = config->get_char_filter_configs(); + append_canonical_value(std::to_string(char_filters.size()), &base); + for (const auto& char_filter : char_filters) { + append_component("char_filter", char_filter, &base); + } + const auto token_filters = config->get_token_filter_configs(); + const auto base_token_filter_count = std::count_if( + token_filters.begin(), token_filters.end(), + [](const auto& token_filter) { return token_filter->get_name() != "common_grams"; }); + append_canonical_value(std::to_string(base_token_filter_count), &base); + for (const auto& token_filter : token_filters) { + if (token_filter->get_name() != "common_grams") { + append_component("token_filter", token_filter, &base); + } + } + return sha256(base); +} + +CommonGramsQueryIdentity build_common_grams_identity(std::string dictionary_identity, + std::string base_analyzer_fingerprint) { + std::string common_grams; + append_canonical_value("doris-common-grams:v1", &common_grams); + append_canonical_value(std::to_string(COMMON_GRAMS_SEMANTICS_VERSION_V1), &common_grams); + append_canonical_value(std::to_string(COMMON_GRAMS_KEY_VERSION_V1), &common_grams); + append_canonical_value(WORDSET_FORMAT_V1, &common_grams); + append_canonical_value(dictionary_identity, &common_grams); + return {.common_grams_dictionary_identity = std::move(dictionary_identity), + .base_analyzer_fingerprint = std::move(base_analyzer_fingerprint), + .common_grams_fingerprint = sha256(common_grams)}; +} + +std::array, 5> build_purpose_analyzers( + const ImmutableCustomAnalyzerConfigPtr& config, + const std::shared_ptr& common_words) { + DORIS_CHECK(config != nullptr); + const bool has_common_grams = config_uses_common_grams(config); + if (!has_common_grams) { + auto analyzer = CustomAnalyzer::build_custom_analyzer(config); + return {analyzer, analyzer, analyzer, analyzer, analyzer}; + } + return {CustomAnalyzer::build_custom_analyzer(config, AnalysisPurpose::kIndex, common_words), + CustomAnalyzer::build_custom_analyzer(config, AnalysisPurpose::kSniiTransientIndex, + common_words), + CustomAnalyzer::build_custom_analyzer(config, AnalysisPurpose::kPlainQuery, + common_words), + CustomAnalyzer::build_custom_analyzer(config, AnalysisPurpose::kExactPhraseQuery, + common_words), + CustomAnalyzer::build_custom_analyzer(config, AnalysisPurpose::kPhrasePrefixQuery, + common_words)}; +} + +} // namespace CustomAnalyzer::CustomAnalyzer(Builder* builder) { _tokenizer = builder->_tokenizer; @@ -74,7 +178,8 @@ TokenStreamComponentsPtr CustomAnalyzer::create_components() { return std::make_shared(tk, ts); } -CustomAnalyzerPtr CustomAnalyzer::build_custom_analyzer(const CustomAnalyzerConfigPtr& config) { +CustomAnalyzerPtr CustomAnalyzer::build_custom_analyzer( + const ImmutableCustomAnalyzerConfigPtr& config) { if (config == nullptr) { throw Exception(ErrorCode::ILLEGAL_STATE, "Null configuration detected."); } @@ -90,6 +195,134 @@ CustomAnalyzerPtr CustomAnalyzer::build_custom_analyzer(const CustomAnalyzerConf return builder.build(); } +CustomAnalyzerPtr CustomAnalyzer::build_custom_analyzer( + const ImmutableCustomAnalyzerConfigPtr& config, AnalysisPurpose purpose) { + return build_custom_analyzer(config, purpose, CommonWordSet::default_word_set()); +} + +CustomAnalyzerPtr CustomAnalyzer::build_custom_analyzer( + const ImmutableCustomAnalyzerConfigPtr& config, AnalysisPurpose purpose, + const std::shared_ptr& common_words) { + if (config == nullptr) { + throw Exception(ErrorCode::ILLEGAL_STATE, "Null configuration detected."); + } + + CustomAnalyzer::Builder builder; + for (const auto& filter_config : config->get_char_filter_configs()) { + builder.add_char_filter(filter_config->get_name(), filter_config->get_params()); + } + builder.with_tokenizer(config->get_tokenizer_config()->get_name(), + config->get_tokenizer_config()->get_params()); + + const auto filter_configs = config->get_token_filter_configs(); + const size_t common_grams_count = + std::count_if(filter_configs.begin(), filter_configs.end(), + [](const auto& entry) { return entry->get_name() == "common_grams"; }); + if (common_grams_count == 0) { + for (const auto& filter_config : filter_configs) { + builder.add_token_filter(filter_config->get_name(), filter_config->get_params()); + } + return builder.build(); + } + if (common_grams_count != 1 || filter_configs.back()->get_name() != "common_grams") { + throw Exception(ErrorCode::INVERTED_INDEX_ANALYZER_ERROR, + "common_grams must appear exactly once as the terminal token filter"); + } + if (builder._tokenizer->position_capability() != PositionCapability::kAlwaysUnitIncrement) { + throw Exception(ErrorCode::INVERTED_INDEX_ANALYZER_ERROR, + "CommonGrams tokenizer does not guarantee unit position increments"); + } + + for (size_t i = 0; i + 1 < filter_configs.size(); ++i) { + auto factory = AnalysisFactoryMgr::instance().create( + filter_configs[i]->get_name(), filter_configs[i]->get_params()); + if (factory->position_capability() != PositionCapability::kAlwaysUnitIncrement) { + throw Exception(ErrorCode::INVERTED_INDEX_ANALYZER_ERROR, + "CommonGrams token filter '{}' does not guarantee unit position " + "increments", + filter_configs[i]->get_name()); + } + builder._token_filters.push_back(std::move(factory)); + } + + auto common_grams = AnalysisFactoryMgr::instance().create( + filter_configs.back()->get_name(), filter_configs.back()->get_params()); + auto common_grams_factory = std::dynamic_pointer_cast(common_grams); + DORIS_CHECK(common_grams_factory != nullptr); + common_grams_factory->set_common_words(common_words); + switch (purpose) { + case AnalysisPurpose::kIndex: + common_grams_factory->set_output_mode(CommonGramsOutputMode::kEscapedV1Index); + builder._token_filters.push_back(std::move(common_grams)); + break; + case AnalysisPurpose::kSniiTransientIndex: + common_grams_factory->set_output_mode(CommonGramsOutputMode::kEscapedV1SpimiIndex); + builder._token_filters.push_back(std::move(common_grams)); + break; + case AnalysisPurpose::kPlainQuery: { + auto factory = std::make_shared(); + factory->initialize({}); + builder._token_filters.push_back(std::move(factory)); + break; + } + case AnalysisPurpose::kExactPhraseQuery: { + builder._token_filters.push_back(std::move(common_grams)); + auto factory = std::make_shared(common_words); + factory->initialize({}); + builder._token_filters.push_back(std::move(factory)); + break; + } + case AnalysisPurpose::kPhrasePrefixQuery: { + builder._token_filters.push_back(std::move(common_grams)); + auto factory = std::make_shared(common_words); + factory->initialize({}); + builder._token_filters.push_back(std::move(factory)); + break; + } + } + return builder.build(); +} + +CustomAnalyzerProvider::CustomAnalyzerProvider( + ImmutableCustomAnalyzerConfigPtr config, + std::map outer_char_filter_map) + : _config(std::move(config)), + _base_analyzer_fingerprint( + calculate_base_analyzer_fingerprint(_config, outer_char_filter_map)), + _uses_common_grams(config_uses_common_grams(_config)) { + _common_words = CommonWordSet::default_word_set(); + _analyzers = build_purpose_analyzers(_config, _common_words); + if (_uses_common_grams) { + // Content-derived, so a BE reading a segment grammed against a different word list sees a + // mismatched identity and falls back to the plain plan instead of trusting its grams. + _common_grams_identity = + build_common_grams_identity(_common_words->identity(), _base_analyzer_fingerprint); + } +} + +std::string CustomAnalyzerProvider::calculate_base_analyzer_fingerprint( + const ImmutableCustomAnalyzerConfigPtr& config, + const std::map& outer_char_filter_map) { + return calculate_base_analyzer_fingerprint_impl(config, outer_char_filter_map); +} + +std::shared_ptr CustomAnalyzerProvider::get_analyzer( + AnalysisPurpose purpose) const { + switch (purpose) { + case AnalysisPurpose::kIndex: + return _analyzers[0]; + case AnalysisPurpose::kSniiTransientIndex: + return _analyzers[1]; + case AnalysisPurpose::kPlainQuery: + return _analyzers[2]; + case AnalysisPurpose::kExactPhraseQuery: + return _analyzers[3]; + case AnalysisPurpose::kPhrasePrefixQuery: + return _analyzers[4]; + } + __builtin_unreachable(); +} + void CustomAnalyzer::Builder::with_tokenizer(const std::string& name, const Settings& params) { _tokenizer = AnalysisFactoryMgr::instance().create(name, params); } diff --git a/be/src/storage/index/inverted/analyzer/custom_analyzer.h b/be/src/storage/index/inverted/analyzer/custom_analyzer.h index 68565355496d68..9f89a442912222 100644 --- a/be/src/storage/index/inverted/analyzer/custom_analyzer.h +++ b/be/src/storage/index/inverted/analyzer/custom_analyzer.h @@ -17,6 +17,11 @@ #pragma once +#include +#include +#include + +#include "storage/index/inverted/analyzer/analyzer_provider.h" #include "storage/index/inverted/analyzer/custom_analyzer_config.h" #include "storage/index/inverted/char_filter/char_filter_factory.h" #include "storage/index/inverted/setting.h" @@ -25,6 +30,7 @@ namespace doris::segment_v2::inverted_index { +class CommonWordSet; class CustomAnalyzer; using CustomAnalyzerPtr = std::shared_ptr; @@ -60,7 +66,12 @@ class CustomAnalyzer : public Analyzer { TokenStream* tokenStream(const TCHAR* fieldName, const ReaderPtr& reader) override; TokenStream* reusableTokenStream(const TCHAR* fieldName, const ReaderPtr& reader) override; - static CustomAnalyzerPtr build_custom_analyzer(const CustomAnalyzerConfigPtr& config); + static CustomAnalyzerPtr build_custom_analyzer(const ImmutableCustomAnalyzerConfigPtr& config); + static CustomAnalyzerPtr build_custom_analyzer(const ImmutableCustomAnalyzerConfigPtr& config, + AnalysisPurpose purpose); + static CustomAnalyzerPtr build_custom_analyzer( + const ImmutableCustomAnalyzerConfigPtr& config, AnalysisPurpose purpose, + const std::shared_ptr& common_words); private: ReaderPtr init_reader(ReaderPtr reader); @@ -73,4 +84,39 @@ class CustomAnalyzer : public Analyzer { TokenStreamComponentsPtr _reuse_token_stream; }; +class CustomAnalyzerProvider final : public AnalyzerProvider { +public: + // The CommonGrams word list is not a parameter: it is the BE-local + // CommonWordSet::default_word_set(), and the dictionary identity stamped into segments comes + // from that set's content. An index policy cannot choose either one. + explicit CustomAnalyzerProvider(ImmutableCustomAnalyzerConfigPtr config, + std::map outer_char_filter_map = {}); + + std::shared_ptr get_analyzer( + AnalysisPurpose purpose) const override; + std::string_view base_analyzer_fingerprint() const override { + return _base_analyzer_fingerprint; + } + bool uses_common_grams() const override { return _uses_common_grams; } + const CommonGramsQueryIdentity* common_grams_identity() const override { + return _common_grams_identity ? &*_common_grams_identity : nullptr; + } + const CommonWordSet* common_grams_word_set() const override { + return _uses_common_grams ? _common_words.get() : nullptr; + } + const std::shared_ptr& common_words() const { return _common_words; } + + static std::string calculate_base_analyzer_fingerprint( + const ImmutableCustomAnalyzerConfigPtr& config, + const std::map& outer_char_filter_map = {}); + +private: + ImmutableCustomAnalyzerConfigPtr _config; + const std::string _base_analyzer_fingerprint; + std::shared_ptr _common_words; + bool _uses_common_grams = false; + std::optional _common_grams_identity; + std::array, 5> _analyzers; +}; + } // namespace doris::segment_v2::inverted_index \ No newline at end of file diff --git a/be/src/storage/index/inverted/analyzer/custom_analyzer_config.cpp b/be/src/storage/index/inverted/analyzer/custom_analyzer_config.cpp index b946c13e17d207..0a4c6a5da3c09e 100644 --- a/be/src/storage/index/inverted/analyzer/custom_analyzer_config.cpp +++ b/be/src/storage/index/inverted/analyzer/custom_analyzer_config.cpp @@ -27,15 +27,15 @@ CustomAnalyzerConfig::CustomAnalyzerConfig(Builder* builder) { _token_filters = builder->_token_filters; } -ComponentConfigPtr CustomAnalyzerConfig::get_tokenizer_config() { +ComponentConfigPtr CustomAnalyzerConfig::get_tokenizer_config() const { return _tokenizer_config; } -std::vector CustomAnalyzerConfig::get_char_filter_configs() { +std::vector CustomAnalyzerConfig::get_char_filter_configs() const { return _char_filters; } -std::vector CustomAnalyzerConfig::get_token_filter_configs() { +std::vector CustomAnalyzerConfig::get_token_filter_configs() const { return _token_filters; } diff --git a/be/src/storage/index/inverted/analyzer/custom_analyzer_config.h b/be/src/storage/index/inverted/analyzer/custom_analyzer_config.h index 619d9ae78c9dac..7f6d55e8d2d11a 100644 --- a/be/src/storage/index/inverted/analyzer/custom_analyzer_config.h +++ b/be/src/storage/index/inverted/analyzer/custom_analyzer_config.h @@ -29,6 +29,7 @@ using ComponentConfigPtr = std::shared_ptr; class CustomAnalyzerConfig; using CustomAnalyzerConfigPtr = std::shared_ptr; +using ImmutableCustomAnalyzerConfigPtr = std::shared_ptr; class CustomAnalyzerConfig { public: @@ -53,9 +54,9 @@ class CustomAnalyzerConfig { CustomAnalyzerConfig(Builder* builder); ~CustomAnalyzerConfig() = default; - ComponentConfigPtr get_tokenizer_config(); - std::vector get_char_filter_configs(); - std::vector get_token_filter_configs(); + ComponentConfigPtr get_tokenizer_config() const; + std::vector get_char_filter_configs() const; + std::vector get_token_filter_configs() const; private: ComponentConfigPtr _tokenizer_config; diff --git a/be/src/storage/index/inverted/analyzer/segment_analyzer_context.cpp b/be/src/storage/index/inverted/analyzer/segment_analyzer_context.cpp new file mode 100644 index 00000000000000..38b4a320fb17ed --- /dev/null +++ b/be/src/storage/index/inverted/analyzer/segment_analyzer_context.cpp @@ -0,0 +1,110 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/inverted/analyzer/segment_analyzer_context.h" + +#include + +#include "common/exception.h" +#include "runtime/index_policy/index_policy_mgr.h" + +namespace doris::segment_v2::inverted_index { +namespace { + +Result> analyzer_bypass(std::string_view reason) { + return ResultError(Status::Error( + "segment analyzer unavailable: {}", reason)); +} + +} // namespace + +Result> maybe_rebuild_segment_analyzer_context( + const InvertedIndexAnalyzerCtx* request_context, + const std::optional& segment_metadata, + const std::map& physical_index_properties, + IndexPolicyMgr* index_policy_mgr) { + return maybe_rebuild_segment_analyzer_context(request_context, + segment_metadata ? &*segment_metadata : nullptr, + physical_index_properties, index_policy_mgr); +} + +Result> maybe_rebuild_segment_analyzer_context( + const InvertedIndexAnalyzerCtx* request_context, + const CommonGramsSegmentMetadata* segment_metadata, + const std::map& physical_index_properties, + IndexPolicyMgr* index_policy_mgr) { + if (segment_metadata == nullptr) { + return std::optional {}; + } + if (segment_metadata->base_analyzer_fingerprint.empty()) { + return analyzer_bypass("typed metadata has no base analyzer fingerprint"); + } + return maybe_rebuild_segment_analyzer_context(request_context, + segment_metadata->base_analyzer_fingerprint, + physical_index_properties, index_policy_mgr); +} + +Result> maybe_rebuild_segment_analyzer_context( + const InvertedIndexAnalyzerCtx* request_context, std::string_view segment_base_fingerprint, + const std::map& physical_index_properties, + IndexPolicyMgr* index_policy_mgr) { + DORIS_CHECK(!segment_base_fingerprint.empty()); + if (request_context == nullptr || request_context->analyzer_provider == nullptr || + !request_context->should_tokenize()) { + return analyzer_bypass("query context cannot reconstruct the physical token stream"); + } + + std::string_view request_base_fingerprint = + request_context->analyzer_provider->base_analyzer_fingerprint(); + if (request_base_fingerprint.empty()) { + const auto* identity = request_context->get_common_grams_identity(); + if (identity != nullptr) { + request_base_fingerprint = identity->base_analyzer_fingerprint; + } + } + if (request_base_fingerprint == segment_base_fingerprint) { + return std::optional {}; + } + if (index_policy_mgr == nullptr) { + return analyzer_bypass("index policy manager is not initialized"); + } + + const CharFilterMap physical_char_filter_map = + get_parser_char_filter_map_from_properties(physical_index_properties); + AnalyzerProviderPtr provider; + try { + provider = index_policy_mgr->get_analyzer_provider_by_base_fingerprint( + segment_base_fingerprint, physical_char_filter_map); + } catch (const CLuceneError& error) { + return analyzer_bypass(error.what()); + } catch (const Exception& error) { + return analyzer_bypass(error.what()); + } + if (provider == nullptr) { + return analyzer_bypass("no installed policy matches the segment base fingerprint"); + } + DORIS_CHECK(provider->base_analyzer_fingerprint() == segment_base_fingerprint); + + InvertedIndexAnalyzerCtx effective_context = *request_context; + effective_context.char_filter_map = physical_char_filter_map; + effective_context.analyzer.reset(); + effective_context.analyzer_provider = std::move(provider); + effective_context.common_grams_identity.reset(); + return std::optional(std::move(effective_context)); +} + +} // namespace doris::segment_v2::inverted_index diff --git a/be/src/storage/index/inverted/analyzer/segment_analyzer_context.h b/be/src/storage/index/inverted/analyzer/segment_analyzer_context.h new file mode 100644 index 00000000000000..06be7eeb9e4811 --- /dev/null +++ b/be/src/storage/index/inverted/analyzer/segment_analyzer_context.h @@ -0,0 +1,57 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/inverted/common_grams/common_grams_segment_metadata.h" +#include "storage/index/inverted/inverted_index_parser.h" + +namespace doris { + +class IndexPolicyMgr; + +namespace segment_v2::inverted_index { + +// A missing metadata record is a legacy segment and keeps the request analyzer. A present record +// must identify its base analyzer; otherwise the caller must bypass the inverted index. +Result> maybe_rebuild_segment_analyzer_context( + const InvertedIndexAnalyzerCtx* request_context, + const std::optional& segment_metadata, + const std::map& physical_index_properties, + IndexPolicyMgr* index_policy_mgr); + +Result> maybe_rebuild_segment_analyzer_context( + const InvertedIndexAnalyzerCtx* request_context, + const CommonGramsSegmentMetadata* segment_metadata, + const std::map& physical_index_properties, + IndexPolicyMgr* index_policy_mgr); + +// Returns nullopt when the request analyzer already matches the persisted segment analyzer. +// A returned context owns a fresh provider and must remain local to one query execution. +Result> maybe_rebuild_segment_analyzer_context( + const InvertedIndexAnalyzerCtx* request_context, std::string_view segment_base_fingerprint, + const std::map& physical_index_properties, + IndexPolicyMgr* index_policy_mgr); + +} // namespace segment_v2::inverted_index +} // namespace doris diff --git a/be/src/storage/index/inverted/common/single_flight.h b/be/src/storage/index/inverted/common/single_flight.h new file mode 100644 index 00000000000000..3d360afc4a2b26 --- /dev/null +++ b/be/src/storage/index/inverted/common/single_flight.h @@ -0,0 +1,109 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/check.h" + +namespace doris::segment_v2::inverted_index { + +// Collapses concurrent operations with the same key into one execution. No work +// or follower wake-up runs while the selected shard mutex is held. +template +class SingleFlight { +public: + using ResultFuture = std::shared_future; + + std::optional join_or_lead(const std::string& key) { + auto& shard = _shard_for(key); + std::lock_guard guard(shard.mutex); + if (auto it = shard.inflight.find(key); it != shard.inflight.end()) { + return it->second->future; + } + auto flight = std::make_shared(); + flight->future = flight->promise.get_future().share(); + shard.inflight.emplace(key, std::move(flight)); + return std::nullopt; + } + + void publish(const std::string& key, Result result) { + auto& shard = _shard_for(key); + std::shared_ptr flight; + { + std::lock_guard guard(shard.mutex); + auto it = shard.inflight.find(key); + if (it == shard.inflight.end() || it->second->publishing) { + return; + } + flight = it->second; + flight->publishing = true; + } + flight->promise.set_value(std::move(result)); + { + std::lock_guard guard(shard.mutex); + auto it = shard.inflight.find(key); + DORIS_CHECK(it != shard.inflight.end()); + DORIS_CHECK(it->second == flight); + shard.inflight.erase(it); + } + } + + size_t inflight_size() const { + std::array, kShardCount> guards; + for (size_t i = 0; i < kShardCount; ++i) { + guards[i] = std::unique_lock(_shards[i].mutex); + } + + size_t size = 0; + for (const auto& shard : _shards) { + size += shard.inflight.size(); + } + return size; + } + +private: + struct Flight { + std::promise promise; + ResultFuture future; + bool publishing = false; + }; + + struct Shard { + mutable std::mutex mutex; + std::unordered_map> inflight; + }; + + static constexpr size_t kShardCount = 64; + + Shard& _shard_for(const std::string& key) { + return _shards[std::hash {}(key) % kShardCount]; + } + + std::array _shards; +}; + +} // namespace doris::segment_v2::inverted_index diff --git a/be/src/storage/index/inverted/common_grams/common_grams_key_codec.cpp b/be/src/storage/index/inverted/common_grams/common_grams_key_codec.cpp new file mode 100644 index 00000000000000..98f0e3ac918643 --- /dev/null +++ b/be/src/storage/index/inverted/common_grams/common_grams_key_codec.cpp @@ -0,0 +1,279 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/inverted/common_grams/common_grams_key_codec.h" + +#include +#include + +#include "util/utf8_check.h" + +namespace doris::segment_v2::inverted_index { +namespace { + +constexpr size_t COMMON_GRAM_LENGTH_HEX_BYTES = 8; +constexpr size_t COMMON_GRAM_SEPARATOR_BYTES = 1; +constexpr size_t COMMON_GRAM_FIXED_BYTES = + CG_V1_MARKER.size() + COMMON_GRAM_LENGTH_HEX_BYTES + COMMON_GRAM_SEPARATOR_BYTES; +constexpr std::string_view HEX_DIGITS = "0123456789abcdef"; +constexpr std::string_view LEGACY_PHRASE_BIGRAM_MARKER = + "\x1f" + "SNII_PHRASE_BIGRAM" + "\x1f"; + +ResultError analyzer_error(std::string_view message) { + return ResultError(Status::Error("{}", message)); +} + +bool prefixes_overlap(std::string_view left, std::string_view right) { + return left.starts_with(right) || right.starts_with(left); +} + +} // namespace + +Status validate_common_grams_logical_term(std::string_view term, std::string_view component) { + if (term.find('\0') != std::string_view::npos) { + return Status::Error( + "CommonGrams {} contains NUL", component); + } + if (!validate_utf8(term.data(), term.size())) { + return Status::Error( + "CommonGrams {} is not valid UTF-8", component); + } + if (term.size() > COMMON_GRAM_MAX_ENCODED_BYTES) { + return Status::Error( + "CommonGrams {} has {} UTF-8 bytes, exceeding {}", component, term.size(), + COMMON_GRAM_MAX_ENCODED_BYTES); + } + return Status::OK(); +} + +Result encode_plain_term(std::string_view term, PlainTermKeyVersion version) { + std::string encoded; + auto encoded_result = try_encode_plain_term(term, version, &encoded); + if (!encoded_result.has_value()) { + return ResultError(std::move(encoded_result.error())); + } + if (!encoded_result.value()) { + return analyzer_error("escaped plain term would exceed the 16383-byte key limit"); + } + return encoded; +} + +Result try_encode_plain_term(std::string_view term, PlainTermKeyVersion version, + std::string* output) { + DORIS_CHECK(output != nullptr); + output->clear(); + auto status = validate_common_grams_logical_term(term, "plain term"); + if (!status.ok()) { + return ResultError(std::move(status)); + } + + switch (version) { + case PlainTermKeyVersion::kLegacyRaw: + case PlainTermKeyVersion::kRawNoInternal: + output->assign(term); + return true; + case PlainTermKeyVersion::kEscapedV1: + if (term.empty() || (term.front() != PLAIN_ESCAPE_PREFIX && term.front() != '\x1f')) { + output->assign(term); + return true; + } + return try_encode_escaped_plain_term_prevalidated(term, *output); + } + return analyzer_error("unknown plain-term key version"); +} + +Result> try_encode_plain_term_view(std::string_view term, + PlainTermKeyVersion version, + std::string* scratch) { + DORIS_CHECK(scratch != nullptr); + scratch->clear(); + auto status = validate_common_grams_logical_term(term, "plain term"); + if (!status.ok()) { + return ResultError(std::move(status)); + } + + switch (version) { + case PlainTermKeyVersion::kLegacyRaw: + case PlainTermKeyVersion::kRawNoInternal: + return std::optional(term); + case PlainTermKeyVersion::kEscapedV1: + if (term.empty() || (term.front() != PLAIN_ESCAPE_PREFIX && term.front() != '\x1f')) { + return std::optional(term); + } + if (!try_encode_escaped_plain_term_prevalidated(term, *scratch)) { + return std::optional(); + } + return std::optional(*scratch); + } + return analyzer_error("unknown plain-term key version"); +} + +bool try_encode_escaped_plain_term_prevalidated(std::string_view logical_term, + std::string& output) { + DCHECK(!logical_term.empty()); + DCHECK(logical_term.front() == PLAIN_ESCAPE_PREFIX || logical_term.front() == '\x1f'); + DCHECK(validate_common_grams_logical_term(logical_term, "plain term").ok()); + output.clear(); + if (logical_term.size() == COMMON_GRAM_MAX_ENCODED_BYTES) { + return false; + } + output.reserve(logical_term.size() + 1); + output.push_back(PLAIN_ESCAPE_PREFIX); + output.push_back(logical_term.front() == PLAIN_ESCAPE_PREFIX ? 'E' : 'G'); + output.append(logical_term.substr(1)); + return true; +} + +Result decode_plain_term_view(std::string_view term, PlainTermKeyVersion version, + std::string* scratch) { + DORIS_CHECK(scratch != nullptr); + scratch->clear(); + if (term.size() > COMMON_GRAM_MAX_ENCODED_BYTES) { + return analyzer_error("encoded plain term exceeds the 16383-byte key limit"); + } + + switch (version) { + case PlainTermKeyVersion::kLegacyRaw: + case PlainTermKeyVersion::kRawNoInternal: { + auto status = validate_common_grams_logical_term(term, "plain term"); + if (!status.ok()) { + return ResultError(std::move(status)); + } + return term; + } + case PlainTermKeyVersion::kEscapedV1: { + if (term.empty() || term.front() != PLAIN_ESCAPE_PREFIX) { + if (!term.empty() && term.front() == '\x1f') { + return analyzer_error("escaped plain term enters the internal namespace"); + } + auto status = validate_common_grams_logical_term(term, "plain term"); + if (!status.ok()) { + return ResultError(std::move(status)); + } + return term; + } + if (term.size() < 2 || (term[1] != 'E' && term[1] != 'G')) { + return analyzer_error("invalid plain_term_escape:v1 key"); + } + scratch->reserve(term.size() - 1); + scratch->push_back(term[1] == 'E' ? PLAIN_ESCAPE_PREFIX : '\x1f'); + scratch->append(term.substr(2)); + auto status = validate_common_grams_logical_term(*scratch, "plain term"); + if (!status.ok()) { + return ResultError(std::move(status)); + } + return std::string_view(*scratch); + } + } + return analyzer_error("unknown plain-term key version"); +} + +Result decode_plain_term(std::string_view term, PlainTermKeyVersion version) { + std::string scratch; + auto decoded = decode_plain_term_view(term, version, &scratch); + if (!decoded.has_value()) { + return ResultError(std::move(decoded.error())); + } + return std::string(*decoded); +} + +bool is_internal_term_key(std::string_view physical_term) { + return physical_term.starts_with(INTERNAL_TERM_NAMESPACE_BEGIN); +} + +bool legacy_raw_exact_requires_bypass(std::string_view logical_term) { + return logical_term.starts_with(CG_V1_MARKER) || + logical_term.starts_with(LEGACY_PHRASE_BIGRAM_MARKER); +} + +bool legacy_raw_prefix_requires_bypass(std::string_view logical_prefix) { + return prefixes_overlap(logical_prefix, CG_V1_MARKER) || + prefixes_overlap(logical_prefix, LEGACY_PHRASE_BIGRAM_MARKER); +} + +bool is_common_gram_encodable(std::string_view left, std::string_view right) { + if (!validate_common_grams_logical_term(left, "left term").ok() || + !validate_common_grams_logical_term(right, "right term").ok()) { + return false; + } + return is_common_gram_encodable_prevalidated(left, right); +} + +bool is_common_gram_encodable_prevalidated(std::string_view left, std::string_view right) { + DCHECK(validate_common_grams_logical_term(left, "left term").ok()); + DCHECK(validate_common_grams_logical_term(right, "right term").ok()); + return common_gram_component_sizes_encodable(left.size(), right.size()); +} + +bool common_gram_component_sizes_encodable(size_t left_size, size_t right_size) { + return left_size <= COMMON_GRAM_MAX_ENCODED_BYTES - COMMON_GRAM_FIXED_BYTES && + right_size <= COMMON_GRAM_MAX_ENCODED_BYTES - COMMON_GRAM_FIXED_BYTES - left_size; +} + +Result try_encode_common_gram(std::string_view left, std::string_view right, + std::string* output) { + DORIS_CHECK(output != nullptr); + output->clear(); + auto left_status = validate_common_grams_logical_term(left, "left term"); + if (!left_status.ok()) { + return ResultError(std::move(left_status)); + } + auto right_status = validate_common_grams_logical_term(right, "right term"); + if (!right_status.ok()) { + return ResultError(std::move(right_status)); + } + return try_encode_common_gram_prevalidated(left, right, *output); +} + +bool try_encode_common_gram_prevalidated(std::string_view left, std::string_view right, + std::string& output) { + DCHECK(validate_common_grams_logical_term(left, "left term").ok()); + DCHECK(validate_common_grams_logical_term(right, "right term").ok()); + output.clear(); + if (COMMON_GRAM_FIXED_BYTES + left.size() + right.size() > COMMON_GRAM_MAX_ENCODED_BYTES) { + return false; + } + + output.reserve(COMMON_GRAM_FIXED_BYTES + left.size() + right.size()); + output.append(CG_V1_MARKER); + char length_and_separator[COMMON_GRAM_LENGTH_HEX_BYTES + COMMON_GRAM_SEPARATOR_BYTES]; + for (size_t i = 0; i < COMMON_GRAM_LENGTH_HEX_BYTES; ++i) { + const size_t shift = (COMMON_GRAM_LENGTH_HEX_BYTES - i - 1) * 4; + length_and_separator[i] = HEX_DIGITS[(left.size() >> shift) & 0xf]; + } + length_and_separator[COMMON_GRAM_LENGTH_HEX_BYTES] = ':'; + output.append(length_and_separator, sizeof(length_and_separator)); + output.append(left); + output.append(right); + return true; +} + +Result encode_common_gram(std::string_view left, std::string_view right) { + std::string encoded; + auto encoded_result = try_encode_common_gram(left, right, &encoded); + if (!encoded_result.has_value()) { + return ResultError(std::move(encoded_result.error())); + } + if (!encoded_result.value()) { + return analyzer_error("encoded common gram would exceed the 16383-byte key limit"); + } + return encoded; +} + +} // namespace doris::segment_v2::inverted_index diff --git a/be/src/storage/index/inverted/common_grams/common_grams_key_codec.h b/be/src/storage/index/inverted/common_grams/common_grams_key_codec.h new file mode 100644 index 00000000000000..07869877a55d01 --- /dev/null +++ b/be/src/storage/index/inverted/common_grams/common_grams_key_codec.h @@ -0,0 +1,72 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include + +#include "common/status.h" + +namespace doris::segment_v2::inverted_index { + +inline constexpr size_t COMMON_GRAM_MAX_ENCODED_BYTES = 16383; +inline constexpr char PLAIN_ESCAPE_PREFIX = '\x1e'; +inline constexpr std::string_view INTERNAL_TERM_NAMESPACE_BEGIN {"\x1f", 1}; +inline constexpr std::string_view INTERNAL_TERM_NAMESPACE_END {"\x20", 1}; +inline constexpr std::string_view CG_V1_MARKER = + "\x1f" + "DORIS_COMMON_GRAM_V1" + "\x1f"; +inline constexpr std::string_view CG_V1_MARKER_END = + "\x1f" + "DORIS_COMMON_GRAM_V1" + "\x20"; + +enum class PlainTermKeyVersion : uint8_t { + kLegacyRaw = 0, + kEscapedV1 = 1, + kRawNoInternal = 2, +}; + +Result encode_plain_term(std::string_view term, PlainTermKeyVersion version); +Result try_encode_plain_term(std::string_view term, PlainTermKeyVersion version, + std::string* output); +Result> try_encode_plain_term_view(std::string_view term, + PlainTermKeyVersion version, + std::string* scratch); +bool try_encode_escaped_plain_term_prevalidated(std::string_view logical_term, std::string& output); +Result decode_plain_term_view(std::string_view term, PlainTermKeyVersion version, + std::string* scratch); +Result decode_plain_term(std::string_view term, PlainTermKeyVersion version); +bool is_internal_term_key(std::string_view physical_term); +bool legacy_raw_exact_requires_bypass(std::string_view logical_term); +bool legacy_raw_prefix_requires_bypass(std::string_view logical_prefix); +Status validate_common_grams_logical_term(std::string_view term, std::string_view component); +Result try_encode_common_gram(std::string_view left, std::string_view right, + std::string* output); +bool try_encode_common_gram_prevalidated(std::string_view left, std::string_view right, + std::string& output); +bool common_gram_component_sizes_encodable(size_t left_size, size_t right_size); +bool is_common_gram_encodable_prevalidated(std::string_view left, std::string_view right); +Result encode_common_gram(std::string_view left, std::string_view right); +bool is_common_gram_encodable(std::string_view left, std::string_view right); + +} // namespace doris::segment_v2::inverted_index diff --git a/be/src/storage/index/inverted/common_grams/common_grams_query_cost.h b/be/src/storage/index/inverted/common_grams/common_grams_query_cost.h new file mode 100644 index 00000000000000..2738c52150a705 --- /dev/null +++ b/be/src/storage/index/inverted/common_grams/common_grams_query_cost.h @@ -0,0 +1,53 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +namespace doris::segment_v2::inverted_index { + +struct CommonGramsPlanRawCost { + uint64_t posting_bytes_or_df_sum = 0; + uint64_t estimated_candidate_df = 0; + uint32_t clause_count = 0; +}; + +struct CommonGramsPlanCostModel { + uint32_t position_verify_factor = 0; + uint32_t common_grams_cost_ratio_percent = 85; + uint64_t generation = 0; +}; + +inline uint64_t estimate_common_grams_plan_cost(const CommonGramsPlanRawCost& input, + uint32_t position_verify_factor) { + const unsigned __int128 estimate = + static_cast(input.posting_bytes_or_df_sum) + + static_cast(input.estimated_candidate_df) * position_verify_factor * + input.clause_count; + return estimate > std::numeric_limits::max() ? std::numeric_limits::max() + : static_cast(estimate); +} + +inline bool common_grams_plan_cost_wins(uint64_t plain_cost, uint64_t common_grams_cost, + uint32_t common_grams_cost_ratio_percent) { + return static_cast(common_grams_cost) * 100 <= + static_cast(plain_cost) * common_grams_cost_ratio_percent; +} + +} // namespace doris::segment_v2::inverted_index diff --git a/be/src/storage/index/inverted/common_grams/common_grams_segment_metadata.cpp b/be/src/storage/index/inverted/common_grams/common_grams_segment_metadata.cpp new file mode 100644 index 00000000000000..c8dcddecf70608 --- /dev/null +++ b/be/src/storage/index/inverted/common_grams/common_grams_segment_metadata.cpp @@ -0,0 +1,158 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/inverted/common_grams/common_grams_segment_metadata.h" + +namespace doris::segment_v2::inverted_index { + +CommonGramsSegmentMetadata make_common_grams_segment_metadata( + const CommonGramsQueryIdentity& identity) { + CommonGramsSegmentMetadata metadata; + metadata.plain_term_key_version = PlainTermKeyVersion::kEscapedV1; + metadata.common_grams_coverage = CommonGramsCoverage::kComplete; + metadata.common_grams_semantics_version = COMMON_GRAMS_SEMANTICS_VERSION_V1; + metadata.common_grams_key_version = COMMON_GRAMS_KEY_VERSION_V1; + metadata.common_grams_dictionary_identity = identity.common_grams_dictionary_identity; + metadata.base_analyzer_fingerprint = identity.base_analyzer_fingerprint; + metadata.common_grams_fingerprint = identity.common_grams_fingerprint; + metadata.scoring_coverage = ScoringCoverage::kComplete; + metadata.scoring_stats_version = COMMON_GRAMS_SCORING_STATS_VERSION_V1; + metadata.norm_semantics_version = COMMON_GRAMS_NORM_SEMANTICS_VERSION_V1; + return metadata; +} + +bool common_grams_identity_matches(const CommonGramsSegmentMetadata& metadata, + const CommonGramsQueryIdentity& identity) { + return metadata.common_grams_dictionary_identity == identity.common_grams_dictionary_identity && + metadata.base_analyzer_fingerprint == identity.base_analyzer_fingerprint && + metadata.common_grams_fingerprint == identity.common_grams_fingerprint; +} + +Status validate_common_grams_segment_metadata(const CommonGramsSegmentMetadata& metadata) { + switch (metadata.plain_term_key_version) { + case PlainTermKeyVersion::kLegacyRaw: + case PlainTermKeyVersion::kEscapedV1: + case PlainTermKeyVersion::kRawNoInternal: + break; + default: + return Status::Error( + "common_grams_metadata: invalid plain-term key version"); + } + + switch (metadata.common_grams_coverage) { + case CommonGramsCoverage::kNone: + case CommonGramsCoverage::kComplete: + case CommonGramsCoverage::kMixed: + break; + default: + return Status::Error( + "common_grams_metadata: invalid coverage"); + } + + switch (metadata.scoring_coverage) { + case ScoringCoverage::kNone: + case ScoringCoverage::kComplete: + break; + default: + return Status::Error( + "common_grams_metadata: invalid scoring coverage"); + } + + if (metadata.plain_term_key_version == PlainTermKeyVersion::kRawNoInternal && + metadata.common_grams_coverage != CommonGramsCoverage::kNone) { + return Status::Error( + "common_grams_metadata: raw-no-internal segment has gram coverage"); + } + + if (metadata.common_grams_coverage == CommonGramsCoverage::kComplete && + (metadata.plain_term_key_version != PlainTermKeyVersion::kEscapedV1 || + metadata.common_grams_semantics_version == 0 || metadata.common_grams_key_version == 0 || + metadata.common_grams_dictionary_identity.empty() || + metadata.base_analyzer_fingerprint.empty() || metadata.common_grams_fingerprint.empty())) { + return Status::Error( + "common_grams_metadata: incomplete complete-coverage identity"); + } + + if (metadata.scoring_coverage == ScoringCoverage::kComplete && + (metadata.scoring_stats_version == 0 || metadata.norm_semantics_version == 0 || + metadata.base_analyzer_fingerprint.empty())) { + return Status::Error( + "common_grams_metadata: incomplete scoring identity"); + } + return Status::OK(); +} + +Status validate_snii_scoring_metadata(const CommonGramsSegmentMetadata* metadata, + uint64_t physical_doc_count, + uint64_t physical_sum_total_term_freq, bool has_scoring_tier, + bool has_positions, bool has_norms) { + if (metadata == nullptr) { + return Status::Error( + "SNII semantic scoring metadata is missing"); + } + RETURN_IF_ERROR(validate_common_grams_segment_metadata(*metadata)); + if (metadata->scoring_coverage != ScoringCoverage::kComplete || + metadata->scoring_stats_version != COMMON_GRAMS_SCORING_STATS_VERSION_V1 || + metadata->norm_semantics_version != COMMON_GRAMS_NORM_SEMANTICS_VERSION_V1) { + return Status::Error( + "SNII scoring metadata uses unsupported semantics"); + } + if (!has_scoring_tier || !has_positions || !has_norms) { + return Status::Error( + "SNII complete scoring metadata requires the scoring tier, positions, and norms"); + } + if (metadata->scoring_doc_count != physical_doc_count) { + return Status::Error( + "SNII semantic scoring document count {} differs from physical document count {}", + metadata->scoring_doc_count, physical_doc_count); + } + if (metadata->scoring_token_count > physical_sum_total_term_freq) { + return Status::Error( + "SNII semantic scoring token count {} exceeds physical term frequency {}", + metadata->scoring_token_count, physical_sum_total_term_freq); + } + if (metadata->plain_term_key_version == PlainTermKeyVersion::kRawNoInternal && + metadata->common_grams_coverage == CommonGramsCoverage::kNone && + metadata->scoring_token_count != physical_sum_total_term_freq) { + return Status::Error( + "SNII semantic plain token count {} differs from physical term frequency {}", + metadata->scoring_token_count, physical_sum_total_term_freq); + } + if (physical_sum_total_term_freq != 0 && metadata->scoring_token_count == 0) { + return Status::Error( + "SNII non-empty physical postings have zero semantic scoring tokens"); + } + return Status::OK(); +} + +bool is_common_grams_query_compatible(const CommonGramsSegmentMetadata& metadata, + const CommonGramsQueryIdentity& identity) { + return is_common_grams_query_compatible(metadata, identity, CommonGramsCoverage::kComplete); +} + +bool is_common_grams_query_compatible(const CommonGramsSegmentMetadata& metadata, + const CommonGramsQueryIdentity& identity, + CommonGramsCoverage required_coverage) { + return metadata.plain_term_key_version == PlainTermKeyVersion::kEscapedV1 && + metadata.common_grams_coverage == required_coverage && + required_coverage != CommonGramsCoverage::kNone && + metadata.common_grams_semantics_version == COMMON_GRAMS_SEMANTICS_VERSION_V1 && + metadata.common_grams_key_version == COMMON_GRAMS_KEY_VERSION_V1 && + common_grams_identity_matches(metadata, identity); +} + +} // namespace doris::segment_v2::inverted_index diff --git a/be/src/storage/index/inverted/common_grams/common_grams_segment_metadata.h b/be/src/storage/index/inverted/common_grams/common_grams_segment_metadata.h new file mode 100644 index 00000000000000..b5864bd340fc87 --- /dev/null +++ b/be/src/storage/index/inverted/common_grams/common_grams_segment_metadata.h @@ -0,0 +1,87 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include "common/status.h" +#include "storage/index/inverted/common_grams/common_grams_key_codec.h" + +namespace doris::segment_v2::inverted_index { + +inline constexpr uint32_t COMMON_GRAMS_SEGMENT_METADATA_VERSION = 1; +inline constexpr uint32_t COMMON_GRAMS_SEMANTICS_VERSION_V1 = 1; +inline constexpr uint32_t COMMON_GRAMS_KEY_VERSION_V1 = 1; +inline constexpr uint32_t COMMON_GRAMS_SCORING_STATS_VERSION_V1 = 1; +inline constexpr uint32_t COMMON_GRAMS_NORM_SEMANTICS_VERSION_V1 = 1; + +enum class CommonGramsCoverage : uint8_t { + kNone = 0, + kComplete = 1, + kMixed = 2, +}; + +enum class ScoringCoverage : uint8_t { + kNone = 0, + kComplete = 1, +}; + +// SNII metadata. A missing record is a legacy segment; a present record must +// validate before any capability is used. +struct CommonGramsSegmentMetadata { + PlainTermKeyVersion plain_term_key_version = PlainTermKeyVersion::kLegacyRaw; + CommonGramsCoverage common_grams_coverage = CommonGramsCoverage::kNone; + uint32_t common_grams_semantics_version = 0; + uint32_t common_grams_key_version = 0; + std::string common_grams_dictionary_identity; + std::string base_analyzer_fingerprint; + std::string common_grams_fingerprint; + ScoringCoverage scoring_coverage = ScoringCoverage::kNone; + uint32_t scoring_stats_version = 0; + uint32_t norm_semantics_version = 0; + uint64_t scoring_doc_count = 0; + uint64_t scoring_token_count = 0; + + bool operator==(const CommonGramsSegmentMetadata&) const = default; +}; + +struct CommonGramsQueryIdentity { + std::string common_grams_dictionary_identity; + std::string base_analyzer_fingerprint; + std::string common_grams_fingerprint; + + bool operator==(const CommonGramsQueryIdentity&) const = default; +}; + +CommonGramsSegmentMetadata make_common_grams_segment_metadata( + const CommonGramsQueryIdentity& identity); +bool common_grams_identity_matches(const CommonGramsSegmentMetadata& metadata, + const CommonGramsQueryIdentity& identity); +Status validate_common_grams_segment_metadata(const CommonGramsSegmentMetadata& metadata); +Status validate_snii_scoring_metadata(const CommonGramsSegmentMetadata* metadata, + uint64_t physical_doc_count, + uint64_t physical_sum_total_term_freq, bool has_scoring_tier, + bool has_positions, bool has_norms); +bool is_common_grams_query_compatible(const CommonGramsSegmentMetadata& metadata, + const CommonGramsQueryIdentity& identity); +bool is_common_grams_query_compatible(const CommonGramsSegmentMetadata& metadata, + const CommonGramsQueryIdentity& identity, + CommonGramsCoverage required_coverage); + +} // namespace doris::segment_v2::inverted_index diff --git a/be/src/storage/index/inverted/common_grams/common_word_set.cpp b/be/src/storage/index/inverted/common_grams/common_word_set.cpp new file mode 100644 index 00000000000000..4e74cac590778d --- /dev/null +++ b/be/src/storage/index/inverted/common_grams/common_word_set.cpp @@ -0,0 +1,197 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/inverted/common_grams/common_word_set.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/config.h" +#include "common/logging.h" +#include "storage/index/inverted/common_grams/common_grams_key_codec.h" +#include "util/md5.h" +#include "util/utf8_check.h" + +namespace doris::segment_v2::inverted_index { +namespace { + +#ifdef BE_TEST +std::atomic g_common_word_membership_lookups {0}; +std::atomic g_common_word_hash_lookups {0}; +#endif + +ResultError wordset_error(std::string_view message) { + return ResultError(Status::Error("{}", message)); +} + +} // namespace + +CommonWordSet::CommonWordSet(WordContainer words, std::string identity) + : _words(std::move(words)), _identity(std::move(identity)) { + DORIS_CHECK(!_identity.empty()); + for (const std::string& word : _words) { + DORIS_CHECK(!word.empty()); + DORIS_CHECK_LE(word.size(), std::numeric_limits::max()); + const auto bytes = static_cast(word.size()); + _min_word_bytes = std::min(_min_word_bytes, bytes); + _max_word_bytes = std::max(_max_word_bytes, bytes); + const uint8_t first = static_cast(word.front()); + _first_byte_mask[first >> 6] |= uint64_t {1} << (first & 63); + } +} + +const CommonWordSet& CommonWordSet::builtin_english_stop_words_v1() { + static const CommonWordSet words( + WordContainer { + "a", "an", "and", "are", "as", "at", "be", "but", "by", + "for", "if", "in", "into", "is", "it", "no", "not", "of", + "on", "or", "such", "that", "the", "their", "then", "there", "these", + "they", "this", "to", "was", "will", "with", + }, + std::string(BUILTIN_COMMON_WORDS_RESOURCE)); + return words; +} + +Result CommonWordSet::parse_words(std::string_view content) { + // Digest first: the parse loop below advances `content`, so hashing it afterwards would hash + // an empty view. Content-derived so a segment records exactly which list grammed it, and + // digesting the raw bytes (not the parsed set) means a comment-only edit also yields a new + // identity -- the safe direction, since that re-plans instead of risking a stale match. + Md5Digest digest; + digest.update(content.data(), content.size()); + digest.digest(); + const std::string identity = "wordset:md5:" + digest.hex(); + + if (content.find('\0') != std::string_view::npos) { + return wordset_error("CommonGrams word list contains NUL"); + } + if (!validate_utf8(content.data(), content.size())) { + return wordset_error("CommonGrams word list is not valid UTF-8"); + } + + WordContainer words; + while (!content.empty()) { + const size_t newline = content.find('\n'); + std::string_view term = content.substr(0, newline); + if (newline != std::string_view::npos && !term.empty() && term.back() == '\r') { + term.remove_suffix(1); + } + if (!term.empty() && term.front() != '#') { + if (term.size() > COMMON_GRAM_MAX_ENCODED_BYTES) { + return wordset_error("wordset:v1 term exceeds the 16383-byte token limit"); + } + words.emplace(term); + } + if (newline == std::string_view::npos) { + break; + } + content.remove_prefix(newline + 1); + } + return CommonWordSet(std::move(words), identity); +} + +std::string CommonWordSet::default_word_set_path() { + return config::inverted_index_dict_path + "/common_grams/default_words.txt"; +} + +std::shared_ptr CommonWordSet::default_word_set() { + // Lazy singleton: the word list is immutable for the process, so every analyzer shares one + // copy and the fallback warning below is emitted at most once no matter how many analyzers + // are built. A later edit to inverted_index_dict_path is therefore ignored, which is intended + // -- swapping the list under live segments would change what their grams mean. + static const std::shared_ptr instance = [] { + auto builtin = [] { + return std::shared_ptr(&builtin_english_stop_words_v1(), + [](const CommonWordSet*) {}); + }; + const std::string path = default_word_set_path(); + std::ifstream input(path, std::ios::binary); + if (!input.is_open()) { + // Expected on a stock install: shipping no file means "use the built-in list", so this + // is INFO. A file that exists but cannot be read or parsed is an operator mistake and + // warns below. + LOG(INFO) << "No CommonGrams word list at " << path + << ", using the built-in English stop words"; + return builtin(); + } + std::ostringstream buffer; + buffer << input.rdbuf(); + if (input.bad()) { + LOG(WARNING) << "CommonGrams word list at " << path + << " could not be read, falling back to the built-in English stop words"; + return builtin(); + } + const std::string content = buffer.str(); + auto parsed = parse_words(content); + if (!parsed.has_value()) { + LOG(WARNING) << "CommonGrams word list at " << path << " is invalid: " << parsed.error() + << ", falling back to the built-in English stop words"; + return builtin(); + } + LOG(INFO) << "Loaded " << parsed.value().size() << " CommonGrams words from " << path; + return std::shared_ptr( + std::make_shared(std::move(parsed.value()))); + }(); + return instance; +} + +bool CommonWordSet::contains(std::string_view term) const { +#ifdef BE_TEST + g_common_word_membership_lookups.fetch_add(1, std::memory_order_relaxed); +#endif + if (term.size() < _min_word_bytes || term.size() > _max_word_bytes) { + return false; + } + const auto first = static_cast(term.front()); + if ((_first_byte_mask[first >> 6] & (uint64_t {1} << (first & 63))) == 0) { + return false; + } +#ifdef BE_TEST + g_common_word_hash_lookups.fetch_add(1, std::memory_order_relaxed); +#endif + return _words.contains(term); +} + +#ifdef BE_TEST +namespace common_grams_testing { + +uint64_t common_word_membership_lookup_count() { + return g_common_word_membership_lookups.load(std::memory_order_relaxed); +} + +void reset_common_word_membership_lookup_count() { + g_common_word_membership_lookups.store(0, std::memory_order_relaxed); +} + +uint64_t common_word_hash_lookup_count() { + return g_common_word_hash_lookups.load(std::memory_order_relaxed); +} + +void reset_common_word_hash_lookup_count() { + g_common_word_hash_lookups.store(0, std::memory_order_relaxed); +} + +} // namespace common_grams_testing +#endif + +} // namespace doris::segment_v2::inverted_index diff --git a/be/src/storage/index/inverted/common_grams/common_word_set.h b/be/src/storage/index/inverted/common_grams/common_word_set.h new file mode 100644 index 00000000000000..b6aac21f48b501 --- /dev/null +++ b/be/src/storage/index/inverted/common_grams/common_word_set.h @@ -0,0 +1,94 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/status.h" + +namespace doris::segment_v2::inverted_index { + +inline constexpr std::string_view BUILTIN_COMMON_WORDS_RESOURCE = "builtin:lucene_english_stop:v1"; +inline constexpr std::string_view WORDSET_FORMAT_V1 = "wordset:v1"; + +class CommonWordSet { +public: + static const CommonWordSet& builtin_english_stop_words_v1(); + + // Parses a newline-separated word list. Blank lines and '#' comments are skipped. + static Result parse_words(std::string_view content); + + // The BE-local word list every CommonGrams analyzer grams against, read once from + // /common_grams/default_words.txt -- the same layout the icu, ik and + // pinyin dictionaries use. Falls back to builtin_english_stop_words_v1() when the file is + // absent or unparseable. The set is deliberately not selectable per index policy: every replica + // of a tablet must gram identically, and a policy-supplied list would have to be distributed + // and acknowledged before any index could use it. + static std::shared_ptr default_word_set(); + + // Exposed so tests can assert the layout without duplicating the literal. + static std::string default_word_set_path(); + + // Stamped into a segment's CommonGrams metadata and compared against the querying analyzer's + // identity, so a segment is only read with gram expectations that match how it was written. + // Because the word list is now a BE-local file, this MUST derive from the content: two BEs + // pointed at different files, or one BE after the file is edited, would otherwise stamp the + // same identity onto incompatible segments and silently mis-plan phrase queries. + const std::string& identity() const { return _identity; } + + bool contains(std::string_view term) const; + size_t size() const { return _words.size(); } + +private: + struct TransparentStringHash { + using is_transparent = void; + + size_t operator()(std::string_view value) const { + return std::hash {}(value); + } + }; + + using WordContainer = phmap::flat_hash_set>; + + CommonWordSet(WordContainer words, std::string identity); + + WordContainer _words; + // Never empty. See identity() for why this is content-derived rather than a fixed constant. + std::string _identity; + std::array _first_byte_mask {}; + uint16_t _min_word_bytes = std::numeric_limits::max(); + uint16_t _max_word_bytes = 0; +}; + +#ifdef BE_TEST +namespace common_grams_testing { +uint64_t common_word_membership_lookup_count(); +void reset_common_word_membership_lookup_count(); +} // namespace common_grams_testing +#endif + +} // namespace doris::segment_v2::inverted_index diff --git a/be/src/storage/index/inverted/inverted_index_cache.cpp b/be/src/storage/index/inverted/inverted_index_cache.cpp index e759a0163ac7d3..402637c3798c6f 100644 --- a/be/src/storage/index/inverted/inverted_index_cache.cpp +++ b/be/src/storage/index/inverted/inverted_index_cache.cpp @@ -27,10 +27,50 @@ #include "runtime/exec_env.h" #include "runtime/thread_context.h" +#include "util/coding.h" #include "util/defer_op.h" namespace doris::segment_v2 { +namespace { + +void append_length_prefixed(std::string_view value, std::string* output) { + put_fixed64_le(output, value.size()); + output->append(value); +} + +} // namespace + +std::string InvertedIndexRawQuerySemantic::encode() const { + std::string output; + output.reserve(sizeof(cache_semantics_version) + sizeof(uint64_t) + raw_query_bytes.size() + + sizeof(query_type) + sizeof(slop) + sizeof(ordered) + sizeof(max_expansions) + + sizeof(common_grams_cache_generation) + sizeof(uint64_t)); + put_fixed32_le(&output, cache_semantics_version); + append_length_prefixed(raw_query_bytes, &output); + put_fixed32_le(&output, static_cast(query_type)); + put_fixed32_le(&output, static_cast(slop)); + output.push_back(static_cast(ordered)); + put_fixed32_le(&output, static_cast(max_expansions)); + put_fixed64_le(&output, common_grams_cache_generation); + return output; +} + +std::string InvertedIndexQueryCache::CacheKey::encode() const { + if (query_type_to_string(query_type).empty()) { + return {}; + } + std::string output; + const std::string index_path_string = index_path.string(); + output.reserve(3 * sizeof(uint64_t) + index_path_string.size() + column_name.size() + + sizeof(query_type) + value.size()); + append_length_prefixed(index_path_string, &output); + append_length_prefixed(column_name, &output); + put_fixed32_le(&output, static_cast(query_type)); + append_length_prefixed(value, &output); + return output; +} + InvertedIndexSearcherCache* InvertedIndexSearcherCache::create_global_instance( size_t capacity, uint32_t num_shards) { return new InvertedIndexSearcherCache(capacity, num_shards); diff --git a/be/src/storage/index/inverted/inverted_index_cache.h b/be/src/storage/index/inverted/inverted_index_cache.h index 0e33fe747b3523..476a44fae7b26e 100644 --- a/be/src/storage/index/inverted/inverted_index_cache.h +++ b/be/src/storage/index/inverted/inverted_index_cache.h @@ -26,6 +26,7 @@ #include #include #include +#include #include "common/config.h" #include "common/status.h" @@ -35,6 +36,7 @@ #include "runtime/memory/lru_cache_policy.h" #include "runtime/memory/mem_tracker.h" #include "storage/index/inverted/inverted_index_searcher.h" +#include "storage/index/snii/reader/logical_index_reader.h" #include "util/lru_cache.h" #include "util/slice.h" #include "util/time.h" @@ -42,6 +44,7 @@ namespace doris { namespace segment_v2 { class InvertedIndexCacheHandle; +class IndexFileReader; class InvertedIndexSearcherCache { public: @@ -56,6 +59,8 @@ class InvertedIndexSearcherCache { class CacheValue : public LRUCacheValueBase { public: IndexSearcherPtr index_searcher; + std::shared_ptr snii_index_file_reader; + std::unique_ptr snii_logical_reader; size_t size = 0; int64_t last_visit_time; @@ -65,6 +70,14 @@ class InvertedIndexSearcherCache { size = mem_size; last_visit_time = visit_time; } + explicit CacheValue(std::unique_ptr logical_reader, + size_t mem_size, int64_t visit_time, + std::shared_ptr index_file_reader) + : snii_index_file_reader(std::move(index_file_reader)), + snii_logical_reader(std::move(logical_reader)) { + size = mem_size; + last_visit_time = visit_time; + } }; // Create global instance of this class. // "capacity" is the capacity of lru cache. @@ -166,6 +179,11 @@ class InvertedIndexCacheHandle { return ((InvertedIndexSearcherCache::CacheValue*)_cache->value(_handle))->index_searcher; } + doris::snii::reader::LogicalIndexReader* get_snii_logical_reader() { + return ((InvertedIndexSearcherCache::CacheValue*)_cache->value(_handle)) + ->snii_logical_reader.get(); + } + InvertedIndexSearcherCache::CacheValue* get_index_cache_value() { return ((InvertedIndexSearcherCache::CacheValue*)_cache->value(_handle)); } @@ -180,6 +198,23 @@ class InvertedIndexCacheHandle { class InvertedIndexQueryCacheHandle; +inline constexpr uint32_t INVERTED_INDEX_QUERY_CACHE_SEMANTICS_VERSION = 1; + +// Stable identity shared by result-cache and row-accurate single-flight. It intentionally contains +// no analyzer output or internal plan kind: those are segment-local implementation details below +// the cache lookup. +struct InvertedIndexRawQuerySemantic { + std::string_view raw_query_bytes; + InvertedIndexQueryType query_type; + int32_t slop = 0; + bool ordered = false; + int32_t max_expansions = 0; + uint32_t cache_semantics_version = INVERTED_INDEX_QUERY_CACHE_SEMANTICS_VERSION; + uint64_t common_grams_cache_generation = 0; + + std::string encode() const; +}; + class InvertedIndexQueryCache : public LRUCachePolicy { public: using LRUCachePolicy::insert; @@ -191,21 +226,8 @@ class InvertedIndexQueryCache : public LRUCachePolicy { InvertedIndexQueryType query_type; // query type std::string value; // query value - // Encode to a flat binary which can be used as LRUCache's key - std::string encode() const { - std::string key_buf(index_path.string()); - key_buf.append("/"); - key_buf.append(column_name); - key_buf.append("/"); - auto query_type_str = query_type_to_string(query_type); - if (query_type_str.empty()) { - return ""; - } - key_buf.append(query_type_str); - key_buf.append("/"); - key_buf.append(value); - return key_buf; - } + // Encode to an unambiguous flat binary which can be used as LRUCache's key. + std::string encode() const; }; class CacheValue : public LRUCacheValueBase { diff --git a/be/src/storage/index/inverted/inverted_index_fs_directory.cpp b/be/src/storage/index/inverted/inverted_index_fs_directory.cpp index 30f168e8b14e02..e332e3cde8069e 100644 --- a/be/src/storage/index/inverted/inverted_index_fs_directory.cpp +++ b/be/src/storage/index/inverted/inverted_index_fs_directory.cpp @@ -97,11 +97,22 @@ bool DorisFSDirectory::FSIndexInput::open(const io::FileSystemSPtr& fs, const ch auto h = std::make_shared(path); io::FileReaderOptions reader_options; - reader_options.cache_type = config::enable_file_cache ? io::FileCachePolicy::FILE_BLOCK_CACHE - : io::FileCachePolicy::NO_CACHE; + // DIAGNOSTIC: inverted_index_read_bypass_file_cache forces NO_CACHE (precise + // S3 range GETs) for inverted-index reads (both CLucene here and SNII), so a + // fair direct-vs-direct engine comparison can bypass the 1MiB block cache + // without disabling the global enable_file_cache (cloud mode requires it). + reader_options.cache_type = + (config::enable_file_cache && !config::inverted_index_read_bypass_file_cache) + ? io::FileCachePolicy::FILE_BLOCK_CACHE + : io::FileCachePolicy::NO_CACHE; reader_options.is_doris_table = true; reader_options.file_size = file_size; reader_options.tablet_id = tablet_id; + // NO_CACHE on a remote fs means every range GET bypasses CachedRemoteFileReader, + // the only layer that normally accounts physical remote bytes; readInternal then + // counts them itself. Local NO_CACHE reads must stay excluded. + h->_direct_remote_io = reader_options.cache_type == io::FileCachePolicy::NO_CACHE && + fs->type() != io::FileSystemType::LOCAL; Status st = fs->open_file(path, &h->_reader, &reader_options); DBUG_EXECUTE_IF("inverted file read error: index file not found", { st = Status::Error("index file not found"); }) @@ -179,20 +190,19 @@ void DorisFSDirectory::FSIndexInput::close() { } void DorisFSDirectory::FSIndexInput::setIoContext(const void* io_ctx) { + // Copy the caller's full IOContext (expiration_time for TTL cache classification, + // is_warmup, is_disposable, read_file_cache, bypass_peer_read, the miss policy and + // the remote-scan cache-write limiter all propagate); a field whitelist here silently + // drops newly added flags. Only the per-stream identity bits are re-stamped below. + const bool is_index_data = _io_ctx.is_index_data; if (io_ctx) { const auto& ctx = static_cast(io_ctx); - _io_ctx.reader_type = ctx->reader_type; - _io_ctx.query_id = ctx->query_id; - _io_ctx.file_cache_stats = ctx->file_cache_stats; - _io_ctx.file_cache_miss_policy = ctx->file_cache_miss_policy; - _io_ctx.remote_scan_cache_write_limiter = ctx->remote_scan_cache_write_limiter; + _io_ctx = *ctx; } else { - _io_ctx.reader_type = ReaderType::UNKNOWN; - _io_ctx.query_id = nullptr; - _io_ctx.file_cache_stats = nullptr; - _io_ctx.file_cache_miss_policy = io::FileCacheMissPolicy::READ_THROUGH_AND_WRITE_BACK; - _io_ctx.remote_scan_cache_write_limiter = nullptr; + _io_ctx = io::IOContext {}; } + _io_ctx.is_index_data = is_index_data; + _io_ctx.is_inverted_index = true; } const void* DorisFSDirectory::FSIndexInput::getIoContext() { @@ -251,6 +261,15 @@ void DorisFSDirectory::FSIndexInput::readInternal(uint8_t* b, const int32_t len) if (_io_ctx.file_cache_stats != nullptr) { _io_ctx.file_cache_stats->inverted_index_io_timer += inverted_index_io_timer; + _io_ctx.file_cache_stats->inverted_index_request_bytes += len; + _io_ctx.file_cache_stats->inverted_index_read_bytes += len; + ++_io_ctx.file_cache_stats->inverted_index_range_read_count; + ++_io_ctx.file_cache_stats->inverted_index_serial_read_rounds; + if (_handle->_direct_remote_io) { + // Cache-bypassed remote read: no CachedRemoteFileReader below to count + // the GET, so account the physical remote bytes here. + _io_ctx.file_cache_stats->inverted_index_remote_physical_read_bytes += len; + } } } diff --git a/be/src/storage/index/inverted/inverted_index_fs_directory.h b/be/src/storage/index/inverted/inverted_index_fs_directory.h index 79854df88d235d..64719880e7caa9 100644 --- a/be/src/storage/index/inverted/inverted_index_fs_directory.h +++ b/be/src/storage/index/inverted/inverted_index_fs_directory.h @@ -170,6 +170,11 @@ class DorisFSDirectory::FSIndexInput : public lucene::store::BufferedIndexInput std::mutex _shared_lock; //std::mutex* _shared_lock = nullptr; char path[4096]; + // True when _reader serves ranges straight from remote storage with no + // CachedRemoteFileReader in between (NO_CACHE on a non-local fs): only + // that layer would normally account physical remote bytes, so + // readInternal then counts them itself. + bool _direct_remote_io = false; SharedHandle(const char* path); ~SharedHandle() override; }; diff --git a/be/src/storage/index/inverted/inverted_index_iterator.cpp b/be/src/storage/index/inverted/inverted_index_iterator.cpp index 6e93d70d025964..6fbaa5bc944ec3 100644 --- a/be/src/storage/index/inverted/inverted_index_iterator.cpp +++ b/be/src/storage/index/inverted/inverted_index_iterator.cpp @@ -46,12 +46,13 @@ void InvertedIndexIterator::add_reader(InvertedIndexReaderType type, VLOG_DEBUG << "InvertedIndexIterator add_reader: type=" << static_cast(type) << ", analyzer_key=" << analyzer_key; - const size_t entry_index = _reader_entries.size(); - _reader_entries.push_back( - ReaderEntry {.type = type, .analyzer_key = std::move(analyzer_key), .reader = reader}); - - // Update index for O(1) lookup - _key_to_entries[_reader_entries.back().analyzer_key].push_back(entry_index); + auto status = add_inverted_index_selection_candidate( + InvertedIndexSelectionCandidate {.index_id = cast_set(reader->get_index_id()), + .reader_type = type, + .analyzer_key = std::move(analyzer_key)}, + &_selection_candidates, &_key_to_entries); + DORIS_CHECK(status.ok()) << status; + _readers.push_back(reader); } Status InvertedIndexIterator::read_from_index(const IndexParam& param) { @@ -101,6 +102,11 @@ Status InvertedIndexIterator::read_from_index(const IndexParam& param) { // Note: analyzer_ctx is now passed via i_param->analyzer_ctx auto execute_query = [&]() { + if (i_param->null_bitmap_cache_handle != nullptr) { + return reader->query_with_null_bitmap( + _context, i_param->column_name, i_param->query_value, i_param->query_type, + i_param->roaring, i_param->null_bitmap_cache_handle, i_param->analyzer_ctx); + } return reader->query(_context, i_param->column_name, i_param->query_value, i_param->query_type, i_param->roaring, i_param->analyzer_ctx); }; @@ -122,14 +128,12 @@ Status InvertedIndexIterator::read_from_index(const IndexParam& param) { } Status InvertedIndexIterator::read_null_bitmap(InvertedIndexQueryCacheHandle* cache_handle) { - // For null bitmap, use any available reader (empty = auto-select) - auto reader = DORIS_TRY(select_best_reader("")); + auto reader = DORIS_TRY(select_any_reader()); return reader->read_null_bitmap(_context, cache_handle, nullptr); } Result InvertedIndexIterator::has_null() { - // For has_null check, use any available reader (empty = auto-select) - auto reader = DORIS_TRY(select_best_reader("")); + auto reader = DORIS_TRY(select_any_reader()); return reader->has_null(); } @@ -149,177 +153,48 @@ Status InvertedIndexIterator::try_read_from_inverted_index(const InvertedIndexRe return Status::OK(); } -// When multiple candidates of the preferred type exist, pick the one with -// the smallest index_id so that the choice is deterministic regardless of -// the order indexes appear in the rowset schema. Different segments may -// have different index orderings (e.g. after sequential BUILD INDEX -// operations), and relying on iteration order would cause inconsistent -// query results across segments. -static const ReaderEntry* pick_preferred(const std::vector& candidates, - InvertedIndexReaderType preferred_type) { - const ReaderEntry* best = nullptr; - for (const auto* entry : candidates) { - if (entry->type == preferred_type) { - if (best == nullptr || entry->reader->get_index_id() < best->reader->get_index_id()) { - best = entry; - } - } - } - return best; -} - -static const ReaderEntry* pick_smallest_index_id( - const std::vector& candidates) { - const ReaderEntry* best = candidates.front(); - for (const auto* entry : candidates) { - if (entry->reader->get_index_id() < best->reader->get_index_id()) { - best = entry; - } - } - return best; -} - -Result InvertedIndexIterator::select_for_text( - const AnalyzerMatchResult& match, InvertedIndexQueryType query_type, - const std::string& analyzer_key) { - // Bypass: explicit analyzer specified but not found - if (match.empty() && AnalyzerKeyMatcher::is_explicit(analyzer_key)) { - return ResultError(Status::Error( - "No inverted index reader found for analyzer '{}'. " - "The index for this analyzer may not be built yet.", - analyzer_key)); - } - - if (match.empty()) { - return ResultError(Status::Error( - "No available inverted index readers for text column.")); - } - - // MATCH queries prefer FULLTEXT - if (is_match_query(query_type)) { - if (auto* best = pick_preferred(match.candidates, InvertedIndexReaderType::FULLTEXT)) { - return best->reader; - } - } - - // EQUAL queries prefer STRING_TYPE for exact match - if (is_equal_query(query_type)) { - if (auto* best = pick_preferred(match.candidates, InvertedIndexReaderType::STRING_TYPE)) { - return best->reader; - } - } - - // Default: smallest index_id for deterministic selection - return pick_smallest_index_id(match.candidates)->reader; -} - -Result InvertedIndexIterator::select_for_numeric( - const AnalyzerMatchResult& match, InvertedIndexQueryType query_type) { - if (match.empty()) { - return ResultError(Status::Error( - "No available inverted index readers for numeric column.")); - } - - // RANGE queries prefer BKD - if (is_range_query(query_type)) { - if (const auto* best = pick_preferred(match.candidates, InvertedIndexReaderType::BKD)) { - return best->reader; - } - } - - // Fallback priority: BKD > STRING_TYPE > smallest index_id - if (const auto* best = pick_preferred(match.candidates, InvertedIndexReaderType::BKD)) { - return best->reader; - } - if (const auto* best = pick_preferred(match.candidates, InvertedIndexReaderType::STRING_TYPE)) { - return best->reader; - } - - // Last resort: smallest index_id for deterministic selection - return pick_smallest_index_id(match.candidates)->reader; -} - Result InvertedIndexIterator::select_best_reader( const DataTypePtr& column_type, InvertedIndexQueryType query_type, const std::string& analyzer_key) { - if (_reader_entries.empty()) { - return ResultError(Status::Error( - "No available inverted index readers. Check if index is properly initialized.")); - } - - // Normalize once at entry point const std::string normalized_key = ensure_normalized_key(analyzer_key); + const auto field_type = get_inverted_index_leaf_field_type(column_type); + auto selection = select_best_inverted_index_candidate(_selection_candidates, _key_to_entries, + field_type, query_type, normalized_key); + if (!selection.has_value()) { + return ResultError(std::move(selection.error())); + } + const size_t selected = *selection; + DORIS_CHECK(selected < _readers.size()); + return _readers[selected]; +} - // Single reader optimization - if (_reader_entries.size() == 1) { - const auto& entry = _reader_entries.front(); - if (AnalyzerKeyMatcher::is_explicit(normalized_key) && - entry.analyzer_key != normalized_key) { - return ResultError(Status::Error( - "No inverted index reader found for analyzer '{}'. " - "Available analyzer: '{}'.", - normalized_key, entry.analyzer_key)); - } - return entry.reader; - } - - // Match analyzer key using AnalyzerKeyMatcher - auto match = AnalyzerKeyMatcher::match(normalized_key, _reader_entries, _key_to_entries); - - // Dispatch by column type - const auto field_type = column_type->get_storage_field_type(); - - if (is_string_type(field_type)) { - return select_for_text(match, query_type, normalized_key); - } - - if (field_is_numeric_type(field_type)) { - return select_for_numeric(match, query_type); - } - - // Default: return deterministic candidate or error - if (match.empty()) { - return ResultError(Status::Error( - "No available inverted index readers for column type.")); +Result InvertedIndexIterator::select_any_reader() { + auto selection = select_best_inverted_index_candidate( + _selection_candidates, _key_to_entries, FieldType::OLAP_FIELD_TYPE_UNKNOWN, + InvertedIndexQueryType::UNKNOWN_QUERY, ""); + if (!selection.has_value()) { + return ResultError(std::move(selection.error())); } - return pick_smallest_index_id(match.candidates)->reader; + const size_t selected = *selection; + DORIS_CHECK(selected < _readers.size()); + return _readers[selected]; } Result InvertedIndexIterator::select_best_reader( const std::string& analyzer_key) { - if (_reader_entries.empty()) { - return ResultError(Status::Error( - "No available inverted index readers. Check if index is properly initialized.")); + if (analyzer_key.empty()) { + return select_any_reader(); } - const std::string normalized_key = ensure_normalized_key(analyzer_key); - - // Single reader optimization - if (_reader_entries.size() == 1) { - const auto& entry = _reader_entries.front(); - if (AnalyzerKeyMatcher::is_explicit(normalized_key) && - entry.analyzer_key != normalized_key) { - return ResultError(Status::Error( - "No inverted index reader found for analyzer '{}'. " - "Available analyzer: '{}'.", - normalized_key, entry.analyzer_key)); - } - return entry.reader; - } - - // Match and return deterministic candidate - auto match = AnalyzerKeyMatcher::match(normalized_key, _reader_entries, _key_to_entries); - - if (match.empty()) { - if (AnalyzerKeyMatcher::is_explicit(normalized_key)) { - return ResultError(Status::Error( - "No inverted index reader found for analyzer '{}'.", normalized_key)); - } - return ResultError(Status::Error( - "No available inverted index readers.")); - } - - return pick_smallest_index_id(match.candidates)->reader; + auto selection = select_best_inverted_index_candidate( + _selection_candidates, _key_to_entries, FieldType::OLAP_FIELD_TYPE_UNKNOWN, + InvertedIndexQueryType::UNKNOWN_QUERY, normalized_key); + if (!selection.has_value()) { + return ResultError(std::move(selection.error())); + } + const size_t selected = *selection; + DORIS_CHECK(selected < _readers.size()); + return _readers[selected]; } IndexReaderPtr InvertedIndexIterator::get_reader(IndexReaderType type) const { @@ -327,9 +202,10 @@ IndexReaderPtr InvertedIndexIterator::get_reader(IndexReaderType type) const { if (inverted_type == nullptr) { return nullptr; } - for (const auto& entry : _reader_entries) { - if (entry.type == *inverted_type) { - return entry.reader; + for (size_t i = 0; i < _selection_candidates.size(); ++i) { + if (_selection_candidates[i].reader_type == *inverted_type) { + DORIS_CHECK(i < _readers.size()); + return _readers[i]; } } return nullptr; diff --git a/be/src/storage/index/inverted/inverted_index_iterator.h b/be/src/storage/index/inverted/inverted_index_iterator.h index afc4a663670633..5f7c2250cc0836 100644 --- a/be/src/storage/index/inverted/inverted_index_iterator.h +++ b/be/src/storage/index/inverted/inverted_index_iterator.h @@ -20,10 +20,10 @@ #include #include "core/field.h" -#include "storage/index/analyzer_key_matcher.h" #include "storage/index/index_iterator.h" #include "storage/index/inverted/inverted_index_parser.h" #include "storage/index/inverted/inverted_index_reader.h" +#include "storage/index/inverted/inverted_index_selector.h" namespace doris::segment_v2 { @@ -35,19 +35,13 @@ struct InvertedIndexParam { uint32_t num_rows; std::shared_ptr roaring; bool skip_try = false; + // Non-null only when the caller consumes both the query result and this reader's null bitmap. + InvertedIndexQueryCacheHandle* null_bitmap_cache_handle = nullptr; // Pointer to analyzer context (can be nullptr if not needed) // Used by FullTextIndexReader for tokenization const InvertedIndexAnalyzerCtx* analyzer_ctx = nullptr; }; -// Entry representing an inverted index reader with its type and analyzer key. -// Used by InvertedIndexIterator and AnalyzerKeyMatcher for reader selection. -struct ReaderEntry { - InvertedIndexReaderType type; - std::string analyzer_key; - InvertedIndexReaderPtr reader; -}; - class InvertedIndexIterator : public IndexIterator { public: InvertedIndexIterator(); @@ -67,6 +61,10 @@ class InvertedIndexIterator : public IndexIterator { [[nodiscard]] Result select_best_reader( const DataTypePtr& column_type, InvertedIndexQueryType query_type, const std::string& analyzer_key); + + [[nodiscard]] Result select_any_reader(); + + // Temporary compatibility for variant fields whose runtime binding has no type. [[nodiscard]] Result select_best_reader( const std::string& analyzer_key); @@ -81,27 +79,16 @@ class InvertedIndexIterator : public IndexIterator { // Empty input stays empty (means "user did not specify"). static std::string ensure_normalized_key(const std::string& analyzer_key); - // Select best reader for text (string) columns. - // Handles FULLTEXT vs STRING_TYPE priority based on query type. - // Returns BYPASS error if explicit analyzer not found. - [[nodiscard]] Result select_for_text(const AnalyzerMatchResult& match, - InvertedIndexQueryType query_type, - const std::string& analyzer_key); - - // Select best reader for numeric columns. - // Handles BKD priority for range queries. - [[nodiscard]] Result select_for_numeric( - const AnalyzerMatchResult& match, InvertedIndexQueryType query_type); - - // THREAD SAFETY: _reader_entries and _key_to_entries are populated during initialization + // THREAD SAFETY: reader metadata and _key_to_entries are populated during initialization // phase (via add_reader) and only read during query phase (via read_from_index/select_best_reader). // These two phases are guaranteed not to overlap, so no synchronization is needed. // Do NOT call add_reader() after any read_from_index() call on the same iterator. - std::vector _reader_entries; + std::vector _selection_candidates; + std::vector _readers; - // Index for O(1) lookup by analyzer_key. Maps normalized key to indices in _reader_entries. + // Index for O(1) lookup by analyzer_key. Maps normalized key to candidate indices. // Built incrementally in add_reader(). - std::unordered_map> _key_to_entries; + InvertedIndexSelectionKeyIndex _key_to_entries; }; } // namespace doris::segment_v2 \ No newline at end of file diff --git a/be/src/storage/index/inverted/inverted_index_parser.cpp b/be/src/storage/index/inverted/inverted_index_parser.cpp index 47819cc62f6397..1bd0a41454de80 100644 --- a/be/src/storage/index/inverted/inverted_index_parser.cpp +++ b/be/src/storage/index/inverted/inverted_index_parser.cpp @@ -190,16 +190,11 @@ std::string normalize_analyzer_key(std::string_view analyzer) { std::string build_analyzer_key_from_properties( const std::map& properties) { - // Build analyzer key from index properties for reader registration. - // This determines how the index is stored/identified. - - // 1. Check for custom analyzer name - auto custom_it = properties.find(INVERTED_INDEX_ANALYZER_NAME_KEY); - if (custom_it != properties.end() && !custom_it->second.empty()) { - return to_lower(custom_it->second); + auto analyzer = properties.find(INVERTED_INDEX_ANALYZER_NAME_KEY); + if (analyzer != properties.end() && !analyzer->second.empty()) { + return normalize_analyzer_key(analyzer->second); } - // 2. Fall back to parser type std::string parser; auto parser_it = properties.find(INVERTED_INDEX_PARSER_KEY); if (parser_it != properties.end()) { @@ -211,11 +206,10 @@ std::string build_analyzer_key_from_properties( } } - // 3. Return normalized parser or "" for no explicit configuration if (parser.empty()) { - return ""; // No explicit parser - empty key means "no configuration" + return ""; } - return to_lower(parser); + return normalize_analyzer_key(parser); } // ============================================================================ diff --git a/be/src/storage/index/inverted/inverted_index_parser.h b/be/src/storage/index/inverted/inverted_index_parser.h index d2d3df47abd0a3..c2ab8aa6727927 100644 --- a/be/src/storage/index/inverted/inverted_index_parser.h +++ b/be/src/storage/index/inverted/inverted_index_parser.h @@ -19,9 +19,12 @@ #include #include +#include #include #include +#include "storage/index/inverted/analyzer/analyzer_provider.h" +#include "storage/index/inverted/common_grams/common_grams_segment_metadata.h" #include "util/debug_points.h" namespace lucene { @@ -119,6 +122,36 @@ struct InvertedIndexAnalyzerCtx { // Used for creating reader and tokenization CharFilterMap char_filter_map; std::shared_ptr analyzer; + segment_v2::inverted_index::AnalyzerProviderPtr analyzer_provider; + std::optional common_grams_identity; + + std::shared_ptr get_analyzer( + segment_v2::inverted_index::AnalysisPurpose purpose) const { + if (analyzer_provider != nullptr) { + return analyzer_provider->get_analyzer(purpose); + } + return analyzer; + } + + const segment_v2::inverted_index::CommonGramsQueryIdentity* get_common_grams_identity() const { + if (common_grams_identity.has_value()) { + return &*common_grams_identity; + } + return analyzer_provider == nullptr ? nullptr : analyzer_provider->common_grams_identity(); + } + + bool has_complete_common_grams_identity() const { + const auto* identity = get_common_grams_identity(); + return identity != nullptr && !identity->common_grams_dictionary_identity.empty() && + !identity->base_analyzer_fingerprint.empty() && + !identity->common_grams_fingerprint.empty(); + } + + // Raw-query cache and single-flight keys intentionally exclude analyzer output. A tokenizing + // provider therefore needs a complete immutable identity before those results may be shared. + bool can_share_raw_query_semantics() const { + return !should_tokenize() || has_complete_common_grams_identity(); + } // Returns true if tokenization should be performed. // Decision is based on parser_type (from index properties): @@ -170,7 +203,7 @@ std::string get_parser_dict_compression_from_properties( std::string get_analyzer_name_from_properties(const std::map& properties); // Build a normalized analyzer key from index properties. -// Checks custom_analyzer first, then falls back to parser type. +// Precedence is analyzer, then parser type. Normalizer is not an index-selection key. std::string build_analyzer_key_from_properties( const std::map& properties); diff --git a/be/src/storage/index/inverted/inverted_index_reader.cpp b/be/src/storage/index/inverted/inverted_index_reader.cpp index 0e83238064297e..23d568aa8dcd8f 100644 --- a/be/src/storage/index/inverted/inverted_index_reader.cpp +++ b/be/src/storage/index/inverted/inverted_index_reader.cpp @@ -154,6 +154,20 @@ std::string InvertedIndexReader::get_index_file_path() { return _index_file_reader->get_index_file_path(&_index_meta); } +Status InvertedIndexReader::query_with_null_bitmap( + const IndexQueryContextPtr& context, const std::string& column_name, + const Field& query_value, InvertedIndexQueryType query_type, + std::shared_ptr& bit_map, + InvertedIndexQueryCacheHandle* null_bitmap_cache_handle, + const InvertedIndexAnalyzerCtx* analyzer_ctx) { + DORIS_CHECK(null_bitmap_cache_handle != nullptr); + RETURN_IF_ERROR(query(context, column_name, query_value, query_type, bit_map, analyzer_ctx)); + if (!has_null()) { + return Status::OK(); + } + return read_null_bitmap(context, null_bitmap_cache_handle); +} + Status InvertedIndexReader::read_null_bitmap(const IndexQueryContextPtr& context, InvertedIndexQueryCacheHandle* cache_handle, lucene::store::Directory* dir) { @@ -219,15 +233,16 @@ bool InvertedIndexReader::handle_query_cache(const IndexQueryContextPtr& context InvertedIndexQueryCache* cache, const InvertedIndexQueryCache::CacheKey& cache_key, InvertedIndexQueryCacheHandle* cache_handler, - std::shared_ptr& bit_map) { + std::shared_ptr& bit_map, + bool enabled) { const auto& query_options = context->runtime_state->query_options(); - - bool cache_hit = false; - if (query_options.enable_inverted_index_query_cache) { - SCOPED_RAW_TIMER(&context->stats->inverted_index_lookup_timer); - cache_hit = cache->lookup(cache_key, cache_handler); + if (!enabled || !query_options.enable_inverted_index_query_cache) { + return false; } + context->stats->inverted_index_query_cache_lookup++; + SCOPED_RAW_TIMER(&context->stats->inverted_index_lookup_timer); + const bool cache_hit = cache->lookup(cache_key, cache_handler); if (cache_hit) { DBUG_EXECUTE_IF("InvertedIndexReader.handle_query_cache_hit", { return Status::Error("handle query cache hit"); @@ -245,6 +260,19 @@ bool InvertedIndexReader::handle_query_cache(const IndexQueryContextPtr& context return false; } +void InvertedIndexReader::insert_query_cache(const IndexQueryContextPtr& context, + InvertedIndexQueryCache* cache, + const InvertedIndexQueryCache::CacheKey& cache_key, + std::shared_ptr bit_map, + InvertedIndexQueryCacheHandle* cache_handler, + bool enabled) { + if (!enabled || !context->runtime_state->query_options().enable_inverted_index_query_cache) { + return; + } + cache->insert(cache_key, std::move(bit_map), cache_handler); + context->stats->inverted_index_query_cache_insert++; +} + Status InvertedIndexReader::handle_searcher_cache( const IndexQueryContextPtr& context, InvertedIndexCacheHandle* inverted_index_cache_handle) { diff --git a/be/src/storage/index/inverted/inverted_index_reader.h b/be/src/storage/index/inverted/inverted_index_reader.h index 0e2f6a120d41e3..6960e135e572b5 100644 --- a/be/src/storage/index/inverted/inverted_index_reader.h +++ b/be/src/storage/index/inverted/inverted_index_reader.h @@ -226,13 +226,19 @@ class InvertedIndexReader : public IndexReader { const Field& query_value, InvertedIndexQueryType query_type, std::shared_ptr& bit_map, const InvertedIndexAnalyzerCtx* analyzer_ctx = nullptr) = 0; + virtual Status query_with_null_bitmap(const IndexQueryContextPtr& context, + const std::string& column_name, const Field& query_value, + InvertedIndexQueryType query_type, + std::shared_ptr& bit_map, + InvertedIndexQueryCacheHandle* null_bitmap_cache_handle, + const InvertedIndexAnalyzerCtx* analyzer_ctx = nullptr); virtual Status try_query(const IndexQueryContextPtr& context, const std::string& column_name, const Field& query_value, InvertedIndexQueryType query_type, size_t* count) = 0; - Status read_null_bitmap(const IndexQueryContextPtr& context, - InvertedIndexQueryCacheHandle* cache_handle, - lucene::store::Directory* dir = nullptr); + virtual Status read_null_bitmap(const IndexQueryContextPtr& context, + InvertedIndexQueryCacheHandle* cache_handle, + lucene::store::Directory* dir = nullptr); virtual InvertedIndexReaderType type() = 0; @@ -249,7 +255,11 @@ class InvertedIndexReader : public IndexReader { bool handle_query_cache(const IndexQueryContextPtr& context, InvertedIndexQueryCache* cache, const InvertedIndexQueryCache::CacheKey& cache_key, InvertedIndexQueryCacheHandle* cache_handler, - std::shared_ptr& bit_map); + std::shared_ptr& bit_map, bool enabled = true); + void insert_query_cache(const IndexQueryContextPtr& context, InvertedIndexQueryCache* cache, + const InvertedIndexQueryCache::CacheKey& cache_key, + std::shared_ptr bit_map, + InvertedIndexQueryCacheHandle* cache_handler, bool enabled = true); virtual Status handle_searcher_cache(const IndexQueryContextPtr& context, InvertedIndexCacheHandle* inverted_index_cache_handle); @@ -335,7 +345,6 @@ class InvertedIndexVisitor : public lucene::util::bkd::bkd_reader::intersect_vis std::string query_min; std::string query_max; -public: InvertedIndexVisitor(const void* io_ctx, lucene::util::bkd::bkd_reader* r, roaring::Roaring* hits, bool only_count = false); ~InvertedIndexVisitor() override = default; diff --git a/be/src/storage/index/inverted/inverted_index_selector.cpp b/be/src/storage/index/inverted/inverted_index_selector.cpp new file mode 100644 index 00000000000000..dac0ec9b41b1cb --- /dev/null +++ b/be/src/storage/index/inverted/inverted_index_selector.cpp @@ -0,0 +1,145 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/inverted/inverted_index_selector.h" + +#include + +#include "common/logging.h" +#include "core/data_type/data_type_array.h" +#include "core/data_type/data_type_nullable.h" +#include "storage/index/inverted/analyzer/analyzer.h" +#include "storage/utils.h" + +namespace doris::segment_v2 { + +Status add_inverted_index_selection_candidate( + InvertedIndexSelectionCandidate candidate, + std::vector* candidates, + InvertedIndexSelectionKeyIndex* key_index) { + DORIS_CHECK(candidates != nullptr); + DORIS_CHECK(key_index != nullptr); + for (const auto& existing : *candidates) { + if (existing.index_id == candidate.index_id) { + return Status::Error( + "Duplicate inverted index id {} in one field", candidate.index_id); + } + } + + const size_t candidate_index = candidates->size(); + candidates->push_back(std::move(candidate)); + (*key_index)[candidates->back().analyzer_key].push_back(candidate_index); + return Status::OK(); +} + +Result select_best_inverted_index_candidate( + const std::vector& candidates, + const InvertedIndexSelectionKeyIndex& key_index, FieldType field_type, + InvertedIndexQueryType query_type, std::string_view normalized_analyzer_key) { + if (candidates.empty()) { + return ResultError(Status::Error( + "No available inverted index candidates")); + } + + const std::vector* exact_candidates = nullptr; + if (!normalized_analyzer_key.empty()) { + const auto exact = key_index.find(std::string(normalized_analyzer_key)); + if (exact == key_index.end() || exact->second.empty()) { + return ResultError(Status::Error( + "No inverted index found for analyzer '{}'", normalized_analyzer_key)); + } + exact_candidates = &exact->second; + } + + const size_t candidate_count = + exact_candidates == nullptr ? candidates.size() : exact_candidates->size(); + auto candidate_at = + [&](size_t ordinal) -> std::pair { + const size_t index = exact_candidates == nullptr ? ordinal : (*exact_candidates)[ordinal]; + DORIS_CHECK(index < candidates.size()); + return {index, candidates[index]}; + }; + auto pick = + [&](std::optional preferred_type) -> std::optional { + std::optional best; + for (size_t ordinal = 0; ordinal < candidate_count; ++ordinal) { + const auto [index, candidate] = candidate_at(ordinal); + if (preferred_type.has_value() && candidate.reader_type != *preferred_type) { + continue; + } + if (!best.has_value() || candidate.index_id < candidates[*best].index_id) { + best = index; + } + } + return best; + }; + + if (is_string_type(field_type)) { + if (is_match_query(query_type)) { + if (auto best = pick(InvertedIndexReaderType::FULLTEXT); best.has_value()) { + return *best; + } + } + if (is_equal_query(query_type)) { + if (auto best = pick(InvertedIndexReaderType::STRING_TYPE); best.has_value()) { + return *best; + } + } + } else if (field_is_numeric_type(field_type)) { + if (is_range_query(query_type)) { + if (auto best = pick(InvertedIndexReaderType::BKD); best.has_value()) { + return *best; + } + } + if (auto best = pick(InvertedIndexReaderType::BKD); best.has_value()) { + return *best; + } + if (auto best = pick(InvertedIndexReaderType::STRING_TYPE); best.has_value()) { + return *best; + } + } + + auto best = pick(std::nullopt); + DORIS_CHECK(best.has_value()); + return *best; +} + +FieldType get_inverted_index_leaf_field_type(const DataTypePtr& column_type) { + DORIS_CHECK(column_type != nullptr); + DataTypePtr leaf_type = remove_nullable(column_type); + while (leaf_type->get_storage_field_type() == FieldType::OLAP_FIELD_TYPE_ARRAY) { + const auto* array_type = dynamic_cast(leaf_type.get()); + DORIS_CHECK(array_type != nullptr); + leaf_type = remove_nullable(array_type->get_nested_type()); + } + return leaf_type->get_storage_field_type(); +} + +InvertedIndexReaderType infer_inverted_index_reader_type( + FieldType field_type, const std::map& properties) { + if (is_string_type(field_type)) { + return inverted_index::InvertedIndexAnalyzer::should_analyzer(properties) + ? InvertedIndexReaderType::FULLTEXT + : InvertedIndexReaderType::STRING_TYPE; + } + if (field_is_numeric_type(field_type)) { + return InvertedIndexReaderType::BKD; + } + return InvertedIndexReaderType::UNKNOWN; +} + +} // namespace doris::segment_v2 diff --git a/be/src/storage/index/inverted/inverted_index_selector.h b/be/src/storage/index/inverted/inverted_index_selector.h new file mode 100644 index 00000000000000..c295a9db3bc33e --- /dev/null +++ b/be/src/storage/index/inverted/inverted_index_selector.h @@ -0,0 +1,57 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "core/data_type/data_type.h" +#include "storage/index/inverted/inverted_index_query_type.h" +#include "storage/olap_common.h" + +namespace doris::segment_v2 { + +struct InvertedIndexSelectionCandidate { + int64_t index_id; + InvertedIndexReaderType reader_type; + std::string analyzer_key; +}; + +using InvertedIndexSelectionKeyIndex = std::unordered_map>; + +Status add_inverted_index_selection_candidate( + InvertedIndexSelectionCandidate candidate, + std::vector* candidates, + InvertedIndexSelectionKeyIndex* key_index); + +[[nodiscard]] Result select_best_inverted_index_candidate( + const std::vector& candidates, + const InvertedIndexSelectionKeyIndex& key_index, FieldType field_type, + InvertedIndexQueryType query_type, std::string_view normalized_analyzer_key); + +FieldType get_inverted_index_leaf_field_type(const DataTypePtr& column_type); + +InvertedIndexReaderType infer_inverted_index_reader_type( + FieldType field_type, const std::map& properties); + +} // namespace doris::segment_v2 diff --git a/be/src/storage/index/inverted/query/conjunction_query.cpp b/be/src/storage/index/inverted/query/conjunction_query.cpp index 8fc3ba0b16148d..eb25605d938803 100644 --- a/be/src/storage/index/inverted/query/conjunction_query.cpp +++ b/be/src/storage/index/inverted/query/conjunction_query.cpp @@ -17,8 +17,8 @@ #include "storage/index/inverted/query/conjunction_query.h" -#include "storage/compaction/collection_statistics.h" #include "storage/index/inverted/query/query_helper.h" +#include "storage/index/inverted/similarity/collection_statistics.h" #include "storage/index/inverted/util/mock_iterator.h" #include "storage/index/inverted/util/string_helper.h" diff --git a/be/src/storage/index/inverted/query/query_info.h b/be/src/storage/index/inverted/query/query_info.h index 829faaca4d7f24..685d3eee951b84 100644 --- a/be/src/storage/index/inverted/query/query_info.h +++ b/be/src/storage/index/inverted/query/query_info.h @@ -17,17 +17,25 @@ #pragma once +#include +#include #include #include namespace doris::segment_v2 { +enum class TermKeyKind : uint8_t { + kPlain = 0, + kCommonGram = 1, +}; + class TermInfo { public: using Term = std::variant>; Term term; int32_t position = 0; + TermKeyKind key_kind = TermKeyKind::kPlain; bool is_single_term() const { return std::holds_alternative(term); } bool is_multi_terms() const { return std::holds_alternative>(term); } @@ -49,6 +57,15 @@ class InvertedIndexQueryInfo { // for test bool use_mock_iter = false; + bool has_common_gram() const { + for (const auto& term_info : term_infos) { + if (term_info.key_kind == TermKeyKind::kCommonGram) { + return true; + } + } + return false; + } + std::string generate_tokens_key() const { std::string key; for (const auto& token : term_infos) { diff --git a/be/src/storage/index/inverted/setting.h b/be/src/storage/index/inverted/setting.h index 51782ab0b2de5d..8e2a7ce1a72dc4 100644 --- a/be/src/storage/index/inverted/setting.h +++ b/be/src/storage/index/inverted/setting.h @@ -19,12 +19,14 @@ #include +#include #include #include #include #include #include #include +#include #include "common/exception.h" @@ -168,6 +170,12 @@ class Settings { return result; } + std::vector> sorted_entries() const { + std::vector> entries(_args.begin(), _args.end()); + std::sort(entries.begin(), entries.end()); + return entries; + } + private: std::unordered_map _args; }; diff --git a/be/src/storage/index/inverted/similarity/collection_statistics.cpp b/be/src/storage/index/inverted/similarity/collection_statistics.cpp new file mode 100644 index 00000000000000..87f42e31f8e75e --- /dev/null +++ b/be/src/storage/index/inverted/similarity/collection_statistics.cpp @@ -0,0 +1,529 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/inverted/similarity/collection_statistics.h" + +#include +#include +#include + +#include "common/exception.h" +#include "exprs/vexpr.h" +#include "exprs/vexpr_context.h" +#include "exprs/vliteral.h" +#include "exprs/vslot_ref.h" +#include "storage/index/index_file_reader.h" +#include "storage/index/index_reader_helper.h" +#include "storage/index/inverted/analyzer/analyzer.h" +#include "storage/index/inverted/util/string_helper.h" +#include "storage/index/inverted/util/term_iterator.h" +#include "storage/index/snii/format/dict_entry.h" +#include "storage/index/snii/reader/dict_block_cache.h" +#include "storage/index/snii/snii_doris_adapter.h" +#include "storage/rowset/rowset.h" +#include "storage/rowset/rowset_reader.h" +#include "util/uid_util.h" + +namespace doris { +namespace { + +struct SniiScoringSegmentStats { + uint64_t doc_count = 0; + uint64_t token_count = 0; + segment_v2::inverted_index::PlainTermKeyVersion plain_term_key_version = + segment_v2::inverted_index::PlainTermKeyVersion::kLegacyRaw; + std::string base_analyzer_fingerprint; +}; + +Result resolve_snii_scoring_segment( + const std::optional& metadata, + uint64_t index_doc_count, uint64_t physical_sum_total_term_freq, bool has_scoring_tier, + bool has_positions, bool has_semantic_norms) { + using namespace segment_v2::inverted_index; + const auto* metadata_ptr = metadata ? &*metadata : nullptr; + auto validation_status = validate_snii_scoring_metadata( + metadata_ptr, index_doc_count, physical_sum_total_term_freq, has_scoring_tier, + has_positions, has_semantic_norms); + if (!validation_status.ok()) { + return ResultError(std::move(validation_status)); + } + DORIS_CHECK(metadata_ptr != nullptr); + + return SniiScoringSegmentStats { + .doc_count = metadata_ptr->scoring_doc_count, + .token_count = metadata_ptr->scoring_token_count, + .plain_term_key_version = metadata_ptr->plain_term_key_version, + .base_analyzer_fingerprint = metadata_ptr->base_analyzer_fingerprint}; +} + +void add_term_doc_frequency( + std::unordered_map>* + logical_frequencies, + const std::wstring& field, const std::wstring& logical_term, uint64_t doc_frequency) { + DORIS_CHECK(logical_frequencies != nullptr); + (*logical_frequencies)[field][logical_term] += doc_frequency; +} + +} // namespace + +Status CollectionStatistics::collect(RuntimeState* state, + const std::vector& rs_splits, + const TabletSchemaSPtr& tablet_schema, + const VExprContextSPtrs& common_expr_ctxs_push_down, + io::IOContext* io_ctx) { + std::vector rowsets; + rowsets.reserve(rs_splits.size()); + for (const auto& rs_split : rs_splits) { + DORIS_CHECK(rs_split.rs_reader != nullptr); + auto rowset = rs_split.rs_reader->rowset(); + DORIS_CHECK(rowset != nullptr); + rowsets.emplace_back(std::move(rowset)); + } + return collect_full_collection(state, rowsets, tablet_schema, common_expr_ctxs_push_down, + io_ctx); +} + +Status CollectionStatistics::collect_full_collection( + RuntimeState* state, const std::vector& rowsets, + const TabletSchemaSPtr& tablet_schema, const VExprContextSPtrs& common_expr_ctxs_push_down, + io::IOContext* io_ctx) { + clear(); + std::unordered_map collect_infos; + RETURN_IF_ERROR( + extract_collect_info(state, common_expr_ctxs_push_down, tablet_schema, &collect_infos)); + if (collect_infos.empty()) { + LOG(WARNING) << "Index statistics collection: no collect info extracted."; + return Status::OK(); + } + + for (const auto& rowset : rowsets) { + DORIS_CHECK(rowset != nullptr); + const auto num_segments = rowset->num_segments(); + for (int64_t seg_id = 0; seg_id < num_segments; ++seg_id) { + auto status = + process_segment(rowset, seg_id, tablet_schema.get(), collect_infos, io_ctx); + if (!status.ok()) { + if (tablet_schema->get_inverted_index_storage_format() != + InvertedIndexStorageFormatPB::SNII && + (status.code() == ErrorCode::INVERTED_INDEX_FILE_NOT_FOUND || + status.code() == ErrorCode::INVERTED_INDEX_BYPASS)) { + LOG(ERROR) << "Index statistics collection failed: " << status.to_string(); + continue; + } + clear(); + return status; + } + } + } + + // Build a single-line log with query_id, tablet_ids, and per-field term statistics + if (VLOG_IS_ON(1)) { + std::set tablet_ids; + for (const auto& rowset : rowsets) { + DORIS_CHECK(rowset != nullptr); + tablet_ids.insert(rowset->rowset_meta()->tablet_id()); + } + + std::ostringstream oss; + oss << "CollectionStatistics: query_id=" << print_id(state->query_id()); + + oss << ", tablet_ids=["; + bool first_tablet = true; + for (int64_t tid : tablet_ids) { + if (!first_tablet) oss << ","; + oss << tid; + first_tablet = false; + } + oss << "]"; + + oss << ", total_num_docs=" << _total_num_docs; + + for (const auto& [ws_field_name, num_tokens] : _total_num_tokens) { + oss << ", {field=" << StringHelper::to_string(ws_field_name) + << ", num_tokens=" << num_tokens << ", terms=["; + + auto field_term_doc_freqs = _term_doc_freqs.find(ws_field_name); + if (field_term_doc_freqs != _term_doc_freqs.end()) { + bool first_term = true; + for (const auto& [term, doc_freq] : field_term_doc_freqs->second) { + if (!first_term) oss << ", "; + oss << "(" << StringHelper::to_string(term) << ":" << doc_freq << ")"; + first_term = false; + } + } + oss << "]}"; + } + + VLOG(1) << oss.str(); + } + + return Status::OK(); +} + +Status CollectionStatistics::extract_collect_info( + RuntimeState* state, const VExprContextSPtrs& common_expr_ctxs_push_down, + const TabletSchemaSPtr& tablet_schema, CollectInfoMap* collect_infos) { + DCHECK(collect_infos != nullptr); + + std::unordered_map collectors; + collectors[TExprNodeType::MATCH_PRED] = std::make_unique(); + collectors[TExprNodeType::SEARCH_EXPR] = std::make_unique(); + + for (const auto& root_expr_ctx : common_expr_ctxs_push_down) { + const auto& root_expr = root_expr_ctx->root(); + if (root_expr == nullptr) { + continue; + } + + std::stack stack; + stack.emplace(root_expr); + + while (!stack.empty()) { + auto expr = stack.top(); + stack.pop(); + + if (!expr) { + continue; + } + + auto collector_it = collectors.find(expr->node_type()); + if (collector_it != collectors.end()) { + RETURN_IF_ERROR( + collector_it->second->collect(state, tablet_schema, expr, collect_infos)); + } + + const auto& children = expr->children(); + for (auto child = children.rbegin(); child != children.rend(); ++child) { + stack.push(*child); + } + } + } + + LOG(INFO) << "Extracted collect info for " << collect_infos->size() << " fields"; + + return Status::OK(); +} + +Status CollectionStatistics::process_segment(const RowsetSharedPtr& rowset, int64_t seg_id, + const TabletSchema* tablet_schema, + const CollectInfoMap& collect_infos, + io::IOContext* io_ctx) { + auto seg_path = DORIS_TRY(rowset->segment_path(seg_id)); + auto rowset_meta = rowset->rowset_meta(); + + auto idx_file_reader = std::make_unique( + rowset_meta->fs(), + std::string {InvertedIndexDescriptor::get_index_file_path_prefix(seg_path)}, + tablet_schema->get_inverted_index_storage_format(), + rowset_meta->inverted_index_file_info(static_cast(seg_id)), + rowset_meta->tablet_id()); + const bool is_snii = + idx_file_reader->get_storage_format() == InvertedIndexStorageFormatPB::SNII; + auto init_status = idx_file_reader->init(config::inverted_index_read_buffer_size, io_ctx); + if (!init_status.ok()) { + if (is_snii && (init_status.code() == ErrorCode::NOT_FOUND || + init_status.code() == ErrorCode::INVERTED_INDEX_FILE_NOT_FOUND || + init_status.code() == ErrorCode::INVERTED_INDEX_BYPASS)) { + return Status::Error( + "SNII scoring requires every collection segment: {}", init_status.msg()); + } + return init_status; + } + + if (is_snii) { + segment_v2::snii_doris::DorisSniiFileReader::ScopedIOContext io_context_scope(io_ctx); + SniiScoringSegmentAccumulator segment_accumulator; + for (const auto& [ws_field_name, collect_info] : collect_infos) { + auto logical_reader_result = + idx_file_reader->open_snii_index(collect_info.index_meta, io_ctx); + if (!logical_reader_result.has_value()) { + auto status = std::move(logical_reader_result.error()); + if (status.code() == ErrorCode::INVERTED_INDEX_SNII_NOT_FOUND || + status.code() == ErrorCode::INVERTED_INDEX_FILE_NOT_FOUND || + status.code() == ErrorCode::INVERTED_INDEX_BYPASS) { + return Status::Error( + "SNII scoring requires every logical index in the collection: {}", + status.msg()); + } + return status; + } + auto logical_reader = std::move(logical_reader_result.value()); + const auto* common_grams_metadata = logical_reader->common_grams_metadata(); + segment_v2::inverted_index::PlainTermKeyVersion key_version; + RETURN_IF_ERROR(admit_snii_scoring_segment( + ws_field_name, + common_grams_metadata == nullptr + ? std::nullopt + : std::optional( + *common_grams_metadata), + collect_info.expected_base_analyzer_fingerprint, + logical_reader->stats().doc_count, logical_reader->stats().sum_total_term_freq, + logical_reader->tier() == ::doris::snii::format::IndexTier::kT3, + logical_reader->has_positions(), + logical_reader->section_refs().norms.length != 0, &key_version, + &segment_accumulator)); + DORIS_CHECK(common_grams_metadata != nullptr); + + ::doris::snii::reader::DictBlockCache dict_block_cache; + for (const auto& logical_term_bytes : collect_info.unique_terms) { + std::string physical_term; + const bool term_present = + DORIS_TRY(segment_v2::inverted_index::try_encode_plain_term( + logical_term_bytes, key_version, &physical_term)); + const auto logical_term = + segment_v2::inverted_index::StringHelper::to_wstring(logical_term_bytes); + if (!term_present) { + add_term_doc_frequency(&segment_accumulator.term_doc_freqs, ws_field_name, + logical_term, 0); + continue; + } + + bool found = false; + ::doris::snii::format::DictEntry entry; + uint64_t frq_base = 0; + uint64_t prx_base = 0; + RETURN_IF_ERROR(logical_reader->lookup(physical_term, &found, &entry, &frq_base, + &prx_base, &dict_block_cache)); + if (found && entry.df > common_grams_metadata->scoring_doc_count) { + return Status::Error( + "SNII term document frequency {} exceeds scoring document count {}", + entry.df, common_grams_metadata->scoring_doc_count); + } + add_term_doc_frequency(&segment_accumulator.term_doc_freqs, ws_field_name, + logical_term, found ? entry.df : 0); + } + } + + commit_snii_scoring_segment(std::move(segment_accumulator)); + return Status::OK(); + } + + int32_t total_segment_docs = 0; + + for (const auto& [ws_field_name, collect_info] : collect_infos) { + lucene::search::IndexSearcher* index_searcher = nullptr; + lucene::index::IndexReader* index_reader = nullptr; + +#ifdef BE_TEST + auto compound_reader = DORIS_TRY(idx_file_reader->open(collect_info.index_meta, io_ctx)); + auto* reader = lucene::index::IndexReader::open(compound_reader.get()); + auto owned_index_searcher = std::make_shared(reader, true); + index_searcher = owned_index_searcher.get(); + index_reader = index_searcher->getReader(); +#else + InvertedIndexCacheHandle inverted_index_cache_handle; + auto index_file_key = idx_file_reader->get_index_file_cache_key(collect_info.index_meta); + InvertedIndexSearcherCache::CacheKey searcher_cache_key(index_file_key); + + if (!InvertedIndexSearcherCache::instance()->lookup(searcher_cache_key, + &inverted_index_cache_handle)) { + auto compound_reader = + DORIS_TRY(idx_file_reader->open(collect_info.index_meta, io_ctx)); + auto* reader = lucene::index::IndexReader::open(compound_reader.get()); + size_t reader_size = reader->getTermInfosRAMUsed(); + auto searcher_ptr = std::make_shared(reader, true); + auto* cache_value = new InvertedIndexSearcherCache::CacheValue( + std::move(searcher_ptr), reader_size, UnixMillis()); + InvertedIndexSearcherCache::instance()->insert(searcher_cache_key, cache_value, + &inverted_index_cache_handle); + } + + auto searcher_variant = inverted_index_cache_handle.get_index_searcher(); + auto index_searcher_ptr = std::get(searcher_variant); + index_searcher = index_searcher_ptr.get(); + index_reader = index_searcher->getReader(); +#endif + total_segment_docs = std::max(total_segment_docs, index_reader->maxDoc()); + _total_num_tokens[ws_field_name] += + index_reader->sumTotalTermFreq(ws_field_name.c_str()).value_or(0); + + for (const auto& logical_term_bytes : collect_info.unique_terms) { + const auto logical_term = + segment_v2::inverted_index::StringHelper::to_wstring(logical_term_bytes); + auto iter = TermIterator::create(io_ctx, false, index_reader, ws_field_name, + logical_term_bytes); + add_term_doc_frequency(&_term_doc_freqs, ws_field_name, logical_term, iter->doc_freq()); + } + } + + _total_num_docs += static_cast(total_segment_docs); + _avg_dl_by_col.clear(); + _idf_by_col_term.clear(); + + return Status::OK(); +} + +Status CollectionStatistics::admit_snii_scoring_segment( + const std::wstring& field_name, + const std::optional& metadata, + std::string_view expected_base_analyzer_fingerprint, uint64_t index_doc_count, + uint64_t physical_sum_total_term_freq, bool has_scoring_tier, bool has_positions, + bool has_semantic_norms, + segment_v2::inverted_index::PlainTermKeyVersion* plain_term_key_version, + SniiScoringSegmentAccumulator* segment_accumulator) { + DORIS_CHECK(plain_term_key_version != nullptr); + DORIS_CHECK(segment_accumulator != nullptr); + auto segment_stats = + resolve_snii_scoring_segment(metadata, index_doc_count, physical_sum_total_term_freq, + has_scoring_tier, has_positions, has_semantic_norms); + if (!segment_stats.has_value()) { + clear(); + return segment_stats.error(); + } + + const auto& base_analyzer_fingerprint = segment_stats->base_analyzer_fingerprint; + if (base_analyzer_fingerprint != expected_base_analyzer_fingerprint) { + clear(); + return Status::Error( + "SNII scoring segment base analyzer does not match the request analyzer for field " + "{}", + StringHelper::to_string(field_name)); + } + auto staged_fingerprint = segment_accumulator->base_analyzer_fingerprints.find(field_name); + if (staged_fingerprint != segment_accumulator->base_analyzer_fingerprints.end() && + staged_fingerprint->second != base_analyzer_fingerprint) { + clear(); + return Status::Error( + "SNII scoring cannot combine segments with different base analyzers for field {}", + StringHelper::to_string(field_name)); + } + auto collected_fingerprint = _snii_base_analyzer_fingerprints.find(field_name); + if (collected_fingerprint != _snii_base_analyzer_fingerprints.end() && + collected_fingerprint->second != base_analyzer_fingerprint) { + clear(); + return Status::Error( + "SNII scoring cannot combine segments with different base analyzers for field {}", + StringHelper::to_string(field_name)); + } + segment_accumulator->base_analyzer_fingerprints.insert_or_assign(field_name, + base_analyzer_fingerprint); + + if (!segment_accumulator->token_counts.empty() && + segment_accumulator->doc_count != segment_stats->doc_count) { + clear(); + return Status::Error( + "SNII scoring fields in one segment have different document counts: {} and {}", + segment_accumulator->doc_count, segment_stats->doc_count); + } + segment_accumulator->doc_count = segment_stats->doc_count; + segment_accumulator->token_counts[field_name] += segment_stats->token_count; + *plain_term_key_version = segment_stats->plain_term_key_version; + return Status::OK(); +} + +void CollectionStatistics::commit_snii_scoring_segment( + SniiScoringSegmentAccumulator&& segment_accumulator) { + for (const auto& [field_name, base_analyzer_fingerprint] : + segment_accumulator.base_analyzer_fingerprints) { + auto [collected_fingerprint, inserted] = + _snii_base_analyzer_fingerprints.try_emplace(field_name, base_analyzer_fingerprint); + DORIS_CHECK(inserted || collected_fingerprint->second == base_analyzer_fingerprint); + } + for (const auto& [field_name, token_count] : segment_accumulator.token_counts) { + _total_num_tokens[field_name] += token_count; + } + for (const auto& [field_name, term_doc_freqs] : segment_accumulator.term_doc_freqs) { + for (const auto& [term, doc_freq] : term_doc_freqs) { + _term_doc_freqs[field_name][term] += doc_freq; + } + } + _total_num_docs += segment_accumulator.doc_count; + _avg_dl_by_col.clear(); + _idf_by_col_term.clear(); +} + +void CollectionStatistics::clear() { + _total_num_docs = 0; + _total_num_tokens.clear(); + _term_doc_freqs.clear(); + _snii_base_analyzer_fingerprints.clear(); + _avg_dl_by_col.clear(); + _idf_by_col_term.clear(); +} + +uint64_t CollectionStatistics::get_term_doc_freq_by_col(const std::wstring& lucene_col_name, + const std::wstring& term) { + const auto field = _term_doc_freqs.find(lucene_col_name); + if (field == _term_doc_freqs.end()) { + throw Exception(ErrorCode::INVERTED_INDEX_CLUCENE_ERROR, + "Index statistics collection failed: Not such column {}", + StringHelper::to_string(lucene_col_name)); + } + + const auto term_frequency = field->second.find(term); + if (term_frequency == field->second.end()) { + throw Exception(ErrorCode::INVERTED_INDEX_CLUCENE_ERROR, + "Index statistics collection failed: Not such term {}", + StringHelper::to_string(term)); + } + + return term_frequency->second; +} + +uint64_t CollectionStatistics::get_total_term_cnt_by_col(const std::wstring& lucene_col_name) { + const auto token_count = _total_num_tokens.find(lucene_col_name); + if (token_count == _total_num_tokens.end()) { + throw Exception(ErrorCode::INVERTED_INDEX_CLUCENE_ERROR, + "Index statistics collection failed: Not such column {}", + StringHelper::to_string(lucene_col_name)); + } + + return token_count->second; +} + +uint64_t CollectionStatistics::get_doc_num() const { + if (_total_num_docs == 0) { + throw Exception( + ErrorCode::INVERTED_INDEX_CLUCENE_ERROR, + "Index statistics collection failed: No data available for SimilarityCollector"); + } + + return _total_num_docs; +} + +float CollectionStatistics::get_or_calculate_avg_dl(const std::wstring& lucene_col_name) { + auto iter = _avg_dl_by_col.find(lucene_col_name); + if (iter != _avg_dl_by_col.end()) { + return iter->second; + } + + const uint64_t total_term_cnt = get_total_term_cnt_by_col(lucene_col_name); + const uint64_t total_doc_cnt = get_doc_num(); + float avg_dl = total_doc_cnt > 0 ? float((double)total_term_cnt / (double)total_doc_cnt) : 0.0F; + _avg_dl_by_col[lucene_col_name] = avg_dl; + return avg_dl; +} + +float CollectionStatistics::get_or_calculate_idf(const std::wstring& lucene_col_name, + const std::wstring& term) { + auto iter = _idf_by_col_term.find(lucene_col_name); + if (iter != _idf_by_col_term.end()) { + auto term_iter = iter->second.find(term); + if (term_iter != iter->second.end()) { + return term_iter->second; + } + } + + const uint64_t doc_num = get_doc_num(); + const uint64_t doc_freq = get_term_doc_freq_by_col(lucene_col_name, term); + auto idf = (float)std::log(1 + ((double)doc_num - (double)doc_freq + (double)0.5) / + ((double)doc_freq + (double)0.5)); + _idf_by_col_term[lucene_col_name][term] = idf; + return idf; +} + +} // namespace doris diff --git a/be/src/storage/compaction/collection_statistics.h b/be/src/storage/index/inverted/similarity/collection_statistics.h similarity index 65% rename from be/src/storage/compaction/collection_statistics.h rename to be/src/storage/index/inverted/similarity/collection_statistics.h index b93d5a424ae1ab..151bb9945d13a7 100644 --- a/be/src/storage/compaction/collection_statistics.h +++ b/be/src/storage/index/inverted/similarity/collection_statistics.h @@ -16,16 +16,24 @@ // under the License. #pragma once +#include #include +#include +#include +#include #include +#include #include +#include +#include #include "common/be_mock_util.h" #include "exprs/vexpr_fwd.h" #include "runtime/runtime_state.h" +#include "storage/index/inverted/common_grams/common_grams_segment_metadata.h" #include "storage/index/inverted/query/query_info.h" +#include "storage/index/inverted/similarity/predicate_collector.h" #include "storage/olap_common.h" -#include "storage/predicate_collector.h" namespace doris { @@ -52,19 +60,40 @@ class CollectionStatistics { Status collect(RuntimeState* state, const std::vector& rs_splits, const TabletSchemaSPtr& tablet_schema, const VExprContextSPtrs& common_expr_ctxs_push_down, io::IOContext* io_ctx); + Status collect_full_collection(RuntimeState* state, const std::vector& rowsets, + const TabletSchemaSPtr& tablet_schema, + const VExprContextSPtrs& common_expr_ctxs_push_down, + io::IOContext* io_ctx); MOCK_FUNCTION float get_or_calculate_idf(const std::wstring& lucene_col_name, const std::wstring& term); MOCK_FUNCTION float get_or_calculate_avg_dl(const std::wstring& lucene_col_name); private: + struct SniiScoringSegmentAccumulator { + uint64_t doc_count = 0; + std::unordered_map token_counts; + std::unordered_map> term_doc_freqs; + std::unordered_map base_analyzer_fingerprints; + }; + Status extract_collect_info(RuntimeState* state, const VExprContextSPtrs& common_expr_ctxs_push_down, const TabletSchemaSPtr& tablet_schema, CollectInfoMap* collect_infos); - Status process_segment(const RowsetSharedPtr& rowset, int32_t seg_id, + Status process_segment(const RowsetSharedPtr& rowset, int64_t seg_id, const TabletSchema* tablet_schema, const CollectInfoMap& collect_infos, io::IOContext* io_ctx); + Status admit_snii_scoring_segment( + const std::wstring& field_name, + const std::optional& metadata, + std::string_view expected_base_analyzer_fingerprint, uint64_t index_doc_count, + uint64_t physical_sum_total_term_freq, bool has_scoring_tier, bool has_positions, + bool has_semantic_norms, + segment_v2::inverted_index::PlainTermKeyVersion* plain_term_key_version, + SniiScoringSegmentAccumulator* segment_accumulator); + void commit_snii_scoring_segment(SniiScoringSegmentAccumulator&& segment_accumulator); + void clear(); uint64_t get_term_doc_freq_by_col(const std::wstring& lucene_col_name, const std::wstring& term); @@ -74,6 +103,7 @@ class CollectionStatistics { uint64_t _total_num_docs = 0; std::unordered_map _total_num_tokens; std::unordered_map> _term_doc_freqs; + std::unordered_map _snii_base_analyzer_fingerprints; std::unordered_map _avg_dl_by_col; std::unordered_map> _idf_by_col_term; diff --git a/be/src/storage/index/inverted/similarity/predicate_collector.cpp b/be/src/storage/index/inverted/similarity/predicate_collector.cpp new file mode 100644 index 00000000000000..223e676b7a9254 --- /dev/null +++ b/be/src/storage/index/inverted/similarity/predicate_collector.cpp @@ -0,0 +1,594 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/inverted/similarity/predicate_collector.h" + +#include + +#include + +#include "exec/common/variant_util.h" +#include "exprs/vexpr.h" +#include "exprs/vexpr_context.h" +#include "exprs/vliteral.h" +#include "exprs/vsearch.h" +#include "exprs/vslot_ref.h" +#include "gen_cpp/Exprs_types.h" +#include "storage/index/index_reader_helper.h" +#include "storage/index/inverted/analyzer/analyzer.h" +#include "storage/index/inverted/inverted_index_selector.h" +#include "storage/index/inverted/util/string_helper.h" +#include "storage/tablet/tablet_schema.h" +#include "storage/utils.h" + +namespace doris { + +using namespace segment_v2; + +namespace { + +InvertedIndexAnalyzerCtx analyzer_context_from_properties( + const std::map& properties) { + InvertedIndexAnalyzerConfig config; + config.analyzer_name = get_analyzer_name_from_properties(properties); + config.parser_type = get_inverted_index_parser_type_from_string( + get_parser_string_from_properties(properties)); + config.parser_mode = get_parser_mode_string_from_properties(properties); + config.lower_case = get_parser_lowercase_from_properties(properties); + config.stop_words = get_parser_stopwords_from_properties(properties); + config.char_filter_map = get_parser_char_filter_map_from_properties(properties); + + InvertedIndexAnalyzerCtx analyzer_ctx; + analyzer_ctx.analyzer_name = config.analyzer_name; + analyzer_ctx.parser_type = config.parser_type; + analyzer_ctx.char_filter_map = config.char_filter_map; + analyzer_ctx.analyzer_provider = + inverted_index::InvertedIndexAnalyzer::create_analyzer_provider(&config); + return analyzer_ctx; +} + +std::vector analyze_plain_query(const std::string& value, + const InvertedIndexAnalyzerCtx& analyzer_ctx) { + DORIS_CHECK(analyzer_ctx.analyzer_provider != nullptr); + auto analyzer = analyzer_ctx.analyzer_provider->get_analyzer( + inverted_index::AnalysisPurpose::kPlainQuery); + auto reader = + inverted_index::InvertedIndexAnalyzer::create_reader(analyzer_ctx.char_filter_map); + reader->init(value.data(), static_cast(value.size()), true); + return inverted_index::InvertedIndexAnalyzer::get_analyse_result(reader, analyzer.get()); +} + +Status append_scoring_leaf(CollectInfo* collect_info, const std::vector& term_infos, + std::string_view base_analyzer_fingerprint) { + DORIS_CHECK(collect_info != nullptr); + if (!collect_info->logical_scoring_leaves.empty() && + collect_info->expected_base_analyzer_fingerprint != base_analyzer_fingerprint) { + return Status::Error( + "Scoring predicates for one field use different base analyzers"); + } + if (collect_info->logical_scoring_leaves.empty()) { + collect_info->expected_base_analyzer_fingerprint = base_analyzer_fingerprint; + } + + LogicalScoringLeaf leaf; + leaf.clauses.reserve(term_infos.size()); + for (const auto& term_info : term_infos) { + DORIS_CHECK(term_info.is_single_term()); + DORIS_CHECK(term_info.key_kind == TermKeyKind::kPlain); + const auto& term = term_info.get_single_term(); + auto [slot, inserted] = collect_info->unique_term_slots.try_emplace( + term, static_cast(collect_info->unique_terms.size())); + if (inserted) { + collect_info->unique_terms.push_back(term); + } + leaf.clauses.emplace_back( + LogicalScoringClause {.df_slot = slot->second, .position = term_info.position}); + } + collect_info->logical_scoring_leaves.emplace_back(std::move(leaf)); + return Status::OK(); +} + +InvertedIndexQueryType match_query_type(TExprOpcode::type opcode) { + switch (opcode) { + case TExprOpcode::MATCH_ANY: + return InvertedIndexQueryType::MATCH_ANY_QUERY; + case TExprOpcode::MATCH_ALL: + return InvertedIndexQueryType::MATCH_ALL_QUERY; + case TExprOpcode::MATCH_PHRASE: + return InvertedIndexQueryType::MATCH_PHRASE_QUERY; + case TExprOpcode::MATCH_PHRASE_PREFIX: + return InvertedIndexQueryType::MATCH_PHRASE_PREFIX_QUERY; + case TExprOpcode::MATCH_REGEXP: + return InvertedIndexQueryType::MATCH_REGEXP_QUERY; + case TExprOpcode::MATCH_PHRASE_EDGE: + return InvertedIndexQueryType::MATCH_PHRASE_EDGE_QUERY; + default: + return InvertedIndexQueryType::UNKNOWN_QUERY; + } +} + +InvertedIndexQueryType search_query_type(std::string_view clause_type) { + if (clause_type == "EXACT") { + return InvertedIndexQueryType::EQUAL_QUERY; + } + if (clause_type == "PHRASE") { + return InvertedIndexQueryType::MATCH_PHRASE_QUERY; + } + if (clause_type == "ALL") { + return InvertedIndexQueryType::MATCH_ALL_QUERY; + } + return InvertedIndexQueryType::MATCH_ANY_QUERY; +} + +Result select_index_meta(const std::vector& index_metas, + FieldType field_type, + InvertedIndexQueryType query_type, + std::string_view analyzer_key) { + std::vector candidates; + candidates.reserve(index_metas.size()); + InvertedIndexSelectionKeyIndex key_index; + for (const auto* index_meta : index_metas) { + auto status = add_inverted_index_selection_candidate( + InvertedIndexSelectionCandidate {.index_id = index_meta->index_id(), + .reader_type = infer_inverted_index_reader_type( + field_type, index_meta->properties()), + .analyzer_key = build_analyzer_key_from_properties( + index_meta->properties())}, + &candidates, &key_index); + if (!status.ok()) { + return ResultError(std::move(status)); + } + } + + auto selection = select_best_inverted_index_candidate( + candidates, key_index, field_type, query_type, normalize_analyzer_key(analyzer_key)); + if (!selection.has_value()) { + return ResultError(std::move(selection.error())); + } + const size_t selected = *selection; + DORIS_CHECK(selected < index_metas.size()); + return index_metas[selected]; +} + +Status validate_same_physical_index(const CollectInfo& collect_info, + const TabletIndex& selected_index) { + DORIS_CHECK(collect_info.index_meta != nullptr); + if (collect_info.index_meta->index_id() != selected_index.index_id() || + collect_info.index_meta->get_index_suffix() != selected_index.get_index_suffix()) { + return Status::Error( + "Scoring predicates for one field select different inverted indexes: {} and {}", + collect_info.index_meta->index_id(), selected_index.index_id()); + } + return Status::OK(); +} + +struct ScoringIndexCandidates { + FieldType field_type = FieldType::OLAP_FIELD_TYPE_UNKNOWN; + std::string index_suffix_path; + std::vector index_metas; + std::vector> owned_index_metas; +}; + +FieldType scoring_leaf_type(const TabletColumn& column) { + const TabletColumn* leaf = &column; + while (leaf->is_array_type()) { + DORIS_CHECK_EQ(leaf->get_subtype_count(), 1); + leaf = &leaf->get_sub_column(0); + } + return leaf->type(); +} + +ScoringIndexCandidates resolve_text_scoring_index_candidates(const TabletSchemaSPtr& tablet_schema, + const TabletColumn& column) { + ScoringIndexCandidates candidates {.field_type = scoring_leaf_type(column), + .index_suffix_path = column.suffix_path(), + .index_metas = tablet_schema->inverted_indexs(column), + .owned_index_metas = {}}; + + // The collector has tablet-schema context but no segment-side variant + // inference. Resolve only shapes that are deterministic from schema: + // typed/materialized paths, field-pattern templates, and a plain parent + // index inherited by the dynamic VARIANT placeholder. + if (!candidates.index_metas.empty() || !column.is_extracted_column()) { + return candidates; + } + + TabletSchema::SubColumnInfo sub_column_info; + const std::string relative_path = column.path_info_ptr()->copy_pop_front().get_path(); + if (variant_util::generate_sub_column_info(*tablet_schema, column.parent_unique_id(), + relative_path, &sub_column_info) && + !sub_column_info.indexes.empty()) { + candidates.field_type = scoring_leaf_type(sub_column_info.column); + candidates.index_suffix_path = sub_column_info.column.suffix_path(); + for (auto& index : sub_column_info.indexes) { + candidates.index_metas.push_back(index.get()); + candidates.owned_index_metas.emplace_back(std::move(index)); + } + return candidates; + } + + if (!column.is_variant_type()) { + return candidates; + } + + // MATCH and score-bearing SEARCH clauses have text semantics. When a dynamic + // VARIANT path has no materialized type, those semantics provide the missing + // leaf-type proof for selecting its plain parent full-text index. Typed paths + // returned above keep their schema type, so numeric BKD leaves remain rejected. + candidates.field_type = FieldType::OLAP_FIELD_TYPE_STRING; + const auto parent_indexes = tablet_schema->inverted_indexs(column.parent_unique_id()); + for (const auto* index : parent_indexes) { + if (!index->field_pattern().empty()) { + continue; + } + auto owned_index = std::make_shared(*index); + owned_index->set_escaped_escaped_index_suffix_path(column.path_info_ptr()->get_path()); + candidates.index_metas.push_back(owned_index.get()); + candidates.owned_index_metas.emplace_back(std::move(owned_index)); + } + return candidates; +} + +Status validate_scoring_leaf_type(const ScoringIndexCandidates& candidates, + std::string_view field_name) { + if (candidates.field_type == FieldType::OLAP_FIELD_TYPE_VARIANT) { + return Status::Error( + "Index statistics collection failed: Cannot prove scoring leaf type for field={}", + field_name); + } + return Status::OK(); +} + +void preserve_selected_index_metadata(const ScoringIndexCandidates& candidates, + const TabletIndex* selected_index, + CollectInfo* collect_info) { + DORIS_CHECK(selected_index != nullptr); + DORIS_CHECK(collect_info != nullptr); + collect_info->index_meta = selected_index; + for (const auto& owned_index : candidates.owned_index_metas) { + if (owned_index.get() == selected_index) { + collect_info->owned_index_meta = owned_index; + return; + } + } +} + +Result resolve_search_scoring_index_candidates( + const TabletSchemaSPtr& tablet_schema, const std::string& field_name, + const TSearchFieldBinding* field_binding) { + const int32_t column_index = tablet_schema->field_index(field_name); + if (column_index >= 0) { + return resolve_text_scoring_index_candidates(tablet_schema, + tablet_schema->column(column_index)); + } + + if (field_binding == nullptr || !field_binding->__isset.is_variant_subcolumn || + !field_binding->is_variant_subcolumn || !field_binding->__isset.parent_field_name || + field_binding->parent_field_name.empty() || !field_binding->__isset.subcolumn_path || + field_binding->subcolumn_path.empty()) { + return ResultError(Status::Error( + "Index statistics collection failed: Cannot resolve search field={}", field_name)); + } + + const int32_t parent_column_index = + tablet_schema->field_index(field_binding->parent_field_name); + if (parent_column_index < 0) { + return ResultError(Status::Error( + "Index statistics collection failed: Cannot resolve parent={} for search field={}", + field_binding->parent_field_name, field_name)); + } + const auto& parent_column = tablet_schema->column(parent_column_index); + if (!parent_column.is_variant_type()) { + return ResultError(Status::Error( + "Index statistics collection failed: Search field={} parent={} is not VARIANT", + field_name, field_binding->parent_field_name)); + } + + TabletColumn dynamic_column; + dynamic_column.set_unique_id(-1); + dynamic_column.set_name(field_name); + dynamic_column.set_type(FieldType::OLAP_FIELD_TYPE_VARIANT); + dynamic_column.set_parent_unique_id(parent_column.unique_id()); + dynamic_column.set_path_info( + PathInData(field_binding->parent_field_name + "." + field_binding->subcolumn_path)); + return resolve_text_scoring_index_candidates(tablet_schema, dynamic_column); +} + +} // namespace + +VSlotRef* PredicateCollector::find_slot_ref(const VExprSPtr& expr) const { + if (!expr) { + return nullptr; + } + + auto cur = VExpr::expr_without_cast(expr); + if (cur->node_type() == TExprNodeType::SLOT_REF) { + return static_cast(cur.get()); + } + + for (const auto& ch : cur->children()) { + if (auto* s = find_slot_ref(ch)) { + return s; + } + } + + return nullptr; +} + +std::string PredicateCollector::build_field_name(int32_t col_unique_id, + const std::string& suffix_path) const { + std::string field_name = std::to_string(col_unique_id); + if (!suffix_path.empty()) { + field_name += "." + suffix_path; + } + return field_name; +} + +Status MatchPredicateCollector::collect(RuntimeState* state, const TabletSchemaSPtr& tablet_schema, + const VExprSPtr& expr, CollectInfoMap* collect_infos) { + DCHECK(collect_infos != nullptr); + + auto* left_slot_ref = find_slot_ref(expr->children()[0]); + if (left_slot_ref == nullptr) { + return Status::Error( + "Index statistics collection failed: Cannot find slot reference in match predicate " + "left expression"); + } + + auto* right_literal = static_cast(expr->children()[1].get()); + DCHECK(right_literal != nullptr); + + const auto* sd = state->desc_tbl().get_slot_descriptor(left_slot_ref->slot_id()); + if (sd == nullptr) { + return Status::Error( + "Index statistics collection failed: Cannot find slot descriptor for slot_id={}", + left_slot_ref->slot_id()); + } + + int32_t col_idx = tablet_schema->field_index(left_slot_ref->column_name()); + if (col_idx == -1) { + return Status::Error( + "Index statistics collection failed: Cannot find column index for column={}", + left_slot_ref->column_name()); + } + + const auto& column = tablet_schema->column(col_idx); + auto candidates = resolve_text_scoring_index_candidates(tablet_schema, column); + RETURN_IF_ERROR(validate_scoring_leaf_type(candidates, left_slot_ref->column_name())); + +#ifndef BE_TEST + if (candidates.index_metas.empty()) { + return Status::Error( + "Index statistics collection failed: Score query is not supported without inverted " + "index for column={}", + left_slot_ref->column_name()); + } +#else + if (candidates.index_metas.empty()) { + return Status::OK(); + } +#endif + + const auto* analyzer_ctx = expr->query_analyzer_ctx(); + DORIS_CHECK(analyzer_ctx != nullptr); + DORIS_CHECK(analyzer_ctx->analyzer_provider != nullptr); + const auto query_type = match_query_type(expr->op()); + DORIS_CHECK(query_type != InvertedIndexQueryType::UNKNOWN_QUERY); + const auto* index_meta = + DORIS_TRY(select_index_meta(candidates.index_metas, candidates.field_type, query_type, + analyzer_ctx->analyzer_name)); + if (!InvertedIndexAnalyzer::should_analyzer(index_meta->properties()) || + !IndexReaderHelper::is_need_similarity_score(expr->op(), index_meta)) { + return Status::OK(); + } + + auto options = DataTypeSerDe::get_default_format_options(); + options.timezone = &state->timezone_obj(); + auto term_infos = analyze_plain_query(right_literal->value(options), *analyzer_ctx); + if (expr->op() == TExprOpcode::MATCH_PHRASE_PREFIX && !term_infos.empty()) { + term_infos.pop_back(); + } + const auto base_analyzer_fingerprint = + analyzer_ctx->analyzer_provider->base_analyzer_fingerprint(); + + std::string field_name = + build_field_name(index_meta->col_unique_ids()[0], candidates.index_suffix_path); + std::wstring ws_field_name = StringHelper::to_wstring(field_name); + + auto iter = collect_infos->find(ws_field_name); + if (iter == collect_infos->end()) { + CollectInfo collect_info; + RETURN_IF_ERROR(append_scoring_leaf(&collect_info, term_infos, base_analyzer_fingerprint)); + preserve_selected_index_metadata(candidates, index_meta, &collect_info); + (*collect_infos)[ws_field_name] = std::move(collect_info); + } else { + RETURN_IF_ERROR(validate_same_physical_index(iter->second, *index_meta)); + RETURN_IF_ERROR(append_scoring_leaf(&iter->second, term_infos, base_analyzer_fingerprint)); + } + + return Status::OK(); +} + +Status SearchPredicateCollector::collect(RuntimeState* state, const TabletSchemaSPtr& tablet_schema, + const VExprSPtr& expr, CollectInfoMap* collect_infos) { + DCHECK(collect_infos != nullptr); + + auto* search_expr = dynamic_cast(expr.get()); + if (search_expr == nullptr) { + return Status::InternalError("SearchPredicateCollector: expr is not VSearchExpr type"); + } + + const TSearchParam& search_param = search_expr->get_search_param(); + FieldBindingMap field_bindings; + field_bindings.reserve(search_param.field_bindings.size()); + for (const auto& field_binding : search_param.field_bindings) { + field_bindings[field_binding.field_name] = &field_binding; + } + + RETURN_IF_ERROR(collect_from_clause(search_param.root, state, tablet_schema, field_bindings, + collect_infos)); + + return Status::OK(); +} + +Status SearchPredicateCollector::collect_from_clause(const TSearchClause& clause, + RuntimeState* state, + const TabletSchemaSPtr& tablet_schema, + const FieldBindingMap& field_bindings, + CollectInfoMap* collect_infos) { + const std::string& clause_type = clause.clause_type; + if (clause_type == "NESTED") { + return Status::Error( + "Scoring nested search clauses is not supported"); + } + ClauseTypeCategory category = get_clause_type_category(clause_type); + + if (category == ClauseTypeCategory::COMPOUND) { + if (clause.__isset.children) { + for (const auto& child_clause : clause.children) { + RETURN_IF_ERROR(collect_from_clause(child_clause, state, tablet_schema, + field_bindings, collect_infos)); + } + } + return Status::OK(); + } + + return collect_from_leaf(clause, state, tablet_schema, field_bindings, collect_infos); +} + +Status SearchPredicateCollector::collect_from_leaf(const TSearchClause& clause, RuntimeState* state, + const TabletSchemaSPtr& tablet_schema, + const FieldBindingMap& field_bindings, + CollectInfoMap* collect_infos) { + if (!clause.__isset.field_name || !clause.__isset.value) { + return Status::InvalidArgument("Search clause missing field_name or value"); + } + + const std::string& field_name = clause.field_name; + const std::string& value = clause.value; + const std::string& clause_type = clause.clause_type; + + if (!is_score_query_type(clause_type)) { + return Status::OK(); + } + + const auto field_binding_iter = field_bindings.find(field_name); + const auto* field_binding = + field_binding_iter == field_bindings.end() ? nullptr : field_binding_iter->second; + auto candidates = DORIS_TRY( + resolve_search_scoring_index_candidates(tablet_schema, field_name, field_binding)); + RETURN_IF_ERROR(validate_scoring_leaf_type(candidates, field_name)); + if (candidates.index_metas.empty()) { + return Status::Error( + "Index statistics collection failed: Score query is not supported without " + "inverted index for search field={}", + field_name); + } + + ClauseTypeCategory category = get_clause_type_category(clause_type); + auto query_type = search_query_type(clause_type); + std::string analyzer_key; + if (query_type != InvertedIndexQueryType::EQUAL_QUERY && field_binding != nullptr && + field_binding->__isset.index_properties && !field_binding->index_properties.empty() && + is_string_type(candidates.field_type)) { + analyzer_key = build_analyzer_key_from_properties(field_binding->index_properties); + } + + auto selected_index = select_index_meta(candidates.index_metas, candidates.field_type, + query_type, analyzer_key); + if (!selected_index.has_value()) { + return Status::Error( + "Index statistics collection failed: Cannot select scoring index for search " + "field={}: {}", + field_name, selected_index.error().to_string()); + } + const auto* index_meta = *selected_index; + if (infer_inverted_index_reader_type(candidates.field_type, index_meta->properties()) == + InvertedIndexReaderType::BKD) { + return Status::Error( + "Index statistics collection failed: BM25 scoring does not support numeric BKD " + "search field={}", + field_name); + } + + const auto& analysis_properties = index_meta->properties(); + + std::vector term_infos; + std::string_view base_analyzer_fingerprint; + std::optional analyzer_ctx; + if (InvertedIndexAnalyzer::should_analyzer(analysis_properties)) { + analyzer_ctx.emplace(analyzer_context_from_properties(analysis_properties)); + base_analyzer_fingerprint = analyzer_ctx->analyzer_provider->base_analyzer_fingerprint(); + } + + if (clause_type == "MATCH") { + term_infos.emplace_back(value); + } else if (category == ClauseTypeCategory::TOKENIZED) { + if (analyzer_ctx.has_value()) { + term_infos = analyze_plain_query(value, *analyzer_ctx); + } else { + term_infos.emplace_back(value); + } + } else if (category == ClauseTypeCategory::NON_TOKENIZED) { + if (clause_type == "TERM" && analyzer_ctx.has_value()) { + term_infos = analyze_plain_query(value, *analyzer_ctx); + } else { + term_infos.emplace_back(value); + } + } + + std::string lucene_field_name = + build_field_name(index_meta->col_unique_ids()[0], candidates.index_suffix_path); + std::wstring ws_field_name = StringHelper::to_wstring(lucene_field_name); + + auto iter = collect_infos->find(ws_field_name); + if (iter == collect_infos->end()) { + CollectInfo collect_info; + RETURN_IF_ERROR(append_scoring_leaf(&collect_info, term_infos, base_analyzer_fingerprint)); + preserve_selected_index_metadata(candidates, index_meta, &collect_info); + (*collect_infos)[ws_field_name] = std::move(collect_info); + } else { + RETURN_IF_ERROR(validate_same_physical_index(iter->second, *index_meta)); + RETURN_IF_ERROR(append_scoring_leaf(&iter->second, term_infos, base_analyzer_fingerprint)); + } + + return Status::OK(); +} + +bool SearchPredicateCollector::is_score_query_type(const std::string& clause_type) const { + return clause_type == "TERM" || clause_type == "EXACT" || clause_type == "PHRASE" || + clause_type == "MATCH" || clause_type == "ANY" || clause_type == "ALL"; +} + +SearchPredicateCollector::ClauseTypeCategory SearchPredicateCollector::get_clause_type_category( + const std::string& clause_type) const { + if (clause_type == "AND" || clause_type == "OR" || clause_type == "NOT" || + clause_type == "OCCUR_BOOLEAN") { + return ClauseTypeCategory::COMPOUND; + } else if (clause_type == "TERM" || clause_type == "EXACT") { + return ClauseTypeCategory::NON_TOKENIZED; + } else if (clause_type == "PHRASE" || clause_type == "MATCH" || clause_type == "ANY" || + clause_type == "ALL") { + return ClauseTypeCategory::TOKENIZED; + } else { + LOG(WARNING) << "Unknown clause type '" << clause_type + << "', defaulting to NON_TOKENIZED category"; + return ClauseTypeCategory::NON_TOKENIZED; + } +} + +} // namespace doris diff --git a/be/src/storage/predicate_collector.h b/be/src/storage/index/inverted/similarity/predicate_collector.h similarity index 80% rename from be/src/storage/predicate_collector.h rename to be/src/storage/index/inverted/similarity/predicate_collector.h index c96e0af9c45ed5..c43865a63a52d3 100644 --- a/be/src/storage/predicate_collector.h +++ b/be/src/storage/index/inverted/similarity/predicate_collector.h @@ -17,10 +17,11 @@ #pragma once +#include #include -#include #include #include +#include #include "common/status.h" #include "exprs/vexpr_fwd.h" @@ -35,14 +36,20 @@ class TabletIndex; class TabletSchema; using TabletSchemaSPtr = std::shared_ptr; -struct TermInfoComparer { - bool operator()(const segment_v2::TermInfo& lhs, const segment_v2::TermInfo& rhs) const { - return lhs.term < rhs.term; - } +struct LogicalScoringClause { + uint32_t df_slot = 0; + int32_t position = 0; +}; + +struct LogicalScoringLeaf { + std::vector clauses; }; struct CollectInfo { - std::set term_infos; + std::vector unique_terms; + std::unordered_map unique_term_slots; + std::vector logical_scoring_leaves; + std::string expected_base_analyzer_fingerprint; std::shared_ptr owned_index_meta; const TabletIndex* index_meta = nullptr; }; @@ -73,16 +80,19 @@ class SearchPredicateCollector : public PredicateCollector { private: enum class ClauseTypeCategory { NON_TOKENIZED, TOKENIZED, COMPOUND }; + using FieldBindingMap = std::unordered_map; Status collect_from_clause(const TSearchClause& clause, RuntimeState* state, const TabletSchemaSPtr& tablet_schema, + const FieldBindingMap& field_bindings, CollectInfoMap* collect_infos); Status collect_from_leaf(const TSearchClause& clause, RuntimeState* state, - const TabletSchemaSPtr& tablet_schema, CollectInfoMap* collect_infos); + const TabletSchemaSPtr& tablet_schema, + const FieldBindingMap& field_bindings, CollectInfoMap* collect_infos); bool is_score_query_type(const std::string& clause_type) const; ClauseTypeCategory get_clause_type_category(const std::string& clause_type) const; }; using PredicateCollectorPtr = std::unique_ptr; -} // namespace doris \ No newline at end of file +} // namespace doris diff --git a/be/src/storage/index/inverted/token_filter/ascii_folding_filter_factory.h b/be/src/storage/index/inverted/token_filter/ascii_folding_filter_factory.h index 9710f0f3cadede..ee3321d96a27c7 100644 --- a/be/src/storage/index/inverted/token_filter/ascii_folding_filter_factory.h +++ b/be/src/storage/index/inverted/token_filter/ascii_folding_filter_factory.h @@ -35,6 +35,11 @@ class ASCIIFoldingFilterFactory : public TokenFilterFactory { return std::make_shared(in, _preserve_original); } + PositionCapability position_capability() const override { + return _preserve_original ? PositionCapability::kUnknown + : PositionCapability::kAlwaysUnitIncrement; + } + private: bool _preserve_original = false; }; diff --git a/be/src/storage/index/inverted/token_filter/common_grams_filter.cpp b/be/src/storage/index/inverted/token_filter/common_grams_filter.cpp new file mode 100644 index 00000000000000..2781ea58842975 --- /dev/null +++ b/be/src/storage/index/inverted/token_filter/common_grams_filter.cpp @@ -0,0 +1,384 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/inverted/token_filter/common_grams_filter.h" + +#include +#include + +#include "common/exception.h" +#include "common/logging.h" +#include "storage/index/inverted/common_grams/common_grams_key_codec.h" + +namespace doris::segment_v2::inverted_index { +namespace { + +void validate_input_token_shape(const Token& token, std::string_view term) { + if (token.getPositionIncrement() != 1) { + throw Exception(ErrorCode::INVERTED_INDEX_ANALYZER_ERROR, + "CommonGrams requires position increment 1, got {}", + token.getPositionIncrement()); + } + if (term.empty()) { + throw Exception(ErrorCode::INVERTED_INDEX_ANALYZER_ERROR, + "CommonGrams requires non-empty input tokens"); + } +} + +void validate_input_token(const Token& token, std::string_view term) { + validate_input_token_shape(token, term); + auto status = validate_common_grams_logical_term(term, "input token"); + if (!status.ok()) { + throw Exception(status); + } +} + +void set_output_token(Token* token, std::string_view term, int32_t position_increment, + int32_t start_offset, int32_t end_offset, const TCHAR* type) { + token->clear(); + token->setTextNoCopy(term.data(), static_cast(term.size())); + token->setPositionIncrement(position_increment); + token->setStartOffset(start_offset); + token->setEndOffset(end_offset); + token->setType(type); +} + +bool is_common_word(const CommonGramsBufferedToken& token, const CommonWordSet& common_words, + std::optional* cached_membership) { + if (!cached_membership->has_value()) { + cached_membership->emplace(common_words.contains(token.term)); + } + return cached_membership->value(); +} + +} // namespace + +bool common_grams_query_may_use_gram(std::span terms, CommonGramsQueryMode mode, + const CommonWordSet& common_words) { + if (terms.size() < 2) { + return false; + } + const size_t relevant_term_count = + mode == CommonGramsQueryMode::kPhrasePrefix ? terms.size() - 1 : terms.size(); + for (size_t i = 0; i < relevant_term_count; ++i) { + if (common_words.contains(terms[i])) { + return true; + } + } + return false; +} + +CommonGramsFilter::CommonGramsFilter(TokenStreamPtr in, + std::shared_ptr common_words, + CommonGramsOutputMode output_mode) + : DorisTokenFilter(std::move(in)), + _common_words(std::move(common_words)), + _output_mode(output_mode) { + DORIS_CHECK(_common_words != nullptr); +} + +bool CommonGramsFilter::read_input(CommonGramsBufferedToken* token, + std::optional* is_common) { + if (_in->next(&_input_token) == nullptr) { + return false; + } + std::string_view term(_input_token.termBuffer(), _input_token.termLength()); + validate_input_token(_input_token, term); + token->term.assign(term); + token->start_offset = _input_token.startOffset(); + token->end_offset = _input_token.endOffset(); + token->type = _input_token.type(); + is_common->reset(); + return true; +} + +Token* CommonGramsFilter::emit_unigram(Token* token, const CommonGramsBufferedToken& buffered) { + const std::string_view output_term = encode_plain_term(buffered.term); + set_output_token(token, output_term, 1, buffered.start_offset, buffered.end_offset, + buffered.type); + return token; +} + +std::string_view CommonGramsFilter::encode_plain_term(std::string_view term) { + if (_output_mode != CommonGramsOutputMode::kLogical && !term.empty() && + (term.front() == PLAIN_ESCAPE_PREFIX || term.front() == '\x1f')) { + if (!try_encode_escaped_plain_term_prevalidated(term, _encoded_plain_term)) { + throw Exception(Status::Error( + "CommonGrams escaped plain term would exceed the 16383-byte key limit; " + "set enable_common_grams_index_build=false and retry the import in a new " + "transaction")); + } + return _encoded_plain_term; + } + return term; +} + +std::string_view CommonGramsFilter::encode_snii_plain_term(std::string_view term) { + DORIS_CHECK(_output_mode == CommonGramsOutputMode::kEscapedV1SpimiIndex); + DORIS_CHECK(!term.empty()); + if (term.front() != PLAIN_ESCAPE_PREFIX && term.front() != '\x1f') { + return term; + } + if (term.size() == COMMON_GRAM_MAX_ENCODED_BYTES) { + throw Exception(Status::Error( + "CommonGrams escaped plain term would exceed the 16383-byte key limit; " + "set enable_common_grams_index_build=false and retry the import in a new " + "transaction")); + } + DORIS_CHECK_LT(term.size(), COMMON_GRAM_MAX_ENCODED_BYTES); + _encoded_plain_term.clear(); + _encoded_plain_term.reserve(term.size() + 1); + _encoded_plain_term.push_back(PLAIN_ESCAPE_PREFIX); + _encoded_plain_term.push_back(term.front() == PLAIN_ESCAPE_PREFIX ? 'E' : 'G'); + _encoded_plain_term.append(term.substr(1)); + return _encoded_plain_term; +} + +Token* CommonGramsFilter::emit_gram(Token* token, int32_t start_offset, int32_t end_offset) { + set_output_token(token, _gram, 0, start_offset, end_offset, COMMON_GRAM_TOKEN_TYPE); + return token; +} + +Token* CommonGramsFilter::next(Token* token) { + if (_emit_current) { + _emit_current = false; + return emit_unigram(token, _current); + } + + if (!_has_current) { + if (!read_input(&_current, &_current_is_common)) { + return nullptr; + } + _has_current = true; + return emit_unigram(token, _current); + } + + if (!read_input(&_lookahead, &_lookahead_is_common)) { + _has_current = false; + return nullptr; + } + + const bool uses_gram = is_common_word(_current, *_common_words, &_current_is_common) || + is_common_word(_lookahead, *_common_words, &_lookahead_is_common); + if (uses_gram) { + const bool encoded = + try_encode_common_gram_prevalidated(_current.term, _lookahead.term, _gram); + if (encoded) { + const int32_t start_offset = _current.start_offset; + const int32_t end_offset = _lookahead.end_offset; + std::swap(_current, _lookahead); + std::swap(_current_is_common, _lookahead_is_common); + _emit_current = true; + return emit_gram(token, start_offset, end_offset); + } + } + + std::swap(_current, _lookahead); + std::swap(_current_is_common, _lookahead_is_common); + return emit_unigram(token, _current); +} + +bool CommonGramsFilter::next_snii_index_event(SniiCommonGramsIndexEvent* event) { + DORIS_CHECK(event != nullptr); + DORIS_CHECK(_output_mode == CommonGramsOutputMode::kEscapedV1SpimiIndex); + if (_in->next(&_input_token) == nullptr) { + return false; + } + + const std::string_view logical_term(_input_token.termBuffer(), + _input_token.termLength()); + validate_input_token_shape(_input_token, logical_term); + const bool requires_prevalidation = + logical_term.size() > COMMON_GRAM_MAX_ENCODED_BYTES || + (logical_term.size() == COMMON_GRAM_MAX_ENCODED_BYTES && + (logical_term.front() == PLAIN_ESCAPE_PREFIX || logical_term.front() == '\x1f')); + if (requires_prevalidation) { + auto status = validate_common_grams_logical_term(logical_term, "input token"); + if (!status.ok()) { + throw Exception(status); + } + } + event->logical_term = logical_term; + event->plain_term = encode_snii_plain_term(logical_term); + return true; +} + +void CommonGramsFilter::reset() { + DorisTokenFilter::reset(); + _current.term.clear(); + _current.start_offset = 0; + _current.end_offset = 0; + _current.type = Token::getDefaultType(); + _lookahead.term.clear(); + _lookahead.start_offset = 0; + _lookahead.end_offset = 0; + _lookahead.type = Token::getDefaultType(); + _current_is_common.reset(); + _lookahead_is_common.reset(); + _has_current = false; + _emit_current = false; + _gram.clear(); + _encoded_plain_term.clear(); +} + +Token* CommonGramsPositionFilter::next(Token* token) { + if (_in->next(token) == nullptr) { + return nullptr; + } + const std::string_view term(token->termBuffer(), token->termLength()); + validate_input_token(*token, term); + return token; +} + +CommonGramsQueryFilterBase::CommonGramsQueryFilterBase( + TokenStreamPtr in, std::shared_ptr common_words, + CommonGramsQueryMode mode) + : DorisTokenFilter(std::move(in)), _common_words(std::move(common_words)), _mode(mode) { + DORIS_CHECK(_common_words != nullptr); +} + +bool CommonGramsQueryFilterBase::pair_uses_gram( + const std::vector& unigrams, std::optional* left_is_common, + size_t pair_index) const { + const bool is_prefix_boundary = + _mode == CommonGramsQueryMode::kPhrasePrefix && pair_index + 2 == unigrams.size(); + if (is_prefix_boundary) { + return is_common_word(unigrams[pair_index], *_common_words, left_is_common); + } + + std::optional right_is_common; + const bool uses_gram = + is_common_word(unigrams[pair_index], *_common_words, left_is_common) || + is_common_word(unigrams[pair_index + 1], *_common_words, &right_is_common); + *left_is_common = right_is_common; + return uses_gram; +} + +void CommonGramsQueryFilterBase::append_plain_output(std::vector* output, + const CommonGramsBufferedToken& token) { + output->push_back(token); +} + +bool CommonGramsQueryFilterBase::append_gram_output( + std::vector* output, + const std::optional& indexed_gram, + const CommonGramsBufferedToken& left, const CommonGramsBufferedToken& right) { + if (!indexed_gram.has_value()) { + DCHECK(!is_common_gram_encodable(left.term, right.term)); + return false; + } + CommonGramsBufferedToken query_gram = *indexed_gram; + query_gram.type = COMMON_GRAM_TOKEN_TYPE; + output->push_back(std::move(query_gram)); + return true; +} + +void CommonGramsQueryFilterBase::prepare_output() { + std::vector unigrams; + std::vector> indexed_grams; + Token token; + while (_in->next(&token) != nullptr) { + const std::string_view term(token.termBuffer(), token.termLength()); + if (token.getPositionIncrement() == 0) { + if (!is_common_gram_token_type(token.type()) || unigrams.empty() || + indexed_grams.back().has_value()) { + throw Exception(ErrorCode::INVERTED_INDEX_ANALYZER_ERROR, + "Invalid indexed CommonGrams token sequence"); + } + auto status = validate_common_grams_logical_term(term, "indexed gram"); + if (!status.ok()) { + throw Exception(status); + } + indexed_grams.back() = CommonGramsBufferedToken {.term = std::string(term), + .start_offset = token.startOffset(), + .end_offset = token.endOffset(), + .type = token.type()}; + continue; + } + validate_input_token(token, term); + unigrams.push_back({.term = std::string(term), + .start_offset = token.startOffset(), + .end_offset = token.endOffset(), + .type = token.type()}); + indexed_grams.emplace_back(); + } + + std::vector output; + if (unigrams.size() < 2) { + _output = std::move(unigrams); + _prepared = true; + return; + } + + bool last_pair_used_gram = false; + std::optional left_is_common; + for (size_t i = 0; i + 1 < unigrams.size(); ++i) { + last_pair_used_gram = pair_uses_gram(unigrams, &left_is_common, i); + if (last_pair_used_gram) { + if (!append_gram_output(&output, indexed_grams[i], unigrams[i], unigrams[i + 1])) { + output.clear(); + for (const auto& unigram : unigrams) { + append_plain_output(&output, unigram); + } + _output = std::move(output); + _prepared = true; + return; + } + } else { + append_plain_output(&output, unigrams[i]); + } + } + if (!last_pair_used_gram) { + append_plain_output(&output, unigrams.back()); + } + _output = std::move(output); + _prepared = true; +} + +Token* CommonGramsQueryFilterBase::emit(Token* token, const CommonGramsBufferedToken& buffered) { + set_output_token(token, buffered.term, 1, buffered.start_offset, buffered.end_offset, + buffered.type); + return token; +} + +Token* CommonGramsQueryFilterBase::next(Token* token) { + if (_failure != nullptr) { + std::rethrow_exception(_failure); + } + if (!_prepared) { + try { + prepare_output(); + } catch (...) { + _failure = std::current_exception(); + throw; + } + } + if (_next_output == _output.size()) { + return nullptr; + } + return emit(token, _output[_next_output++]); +} + +void CommonGramsQueryFilterBase::reset() { + DorisTokenFilter::reset(); + _output.clear(); + _next_output = 0; + _prepared = false; + _failure = nullptr; +} + +} // namespace doris::segment_v2::inverted_index diff --git a/be/src/storage/index/inverted/token_filter/common_grams_filter.h b/be/src/storage/index/inverted/token_filter/common_grams_filter.h new file mode 100644 index 00000000000000..5fae9ce79890ac --- /dev/null +++ b/be/src/storage/index/inverted/token_filter/common_grams_filter.h @@ -0,0 +1,152 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include "storage/index/inverted/common_grams/common_word_set.h" +#include "storage/index/inverted/token_filter/token_filter.h" + +namespace doris::segment_v2::inverted_index { + +inline constexpr const TCHAR* COMMON_GRAM_TOKEN_TYPE = L"common_gram"; + +inline bool is_common_gram_token_type(const TCHAR* type) { + return std::wstring_view(type) == COMMON_GRAM_TOKEN_TYPE; +} + +struct CommonGramsBufferedToken { + std::string term; + int32_t start_offset = 0; + int32_t end_offset = 0; + const TCHAR* type = Token::getDefaultType(); +}; + +enum class CommonGramsOutputMode { + kLogical, + kEscapedV1Index, + kEscapedV1SpimiIndex, +}; + +struct SniiCommonGramsIndexEvent { + // Analyzer output before the EscapedV1 namespace transform. The view has the + // same lifetime as plain_term and is used for validate-on-first-intern. + std::string_view logical_term; + // EscapedV1 physical plain key. The view remains valid until the next event + // call or reset; the SNII writer interns it synchronously. + std::string_view plain_term; +}; + +class CommonGramsFilter final : public DorisTokenFilter { +public: + CommonGramsFilter(TokenStreamPtr in, std::shared_ptr common_words, + CommonGramsOutputMode output_mode = CommonGramsOutputMode::kLogical); + + Token* next(Token* token) override; + bool next_snii_index_event(SniiCommonGramsIndexEvent* event); + void reset() override; + const CommonWordSet& common_words() const { return *_common_words; } + +private: + bool read_input(CommonGramsBufferedToken* token, std::optional* is_common); + std::string_view encode_plain_term(std::string_view term); + std::string_view encode_snii_plain_term(std::string_view term); + Token* emit_unigram(Token* token, const CommonGramsBufferedToken& buffered); + Token* emit_gram(Token* token, int32_t start_offset, int32_t end_offset); + + std::shared_ptr _common_words; + CommonGramsOutputMode _output_mode; + Token _input_token; + CommonGramsBufferedToken _current; + CommonGramsBufferedToken _lookahead; + std::optional _current_is_common; + std::optional _lookahead_is_common; + bool _has_current = false; + bool _emit_current = false; + std::string _gram; + std::string _encoded_plain_term; +}; + +class CommonGramsPositionFilter final : public DorisTokenFilter { +public: + explicit CommonGramsPositionFilter(TokenStreamPtr in) : DorisTokenFilter(std::move(in)) {} + + Token* next(Token* token) override; +}; + +enum class CommonGramsQueryMode { + kExact, + kPhrasePrefix, +}; + +// A false result proves that the purpose-specific query filter cannot select a +// gram. A true result stays conservative because key encoding may still force +// the filter to replay the complete plain stream. +bool common_grams_query_may_use_gram(std::span terms, CommonGramsQueryMode mode, + const CommonWordSet& common_words); + +class CommonGramsQueryFilterBase : public DorisTokenFilter { +public: + CommonGramsQueryFilterBase(TokenStreamPtr in, std::shared_ptr common_words, + CommonGramsQueryMode mode); + + Token* next(Token* token) override; + void reset() override; + +private: + void prepare_output(); + bool pair_uses_gram(const std::vector& unigrams, + std::optional* left_is_common, size_t pair_index) const; + static void append_plain_output(std::vector* output, + const CommonGramsBufferedToken& token); + static bool append_gram_output(std::vector* output, + const std::optional& indexed_gram, + const CommonGramsBufferedToken& left, + const CommonGramsBufferedToken& right); + static Token* emit(Token* token, const CommonGramsBufferedToken& buffered); + + std::shared_ptr _common_words; + CommonGramsQueryMode _mode; + std::vector _output; + size_t _next_output = 0; + bool _prepared = false; + std::exception_ptr _failure; +}; + +class CommonGramsQueryFilter final : public CommonGramsQueryFilterBase { +public: + CommonGramsQueryFilter(TokenStreamPtr in, std::shared_ptr common_words) + : CommonGramsQueryFilterBase(std::move(in), std::move(common_words), + CommonGramsQueryMode::kExact) {} +}; + +class CommonGramsPhrasePrefixFilter final : public CommonGramsQueryFilterBase { +public: + CommonGramsPhrasePrefixFilter(TokenStreamPtr in, + std::shared_ptr common_words) + : CommonGramsQueryFilterBase(std::move(in), std::move(common_words), + CommonGramsQueryMode::kPhrasePrefix) {} +}; + +} // namespace doris::segment_v2::inverted_index diff --git a/be/src/storage/index/inverted/token_filter/common_grams_filter_factory.h b/be/src/storage/index/inverted/token_filter/common_grams_filter_factory.h new file mode 100644 index 00000000000000..7803c7f00b6004 --- /dev/null +++ b/be/src/storage/index/inverted/token_filter/common_grams_filter_factory.h @@ -0,0 +1,107 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include "common/exception.h" +#include "storage/index/inverted/token_filter/common_grams_filter.h" +#include "storage/index/inverted/token_filter/token_filter_factory.h" + +namespace doris::segment_v2::inverted_index { + +class CommonGramsFilterFactory final : public TokenFilterFactory { +public: + explicit CommonGramsFilterFactory(std::shared_ptr common_words = nullptr) + : _common_words(std::move(common_words)) {} + + void initialize(const Settings& settings) override { + if (!settings.empty()) { + throw Exception(ErrorCode::INVERTED_INDEX_ANALYZER_ERROR, + "common_grams does not accept settings"); + } + if (_common_words == nullptr) { + _common_words = CommonWordSet::default_word_set(); + } + } + + TokenFilterPtr create(const TokenStreamPtr& in) override { + return std::make_shared(in, _common_words, _output_mode); + } + + void set_common_words(std::shared_ptr common_words) { + _common_words = std::move(common_words); + } + + void set_output_mode(CommonGramsOutputMode output_mode) { _output_mode = output_mode; } + + const std::shared_ptr& common_words() const { return _common_words; } + +private: + std::shared_ptr _common_words; + CommonGramsOutputMode _output_mode = CommonGramsOutputMode::kLogical; +}; + +class CommonGramsPositionFilterFactory final : public TokenFilterFactory { +public: + void initialize(const Settings&) override {} + + TokenFilterPtr create(const TokenStreamPtr& in) override { + return std::make_shared(in); + } + + PositionCapability position_capability() const override { + return PositionCapability::kAlwaysUnitIncrement; + } +}; + +class CommonGramsQueryFilterFactory final : public TokenFilterFactory { +public: + explicit CommonGramsQueryFilterFactory(std::shared_ptr common_words) + : _common_words(std::move(common_words)) {} + + void initialize(const Settings&) override {} + + TokenFilterPtr create(const TokenStreamPtr& in) override { + return std::make_shared(in, _common_words); + } + + const std::shared_ptr& common_words() const { return _common_words; } + +private: + std::shared_ptr _common_words; +}; + +class CommonGramsPhrasePrefixFilterFactory final : public TokenFilterFactory { +public: + explicit CommonGramsPhrasePrefixFilterFactory(std::shared_ptr common_words) + : _common_words(std::move(common_words)) {} + + void initialize(const Settings&) override {} + + TokenFilterPtr create(const TokenStreamPtr& in) override { + return std::make_shared(in, _common_words); + } + + const std::shared_ptr& common_words() const { return _common_words; } + +private: + std::shared_ptr _common_words; +}; + +} // namespace doris::segment_v2::inverted_index diff --git a/be/src/storage/index/inverted/token_filter/empty_token_filter_factory.h b/be/src/storage/index/inverted/token_filter/empty_token_filter_factory.h index 2855546e78bdb1..d1c42966f14a11 100644 --- a/be/src/storage/index/inverted/token_filter/empty_token_filter_factory.h +++ b/be/src/storage/index/inverted/token_filter/empty_token_filter_factory.h @@ -46,6 +46,10 @@ class EmptyTokenFilterFactory : public TokenFilterFactory { token_filter->initialize(); return token_filter; } + + PositionCapability position_capability() const override { + return PositionCapability::kAlwaysUnitIncrement; + } }; } // namespace doris::segment_v2::inverted_index \ No newline at end of file diff --git a/be/src/storage/index/inverted/token_filter/icu_normalizer_filter_factory.h b/be/src/storage/index/inverted/token_filter/icu_normalizer_filter_factory.h index c0cedb8c483cbd..27dc897e818a34 100644 --- a/be/src/storage/index/inverted/token_filter/icu_normalizer_filter_factory.h +++ b/be/src/storage/index/inverted/token_filter/icu_normalizer_filter_factory.h @@ -82,6 +82,10 @@ class ICUNormalizerFilterFactory : public TokenFilterFactory { return std::make_shared(in, _normalizer); } + PositionCapability position_capability() const override { + return PositionCapability::kAlwaysUnitIncrement; + } + private: static const icu::Normalizer2* get_normalizer(const std::string& name, UErrorCode& status) { std::string lower_name = to_lower_copy(trim_copy(name)); diff --git a/be/src/storage/index/inverted/token_filter/lower_case_filter.h b/be/src/storage/index/inverted/token_filter/lower_case_filter.h index dd55d48060d03a..ff737209a1ea09 100644 --- a/be/src/storage/index/inverted/token_filter/lower_case_filter.h +++ b/be/src/storage/index/inverted/token_filter/lower_case_filter.h @@ -19,10 +19,40 @@ #include +#ifdef BE_TEST +#include +#endif + +#include "common/cast_set.h" +#include "common/exception.h" #include "storage/index/inverted/token_filter/token_filter.h" +#include "util/utf8_check.h" namespace doris::segment_v2::inverted_index { +#ifdef BE_TEST +namespace lower_case_testing { + +inline std::atomic& unicode_path_counter() { + static std::atomic counter {0}; + return counter; +} + +inline uint64_t unicode_path_count() { + return unicode_path_counter().load(std::memory_order_relaxed); +} + +inline void reset_unicode_path_count() { + unicode_path_counter().store(0, std::memory_order_relaxed); +} + +inline void note_unicode_path() { + unicode_path_counter().fetch_add(1, std::memory_order_relaxed); +} + +} // namespace lower_case_testing +#endif + /** * @brief A token filter that converts Unicode text to lowercase using ICU library. * @@ -40,7 +70,7 @@ class LowerCaseFilter : public DorisTokenFilter { UErrorCode status = U_ZERO_ERROR; auto* ucsm = ucasemap_open("", 0, &status); if (U_FAILURE(status)) { - throw Exception(ErrorCode::RUNTIME_ERROR, + throw Exception(ErrorCode::INVERTED_INDEX_ANALYZER_ERROR, "Failed to open UCaseMap. ICU Error: " + std::to_string(status) + " - " + u_errorName(status)); } @@ -52,21 +82,53 @@ class LowerCaseFilter : public DorisTokenFilter { return nullptr; } std::string_view term(t->termBuffer(), t->termLength()); - - size_t max_len = term.size() * 2; - if (_lower_term.size() < max_len) { - _lower_term.resize(max_len); + bool has_ascii_upper = false; + bool all_ascii = true; + for (const char value : term) { + const auto byte = static_cast(value); + if (byte >= 0x80) { + all_ascii = false; + break; + } + has_ascii_upper |= byte >= 'A' && byte <= 'Z'; + } + if (all_ascii) { + if (!has_ascii_upper) { + return t; + } + _lower_term.resize(term.size()); + for (size_t i = 0; i < term.size(); ++i) { + const auto byte = static_cast(term[i]); + _lower_term[i] = + static_cast(byte >= 'A' && byte <= 'Z' ? byte + ('a' - 'A') : byte); + } + set_text(t, _lower_term); + return t; + } +#ifdef BE_TEST + lower_case_testing::note_unicode_path(); +#endif + if (!validate_utf8(term.data(), term.size())) { + throw Exception(ErrorCode::INVERTED_INDEX_ANALYZER_ERROR, + "Failed to lowercase token: invalid UTF-8"); } + _lower_term.resize(term.size()); UErrorCode status = U_ZERO_ERROR; - int32_t result_len = ucasemap_utf8ToLower(_ucsm.get(), _lower_term.data(), max_len, - term.data(), term.size(), &status); + int32_t result_len = ucasemap_utf8ToLower( + _ucsm.get(), _lower_term.data(), cast_set(_lower_term.size()), term.data(), + cast_set(term.size()), &status); + if (status == U_BUFFER_OVERFLOW_ERROR) { + _lower_term.resize(cast_set(result_len)); + status = U_ZERO_ERROR; + result_len = ucasemap_utf8ToLower(_ucsm.get(), _lower_term.data(), + cast_set(_lower_term.size()), term.data(), + cast_set(term.size()), &status); + } if (U_FAILURE(status)) { - LOG(WARNING) << "Failed to convert to lowercase. " - << "Term: '" << term << "', " - << "ICU Error: " << status << " - " << u_errorName(status) - << ", Buffer size: " << max_len << std::endl; - return nullptr; + throw Exception(ErrorCode::INVERTED_INDEX_ANALYZER_ERROR, + "Failed to convert token to lowercase. ICU Error: {} - {}", + static_cast(status), u_errorName(status)); } set_text(t, std::string_view(_lower_term.data(), result_len)); diff --git a/be/src/storage/index/inverted/token_filter/lower_case_filter_factory.h b/be/src/storage/index/inverted/token_filter/lower_case_filter_factory.h index 03467c5114a6b4..b24ad4b0822944 100644 --- a/be/src/storage/index/inverted/token_filter/lower_case_filter_factory.h +++ b/be/src/storage/index/inverted/token_filter/lower_case_filter_factory.h @@ -34,6 +34,10 @@ class LowerCaseFilterFactory : public TokenFilterFactory { token_filter->initialize(); return token_filter; } + + PositionCapability position_capability() const override { + return PositionCapability::kAlwaysUnitIncrement; + } }; } // namespace doris::segment_v2::inverted_index \ No newline at end of file diff --git a/be/src/storage/index/inverted/token_filter/token_filter_factory.h b/be/src/storage/index/inverted/token_filter/token_filter_factory.h index ebbd8836715e8d..5e1bd93236977a 100644 --- a/be/src/storage/index/inverted/token_filter/token_filter_factory.h +++ b/be/src/storage/index/inverted/token_filter/token_filter_factory.h @@ -28,6 +28,7 @@ class TokenFilterFactory : public AbstractAnalysisFactory { ~TokenFilterFactory() override = default; virtual TokenFilterPtr create(const TokenStreamPtr& in) = 0; + virtual PositionCapability position_capability() const { return PositionCapability::kUnknown; } }; using TokenFilterFactoryPtr = std::shared_ptr; diff --git a/be/src/storage/index/inverted/tokenizer/char/char_group_tokenizer_factory.h b/be/src/storage/index/inverted/tokenizer/char/char_group_tokenizer_factory.h index 030c0c2512f850..68273ad1366783 100644 --- a/be/src/storage/index/inverted/tokenizer/char/char_group_tokenizer_factory.h +++ b/be/src/storage/index/inverted/tokenizer/char/char_group_tokenizer_factory.h @@ -30,6 +30,10 @@ class CharGroupTokenizerFactory : public TokenizerFactory { TokenizerPtr create() override; + PositionCapability position_capability() const override { + return PositionCapability::kAlwaysUnitIncrement; + } + private: static UChar32 parse_escaped_char(const icu::UnicodeString& unicode_str); diff --git a/be/src/storage/index/inverted/tokenizer/char/char_tokenizer.cpp b/be/src/storage/index/inverted/tokenizer/char/char_tokenizer.cpp index d7b774ef0966a4..54ac1d72c9de82 100644 --- a/be/src/storage/index/inverted/tokenizer/char/char_tokenizer.cpp +++ b/be/src/storage/index/inverted/tokenizer/char/char_tokenizer.cpp @@ -17,10 +17,32 @@ #include "storage/index/inverted/tokenizer/char/char_tokenizer.h" +#ifdef BE_TEST +#include +#endif + #include "common/exception.h" namespace doris::segment_v2::inverted_index { +#ifdef BE_TEST +namespace { +std::atomic g_non_ascii_decode_count {0}; +} // namespace + +namespace char_tokenizer_testing { + +uint64_t non_ascii_decode_count() { + return g_non_ascii_decode_count.load(std::memory_order_relaxed); +} + +void reset_non_ascii_decode_count() { + g_non_ascii_decode_count.store(0, std::memory_order_relaxed); +} + +} // namespace char_tokenizer_testing +#endif + void CharTokenizer::initialize(int32_t max_token_len) { if (max_token_len > MAX_TOKEN_LENGTH_LIMIT || max_token_len <= 0) { throw Exception(ErrorCode::INVALID_ARGUMENT, @@ -29,6 +51,33 @@ void CharTokenizer::initialize(int32_t max_token_len) { " passed: " + std::to_string(max_token_len)); } _max_token_len = max_token_len; + for (size_t value = 0; value < _ascii_char_classes.size(); ++value) { + const auto c = static_cast(value); + _ascii_char_classes[value] = + is_cjk_char(c) + ? AsciiCharClass::kCjk + : (is_token_char(c) ? AsciiCharClass::kToken : AsciiCharClass::kDelimiter); + } +} + +CharTokenizer::AsciiCharClass CharTokenizer::read_next_char_class() { + const auto first_byte = static_cast(_char_buffer[_buffer_index]); + if (first_byte < _ascii_char_classes.size()) { + ++_buffer_index; + return _ascii_char_classes[first_byte]; + } +#ifdef BE_TEST + g_non_ascii_decode_count.fetch_add(1, std::memory_order_relaxed); +#endif + UChar32 c = U_UNASSIGNED; + U8_NEXT(_char_buffer, _buffer_index, _data_len, c); + if (c < 0) { + return AsciiCharClass::kInvalid; + } + if (is_cjk_char(c)) { + return AsciiCharClass::kCjk; + } + return is_token_char(c) ? AsciiCharClass::kToken : AsciiCharClass::kDelimiter; } Token* CharTokenizer::next(Token* token) { @@ -46,14 +95,13 @@ Token* CharTokenizer::next(Token* token) { break; } - UChar32 c = U_UNASSIGNED; const int32_t prev_i = _buffer_index; - U8_NEXT(_char_buffer, _buffer_index, _data_len, c); - if (c < 0) { + const AsciiCharClass char_class = read_next_char_class(); + if (char_class == AsciiCharClass::kInvalid) { continue; } - if (is_cjk_char(c)) { + if (char_class == AsciiCharClass::kCjk) { if (start == -1) { start = prev_i; end = _buffer_index - 1; @@ -61,7 +109,7 @@ Token* CharTokenizer::next(Token* token) { _buffer_index = prev_i; } break; - } else if (is_token_char(c)) { + } else if (char_class == AsciiCharClass::kToken) { if (start == -1) { start = prev_i; } diff --git a/be/src/storage/index/inverted/tokenizer/char/char_tokenizer.h b/be/src/storage/index/inverted/tokenizer/char/char_tokenizer.h index 66a9979a9fc97b..bae2f6c55d734f 100644 --- a/be/src/storage/index/inverted/tokenizer/char/char_tokenizer.h +++ b/be/src/storage/index/inverted/tokenizer/char/char_tokenizer.h @@ -17,6 +17,9 @@ #pragma once +#include +#include + #include "storage/index/inverted/tokenizer/tokenizer.h" namespace doris::segment_v2::inverted_index { @@ -36,8 +39,13 @@ class CharTokenizer : public DorisTokenizer { static constexpr int32_t DEFAULT_MAX_WORD_LEN = 255; private: + enum class AsciiCharClass : uint8_t { kDelimiter, kToken, kCjk, kInvalid }; + static constexpr int32_t MAX_TOKEN_LENGTH_LIMIT = 16383; + AsciiCharClass read_next_char_class(); + + std::array _ascii_char_classes {}; int32_t _max_token_len = 0; int32_t _buffer_index = 0; diff --git a/be/src/storage/index/inverted/tokenizer/empty/empty_tokenizer_factory.h b/be/src/storage/index/inverted/tokenizer/empty/empty_tokenizer_factory.h index 27229ba99f1184..b540dfa45a3f61 100644 --- a/be/src/storage/index/inverted/tokenizer/empty/empty_tokenizer_factory.h +++ b/be/src/storage/index/inverted/tokenizer/empty/empty_tokenizer_factory.h @@ -67,6 +67,10 @@ class EmptyTokenizerFactory : public TokenizerFactory { tokenizer->initialize(); return tokenizer; } + + PositionCapability position_capability() const override { + return PositionCapability::kAlwaysUnitIncrement; + } }; } // namespace doris::segment_v2::inverted_index \ No newline at end of file diff --git a/be/src/storage/index/inverted/tokenizer/ngram/edge_ngram_tokenizer_factory.h b/be/src/storage/index/inverted/tokenizer/ngram/edge_ngram_tokenizer_factory.h index 8d63b5cd230582..a0b88153a7a11f 100644 --- a/be/src/storage/index/inverted/tokenizer/ngram/edge_ngram_tokenizer_factory.h +++ b/be/src/storage/index/inverted/tokenizer/ngram/edge_ngram_tokenizer_factory.h @@ -56,6 +56,10 @@ class EdgeNGramTokenizerFactory : public TokenizerFactory { } } + PositionCapability position_capability() const override { + return PositionCapability::kAlwaysUnitIncrement; + } + private: int32_t _min_gram = 0; int32_t _max_gram = 0; diff --git a/be/src/storage/index/inverted/tokenizer/ngram/ngram_tokenizer_factory.h b/be/src/storage/index/inverted/tokenizer/ngram/ngram_tokenizer_factory.h index d064749d9d51a5..2ee428e32ff77f 100644 --- a/be/src/storage/index/inverted/tokenizer/ngram/ngram_tokenizer_factory.h +++ b/be/src/storage/index/inverted/tokenizer/ngram/ngram_tokenizer_factory.h @@ -50,6 +50,10 @@ class NGramTokenizerFactory : public TokenizerFactory { } } + PositionCapability position_capability() const override { + return PositionCapability::kAlwaysUnitIncrement; + } + static void initialize_matchers(); static CharMatcherPtr parse_token_chars(const Settings& settings); diff --git a/be/src/storage/index/inverted/tokenizer/tokenizer_factory.h b/be/src/storage/index/inverted/tokenizer/tokenizer_factory.h index b788990ecf09be..654726a9b976bb 100644 --- a/be/src/storage/index/inverted/tokenizer/tokenizer_factory.h +++ b/be/src/storage/index/inverted/tokenizer/tokenizer_factory.h @@ -28,6 +28,7 @@ class TokenizerFactory : public AbstractAnalysisFactory { ~TokenizerFactory() override = default; virtual TokenizerPtr create() = 0; + virtual PositionCapability position_capability() const { return PositionCapability::kUnknown; } }; using TokenizerFactoryPtr = std::shared_ptr; diff --git a/be/src/storage/index/snii/common/slice.h b/be/src/storage/index/snii/common/slice.h new file mode 100644 index 00000000000000..e5b80932944df3 --- /dev/null +++ b/be/src/storage/index/snii/common/slice.h @@ -0,0 +1,56 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include + +namespace doris::snii { + +// Read-only byte view (does not own memory). Lifetime is managed by the underlying buffer. +class Slice { +public: + Slice() = default; + Slice(const uint8_t* d, size_t n) : data_(d), size_(n) {} + explicit Slice(const std::vector& v) : data_(v.data()), size_(v.size()) {} + explicit Slice(std::string_view sv) + : data_(reinterpret_cast(sv.data())), size_(sv.size()) {} + + const uint8_t* data() const { return data_; } + size_t size() const { return size_; } + bool empty() const { return size_ == 0; } + + uint8_t operator[](size_t i) const { + assert(i < size_); + return data_[i]; + } + + Slice subslice(size_t off, size_t n) const { + assert(off + n <= size_); + return Slice(data_ + off, n); + } + +private: + const uint8_t* data_ = nullptr; + size_t size_ = 0; +}; + +} // namespace doris::snii diff --git a/be/src/storage/index/snii/common/uninitialized_buffer.h b/be/src/storage/index/snii/common/uninitialized_buffer.h new file mode 100644 index 00000000000000..1b1855db566d0c --- /dev/null +++ b/be/src/storage/index/snii/common/uninitialized_buffer.h @@ -0,0 +1,82 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include + +// Single auditable seam for "resize a decode buffer that is about to be fully +// overwritten". Collapses the scattered `assign(n, 0)` / `resize(n)` calls in the +// PFOR/zstd decode paths into one intent-revealing entry point and removes the +// redundant zero-fill on warm-reused buffers. +namespace doris::snii { + +// Resize a vector of trivially-copyable T to `n` without any zero-fill beyond what +// std::vector itself mandates. +// +// CONTRACT: the caller MUST fully overwrite [0, n) before reading any element. The +// PFOR (`pfor_decode`/`bitunpack`, including the w==0 memset and exception patch +// paths) and zstd (`ZSTD_decompress` + length check) decoders all satisfy this. +// +// For a warm-reused buffer whose current size() >= n this performs NO +// value-initialization (resize shrinks in place). For a cold/grown buffer +// std::vector still value-initializes the new tail -- that is unavoidable for +// std::vector and is intentional here. Do NOT use on any buffer that may be read +// before being fully overwritten. +template +inline void resize_uninitialized(std::vector& v, std::size_t n) { + static_assert(std::is_trivially_copyable_v, + "resize_uninitialized requires a trivially-copyable element type"); + v.resize(n); +} + +// Allocator that default-initializes (instead of value-initializes) on the no-arg +// construct path: for trivial T this skips zeroing entirely. Use only for buffers +// that are fully overwritten before any read. +template +struct default_init_allocator : std::allocator { + template + struct rebind { + using other = default_init_allocator; + }; + using std::allocator::allocator; + + template + void construct(U* p, Args&&... args) { + if constexpr (sizeof...(Args) == 0) { + ::new (static_cast(p)) U; // default-init: no zeroing for trivial U + } else { + ::new (static_cast(p)) U(std::forward(args)...); + } + } +}; + +// A vector whose grow path default-initializes, so even a cold grow avoids the +// zero-fill that std::vector would perform for trivial T. +template +using uninitialized_vector = std::vector>; + +template +inline void resize_uninitialized(uninitialized_vector& v, std::size_t n) { + v.resize(n); +} + +} // namespace doris::snii diff --git a/be/src/storage/index/snii/compaction/eligibility.cpp b/be/src/storage/index/snii/compaction/eligibility.cpp new file mode 100644 index 00000000000000..92b8676e979ad7 --- /dev/null +++ b/be/src/storage/index/snii/compaction/eligibility.cpp @@ -0,0 +1,386 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/compaction/eligibility.h" + +#include +#include +#include +#include + +#include "CLucene.h" +#include "common/check.h" +#include "common/config.h" +#include "common/exception.h" +#include "storage/index/inverted/analyzer/analyzer.h" +#include "storage/index/inverted/common_grams/common_grams_segment_metadata.h" +#include "storage/index/snii/format/format_constants.h" +#include "storage/index/snii/format/phrase_bigram.h" +#include "storage/index/snii/reader/logical_index_reader.h" +#include "storage/tablet/tablet_schema.h" + +namespace doris::snii::compaction { + +namespace inverted_index = segment_v2::inverted_index; + +namespace { + +Status reject(std::string_view reason) { + return Status::Error( + "SNII streamed compaction is not eligible: {}", reason); +} + +InvertedIndexAnalyzerConfig analyzer_config_from_properties( + const std::map& properties) { + InvertedIndexAnalyzerConfig config; + config.analyzer_name = get_analyzer_name_from_properties(properties); + config.parser_type = get_inverted_index_parser_type_from_string( + get_parser_string_from_properties(properties)); + config.parser_mode = get_parser_mode_string_from_properties(properties); + config.char_filter_map = get_parser_char_filter_map_from_properties(properties); + config.lower_case = get_parser_lowercase_from_properties(properties); + config.stop_words = get_parser_stopwords_from_properties(properties); + return config; +} + +Status validate_source_shape(const reader::LogicalIndexReader& source, size_t source_ordinal) { + if (source.tier() != format::IndexTier::kT2) { + return reject(fmt::format("source {} is not T2", source_ordinal)); + } + if (!source.has_positions()) { + return reject(fmt::format("source {} has no positions", source_ordinal)); + } + const auto& norms = source.section_refs().norms; + if (norms.offset != 0 || norms.length != 0) { + return reject(fmt::format("source {} carries scoring norms", source_ordinal)); + } + if (source.common_grams_metadata() != nullptr) { + return reject( + fmt::format("source {} carries CommonGrams/scoring metadata", source_ordinal)); + } + + const auto& stats = source.stats(); + if (stats.indexed_doc_count > stats.doc_count || + stats.null_count != stats.doc_count - stats.indexed_doc_count) { + return reject(fmt::format("source {} document statistics violate indexed + null = doc", + source_ordinal)); + } + return Status::OK(); +} + +inverted_index::CommonGramsSegmentMetadata static_common_grams_metadata( + const inverted_index::CommonGramsSegmentMetadata& metadata) { + auto seed = metadata; + seed.scoring_doc_count = 0; + seed.scoring_token_count = 0; + return seed; +} + +Status validate_common_grams_source_shape(const reader::LogicalIndexReader& source, + size_t source_ordinal) { + if (source.tier() != format::IndexTier::kT3 || !source.has_positions()) { + return reject(fmt::format("source {} is not CommonGrams T3", source_ordinal)); + } + if (source.section_refs().norms.length == 0) { + return reject(fmt::format("source {} has no scoring norms", source_ordinal)); + } + const auto* metadata = source.common_grams_metadata(); + if (metadata == nullptr) { + return reject(fmt::format("source {} has no CommonGrams metadata", source_ordinal)); + } + const Status metadata_status = + inverted_index::validate_common_grams_segment_metadata(*metadata); + const bool complete_shape = + metadata->common_grams_coverage == inverted_index::CommonGramsCoverage::kComplete && + source.common_grams_posting_policy() == format::CommonGramsPostingPolicy::kNone; + const bool hybrid_shape = + metadata->common_grams_coverage == inverted_index::CommonGramsCoverage::kMixed && + source.common_grams_posting_policy() == format::CommonGramsPostingPolicy::kHybridV1; + if (!metadata_status.ok() || (!complete_shape && !hybrid_shape) || + metadata->plain_term_key_version != inverted_index::PlainTermKeyVersion::kEscapedV1 || + metadata->common_grams_semantics_version != + inverted_index::COMMON_GRAMS_SEMANTICS_VERSION_V1 || + metadata->common_grams_key_version != inverted_index::COMMON_GRAMS_KEY_VERSION_V1 || + metadata->scoring_coverage != inverted_index::ScoringCoverage::kComplete || + metadata->scoring_stats_version != inverted_index::COMMON_GRAMS_SCORING_STATS_VERSION_V1 || + metadata->norm_semantics_version != + inverted_index::COMMON_GRAMS_NORM_SEMANTICS_VERSION_V1) { + return reject(fmt::format("source {} has incomplete or unsupported CommonGrams metadata", + source_ordinal)); + } + const Status scoring_status = inverted_index::validate_snii_scoring_metadata( + metadata, source.stats().doc_count, source.stats().sum_total_term_freq, + /*has_scoring_tier=*/true, /*has_positions=*/true, /*has_norms=*/true); + if (!scoring_status.ok()) { + return reject(fmt::format("source {} has incomplete scoring metadata: {}", source_ordinal, + scoring_status.to_string())); + } + const auto& stats = source.stats(); + if (stats.indexed_doc_count > stats.doc_count || + stats.null_count != stats.doc_count - stats.indexed_doc_count) { + return reject(fmt::format("source {} document statistics violate indexed + null = doc", + source_ordinal)); + } + return Status::OK(); +} + +Status reject_legacy_bigram(const reader::LogicalIndexReader& source, size_t source_ordinal) { + bool found_legacy_bigram = false; + RETURN_IF_ERROR(source.visit_prefix_terms( + format::kPhraseBigramTermMarker, + [&found_legacy_bigram](reader::LogicalIndexReader::PrefixHit&&, bool* stop) { + found_legacy_bigram = true; + *stop = true; + return Status::OK(); + })); + if (found_legacy_bigram) { + return reject( + fmt::format("source {} contains a legacy phrase-bigram term", source_ordinal)); + } + return Status::OK(); +} + +Status resolve_destination_analyzer(const TabletIndex& destination_index, + const AnalyzerProviderFactory& analyzer_provider_factory, + inverted_index::AnalyzerProviderPtr* analyzer_provider) { + analyzer_provider->reset(); + const auto& properties = destination_index.properties(); + if (!inverted_index::InvertedIndexAnalyzer::should_analyzer(properties)) { + return Status::OK(); + } + const InvertedIndexAnalyzerConfig analyzer_config = analyzer_config_from_properties(properties); + try { + *analyzer_provider = + analyzer_provider_factory + ? analyzer_provider_factory(analyzer_config) + : inverted_index::InvertedIndexAnalyzer::create_analyzer_provider( + &analyzer_config); + } catch (const CLuceneError& error) { + return reject(fmt::format("destination analyzer resolution failed: {}", error.what())); + } catch (const Exception& error) { + return reject(fmt::format("destination analyzer resolution failed: {}", error.what())); + } + DORIS_CHECK(*analyzer_provider != nullptr); + return Status::OK(); +} + +Status validate_destination_policy(const TabletIndex& destination_index, + const AnalyzerProviderFactory& analyzer_provider_factory) { + const auto& properties = destination_index.properties(); + if (get_parser_phrase_support_string_from_properties(properties) != + INVERTED_INDEX_PARSER_PHRASE_SUPPORT_YES) { + return reject("destination does not request phrase positions"); + } + if (!inverted_index::InvertedIndexAnalyzer::should_analyzer(properties)) { + return Status::OK(); + } + + inverted_index::AnalyzerProviderPtr analyzer_provider; + RETURN_IF_ERROR(resolve_destination_analyzer(destination_index, analyzer_provider_factory, + &analyzer_provider)); + DORIS_CHECK(analyzer_provider != nullptr); + if (config::enable_common_grams_index_build && analyzer_provider->uses_common_grams()) { + return reject("destination analyzer policy would build CommonGrams"); + } + return Status::OK(); +} + +} // namespace + +Status validate_plain_t2_source(const reader::LogicalIndexReader& source, size_t source_ordinal) { + return validate_source_shape(source, source_ordinal); +} + +Status validate_plain_t2_source_eligibility(const reader::LogicalIndexReader& source, + size_t source_ordinal) { + RETURN_IF_ERROR(validate_source_shape(source, source_ordinal)); + return reject_legacy_bigram(source, source_ordinal); +} + +Status validate_snii_source_eligibility(const reader::LogicalIndexReader& source, + size_t source_ordinal, + const SniiCompactionEligibility& eligibility) { + if (eligibility.kind == SniiStreamedMergeKind::kPlainT2) { + return validate_plain_t2_source_eligibility(source, source_ordinal); + } + RETURN_IF_ERROR(validate_common_grams_source_shape(source, source_ordinal)); + RETURN_IF_ERROR(reject_legacy_bigram(source, source_ordinal)); + if (!eligibility.common_grams_metadata_seed.has_value()) { + return reject("CommonGrams eligibility has no metadata identity seed"); + } + if (source.common_grams_posting_policy() != eligibility.common_grams_posting_policy) { + return reject(fmt::format("source {} CommonGrams posting policy differs from eligibility", + source_ordinal)); + } + if (static_common_grams_metadata(*source.common_grams_metadata()) != + *eligibility.common_grams_metadata_seed) { + return reject(fmt::format("source {} CommonGrams static identity differs from eligibility", + source_ordinal)); + } + return Status::OK(); +} + +Status validate_plain_t2_compaction_eligibility( + std::span sources, const TabletIndex& destination_index, + const AnalyzerProviderFactory& analyzer_provider_factory) { + if (sources.empty()) { + return reject("no source logical indexes"); + } + if (!destination_index.is_inverted_index()) { + return reject("destination is not an inverted index"); + } + + const TabletIndex& first_index_meta = sources.front().index_meta.get(); + if (!first_index_meta.is_inverted_index()) { + return reject("source 0 is not an inverted index"); + } + const auto& source_properties = first_index_meta.properties(); + for (size_t source_ordinal = 0; source_ordinal < sources.size(); ++source_ordinal) { + const TabletIndex& index_meta = sources[source_ordinal].index_meta.get(); + if (!index_meta.is_inverted_index()) { + return reject(fmt::format("source {} is not an inverted index", source_ordinal)); + } + if (index_meta.properties() != source_properties) { + return reject(fmt::format("source {} properties differ from source 0", source_ordinal)); + } + if (index_meta.index_id() != destination_index.index_id()) { + return reject( + fmt::format("source {} index id differs from destination", source_ordinal)); + } + if (index_meta.get_index_suffix() != destination_index.get_index_suffix()) { + return reject( + fmt::format("source {} index suffix differs from destination", source_ordinal)); + } + } + if (destination_index.properties() != source_properties) { + return reject("destination properties differ from source properties"); + } + + for (size_t source_ordinal = 0; source_ordinal < sources.size(); ++source_ordinal) { + RETURN_IF_ERROR(validate_plain_t2_source_eligibility(sources[source_ordinal].reader.get(), + source_ordinal)); + } + return validate_destination_policy(destination_index, analyzer_provider_factory); +} + +Status validate_snii_compaction_eligibility( + std::span sources, const TabletIndex& destination_index, + SniiCompactionEligibility* out, const AnalyzerProviderFactory& analyzer_provider_factory) { + if (out == nullptr) { + return Status::InvalidArgument("SNII compaction eligibility has null output"); + } + *out = SniiCompactionEligibility {}; + if (sources.empty()) { + return reject("no source logical indexes"); + } + if (!destination_index.is_inverted_index()) { + return reject("destination is not an inverted index"); + } + if (get_parser_phrase_support_string_from_properties(destination_index.properties()) != + INVERTED_INDEX_PARSER_PHRASE_SUPPORT_YES) { + return reject("destination does not request phrase positions"); + } + + const TabletIndex& first_index_meta = sources.front().index_meta.get(); + if (!first_index_meta.is_inverted_index()) { + return reject("source 0 is not an inverted index"); + } + const auto& source_properties = first_index_meta.properties(); + for (size_t source_ordinal = 0; source_ordinal < sources.size(); ++source_ordinal) { + const TabletIndex& index_meta = sources[source_ordinal].index_meta.get(); + if (!index_meta.is_inverted_index()) { + return reject(fmt::format("source {} is not an inverted index", source_ordinal)); + } + if (index_meta.properties() != source_properties) { + return reject(fmt::format("source {} properties differ from source 0", source_ordinal)); + } + if (index_meta.index_id() != destination_index.index_id()) { + return reject( + fmt::format("source {} index id differs from destination", source_ordinal)); + } + if (index_meta.get_index_suffix() != destination_index.get_index_suffix()) { + return reject( + fmt::format("source {} index suffix differs from destination", source_ordinal)); + } + } + if (destination_index.properties() != source_properties) { + return reject("destination properties differ from source properties"); + } + + const format::IndexTier source_tier = sources.front().reader.get().tier(); + if (source_tier == format::IndexTier::kT2) { + for (size_t source_ordinal = 0; source_ordinal < sources.size(); ++source_ordinal) { + if (sources[source_ordinal].reader.get().tier() != format::IndexTier::kT2) { + return reject("source streamed-merge shapes are not homogeneous"); + } + RETURN_IF_ERROR(validate_plain_t2_source_eligibility( + sources[source_ordinal].reader.get(), source_ordinal)); + } + RETURN_IF_ERROR(validate_destination_policy(destination_index, analyzer_provider_factory)); + out->kind = SniiStreamedMergeKind::kPlainT2; + return Status::OK(); + } + if (source_tier != format::IndexTier::kT3) { + return reject("source streamed-merge shape is neither plain T2 nor CommonGrams T3"); + } + + std::optional source_seed; + std::optional source_policy; + for (size_t source_ordinal = 0; source_ordinal < sources.size(); ++source_ordinal) { + const auto& source = sources[source_ordinal].reader.get(); + if (source.tier() != format::IndexTier::kT3) { + return reject("source streamed-merge shapes are not homogeneous"); + } + RETURN_IF_ERROR(validate_common_grams_source_shape(source, source_ordinal)); + RETURN_IF_ERROR(reject_legacy_bigram(source, source_ordinal)); + const auto seed = static_common_grams_metadata(*source.common_grams_metadata()); + if (!source_seed.has_value()) { + source_seed = seed; + source_policy = source.common_grams_posting_policy(); + } else if (source.common_grams_posting_policy() != *source_policy) { + return reject("source CommonGrams posting policies are not homogeneous"); + } else if (*source_seed != seed) { + return reject(fmt::format("source {} CommonGrams static identity differs from source 0", + source_ordinal)); + } + } + + if (!config::enable_common_grams_index_build) { + return reject("destination CommonGrams index build is disabled"); + } + inverted_index::AnalyzerProviderPtr analyzer_provider; + RETURN_IF_ERROR(resolve_destination_analyzer(destination_index, analyzer_provider_factory, + &analyzer_provider)); + if (analyzer_provider == nullptr || !analyzer_provider->uses_common_grams()) { + return reject("destination analyzer does not build CommonGrams"); + } + const auto* destination_identity = analyzer_provider->common_grams_identity(); + if (destination_identity == nullptr) { + return reject("destination analyzer has no complete CommonGrams identity"); + } + if (!source_seed.has_value() || + !inverted_index::common_grams_identity_matches(*source_seed, *destination_identity)) { + return reject("destination CommonGrams identity differs from source identity"); + } + out->kind = SniiStreamedMergeKind::kCommonGramsT3; + out->common_grams_metadata_seed = std::move(source_seed); + DORIS_CHECK(source_policy.has_value()); + out->common_grams_posting_policy = *source_policy; + return Status::OK(); +} + +} // namespace doris::snii::compaction diff --git a/be/src/storage/index/snii/compaction/eligibility.h b/be/src/storage/index/snii/compaction/eligibility.h new file mode 100644 index 00000000000000..bbbfae59204d22 --- /dev/null +++ b/be/src/storage/index/snii/compaction/eligibility.h @@ -0,0 +1,114 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/inverted/analyzer/analyzer_provider.h" +#include "storage/index/inverted/common_grams/common_grams_segment_metadata.h" +#include "storage/index/inverted/inverted_index_parser.h" +#include "storage/index/snii/format/core_metadata.h" + +namespace doris { + +class TabletIndex; + +namespace snii::reader { +class LogicalIndexReader; +} + +namespace snii::compaction { + +// One already-opened SNII source logical index and the TabletIndex metadata +// that produced it. References make null inputs unrepresentable; both objects +// must outlive validate_plain_t2_compaction_eligibility(). +struct PlainT2CompactionSource { + std::reference_wrapper reader; + std::reference_wrapper index_meta; +}; + +// Injectable only at analyzer-provider construction. Production callers omit +// it and use InvertedIndexAnalyzer::create_analyzer_provider; focused tests can +// supply an immutable provider without mutating the process IndexPolicyMgr. +using AnalyzerProviderFactory = std::function; + +enum class SniiStreamedMergeKind : uint8_t { + kPlainT2, + kCommonGramsT3, +}; + +// Validated destination shape for one streamed merge. CommonGrams metadata is a +// static identity seed: destination doc_count is bound when its session starts, +// and semantic token_count is bound after the single postings pass. +struct SniiCompactionEligibility { + SniiStreamedMergeKind kind = SniiStreamedMergeKind::kPlainT2; + std::optional + common_grams_metadata_seed; + format::CommonGramsPostingPolicy common_grams_posting_policy = + format::CommonGramsPostingPolicy::kNone; +}; + +// O(1) physical/semantic validation for one source. The merge planner may call +// this while preparing opened sources; the aggregate validator below reuses the +// exact same predicate. +Status validate_plain_t2_source(const reader::LogicalIndexReader& source, size_t source_ordinal); + +// Gate-only validation. In addition to the O(1) physical shape checks above, +// performs one bounded DICT prefix seek for the legacy hidden-bigram namespace +// so the owner can select raw rebuild before creating streamed output. +Status validate_plain_t2_source_eligibility(const reader::LogicalIndexReader& source, + size_t source_ordinal); + +// Revalidates one opened source against an already selected streamed shape. +// CommonGrams checks include exact current versions and static identity equality +// with the eligibility seed. +Status validate_snii_source_eligibility(const reader::LogicalIndexReader& source, + size_t source_ordinal, + const SniiCompactionEligibility& eligibility); + +// O(source-count), O(1)-metadata validation for the SNII postings-merge fast +// path. OK means every source is a plain positions-only T2 index, all source +// properties are byte-for-byte equal to the current destination properties, +// source/destination index ids and escaped suffixes match, and the current +// destination analyzer policy will not build CommonGrams. +// Rejections use INVERTED_INDEX_NOT_SUPPORTED with a concrete reason so the +// caller can select raw-column rebuild before creating output. +// +// Each source also receives one bounded seek for the legacy hidden-bigram +// marker namespace. This avoids discovering an unsupported legacy segment only +// after the raw-column compaction path has already been skipped. +Status validate_plain_t2_compaction_eligibility( + std::span sources, const TabletIndex& destination_index, + const AnalyzerProviderFactory& analyzer_provider_factory = {}); + +// Accepts either the existing plain positions-only T2 shape or homogeneous, +// complete CommonGrams T3 sources. Mixed shapes and any CommonGrams identity or +// destination-build-policy mismatch return INVERTED_INDEX_NOT_SUPPORTED so the +// caller can rebuild from raw columns before creating streamed output. +Status validate_snii_compaction_eligibility( + std::span sources, const TabletIndex& destination_index, + SniiCompactionEligibility* out, + const AnalyzerProviderFactory& analyzer_provider_factory = {}); + +} // namespace snii::compaction +} // namespace doris diff --git a/be/src/storage/index/snii/compaction/indexed_winner_tree.h b/be/src/storage/index/snii/compaction/indexed_winner_tree.h new file mode 100644 index 00000000000000..e16205a484832c --- /dev/null +++ b/be/src/storage/index/snii/compaction/indexed_winner_tree.h @@ -0,0 +1,103 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include + +#include "common/check.h" + +namespace doris::snii::compaction { + +// Complete binary winner tree over dense source ordinals. Updating one source +// touches only its leaf-to-root path. runner_up() inspects the sibling winners +// on the current winner's path without changing the tree. +template +class IndexedWinnerTree { +public: + static constexpr size_t kNoSource = std::numeric_limits::max(); + + explicit IndexedWinnerTree(Before before) : before_(std::move(before)), nodes_(2, kNoSource) {} + + template + void build(size_t source_count, IsLive&& is_live) { + source_count_ = source_count; + leaf_base_ = 1; + while (leaf_base_ < source_count_) { + DORIS_CHECK_LE(leaf_base_, std::numeric_limits::max() / 2); + leaf_base_ *= 2; + } + nodes_.assign(leaf_base_ * 2, kNoSource); + for (size_t source = 0; source < source_count_; ++source) { + if (is_live(source)) { + nodes_[leaf_base_ + source] = source; + } + } + for (size_t node = leaf_base_; node-- > 1;) { + nodes_[node] = select(nodes_[node * 2], nodes_[node * 2 + 1]); + } + } + + bool empty() const noexcept { return nodes_[1] == kNoSource; } + + size_t winner() const { + DCHECK(!empty()); + return nodes_[1]; + } + + size_t runner_up() const { + const size_t current_winner = winner(); + size_t candidate = kNoSource; + size_t node = leaf_base_ + current_winner; + while (node > 1) { + candidate = select(candidate, nodes_[node ^ 1]); + node /= 2; + } + return candidate; + } + + void update(size_t source, bool live) { + DCHECK_LT(source, source_count_); + size_t node = leaf_base_ + source; + nodes_[node] = live ? source : kNoSource; + while (node > 1) { + node /= 2; + nodes_[node] = select(nodes_[node * 2], nodes_[node * 2 + 1]); + } + } + +private: + size_t select(size_t candidate, size_t challenger) const { + if (candidate == kNoSource) { + return challenger; + } + if (challenger == kNoSource) { + return candidate; + } + return before_(challenger, candidate) ? challenger : candidate; + } + + Before before_; + size_t source_count_ = 0; + size_t leaf_base_ = 1; + boost::container::small_vector nodes_; +}; + +} // namespace doris::snii::compaction diff --git a/be/src/storage/index/snii/compaction/posting_cursor.cpp b/be/src/storage/index/snii/compaction/posting_cursor.cpp new file mode 100644 index 00000000000000..fbb25efc7701a6 --- /dev/null +++ b/be/src/storage/index/snii/compaction/posting_cursor.cpp @@ -0,0 +1,1025 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/compaction/posting_cursor.h" + +#include + +#include +#include +#include +#include + +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/encoding/byte_source.h" +#include "storage/index/snii/format/format_constants.h" +#include "storage/index/snii/format/frq_pod.h" +#include "storage/index/snii/format/prx_pod.h" +#include "storage/index/snii/reader/windowed_posting.h" + +namespace doris::snii::compaction { + +namespace { + +Status checked_add(uint64_t lhs, uint64_t rhs, const char* message, uint64_t* out) { + if (rhs > std::numeric_limits::max() - lhs) { + return Status::Error(message); + } + *out = lhs + rhs; + return Status::OK(); +} + +Status posting_corruption(const char* message, uint32_t source_ordinal) { + return Status::Error( + "posting_cursor: {} (src_ord={})", message, source_ordinal); +} + +} // namespace + +Status validate_posting_region(const format::RegionRef& region, uint64_t file_size) { + if (region.offset > file_size || region.length > file_size - region.offset) { + return Status::Error( + "posting_cursor: posting region outside source file"); + } + return Status::OK(); +} + +bool posting_entry_has_positions(const format::DictEntry& entry) { + return entry.kind == format::DictEntryKind::kInline ? !entry.prx_bytes.empty() + : entry.prx_len != 0; +} + +SniiPostingReadContext::TermLease::~TermLease() { + if (context_ != nullptr) { + context_->release_term(); + } +} + +size_t SniiPostingReadContext::DecoderWorkspace::capacity_bytes() const { + return docs_scratch.capacity() + prx_scratch.capacity() + decompressed.capacity() + + sizeof(uint32_t) * (docids.capacity() + positions_flat.capacity() + + position_offsets.capacity() + frequencies.capacity()) + + sizeof(DestinationPostingRun) * destination_runs.capacity(); +} + +void SniiPostingReadContext::DecoderWorkspace::init_memory_reporter( + writer::MemoryReporter* memory_reporter) { + if (memory_reporter == nullptr) return; + docs_scratch_reservation = memory_reporter->make_reservation(); + prx_scratch_reservation = memory_reporter->make_reservation(); + docids_reservation = memory_reporter->make_reservation(); + positions_reservation = memory_reporter->make_reservation(); + position_offsets_reservation = memory_reporter->make_reservation(); + destination_runs_reservation = memory_reporter->make_reservation(); + frequencies_reservation = memory_reporter->make_reservation(); + decompressed_reservation = memory_reporter->make_reservation(); + reservations_enabled = true; +} + +Status SniiPostingReadContext::DecoderWorkspace::reserve_remapped(size_t document_count, + size_t run_count, + bool retain_frequencies) { + const bool grow_runs = destination_runs.capacity() < run_count; + const bool grow_frequencies = retain_frequencies && frequencies.capacity() < document_count; + if (!reservations_enabled) { + destination_runs.reserve(run_count); + if (retain_frequencies) { + frequencies.reserve(document_count); + } + return Status::OK(); + } + if (!grow_runs && !grow_frequencies) { + DCHECK_EQ(destination_runs_reservation.bytes(), + destination_runs.capacity() * sizeof(DestinationPostingRun)); + DCHECK_EQ(frequencies_reservation.bytes(), frequencies.capacity() * sizeof(uint32_t)); + return Status::OK(); + } + if (run_count > std::numeric_limits::max() / sizeof(DestinationPostingRun) || + document_count > std::numeric_limits::max() / sizeof(uint32_t)) { + return Status::Error( + "posting cursor: remapped workspace size overflows size_t"); + } + writer::MemoryReporter::Reservation replacement_runs; + writer::MemoryReporter::Reservation replacement_frequencies; + if (grow_runs) { + RETURN_IF_ERROR(destination_runs_reservation.prepare_replacement( + run_count * sizeof(DestinationPostingRun), &replacement_runs)); + } + if (grow_frequencies) { + RETURN_IF_ERROR(frequencies_reservation.prepare_replacement( + document_count * sizeof(uint32_t), &replacement_frequencies)); + } + { + std::vector new_runs; + std::vector new_frequencies; + if (grow_runs) { + new_runs.reserve(run_count); + DCHECK_EQ(new_runs.capacity(), run_count); + destination_runs.swap(new_runs); + } + if (grow_frequencies) { + new_frequencies.reserve(document_count); + DCHECK_EQ(new_frequencies.capacity(), document_count); + frequencies.swap(new_frequencies); + } + } + if (grow_runs) { + destination_runs_reservation = std::move(replacement_runs); + } + if (grow_frequencies) { + frequencies_reservation = std::move(replacement_frequencies); + } + return Status::OK(); +} + +Status SniiPostingReadContext::DecoderWorkspace::reserve_docids(size_t count) { + if (!reservations_enabled || docids.capacity() >= count) { + if (reservations_enabled) { + DCHECK_EQ(docids_reservation.bytes(), docids.capacity() * sizeof(uint32_t)); + } + return Status::OK(); + } + const size_t target_bytes = count * sizeof(uint32_t); + writer::MemoryReporter::Reservation replacement; + RETURN_IF_ERROR(docids_reservation.prepare_replacement(target_bytes, &replacement)); + docids.reserve(count); + DCHECK_EQ(docids.capacity(), count); + docids_reservation = std::move(replacement); + return Status::OK(); +} + +Status SniiPostingReadContext::DecoderWorkspace::reserve_csr(std::vector* pos_flat, + size_t position_count, + std::vector* pos_off, + size_t offset_count) { + DCHECK(reservations_enabled); + DCHECK_EQ(pos_flat, &positions_flat); + DCHECK_EQ(pos_off, &position_offsets); + if (position_count > std::numeric_limits::max() / sizeof(uint32_t) || + offset_count > std::numeric_limits::max() / sizeof(uint32_t)) { + return Status::Error( + "posting cursor: position workspace size overflows size_t"); + } + + const bool grow_positions = positions_flat.capacity() < position_count; + const bool grow_offsets = position_offsets.capacity() < offset_count; + if (!grow_positions && !grow_offsets) { + DCHECK_EQ(positions_reservation.bytes(), positions_flat.capacity() * sizeof(uint32_t)); + DCHECK_EQ(position_offsets_reservation.bytes(), + position_offsets.capacity() * sizeof(uint32_t)); + return Status::OK(); + } + + writer::MemoryReporter::Reservation replacement_positions; + writer::MemoryReporter::Reservation replacement_offsets; + if (grow_positions) { + RETURN_IF_ERROR(positions_reservation.prepare_replacement(position_count * sizeof(uint32_t), + &replacement_positions)); + } + if (grow_offsets) { + RETURN_IF_ERROR(position_offsets_reservation.prepare_replacement( + offset_count * sizeof(uint32_t), &replacement_offsets)); + } + + { + std::vector new_positions; + std::vector new_offsets; + if (grow_positions) { + new_positions.reserve(position_count); + DCHECK_EQ(new_positions.capacity(), position_count); + positions_flat.swap(new_positions); + } + if (grow_offsets) { + new_offsets.reserve(offset_count); + DCHECK_EQ(new_offsets.capacity(), offset_count); + position_offsets.swap(new_offsets); + } + } + if (grow_positions) { + positions_reservation = std::move(replacement_positions); + } + if (grow_offsets) { + position_offsets_reservation = std::move(replacement_offsets); + } + return Status::OK(); +} + +Status SniiPostingReadContext::DecoderWorkspace::reserve_decompression( + size_t bytes, std::vector** buffer) { + DCHECK(reservations_enabled); + DCHECK(buffer != nullptr); + if (decompressed.capacity() < bytes) { + writer::MemoryReporter::Reservation replacement; + RETURN_IF_ERROR(decompressed_reservation.prepare_replacement(bytes, &replacement)); + { + std::vector new_decompressed; + new_decompressed.reserve(bytes); + DCHECK_EQ(new_decompressed.capacity(), bytes); + decompressed.swap(new_decompressed); + } + decompressed_reservation = std::move(replacement); + } else { + DCHECK_EQ(decompressed_reservation.bytes(), decompressed.capacity()); + } + *buffer = &decompressed; + return Status::OK(); +} + +void SniiPostingReadContext::DecoderWorkspace::release_large_buffers( + size_t retained_capacity_limit_bytes) { + prelude = format::FrqPreludeReader(); + if (capacity_bytes() <= retained_capacity_limit_bytes) { + return; + } + std::vector().swap(docs_scratch); + std::vector().swap(prx_scratch); + std::vector().swap(decompressed); + std::vector().swap(docids); + std::vector().swap(positions_flat); + std::vector().swap(position_offsets); + std::vector().swap(destination_runs); + std::vector().swap(frequencies); + docs_scratch_reservation.reset(); + prx_scratch_reservation.reset(); + docids_reservation.reset(); + positions_reservation.reset(); + position_offsets_reservation.reset(); + destination_runs_reservation.reset(); + frequencies_reservation.reset(); + decompressed_reservation.reset(); + DCHECK_LE(capacity_bytes(), retained_capacity_limit_bytes); +} + +Status SniiPostingReadContext::poison(Status status) { + DCHECK(!status.ok()); + if (failed_.ok()) { + failed_ = std::move(status); + } + return failed_; +} + +Status SniiPostingReadContext::init() { + if (initialized_) { + return Status::Error( + "posting_read_context: init called twice"); + } + if (index_ == nullptr || index_->reader() == nullptr) { + return Status::Error( + "posting_read_context: null source index"); + } + if (total_read_ahead_budget_bytes_ < 2 || + total_read_ahead_budget_bytes_ > kMaxReadAheadBudgetBytes) { + return Status::Error( + "posting_read_context: total read-ahead budget outside [2, {}]", + kMaxReadAheadBudgetBytes); + } + + posting_region_ = index_->section_refs().posting_region; + RETURN_IF_ERROR(validate_posting_region(posting_region_, index_->reader()->size())); + decoder_workspace_.init_memory_reporter(memory_reporter_); + posting_cache_ = std::make_unique( + index_->reader(), posting_region_.offset, posting_region_.length, + total_read_ahead_budget_bytes_, memory_reporter_); + RETURN_IF_ERROR(posting_cache_->init()); + initialized_ = true; + return Status::OK(); +} + +Status SniiPostingReadContext::validate_next_range(const format::RegionRef& range, + bool has_previous, uint64_t previous_end, + const char* stream, uint64_t* end) const { + DCHECK(end != nullptr); + if (range.length == 0 || range.offset < posting_region_.offset) { + return Status::Error( + "posting_read_context: invalid {} term range", stream); + } + const uint64_t relative_offset = range.offset - posting_region_.offset; + if (relative_offset > posting_region_.length || + range.length > posting_region_.length - relative_offset) { + return Status::Error( + "posting_read_context: {} term range outside posting region", stream); + } + if (has_previous && range.offset < previous_end) { + return Status::Error( + "posting_read_context: {} term ranges are not monotone", stream); + } + *end = range.offset + range.length; + return Status::OK(); +} + +Status SniiPostingReadContext::acquire_term(bool has_docs_range, bool has_prx_range, + const format::RegionRef& docs_range, + const format::RegionRef& prx_range, + std::unique_ptr* lease) { + DCHECK(lease != nullptr); + DCHECK(*lease == nullptr); + if (!initialized_) { + return Status::Error( + "posting_read_context: acquire before init"); + } + if (!failed_.ok()) { + return failed_; + } + if (term_active_) { + return Status::Error( + "posting_read_context: concurrent term cursor"); + } + + uint64_t docs_end = 0; + uint64_t prx_end = 0; + if (has_docs_range) { + Status status = + validate_next_range(docs_range, has_docs_range_, last_docs_end_, "docs", &docs_end); + if (!status.ok()) { + return poison(status); + } + last_docs_end_ = docs_end; + has_docs_range_ = true; + } + if (has_prx_range) { + Status status = + validate_next_range(prx_range, has_prx_range_, last_prx_end_, "prx", &prx_end); + if (!status.ok()) { + return poison(status); + } + last_prx_end_ = prx_end; + has_prx_range_ = true; + } + + term_active_ = true; + lease->reset(new TermLease(this)); + return Status::OK(); +} + +void SniiPostingReadContext::release_term() { + DCHECK(term_active_); + term_active_ = false; + decoder_workspace_.release_large_buffers(retained_decoder_workspace_limit_bytes()); +} + +Status SniiPostingReadContext::poison_active_term(Status status, TermLease* lease) { + DCHECK(lease != nullptr); + DCHECK_EQ(lease->context_, this); + DCHECK(term_active_); + Status first = poison(std::move(status)); + lease->context_ = nullptr; + release_term(); + return first; +} + +uint64_t SniiPostingReadContext::docs_read_calls() const { + DCHECK(initialized_); + return posting_cache_->read_calls(PostingStream::kDocs); +} + +uint64_t SniiPostingReadContext::prx_read_calls() const { + DCHECK(initialized_); + return posting_cache_->read_calls(PostingStream::kPrx); +} + +uint64_t SniiPostingReadContext::docs_buffer_hits() const { + DCHECK(initialized_); + return posting_cache_->buffer_hits(PostingStream::kDocs); +} + +uint64_t SniiPostingReadContext::prx_buffer_hits() const { + DCHECK(initialized_); + return posting_cache_->buffer_hits(PostingStream::kPrx); +} + +uint64_t SniiPostingReadContext::physical_read_ranges() const { + DCHECK(initialized_); + return posting_cache_->physical_read_ranges(); +} + +uint64_t SniiPostingReadContext::physical_read_bytes() const { + DCHECK(initialized_); + return posting_cache_->physical_read_bytes(); +} + +size_t SniiPostingReadContext::resident_read_ahead_capacity_bytes() const { + DCHECK(initialized_); + return posting_cache_->resident_capacity_bytes(); +} + +size_t SniiPostingReadContext::decoder_workspace_capacity_bytes() const { + DCHECK(initialized_); + return decoder_workspace_.capacity_bytes(); +} + +Status SniiPostingCursor::poison(Status status) { + DCHECK(!status.ok()); + if (failed_.ok()) { + if (term_lease_ != nullptr) { + failed_ = read_context_->poison_active_term(std::move(status), term_lease_.get()); + term_lease_.reset(); + } else { + failed_ = std::move(status); + } + } + return failed_; +} + +Status SniiPostingCursor::validate_entry_geometry() { + if (entry_.df == 0) { + return posting_corruption("zero-df dictionary entry", source_ordinal_); + } + if (entry_.kind == format::DictEntryKind::kInline) { + if (entry_.enc != format::DictEntryEnc::kSlim) { + return posting_corruption("inline entry is not slim", source_ordinal_); + } + if (entry_.inline_dd_disk_len != entry_.dd_meta.disk_len || + entry_.inline_dd_disk_len > entry_.frq_bytes.size()) { + return posting_corruption("inline dd geometry mismatch", source_ordinal_); + } + const uint64_t freq_len = entry_.frq_bytes.size() - entry_.inline_dd_disk_len; + if (entry_.freq_meta.disk_len != freq_len) { + return posting_corruption("inline freq geometry mismatch", source_ordinal_); + } + shape_ = Shape::kFlat; + return Status::OK(); + } + + if (entry_.kind != format::DictEntryKind::kPodRef) { + return posting_corruption("unknown dictionary entry kind", source_ordinal_); + } + if (entry_.enc == format::DictEntryEnc::kWindowed) { + if (entry_.prelude_len == 0 || entry_.prelude_len > entry_.frq_docs_len || + entry_.frq_docs_len > entry_.frq_len) { + return posting_corruption("invalid windowed frq geometry", source_ordinal_); + } + shape_ = Shape::kWindowed; + return Status::OK(); + } + if (entry_.enc != format::DictEntryEnc::kSlim) { + return posting_corruption("unknown dictionary entry encoding", source_ordinal_); + } + if (entry_.prelude_len != 0 || entry_.frq_docs_len != entry_.dd_meta.disk_len || + entry_.frq_docs_len > entry_.frq_len) { + return posting_corruption("invalid slim frq geometry", source_ordinal_); + } + if (entry_.freq_meta.disk_len != entry_.frq_len - entry_.frq_docs_len) { + return posting_corruption("slim freq geometry mismatch", source_ordinal_); + } + shape_ = Shape::kFlat; + return Status::OK(); +} + +Status SniiPostingCursor::prepare_flat_ranges() { + if (entry_.kind == format::DictEntryKind::kInline) { + flat_dd_len_ = entry_.inline_dd_disk_len; + flat_prx_len_ = entry_.prx_bytes.size(); + return Status::OK(); + } + + uint64_t frq_len = 0; + RETURN_IF_ERROR(index_->resolve_frq_window(entry_, frq_base_, &flat_dd_abs_, &frq_len)); + if (frq_len != entry_.frq_len) { + return posting_corruption("resolved slim frq length mismatch", source_ordinal_); + } + flat_dd_len_ = entry_.frq_docs_len; + if (term_has_positions_) { + RETURN_IF_ERROR( + index_->resolve_prx_window(entry_, prx_base_, &flat_prx_abs_, &flat_prx_len_)); + if (flat_prx_len_ != entry_.prx_len) { + return posting_corruption("resolved slim prx length mismatch", source_ordinal_); + } + } + return Status::OK(); +} + +Status SniiPostingCursor::prepare_windowed_ranges() { + RETURN_IF_ERROR(index_->resolve_frq_window(entry_, frq_base_, &flat_dd_abs_, &flat_dd_len_)); + if (flat_dd_len_ != entry_.frq_len - entry_.prelude_len || flat_dd_abs_ < entry_.prelude_len) { + return posting_corruption("resolved windowed frq geometry mismatch", source_ordinal_); + } + if (term_has_positions_) { + RETURN_IF_ERROR( + index_->resolve_prx_window(entry_, prx_base_, &flat_prx_abs_, &flat_prx_len_)); + if (flat_prx_len_ != entry_.prx_len) { + return posting_corruption("resolved windowed prx length mismatch", source_ordinal_); + } + } + return Status::OK(); +} + +Status SniiPostingCursor::prepare_windowed() { + DCHECK(workspace_ != nullptr); + Slice prelude_bytes; + RETURN_IF_ERROR(read_context_->posting_cache_->resolve( + PostingStream::kDocs, flat_dd_abs_ - entry_.prelude_len, entry_.prelude_len, + &workspace_->docs_scratch, &prelude_bytes, + read_context_->memory_reporter_ == nullptr ? nullptr + : &workspace_->docs_scratch_reservation)); + RETURN_IF_ERROR(format::FrqPreludeReader::open(prelude_bytes, &workspace_->prelude)); + if (workspace_->prelude.has_prx() != term_has_positions_) { + return posting_corruption("windowed prelude position shape differs from entry", + source_ordinal_); + } + + uint64_t docs_prefix_len = 0; + RETURN_IF_ERROR(checked_add(entry_.prelude_len, workspace_->prelude.dd_block_len(), + "posting_cursor: windowed docs prefix overflow", &docs_prefix_len)); + if (docs_prefix_len != entry_.frq_docs_len) { + return posting_corruption("windowed docs prefix mismatch", source_ordinal_); + } + uint64_t encoded_frq_len = 0; + RETURN_IF_ERROR(checked_add(docs_prefix_len, workspace_->prelude.freq_block_len(), + "posting_cursor: windowed frq length overflow", &encoded_frq_len)); + if (encoded_frq_len != entry_.frq_len) { + return posting_corruption("windowed frq blocks do not tile entry", source_ordinal_); + } + + uint64_t dd_bytes = 0; + uint64_t freq_bytes = 0; + uint64_t prx_bytes = 0; + uint64_t docs = 0; + uint32_t previous_last_docid = 0; + bool has_previous_window = false; + for (uint32_t window = 0; window < workspace_->prelude.n_windows(); ++window) { + format::WindowMeta meta; + RETURN_IF_ERROR(workspace_->prelude.window(window, &meta)); + if (meta.doc_count == 0 || meta.dd_off != dd_bytes || meta.prx_off != prx_bytes || + (workspace_->prelude.has_freq() && meta.freq_off != freq_bytes)) { + return posting_corruption("non-contiguous window metadata", source_ordinal_); + } + if ((!has_previous_window && meta.win_base != 0) || + (has_previous_window && + (meta.win_base != previous_last_docid || meta.last_docid <= previous_last_docid))) { + return posting_corruption("invalid window docid chain", source_ordinal_); + } + if (meta.last_docid >= index_->stats().doc_count) { + return posting_corruption("window last docid outside index", source_ordinal_); + } + RETURN_IF_ERROR(checked_add(dd_bytes, meta.dd_disk_len, + "posting_cursor: window dd bytes overflow", &dd_bytes)); + RETURN_IF_ERROR(checked_add(freq_bytes, meta.freq_disk_len, + "posting_cursor: window freq bytes overflow", &freq_bytes)); + RETURN_IF_ERROR(checked_add(prx_bytes, meta.prx_len, + "posting_cursor: window prx bytes overflow", &prx_bytes)); + RETURN_IF_ERROR(checked_add(docs, meta.doc_count, + "posting_cursor: window doc count overflow", &docs)); + previous_last_docid = meta.last_docid; + has_previous_window = true; + } + if (dd_bytes != workspace_->prelude.dd_block_len() || + freq_bytes != workspace_->prelude.freq_block_len() || prx_bytes != entry_.prx_len || + docs != entry_.df) { + return posting_corruption("window directory totals mismatch", source_ordinal_); + } + return Status::OK(); +} + +Status SniiPostingCursor::init() { + if (initialized_) { + return Status::Error( + "posting_cursor: init called twice"); + } + if (read_context_ == nullptr || index_ == nullptr || rowid_conversion_ == nullptr) { + return Status::Error( + "posting_cursor: null read context or rowid conversion"); + } + if (!read_context_->initialized()) { + return Status::Error( + "posting_cursor: source read context not initialized"); + } + if (!read_context_->failed_status().ok()) { + return read_context_->failed_status(); + } + if (index_->tier() == format::IndexTier::kT1) { + return Status::Error( + "posting_cursor: positions index is required"); + } + if (!index_->has_positions()) { + return posting_corruption("positions tier lacks positions capability", source_ordinal_); + } + if (source_ordinal_ >= rowid_conversion_->source_segment_count()) { + return Status::Error( + "posting_cursor: source ordinal outside rowid conversion"); + } + if (index_->stats().doc_count != + rowid_conversion_->source_segment_doc_counts()[source_ordinal_]) { + return posting_corruption("rowid conversion size differs from source doc count", + source_ordinal_); + } + source_mapping_ = rowid_conversion_->source_mapping(source_ordinal_); + source_has_deletions_ = rowid_conversion_->source_has_deletions(source_ordinal_); + + RETURN_IF_ERROR(validate_entry_geometry()); + const bool has_pod_ranges = entry_.kind == format::DictEntryKind::kPodRef; + format::RegionRef docs_range; + format::RegionRef prx_range; + if (shape_ == Shape::kWindowed) { + RETURN_IF_ERROR(prepare_windowed_ranges()); + docs_range.offset = flat_dd_abs_ - entry_.prelude_len; + docs_range.length = entry_.frq_docs_len; + prx_range.offset = flat_prx_abs_; + prx_range.length = flat_prx_len_; + } else { + RETURN_IF_ERROR(prepare_flat_ranges()); + if (has_pod_ranges) { + docs_range.offset = flat_dd_abs_; + docs_range.length = flat_dd_len_; + prx_range.offset = flat_prx_abs_; + prx_range.length = flat_prx_len_; + } + } + RETURN_IF_ERROR(read_context_->acquire_term(has_pod_ranges, + has_pod_ranges && term_has_positions_, docs_range, + prx_range, &term_lease_)); + workspace_ = &read_context_->decoder_workspace_; + workspace_->destination_runs.clear(); + workspace_->frequencies.clear(); + next_destination_run_ = 0; + if (shape_ == Shape::kWindowed) { + const Status status = prepare_windowed(); + if (!status.ok()) { + return poison(status); + } + } + initialized_ = true; + return Status::OK(); +} + +Status SniiPostingCursor::decode_prx(Slice bytes, format::PrxDecodedShape* shape) { + DCHECK(workspace_ != nullptr); + DCHECK(shape != nullptr); + ByteSource source(bytes); + format::PrxDecodeContext decode_context { + .shape = shape, + .allocation_gate = workspace_->reservations_enabled ? workspace_ : nullptr}; + RETURN_IF_ERROR(format::read_prx_window_csr(&source, &workspace_->positions_flat, + &workspace_->position_offsets, &decode_context)); + if (!source.eof()) { + return posting_corruption("trailing bytes after prx frame", source_ordinal_); + } + return Status::OK(); +} + +Status SniiPostingCursor::decode_dd(Slice bytes, const format::FrqRegionMeta& meta, + uint64_t win_base, uint32_t expected_doc_count) { + DCHECK(workspace_ != nullptr); + if (!workspace_->reservations_enabled) { + return format::decode_dd_region(bytes, meta, win_base, expected_doc_count, + &workspace_->docids); + } + return format::decode_dd_region(bytes, meta, win_base, expected_doc_count, workspace_, + &workspace_->docids); +} + +Status SniiPostingCursor::load_flat_chunk() { + DCHECK(workspace_ != nullptr); + Slice dd_bytes; + Slice prx_bytes; + if (entry_.kind == format::DictEntryKind::kInline) { + dd_bytes = Slice(entry_.frq_bytes.data(), static_cast(flat_dd_len_)); + prx_bytes = Slice(entry_.prx_bytes); + } else { + RETURN_IF_ERROR(read_context_->posting_cache_->resolve( + PostingStream::kDocs, flat_dd_abs_, flat_dd_len_, &workspace_->docs_scratch, + &dd_bytes, + read_context_->memory_reporter_ == nullptr + ? nullptr + : &workspace_->docs_scratch_reservation)); + if (term_has_positions_) { + RETURN_IF_ERROR(read_context_->posting_cache_->resolve( + PostingStream::kPrx, flat_prx_abs_, flat_prx_len_, &workspace_->prx_scratch, + &prx_bytes, + read_context_->memory_reporter_ == nullptr + ? nullptr + : &workspace_->prx_scratch_reservation)); + } + } + + RETURN_IF_ERROR(workspace_->reserve_docids(entry_.df)); + RETURN_IF_ERROR(decode_dd(dd_bytes, entry_.dd_meta, /*win_base=*/0, entry_.df)); + format::PrxDecodedShape prx_shape; + if (term_has_positions_) { + RETURN_IF_ERROR(decode_prx(prx_bytes, &prx_shape)); + } + if (workspace_->docids.size() != entry_.df || workspace_->docids.empty() || + workspace_->docids.back() >= index_->stats().doc_count) { + return posting_corruption("decoded docids differ from flat entry shape", source_ordinal_); + } + if (term_has_positions_ && + (prx_shape.total_docs != entry_.df || + prx_shape.total_positions != workspace_->positions_flat.size() || + prx_shape.has_zero_frequency || + workspace_->position_offsets.size() != static_cast(entry_.df) + 1 || + workspace_->position_offsets.empty() || workspace_->position_offsets.front() != 0 || + workspace_->position_offsets.back() != workspace_->positions_flat.size())) { + return posting_corruption("dd/prx document shape mismatch", source_ordinal_); + } + decoded_docs_ = entry_.df; + if (term_has_positions_) { + decoded_total_freq_ = prx_shape.total_positions; + decoded_max_freq_ = prx_shape.max_frequency; + } + flat_loaded_ = true; + return Status::OK(); +} + +Status SniiPostingCursor::load_windowed_chunk() { + DCHECK(workspace_ != nullptr); + format::WindowMeta meta; + RETURN_IF_ERROR(workspace_->prelude.window(next_window_, &meta)); + reader::WindowAbsRange range; + RETURN_IF_ERROR(reader::windowed_window_range( + *index_, entry_, frq_base_, prx_base_, workspace_->prelude, next_window_, + /*want_positions=*/term_has_positions_, /*want_freq=*/false, &range)); + + Slice dd_bytes; + Slice prx_bytes; + RETURN_IF_ERROR(read_context_->posting_cache_->resolve( + PostingStream::kDocs, range.dd_off, range.dd_len, &workspace_->docs_scratch, &dd_bytes, + read_context_->memory_reporter_ == nullptr ? nullptr + : &workspace_->docs_scratch_reservation)); + if (term_has_positions_) { + RETURN_IF_ERROR(read_context_->posting_cache_->resolve( + PostingStream::kPrx, range.prx_off, range.prx_len, &workspace_->prx_scratch, + &prx_bytes, + read_context_->memory_reporter_ == nullptr ? nullptr + : &workspace_->prx_scratch_reservation)); + } + RETURN_IF_ERROR(workspace_->reserve_docids(meta.doc_count)); + RETURN_IF_ERROR(decode_dd(dd_bytes, + format::FrqRegionMeta {.zstd = meta.dd_zstd, + .uncomp_len = meta.dd_uncomp_len, + .disk_len = meta.dd_disk_len, + .crc = meta.crc_dd, + .verify_crc = meta.verify_crc}, + meta.win_base, meta.doc_count)); + format::PrxDecodedShape prx_shape; + if (term_has_positions_) { + RETURN_IF_ERROR(decode_prx(prx_bytes, &prx_shape)); + } + if (workspace_->docids.size() != meta.doc_count || workspace_->docids.empty() || + workspace_->docids.back() != meta.last_docid || + (next_window_ != 0 && workspace_->docids.front() <= meta.win_base)) { + return posting_corruption("window docid shape or last docid mismatch", source_ordinal_); + } + if (term_has_positions_ && + (prx_shape.total_docs != meta.doc_count || + prx_shape.total_positions != workspace_->positions_flat.size() || + prx_shape.has_zero_frequency || + workspace_->position_offsets.size() != static_cast(meta.doc_count) + 1 || + workspace_->position_offsets.empty() || workspace_->position_offsets.front() != 0 || + workspace_->position_offsets.back() != workspace_->positions_flat.size())) { + return posting_corruption("dd/prx document shape mismatch", source_ordinal_); + } + if (term_has_positions_ && entry_.term_stats_present && + prx_shape.max_frequency != meta.max_freq) { + return posting_corruption("window max frequency mismatch", source_ordinal_); + } + if (decoded_docs_ > entry_.df || meta.doc_count > entry_.df - decoded_docs_) { + return posting_corruption("decoded document count exceeds df", source_ordinal_); + } + decoded_docs_ += meta.doc_count; + if (term_has_positions_) { + if (prx_shape.total_positions > + std::numeric_limits::max() - decoded_total_freq_) { + return posting_corruption("total term frequency overflow", source_ordinal_); + } + decoded_total_freq_ += prx_shape.total_positions; + decoded_max_freq_ = std::max(decoded_max_freq_, prx_shape.max_frequency); + } + ++next_window_; + return Status::OK(); +} + +Status SniiPostingCursor::load_next_chunk(bool* loaded) { + DCHECK(loaded != nullptr); + DCHECK(workspace_ != nullptr); + *loaded = false; + // docids/positions_flat/position_offsets are intentionally NOT cleared here: + // both decode paths size them exactly (resize_uninitialized + post-decode + // shape checks), and clearing first would turn every warm re-decode into a + // cold 0->n grow whose std::vector tail value-initialization memsets the + // whole chunk. On the not-loaded path the stale contents are never read. + + if (shape_ == Shape::kFlat) { + if (flat_loaded_) { + return Status::OK(); + } + RETURN_IF_ERROR(load_flat_chunk()); + *loaded = true; + return Status::OK(); + } + if (next_window_ >= workspace_->prelude.n_windows()) { + return Status::OK(); + } + RETURN_IF_ERROR(load_windowed_chunk()); + *loaded = true; + return Status::OK(); +} + +Status SniiPostingCursor::finish_source() { + if (decoded_docs_ != entry_.df) { + return posting_corruption("decoded document count differs from df", source_ordinal_); + } + if (entry_.term_stats_present && + (decoded_total_freq_ != entry_.ttf_delta || decoded_max_freq_ != entry_.max_freq)) { + return posting_corruption("decoded term statistics mismatch", source_ordinal_); + } + exhausted_ = true; + term_lease_.reset(); + return Status::OK(); +} + +Status SniiPostingCursor::map_decoded_chunk() { + DCHECK(workspace_ != nullptr); + std::vector& runs = workspace_->destination_runs; + std::vector& docids = workspace_->docids; + std::vector& frequencies = workspace_->frequencies; + runs.clear(); + frequencies.clear(); + const size_t document_count = docids.size(); + const size_t max_run_count = + std::min(document_count, rowid_conversion_->destination_segment_doc_counts().size()); + RETURN_IF_ERROR( + workspace_->reserve_remapped(document_count, max_run_count, term_has_positions_)); + + auto append_live_document = [&](uint32_t segment, uint32_t docid, size_t live_docs) { + if (runs.empty() || runs.back().destination_segment != segment) { + if (!runs.empty()) { + runs.back().document_end = static_cast(live_docs); + } + runs.push_back({.destination_segment = segment}); + } + docids[live_docs] = docid; + }; + + if (!source_has_deletions_) { + if (term_has_positions_) { + // Frequencies are adjacent position-offset deltas; fill them in + // their own pass so the remap loop below stays a pure gather. + const uint32_t* offsets = workspace_->position_offsets.data(); + for (size_t ordinal = 0; ordinal < document_count; ++ordinal) { + frequencies.push_back(offsets[ordinal + 1] - offsets[ordinal]); + } + } + // The gather walks source_mapping_ at monotonically increasing but + // sparse indexes the hardware prefetcher cannot follow; the future + // lookup indexes are already decoded, so prefetch them explicitly. + // Without deletions every mapping entry is live, so the destination + // segment can never equal the uint32 sentinel. + constexpr size_t kMapPrefetchDistance = 16; + uint32_t current_segment = std::numeric_limits::max(); + for (size_t ordinal = 0; ordinal < document_count; ++ordinal) { + if (ordinal + kMapPrefetchDistance < document_count) { + __builtin_prefetch(&source_mapping_[docids[ordinal + kMapPrefetchDistance]]); + } + const auto [segment, docid] = source_mapping_[docids[ordinal]]; + if (segment != current_segment) { + if (!runs.empty()) { + runs.back().document_end = static_cast(ordinal); + } + runs.push_back({.destination_segment = segment}); + current_segment = segment; + } + docids[ordinal] = docid; + } + } else if (!term_has_positions_) { + constexpr uint32_t kDeleted = std::numeric_limits::max(); + size_t live_docs = 0; + for (uint32_t source_docid : docids) { + const auto [segment, docid] = source_mapping_[source_docid]; + if (segment != kDeleted) { + append_live_document(segment, docid, live_docs++); + } + } + docids.resize(live_docs); + } else { + constexpr uint32_t kDeleted = std::numeric_limits::max(); + size_t write_position = 0; + size_t live_docs = 0; + workspace_->position_offsets[0] = 0; + for (size_t ordinal = 0; ordinal < docids.size(); ++ordinal) { + const auto [segment, docid] = source_mapping_[docids[ordinal]]; + if (segment == kDeleted) { + continue; + } + const size_t begin = workspace_->position_offsets[ordinal]; + const size_t end = workspace_->position_offsets[ordinal + 1]; + const uint32_t frequency = static_cast(end - begin); + if (write_position != begin) { + std::copy(workspace_->positions_flat.begin() + begin, + workspace_->positions_flat.begin() + end, + workspace_->positions_flat.begin() + write_position); + } + write_position += frequency; + append_live_document(segment, docid, live_docs); + frequencies.push_back(frequency); + workspace_->position_offsets[++live_docs] = static_cast(write_position); + } + docids.resize(live_docs); + workspace_->positions_flat.resize(write_position); + workspace_->position_offsets.resize(live_docs + 1); + } + + if (!runs.empty()) { + runs.back().document_end = static_cast(docids.size()); + } + next_destination_run_ = 0; + return Status::OK(); +} + +void SniiPostingCursor::emit_next_mapped_run(RemappedPostingChunk* chunk) { + DCHECK(chunk != nullptr); + DCHECK(workspace_ != nullptr); + DCHECK_LT(next_destination_run_, workspace_->destination_runs.size()); + + const size_t run_ordinal = next_destination_run_++; + const DestinationPostingRun& run = workspace_->destination_runs[run_ordinal]; + const size_t document_begin = + run_ordinal == 0 ? 0 : workspace_->destination_runs[run_ordinal - 1].document_end; + const size_t document_end = run.document_end; + DCHECK_LT(document_begin, document_end); + DCHECK_LE(document_end, workspace_->docids.size()); + const size_t document_count = document_end - document_begin; + + chunk->destination_segment = run.destination_segment; + chunk->destination_docids = + std::span(workspace_->docids).subspan(document_begin, document_count); + if (!term_has_positions_) { + return; + } + + chunk->freqs = std::span(workspace_->frequencies) + .subspan(document_begin, document_count); + chunk->position_offsets = std::span(workspace_->position_offsets) + .subspan(document_begin, document_count + 1); + const size_t position_begin = chunk->position_offsets.front(); + const size_t position_end = chunk->position_offsets.back(); + DCHECK_LE(position_begin, position_end); + DCHECK_LE(position_end, workspace_->positions_flat.size()); + chunk->positions_flat = std::span(workspace_->positions_flat) + .subspan(position_begin, position_end - position_begin); +} + +Status SniiPostingCursor::next_chunk(RemappedPostingChunk* chunk, bool* has_chunk) { + if (chunk == nullptr || has_chunk == nullptr) { + return Status::Error( + "posting_cursor: null chunk output"); + } + *chunk = {}; + *has_chunk = false; + if (!failed_.ok()) { + return failed_; + } + if (!initialized_) { + return Status::Error( + "posting_cursor: next_chunk before init"); + } + if (exhausted_) { + return Status::OK(); + } + + if (next_destination_run_ < workspace_->destination_runs.size()) { + emit_next_mapped_run(chunk); + *has_chunk = true; + return Status::OK(); + } + + while (true) { + bool loaded = false; + const Status status = load_next_chunk(&loaded); + if (!status.ok()) { + return poison(status); + } + if (!loaded) { + *chunk = {}; + const Status finish_status = finish_source(); + if (!finish_status.ok()) { + return poison(finish_status); + } + return Status::OK(); + } + const Status map_status = map_decoded_chunk(); + if (!map_status.ok()) { + return poison(map_status); + } + if (!workspace_->destination_runs.empty()) { + emit_next_mapped_run(chunk); + *has_chunk = true; + return Status::OK(); + } + *chunk = {}; + } +} + +} // namespace doris::snii::compaction diff --git a/be/src/storage/index/snii/compaction/posting_cursor.h b/be/src/storage/index/snii/compaction/posting_cursor.h new file mode 100644 index 00000000000000..356f08d7a9326a --- /dev/null +++ b/be/src/storage/index/snii/compaction/posting_cursor.h @@ -0,0 +1,258 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/compaction/region_reader.h" +#include "storage/index/snii/compaction/rowid_conversion.h" +#include "storage/index/snii/format/core_metadata.h" +#include "storage/index/snii/format/dict_entry.h" +#include "storage/index/snii/format/frq_prelude.h" +#include "storage/index/snii/format/prx_decode_stats.h" +#include "storage/index/snii/reader/logical_index_reader.h" + +namespace doris::snii::compaction { + +struct DestinationPostingRun { + uint32_t destination_segment = 0; + uint32_t document_end = 0; +}; + +// One destination-homogeneous slice of a decoded physical chunk. Positioned +// runs retain the decoder's offset base; positions_flat covers exactly +// [position_offsets.front(), position_offsets.back()). +struct RemappedPostingChunk { + uint32_t destination_segment = 0; + std::span destination_docids; + std::span freqs; + std::span position_offsets; + std::span positions_flat; +}; + +// Checks [region.offset, region.offset + region.length) against the source +// file without ever evaluating the potentially-overflowing addition. +Status validate_posting_region(const format::RegionRef& region, uint64_t file_size); +bool posting_entry_has_positions(const format::DictEntry& entry); + +class SniiPostingCursor; + +// Persistent read/decode state for every term decoded from one source logical +// index. Docs and positions share one bounded physical chunk cache while their +// decoder vectors and prelude workspace survive term cursor destruction. +class SniiPostingReadContext { +public: + static constexpr size_t kMaxReadAheadBudgetBytes = + 2 * SequentialRegionReader::kDefaultChunkBytes; + static constexpr size_t kMaxRetainedDecoderWorkspaceBytes = 64ULL << 10; + + SniiPostingReadContext(const reader::LogicalIndexReader* index, + size_t total_read_ahead_budget_bytes, + writer::MemoryReporter* memory_reporter = nullptr) + : index_(index), + total_read_ahead_budget_bytes_(total_read_ahead_budget_bytes), + memory_reporter_(memory_reporter) {} + + SniiPostingReadContext(const SniiPostingReadContext&) = delete; + SniiPostingReadContext& operator=(const SniiPostingReadContext&) = delete; + SniiPostingReadContext(SniiPostingReadContext&&) = delete; + SniiPostingReadContext& operator=(SniiPostingReadContext&&) = delete; + + Status init(); + + const reader::LogicalIndexReader* index() const { return index_; } + bool initialized() const { return initialized_; } + size_t total_read_ahead_budget_bytes() const { return total_read_ahead_budget_bytes_; } + uint64_t docs_read_calls() const; + uint64_t prx_read_calls() const; + uint64_t docs_buffer_hits() const; + uint64_t prx_buffer_hits() const; + uint64_t physical_read_ranges() const; + uint64_t physical_read_bytes() const; + size_t resident_read_ahead_capacity_bytes() const; + size_t decoder_workspace_capacity_bytes() const; + size_t retained_decoder_workspace_limit_bytes() const { + return std::min(total_read_ahead_budget_bytes_, kMaxRetainedDecoderWorkspaceBytes); + } + +private: + friend class SniiPostingCursor; + + struct DecoderWorkspace final : public format::PrxCsrAllocationGate { + format::FrqPreludeReader prelude; + writer::MemoryReporter::Reservation docs_scratch_reservation; + writer::MemoryReporter::Reservation prx_scratch_reservation; + writer::MemoryReporter::Reservation docids_reservation; + writer::MemoryReporter::Reservation positions_reservation; + writer::MemoryReporter::Reservation position_offsets_reservation; + writer::MemoryReporter::Reservation destination_runs_reservation; + writer::MemoryReporter::Reservation frequencies_reservation; + writer::MemoryReporter::Reservation decompressed_reservation; + std::vector docs_scratch; + std::vector prx_scratch; + std::vector decompressed; + std::vector docids; + std::vector positions_flat; + std::vector position_offsets; + std::vector destination_runs; + std::vector frequencies; + bool reservations_enabled = false; + + size_t capacity_bytes() const; + void init_memory_reporter(writer::MemoryReporter* memory_reporter); + Status reserve_docids(size_t count); + Status reserve_remapped(size_t document_count, size_t run_count, bool retain_frequencies); + Status reserve_csr(std::vector* pos_flat, size_t position_count, + std::vector* pos_off, size_t offset_count) override; + Status reserve_decompression(size_t bytes, std::vector** buffer) override; + void release_large_buffers(size_t retained_capacity_limit_bytes); + }; + + class TermLease { + public: + ~TermLease(); + + TermLease(const TermLease&) = delete; + TermLease& operator=(const TermLease&) = delete; + + private: + friend class SniiPostingReadContext; + explicit TermLease(SniiPostingReadContext* context) : context_(context) {} + SniiPostingReadContext* context_ = nullptr; + }; + + Status acquire_term(bool has_docs_range, bool has_prx_range, + const format::RegionRef& docs_range, const format::RegionRef& prx_range, + std::unique_ptr* lease); + Status validate_next_range(const format::RegionRef& range, bool has_previous, + uint64_t previous_end, const char* stream, uint64_t* end) const; + Status poison(Status status); + Status poison_active_term(Status status, TermLease* lease); + void release_term(); + const Status& failed_status() const { return failed_; } + + const reader::LogicalIndexReader* index_ = nullptr; + size_t total_read_ahead_budget_bytes_ = 0; + writer::MemoryReporter* memory_reporter_ = nullptr; + format::RegionRef posting_region_; + std::unique_ptr posting_cache_; + DecoderWorkspace decoder_workspace_; + + uint64_t last_docs_end_ = 0; + uint64_t last_prx_end_ = 0; + bool has_docs_range_ = false; + bool has_prx_range_ = false; + bool term_active_ = false; + bool initialized_ = false; + Status failed_ = Status::OK(); +}; + +// Sequential positions-posting decoder for one term in one source SNII index. +// It accepts all v1 physical shapes (inline, slim POD-ref and windowed POD-ref), +// validates the decoded doc/frequency/position stream, applies a validated +// row-id conversion, and yields surviving rows in destination-order chunks. +// +// The borrowed read context and row-id capability must outlive the cursor. A +// cursor is single-use and single-threaded. Only one cursor may hold a context +// term lease at a time. +class SniiPostingCursor { +public: + SniiPostingCursor(SniiPostingReadContext* read_context, format::DictEntry entry, + uint64_t frq_base, uint64_t prx_base, uint32_t source_ordinal, + const ValidatedRowIdConversion* rowid_conversion) + : read_context_(read_context), + index_(read_context == nullptr ? nullptr : read_context->index()), + entry_(std::move(entry)), + frq_base_(frq_base), + prx_base_(prx_base), + source_ordinal_(source_ordinal), + rowid_conversion_(rowid_conversion), + term_has_positions_(posting_entry_has_positions(entry_)) {} + + // Validates index capability, posting locators and fixed entry geometry, + // then acquires the source context's exclusive term lease. Payload decoding + // is lazy: corrupt DD/PRX frames surface from next_chunk(). + Status init(); + + // Returns the next non-empty remapped chunk. Its spans, including an exact + // base-relative positions slice, remain valid until the next call. + // Decoder/corruption failures poison the cursor and are returned unchanged + // by subsequent calls. + Status next_chunk(RemappedPostingChunk* chunk, bool* has_chunk); + bool has_positions() const { return term_has_positions_; } + +private: + enum class Shape : uint8_t { kFlat, kWindowed }; + + Status validate_entry_geometry(); + Status prepare_flat_ranges(); + Status prepare_windowed_ranges(); + Status prepare_windowed(); + Status load_next_chunk(bool* loaded); + Status load_flat_chunk(); + Status load_windowed_chunk(); + Status decode_dd(Slice bytes, const format::FrqRegionMeta& meta, uint64_t win_base, + uint32_t expected_doc_count); + Status decode_prx(Slice bytes, format::PrxDecodedShape* shape); + Status map_decoded_chunk(); + void emit_next_mapped_run(RemappedPostingChunk* chunk); + Status finish_source(); + Status poison(Status status); + + SniiPostingReadContext* read_context_ = nullptr; + const reader::LogicalIndexReader* index_ = nullptr; + format::DictEntry entry_; + uint64_t frq_base_ = 0; + uint64_t prx_base_ = 0; + uint32_t source_ordinal_ = 0; + const ValidatedRowIdConversion* rowid_conversion_ = nullptr; + std::span> source_mapping_; + bool source_has_deletions_ = false; + bool term_has_positions_ = true; + + Shape shape_ = Shape::kFlat; + std::unique_ptr term_lease_; + SniiPostingReadContext::DecoderWorkspace* workspace_ = nullptr; + + uint64_t flat_dd_abs_ = 0; + uint64_t flat_dd_len_ = 0; + uint64_t flat_prx_abs_ = 0; + uint64_t flat_prx_len_ = 0; + uint32_t next_window_ = 0; + bool flat_loaded_ = false; + + uint64_t decoded_docs_ = 0; + uint64_t decoded_total_freq_ = 0; + uint32_t decoded_max_freq_ = 0; + size_t next_destination_run_ = 0; + + bool initialized_ = false; + bool exhausted_ = false; + Status failed_ = Status::OK(); +}; + +} // namespace doris::snii::compaction diff --git a/be/src/storage/index/snii/compaction/posting_run_merger.cpp b/be/src/storage/index/snii/compaction/posting_run_merger.cpp new file mode 100644 index 00000000000000..bdd5b31c8742cd --- /dev/null +++ b/be/src/storage/index/snii/compaction/posting_run_merger.cpp @@ -0,0 +1,463 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/compaction/posting_run_merger.h" + +#include +#include +#include +#include +#include + +#include "common/check.h" + +namespace doris::snii::compaction { + +namespace { + +Status invalid_source(std::string_view reason) { + return Status::Error("posting_run_merger: {}", reason); +} + +Status merge_corruption(std::string_view reason) { + return Status::Error("posting_run_merger: {}", + reason); +} + +bool posting_after(uint32_t lhs_segment, uint32_t lhs_docid, uint32_t rhs_segment, + uint32_t rhs_docid) { + return lhs_segment > rhs_segment || (lhs_segment == rhs_segment && lhs_docid > rhs_docid); +} + +#ifdef BE_TEST +std::atomic posting_run_frontier_update_counter {0}; +std::atomic posting_run_frontier_comparison_counter {0}; +std::atomic posting_run_document_counter {0}; +std::atomic posting_run_emitted_run_counter {0}; +std::atomic posting_run_boundary_search_counter {0}; +std::atomic posting_run_shape_scan_document_counter {0}; +std::atomic posting_run_legacy_fill_call_counter {0}; +std::atomic posting_run_copied_document_counter {0}; +#endif + +size_t lower_bound_docid(std::span docids, size_t begin, uint32_t target) { +#ifdef BE_TEST + posting_run_boundary_search_counter.fetch_add(1, std::memory_order_relaxed); +#endif + const size_t size = docids.size(); + if (begin >= size || docids[begin] >= target) { + return begin; + } + // Gallop before the binary search: when many sources interleave (a full + // compaction merging dozens of sorted segments), destination runs shrink to + // one or two documents, and a plain binary search over the whole remaining + // chunk costs O(log chunk) comparisons per emitted run. Doubling probes + // resolve those short runs in O(1) while keeping O(log run) for long runs. + size_t less = begin; + size_t probe = 1; + while (less + probe < size && docids[less + probe] < target) { + less += probe; + probe *= 2; + } + size_t low = less + 1; + size_t high = std::min(less + probe, size); + while (low < high) { + const size_t middle = low + (high - low) / 2; + if (docids[middle] < target) { + low = middle + 1; + } else { + high = middle; + } + } + return low; +} + +} // namespace + +void MergedPostingRuns::ActivePostingChunk::refresh_frontier() { + DCHECK_LT(ordinal, chunk.destination_docids.size()); + frontier_segment = chunk.destination_segment; + frontier_docid = chunk.destination_docids[ordinal]; +} + +Status MergedPostingRuns::ActivePostingChunk::validate_and_refresh_frontier( + bool retain_positions, std::span destination_doc_counts) { + if (chunk.destination_docids.empty() || ordinal >= chunk.destination_docids.size()) { + return merge_corruption("destination posting run is empty"); + } + if (chunk.destination_segment >= destination_doc_counts.size()) { + return merge_corruption("destination posting run segment is out of range"); + } + if (retain_positions) { + if (chunk.freqs.size() != chunk.destination_docids.size() || + chunk.position_offsets.size() != chunk.destination_docids.size() + 1) { + return merge_corruption("positioned posting run has an invalid shape"); + } + const uint32_t position_begin = chunk.position_offsets.front(); + const uint32_t position_end = chunk.position_offsets.back(); + if (position_end < position_begin || + position_end - position_begin != chunk.positions_flat.size()) { + return merge_corruption("positioned posting run has invalid offsets"); + } + } else if (!chunk.freqs.empty() || !chunk.position_offsets.empty() || + !chunk.positions_flat.empty()) { + return merge_corruption("docs-only posting run has positioned payload"); + } + + uint32_t previous_offset = retain_positions ? chunk.position_offsets.front() : 0; + for (size_t document = 0; document < chunk.destination_docids.size(); ++document) { + const uint32_t docid = chunk.destination_docids[document]; + if (docid >= destination_doc_counts[chunk.destination_segment]) { + return merge_corruption("destination posting run document is out of range"); + } + if (document > 0 && docid <= chunk.destination_docids[document - 1]) { + return merge_corruption("destination posting run is not strictly monotone"); + } + if (retain_positions) { + const uint32_t next_offset = chunk.position_offsets[document + 1]; + if (next_offset < previous_offset || + next_offset - previous_offset != chunk.freqs[document]) { + return merge_corruption("positioned posting run offsets differ from frequencies"); + } + previous_offset = next_offset; + } +#ifdef BE_TEST + posting_run_shape_scan_document_counter.fetch_add(1, std::memory_order_relaxed); +#endif + } + if (has_previous_chunk_posting && + !posting_after(chunk.destination_segment, chunk.destination_docids.front(), + previous_chunk_segment, previous_chunk_docid)) { + return merge_corruption("source posting chunks are not globally monotone"); + } + previous_chunk_segment = chunk.destination_segment; + previous_chunk_docid = chunk.destination_docids.back(); + has_previous_chunk_posting = true; + refresh_frontier(); + return Status::OK(); +} + +bool MergedPostingRuns::FrontierBefore::operator()(size_t lhs, size_t rhs) const { +#ifdef BE_TEST + posting_run_frontier_comparison_counter.fetch_add(1, std::memory_order_relaxed); +#endif + const ActivePostingChunk& lhs_chunk = (*active_chunks)[lhs]; + const ActivePostingChunk& rhs_chunk = (*active_chunks)[rhs]; + if (lhs_chunk.frontier_segment != rhs_chunk.frontier_segment) { + return lhs_chunk.frontier_segment < rhs_chunk.frontier_segment; + } + if (lhs_chunk.frontier_docid != rhs_chunk.frontier_docid) { + return lhs_chunk.frontier_docid < rhs_chunk.frontier_docid; + } + return lhs < rhs; +} + +MergedPostingRuns::MergedPostingRuns(std::vector> cursors, + bool retain_positions, bool counts_as_semantic_token, + std::span destination_doc_counts, + std::span destination_semantic_token_counts) + : cursors_(std::move(cursors)), + active_frontier_(FrontierBefore {.active_chunks = &active_chunks_}), + retain_positions_(retain_positions), + counts_as_semantic_token_(counts_as_semantic_token), + destination_doc_counts_(destination_doc_counts), + destination_semantic_token_counts_(destination_semantic_token_counts) {} + +Status MergedPostingRuns::init() { + if (initialized_) { + return invalid_source("source initialized twice"); + } + if (cursors_.empty() || destination_doc_counts_.empty()) { + return invalid_source("source or destination set is empty"); + } + if (counts_as_semantic_token_ && + destination_semantic_token_counts_.size() != destination_doc_counts_.size()) { + return invalid_source("semantic token counters differ from destination count"); + } + active_chunks_.resize(cursors_.size()); + for (size_t cursor_ordinal = 0; cursor_ordinal < cursors_.size(); ++cursor_ordinal) { + bool has_chunk = false; + RETURN_IF_ERROR(cursors_[cursor_ordinal]->next_chunk(&active_chunks_[cursor_ordinal].chunk, + &has_chunk)); + if (has_chunk) { + RETURN_IF_ERROR(active_chunks_[cursor_ordinal].validate_and_refresh_frontier( + retain_positions_, destination_doc_counts_)); + } + } + active_frontier_.build(cursors_.size(), [this](size_t source) { + return !active_chunks_[source].chunk.destination_docids.empty(); + }); + initialized_ = true; + return Status::OK(); +} + +bool MergedPostingRuns::empty() const { + DCHECK(initialized_); + return active_frontier_.empty(); +} + +uint32_t MergedPostingRuns::next_destination() const { + DCHECK(initialized_); + DCHECK(!active_destination_.has_value()); + DCHECK(!pending_source_.has_value()); + return front_segment(); +} + +Status MergedPostingRuns::begin_destination(uint32_t destination) { + if (!initialized_ || active_destination_.has_value() || pending_source_.has_value() || + active_frontier_.empty()) { + return invalid_source("cannot begin destination"); + } + if (front_segment() != destination) { + return invalid_source("destination differs from frontier"); + } + active_destination_ = destination; + return Status::OK(); +} + +Status MergedPostingRuns::next_run(uint32_t max_docs, writer::PostingRunView* run, bool* has_run) { + if (max_docs == 0 || run == nullptr || has_run == nullptr) { + return invalid_source("invalid next_run arguments"); + } + if (!initialized_ || !active_destination_.has_value()) { + return invalid_source("source has no active destination"); + } + *run = {}; + *has_run = false; + RETURN_IF_ERROR(settle_pending_run()); + if (active_frontier_.empty() || front_segment() != *active_destination_) { + active_destination_.reset(); + return Status::OK(); + } + RETURN_IF_ERROR(select_front_run(max_docs, run)); + *has_run = true; + return Status::OK(); +} + +Status MergedPostingRuns::fill(uint32_t target_docs, writer::TermPostingBuffer* out, + bool* exhausted) { + if (target_docs == 0 || out == nullptr || exhausted == nullptr) { + return invalid_source("invalid fill arguments"); + } + if (!initialized_ || !active_destination_.has_value()) { + return invalid_source("source has no active destination"); + } + if (!out->empty()) { + return invalid_source("output must be empty"); + } +#ifdef BE_TEST + posting_run_legacy_fill_call_counter.fetch_add(1, std::memory_order_relaxed); +#endif + + while (out->document_count() < target_docs) { + writer::PostingRunView run; + bool has_run = false; + RETURN_IF_ERROR(next_run(static_cast(target_docs - out->document_count()), &run, + &has_run)); + if (!has_run) { + *exhausted = true; + return Status::OK(); + } + const size_t position_count = run.positions_flat.size(); + writer::MutableTermPostingSpan destination; + RETURN_IF_ERROR(out->grow_uninitialized(run.docids.size(), retain_positions_, + position_count, &destination)); + std::ranges::copy(run.docids, destination.docids.begin()); + if (retain_positions_) { + std::ranges::copy(run.freqs, destination.freqs.begin()); + std::ranges::copy(run.positions_flat, destination.positions_flat.begin()); + } +#ifdef BE_TEST + posting_run_copied_document_counter.fetch_add(run.docids.size(), std::memory_order_relaxed); +#endif + } + + RETURN_IF_ERROR(settle_pending_run()); + *exhausted = active_frontier_.empty() || front_segment() != *active_destination_; + if (*exhausted) { + active_destination_.reset(); + } + return Status::OK(); +} + +uint32_t MergedPostingRuns::front_segment() const { + return active_chunks_[active_frontier_.winner()].frontier_segment; +} + +Status MergedPostingRuns::select_front_run(size_t max_docs, writer::PostingRunView* run) { + DCHECK_GT(max_docs, 0); + DCHECK(!pending_source_.has_value()); + const size_t cursor_ordinal = active_frontier_.winner(); + std::optional> next_frontier; + const size_t runner_up = active_frontier_.runner_up(); + if (runner_up != IndexedWinnerTree::kNoSource) { + const ActivePostingChunk& next = active_chunks_[runner_up]; + next_frontier = std::pair(next.frontier_segment, next.frontier_docid); + } + + ActivePostingChunk& active = active_chunks_[cursor_ordinal]; + RETURN_IF_ERROR(select_run(&active, max_docs, next_frontier, run)); + pending_source_ = cursor_ordinal; + return Status::OK(); +} + +Status MergedPostingRuns::select_run(ActivePostingChunk* active, size_t max_docs, + std::optional> next_frontier, + writer::PostingRunView* run) { + DCHECK(active != nullptr); + const auto docids = active->chunk.destination_docids; + if (active->chunk.destination_segment != *active_destination_) { + return merge_corruption("posting run differs from the active destination"); + } + const size_t begin = active->ordinal; + size_t end = begin + std::min(max_docs, docids.size() - begin); + if (next_frontier.has_value() && next_frontier->first == active->chunk.destination_segment) { + end = std::min(end, lower_bound_docid(docids, begin, next_frontier->second)); + } + if (end == begin) { + return merge_corruption("destination postings contain an equal merge frontier"); + } + if (has_previous_posting_ && !posting_after(active->frontier_segment, active->frontier_docid, + previous_segment_, previous_docid_)) { + return merge_corruption("destination postings are duplicated or not globally monotone"); + } + + const size_t document_count = end - begin; + size_t position_begin = 0; + size_t position_count = 0; + if (retain_positions_) { + const size_t position_base = active->chunk.position_offsets.front(); + const size_t absolute_position_begin = active->chunk.position_offsets[begin]; + const size_t absolute_position_end = active->chunk.position_offsets[end]; + DCHECK_GE(absolute_position_begin, position_base); + DCHECK_GE(absolute_position_end, absolute_position_begin); + DCHECK_LE(absolute_position_end - position_base, active->chunk.positions_flat.size()); + position_begin = absolute_position_begin - position_base; + position_count = absolute_position_end - absolute_position_begin; + } + + run->docids = docids.subspan(begin, document_count); + run->freqs = retain_positions_ ? active->chunk.freqs.subspan(begin, document_count) + : std::span {}; + run->position_offsets = + retain_positions_ ? active->chunk.position_offsets.subspan(begin, document_count + 1) + : std::span {}; + run->positions_flat = + retain_positions_ ? active->chunk.positions_flat.subspan(position_begin, position_count) + : std::span {}; + + if (counts_as_semantic_token_) { + uint64_t& token_count = destination_semantic_token_counts_[*active_destination_]; + if (position_count > std::numeric_limits::max() - token_count) { + return merge_corruption("semantic token count overflows uint64"); + } + token_count += position_count; + } + previous_segment_ = active->chunk.destination_segment; + previous_docid_ = docids[end - 1]; + has_previous_posting_ = true; + active->ordinal = end; +#ifdef BE_TEST + posting_run_document_counter.fetch_add(document_count, std::memory_order_relaxed); + posting_run_emitted_run_counter.fetch_add(1, std::memory_order_relaxed); +#endif + return Status::OK(); +} + +Status MergedPostingRuns::settle_pending_run() { + if (!pending_source_.has_value()) { + return Status::OK(); + } + const size_t cursor_ordinal = *pending_source_; + RETURN_IF_ERROR(advance_front_source(cursor_ordinal, &active_chunks_[cursor_ordinal])); + pending_source_.reset(); + return Status::OK(); +} + +Status MergedPostingRuns::advance_front_source(size_t cursor_ordinal, ActivePostingChunk* active) { + DCHECK(active != nullptr); + bool has_chunk = active->ordinal < active->chunk.destination_docids.size(); + if (!has_chunk) { + active->chunk = {}; + active->ordinal = 0; + RETURN_IF_ERROR(cursors_[cursor_ordinal]->next_chunk(&active->chunk, &has_chunk)); + } + if (has_chunk) { + if (active->ordinal == 0) { + RETURN_IF_ERROR(active->validate_and_refresh_frontier(retain_positions_, + destination_doc_counts_)); + } else { + active->refresh_frontier(); + } + } + active_frontier_.update(cursor_ordinal, has_chunk); +#ifdef BE_TEST + posting_run_frontier_update_counter.fetch_add(1, std::memory_order_relaxed); +#endif + return Status::OK(); +} + +#ifdef BE_TEST +namespace testing { + +void reset_posting_run_merge_counters() { + posting_run_frontier_update_counter.store(0, std::memory_order_relaxed); + posting_run_frontier_comparison_counter.store(0, std::memory_order_relaxed); + posting_run_document_counter.store(0, std::memory_order_relaxed); + posting_run_emitted_run_counter.store(0, std::memory_order_relaxed); + posting_run_boundary_search_counter.store(0, std::memory_order_relaxed); + posting_run_shape_scan_document_counter.store(0, std::memory_order_relaxed); + posting_run_legacy_fill_call_counter.store(0, std::memory_order_relaxed); + posting_run_copied_document_counter.store(0, std::memory_order_relaxed); +} + +uint64_t posting_run_frontier_updates() { + return posting_run_frontier_update_counter.load(std::memory_order_relaxed); +} + +uint64_t posting_run_frontier_comparisons() { + return posting_run_frontier_comparison_counter.load(std::memory_order_relaxed); +} + +uint64_t posting_run_documents() { + return posting_run_document_counter.load(std::memory_order_relaxed); +} + +uint64_t posting_run_emitted_runs() { + return posting_run_emitted_run_counter.load(std::memory_order_relaxed); +} + +uint64_t posting_run_boundary_searches() { + return posting_run_boundary_search_counter.load(std::memory_order_relaxed); +} + +uint64_t posting_run_shape_scan_documents() { + return posting_run_shape_scan_document_counter.load(std::memory_order_relaxed); +} + +uint64_t posting_run_legacy_fill_calls() { + return posting_run_legacy_fill_call_counter.load(std::memory_order_relaxed); +} + +uint64_t posting_run_copied_documents() { + return posting_run_copied_document_counter.load(std::memory_order_relaxed); +} + +} // namespace testing +#endif + +} // namespace doris::snii::compaction diff --git a/be/src/storage/index/snii/compaction/posting_run_merger.h b/be/src/storage/index/snii/compaction/posting_run_merger.h new file mode 100644 index 00000000000000..7792a862d35f22 --- /dev/null +++ b/be/src/storage/index/snii/compaction/posting_run_merger.h @@ -0,0 +1,116 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/compaction/indexed_winner_tree.h" +#include "storage/index/snii/compaction/posting_cursor.h" +#include "storage/index/snii/writer/posting_window_emitter.h" +#include "storage/index/snii/writer/term_posting_source.h" + +namespace doris::snii::compaction { + +// K-way merge of destination-homogeneous cursor chunks. next_run() borrows the +// selected cursor workspace without copying; the view remains valid until a +// later next_run() call advances that cursor. +class MergedPostingRuns final : public writer::TermPostingSource { + struct ActivePostingChunk { + RemappedPostingChunk chunk; + size_t ordinal = 0; + uint32_t frontier_segment = 0; + uint32_t frontier_docid = 0; + uint32_t previous_chunk_segment = 0; + uint32_t previous_chunk_docid = 0; + bool has_previous_chunk_posting = false; + + void refresh_frontier(); + Status validate_and_refresh_frontier(bool retain_positions, + std::span destination_doc_counts); + }; + + struct FrontierBefore { + const std::vector* active_chunks = nullptr; + + bool operator()(size_t lhs, size_t rhs) const; + }; + +public: + MergedPostingRuns(std::vector> cursors, + bool retain_positions, bool counts_as_semantic_token, + std::span destination_doc_counts, + std::span destination_semantic_token_counts); + + Status init(); + bool empty() const; + uint32_t next_destination() const; + Status begin_destination(uint32_t destination); + Status next_run(uint32_t max_docs, writer::PostingRunView* run, bool* has_run); + + // Transitional streamed-session adapter. Compaction tests use next_run() + // directly; the assembler integration removes this handoff in the next + // milestone. + Status fill(uint32_t target_docs, writer::TermPostingBuffer* out, bool* exhausted) override; + +private: + uint32_t front_segment() const; + Status select_front_run(size_t max_docs, writer::PostingRunView* run); + Status select_run(ActivePostingChunk* active, size_t max_docs, + std::optional> next_frontier, + writer::PostingRunView* run); + Status settle_pending_run(); + Status advance_front_source(size_t cursor_ordinal, ActivePostingChunk* active); + + std::vector> cursors_; + std::vector active_chunks_; + IndexedWinnerTree active_frontier_; + bool retain_positions_ = true; + bool counts_as_semantic_token_ = false; + std::span destination_doc_counts_; + std::span destination_semantic_token_counts_; + std::optional active_destination_; + std::optional pending_source_; + uint32_t previous_segment_ = 0; + uint32_t previous_docid_ = 0; + bool has_previous_posting_ = false; + bool initialized_ = false; +}; + +#ifdef BE_TEST +namespace testing { + +void reset_posting_run_merge_counters(); +uint64_t posting_run_frontier_updates(); +uint64_t posting_run_frontier_comparisons(); +uint64_t posting_run_documents(); +uint64_t posting_run_emitted_runs(); +uint64_t posting_run_boundary_searches(); +uint64_t posting_run_shape_scan_documents(); +uint64_t posting_run_legacy_fill_calls(); +uint64_t posting_run_copied_documents(); + +} // namespace testing +#endif + +} // namespace doris::snii::compaction diff --git a/be/src/storage/index/snii/compaction/region_reader.cpp b/be/src/storage/index/snii/compaction/region_reader.cpp new file mode 100644 index 00000000000000..3832b8505523ef --- /dev/null +++ b/be/src/storage/index/snii/compaction/region_reader.cpp @@ -0,0 +1,274 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/compaction/region_reader.h" + +#include +#include + +namespace doris::snii::compaction { + +namespace { + +Status reserve_read_buffer(size_t target, std::vector* buffer, + writer::MemoryReporter::Reservation* reservation) { + if (reservation == nullptr) return Status::OK(); + if (buffer->capacity() >= target) { + DCHECK_EQ(reservation->bytes(), buffer->capacity()); + return Status::OK(); + } + writer::MemoryReporter::Reservation replacement; + RETURN_IF_ERROR(reservation->prepare_replacement(target, &replacement)); + buffer->reserve(target); + DCHECK_EQ(buffer->capacity(), target); + *reservation = std::move(replacement); + return Status::OK(); +} + +} // namespace + +size_t SharedAlignedRegionCache::stream_index(PostingStream stream) { + const size_t index = static_cast(stream); + DCHECK_LT(index, kStreamCount); + return index; +} + +Status SharedAlignedRegionCache::init() { + if (initialized_) { + return Status::Error( + "shared_region_cache: init called twice"); + } + if (reader_ == nullptr) { + return Status::Error( + "shared_region_cache: null reader"); + } + if (total_budget_bytes_ < kSlotCount) { + return Status::Error( + "shared_region_cache: budget must hold two chunks"); + } + if (region_len_ > std::numeric_limits::max() - region_off_ || + region_off_ > reader_->size() || region_len_ > reader_->size() - region_off_) { + return Status::Error( + "shared_region_cache: region outside source file"); + } + + block_bytes_ = total_budget_bytes_ / kSlotCount; + for (size_t slot_index = 0; slot_index < slots_.size(); ++slot_index) { + Slot& slot = slots_[slot_index]; + if (memory_reporter_ != nullptr) { + slot_reservations_[slot_index] = memory_reporter_->make_reservation(); + RETURN_IF_ERROR(slot_reservations_[slot_index].set_bytes(block_bytes_)); + slot.bytes.reserve(block_bytes_); + DCHECK_EQ(slot.bytes.capacity(), block_bytes_); + } + slot.bytes.resize(block_bytes_); + } + if (resident_capacity_bytes() > total_budget_bytes_) { + return Status::Error( + "shared_region_cache: allocator exceeded cache budget"); + } + initialized_ = true; + return Status::OK(); +} + +size_t SharedAlignedRegionCache::resident_capacity_bytes() const { + size_t capacity = 0; + for (const Slot& slot : slots_) { + capacity += slot.bytes.capacity(); + } + return capacity; +} + +uint64_t SharedAlignedRegionCache::read_calls(PostingStream stream) const { + return read_calls_[stream_index(stream)]; +} + +uint64_t SharedAlignedRegionCache::buffer_hits(PostingStream stream) const { + return buffer_hits_[stream_index(stream)]; +} + +void SharedAlignedRegionCache::unpin(PostingStream stream) { + const size_t index = stream_index(stream); + const int8_t slot_index = stream_slots_[index]; + if (slot_index < 0) { + return; + } + Slot& slot = slots_[static_cast(slot_index)]; + DCHECK_GT(slot.pins, 0); + --slot.pins; + stream_slots_[index] = -1; +} + +Status SharedAlignedRegionCache::read_physical(PostingStream stream, uint64_t abs_off, size_t len, + std::vector* out) { + RETURN_IF_ERROR(reader_->read_at(abs_off, len, out)); + ++physical_read_ranges_; + physical_read_bytes_ += len; + ++read_calls_[stream_index(stream)]; + return Status::OK(); +} + +Status SharedAlignedRegionCache::resolve(PostingStream stream, uint64_t abs_off, uint64_t len, + std::vector* scratch, Slice* out, + writer::MemoryReporter::Reservation* scratch_reservation) { + if (scratch == nullptr || out == nullptr) { + return Status::Error("shared_region_cache: null out"); + } + if (!initialized_) { + return Status::Error( + "shared_region_cache: resolve before init"); + } + *out = Slice(); + if (abs_off < region_off_) { + return Status::Error( + "shared_region_cache: window outside region"); + } + const uint64_t relative_off = abs_off - region_off_; + if (relative_off > region_len_ || len > region_len_ - relative_off) { + return Status::Error( + "shared_region_cache: window outside region"); + } + if (len > std::numeric_limits::max()) { + return Status::Error( + "shared_region_cache: window length out of range"); + } + + unpin(stream); + const size_t want = static_cast(len); + if (want == 0) { + return Status::OK(); + } + + const uint64_t aligned_relative_off = (relative_off / block_bytes_) * block_bytes_; + const uint64_t block_off = region_off_ + aligned_relative_off; + const size_t block_len = static_cast( + std::min(block_bytes_, region_len_ - aligned_relative_off)); + const size_t offset_in_block = static_cast(relative_off - aligned_relative_off); + const bool fits_one_block = want <= block_len - offset_in_block; + if (!fits_one_block) { + RETURN_IF_ERROR(reserve_read_buffer(want, scratch, scratch_reservation)); + RETURN_IF_ERROR(read_physical(stream, abs_off, want, scratch)); + if (scratch->size() != want) { + return Status::Error( + "shared_region_cache: short exact read"); + } + *out = Slice(scratch->data(), want); + return Status::OK(); + } + + const size_t stream_id = stream_index(stream); + for (size_t slot_index = 0; slot_index < slots_.size(); ++slot_index) { + Slot& slot = slots_[slot_index]; + if (slot.valid && slot.offset == block_off && offset_in_block + want <= slot.bytes.size()) { + ++slot.pins; + stream_slots_[stream_id] = static_cast(slot_index); + ++buffer_hits_[stream_id]; + *out = Slice(slot.bytes.data() + offset_in_block, want); + return Status::OK(); + } + } + + size_t slot_index = 0; + while (slot_index < slots_.size() && slots_[slot_index].pins != 0) { + ++slot_index; + } + DCHECK_LT(slot_index, slots_.size()); + Slot& slot = slots_[slot_index]; + slot.valid = false; + RETURN_IF_ERROR(read_physical(stream, block_off, block_len, &slot.bytes)); + if (slot.bytes.size() != block_len) { + return Status::Error( + "shared_region_cache: short chunk read"); + } + slot.offset = block_off; + slot.pins = 1; + slot.valid = true; + stream_slots_[stream_id] = static_cast(slot_index); + *out = Slice(slot.bytes.data() + offset_in_block, want); + return Status::OK(); +} + +Status SequentialRegionReader::resolve(uint64_t abs_off, uint64_t len, + std::vector* scratch, Slice* out) { + if (scratch == nullptr || out == nullptr) { + return Status::Error("region_reader: null out"); + } + if (reader_ == nullptr) { + return Status::Error("region_reader: null reader"); + } + *out = Slice(); + // Subtraction-based bounds avoid wrapping region_offset+region_length or + // abs_off+len at UINT64_MAX. + if (region_len_ > std::numeric_limits::max() - region_off_ || abs_off < region_off_) { + return Status::Error( + "region_reader: window outside region"); + } + const uint64_t relative_off = abs_off - region_off_; + if (relative_off > region_len_ || len > region_len_ - relative_off) { + return Status::Error( + "region_reader: window outside region"); + } + if (len > std::numeric_limits::max()) { + return Status::Error( + "region_reader: window length out of range"); + } + const size_t want = static_cast(len); + if (want == 0) { + return Status::OK(); + } + + // 1. Buffered hit: zero-copy slice, no file read. + if (!buf_.empty() && abs_off >= buf_off_ && abs_off - buf_off_ + want <= buf_.size()) { + ++buffer_hits_; + *out = Slice(buf_.data() + (abs_off - buf_off_), want); + return Status::OK(); + } + + // 3. Oversized or backward miss: one exact range read into the caller's + // scratch, keeping the buffered chunk (and the forward stream position) + // intact. + const bool backward = !buf_.empty() && abs_off < buf_off_; + if (want > chunk_bytes_ || backward) { + RETURN_IF_ERROR(reader_->read_at(abs_off, want, scratch)); + ++read_calls_; + if (scratch->size() != want) { + return Status::Error( + "region_reader: short read"); + } + *out = Slice(scratch->data(), want); + return Status::OK(); + } + + // 2. Forward miss: refill the chunk starting at the window, clamped to the + // region end so read-ahead never reads past the region (whose tail may abut + // other file sections or EOF). want <= chunk and the range check above + // guarantee the window fits the refilled chunk. + const uint64_t remaining = region_len_ - relative_off; + const size_t fill = static_cast(std::min(chunk_bytes_, remaining)); + RETURN_IF_ERROR(reader_->read_at(abs_off, fill, &buf_)); + ++read_calls_; + if (buf_.size() != fill) { + buf_.clear(); + return Status::Error( + "region_reader: short chunk read"); + } + buf_off_ = abs_off; + *out = Slice(buf_.data(), want); + return Status::OK(); +} + +} // namespace doris::snii::compaction diff --git a/be/src/storage/index/snii/compaction/region_reader.h b/be/src/storage/index/snii/compaction/region_reader.h new file mode 100644 index 00000000000000..63304883cd8ed7 --- /dev/null +++ b/be/src/storage/index/snii/compaction/region_reader.h @@ -0,0 +1,156 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/io/file_reader.h" +#include "storage/index/snii/writer/memory_reporter.h" + +// SequentialRegionReader -- chunked sequential read-ahead over ONE contiguous +// byte region of a source file (T2.3, compaction index-merge fast path). +// +// The merge walks a source segment's posting region in ascending offset order +// (the writer laid the per-term [prx][frq] spans out in term order, and the +// term cursor replays terms in that same order), so per-window read_at calls +// would issue thousands of tiny reads over an already-sequential byte stream. +// This reader amortizes them: resolve() serves a window from the buffered +// chunk when possible and only touches the file on a miss. +// +// resolve() preference order (documented contract, pinned by UT): +// 1. window fully inside the buffered chunk -> zero-copy Slice into the +// buffer, NO file read; +// 2. forward miss with len <= chunk_bytes -> ONE chunk read starting at the +// window (clamped to the region end, never past it) and a slice of it; +// 3. oversized (len > chunk_bytes) or backward window -> ONE exact range +// read into *scratch, leaving the buffered chunk untouched (a rare +// backward probe must not thrash the forward stream). +// A window not fully inside [region_offset, region_offset+region_length) is +// Corruption -- posting locators were already validated against the region by +// LogicalIndexReader::resolve_*_window, so an out-of-region request here means +// a caller bug or corrupt state, never a legal miss. +// +// The returned Slice is valid until the NEXT resolve() call (buffer path) or +// until *scratch is next modified (fallback path); callers decode immediately. +// Single-threaded, borrowed FileReader must outlive the region reader. +namespace doris::snii::compaction { + +enum class PostingStream : uint8_t { kDocs = 0, kPrx = 1 }; + +// Two logical monotone posting streams share two aligned physical chunks. A +// stream pins at most its current chunk, so resolving the other stream cannot +// invalidate an outstanding Slice. Requests crossing an aligned chunk use the +// caller's per-stream scratch and do not evict either pinned chunk. +class SharedAlignedRegionCache { +public: + SharedAlignedRegionCache(io::FileReader* reader, uint64_t region_offset, uint64_t region_length, + size_t total_budget_bytes, + writer::MemoryReporter* memory_reporter = nullptr) + : reader_(reader), + region_off_(region_offset), + region_len_(region_length), + total_budget_bytes_(total_budget_bytes), + memory_reporter_(memory_reporter) {} + + Status init(); + Status resolve(PostingStream stream, uint64_t abs_off, uint64_t len, + std::vector* scratch, Slice* out, + writer::MemoryReporter::Reservation* scratch_reservation = nullptr); + + uint64_t physical_read_ranges() const { return physical_read_ranges_; } + uint64_t physical_read_bytes() const { return physical_read_bytes_; } + uint64_t read_calls(PostingStream stream) const; + uint64_t buffer_hits(PostingStream stream) const; + size_t resident_capacity_bytes() const; + +private: + static constexpr size_t kStreamCount = 2; + static constexpr size_t kSlotCount = 2; + + struct Slot { + std::vector bytes; + uint64_t offset = 0; + uint8_t pins = 0; + bool valid = false; + }; + + static size_t stream_index(PostingStream stream); + void unpin(PostingStream stream); + Status read_physical(PostingStream stream, uint64_t abs_off, size_t len, + std::vector* out); + + io::FileReader* reader_ = nullptr; + uint64_t region_off_ = 0; + uint64_t region_len_ = 0; + size_t total_budget_bytes_ = 0; + size_t block_bytes_ = 0; + writer::MemoryReporter* memory_reporter_ = nullptr; + std::array slot_reservations_; + std::array slots_; + std::array stream_slots_ {-1, -1}; + std::array read_calls_ {}; + std::array buffer_hits_ {}; + uint64_t physical_read_ranges_ = 0; + uint64_t physical_read_bytes_ = 0; + bool initialized_ = false; +}; + +class SequentialRegionReader { +public: + // Default chunk: large enough to amortize per-window read overhead over + // slim terms, small enough that k sources x 1 chunk stays negligible next + // to the merge's memory-precheck budget. Configurable per instance (the + // compaction wiring exposes a config knob in T2.6). + static constexpr size_t kDefaultChunkBytes = 4ULL << 20; + + SequentialRegionReader(io::FileReader* reader, uint64_t region_offset, uint64_t region_length, + size_t chunk_bytes = kDefaultChunkBytes) + : reader_(reader), + region_off_(region_offset), + region_len_(region_length), + chunk_bytes_(chunk_bytes == 0 ? kDefaultChunkBytes : chunk_bytes) {} + + // Resolves the absolute byte window [abs_off, abs_off+len) per the + // contract above. len == 0 yields an empty slice without touching the + // file. + Status resolve(uint64_t abs_off, uint64_t len, std::vector* scratch, Slice* out); + + // Observability for tests/profiling: physical reads issued vs windows + // served straight from the buffered chunk. + uint64_t read_calls() const { return read_calls_; } + uint64_t buffer_hits() const { return buffer_hits_; } + +private: + io::FileReader* reader_ = nullptr; + uint64_t region_off_ = 0; + uint64_t region_len_ = 0; + size_t chunk_bytes_ = kDefaultChunkBytes; + + std::vector buf_; // buffered chunk; empty until the first fill + uint64_t buf_off_ = 0; // absolute offset of buf_[0] (valid when buf_ non-empty) + + uint64_t read_calls_ = 0; + uint64_t buffer_hits_ = 0; +}; + +} // namespace doris::snii::compaction diff --git a/be/src/storage/index/snii/compaction/rowid_conversion.cpp b/be/src/storage/index/snii/compaction/rowid_conversion.cpp new file mode 100644 index 00000000000000..e05b80572d20c9 --- /dev/null +++ b/be/src/storage/index/snii/compaction/rowid_conversion.cpp @@ -0,0 +1,240 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/compaction/rowid_conversion.h" + +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "common/check.h" + +namespace doris::snii::compaction { +namespace { + +constexpr uint32_t kDeleted = std::numeric_limits::max(); + +struct HeapEntry { + uint64_t destination_ordinal = 0; + size_t source_ordinal = 0; + size_t source_docid = 0; +}; + +struct HeapEntryGreater { + bool operator()(const HeapEntry& lhs, const HeapEntry& rhs) const { + if (lhs.destination_ordinal != rhs.destination_ordinal) { + return lhs.destination_ordinal > rhs.destination_ordinal; + } + return lhs.source_ordinal > rhs.source_ordinal; + } +}; + +uint64_t destination_ordinal(const std::pair& destination, + const std::vector& destination_segment_prefixes) { + return destination_segment_prefixes[destination.first] + destination.second; +} + +} // namespace + +Status validate_rowid_conversion(const RowIdConversionMap& conversion, + const std::vector& source_segment_doc_counts, + const std::vector& destination_segment_doc_counts) { + if (conversion.size() != source_segment_doc_counts.size()) { + return Status::InvalidArgument( + fmt::format("SNII rowid conversion source segment count mismatch: conversion={}, " + "doc_counts={}", + conversion.size(), source_segment_doc_counts.size())); + } + if (destination_segment_doc_counts.size() > + static_cast(std::numeric_limits::max())) { + return Status::InvalidArgument(fmt::format( + "SNII rowid conversion destination segment count {} exceeds uint32 encoding", + destination_segment_doc_counts.size())); + } + + std::vector destination_segment_prefixes; + destination_segment_prefixes.reserve(destination_segment_doc_counts.size() + 1); + destination_segment_prefixes.push_back(0); + uint64_t destination_doc_count = 0; + for (size_t destination_segment = 0; + destination_segment < destination_segment_doc_counts.size(); ++destination_segment) { + const uint64_t segment_doc_count = destination_segment_doc_counts[destination_segment]; + if (segment_doc_count > std::numeric_limits::max() - destination_doc_count) { + return Status::InvalidArgument(fmt::format( + "SNII rowid conversion destination prefix sum overflows uint64 at segment {}", + destination_segment)); + } + destination_doc_count += segment_doc_count; + destination_segment_prefixes.push_back(destination_doc_count); + } + + for (size_t source = 0; source < conversion.size(); ++source) { + const auto& source_conversion = conversion[source]; + if (source_conversion.size() != source_segment_doc_counts[source]) { + return Status::InvalidArgument(fmt::format( + "SNII rowid conversion source doc count mismatch at source {}: " + "conversion={}, doc_count={}", + source, source_conversion.size(), source_segment_doc_counts[source])); + } + + bool has_previous = false; + uint64_t previous_ordinal = 0; + for (size_t source_docid = 0; source_docid < source_conversion.size(); ++source_docid) { + const auto& destination = source_conversion[source_docid]; + const bool segment_deleted = destination.first == kDeleted; + const bool row_deleted = destination.second == kDeleted; + if (segment_deleted != row_deleted) { + return Status::InvalidArgument(fmt::format( + "SNII rowid conversion entry is partially deleted at source {} doc {}: " + "destination=({}, {})", + source, source_docid, destination.first, destination.second)); + } + if (segment_deleted) { + continue; + } + if (destination.first >= destination_segment_doc_counts.size()) { + return Status::InvalidArgument(fmt::format( + "SNII rowid conversion destination segment {} is out of range at " + "source {} doc {} (segment_count={})", + destination.first, source, source_docid, + destination_segment_doc_counts.size())); + } + if (destination.second >= destination_segment_doc_counts[destination.first]) { + return Status::InvalidArgument(fmt::format( + "SNII rowid conversion destination row {} is out of range at source {} " + "doc {} (destination segment {} has {} docs)", + destination.second, source, source_docid, destination.first, + destination_segment_doc_counts[destination.first])); + } + + const uint64_t ordinal = destination_ordinal(destination, destination_segment_prefixes); + if (has_previous && ordinal <= previous_ordinal) { + return Status::InvalidArgument(fmt::format( + "SNII rowid conversion source {} is not strictly increasing at doc {}: " + "destination ordinal {} follows {}", + source, source_docid, ordinal, previous_ordinal)); + } + previous_ordinal = ordinal; + has_previous = true; + } + } + + std::priority_queue, HeapEntryGreater> heap; + auto push_next_live = [&](size_t source, size_t source_docid) { + const auto& source_conversion = conversion[source]; + while (source_docid < source_conversion.size() && + source_conversion[source_docid].first == kDeleted) { + ++source_docid; + } + if (source_docid < source_conversion.size()) { + heap.push({destination_ordinal(source_conversion[source_docid], + destination_segment_prefixes), + source, source_docid}); + } + }; + + for (size_t source = 0; source < conversion.size(); ++source) { + push_next_live(source, 0); + } + + uint64_t expected_ordinal = 0; + while (!heap.empty()) { + const HeapEntry entry = heap.top(); + heap.pop(); + if (entry.destination_ordinal < expected_ordinal) { + return Status::InvalidArgument(fmt::format( + "SNII rowid conversion has duplicate destination ordinal {} at source {} " + "doc {}", + entry.destination_ordinal, entry.source_ordinal, entry.source_docid)); + } + if (entry.destination_ordinal > expected_ordinal) { + return Status::InvalidArgument(fmt::format( + "SNII rowid conversion is missing destination ordinal {} before ordinal {} " + "at source {} doc {}", + expected_ordinal, entry.destination_ordinal, entry.source_ordinal, + entry.source_docid)); + } + ++expected_ordinal; + push_next_live(entry.source_ordinal, entry.source_docid + 1); + } + + if (expected_ordinal != destination_doc_count) { + return Status::InvalidArgument(fmt::format( + "SNII rowid conversion is missing destination ordinal {}: covered {} of {} " + "destination docs", + expected_ordinal, expected_ordinal, destination_doc_count)); + } + return Status::OK(); +} + +ValidatedRowIdConversion::ValidatedRowIdConversion( + const RowIdConversionMap* conversion, std::vector source_segment_doc_counts, + std::vector destination_segment_doc_counts) + : conversion_(conversion), + source_segment_doc_counts_(std::move(source_segment_doc_counts)), + destination_segment_doc_counts_(std::move(destination_segment_doc_counts)) { + DORIS_CHECK(conversion_ != nullptr); + source_has_deletions_.reserve(conversion_->size()); + for (const auto& source : *conversion_) { + const bool has_deletions = + std::ranges::any_of(source, [](const std::pair& mapping) { + return mapping.first == kDeleted; + }); + source_has_deletions_.push_back(static_cast(has_deletions)); + } +} + +Status ValidatedRowIdConversion::create(const RowIdConversionMap* conversion, + std::span source_segment_doc_counts, + std::span destination_segment_doc_counts, + std::unique_ptr* out) { + if (out == nullptr) { + return Status::InvalidArgument("SNII validated rowid conversion has null out parameter"); + } + out->reset(); + if (conversion == nullptr) { + return Status::InvalidArgument("SNII validated rowid conversion has null conversion"); + } + + std::vector source_counts(source_segment_doc_counts.begin(), + source_segment_doc_counts.end()); + std::vector destination_counts(destination_segment_doc_counts.begin(), + destination_segment_doc_counts.end()); + RETURN_IF_ERROR(validate_rowid_conversion(*conversion, source_counts, destination_counts)); + out->reset(new ValidatedRowIdConversion(conversion, std::move(source_counts), + std::move(destination_counts))); + return Status::OK(); +} + +std::span> ValidatedRowIdConversion::source_mapping( + size_t source_ordinal) const { + DCHECK_LT(source_ordinal, conversion_->size()); + return std::span>((*conversion_)[source_ordinal]); +} + +bool ValidatedRowIdConversion::source_has_deletions(size_t source_ordinal) const { + DCHECK_LT(source_ordinal, source_has_deletions_.size()); + return source_has_deletions_[source_ordinal] != 0; +} + +} // namespace doris::snii::compaction diff --git a/be/src/storage/index/snii/compaction/rowid_conversion.h b/be/src/storage/index/snii/compaction/rowid_conversion.h new file mode 100644 index 00000000000000..ac6a339ba01f4f --- /dev/null +++ b/be/src/storage/index/snii/compaction/rowid_conversion.h @@ -0,0 +1,85 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "common/status.h" + +namespace doris::snii::compaction { + +using RowIdConversionMap = std::vector>>; + +// Capability proving that a complete row-id conversion has passed the global +// shape, bounds, monotonicity and destination-coverage validation. Construction +// is restricted to create(), so merge plans cannot accidentally accept an +// unvalidated conversion. The conversion map is borrowed and must outlive this +// token and every merge plan prepared from it. +class ValidatedRowIdConversion { +public: + ValidatedRowIdConversion(const ValidatedRowIdConversion&) = delete; + ValidatedRowIdConversion& operator=(const ValidatedRowIdConversion&) = delete; + + static Status create(const RowIdConversionMap* conversion, + std::span source_segment_doc_counts, + std::span destination_segment_doc_counts, + std::unique_ptr* out); + + size_t source_segment_count() const { return source_segment_doc_counts_.size(); } + const std::vector& source_segment_doc_counts() const { + return source_segment_doc_counts_; + } + const std::vector& destination_segment_doc_counts() const { + return destination_segment_doc_counts_; + } + bool source_has_deletions(size_t source_ordinal) const; + std::span> source_mapping(size_t source_ordinal) const; + +private: + ValidatedRowIdConversion(const RowIdConversionMap* conversion, + std::vector source_segment_doc_counts, + std::vector destination_segment_doc_counts); + + const RowIdConversionMap* conversion_ = nullptr; + std::vector source_segment_doc_counts_; + std::vector destination_segment_doc_counts_; + std::vector source_has_deletions_; +}; + +// Validates the complete row-id conversion before an SNII index fast merge +// writes any destination bytes. Each inner conversion vector belongs to one +// source segment and is indexed by source docid. A deleted source doc must be +// represented by exactly (UINT32_MAX, UINT32_MAX); every other pair is a live +// (destination segment, destination docid). +// +// In addition to shape and bounds, the function proves that: +// * each source stream is strictly increasing in global destination order; +// * all source streams together contain every destination doc exactly once. +// +// Completeness is checked by a k-way merge of the monotonic source streams. +// Memory is O(source segments + destination segments), never O(output rows). +Status validate_rowid_conversion(const RowIdConversionMap& conversion, + const std::vector& source_segment_doc_counts, + const std::vector& destination_segment_doc_counts); + +} // namespace doris::snii::compaction diff --git a/be/src/storage/index/snii/compaction/snii_index_compaction.cpp b/be/src/storage/index/snii/compaction/snii_index_compaction.cpp new file mode 100644 index 00000000000000..5bba7f822ae01b --- /dev/null +++ b/be/src/storage/index/snii/compaction/snii_index_compaction.cpp @@ -0,0 +1,557 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/compaction/snii_index_compaction.h" + +#include +#include +#include +#include +#include +#include + +#include "common/check.h" +#include "common/logging.h" +#include "storage/index/inverted/common_grams/common_grams_key_codec.h" +#include "storage/index/snii/compaction/eligibility.h" +#include "storage/index/snii/compaction/posting_run_merger.h" +#include "storage/index/snii/compaction/term_cursor.h" +#include "storage/index/snii/compaction/term_merge_frontier.h" +#include "storage/index/snii/format/norms_pod.h" +#include "storage/index/snii/writer/spimi_term_buffer.h" + +namespace doris::snii::compaction { + +namespace { + +Status invalid_plan(std::string_view reason) { + return Status::Error("snii_compaction: {}", reason); +} + +Status merge_corruption(std::string_view reason) { + return Status::Error("snii_compaction: {}", + reason); +} + +bool is_well_formed_common_gram(std::string_view term) { + namespace inverted_index = segment_v2::inverted_index; + if (!term.starts_with(inverted_index::CG_V1_MARKER) || + term.size() > inverted_index::COMMON_GRAM_MAX_ENCODED_BYTES) { + return false; + } + constexpr size_t kLengthBytes = 8; + const size_t length_offset = inverted_index::CG_V1_MARKER.size(); + if (term.size() < length_offset + kLengthBytes + 1) { + return false; + } + uint32_t left_length = 0; + for (size_t i = 0; i < kLengthBytes; ++i) { + const char digit = term[length_offset + i]; + if (!((digit >= '0' && digit <= '9') || (digit >= 'a' && digit <= 'f'))) { + return false; + } + left_length = (left_length << 4) | + static_cast(digit <= '9' ? digit - '0' : digit - 'a' + 10); + } + const size_t separator = length_offset + kLengthBytes; + if (term[separator] != ':') { + return false; + } + const std::string_view components = term.substr(separator + 1); + if (left_length > components.size()) { + return false; + } + return inverted_index::validate_common_grams_logical_term(components.substr(0, left_length), + "left term") + .ok() && + inverted_index::validate_common_grams_logical_term(components.substr(left_length), + "right term") + .ok(); +} + +template +Status reserve_tracked_vector(std::vector* values, size_t additional, + writer::MemoryReporter::Reservation* reservation) { + if (reservation == nullptr || additional == 0) return Status::OK(); + if (additional > std::numeric_limits::max() - values->size()) { + return Status::Error( + "snii_compaction: destination posting vector size overflows"); + } + const size_t required = values->size() + additional; + if (required <= values->capacity()) { + DCHECK_EQ(reservation->bytes(), values->capacity() * sizeof(T)); + return Status::OK(); + } + size_t target = std::max(64, required); + if (values->capacity() != 0 && values->capacity() <= std::numeric_limits::max() / 2) { + target = std::max(target, values->capacity() * 2); + } + if (target > static_cast(std::numeric_limits::max()) / sizeof(T)) { + return Status::Error( + "snii_compaction: destination posting reservation exceeds int64"); + } + writer::MemoryReporter::Reservation replacement; + RETURN_IF_ERROR(reservation->prepare_replacement(target * sizeof(T), &replacement)); + values->reserve(target); + DCHECK_EQ(values->capacity(), target); + *reservation = std::move(replacement); + return Status::OK(); +} +} // namespace + +SniiPlainT2MergePlan::SniiPlainT2MergePlan( + std::vector source_indexes, + const ValidatedRowIdConversion* rowid_conversion, + std::vector destination_segment_num_rows, + std::vector destination_null_reservations, + std::vector> destination_null_docids, + SniiCompactionEligibility eligibility, + std::vector destination_norm_reservations, + std::vector> destination_encoded_norms, + std::shared_ptr memory_reporter, + std::vector> read_contexts) + : source_indexes_(std::move(source_indexes)), + rowid_conversion_(rowid_conversion), + destination_segment_num_rows_(std::move(destination_segment_num_rows)), + memory_reporter_(std::move(memory_reporter)), + destination_null_reservations_(std::move(destination_null_reservations)), + destination_null_docids_(std::move(destination_null_docids)), + destination_null_docids_taken_(destination_null_docids_.size(), false), + eligibility_(std::move(eligibility)), + destination_norm_reservations_(std::move(destination_norm_reservations)), + destination_encoded_norms_(std::move(destination_encoded_norms)), + destination_encoded_norms_taken_(destination_encoded_norms_.size(), false), + destination_semantic_token_counts_(destination_segment_num_rows_.size(), 0), + read_contexts_(std::move(read_contexts)) {} + +Status SniiPlainT2MergePlan::prepare(std::vector source_indexes, + const ValidatedRowIdConversion& rowid_conversion, + size_t total_read_ahead_budget_bytes, + std::unique_ptr* out) { + return prepare(std::move(source_indexes), rowid_conversion, total_read_ahead_budget_bytes, + nullptr, out); +} + +Status SniiPlainT2MergePlan::prepare(std::vector source_indexes, + const ValidatedRowIdConversion& rowid_conversion, + size_t total_read_ahead_budget_bytes, + std::shared_ptr memory_reporter, + std::unique_ptr* out) { + SniiCompactionEligibility eligibility; + eligibility.kind = SniiStreamedMergeKind::kPlainT2; + return prepare(std::move(source_indexes), rowid_conversion, eligibility, + total_read_ahead_budget_bytes, std::move(memory_reporter), out); +} + +Status SniiPlainT2MergePlan::prepare(std::vector source_indexes, + const ValidatedRowIdConversion& rowid_conversion, + const SniiCompactionEligibility& eligibility, + size_t total_read_ahead_budget_bytes, + std::unique_ptr* out) { + return prepare(std::move(source_indexes), rowid_conversion, eligibility, + total_read_ahead_budget_bytes, nullptr, out); +} + +Status SniiPlainT2MergePlan::prepare(std::vector source_indexes, + const ValidatedRowIdConversion& rowid_conversion, + const SniiCompactionEligibility& eligibility, + size_t total_read_ahead_budget_bytes, + std::shared_ptr memory_reporter, + std::unique_ptr* out) { + if (out == nullptr) { + return invalid_plan("null plan out parameter"); + } + out->reset(); + if (source_indexes.empty()) { + return invalid_plan("no source indexes"); + } + const std::vector& destination_segment_num_rows = + rowid_conversion.destination_segment_doc_counts(); + if (destination_segment_num_rows.empty()) { + return invalid_plan("no destination segments"); + } + if (source_indexes.size() != rowid_conversion.source_segment_count()) { + return invalid_plan("source index count differs from row-id conversion"); + } + // Division first avoids source_count * minimum overflow. Rejecting tiny + // allocations is an IO gate: the raw rebuild is cheaper than issuing a + // range request for every small posting window. + if (source_indexes.size() > total_read_ahead_budget_bytes / kMinReadAheadBudgetPerSource) { + return invalid_plan("read-ahead budget is below the per-source IO floor"); + } + const size_t per_source_read_ahead_budget = + std::min(SniiPostingReadContext::kMaxReadAheadBudgetBytes, + total_read_ahead_budget_bytes / source_indexes.size()); + DORIS_CHECK_GE(per_source_read_ahead_budget, kMinReadAheadBudgetPerSource); + DORIS_CHECK_LE(per_source_read_ahead_budget, + total_read_ahead_budget_bytes / source_indexes.size()); + + for (size_t source_ordinal = 0; source_ordinal < source_indexes.size(); ++source_ordinal) { + const reader::LogicalIndexReader* source = source_indexes[source_ordinal]; + if (source == nullptr) { + return invalid_plan("null source index"); + } + RETURN_IF_ERROR(validate_snii_source_eligibility(*source, source_ordinal, eligibility)); + if (source->stats().doc_count > std::numeric_limits::max()) { + return merge_corruption("source doc count exceeds the SNII uint32 docid domain"); + } + if (source->stats().doc_count != + rowid_conversion.source_segment_doc_counts()[source_ordinal]) { + return invalid_plan("source doc count differs from validated row-id conversion"); + } + } + + std::vector destination_null_reservations; + destination_null_reservations.reserve(destination_segment_num_rows.size()); + for (size_t destination_ordinal = 0; destination_ordinal < destination_segment_num_rows.size(); + ++destination_ordinal) { + destination_null_reservations.push_back(memory_reporter == nullptr + ? writer::MemoryReporter::Reservation() + : memory_reporter->make_reservation()); + } + std::vector> destination_null_docids(destination_segment_num_rows.size()); + for (size_t source_ordinal = 0; source_ordinal < source_indexes.size(); ++source_ordinal) { + reader::NullDocidsScanMemory scan_memory; + RETURN_IF_ERROR(source_indexes[source_ordinal]->null_docids_scan_memory(&scan_memory)); + writer::MemoryReporter::Reservation source_output_reservation = + memory_reporter == nullptr ? writer::MemoryReporter::Reservation() + : memory_reporter->make_reservation(); + writer::MemoryReporter::Reservation source_frame_reservation = + memory_reporter == nullptr ? writer::MemoryReporter::Reservation() + : memory_reporter->make_reservation(); + writer::MemoryReporter::Reservation source_decode_reservation = + memory_reporter == nullptr ? writer::MemoryReporter::Reservation() + : memory_reporter->make_reservation(); + if (memory_reporter != nullptr) { + RETURN_IF_ERROR(source_output_reservation.set_bytes(scan_memory.output_bytes)); + RETURN_IF_ERROR(source_frame_reservation.set_bytes(scan_memory.frame_bytes)); + } + std::vector source_null_docids; + source_null_docids.reserve(source_indexes[source_ordinal]->stats().null_count); + if (memory_reporter != nullptr) { + DORIS_CHECK_EQ(source_null_docids.capacity() * sizeof(uint32_t), + source_output_reservation.bytes()); + } + RETURN_IF_ERROR(source_indexes[source_ordinal]->read_null_docids( + &source_null_docids, [&](uint64_t bytes) { + return memory_reporter == nullptr ? Status::OK() + : source_decode_reservation.set_bytes(bytes); + })); + source_frame_reservation.reset(); + source_decode_reservation.reset(); + const auto source_mapping = rowid_conversion.source_mapping(source_ordinal); + for (uint32_t source_docid : source_null_docids) { + DCHECK_LT(source_docid, source_mapping.size()); + const auto [destination_segment, destination_docid] = source_mapping[source_docid]; + const bool segment_deleted = + destination_segment == std::numeric_limits::max(); + const bool doc_deleted = destination_docid == std::numeric_limits::max(); + DCHECK_EQ(segment_deleted, doc_deleted); + if (!segment_deleted) { + DCHECK_LT(destination_segment, destination_null_docids.size()); + DCHECK_LT(destination_docid, destination_segment_num_rows[destination_segment]); + RETURN_IF_ERROR(reserve_tracked_vector( + &destination_null_docids[destination_segment], 1, + memory_reporter == nullptr + ? nullptr + : &destination_null_reservations[destination_segment])); + destination_null_docids[destination_segment].push_back(destination_docid); + } + } + } + for (auto& null_docids : destination_null_docids) { + std::ranges::sort(null_docids); + DCHECK(std::adjacent_find(null_docids.begin(), null_docids.end()) == null_docids.end()); + } + + std::vector destination_norm_reservations; + std::vector> destination_encoded_norms; + if (eligibility.kind == SniiStreamedMergeKind::kCommonGramsT3) { + DORIS_CHECK(eligibility.common_grams_metadata_seed.has_value()); + destination_norm_reservations.reserve(destination_segment_num_rows.size()); + destination_encoded_norms.resize(destination_segment_num_rows.size()); + for (size_t destination_ordinal = 0; + destination_ordinal < destination_segment_num_rows.size(); ++destination_ordinal) { + destination_norm_reservations.push_back(memory_reporter == nullptr + ? writer::MemoryReporter::Reservation() + : memory_reporter->make_reservation()); + const uint32_t doc_count = destination_segment_num_rows[destination_ordinal]; + auto& norms = destination_encoded_norms[destination_ordinal]; + RETURN_IF_ERROR(reserve_tracked_vector( + &norms, doc_count, + memory_reporter == nullptr ? nullptr : &destination_norm_reservations.back())); + norms.resize(doc_count); + if (memory_reporter != nullptr) { + DORIS_CHECK_EQ(destination_norm_reservations.back().bytes(), norms.capacity()); + } + } + for (size_t source_ordinal = 0; source_ordinal < source_indexes.size(); ++source_ordinal) { + writer::MemoryReporter::Reservation source_norms_reservation = + memory_reporter == nullptr ? writer::MemoryReporter::Reservation() + : memory_reporter->make_reservation(); + if (memory_reporter != nullptr) { + RETURN_IF_ERROR(source_norms_reservation.set_bytes( + source_indexes[source_ordinal]->compaction_norms_cache_charge())); + } + format::NormsPodReader source_norms; + RETURN_IF_ERROR(source_indexes[source_ordinal]->open_norms(&source_norms)); + const auto source_mapping = rowid_conversion.source_mapping(source_ordinal); + // The norms POD is only CRC-self-consistent; nothing upstream ties its + // doc_count to the validated conversion, and the loop below indexes + // source_mapping by it. Reconcile loudly (once per source, cold path). + if (source_norms.doc_count() != source_mapping.size()) { + return merge_corruption("norms doc count differs from validated row-id conversion"); + } + for (uint32_t source_docid = 0; source_docid < source_norms.doc_count(); + ++source_docid) { + const auto [destination_segment, destination_docid] = source_mapping[source_docid]; + const bool deleted = destination_segment == std::numeric_limits::max(); + DCHECK_EQ(deleted, destination_docid == std::numeric_limits::max()); + if (deleted) { + continue; + } + DCHECK_LT(destination_segment, destination_encoded_norms.size()); + DCHECK_LT(destination_docid, destination_encoded_norms[destination_segment].size()); + destination_encoded_norms[destination_segment][destination_docid] = + source_norms.encoded_norm(source_docid); + } + source_indexes[source_ordinal]->release_compaction_norms(); + source_norms_reservation.reset(); + } + } + + std::vector> read_contexts; + read_contexts.reserve(source_indexes.size()); + for (const reader::LogicalIndexReader* source : source_indexes) { + auto context = std::make_unique( + source, per_source_read_ahead_budget, memory_reporter.get()); + RETURN_IF_ERROR(context->init()); + read_contexts.push_back(std::move(context)); + } + + out->reset(new SniiPlainT2MergePlan( + std::move(source_indexes), &rowid_conversion, destination_segment_num_rows, + std::move(destination_null_reservations), std::move(destination_null_docids), + eligibility, std::move(destination_norm_reservations), + std::move(destination_encoded_norms), std::move(memory_reporter), + std::move(read_contexts))); + return Status::OK(); +} + +const std::vector& SniiPlainT2MergePlan::destination_null_docids( + size_t destination_segment) const { + DORIS_CHECK_LT(destination_segment, destination_null_docids_.size()); + return destination_null_docids_[destination_segment]; +} + +writer::TrackedNullDocids SniiPlainT2MergePlan::take_destination_null_docids( + size_t destination_segment) { + DORIS_CHECK_LT(destination_segment, destination_null_docids_.size()); + DORIS_CHECK(!destination_null_docids_taken_[destination_segment]); + destination_null_docids_taken_[destination_segment] = true; + return writer::TrackedNullDocids(std::move(destination_null_reservations_[destination_segment]), + std::move(destination_null_docids_[destination_segment])); +} + +const std::vector& SniiPlainT2MergePlan::destination_encoded_norms( + size_t destination_segment) const { + DORIS_CHECK(eligibility_.kind == SniiStreamedMergeKind::kCommonGramsT3); + DORIS_CHECK_LT(destination_segment, destination_encoded_norms_.size()); + return destination_encoded_norms_[destination_segment]; +} + +writer::TrackedEncodedNorms SniiPlainT2MergePlan::take_destination_encoded_norms( + size_t destination_segment) { + DORIS_CHECK(eligibility_.kind == SniiStreamedMergeKind::kCommonGramsT3); + DORIS_CHECK_LT(destination_segment, destination_encoded_norms_.size()); + DORIS_CHECK(!destination_encoded_norms_taken_[destination_segment]); + destination_encoded_norms_taken_[destination_segment] = true; + return writer::TrackedEncodedNorms( + std::move(destination_norm_reservations_[destination_segment]), + std::move(destination_encoded_norms_[destination_segment])); +} + +format::IndexConfig SniiPlainT2MergePlan::destination_index_config() const { + return eligibility_.kind == SniiStreamedMergeKind::kCommonGramsT3 + ? format::IndexConfig::kDocsPositionsScoring + : format::IndexConfig::kDocsPositions; +} + +std::optional +SniiPlainT2MergePlan::destination_common_grams_metadata(size_t destination_segment) const { + if (eligibility_.kind == SniiStreamedMergeKind::kPlainT2) { + return std::nullopt; + } + DORIS_CHECK(eligibility_.common_grams_metadata_seed.has_value()); + DORIS_CHECK_LT(destination_segment, destination_segment_num_rows_.size()); + auto metadata = *eligibility_.common_grams_metadata_seed; + metadata.scoring_doc_count = destination_segment_num_rows_[destination_segment]; + metadata.scoring_token_count = 0; + return metadata; +} + +Status SniiPlainT2MergePlan::poison(Status status) { + DORIS_CHECK(!status.ok()); + if (failed_.ok()) { + failed_ = std::move(status); + } + return failed_; +} + +Status SniiPlainT2MergePlan::take_front_source(TermMergeFrontier* frontier, CurrentTerm* current) { + SniiSegmentTermCursor* source = frontier->front(); + const uint32_t source_ordinal = source->source_ordinal(); + DCHECK_LT(source_ordinal, source_indexes_.size()); + const uint64_t frq_base = source->frq_base(); + const uint64_t prx_base = source->prx_base(); + format::DictEntry entry = source->take_entry(); + if (current->posting_cursors.empty()) { + current->term = std::move(entry.term); + } + + const bool source_has_positions = posting_entry_has_positions(entry); + if (!current->common_gram && !source_has_positions) { + return merge_corruption("ordinary term has docs-only posting shape"); + } + if (!current->has_positions.has_value()) { + current->has_positions = source_has_positions; + } else if (*current->has_positions != source_has_positions) { + return merge_corruption("same term has inconsistent position shape across sources"); + } + + auto cursor = std::make_unique(read_contexts_[source_ordinal].get(), + std::move(entry), frq_base, prx_base, + source_ordinal, rowid_conversion_); + RETURN_IF_ERROR(cursor->init()); + current->posting_cursors.push_back(std::move(cursor)); + return frontier->advance_front(); +} + +Status SniiPlainT2MergePlan::take_current_term(TermMergeFrontier* frontier, CurrentTerm* current) { + DCHECK(frontier != nullptr); + DCHECK(current != nullptr); + DCHECK(!frontier->empty()); + const std::string_view group_term = frontier->front()->term(); + current->posting_cursors.reserve(source_indexes_.size()); + if (eligibility_.kind == SniiStreamedMergeKind::kCommonGramsT3) { + current->common_gram = segment_v2::inverted_index::is_internal_term_key(group_term); + if (current->common_gram && !is_well_formed_common_gram(group_term)) { + return merge_corruption("CommonGrams source contains an unknown internal term marker"); + } + current->counts_as_semantic_token = !current->common_gram; + } + + do { + RETURN_IF_ERROR(take_front_source(frontier, current)); + } while (!frontier->empty() && frontier->front()->term() == current->term); + DCHECK(current->has_positions.has_value()); + return Status::OK(); +} + +Status SniiPlainT2MergePlan::write_current_term( + CurrentTerm current, std::span sessions) { + MergedPostingRuns posting_source(std::move(current.posting_cursors), *current.has_positions, + current.counts_as_semantic_token, + destination_segment_num_rows_, + destination_semantic_token_counts_); + RETURN_IF_ERROR(posting_source.init()); + while (!posting_source.empty()) { + const uint32_t destination = posting_source.next_destination(); + DCHECK_LT(destination, sessions.size()); + RETURN_IF_ERROR(posting_source.begin_destination(destination)); + writer::StreamedTermPostings postings {.term = current.term, + .retain_positions = *current.has_positions, + .source = &posting_source}; + RETURN_IF_ERROR(sessions[destination]->push_term(std::move(postings))); + } + return Status::OK(); +} + +Status SniiPlainT2MergePlan::merge_terms( + std::span sessions) { + std::vector> term_cursors; + std::vector term_cursor_ptrs; + term_cursors.reserve(source_indexes_.size()); + term_cursor_ptrs.reserve(source_indexes_.size()); + for (size_t source_ordinal = 0; source_ordinal < source_indexes_.size(); ++source_ordinal) { + term_cursors.push_back(std::make_unique( + source_indexes_[source_ordinal], static_cast(source_ordinal), + memory_reporter_.get())); + term_cursor_ptrs.push_back(term_cursors.back().get()); + } + TermMergeFrontier term_frontier; + RETURN_IF_ERROR(term_frontier.init(std::move(term_cursor_ptrs))); + + while (!term_frontier.empty()) { + CurrentTerm current; + RETURN_IF_ERROR(take_current_term(&term_frontier, ¤t)); + RETURN_IF_ERROR(write_current_term(std::move(current), sessions)); + } + + if (eligibility_.kind == SniiStreamedMergeKind::kCommonGramsT3) { + for (size_t destination_ordinal = 0; destination_ordinal < sessions.size(); + ++destination_ordinal) { + RETURN_IF_ERROR(sessions[destination_ordinal]->set_semantic_token_count( + destination_semantic_token_counts_[destination_ordinal])); + } + } + for (writer::SniiStreamedIndexSession* session : sessions) { + RETURN_IF_ERROR(session->finish()); + } + return Status::OK(); +} + +Status SniiPlainT2MergePlan::execute(std::span sessions) { + const auto abort_sessions = [&sessions](const Status& cause) { + DCHECK(!cause.ok()); + for (writer::SniiStreamedIndexSession* session : sessions) { + if (session != nullptr) { + session->abort(cause); + } + } + }; + if (!failed_.ok()) { + abort_sessions(failed_); + return failed_; + } + if (executed_) { + const Status status = invalid_plan("merge plan executed twice"); + abort_sessions(status); + return poison(status); + } + if (sessions.size() != destination_segment_num_rows_.size()) { + const Status status = invalid_plan("destination session count mismatch"); + abort_sessions(status); + return poison(status); + } + for (writer::SniiStreamedIndexSession* session : sessions) { + if (session == nullptr) { + const Status status = invalid_plan("null destination session"); + abort_sessions(status); + return poison(status); + } + } + executed_ = true; + const Status status = merge_terms(sessions); + if (!status.ok()) { + abort_sessions(status); + return poison(status); + } + return Status::OK(); +} + +} // namespace doris::snii::compaction diff --git a/be/src/storage/index/snii/compaction/snii_index_compaction.h b/be/src/storage/index/snii/compaction/snii_index_compaction.h new file mode 100644 index 00000000000000..e012ae4e294111 --- /dev/null +++ b/be/src/storage/index/snii/compaction/snii_index_compaction.h @@ -0,0 +1,146 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/compaction/eligibility.h" +#include "storage/index/snii/compaction/posting_cursor.h" +#include "storage/index/snii/compaction/rowid_conversion.h" +#include "storage/index/snii/reader/logical_index_reader.h" +#include "storage/index/snii/writer/memory_reporter.h" +#include "storage/index/snii/writer/snii_compound_writer.h" + +namespace doris::snii::compaction { + +class TermMergeFrontier; + +// Prepared, one-shot merge of one plain positions-only logical index across +// source segments. prepare() performs every O(1)-metadata, row-id and NULL +// preflight before the caller creates destination streamed sessions. execute() +// then performs exactly one dictionary/posting pass and seals every session. +// +// Source readers and the validated row-id conversion token are borrowed and +// must remain stable through execute(). The plan owns the per-source read +// contexts, destination row counts and remapped NULL docids. Its aggregate +// read-ahead allocation never exceeds the explicit prepare() budget. +class SniiPlainT2MergePlan { +public: + // Below this aggregate-per-source budget, two posting streams would issue + // tiny range reads and turn a memory fallback into an IO regression. + static constexpr size_t kMinReadAheadBudgetPerSource = 64U << 10; + + SniiPlainT2MergePlan(const SniiPlainT2MergePlan&) = delete; + SniiPlainT2MergePlan& operator=(const SniiPlainT2MergePlan&) = delete; + SniiPlainT2MergePlan(SniiPlainT2MergePlan&&) = delete; + SniiPlainT2MergePlan& operator=(SniiPlainT2MergePlan&&) = delete; + + static Status prepare(std::vector source_indexes, + const ValidatedRowIdConversion& rowid_conversion, + size_t total_read_ahead_budget_bytes, + std::unique_ptr* out); + static Status prepare(std::vector source_indexes, + const ValidatedRowIdConversion& rowid_conversion, + size_t total_read_ahead_budget_bytes, + std::shared_ptr memory_reporter, + std::unique_ptr* out); + static Status prepare(std::vector source_indexes, + const ValidatedRowIdConversion& rowid_conversion, + const SniiCompactionEligibility& eligibility, + size_t total_read_ahead_budget_bytes, + std::unique_ptr* out); + static Status prepare(std::vector source_indexes, + const ValidatedRowIdConversion& rowid_conversion, + const SniiCompactionEligibility& eligibility, + size_t total_read_ahead_budget_bytes, + std::shared_ptr memory_reporter, + std::unique_ptr* out); + + const std::vector& destination_null_docids(size_t destination_segment) const; + writer::TrackedNullDocids take_destination_null_docids(size_t destination_segment); + const std::vector& destination_encoded_norms(size_t destination_segment) const; + writer::TrackedEncodedNorms take_destination_encoded_norms(size_t destination_segment); + format::IndexConfig destination_index_config() const; + std::optional + destination_common_grams_metadata(size_t destination_segment) const; + format::CommonGramsPostingPolicy destination_common_grams_posting_policy() const { + return eligibility_.common_grams_posting_policy; + } + size_t destination_segment_count() const { return destination_segment_num_rows_.size(); } + + // Sessions must correspond one-for-one with destination segments and must + // already carry the doc_count and destination_null_docids() returned by this + // plan. Any failure is terminal: the first error is sticky, sessions remain + // unsealable, and the whole unpublished compaction output must be discarded. + Status execute(std::span sessions); + +private: + struct CurrentTerm { + std::string term; + std::vector> posting_cursors; + std::optional has_positions; + bool common_gram = false; + bool counts_as_semantic_token = false; + }; + + SniiPlainT2MergePlan( + std::vector source_indexes, + const ValidatedRowIdConversion* rowid_conversion, + std::vector destination_segment_num_rows, + std::vector destination_null_reservations, + std::vector> destination_null_docids, + SniiCompactionEligibility eligibility, + std::vector destination_norm_reservations, + std::vector> destination_encoded_norms, + std::shared_ptr memory_reporter, + std::vector> read_contexts); + + Status take_front_source(TermMergeFrontier* frontier, CurrentTerm* current); + Status take_current_term(TermMergeFrontier* frontier, CurrentTerm* current); + Status write_current_term(CurrentTerm current, + std::span sessions); + Status merge_terms(std::span sessions); + Status poison(Status status); + + std::vector source_indexes_; + const ValidatedRowIdConversion* rowid_conversion_ = nullptr; + std::vector destination_segment_num_rows_; + std::shared_ptr memory_reporter_; + // Reservations precede their vectors so physical memory is destroyed first. + std::vector destination_null_reservations_; + std::vector> destination_null_docids_; + std::vector destination_null_docids_taken_; + SniiCompactionEligibility eligibility_; + // Reservations precede their vectors so physical memory is destroyed first. + std::vector destination_norm_reservations_; + std::vector> destination_encoded_norms_; + std::vector destination_encoded_norms_taken_; + std::vector destination_semantic_token_counts_; + std::vector> read_contexts_; + bool executed_ = false; + Status failed_ = Status::OK(); +}; + +} // namespace doris::snii::compaction diff --git a/be/src/storage/index/snii/compaction/term_cursor.cpp b/be/src/storage/index/snii/compaction/term_cursor.cpp new file mode 100644 index 00000000000000..1c25f1b57ff06a --- /dev/null +++ b/be/src/storage/index/snii/compaction/term_cursor.cpp @@ -0,0 +1,164 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/compaction/term_cursor.h" + +#include + +#include "storage/index/snii/format/phrase_bigram.h" +#include "storage/index/snii/format/sampled_term_index.h" + +namespace doris::snii::compaction { + +namespace { + +Status add_entry_memory(uint64_t bytes, uint64_t* total) { + if (bytes > std::numeric_limits::max() - *total) { + return Status::Error( + "term_cursor: decoded dictionary entries exceed uint64 memory accounting"); + } + *total += bytes; + return Status::OK(); +} + +Status decoded_entries_memory_bytes(const std::vector& entries, uint64_t* out) { + if (entries.capacity() > std::numeric_limits::max() / sizeof(format::DictEntry)) { + return Status::Error( + "term_cursor: decoded dictionary entry slots exceed uint64 memory accounting"); + } + uint64_t bytes = entries.capacity() * sizeof(format::DictEntry); + for (const format::DictEntry& entry : entries) { + RETURN_IF_ERROR(add_entry_memory(format::std_string_heap_bytes(entry.term), &bytes)); + RETURN_IF_ERROR(add_entry_memory(entry.frq_bytes.capacity(), &bytes)); + RETURN_IF_ERROR(add_entry_memory(entry.prx_bytes.capacity(), &bytes)); + } + *out = bytes; + return Status::OK(); +} + +} // namespace + +uint64_t big_endian_term_prefix(std::string_view term) { + uint64_t prefix = 0; + for (size_t i = 0; i < term.size() && i < sizeof(prefix); ++i) { + prefix |= static_cast(static_cast(term[i])) << (56 - i * 8); + } + return prefix; +} + +Status SniiSegmentTermCursor::next(bool* has_term) { + if (has_term == nullptr) { + return Status::Error("term_cursor: null has_term"); + } + *has_term = false; + if (!failed_.ok()) { + return failed_; + } + if (index_ == nullptr) { + return Status::Error("term_cursor: null index"); + } + if (exhausted_) { + return Status::OK(); + } + + if (started_) { + ++pos_; + } else { + started_ = true; + } + // Cross a block boundary (or start): materialize the next DICT block. The + // loop form also tolerates a (format-legal but writer-never-produced) + // empty block. + while (pos_ >= entries_.size()) { + if (next_block_ >= index_->n_dict_blocks()) { + exhausted_ = true; + entries_.clear(); + return Status::OK(); + } + + writer::MemoryReporter::Reservation decode_reservation = + memory_reporter_ == nullptr ? writer::MemoryReporter::Reservation() + : memory_reporter_->make_reservation(); + if (memory_reporter_ != nullptr) { + reader::DictBlockScanMemory memory; + Status st = index_->dict_block_scan_memory(next_block_, &memory); + if (!st.ok()) { + failed_ = st; + return failed_; + } + st = decode_reservation.set_bytes(memory.decode_bytes); + if (!st.ok()) { + failed_ = st; + return failed_; + } + + previous_entries_reservation_.reset(); + previous_entries_reservation_ = std::move(entries_reservation_); + entries_reservation_ = memory_reporter_->make_reservation(); + st = entries_reservation_.set_bytes(memory.entries_bytes); + if (!st.ok()) { + failed_ = st; + return failed_; + } + } + const Status st = index_->decode_dict_block(next_block_, &entries_, &frq_base_, &prx_base_); + if (!st.ok()) { + failed_ = st; + return failed_; + } + if (memory_reporter_ != nullptr) { + uint64_t actual_entries_bytes = 0; + Status memory_status = decoded_entries_memory_bytes(entries_, &actual_entries_bytes); + if (memory_status.ok()) { + memory_status = entries_reservation_.set_bytes(actual_entries_bytes); + } + if (!memory_status.ok()) { + std::vector().swap(entries_); + entries_reservation_.reset(); + failed_ = memory_status; + return failed_; + } + } + ++next_block_; + pos_ = 0; + } + + const format::DictEntry& e = entries_[pos_]; + // Hidden bigram / sentinel gate: classify by the FULL marker so a user term + // that merely begins with a raw 0x1F byte passes through (design ruling on + // the 0x1F-prefix corner). Any full-marker term aborts this column's merge + // -- the caller must fall back to rebuild (v1 excludes legacy bigrams). + if (format::is_phrase_bigram_term(e.term)) { + failed_ = Status::Error( + "term_cursor: source dictionary contains a legacy phrase-bigram/sentinel term; " + "column must fall back to index rebuild (src_ord={})", + source_ordinal_); + return failed_; + } + if (has_prev_ && e.term <= prev_term_) { + failed_ = Status::Error( + "term_cursor: dictionary term order violated (src_ord={})", source_ordinal_); + return failed_; + } + prev_term_ = e.term; + term_prefix_ = big_endian_term_prefix(e.term); + has_prev_ = true; + *has_term = true; + return Status::OK(); +} + +} // namespace doris::snii::compaction diff --git a/be/src/storage/index/snii/compaction/term_cursor.h b/be/src/storage/index/snii/compaction/term_cursor.h new file mode 100644 index 00000000000000..2c73e8442c7f17 --- /dev/null +++ b/be/src/storage/index/snii/compaction/term_cursor.h @@ -0,0 +1,124 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/format/dict_entry.h" +#include "storage/index/snii/reader/logical_index_reader.h" +#include "storage/index/snii/writer/memory_reporter.h" + +// SniiSegmentTermCursor -- pull-model full-dictionary scan over ONE source +// segment's logical index (T2.3, compaction index-merge fast path). +// +// The cursor walks the source's DICT blocks in ordinal order and yields every +// DictEntry in lexicographic term order, one block resident at a time (never +// the whole vocabulary). Entries are passed through UNINTERPRETED: the locator +// (inline bytes / slim pod_ref / windowed pod_ref) and the per-block +// kNoTermStats flag (DictEntry::term_stats_present) reach the downstream +// decoder exactly as the reader produced them -- the merge pump (T2.4) decides +// how to decode, and kNoTermStats inputs must have their ttf/max_freq recomputed +// from the actual freq stream (correctness invariant 2), so this layer must not +// synthesize stats. +// +// Hidden-term gate (base-drift addendum ruling): the v1 merge fast path does +// NOT merge legacy phrase-bigram postings. Any dictionary term carrying the +// FULL 0x1F bigram marker (hidden bigram pair or the bare-marker sentinel, +// classified by format::is_phrase_bigram_term -- NOT by a raw leading 0x1F +// byte, which a legitimate user term may begin with) makes next() return +// INVERTED_INDEX_NOT_SUPPORTED so the caller aborts THIS column's merge and +// falls back to a full rebuild. The error is deliberately raised from next() +// (not swallowed by skipping) because silently dropping hidden postings would +// change phrase semantics on the merged output. +namespace doris::snii::compaction { + +uint64_t big_endian_term_prefix(std::string_view term); + +class SniiSegmentTermCursor { +public: + // `index` is borrowed and must outlive the cursor. `source_ordinal` is the + // caller's stable id for this source segment (frontier tie-break + docid + // remapping key downstream). + SniiSegmentTermCursor(const reader::LogicalIndexReader* index, uint32_t source_ordinal, + writer::MemoryReporter* memory_reporter = nullptr) + : index_(index), + source_ordinal_(source_ordinal), + memory_reporter_(memory_reporter), + previous_entries_reservation_(memory_reporter == nullptr + ? writer::MemoryReporter::Reservation() + : memory_reporter->make_reservation()), + entries_reservation_(memory_reporter == nullptr + ? writer::MemoryReporter::Reservation() + : memory_reporter->make_reservation()) {} + + // Advances to the next dictionary term. *has_term=false once the + // dictionary is exhausted (an empty index yields it on the first call). + // Errors: + // INVERTED_INDEX_NOT_SUPPORTED -- hidden bigram/sentinel term (see above); + // Corruption -- the dictionary violated strict lexicographic order. + // After an error the cursor is poisoned and keeps returning the error. + Status next(bool* has_term); + + // Accessors for the CURRENT term; valid only after next() returned + // *has_term=true and, for term()/entry(), before take_entry(). + const std::string& term() const { return entries_[pos_].term; } + uint64_t term_prefix() const { return term_prefix_; } + const format::DictEntry& entry() const { return entries_[pos_]; } + // Moves the current entry out (term + locator + any inline posting bytes). + // The cursor stays positioned, but term()/entry() must not be used again + // until the next next() call. + format::DictEntry take_entry() { return std::move(entries_[pos_]); } + + // frq/prx bases of the DICT block owning the current entry -- required to + // resolve pod_ref locators against the source's posting region. + uint64_t frq_base() const { return frq_base_; } + uint64_t prx_base() const { return prx_base_; } + uint32_t source_ordinal() const { return source_ordinal_; } + +private: + const reader::LogicalIndexReader* index_ = nullptr; + uint32_t source_ordinal_ = 0; + writer::MemoryReporter* memory_reporter_ = nullptr; + // Keep the preceding block's charge for one additional block transition. + // The frontier advances a source before the current term's posting cursor + // has released the DictEntry moved out of that block. + writer::MemoryReporter::Reservation previous_entries_reservation_; + writer::MemoryReporter::Reservation entries_reservation_; + + uint32_t next_block_ = 0; // next DICT block ordinal to decode + std::vector entries_; // current block, materialized + size_t pos_ = 0; // current entry within entries_ + uint64_t frq_base_ = 0; + uint64_t prx_base_ = 0; + uint64_t term_prefix_ = 0; + + bool started_ = false; // first next() must not pre-increment pos_ + bool exhausted_ = false; + Status failed_ = Status::OK(); // sticky error (poisoned cursor) + // Strict-order guard across block boundaries: DICT blocks and the merge + // frontier both assume a strictly increasing term sequence; a violation + // means a corrupt dictionary and must fail the merge, not scramble output. + std::string prev_term_; + bool has_prev_ = false; +}; + +} // namespace doris::snii::compaction diff --git a/be/src/storage/index/snii/compaction/term_merge_frontier.cpp b/be/src/storage/index/snii/compaction/term_merge_frontier.cpp new file mode 100644 index 00000000000000..8c5bd6ebc8c539 --- /dev/null +++ b/be/src/storage/index/snii/compaction/term_merge_frontier.cpp @@ -0,0 +1,103 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/compaction/term_merge_frontier.h" + +#include + +#include "common/check.h" + +namespace doris::snii::compaction { + +bool TermMergeFrontier::Before::operator()(size_t lhs, size_t rhs) const { + const uint64_t lhs_prefix = (*cursors)[lhs]->term_prefix(); + const uint64_t rhs_prefix = (*cursors)[rhs]->term_prefix(); + if (lhs_prefix != rhs_prefix) { + return lhs_prefix < rhs_prefix; + } + const std::string& lhs_term = (*cursors)[lhs]->term(); + const std::string& rhs_term = (*cursors)[rhs]->term(); + if (lhs_term != rhs_term) { + return lhs_term < rhs_term; + } + return (*cursors)[lhs]->source_ordinal() < (*cursors)[rhs]->source_ordinal(); +} + +Status TermMergeFrontier::init(std::vector cursors) { + if (initialized_) { + return Status::Error( + "term_merge_frontier: init called twice"); + } + initialized_ = true; + cursors_ = std::move(cursors); + std::vector live(cursors_.size(), 0); + for (size_t source = 0; source < cursors_.size(); ++source) { + SniiSegmentTermCursor* cursor = cursors_[source]; + if (cursor == nullptr) { + failed_ = Status::Error( + "term_merge_frontier: null cursor"); + return failed_; + } + bool has_term = false; + const Status status = cursor->next(&has_term); + if (!status.ok()) { + failed_ = status; + return failed_; + } + live[source] = has_term; + } + frontier_.build(cursors_.size(), [&live](size_t source) { return live[source] != 0; }); + return Status::OK(); +} + +bool TermMergeFrontier::empty() const { + DCHECK(initialized_); + DCHECK(failed_.ok()); + return frontier_.empty(); +} + +SniiSegmentTermCursor* TermMergeFrontier::front() const { + DCHECK(initialized_); + DCHECK(failed_.ok()); + return cursors_[frontier_.winner()]; +} + +Status TermMergeFrontier::advance_front() { + if (!initialized_) { + return Status::Error( + "term_merge_frontier: advance before init"); + } + if (!failed_.ok()) { + return failed_; + } + if (frontier_.empty()) { + return Status::Error( + "term_merge_frontier: advance on empty frontier"); + } + + const size_t source = frontier_.winner(); + bool has_term = false; + const Status status = cursors_[source]->next(&has_term); + if (!status.ok()) { + failed_ = status; + return failed_; + } + frontier_.update(source, has_term); + return Status::OK(); +} + +} // namespace doris::snii::compaction diff --git a/be/src/storage/index/snii/compaction/term_merge_frontier.h b/be/src/storage/index/snii/compaction/term_merge_frontier.h new file mode 100644 index 00000000000000..8bb36a3a849dca --- /dev/null +++ b/be/src/storage/index/snii/compaction/term_merge_frontier.h @@ -0,0 +1,56 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include "common/status.h" +#include "storage/index/snii/compaction/indexed_winner_tree.h" +#include "storage/index/snii/compaction/term_cursor.h" + +namespace doris::snii::compaction { + +// K-way term frontier. Each source cursor is a dense winner-tree leaf. +// The caller consumes front()->entry() before advance_front(); advancing a +// source updates one leaf-to-root path and never materializes an intermediate +// merged-term group. +class TermMergeFrontier { + struct Before { + const std::vector* cursors = nullptr; + + bool operator()(size_t lhs, size_t rhs) const; + }; + +public: + TermMergeFrontier() : frontier_(Before {.cursors = &cursors_}) {} + + Status init(std::vector cursors); + + bool empty() const; + SniiSegmentTermCursor* front() const; + Status advance_front(); + +private: + std::vector cursors_; + IndexedWinnerTree frontier_; + bool initialized_ = false; + Status failed_ = Status::OK(); +}; + +} // namespace doris::snii::compaction diff --git a/be/src/storage/index/snii/encoding/byte_sink.cpp b/be/src/storage/index/snii/encoding/byte_sink.cpp new file mode 100644 index 00000000000000..7d91a6552f6658 --- /dev/null +++ b/be/src/storage/index/snii/encoding/byte_sink.cpp @@ -0,0 +1,60 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/encoding/byte_sink.h" + +#include "storage/index/snii/encoding/varint.h" + +namespace doris::snii { + +void ByteSink::put_fixed16(uint16_t v) { + for (int i = 0; i < 2; ++i) buf_.push_back(static_cast(v >> (8 * i))); +} + +void ByteSink::put_fixed32(uint32_t v) { + for (int i = 0; i < 4; ++i) buf_.push_back(static_cast(v >> (8 * i))); +} + +void ByteSink::put_fixed64(uint64_t v) { + for (int i = 0; i < 8; ++i) buf_.push_back(static_cast(v >> (8 * i))); +} + +void ByteSink::put_varint32(uint32_t v) { + while (v >= 0x80) { + buf_.push_back(static_cast(v) | 0x80); + v >>= 7; + } + buf_.push_back(static_cast(v)); +} + +void ByteSink::put_varint64(uint64_t v) { + while (v >= 0x80) { + buf_.push_back(static_cast(v) | 0x80); + v >>= 7; + } + buf_.push_back(static_cast(v)); +} + +void ByteSink::put_zigzag(int64_t v) { + put_varint64(zigzag_encode(v)); +} + +void ByteSink::put_bytes(Slice s) { + buf_.insert(buf_.end(), s.data(), s.data() + s.size()); +} + +} // namespace doris::snii diff --git a/be/src/storage/index/snii/encoding/byte_sink.h b/be/src/storage/index/snii/encoding/byte_sink.h new file mode 100644 index 00000000000000..e96ae112e20a31 --- /dev/null +++ b/be/src/storage/index/snii/encoding/byte_sink.h @@ -0,0 +1,71 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include "storage/index/snii/common/slice.h" + +namespace doris::snii { + +// append-only write cursor: all section serialization goes through this; manual byte assembly is forbidden. +// All multi-byte fixed-width fields are little-endian. +class ByteSink { +public: + void put_u8(uint8_t v) { buf_.push_back(v); } + void put_fixed16(uint16_t v); + void put_fixed32(uint32_t v); + void put_fixed64(uint64_t v); + void put_varint32(uint32_t v); + void put_varint64(uint64_t v); + void put_zigzag(int64_t v); + void put_bytes(Slice s); + + size_t size() const { return buf_.size(); } + const std::vector& buffer() const { return buf_; } + Slice view() const { return Slice(buf_); } + + // Reserves capacity for `additional` MORE bytes on top of the current size(), + // so a caller about to append a known-length run pays at most one reallocation + // instead of the geometric-growth reallocs a byte-at-a-time put_u8 loop would + // trigger. The argument is RELATIVE (absolute target = size() + additional): a + // sink REUSED across encodes (e.g. the shared PFOR-run `out`) keeps accumulating + // correctly rather than no-op'ing on a repeated per-run reserve of the same + // value. Affects only the backing buffer's capacity -- the emitted bytes are + // unchanged. + void reserve(size_t additional) { buf_.reserve(buf_.size() + additional); } + + // Resets the cursor to empty while RETAINING the backing capacity, so a sink can + // be reused across many small encodes (e.g. per-window region/prx scratch in the + // windowed posting builder) without re-allocating each time -- this avoids the + // cumulative small-allocation churn that fragments the heap arena and inflates + // peak RSS during the merge of a high-df term split into thousands of windows. + void clear() { buf_.clear(); } + + // Moves the backing buffer OUT to the caller (the sink is left empty), so an encoded + // section can be handed off without the copy (+ copy-induced capacity slack) that + // reading buffer() and copy-assigning would incur. Use only when the sink is not + // reused afterward (a stack-local about to die, or one that is clear()'d next). + std::vector take() { return std::move(buf_); } + +private: + std::vector buf_; +}; + +} // namespace doris::snii diff --git a/be/src/storage/index/snii/encoding/byte_source.cpp b/be/src/storage/index/snii/encoding/byte_source.cpp new file mode 100644 index 00000000000000..93619aa97064e7 --- /dev/null +++ b/be/src/storage/index/snii/encoding/byte_source.cpp @@ -0,0 +1,179 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/encoding/byte_source.h" + +#include + +#include "storage/index/snii/encoding/varint.h" + +namespace doris::snii { + +Status ByteSource::get_u8(uint8_t* v) { + if (remaining() < 1) + return Status::Error("get_u8 overrun"); + *v = s_[pos_++]; + return Status::OK(); +} + +Status ByteSource::get_fixed16(uint16_t* v) { + if (remaining() < 2) + return Status::Error( + "get_fixed16 overrun"); + uint16_t r = 0; + for (int i = 0; i < 2; ++i) r |= static_cast(s_[pos_ + i]) << (8 * i); + pos_ += 2; + *v = r; + return Status::OK(); +} + +Status ByteSource::get_fixed32(uint32_t* v) { + if (remaining() < 4) + return Status::Error( + "get_fixed32 overrun"); + uint32_t r = 0; + for (int i = 0; i < 4; ++i) r |= static_cast(s_[pos_ + i]) << (8 * i); + pos_ += 4; + *v = r; + return Status::OK(); +} + +Status ByteSource::get_fixed64(uint64_t* v) { + if (remaining() < 8) + return Status::Error( + "get_fixed64 overrun"); + uint64_t r = 0; + for (int i = 0; i < 8; ++i) r |= static_cast(s_[pos_ + i]) << (8 * i); + pos_ += 8; + *v = r; + return Status::OK(); +} + +Status ByteSource::get_varint64(uint64_t* v) { + const uint8_t* p = s_.data() + pos_; + const uint8_t* next = nullptr; + RETURN_IF_ERROR(decode_varint64(p, s_.data() + s_.size(), v, &next)); + pos_ = static_cast(next - s_.data()); + return Status::OK(); +} + +Status ByteSource::get_varint32(uint32_t* v) { + uint64_t tmp; + RETURN_IF_ERROR(get_varint64(&tmp)); + if (tmp > 0xFFFFFFFFu) + return Status::Error("varint32 overflow"); + *v = static_cast(tmp); + return Status::OK(); +} + +Status ByteSource::decode_delta_run(size_t count, std::vector* out) { + if (out == nullptr) { + return Status::Error( + "byte_source: null delta run output"); + } + if (count > std::numeric_limits::max() - out->size()) { + return Status::Error( + "byte_source: delta run size overflow"); + } + const uint8_t* const begin = s_.data(); + const uint8_t* const end = begin + s_.size(); + const uint8_t* p = begin + pos_; + const size_t original_size = out->size(); + out->reserve(out->size() + count); + uint32_t running = 0; + const auto fail = [&](const char* message) { + out->resize(original_size); + return Status::Error(message); + }; + for (size_t i = 0; i < count; ++i) { + if (p >= end) { + return fail("byte_source: delta run past end"); + } + uint32_t b = *p++; + uint32_t delta = b & 0x7FU; + if (b >= 0x80) { // multi-byte (rare for position deltas) + uint32_t shift = 7; + for (;;) { + if (p >= end) { + return fail("byte_source: delta run past end"); + } + b = *p++; + if (shift == 28 && (b & 0xF0U) != 0) { + return fail("byte_source: delta varint32 overflow"); + } + delta |= (b & 0x7FU) << shift; + if ((b & 0x80) == 0) { + break; + } + if (shift == 28) { + return fail("byte_source: delta varint32 overflow"); + } + shift += 7; + } + } + if (delta > std::numeric_limits::max() - running) { + return fail("byte_source: delta prefix sum overflow"); + } + running += delta; + out->push_back(running); + } + pos_ = static_cast(p - begin); + return Status::OK(); +} + +Status ByteSource::skip_varints(size_t count) { + const uint8_t* const begin = s_.data(); + const uint8_t* const end = begin + s_.size(); + const uint8_t* p = begin + pos_; + // Each varint ends at the first byte whose continuation bit (0x80) is clear. + // Scanning for `count` such terminators skips the values with one branch per + // byte -- no shift/accumulate/store and no per-value bounds Status. (A SIMD + // bulk terminator-count was tried and reverted: the skipped position runs + // between selected docs are almost always 1-3 varints -- far below a 16-byte + // block -- so the vector path never amortized, and the larger body stopped + // this function from inlining into the CSR reader, a net CPU regression on + // the httplogs/agentlogs phrase-prefix profiles.) + for (size_t k = 0; k < count; ++k) { + while (p < end && (*p & 0x80) != 0) { + ++p; + } + if (p >= end) { + return Status::Error( + "byte_source: varint skip past end"); + } + ++p; // consume the terminator byte + } + pos_ = static_cast(p - begin); + return Status::OK(); +} + +Status ByteSource::get_zigzag(int64_t* v) { + uint64_t tmp; + RETURN_IF_ERROR(get_varint64(&tmp)); + *v = zigzag_decode(tmp); + return Status::OK(); +} + +Status ByteSource::get_bytes(size_t n, Slice* out) { + if (remaining() < n) + return Status::Error("get_bytes overrun"); + *out = s_.subslice(pos_, n); + pos_ += n; + return Status::OK(); +} + +} // namespace doris::snii diff --git a/be/src/storage/index/snii/encoding/byte_source.h b/be/src/storage/index/snii/encoding/byte_source.h new file mode 100644 index 00000000000000..0490e6949b8de9 --- /dev/null +++ b/be/src/storage/index/snii/encoding/byte_source.h @@ -0,0 +1,91 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/common/slice.h" + +namespace doris::snii { + +// Slice read cursor: all section deserialization goes through this; any overrun returns Corruption. +class ByteSource { +public: + explicit ByteSource(Slice s) : s_(s) {} + + Status get_u8(uint8_t* v); + Status get_fixed16(uint16_t* v); + Status get_fixed32(uint32_t* v); + Status get_fixed64(uint64_t* v); + Status get_varint32(uint32_t* v); + Status get_varint64(uint64_t* v); + + // Single-byte fast-path varint32. The CSR position reader decodes one count + // header (`pos_count`) per doc in a window -- for every doc, selected or + // skipped -- and that per-doc read, routed through the out-of-line + // get_varint32 -> get_varint64 -> decode_varint64 chain, dominates the loop + // once positions themselves are skipped/inlined. Almost all counts are < 128 + // (one byte), so inline that case here and fall back to the full decoder for + // multi-byte values and bounds handling. + Status get_varint32_fast(uint32_t* v) { + const size_t p = pos_; + if (p < s_.size()) { + const uint8_t b0 = s_[p]; + if (b0 < 0x80) { + *v = b0; + pos_ = p + 1; + return Status::OK(); + } + } + return get_varint32(v); + } + // Advances past `count` LEB128 varints WITHOUT decoding their values -- just + // scans continuation bytes. Cheaper than get_varint* per value when the + // decoded value is unused (e.g. skipping a non-selected doc's position + // deltas in a CSR window, where the vast majority of docs in a window are + // not in the candidate set). Returns Corruption on truncation. + Status skip_varints(size_t count); + + // Decodes `count` LEB128 varints, treats them as ASCENDING deltas + // (running prefix sum starting at 0), and APPENDS the running values to + // `out`. A tight inline decoder -- no per-value get_varint32/get_varint64/ + // decode_varint64 call chain, no per-value Status, and a single-byte fast + // path (position deltas are almost always < 128). This is the hot loop of + // the CSR position reader for candidate (selected) docs. Returns Corruption + // on truncation, a >32-bit value, or a uint32 prefix-sum overflow. Failure + // leaves the cursor and output vector unchanged. + Status decode_delta_run(size_t count, std::vector* out); + Status get_zigzag(int64_t* v); + Status get_bytes(size_t n, Slice* out); + + size_t remaining() const { return s_.size() - pos_; } + size_t position() const { return pos_; } + bool eof() const { return pos_ == s_.size(); } + + // Returns a sub-view starting at absolute offset start with length len (used by framer etc. to rewind over the CRC coverage region). + Slice slice_from(size_t start, size_t len) const { return s_.subslice(start, len); } + +private: + Slice s_; + size_t pos_ = 0; +}; + +} // namespace doris::snii diff --git a/be/src/storage/index/snii/encoding/crc32c.cpp b/be/src/storage/index/snii/encoding/crc32c.cpp new file mode 100644 index 00000000000000..39d7c6f58fe487 --- /dev/null +++ b/be/src/storage/index/snii/encoding/crc32c.cpp @@ -0,0 +1,278 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/encoding/crc32c.h" + +// T21 test-seam implementation. Production crc32c()/crc32c_extend() (in the header) +// delegate to the bundled Google crc32c thirdparty, which already runs a +// runtime-dispatched, hardware-accelerated and interleaved CRC32C. The reference +// sub-paths defined here -- portable slice-by-8, serial SSE4.2 hardware, and the +// 3-way interleaved SSE4.2 hardware algorithm T21 specifies -- exist ONLY so that +// unit tests can prove, byte-for-byte across all sizes and alignments, that the +// production path equals the canonical CRC32C (same bit-reflected Castagnoli +// polynomial) and that the hardware path is engaged. All of them are compiled out +// of release builds by the BE_TEST gate, so production pays nothing: no extra code, +// no static slice-by-8 table, no startup CPUID probe. Every function here is a pure +// function with no shared mutable state, so it is trivially thread-safe. +#ifdef BE_TEST + +#include +#include +#include + +#if defined(__x86_64__) || defined(_M_X64) +#define SNII_CRC32C_X86 1 +#include // __get_cpuid, bit_SSE4_2 +#include // _mm_crc32_u8/u32/u64 (SSE4.2) +#endif + +namespace doris::snii { +namespace { + +// Bit-reflected Castagnoli polynomial (CRC32C / iSCSI). Identical to the constant +// the removed in-tree implementation used, so every reference path below yields +// the same on-disk checksum value as the production library. +constexpr uint32_t kPoly = 0x82F63B78U; + +// Below this length the 3-way path's lane setup and GF(2) shift-combine outweigh +// the throughput win, so crc32c_hw3 falls back to the serial hardware path. Small +// buffers (inline prx windows, small pod_ref regions) therefore never pay the +// combine cost. Exposed to tests via crc32c_interleave_threshold(). +constexpr size_t kInterleaveThreshold = 1024; + +// Builds the slice-by-8 lookup tables. Column 0 is the classic byte table; each +// successive column folds in one more byte of look-ahead, letting the inner loop +// consume 8 bytes per iteration with 8 table reads + XORs instead of 8 dependent +// shift/lookup steps. The checksum value is identical to the byte-at-a-time loop. +std::array, 8> make_slice8_table() { + std::array, 8> t {}; + for (uint32_t i = 0; i < 256; ++i) { + uint32_t c = i; + for (int k = 0; k < 8; ++k) { + c = (c & 1) ? (kPoly ^ (c >> 1)) : (c >> 1); + } + t[0][i] = c; + } + for (uint32_t i = 0; i < 256; ++i) { + uint32_t c = t[0][i]; + for (int s = 1; s < 8; ++s) { + c = t[0][c & 0xFF] ^ (c >> 8); + t[s][i] = c; + } + } + return t; +} + +const std::array, 8> kSlice8 = make_slice8_table(); + +inline uint32_t load_le32(const uint8_t* p) { + return static_cast(p[0]) | (static_cast(p[1]) << 8) | + (static_cast(p[2]) << 16) | (static_cast(p[3]) << 24); +} + +// Pure software slice-by-8 (used as the portable path and the hardware fallback). +// Operates on the raw (pre/post-inversion applied by the callers) CRC register. +uint32_t crc32c_slice8(uint32_t crc, const uint8_t* p, size_t n) { + while (n >= 8) { + crc ^= load_le32(p); + const uint32_t hi = load_le32(p + 4); + crc = kSlice8[7][crc & 0xFF] ^ kSlice8[6][(crc >> 8) & 0xFF] ^ + kSlice8[5][(crc >> 16) & 0xFF] ^ kSlice8[4][crc >> 24] ^ kSlice8[3][hi & 0xFF] ^ + kSlice8[2][(hi >> 8) & 0xFF] ^ kSlice8[1][(hi >> 16) & 0xFF] ^ kSlice8[0][hi >> 24]; + p += 8; + n -= 8; + } + while (n--) { + crc = kSlice8[0][(crc ^ *p++) & 0xFF] ^ (crc >> 8); + } + return crc; +} + +#if SNII_CRC32C_X86 +// Serial hardware CRC32C via the SSE4.2 crc32 instruction. The intrinsics operate +// on the same bit-reflected Castagnoli polynomial as the tables, so the result is +// byte-identical. This TU is compiled without -msse4.2, so gate the intrinsics +// behind a function-level target attribute and a runtime CPUID check. This is the +// authoritative serial hardware path that crc32c_hw3 reuses for each lane and tail. +__attribute__((target("sse4.2"))) uint32_t crc32c_hw_serial(uint32_t crc, const uint8_t* p, + size_t n) { + while (n >= 8) { + uint64_t v; + std::memcpy(&v, p, sizeof(v)); // unaligned-safe; x86 folds to a plain load + crc = static_cast(_mm_crc32_u64(crc, v)); + p += 8; + n -= 8; + } + if (n >= 4) { + crc = _mm_crc32_u32(crc, load_le32(p)); + p += 4; + n -= 4; + } + while (n--) { + crc = _mm_crc32_u8(crc, *p++); + } + return crc; +} + +// GF(2) 32x32 bit-matrix helpers (zlib crc32_combine style). A matrix column mat[i] +// is the image of the i-th unit CRC register; gf2_matrix_times sums (XORs) the +// columns selected by the set bits of vec. +uint32_t gf2_matrix_times(const uint32_t* mat, uint32_t vec) { + uint32_t sum = 0; + while (vec != 0) { + if (vec & 1) { + sum ^= *mat; + } + vec >>= 1; + ++mat; + } + return sum; +} + +void gf2_matrix_square(uint32_t* square, const uint32_t* mat) { + for (int n = 0; n < 32; ++n) { + square[n] = gf2_matrix_times(mat, mat[n]); + } +} + +// crc32c_shift(crc, bytes): advance the raw CRC32C register as if `bytes` zero +// bytes were appended, i.e. crc . x^(8*bytes) mod P in the bit-reflected domain. +// This is the linear operator the 3-way combine applies to a lane's partial CRC so +// it lines up with the following lanes -- a pure table/matrix computation with no +// PCLMULQDQ, hence no extra CPUID gate. bytes == 0 returns crc unchanged. +uint32_t crc32c_shift(uint32_t crc, size_t bytes) { + uint32_t even[32]; // operator for 2^k zero bits, doubled each round + uint32_t odd[32]; // operator for 2^(k-1) zero bits + + // odd = operator for a single zero bit: column 0 is the polynomial, columns + // 1..31 shift the register right by one (bit i maps to bit i-1). + odd[0] = kPoly; + uint32_t row = 1; + for (int n = 1; n < 32; ++n) { + odd[n] = row; + row <<= 1; + } + gf2_matrix_square(even, odd); // even = two zero bits + gf2_matrix_square(odd, even); // odd = four zero bits + + size_t len = bytes; + do { + gf2_matrix_square(even, odd); // first pass: even = one zero byte (8 bits) + if (len & 1) { + crc = gf2_matrix_times(even, crc); + } + len >>= 1; + if (len == 0) { + break; + } + gf2_matrix_square(odd, even); + if (len & 1) { + crc = gf2_matrix_times(odd, crc); + } + len >>= 1; + } while (len != 0); + return crc; +} + +// 3-way interleaved hardware CRC32C (rocksdb/folly/Intel style). Splits the buffer +// into three equal, 8-byte-aligned lanes processed by independent _mm_crc32_u64 +// accumulators (breaking the ~3-cycle loop-carried dependency of the serial path), +// then stitches them with the GF(2) shift-combine and finishes the remainder +// serially. Byte-identical to crc32c_hw_serial / crc32c_slice8. Below the +// threshold it defers to the serial path so small buffers skip the combine cost. +uint32_t crc32c_hw3(uint32_t crc, const uint8_t* p, size_t n) { + if (n < kInterleaveThreshold) { + return crc32c_hw_serial(crc, p, n); + } + // 8-byte-aligned lane length keeps every lane on the u64 fast path. n >= 1024 + // guarantees L >= 336 > 0, so 3L <= n and the tail (n - 3L) is well defined. + const size_t lane = (n / 3) & ~static_cast(7); + const uint32_t crc_a = crc32c_hw_serial(crc, p, lane); // seeded lane + const uint32_t crc_b = crc32c_hw_serial(0, p + lane, lane); // raw lane + const uint32_t crc_c = crc32c_hw_serial(0, p + 2 * lane, lane); // raw lane + // crc(seed, A||B) == shift(crc(seed, A), |B|) ^ crc(0, B), applied twice. + uint32_t comb = crc32c_shift(crc_a, lane) ^ crc_b; + comb = crc32c_shift(comb, lane) ^ crc_c; + return crc32c_hw_serial(comb, p + 3 * lane, n - 3 * lane); // serial tail +} + +bool detect_sse42() { + unsigned int eax = 0, ebx = 0, ecx = 0, edx = 0; + if (!__get_cpuid(1, &eax, &ebx, &ecx, &edx)) { + return false; + } + return (ecx & bit_SSE4_2) != 0; +} + +const bool kHasSse42 = detect_sse42(); +#endif // SNII_CRC32C_X86 + +} // namespace + +namespace detail { + +// Portable software path. Always available; the canonical scalar reference for +// every other path and for the on-disk checksum value. +uint32_t crc32c_slice8_extend(uint32_t crc, Slice data) { + crc = ~crc; + crc = crc32c_slice8(crc, data.data(), data.size()); + return ~crc; +} + +// Serial hardware path (falls back to slice8 without SSE4.2). +uint32_t crc32c_hw_serial_extend(uint32_t crc, Slice data) { + crc = ~crc; +#if SNII_CRC32C_X86 + if (kHasSse42) { + crc = crc32c_hw_serial(crc, data.data(), data.size()); + return ~crc; + } +#endif + crc = crc32c_slice8(crc, data.data(), data.size()); + return ~crc; +} + +// 3-way interleaved hardware path (falls back to slice8 without SSE4.2; internally +// falls back to the serial path below the interleave threshold). +uint32_t crc32c_hw3_extend(uint32_t crc, Slice data) { + crc = ~crc; +#if SNII_CRC32C_X86 + if (kHasSse42) { + crc = crc32c_hw3(crc, data.data(), data.size()); + return ~crc; + } +#endif + crc = crc32c_slice8(crc, data.data(), data.size()); + return ~crc; +} + +size_t crc32c_interleave_threshold() { + return kInterleaveThreshold; +} + +bool crc32c_has_hw() { +#if SNII_CRC32C_X86 + return kHasSse42; +#else + return false; +#endif +} + +} // namespace detail +} // namespace doris::snii + +#endif // BE_TEST diff --git a/be/src/storage/index/snii/encoding/crc32c.h b/be/src/storage/index/snii/encoding/crc32c.h new file mode 100644 index 00000000000000..775a781c3d39f0 --- /dev/null +++ b/be/src/storage/index/snii/encoding/crc32c.h @@ -0,0 +1,72 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include +#include + +#include "storage/index/snii/common/slice.h" + +namespace doris::snii { + +// CRC32C (Castagnoli, polynomial 0x1EDC6F41). Used to checksum the tail of each +// format block. Thin inline adapter over Doris's bundled Google crc32c thirdparty +// (crc32c::Extend / crc32c::Crc32c). That library computes the same canonical +// CRC32C (same reflected polynomial, same standard pre/post inversion), so every +// on-disk checksum stays byte-identical to the previous in-tree slice-by-8 / +// SSE4.2 implementation -- this is an implementation swap, not a format change. +// The leading :: keeps the crc32c namespace distinct from crc32c() below. +inline uint32_t crc32c_extend(uint32_t crc, Slice data) { + return ::crc32c::Extend(crc, data.data(), data.size()); +} + +inline uint32_t crc32c(Slice data) { + return ::crc32c::Crc32c(data.data(), data.size()); +} + +#ifdef BE_TEST +// T21 test seam. The production crc32c()/crc32c_extend() above delegate to the +// bundled Google crc32c thirdparty (see commit d0416bb4129), which already runs a +// runtime-dispatched, hardware-accelerated and interleaved CRC32C -- so T21's +// "hardware interleaved CRC" goal is already met (and exceeded: that library adds +// a PCLMULQDQ fold a hand-rolled 3-way _mm_crc32_u64 lacks). Rather than regress +// that reuse, the reference sub-paths below let unit tests prove, byte-for-byte +// across all sizes/alignments, that the production path equals the canonical +// CRC32C and that the hardware path is engaged: +// * crc32c_slice8_extend -- portable software slice-by-8 (always available); +// * crc32c_hw_serial_extend -- serial SSE4.2 _mm_crc32 hardware path; +// * crc32c_hw3_extend -- 3-way interleaved SSE4.2 hardware path with a +// GF(2) shift-combine and a 1024-byte fall-back to +// the serial path (the algorithm T21 specifies). +// hw_serial/hw3 fall back to slice8 when SSE4.2 is absent. Each *_extend applies +// the standard ~crc pre/post inversion, so *_extend(0, d) == crc32c(d). The whole +// seam plus its static slice-by-8 table and startup CPUID probe are compiled out +// of release builds by this BE_TEST gate, so production carries no extra code. +// Pure functions with no shared mutable state (CONCURRENCY: N/A). +namespace detail { +uint32_t crc32c_slice8_extend(uint32_t crc, Slice data); +uint32_t crc32c_hw_serial_extend(uint32_t crc, Slice data); +uint32_t crc32c_hw3_extend(uint32_t crc, Slice data); +size_t crc32c_interleave_threshold(); +bool crc32c_has_hw(); +} // namespace detail +#endif // BE_TEST + +} // namespace doris::snii diff --git a/be/src/storage/index/snii/encoding/pfor.cpp b/be/src/storage/index/snii/encoding/pfor.cpp new file mode 100644 index 00000000000000..dcc620fcc54c8f --- /dev/null +++ b/be/src/storage/index/snii/encoding/pfor.cpp @@ -0,0 +1,452 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/encoding/pfor.h" + +#include +#include +#include +#include +#include + +#include "storage/index/snii/common/slice.h" + +namespace doris::snii { +namespace { + +// Unaligned little-endian 64-bit load from a raw byte pointer (single +// instruction on x86; memcpy is the portable, UB-free spelling the compiler +// folds to a mov). +inline uint64_t load_u64_le(const uint8_t* p) { + uint64_t v; + std::memcpy(&v, p, sizeof(v)); +#if defined(__BIG_ENDIAN__) || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) + v = __builtin_bswap64(v); +#endif + return v; +} + +// TEST-ONLY seam backing doris::snii::testing::pfor_width_evals(). value_width() +// is the SINGLE per-value bit-width evaluation point on the encode path, so this +// counter equals the number of values processed per run -- the deterministic +// signal that the histogram path scans each value exactly once (vs the former +// O(maxw*n) re-scan). Compiled out entirely in non-test builds so production pays +// nothing; a relaxed atomic under BE_TEST keeps it data-race-free under TSAN. +#ifdef BE_TEST +std::atomic g_width_evals {0}; +#endif + +// Number of significant bits of v (its minimal bit_width); value_width(0) == 0 is +// preserved (matching the former bits_for). clz(0) is undefined behaviour, so +// v == 0 is mapped explicitly here -- never via clz(v | 1), which would mis-score +// 0 as width 1 and corrupt the histogram. +inline uint8_t value_width(uint32_t v) { +#ifdef BE_TEST + g_width_evals.fetch_add(1, std::memory_order_relaxed); +#endif + return v ? static_cast(32 - __builtin_clz(v)) : 0; +} + +// Choose the bit_width that minimizes total bytes (packed + exceptions), with +// exception cost estimated at ~6 bytes each. A single O(n) pass builds a bit-width +// histogram (recording each value's width into widths[] for the encoder to reuse), +// then an O(maxw) suffix-sum gives the exception count per candidate width -- +// replacing the former O(maxw*n) re-scan. The cost formula and the ascending-w +// strict-'<' tie-break are kept identical, so the chosen width (and hence every +// encoded byte) is unchanged. +uint8_t choose_width(const uint32_t* v, size_t n, uint8_t* widths) { + // hist[b] = #values whose bit-width == b. n is capped at kFrqBaseUnit (256) on + // the production path; uint32_t buckets keep a direct caller with a larger run + // correct without affecting the chosen width. + uint32_t hist[33] = {0}; + uint8_t maxw = 0; + for (size_t i = 0; i < n; ++i) { + const uint8_t b = value_width(v[i]); + widths[i] = b; + ++hist[b]; + maxw = std::max(b, maxw); + } + // suffix[k] = #values whose bit-width >= k, so the exceptions for candidate + // width w (values needing more than w bits) are exactly suffix[w + 1]. + size_t suffix[34] = {0}; + for (int k = 32; k >= 0; --k) { + suffix[k] = suffix[k + 1] + hist[k]; + } + uint8_t best = maxw; + size_t best_cost = SIZE_MAX; + for (uint8_t w = 0; w <= maxw; ++w) { + const size_t exc = suffix[w + 1]; + const size_t cost = (static_cast(w) * n + 7) / 8 + exc * 6; + if (cost < best_cost) { + best_cost = cost; + best = w; + } + } + return best; +} + +uint32_t low_mask(uint8_t w) { + return (w >= 32) ? 0xFFFFFFFFU : ((1U << w) - 1U); +} + +// Bit-pack the low w bits of each value, writing 0 at exception positions +// (widths[i] > w) instead of the value's low bits. This is byte-identical to +// packing a copy in which those slots were pre-zeroed (the former `low[i] = 0` +// placeholder), so the encoder no longer materializes that copy. +void bitpack_masked(const uint32_t* v, const uint8_t* widths, size_t n, uint8_t w, ByteSink* out) { + if (w == 0) { + return; + } + // Pre-size for the exact packed byte count (ceil(w*n/8)) so the per-byte put_u8 + // loop below never reallocates mid-pack. ByteSink::reserve is RELATIVE to out's + // current size, and `out` is reused across the PFOR runs of one region, so this + // accumulates run-to-run instead of no-op'ing. Capacity-only: the packed bytes + // and their values are byte-identical to the un-reserved path. + const size_t packed = (static_cast(w) * n + 7) / 8; + out->reserve(packed); + const uint32_t mask = low_mask(w); + uint64_t acc = 0; + int filled = 0; + for (size_t i = 0; i < n; ++i) { + const uint32_t lo = (widths[i] > w) ? 0U : (v[i] & mask); + acc |= static_cast(lo) << filled; + filled += w; + while (filled >= 8) { + out->put_u8(static_cast(acc)); + acc >>= 8; + filled -= 8; + } + } + if (filled > 0) { + out->put_u8(static_cast(acc)); + } +} + +void bitunpack_tail(const uint8_t* base, size_t packed, size_t n, uint8_t w, size_t i, + uint64_t mask, uint32_t* out) { + for (; i < n; ++i) { + const size_t bit_off = static_cast(w) * i; + const size_t byte_off = bit_off >> 3; + uint64_t word = 0; + for (size_t b = byte_off; b < packed && b < byte_off + 8; ++b) { + word |= static_cast(base[b]) << ((b - byte_off) * 8); + } + out[i] = static_cast((word >> (bit_off & 7)) & mask); + } +} + +void bitunpack_w1(const uint8_t* base, size_t n, uint32_t* out) { + size_t i = 0; + size_t byte = 0; + for (; i + 8 <= n; i += 8, ++byte) { + const uint8_t v = base[byte]; + out[i] = v & 1U; + out[i + 1] = (v >> 1) & 1U; + out[i + 2] = (v >> 2) & 1U; + out[i + 3] = (v >> 3) & 1U; + out[i + 4] = (v >> 4) & 1U; + out[i + 5] = (v >> 5) & 1U; + out[i + 6] = (v >> 6) & 1U; + out[i + 7] = (v >> 7) & 1U; + } + if (i < n) { + const uint8_t v = base[byte]; + for (uint8_t bit = 0; i < n; ++i, ++bit) { + out[i] = (v >> bit) & 1U; + } + } +} + +void bitunpack_w2(const uint8_t* base, size_t n, uint32_t* out) { + size_t i = 0; + size_t byte = 0; + for (; i + 4 <= n; i += 4, ++byte) { + const uint8_t v = base[byte]; + out[i] = v & 3U; + out[i + 1] = (v >> 2) & 3U; + out[i + 2] = (v >> 4) & 3U; + out[i + 3] = (v >> 6) & 3U; + } + if (i < n) { + const uint8_t v = base[byte]; + for (uint8_t shift = 0; i < n; ++i, shift += 2) { + out[i] = (v >> shift) & 3U; + } + } +} + +void bitunpack_w3(const uint8_t* base, size_t packed, size_t n, uint32_t* out) { + size_t i = 0; + size_t byte = 0; + for (; i + 8 <= n; i += 8, byte += 3) { + const uint32_t b0 = base[byte]; + const uint32_t b1 = base[byte + 1]; + const uint32_t b2 = base[byte + 2]; + out[i] = b0 & 7U; + out[i + 1] = (b0 >> 3) & 7U; + out[i + 2] = ((b0 >> 6) | (b1 << 2)) & 7U; + out[i + 3] = (b1 >> 1) & 7U; + out[i + 4] = (b1 >> 4) & 7U; + out[i + 5] = ((b1 >> 7) | (b2 << 1)) & 7U; + out[i + 6] = (b2 >> 2) & 7U; + out[i + 7] = (b2 >> 5) & 7U; + } + bitunpack_tail(base, packed, n, 3, i, 7U, out); +} + +void bitunpack_w4(const uint8_t* base, size_t n, uint32_t* out) { + size_t i = 0; + size_t byte = 0; + for (; i + 2 <= n; i += 2, ++byte) { + const uint8_t v = base[byte]; + out[i] = v & 15U; + out[i + 1] = (v >> 4) & 15U; + } + if (i < n) { + out[i] = base[byte] & 15U; + } +} + +void bitunpack_w5(const uint8_t* base, size_t packed, size_t n, uint32_t* out) { + size_t i = 0; + size_t byte = 0; + for (; i + 8 <= n; i += 8, byte += 5) { + const uint32_t b0 = base[byte]; + const uint32_t b1 = base[byte + 1]; + const uint32_t b2 = base[byte + 2]; + const uint32_t b3 = base[byte + 3]; + const uint32_t b4 = base[byte + 4]; + out[i] = b0 & 31U; + out[i + 1] = ((b0 >> 5) | (b1 << 3)) & 31U; + out[i + 2] = (b1 >> 2) & 31U; + out[i + 3] = ((b1 >> 7) | (b2 << 1)) & 31U; + out[i + 4] = ((b2 >> 4) | (b3 << 4)) & 31U; + out[i + 5] = (b3 >> 1) & 31U; + out[i + 6] = ((b3 >> 6) | (b4 << 2)) & 31U; + out[i + 7] = (b4 >> 3) & 31U; + } + bitunpack_tail(base, packed, n, 5, i, 31U, out); +} + +void bitunpack_w6(const uint8_t* base, size_t packed, size_t n, uint32_t* out) { + size_t i = 0; + size_t byte = 0; + for (; i + 4 <= n; i += 4, byte += 3) { + const uint32_t b0 = base[byte]; + const uint32_t b1 = base[byte + 1]; + const uint32_t b2 = base[byte + 2]; + out[i] = b0 & 63U; + out[i + 1] = ((b0 >> 6) | (b1 << 2)) & 63U; + out[i + 2] = ((b1 >> 4) | (b2 << 4)) & 63U; + out[i + 3] = (b2 >> 2) & 63U; + } + bitunpack_tail(base, packed, n, 6, i, 63U, out); +} + +void bitunpack_w7(const uint8_t* base, size_t packed, size_t n, uint32_t* out) { + size_t i = 0; + size_t byte = 0; + for (; i + 8 <= n; i += 8, byte += 7) { + const uint32_t b0 = base[byte]; + const uint32_t b1 = base[byte + 1]; + const uint32_t b2 = base[byte + 2]; + const uint32_t b3 = base[byte + 3]; + const uint32_t b4 = base[byte + 4]; + const uint32_t b5 = base[byte + 5]; + const uint32_t b6 = base[byte + 6]; + out[i] = b0 & 127U; + out[i + 1] = ((b0 >> 7) | (b1 << 1)) & 127U; + out[i + 2] = ((b1 >> 6) | (b2 << 2)) & 127U; + out[i + 3] = ((b2 >> 5) | (b3 << 3)) & 127U; + out[i + 4] = ((b3 >> 4) | (b4 << 4)) & 127U; + out[i + 5] = ((b4 >> 3) | (b5 << 5)) & 127U; + out[i + 6] = ((b5 >> 2) | (b6 << 6)) & 127U; + out[i + 7] = (b6 >> 1) & 127U; + } + bitunpack_tail(base, packed, n, 7, i, 127U, out); +} + +void bitunpack_w8(const uint8_t* base, size_t n, uint32_t* out) { + for (size_t i = 0; i < n; ++i) { + out[i] = base[i]; + } +} + +void bitunpack_generic(const uint8_t* base, size_t packed, size_t n, uint8_t w, uint32_t* out) { + const uint64_t mask = low_mask(w); + size_t i = 0; + if (packed >= 8) { + const size_t last_safe_byte = packed - 8; + for (; i < n; ++i) { + const size_t bit_off = static_cast(w) * i; + const size_t byte_off = bit_off >> 3; + if (byte_off > last_safe_byte) { + break; + } + out[i] = static_cast((load_u64_le(base + byte_off) >> (bit_off & 7)) & mask); + } + } + bitunpack_tail(base, packed, n, w, i, mask, out); +} + +Status bitunpack(ByteSource* src, size_t n, uint8_t w, uint32_t* out) { + if (w == 0) { + std::memset(out, 0, n * sizeof(uint32_t)); + return Status::OK(); + } + // Pull the packed run once and unpack from the contiguous slice; this keeps + // the hot decode path free of per-byte ByteSource calls. + const size_t packed = (static_cast(w) * n + 7) / 8; + Slice buf; + RETURN_IF_ERROR(src->get_bytes(packed, &buf)); + const uint8_t* base = buf.data(); + + switch (w) { + case 1: + bitunpack_w1(base, n, out); + break; + case 2: + bitunpack_w2(base, n, out); + break; + case 3: + bitunpack_w3(base, packed, n, out); + break; + case 4: + bitunpack_w4(base, n, out); + break; + case 5: + bitunpack_w5(base, packed, n, out); + break; + case 6: + bitunpack_w6(base, packed, n, out); + break; + case 7: + bitunpack_w7(base, packed, n, out); + break; + case 8: + bitunpack_w8(base, n, out); + break; + default: + bitunpack_generic(base, packed, n, w, out); + break; + } + return Status::OK(); +} + +} // namespace + +namespace testing { +// Test-only op-count seam; see pfor.h. Reports/resets the per-value bit-width +// evaluation counter, and is a no-op in non-test builds where the counter is +// compiled out. +uint64_t pfor_width_evals() { +#ifdef BE_TEST + return g_width_evals.load(std::memory_order_relaxed); +#else + return 0; +#endif +} +void reset_pfor_width_evals() { +#ifdef BE_TEST + g_width_evals.store(0, std::memory_order_relaxed); +#endif +} +} // namespace testing + +void pfor_encode(const uint32_t* values, size_t n, ByteSink* out) { + // n is hard-capped at kFrqBaseUnit (256) by encode_pfor_runs, so the stack + // buffer is used on every production call; the heap fallback only guards a + // direct caller passing a larger run. + uint8_t widths_stack[256]; + std::vector widths_heap; + uint8_t* widths = widths_stack; + if (n > sizeof(widths_stack)) { + widths_heap.resize(n); + widths = widths_heap.data(); + } + + const uint8_t w = choose_width(values, n, widths); + out->put_u8(w); + + // Count exceptions for the varint header by reusing the cached per-value + // widths -- no further bit-width evaluation. + uint32_t n_exc = 0; + for (size_t i = 0; i < n; ++i) { + n_exc += (widths[i] > w); + } + out->put_varint32(n_exc); + + // Pack the low w bits, writing 0 at exception slots (byte-identical to the + // former zeroed-copy approach) so no separate `low` buffer is materialized. + bitpack_masked(values, widths, n, w, out); + + // Exception table: (index_delta, full_value) in ascending index order. + uint32_t prev = 0; + for (size_t i = 0; i < n; ++i) { + if (widths[i] > w) { + out->put_varint32(static_cast(i) - prev); + out->put_varint32(values[i]); + prev = static_cast(i); + } + } +} + +Status pfor_decode(ByteSource* src, size_t n, uint32_t* out) { + uint8_t w; + RETURN_IF_ERROR(src->get_u8(&w)); + uint32_t n_exc; + RETURN_IF_ERROR(src->get_varint32(&n_exc)); + RETURN_IF_ERROR(bitunpack(src, n, w, out)); + uint32_t idx = 0; + for (uint32_t i = 0; i < n_exc; ++i) { + uint32_t d, val; + RETURN_IF_ERROR(src->get_varint32(&d)); + RETURN_IF_ERROR(src->get_varint32(&val)); + idx += d; + if (idx >= n) { + return Status::Error( + "pfor exception index out of range"); + } + out[idx] = val; + } + return Status::OK(); +} + +Status pfor_skip(ByteSource* src, size_t n) { + uint8_t w = 0; + RETURN_IF_ERROR(src->get_u8(&w)); + uint32_t n_exc = 0; + RETURN_IF_ERROR(src->get_varint32(&n_exc)); + const size_t packed = (static_cast(w) * n + 7) / 8; + Slice unused; + RETURN_IF_ERROR(src->get_bytes(packed, &unused)); + uint32_t idx = 0; + for (uint32_t i = 0; i < n_exc; ++i) { + uint32_t d = 0; + uint32_t val = 0; + RETURN_IF_ERROR(src->get_varint32(&d)); + RETURN_IF_ERROR(src->get_varint32(&val)); + idx += d; + if (idx >= n) { + return Status::Error( + "pfor exception index out of range"); + } + } + return Status::OK(); +} + +} // namespace doris::snii diff --git a/be/src/storage/index/snii/encoding/pfor.h b/be/src/storage/index/snii/encoding/pfor.h new file mode 100644 index 00000000000000..709d35748249f1 --- /dev/null +++ b/be/src/storage/index/snii/encoding/pfor.h @@ -0,0 +1,53 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include "common/status.h" +#include "storage/index/snii/encoding/byte_sink.h" +#include "storage/index/snii/encoding/byte_source.h" + +namespace doris::snii { + +// PFOR integer block encoder/decoder (unsigned uint32 array). +// Encoded layout: [u8 bit_width][varint n_exceptions][bit-packed low +// bits][exception table]. Selects the bit_width that minimizes total byte size; +// values exceeding it go into the exception table (index_delta, full_value). +// delta/zigzag is handled by the upper layer (.frq window); PFOR only processes +// unsigned integer arrays. +void pfor_encode(const uint32_t* values, size_t n, ByteSink* out); +Status pfor_decode(ByteSource* src, size_t n, uint32_t* out); +Status pfor_skip(ByteSource* src, size_t n); + +} // namespace doris::snii + +// Test-only instrumentation seam (mirrors the dict-block decode-counter pattern). +// pfor_width_evals() returns a process-global count of per-value bit-width +// evaluations performed by pfor_encode since the last reset -- one per +// value_width() call, the single evaluation point on the encode path. Deterministic +// perf tests assert it equals the number of encoded values per run, proving the +// histogram path scans each value exactly once (vs the former O(maxw*n) re-scan). +// Compiled to a no-op in non-test builds; reset between tests. +namespace doris::snii::testing { + +uint64_t pfor_width_evals(); +void reset_pfor_width_evals(); + +} // namespace doris::snii::testing diff --git a/be/src/storage/index/snii/encoding/section_framer.cpp b/be/src/storage/index/snii/encoding/section_framer.cpp new file mode 100644 index 00000000000000..e67b618758dae8 --- /dev/null +++ b/be/src/storage/index/snii/encoding/section_framer.cpp @@ -0,0 +1,59 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/encoding/section_framer.h" + +#include "storage/index/snii/encoding/crc32c.h" + +namespace doris::snii { + +void SectionFramer::write(ByteSink& sink, uint8_t section_type, Slice payload) { + // Single-copy framing: write [type][varint64 len][payload] straight into the + // target sink, then crc exactly those bytes. view() is taken AFTER the payload + // and BEFORE the crc, so subslice([start, framed_len)) is over a settled, + // contiguous buffer with no pending realloc/aliasing. Byte-identical to the + // former temp-ByteSink assembly, minus one heap alloc + one payload copy. + const size_t start = sink.size(); + sink.put_u8(section_type); + sink.put_varint64(payload.size()); + sink.put_bytes(payload); + const size_t framed_len = sink.size() - start; + const uint32_t crc = crc32c(sink.view().subslice(start, framed_len)); + sink.put_fixed32(crc); +} + +Status SectionFramer::read(ByteSource& src, FramedSection* out) { + size_t start = src.position(); + uint8_t type; + RETURN_IF_ERROR(src.get_u8(&type)); + uint64_t len; + RETURN_IF_ERROR(src.get_varint64(&len)); + Slice payload; + RETURN_IF_ERROR(src.get_bytes(static_cast(len), &payload)); + size_t framed_len = src.position() - start; + uint32_t stored; + RETURN_IF_ERROR(src.get_fixed32(&stored)); + if (crc32c(src.slice_from(start, framed_len)) != stored) { + return Status::Error( + "section crc mismatch"); + } + out->type = type; + out->payload = payload; + return Status::OK(); +} + +} // namespace doris::snii diff --git a/be/src/storage/index/snii/encoding/section_framer.h b/be/src/storage/index/snii/encoding/section_framer.h new file mode 100644 index 00000000000000..9a248381ea97b6 --- /dev/null +++ b/be/src/storage/index/snii/encoding/section_framer.h @@ -0,0 +1,44 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include "common/status.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/encoding/byte_sink.h" +#include "storage/index/snii/encoding/byte_source.h" + +namespace doris::snii { + +// A framed section: type + payload view. +struct FramedSection { + uint8_t type = 0; + Slice payload; +}; + +// Unified section framing: [u8 type][varint64 len][payload][fixed32 crc32c(type+len+payload)]. +// All full-format sections reuse this encode/checksum path to avoid ad-hoc hand-assembly. +// Unknown optional sections are dispatched by the caller based on type; read still verifies the CRC and skips the payload. +class SectionFramer { +public: + static void write(ByteSink& sink, uint8_t section_type, Slice payload); + static Status read(ByteSource& src, FramedSection* out); +}; + +} // namespace doris::snii diff --git a/be/src/storage/index/snii/encoding/varint.cpp b/be/src/storage/index/snii/encoding/varint.cpp new file mode 100644 index 00000000000000..a53a08de6b8d3b --- /dev/null +++ b/be/src/storage/index/snii/encoding/varint.cpp @@ -0,0 +1,73 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/encoding/varint.h" + +namespace doris::snii { + +size_t varint_len(uint64_t v) { + size_t n = 1; + while (v >= 0x80) { + v >>= 7; + ++n; + } + return n; +} + +size_t encode_varint64(uint64_t v, uint8_t* out) { + size_t i = 0; + while (v >= 0x80) { + out[i++] = static_cast(v) | 0x80; + v >>= 7; + } + out[i++] = static_cast(v); + return i; +} + +size_t encode_varint32(uint32_t v, uint8_t* out) { + return encode_varint64(v, out); +} + +Status decode_varint64(const uint8_t* p, const uint8_t* end, uint64_t* v, const uint8_t** next) { + uint64_t result = 0; + int shift = 0; + while (p < end) { + uint8_t b = *p++; + result |= static_cast(b & 0x7F) << shift; + if ((b & 0x80) == 0) { + *v = result; + *next = p; + return Status::OK(); + } + shift += 7; + if (shift >= 64) + return Status::Error( + "varint64 overflow"); + } + return Status::Error("varint truncated"); +} + +Status decode_varint32(const uint8_t* p, const uint8_t* end, uint32_t* v, const uint8_t** next) { + uint64_t tmp; + RETURN_IF_ERROR(decode_varint64(p, end, &tmp, next)); + if (tmp > 0xFFFFFFFFu) + return Status::Error("varint32 overflow"); + *v = static_cast(tmp); + return Status::OK(); +} + +} // namespace doris::snii diff --git a/be/src/storage/index/snii/encoding/varint.h b/be/src/storage/index/snii/encoding/varint.h new file mode 100644 index 00000000000000..978812ea6e66f6 --- /dev/null +++ b/be/src/storage/index/snii/encoding/varint.h @@ -0,0 +1,43 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include "common/status.h" + +namespace doris::snii { + +// LEB128 variable-length integer encoding + zigzag. out buffer must be >=10 bytes; returns number of bytes written. +size_t varint_len(uint64_t v); +size_t encode_varint32(uint32_t v, uint8_t* out); +size_t encode_varint64(uint64_t v, uint8_t* out); + +// Decode a varint from the range [p, end); on success *next points to the next byte after the consumed input. +Status decode_varint32(const uint8_t* p, const uint8_t* end, uint32_t* v, const uint8_t** next); +Status decode_varint64(const uint8_t* p, const uint8_t* end, uint64_t* v, const uint8_t** next); + +inline uint64_t zigzag_encode(int64_t v) { + return (static_cast(v) << 1) ^ static_cast(v >> 63); +} +inline int64_t zigzag_decode(uint64_t v) { + return static_cast(v >> 1) ^ -static_cast(v & 1); +} + +} // namespace doris::snii diff --git a/be/src/storage/index/snii/encoding/zstd_codec.cpp b/be/src/storage/index/snii/encoding/zstd_codec.cpp new file mode 100644 index 00000000000000..6c01c0cb2ebdef --- /dev/null +++ b/be/src/storage/index/snii/encoding/zstd_codec.cpp @@ -0,0 +1,55 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/encoding/zstd_codec.h" + +#include + +#include + +#include "storage/index/snii/common/uninitialized_buffer.h" + +namespace doris::snii { + +Status zstd_compress(Slice input, int level, std::vector* out) { + size_t bound = ZSTD_compressBound(input.size()); + out->resize(bound); + size_t n = ZSTD_compress(out->data(), bound, input.data(), input.size(), level); + if (ZSTD_isError(n)) { + return Status::Error(std::string("zstd compress: ") + + ZSTD_getErrorName(n)); + } + out->resize(n); + return Status::OK(); +} + +Status zstd_decompress(Slice input, size_t expected_uncomp_len, std::vector* out) { + // Sized then fully overwritten by ZSTD_decompress (length-checked below). + resize_uninitialized(*out, expected_uncomp_len); + size_t n = ZSTD_decompress(out->data(), expected_uncomp_len, input.data(), input.size()); + if (ZSTD_isError(n)) { + return Status::Error( + std::string("zstd decompress: ") + ZSTD_getErrorName(n)); + } + if (n != expected_uncomp_len) { + return Status::Error( + "zstd decompressed length mismatch"); + } + return Status::OK(); +} + +} // namespace doris::snii diff --git a/be/src/storage/index/snii/encoding/zstd_codec.h b/be/src/storage/index/snii/encoding/zstd_codec.h new file mode 100644 index 00000000000000..c8291ce024fca7 --- /dev/null +++ b/be/src/storage/index/snii/encoding/zstd_codec.h @@ -0,0 +1,33 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/common/slice.h" + +namespace doris::snii { + +// Thin ZSTD wrapper. Used for compressing large payloads such as .prx windows. Decompression requires the caller to supply the original uncompressed length (from the block header). +Status zstd_compress(Slice input, int level, std::vector* out); +Status zstd_decompress(Slice input, size_t expected_uncomp_len, std::vector* out); + +} // namespace doris::snii diff --git a/be/src/storage/index/snii/format/bootstrap_header.cpp b/be/src/storage/index/snii/format/bootstrap_header.cpp new file mode 100644 index 00000000000000..6cdc1869385bec --- /dev/null +++ b/be/src/storage/index/snii/format/bootstrap_header.cpp @@ -0,0 +1,113 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/format/bootstrap_header.h" + +#include "storage/index/snii/encoding/byte_source.h" +#include "storage/index/snii/encoding/crc32c.h" + +namespace doris::snii::format { + +namespace { + +// Number of bytes covered by header_checksum: everything except the trailing +// crc32c. +constexpr size_t kChecksumCoverage = kBootstrapHeaderSize - 4; + +// Writes all fixed fields except the trailing checksum. Field order is the +// on-disk contract; reuse ByteSink fixed-width primitives, never hand-assemble +// bytes. +void encode_fields(const BootstrapHeader& header, ByteSink* sink) { + sink->put_fixed32(header.magic); + sink->put_fixed32((static_cast(header.min_reader_version) << 16) | + header.format_version); + sink->put_fixed32(header.flags); + sink->put_fixed32(kBootstrapHeaderSize); // header_length is always derived + sink->put_u8(header.tail_pointer_size); +} + +} // namespace + +Status encode_bootstrap_header(const BootstrapHeader& header, ByteSink* sink) { + if (sink == nullptr) { + return Status::Error("bootstrap_header: null sink"); + } + ByteSink fields; + encode_fields(header, &fields); + const uint32_t checksum = crc32c(fields.view()); + sink->put_bytes(fields.view()); + sink->put_fixed32(checksum); + return Status::OK(); +} + +Status decode_bootstrap_header(Slice data, BootstrapHeader* out) { + if (out == nullptr) { + return Status::Error("bootstrap_header: null out"); + } + // Reject any size other than the exact fixed header: short input is + // truncation, longer input means stray trailing bytes the parser would + // otherwise ignore. + if (data.size() != kBootstrapHeaderSize) { + return Status::Error( + "bootstrap_header: wrong header size"); + } + + ByteSource src(data); + uint32_t magic = 0; + uint32_t version_pair = 0; + uint32_t flags = 0; + uint32_t header_length = 0; + uint8_t tail_pointer_size = 0; + uint32_t stored_checksum = 0; + RETURN_IF_ERROR(src.get_fixed32(&magic)); + RETURN_IF_ERROR(src.get_fixed32(&version_pair)); + RETURN_IF_ERROR(src.get_fixed32(&flags)); + RETURN_IF_ERROR(src.get_fixed32(&header_length)); + RETURN_IF_ERROR(src.get_u8(&tail_pointer_size)); + RETURN_IF_ERROR(src.get_fixed32(&stored_checksum)); + + if (magic != kContainerMagic) { + return Status::Error( + "bootstrap_header: bad container magic"); + } + const uint32_t computed = crc32c(data.subslice(0, kChecksumCoverage)); + if (computed != stored_checksum) { + return Status::Error( + "bootstrap_header: checksum mismatch"); + } + + const auto min_reader_version = static_cast((version_pair >> 16) & 0xFFFFu); + const auto format_version = static_cast(version_pair & 0xFFFFu); + if (format_version != kFormatVersion) { + return Status::Error( + "bootstrap_header: unsupported container format_version"); + } + if (min_reader_version > kFormatVersion) { + return Status::Error( + "bootstrap_header: container requires a newer reader version"); + } + + out->magic = magic; + out->format_version = format_version; + out->min_reader_version = min_reader_version; + out->flags = flags; + out->header_length = header_length; + out->tail_pointer_size = tail_pointer_size; + return Status::OK(); +} + +} // namespace doris::snii::format diff --git a/be/src/storage/index/snii/format/bootstrap_header.h b/be/src/storage/index/snii/format/bootstrap_header.h new file mode 100644 index 00000000000000..6a8c0e8083d3ac --- /dev/null +++ b/be/src/storage/index/snii/format/bootstrap_header.h @@ -0,0 +1,71 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include "common/status.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/encoding/byte_sink.h" +#include "storage/index/snii/format/format_constants.h" + +namespace doris::snii::format { + +// Fixed container header at the very start of a {rowset_id}_{seg_id}.idx file. +// Identifies the SNII container and carries basic compatibility info so a +// reader can fail fast before touching any streamed section or the tail meta +// region. +// +// On-disk layout (all multi-byte fields little-endian, fixed width; NOT framed +// by SectionFramer because it must be parseable without prior knowledge of the +// file): +// u32 magic == kContainerMagic +// u16 format_version == kFormatVersion +// u16 min_reader_version readers with kFormatVersion < this MUST refuse to +// read u32 flags container-level feature flags u32 +// header_length total bytes of this header including the checksum u8 +// tail_pointer_size size of the fixed tail pointer at EOF (hint for the +// reader) u32 header_checksum crc32c over all preceding header bytes +struct BootstrapHeader { + uint32_t magic = kContainerMagic; + uint16_t format_version = kFormatVersion; + uint16_t min_reader_version = kMinReaderVersion; + uint32_t flags = 0; + uint32_t header_length = 0; + uint8_t tail_pointer_size = 0; +}; + +// Total fixed on-disk size of the header, including the trailing crc32c. +inline constexpr uint32_t kBootstrapHeaderSize = + 4 /*magic*/ + 2 /*format_version*/ + 2 /*min_reader_version*/ + 4 /*flags*/ + + 4 /*header_length*/ + 1 /*tail_pointer_size*/ + 4 /*header_checksum*/; + +// Serializes the header to sink: writes header_length = kBootstrapHeaderSize +// and appends a crc32c over all preceding bytes. The caller's header_length +// field is ignored on input (it is always derived). Returns OK. +Status encode_bootstrap_header(const BootstrapHeader& header, ByteSink* sink); + +// Parses and validates a bootstrap header from the front of data. +// - too short / trailing bytes beyond the fixed header -> kCorruption +// - magic != kContainerMagic -> kCorruption +// - checksum mismatch -> kCorruption +// - format_version != kFormatVersion -> kUnsupported +// - min_reader_version > kFormatVersion -> kUnsupported +Status decode_bootstrap_header(Slice data, BootstrapHeader* out); + +} // namespace doris::snii::format diff --git a/be/src/storage/index/snii/format/bsbf.cpp b/be/src/storage/index/snii/format/bsbf.cpp new file mode 100644 index 00000000000000..d214a99e745ae9 --- /dev/null +++ b/be/src/storage/index/snii/format/bsbf.cpp @@ -0,0 +1,257 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/format/bsbf.h" + +#include + +#include "storage/index/snii/encoding/crc32c.h" + +#if defined(__x86_64__) || defined(_M_X64) +#include +#define SNII_BSBF_X86 1 +#endif + +#define XXH_INLINE_ALL +#include "xxhash.h" + +namespace doris::snii::format { + +const uint32_t kBsbfSalt[kBsbfBitsSetPerBlock] = {0x47b6137bU, 0x44974d91U, 0x8824ad5bU, + 0xa2b7289dU, 0x705495c7U, 0x2df1424bU, + 0x9efc4947U, 0x5c6bfb31U}; + +namespace { + +void store_le32(uint8_t* p, uint32_t v) { + p[0] = static_cast(v); + p[1] = static_cast(v >> 8); + p[2] = static_cast(v >> 16); + p[3] = static_cast(v >> 24); +} +uint32_t load_le32(const uint8_t* p) { + return static_cast(p[0]) | (static_cast(p[1]) << 8) | + (static_cast(p[2]) << 16) | (static_cast(p[3]) << 24); +} + +bool cpu_has_avx2() { +#if defined(SNII_BSBF_X86) + static const bool v = __builtin_cpu_supports("avx2"); + return v; +#else + return false; +#endif +} + +// --- scalar kernels --- +inline void masks_scalar(uint32_t key, uint32_t m[8]) { + for (int i = 0; i < 8; ++i) m[i] = 1u << ((key * kBsbfSalt[i]) >> 27); +} +bool block_contains_scalar(uint64_t hash, const uint8_t* block) { + const uint32_t* w = reinterpret_cast(block); // LE + uint32_t m[8]; + masks_scalar(static_cast(hash), m); + for (int i = 0; i < 8; ++i) + if ((load_le32(reinterpret_cast(w + i)) & m[i]) != m[i]) return false; + return true; +} +void insert_scalar(uint32_t* words, uint32_t block, uint32_t key) { + uint32_t m[8]; + masks_scalar(key, m); + for (int i = 0; i < 8; ++i) words[block * 8 + i] |= m[i]; +} +bool find_scalar(const uint32_t* words, uint32_t block, uint32_t key) { + uint32_t m[8]; + masks_scalar(key, m); + for (int i = 0; i < 8; ++i) + if ((words[block * 8 + i] & m[i]) != m[i]) return false; + return true; +} + +#if defined(SNII_BSBF_X86) +// --- AVX2 kernels: a 256-bit block is one YMM register --- +__attribute__((target("avx2"))) __m256i mask_avx2(uint32_t key) { + const __m256i salt = + _mm256_setr_epi32(static_cast(kBsbfSalt[0]), static_cast(kBsbfSalt[1]), + static_cast(kBsbfSalt[2]), static_cast(kBsbfSalt[3]), + static_cast(kBsbfSalt[4]), static_cast(kBsbfSalt[5]), + static_cast(kBsbfSalt[6]), static_cast(kBsbfSalt[7])); + const __m256i prod = _mm256_mullo_epi32(_mm256_set1_epi32(static_cast(key)), salt); + const __m256i shifts = _mm256_srli_epi32(prod, 27); // top 5 bits -> 0..31 + return _mm256_sllv_epi32(_mm256_set1_epi32(1), shifts); +} +__attribute__((target("avx2"))) bool block_contains_avx2(uint64_t hash, const uint8_t* block) { + const __m256i m = mask_avx2(static_cast(hash)); + const __m256i b = _mm256_loadu_si256(reinterpret_cast(block)); + return _mm256_testc_si256(b, m) != 0; // (~b & m) == 0 -> b contains m +} +__attribute__((target("avx2"))) void insert_avx2(uint32_t* words, uint32_t block, uint32_t key) { + __m256i* p = reinterpret_cast<__m256i*>(words + block * 8); + _mm256_storeu_si256(p, _mm256_or_si256(_mm256_loadu_si256(p), mask_avx2(key))); +} +__attribute__((target("avx2"))) bool find_avx2(const uint32_t* words, uint32_t block, + uint32_t key) { + const __m256i m = mask_avx2(key); + const __m256i b = _mm256_loadu_si256(reinterpret_cast(words + block * 8)); + return _mm256_testc_si256(b, m) != 0; +} +#endif + +} // namespace + +uint64_t bsbf_hash(std::string_view term) { + return XXH64(term.data(), term.size(), /*seed=*/0); +} + +uint32_t bsbf_optimal_num_bytes(uint32_t ndv, double fpp) { + // Parquet OptimalNumOfBits, then >>3 for bytes. + const double m = -8.0 * ndv / std::log(1 - std::pow(fpp, 1.0 / 8)); + uint32_t num_bits; + if (m < 0 || m > static_cast(kBsbfMaxBytes) * 8) { + num_bits = kBsbfMaxBytes << 3; + } else { + num_bits = static_cast(m); + } + if (num_bits < (kBsbfMinBytes << 3)) num_bits = kBsbfMinBytes << 3; + // G16-g: round up to the 32-byte block only, NOT to the next power of 2. + // The block index is fastrange ((h >> 32) * num_blocks >> 32, bsbf.h), + // which is uniform for ANY block count -- the old power-of-2 rounding was + // a sizing convention, not an addressing requirement, and cost up to ~2x + // bitset bytes (e.g. a 31M-term segment: 40.3 MB optimal -> 64 MB rounded) + // while silently over-delivering on fpp. Exact sizing keeps the fpp at + // the kBsbfFpp design point. + const uint32_t block_bits = kBsbfBytesPerBlock << 3; + num_bits = (num_bits + block_bits - 1) / block_bits * block_bits; + if (num_bits > (kBsbfMaxBytes << 3)) num_bits = kBsbfMaxBytes << 3; + return num_bits >> 3; +} + +bool bsbf_block_contains(uint64_t hash, const uint8_t block[kBsbfBytesPerBlock]) { +#if defined(SNII_BSBF_X86) + if (cpu_has_avx2()) return block_contains_avx2(hash, block); +#endif + return block_contains_scalar(hash, block); +} + +Status BsbfBuilder::create(uint32_t ndv, double fpp, BsbfBuilder* out) { + if (out == nullptr) return Status::Error("bsbf: null out"); + if (!(fpp > 0.0 && fpp < 1.0)) + return Status::Error("bsbf: fpp out of (0,1)"); + if (ndv == 0) ndv = 1; + out->num_bytes_ = bsbf_optimal_num_bytes(ndv, fpp); + out->num_blocks_ = out->num_bytes_ / kBsbfBytesPerBlock; + out->ndv_ = ndv; + out->words_.assign(out->num_bytes_ / 4, 0u); + return Status::OK(); +} + +void BsbfBuilder::insert(uint64_t hash) { + const uint32_t block = bsbf_block_index(hash, num_blocks_); + const uint32_t key = static_cast(hash); +#if defined(SNII_BSBF_X86) + if (cpu_has_avx2()) { + insert_avx2(words_.data(), block, key); + return; + } +#endif + insert_scalar(words_.data(), block, key); +} + +bool BsbfBuilder::maybe_contains(uint64_t hash) const { + const uint32_t block = bsbf_block_index(hash, num_blocks_); + const uint32_t key = static_cast(hash); +#if defined(SNII_BSBF_X86) + if (cpu_has_avx2()) return find_avx2(words_.data(), block, key); +#endif + return find_scalar(words_.data(), block, key); +} + +Status BsbfBuilder::serialize(ByteSink* sink) const { + if (sink == nullptr) + return Status::Error("bsbf: null sink"); + if (num_bytes_ == 0) + return Status::Error("bsbf: not built"); + uint8_t hdr[kBsbfHeaderSize] = {0}; + hdr[0] = 'B'; + hdr[1] = 'S'; + hdr[2] = 'B'; + hdr[3] = 'F'; + hdr[4] = 1; // version + hdr[5] = 0; // hash strategy: XXH64 seed 0 + hdr[6] = 0; // index strategy: fastrange + hdr[7] = 0; // pad + store_le32(hdr + 8, num_bytes_); + store_le32(hdr + 12, num_blocks_); + store_le32(hdr + 16, ndv_); + store_le32(hdr + 20, crc32c(Slice(hdr, 20))); // header crc over [0,20) + const uint8_t* bits = reinterpret_cast(words_.data()); + store_le32(hdr + 24, crc32c(Slice(bits, num_bytes_))); // bitset crc + sink->put_bytes(Slice(hdr, kBsbfHeaderSize)); + sink->put_bytes(Slice(bits, num_bytes_)); // contiguous, uncompressed, LE + return Status::OK(); +} + +Status BsbfHeader::parse(Slice h, uint64_t section_base, BsbfHeader* out) { + if (out == nullptr) return Status::Error("bsbf: null out"); + if (h.size() < kBsbfHeaderSize) + return Status::Error("bsbf: short header"); + const uint8_t* p = h.data(); + if (p[0] != 'B' || p[1] != 'S' || p[2] != 'B' || p[3] != 'F') + return Status::Error("bsbf: bad magic"); + if (p[4] != 1) + return Status::Error("bsbf: bad version"); + if (p[5] != 0) + return Status::Error( + "bsbf: unsupported hash strategy"); + if (p[6] != 0) + return Status::Error( + "bsbf: unsupported index strategy"); + if (crc32c(Slice(p, 20)) != load_le32(p + 20)) + return Status::Error( + "bsbf: header crc mismatch"); + const uint32_t nb = load_le32(p + 8); + const uint32_t nblk = load_le32(p + 12); + // G16-g: num_bytes only needs 32-byte block alignment -- the fastrange + // block index is uniform for any block count. Power-of-2 sizes (all + // pre-G16-g filters) remain valid as a subset. + if (nb < kBsbfMinBytes || nb > kBsbfMaxBytes || nb % kBsbfBytesPerBlock != 0) + return Status::Error( + "bsbf: num_bytes out of range or not block-aligned"); + if (nblk != nb / kBsbfBytesPerBlock) + return Status::Error( + "bsbf: num_blocks mismatch"); + out->num_bytes = nb; + out->num_blocks = nblk; + out->bitset_crc = load_le32(p + 24); + out->bitset_base = section_base + kBsbfHeaderSize; + return Status::OK(); +} + +Status bsbf_probe(io::FileReader* reader, const BsbfHeader& header, uint64_t hash, + bool* maybe_present) { + if (reader == nullptr || maybe_present == nullptr) + return Status::Error("bsbf: null arg"); + std::vector blk; + RETURN_IF_ERROR(reader->read_at(header.block_offset(hash), kBsbfBytesPerBlock, &blk)); + if (blk.size() < kBsbfBytesPerBlock) + return Status::Error( + "bsbf: short block read"); + *maybe_present = bsbf_block_contains(hash, blk.data()); + return Status::OK(); +} + +} // namespace doris::snii::format diff --git a/be/src/storage/index/snii/format/bsbf.h b/be/src/storage/index/snii/format/bsbf.h new file mode 100644 index 00000000000000..7877073fedd10b --- /dev/null +++ b/be/src/storage/index/snii/format/bsbf.h @@ -0,0 +1,135 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/encoding/byte_sink.h" +#include "storage/index/snii/io/file_reader.h" + +// Block-split bloom filter (BSBF) -- Apache Parquet split-block spec, with an +// S3-native on-demand single-block probe that none of the reference implementations +// (Apache Parquet, Doris storage, Doris format/parquet) ship. +// +// BIT FORMAT IS PARQUET-CANONICAL (interoperable with Apache Parquet / Doris +// format/parquet for the bitset bytes): +// - 256-bit (32-byte) blocks, 8 bits set per block. +// - key = XXH64(term, seed=0); high 32 bits select the block via FASTRANGE +// `block = ((hash>>32) * num_blocks) >> 32` (no power-of-2 requirement); low 32 +// bits select 8 in-block positions `1 << ((key * SALT[i]) >> 27)`. +// - num_bytes via Parquet OptimalNumOfBytes: power of 2 in [32, 128 MiB]. +// +// SNII WRAPPER (NOT Parquet's variable thrift header): a FIXED 28-byte header, then +// the contiguous, uncompressed, little-endian bitset. Because the header size is a +// constant, the bitset start is a constant offset (`section_base + 28`) and block i +// is at `section_base + 28 + i*32` -- so a single 32-byte block can be range-read on +// demand WITHOUT parsing a variable-length header and WITHOUT loading the whole blob. +namespace doris::snii::format { + +constexpr uint32_t kBsbfBytesPerBlock = 32; // 256-bit block +constexpr uint32_t kBsbfBitsSetPerBlock = 8; // 8 uint32 words / block +constexpr uint32_t kBsbfMinBytes = 32; +constexpr uint32_t kBsbfMaxBytes = 128u * 1024 * 1024; // Parquet kMaximumBloomFilterBytes +constexpr uint32_t kBsbfHeaderSize = 28; // FIXED (constant bitset offset) +// L0/L1 tiering threshold (the "fast-reject absent terms" design): a bsbf section whose total +// size is <= this is loaded WHOLE into the resident reader at open (L0 -> free +// in-memory probe, no per-lookup round); larger filters stay L1 (header-only, probed +// one 32-byte block on demand). 256 KiB fits in a single cloud FileCache block. +constexpr uint32_t kBsbfResidentMaxBytes = 256u * 1024; + +// Canonical Parquet/Doris split-block SALT (8 odd 32-bit constants). +extern const uint32_t kBsbfSalt[kBsbfBitsSetPerBlock]; + +// XXH64(term, seed=0) -- the Parquet-canonical key (NOT XXH3, NOT Doris murmur). +uint64_t bsbf_hash(std::string_view term); + +// Parquet OptimalNumOfBytes(ndv, fpp): power of 2 in [32, 128 MiB]. +uint32_t bsbf_optimal_num_bytes(uint32_t ndv, double fpp); + +// Fastrange block index from a 64-bit hash and the block count. +inline uint32_t bsbf_block_index(uint64_t hash, uint32_t num_blocks) { + return static_cast(((hash >> 32) * num_blocks) >> 32); +} + +// Pure 32-byte-block kernel: does `block` contain the key's 8 bits? SIMD (AVX2) +// accelerated at runtime when available, scalar otherwise. Returns true => the term +// MAY be present (could be a false positive); false => DEFINITELY ABSENT. +bool bsbf_block_contains(uint64_t hash, const uint8_t block[kBsbfBytesPerBlock]); + +// In-memory builder + serializer. +class BsbfBuilder { +public: + BsbfBuilder() = default; + + // Sizes the filter for `ndv` distinct keys at target `fpp`. fpp in (0,1). + static Status create(uint32_t ndv, double fpp, BsbfBuilder* out); + + // Insert a key / term. SIMD-accelerated. + void insert(uint64_t hash); + void insert_term(std::string_view term) { insert(bsbf_hash(term)); } + + // In-memory probe over the resident bitset (build/warm path). SIMD-accelerated. + bool maybe_contains(uint64_t hash) const; + bool maybe_contains_term(std::string_view term) const { + return maybe_contains(bsbf_hash(term)); + } + + // Serialize [28-byte header][contiguous LE bitset] into `sink`. The header carries + // magic/version/hash+index strategy/num_bytes/num_blocks/ndv + header & bitset + // crc32c. The bitset is Parquet-canonical bytes. + Status serialize(ByteSink* sink) const; + + uint32_t num_bytes() const { return num_bytes_; } + uint32_t num_blocks() const { return num_blocks_; } + size_t resident_capacity_bytes() const { return words_.capacity() * sizeof(uint32_t); } + +private: + std::vector words_; // num_bytes_/4, blocks of 8 words + uint32_t num_bytes_ = 0; + uint32_t num_blocks_ = 0; + uint32_t ndv_ = 0; +}; + +// Resident header (28 bytes), parsed once at open. Validates magic/version/crc/bounds. +struct BsbfHeader { + uint32_t num_bytes = 0; + uint32_t num_blocks = 0; + uint32_t bitset_crc = 0; // stored crc32c of the bitset body (for L0 verification) + uint64_t bitset_base = 0; // absolute file offset of block 0 = section_base + 28 + + // Parse a 28-byte header located at `section_base` in the file. The bitset_base + // is set to section_base + kBsbfHeaderSize. + static Status parse(Slice header28, uint64_t section_base, BsbfHeader* out); + + // Absolute file offset of the 32-byte block this hash maps to. + uint64_t block_offset(uint64_t hash) const { + return bitset_base + + static_cast(bsbf_block_index(hash, num_blocks)) * kBsbfBytesPerBlock; + } +}; + +// On-demand probe: read EXACTLY ONE 32-byte block via `reader`, then test. No whole +// blob load, no deep copy. *maybe_present=false means DEFINITELY ABSENT. +Status bsbf_probe(io::FileReader* reader, const BsbfHeader& header, uint64_t hash, + bool* maybe_present); + +} // namespace doris::snii::format diff --git a/be/src/storage/index/snii/format/core_metadata.cpp b/be/src/storage/index/snii/format/core_metadata.cpp new file mode 100644 index 00000000000000..f9007cc96768b3 --- /dev/null +++ b/be/src/storage/index/snii/format/core_metadata.cpp @@ -0,0 +1,285 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/format/core_metadata.h" + +#include +#include +#include +#include + +#include "gen_cpp/snii.pb.h" +#include "storage/index/snii/encoding/byte_source.h" +#include "storage/index/snii/encoding/section_framer.h" + +namespace doris::snii::format { +namespace { + +using segment_v2::inverted_index::CommonGramsCoverage; +using segment_v2::inverted_index::CommonGramsSegmentMetadata; +using segment_v2::inverted_index::PlainTermKeyVersion; +using segment_v2::inverted_index::ScoringCoverage; +using segment_v2::inverted_index::validate_common_grams_segment_metadata; +using segment_v2::inverted_index::validate_snii_scoring_metadata; + +Status corrupted(std::string_view message) { + return Status::Error(message); +} + +Status unsupported(std::string_view message) { + return Status::Error(message); +} + +Status validate_index_config(uint32_t value, IndexConfig* out) { + switch (value) { + case static_cast(IndexConfig::kDocsOnly): + case static_cast(IndexConfig::kDocsPositions): + case static_cast(IndexConfig::kDocsPositionsScoring): + *out = static_cast(value); + return Status::OK(); + default: + return unsupported("core metadata: unsupported index config"); + } +} + +Status validate_posting_policy(uint32_t value, CommonGramsPostingPolicy* out) { + switch (value) { + case 0: + *out = CommonGramsPostingPolicy::kNone; + return Status::OK(); + case 1: + *out = CommonGramsPostingPolicy::kHybridV1; + return Status::OK(); + default: + return unsupported("core metadata: unsupported CommonGrams posting policy"); + } +} + +Status validate_plain_term_key_version(uint32_t value) { + switch (value) { + case static_cast(PlainTermKeyVersion::kLegacyRaw): + case static_cast(PlainTermKeyVersion::kEscapedV1): + case static_cast(PlainTermKeyVersion::kRawNoInternal): + return Status::OK(); + default: + return unsupported("core metadata: unsupported plain-term key version"); + } +} + +Status validate_common_grams_coverage(uint32_t value) { + switch (value) { + case static_cast(CommonGramsCoverage::kNone): + case static_cast(CommonGramsCoverage::kComplete): + case static_cast(CommonGramsCoverage::kMixed): + return Status::OK(); + default: + return unsupported("core metadata: unsupported CommonGrams coverage"); + } +} + +Status validate_scoring_coverage(uint32_t value) { + switch (value) { + case static_cast(ScoringCoverage::kNone): + case static_cast(ScoringCoverage::kComplete): + return Status::OK(); + default: + return unsupported("core metadata: unsupported scoring coverage"); + } +} + +void encode_region_ref(const RegionRef& ref, doris::snii::SniiRegionRefPB* out) { + out->set_offset(ref.offset); + out->set_length(ref.length); +} + +void encode_common_grams(const CommonGramsSegmentMetadata& metadata, + doris::snii::SniiCommonGramsMetadataPB* out) { + out->set_plain_term_key_version(static_cast(metadata.plain_term_key_version)); + out->set_common_grams_coverage(static_cast(metadata.common_grams_coverage)); + out->set_common_grams_semantics_version(metadata.common_grams_semantics_version); + out->set_common_grams_key_version(metadata.common_grams_key_version); + out->set_common_grams_dictionary_identity(metadata.common_grams_dictionary_identity); + out->set_base_analyzer_fingerprint(metadata.base_analyzer_fingerprint); + out->set_common_grams_fingerprint(metadata.common_grams_fingerprint); + out->set_scoring_coverage(static_cast(metadata.scoring_coverage)); + out->set_scoring_stats_version(metadata.scoring_stats_version); + out->set_norm_semantics_version(metadata.norm_semantics_version); + out->set_scoring_doc_count(metadata.scoring_doc_count); + out->set_scoring_token_count(metadata.scoring_token_count); +} + +Status decode_region_ref(const doris::snii::SniiRegionRefPB& input, RegionRef* out) { + if (!input.has_offset() || !input.has_length()) { + return corrupted("core metadata: missing region reference field"); + } + *out = {.offset = input.offset(), .length = input.length()}; + return Status::OK(); +} + +Status decode_common_grams(const doris::snii::SniiCommonGramsMetadataPB& input, + CommonGramsSegmentMetadata* out) { + if (!input.has_plain_term_key_version() || !input.has_common_grams_coverage() || + !input.has_common_grams_semantics_version() || !input.has_common_grams_key_version() || + !input.has_common_grams_dictionary_identity() || !input.has_base_analyzer_fingerprint() || + !input.has_common_grams_fingerprint() || !input.has_scoring_coverage() || + !input.has_scoring_stats_version() || !input.has_norm_semantics_version() || + !input.has_scoring_doc_count() || !input.has_scoring_token_count()) { + return corrupted("core metadata: missing CommonGrams metadata field"); + } + RETURN_IF_ERROR(validate_plain_term_key_version(input.plain_term_key_version())); + RETURN_IF_ERROR(validate_common_grams_coverage(input.common_grams_coverage())); + RETURN_IF_ERROR(validate_scoring_coverage(input.scoring_coverage())); + *out = {.plain_term_key_version = + static_cast(input.plain_term_key_version()), + .common_grams_coverage = + static_cast(input.common_grams_coverage()), + .common_grams_semantics_version = input.common_grams_semantics_version(), + .common_grams_key_version = input.common_grams_key_version(), + .common_grams_dictionary_identity = input.common_grams_dictionary_identity(), + .base_analyzer_fingerprint = input.base_analyzer_fingerprint(), + .common_grams_fingerprint = input.common_grams_fingerprint(), + .scoring_coverage = static_cast(input.scoring_coverage()), + .scoring_stats_version = input.scoring_stats_version(), + .norm_semantics_version = input.norm_semantics_version(), + .scoring_doc_count = input.scoring_doc_count(), + .scoring_token_count = input.scoring_token_count()}; + return validate_common_grams_segment_metadata(*out); +} + +Status decode_core_pb(const doris::snii::SniiCoreMetadataPB& input, CoreMetadata* out) { + if (!input.has_index_config() || !input.has_stats() || !input.has_section_refs()) { + return corrupted("core metadata: missing required field"); + } + RETURN_IF_ERROR(validate_index_config(input.index_config(), &out->index_config)); + + const auto& stats = input.stats(); + if (!stats.has_doc_count() || !stats.has_indexed_doc_count() || !stats.has_term_count() || + !stats.has_sum_total_term_freq() || !stats.has_null_count()) { + return corrupted("core metadata: missing statistics field"); + } + out->stats = {.doc_count = stats.doc_count(), + .indexed_doc_count = stats.indexed_doc_count(), + .term_count = stats.term_count(), + .sum_total_term_freq = stats.sum_total_term_freq(), + .null_count = stats.null_count()}; + + const auto& refs = input.section_refs(); + if (!refs.has_dict_region() || !refs.has_posting_region() || !refs.has_norms() || + !refs.has_null_bitmap() || !refs.has_bsbf()) { + return corrupted("core metadata: missing section reference"); + } + RETURN_IF_ERROR(decode_region_ref(refs.dict_region(), &out->section_refs.dict_region)); + RETURN_IF_ERROR(decode_region_ref(refs.posting_region(), &out->section_refs.posting_region)); + RETURN_IF_ERROR(decode_region_ref(refs.norms(), &out->section_refs.norms)); + RETURN_IF_ERROR(decode_region_ref(refs.null_bitmap(), &out->section_refs.null_bitmap)); + RETURN_IF_ERROR(decode_region_ref(refs.bsbf(), &out->section_refs.bsbf)); + + if (input.has_common_grams()) { + CommonGramsSegmentMetadata common_grams; + RETURN_IF_ERROR(decode_common_grams(input.common_grams(), &common_grams)); + out->common_grams_metadata = std::move(common_grams); + } + + RETURN_IF_ERROR(validate_posting_policy(input.common_grams_posting_policy(), + &out->common_grams_posting_policy)); + if (out->common_grams_posting_policy == CommonGramsPostingPolicy::kHybridV1 && + (!out->common_grams_metadata.has_value() || + out->common_grams_metadata->common_grams_coverage != CommonGramsCoverage::kMixed)) { + return corrupted("core metadata: hybrid policy requires mixed CommonGrams metadata"); + } + const bool has_scoring_tier = out->index_config == IndexConfig::kDocsPositionsScoring; + if (has_scoring_tier) { + if (out->section_refs.norms.length == 0) { + return corrupted("core metadata: scoring index requires a norms region"); + } + } + if (has_scoring_tier || + (out->common_grams_metadata.has_value() && + out->common_grams_metadata->scoring_coverage == ScoringCoverage::kComplete)) { + RETURN_IF_ERROR(validate_snii_scoring_metadata( + out->common_grams_metadata ? &*out->common_grams_metadata : nullptr, + out->stats.doc_count, out->stats.sum_total_term_freq, has_scoring_tier, + has_positions(out->index_config), out->section_refs.norms.length != 0)); + } + return Status::OK(); +} + +} // namespace + +Status encode_core_metadata(const CoreMetadata& metadata, ByteSink* out) { + if (out == nullptr) { + return Status::Error("core metadata: null output"); + } + + doris::snii::SniiCoreMetadataPB core; + core.set_index_config(static_cast(metadata.index_config)); + auto* stats = core.mutable_stats(); + stats->set_doc_count(metadata.stats.doc_count); + stats->set_indexed_doc_count(metadata.stats.indexed_doc_count); + stats->set_term_count(metadata.stats.term_count); + stats->set_sum_total_term_freq(metadata.stats.sum_total_term_freq); + stats->set_null_count(metadata.stats.null_count); + auto* refs = core.mutable_section_refs(); + encode_region_ref(metadata.section_refs.dict_region, refs->mutable_dict_region()); + encode_region_ref(metadata.section_refs.posting_region, refs->mutable_posting_region()); + encode_region_ref(metadata.section_refs.norms, refs->mutable_norms()); + encode_region_ref(metadata.section_refs.null_bitmap, refs->mutable_null_bitmap()); + encode_region_ref(metadata.section_refs.bsbf, refs->mutable_bsbf()); + if (metadata.common_grams_metadata.has_value()) { + encode_common_grams(*metadata.common_grams_metadata, core.mutable_common_grams()); + } + if (metadata.common_grams_posting_policy != CommonGramsPostingPolicy::kNone) { + core.set_common_grams_posting_policy( + static_cast(metadata.common_grams_posting_policy)); + } + + CoreMetadata validated; + RETURN_IF_ERROR(decode_core_pb(core, &validated)); + const size_t size = core.ByteSizeLong(); + if (size > static_cast(std::numeric_limits::max())) { + return corrupted("core metadata: protobuf payload exceeds INT_MAX"); + } + std::string payload(size, '\0'); + if (!core.SerializeToArray(payload.data(), static_cast(size))) { + return corrupted("core metadata: protobuf serialization failed"); + } + SectionFramer::write(*out, static_cast(SectionType::kCoreMetadataPB), Slice(payload)); + return Status::OK(); +} + +Status decode_core_metadata(Slice framed_bytes, CoreMetadata* out) { + if (out == nullptr) { + return Status::Error("core metadata: null output"); + } + *out = {}; + ByteSource source(framed_bytes); + FramedSection section; + RETURN_IF_ERROR(SectionFramer::read(source, §ion)); + if (!source.eof() || section.type != static_cast(SectionType::kCoreMetadataPB)) { + return corrupted("core metadata: invalid frame"); + } + if (section.payload.size() > static_cast(std::numeric_limits::max())) { + return corrupted("core metadata: protobuf payload exceeds INT_MAX"); + } + doris::snii::SniiCoreMetadataPB core; + if (!core.ParseFromArray(section.payload.data(), static_cast(section.payload.size()))) { + return corrupted("core metadata: protobuf parsing failed"); + } + return decode_core_pb(core, out); +} + +} // namespace doris::snii::format diff --git a/be/src/storage/index/snii/format/core_metadata.h b/be/src/storage/index/snii/format/core_metadata.h new file mode 100644 index 00000000000000..ce761901a145ec --- /dev/null +++ b/be/src/storage/index/snii/format/core_metadata.h @@ -0,0 +1,62 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include "common/status.h" +#include "storage/index/inverted/common_grams/common_grams_segment_metadata.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/encoding/byte_sink.h" +#include "storage/index/snii/format/format_constants.h" +#include "storage/index/snii/format/stats_block.h" + +namespace doris::snii::format { + +struct RegionRef { + uint64_t offset = 0; + uint64_t length = 0; +}; + +struct SectionRefs { + RegionRef dict_region; + RegionRef posting_region; + RegionRef norms; + RegionRef null_bitmap; + RegionRef bsbf; +}; + +enum class CommonGramsPostingPolicy : uint8_t { + kNone = 0, + kDocsOnlyV1 = 1, + kHybridV1 = kDocsOnlyV1, +}; + +struct CoreMetadata { + IndexConfig index_config = IndexConfig::kDocsOnly; + StatsBlock stats; + SectionRefs section_refs; + std::optional common_grams_metadata; + CommonGramsPostingPolicy common_grams_posting_policy = CommonGramsPostingPolicy::kNone; +}; + +Status encode_core_metadata(const CoreMetadata& metadata, ByteSink* out); +Status decode_core_metadata(Slice framed_bytes, CoreMetadata* out); + +} // namespace doris::snii::format diff --git a/be/src/storage/index/snii/format/dict_block.cpp b/be/src/storage/index/snii/format/dict_block.cpp new file mode 100644 index 00000000000000..56416924f8abd0 --- /dev/null +++ b/be/src/storage/index/snii/format/dict_block.cpp @@ -0,0 +1,604 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/format/dict_block.h" + +#include +#include + +#include "storage/index/snii/encoding/byte_source.h" +#include "storage/index/snii/encoding/crc32c.h" +#include "storage/index/snii/encoding/varint.h" +#include "storage/index/snii/format/sampled_term_index.h" // std_string_heap_bytes + +namespace doris::snii::format { + +namespace { + +constexpr size_t kFooterBytes = sizeof(uint32_t); // trailing crc32c +constexpr size_t kNAnchorsBytes = sizeof(uint32_t); // n_anchors u32 +constexpr size_t kAnchorOffBytes = sizeof(uint32_t); // per-anchor offset u32 + +size_t estimate_statless_entry_upper_bound(const DictEntry& e, IndexTier tier) { + size_t body = 0; + body += varint_len(static_cast(e.term.size())); + body += varint_len(static_cast(e.term.size())); + body += e.term.size(); + body += 1; // flags + body += varint_len(e.df); + + const bool tier_has_stats = tier >= IndexTier::kT2; + if (e.kind == DictEntryKind::kInline) { + body += varint_len(static_cast(e.frq_bytes.size())) + e.frq_bytes.size(); + body += varint_len(e.inline_dd_disk_len); + body += 1 + varint_len(e.dd_meta.uncomp_len); // win_mode + DD metadata + if (tier_has_stats) { + body += varint_len(e.freq_meta.uncomp_len); + body += varint_len(static_cast(e.prx_bytes.size())) + e.prx_bytes.size(); + } + } else { + body += varint_len(e.frq_off_delta) + varint_len(e.frq_len); + if (e.enc == DictEntryEnc::kWindowed) { + body += varint_len(e.prelude_len) + varint_len(e.frq_docs_len); + } else { + body += varint_len(e.frq_docs_len); + body += 1 + varint_len(e.dd_meta.uncomp_len) + sizeof(uint32_t); + if (tier_has_stats) { + body += varint_len(e.freq_meta.uncomp_len) + sizeof(uint32_t); + } + } + if (tier_has_stats) { + body += varint_len(e.prx_off_delta) + varint_len(e.prx_len); + } + } + return varint_len(static_cast(body)) + body; +} + +// Estimate the encoded upper-bound byte size of one entry (no actual encoding; used by +// estimated_bytes). Take the maximum varint width of each variable-length field plus payload bytes +// to guarantee an upper bound. +size_t estimate_entry_bytes(const DictEntry& e, IndexTier tier, bool term_stats) { + size_t body = 0; + body += varint_len(static_cast(e.term.size())); // prefix_len upper bound + body += varint_len(static_cast(e.term.size())); // suffix_len upper bound + body += e.term.size(); // suffix bytes upper bound + body += 1; // flags + body += 10; // df upper bound + if (term_stats) { + body += 10; // ttf_delta + body += 10; // max_freq + } + if (e.kind == DictEntryKind::kInline) { + body += 10 + e.frq_bytes.size(); + body += 10 + e.prx_bytes.size(); + } else { + body += 10 * 5; // frq_off/frq_len/prelude/prx_off/prx_len upper bound + } + const size_t legacy_estimate = varint_len(static_cast(body)) + body; + if (term_stats) { + return legacy_estimate; + } + return std::max(legacy_estimate, estimate_statless_entry_upper_bound(e, tier)); +} + +} // namespace + +// ---- DictBlockBuilder ---- + +DictBlockBuilder::DictBlockBuilder(IndexTier tier, bool has_positions, uint64_t frq_base, + uint64_t prx_base, uint32_t anchor_interval, bool term_stats) + : tier_(tier), + has_positions_(has_positions), + term_stats_(term_stats), + frq_base_(frq_base), + prx_base_(prx_base), + anchor_interval_(anchor_interval == 0 ? 1 : anchor_interval) {} + +void DictBlockBuilder::add_entry(const DictEntry& entry) { + if (is_anchor(n_entries_)) { + ++n_anchors_; + } + entries_est_ += estimate_entry_bytes(entry, tier_, term_stats_); + entries_.push_back(entry); + ++n_entries_; +} + +void DictBlockBuilder::add_entry(DictEntry&& entry) { + if (is_anchor(n_entries_)) { + ++n_anchors_; + } + // estimate_entry_bytes reads `entry`, so it MUST run before the move below: + // sizing a moved-from (empty) entry would undercount entries_est_ and split + // blocks incorrectly. finish() output is unaffected either way -- it depends + // only on the entries actually queued, not on how they were appended. + entries_est_ += estimate_entry_bytes(entry, tier_, term_stats_); + entries_.push_back(std::move(entry)); + ++n_entries_; +} + +size_t DictBlockBuilder::estimated_bytes() const { + size_t header = varint_len(static_cast(n_entries_)) + 2; // +ver +flags + header += varint_len(frq_base_); + if (has_positions_) { + header += varint_len(prx_base_); + } + const size_t anchors = n_anchors_ * kAnchorOffBytes + kNAnchorsBytes; + return header + entries_est_ + anchors + kFooterBytes; +} + +void DictBlockBuilder::encode_covered(ByteSink* sink) const { + // header. + sink->put_varint64(static_cast(n_entries_)); + sink->put_u8(kDictBlockFormatVer); + sink->put_u8(static_cast((has_positions_ ? dict_block_flags::kHasPositions : 0U) | + (term_stats_ ? 0U : dict_block_flags::kNoTermStats))); + sink->put_varint64(frq_base_); + if (has_positions_) { + sink->put_varint64(prx_base_); + } + + // entries: anchor entries use prev_term="" and record their byte offset within the block. + std::vector anchor_offsets; + anchor_offsets.reserve(n_anchors_); + ByteSink entry_body_scratch; + std::string_view prev; + for (uint32_t i = 0; i < n_entries_; ++i) { + const bool anchor = is_anchor(i); + if (anchor) { + anchor_offsets.push_back(static_cast(sink->size())); + } + const std::string_view prev_term = anchor ? std::string_view {} : prev; + // finish() is void and entry encoding into an in-memory ByteSink cannot fail; + // explicitly discard the (now [[nodiscard]] Status) return. + static_cast(encode_dict_entry(entries_[i], prev_term, tier_, sink, term_stats_, + &entry_body_scratch)); + prev = entries_[i].term; + } + + // anchor_offsets[] + n_anchors. + for (uint32_t off : anchor_offsets) { + sink->put_fixed32(off); + } + sink->put_fixed32(static_cast(anchor_offsets.size())); +} + +void DictBlockBuilder::finish(ByteSink* sink) const { + ByteSink body; // header + entries + anchor_offsets + n_anchors (crc covered region) + encode_covered(&body); + + // Write the entire block (including crc footer) to sink. + sink->put_bytes(body.view()); + sink->put_fixed32(crc32c(body.view())); +} + +std::vector DictBlockBuilder::finish_owned() const { + ByteSink block; + encode_covered(&block); + const uint32_t checksum = crc32c(block.view()); + block.reserve(kFooterBytes); + block.put_fixed32(checksum); + return block.take(); +} + +// ---- DictBlockReader ---- + +namespace { + +// Verify the block length is sufficient and validate the trailing crc; return a Slice of the covered region (excluding crc footer). +Status verify_crc(Slice block, Slice* covered) { + if (block.size() < kFooterBytes + kNAnchorsBytes) { + return Status::Error( + "dict_block: block too short to contain footer"); + } + const size_t covered_len = block.size() - kFooterBytes; + *covered = block.subslice(0, covered_len); + + ByteSource crc_src(block.subslice(covered_len, kFooterBytes)); + uint32_t stored = 0; + RETURN_IF_ERROR(crc_src.get_fixed32(&stored)); + if (crc32c(*covered) != stored) { + return Status::Error( + "dict_block: crc32c checksum mismatch"); + } + return Status::OK(); +} + +// Read and verify that block_flags is consistent with has_positions. +Status check_flags(uint8_t flags, bool has_positions) { + const bool flag_pos = (flags & dict_block_flags::kHasPositions) != 0; + if (flag_pos != has_positions) { + return Status::Error( + "dict_block: has_positions inconsistent with block_flags"); + } + return Status::OK(); +} + +} // namespace + +Status DictBlockReader::open(Slice block, IndexTier tier, bool has_positions, + DictBlockReader* out) { + if (out == nullptr) { + return Status::Error("dict_block: out is null"); + } + *out = DictBlockReader {}; + + // Decode instrumentation seam: one increment per block materialization (the + // CRC verify + anchor parse below, preceded by a zstd decompress for a + // compressed block). A dict-block cache eliminates repeats of exactly this. + testing::note_dict_block_decode(); + + Slice covered; + RETURN_IF_ERROR(verify_crc(block, &covered)); + out->block_ = covered; + out->tier_ = tier; + out->has_positions_ = has_positions; + + // header. + ByteSource src(covered); + uint64_t n_entries = 0; + RETURN_IF_ERROR(src.get_varint64(&n_entries)); + uint8_t ver = 0; + uint8_t flags = 0; + RETURN_IF_ERROR(src.get_u8(&ver)); + RETURN_IF_ERROR(src.get_u8(&flags)); + if (ver != kDictBlockFormatVer) { + return Status::Error( + "dict_block: unsupported entry_format_ver"); + } + RETURN_IF_ERROR(check_flags(flags, has_positions)); + out->term_stats_ = (flags & dict_block_flags::kNoTermStats) == 0; + RETURN_IF_ERROR(src.get_varint64(&out->frq_base_)); + if (has_positions) { + RETURN_IF_ERROR(src.get_varint64(&out->prx_base_)); + } + + out->n_entries_ = static_cast(n_entries); + out->entries_begin_ = src.position(); + + // The anchor table is at the tail of covered: [... anchor_offsets[n] n_anchors(u32)]. + if (covered.size() < kNAnchorsBytes) { + return Status::Error( + "dict_block: missing n_anchors"); + } + ByteSource na_src(covered.subslice(covered.size() - kNAnchorsBytes, kNAnchorsBytes)); + uint32_t n_anchors = 0; + RETURN_IF_ERROR(na_src.get_fixed32(&n_anchors)); + + const size_t anchor_table_bytes = static_cast(n_anchors) * kAnchorOffBytes; + if (covered.size() < kNAnchorsBytes + anchor_table_bytes || + out->entries_begin_ + anchor_table_bytes + kNAnchorsBytes > covered.size()) { + return Status::Error( + "dict_block: anchor table out of range"); + } + const size_t anchor_table_begin = covered.size() - kNAnchorsBytes - anchor_table_bytes; + + ByteSource at_src(covered.subslice(anchor_table_begin, anchor_table_bytes)); + out->anchor_offsets_.resize(n_anchors); + out->anchor_terms_.resize(n_anchors); + for (uint32_t i = 0; i < n_anchors; ++i) { + uint32_t off = 0; + RETURN_IF_ERROR(at_src.get_fixed32(&off)); + if (off >= anchor_table_begin) { + return Status::Error( + "dict_block: anchor offset out of range"); + } + // Anchor offsets must be strictly monotonically increasing, and the first anchor must be exactly the start of the entries region (entry 0 is always an anchor). + // Otherwise scan_from_anchor's segment-length computation seg_end-seg_begin would underflow as size_t and cause an out-of-range read, + // guarding against non-monotonic offset tables with a re-stamped crc (remote on-demand read / cache misalignment scenarios). + if (i == 0) { + if (off != out->entries_begin_) { + return Status::Error( + "dict_block: first anchor offset is not the start of entries"); + } + } else if (off <= out->anchor_offsets_[i - 1]) { + return Status::Error( + "dict_block: anchor offsets are not strictly increasing"); + } + out->anchor_offsets_[i] = off; + // Anchor entries are encoded with prev_term="" and can be decoded independently to retrieve their term. + ByteSource e_src(covered.subslice(off, anchor_table_begin - off)); + DictEntry probe; + RETURN_IF_ERROR(decode_dict_entry(&e_src, std::string_view {}, tier, &probe, + (flags & dict_block_flags::kNoTermStats) == 0)); + out->anchor_terms_[i] = std::move(probe.term); + } + return Status::OK(); +} + +size_t DictBlockReader::heap_bytes() const { + size_t bytes = anchor_offsets_.capacity() * sizeof(uint32_t) + + anchor_terms_.capacity() * sizeof(std::string); + for (const auto& term : anchor_terms_) { + bytes += std_string_heap_bytes(term); + } + return bytes; +} + +bool DictBlockReader::locate_anchor(std::string_view target, size_t* anchor_idx) const { + if (anchor_terms_.empty()) { + return false; + } + if (target < std::string_view(anchor_terms_.front())) { + return false; + } + // The last anchor_term <= target. + size_t lo = 0; + size_t hi = anchor_terms_.size(); // open interval + while (lo + 1 < hi) { + const size_t mid = lo + (hi - lo) / 2; + if (std::string_view(anchor_terms_[mid]) <= target) { + lo = mid; + } else { + hi = mid; + } + } + *anchor_idx = lo; + return true; +} + +Status DictBlockReader::decode_all(std::vector* out) const { + if (out == nullptr) { + return Status::Error("dict_block: out is null"); + } + out->clear(); + out->reserve(n_entries_); + for (size_t a = 0; a < anchor_offsets_.size(); ++a) { + const size_t seg_begin = anchor_offsets_[a]; + const bool is_last = a + 1 == anchor_offsets_.size(); + const size_t seg_end = is_last ? (block_.size() - kNAnchorsBytes - + anchor_offsets_.size() * kAnchorOffBytes) + : anchor_offsets_[a + 1]; + if (seg_end < seg_begin || seg_end > block_.size()) { + return Status::Error( + "dict_block: anchor segment range invalid"); + } + ByteSource src(block_.subslice(seg_begin, seg_end - seg_begin)); + std::string prev; // first entry of a segment is an anchor (prev_term="") + while (!src.eof()) { + DictEntry e; + RETURN_IF_ERROR( + decode_dict_entry(&src, std::string_view(prev), tier_, &e, term_stats_)); + prev = e.term; + out->push_back(std::move(e)); + } + } + if (out->size() != n_entries_) { + return Status::Error( + "dict_block: decoded entry count mismatch"); + } + return Status::OK(); +} + +Status DictBlockReader::scan_from_anchor(size_t anchor_idx, std::string_view target, bool* found, + DictEntry* out) const { + // Byte range of this anchor segment: [anchor_offset, next anchor offset or anchor table start). + const size_t seg_begin = anchor_offsets_[anchor_idx]; + const bool is_last = anchor_idx + 1 == anchor_offsets_.size(); + const size_t seg_end = + is_last ? (block_.size() - kNAnchorsBytes - anchor_offsets_.size() * kAnchorOffBytes) + : anchor_offsets_[anchor_idx + 1]; + + // Fallback: open() has already verified anchor monotonicity; this additionally guards against seg_end block_.size()) { + return Status::Error( + "dict_block: anchor segment range invalid"); + } + ByteSource src(block_.subslice(seg_begin, seg_end - seg_begin)); + std::string prev; // the first entry in the segment is an anchor, prev_term="" + while (!src.eof()) { + // Key-first: decode only the (front-coded) term key, then decide whether + // the body is worth materializing. Non-matching entries skip their body + // entirely -- with anchor_interval=16 that turns ~16 body decodes per + // lookup into 1 (the matched entry) or 0 (a miss). + DictEntry e; + size_t body_start = 0; + uint64_t entry_total = 0; + RETURN_IF_ERROR( + decode_dict_entry_key(&src, std::string_view(prev), &e, &body_start, &entry_total)); + if (e.term == target) { + RETURN_IF_ERROR( + decode_dict_entry_rest(&src, tier_, body_start, entry_total, &e, term_stats_)); + *found = true; + *out = std::move(e); + return Status::OK(); + } + if (std::string_view(e.term) > target) { + *found = false; // already past target; entries are sorted so it does not exist + return Status::OK(); + } + // Before target: skip the body but keep the key as the front-coding base. + RETURN_IF_ERROR(skip_dict_entry_body(&src, body_start, entry_total)); + prev = std::move(e.term); + } + *found = false; + return Status::OK(); +} + +Status DictBlockReader::find_term(std::string_view target, bool* found, DictEntry* out) const { + if (found == nullptr || out == nullptr) { + return Status::Error("dict_block: found / out is null"); + } + *found = false; + size_t anchor_idx = 0; + if (!locate_anchor(target, &anchor_idx)) { + return Status::OK(); + } + return scan_from_anchor(anchor_idx, target, found, out); +} + +Status DictBlockReader::visit_prefix_range(std::string_view prefix, + const std::function& accept_key, + const std::function& on_hit, + bool* prefix_exhausted) const { + if (!on_hit || prefix_exhausted == nullptr) { + return Status::Error( + "dict_block: null visit_prefix_range args"); + } + *prefix_exhausted = false; + if (anchor_offsets_.empty()) { + return Status::OK(); // empty block: nothing to enumerate + } + + // Anchor-jump: start at the anchor segment that may contain prefix. Earlier + // segments hold only terms < prefix (every term is < the next anchor term + // <= prefix), so they are skipped without any decode. An empty prefix or one + // sorting before the first anchor starts at anchor 0. + size_t anchor_idx = 0; + if (!prefix.empty()) { + locate_anchor(prefix, &anchor_idx); // false leaves anchor_idx at 0 + } + + // Scan from the chosen anchor to the end of the entries region: the prefix + // range may span several anchor segments. Anchor entries are encoded with + // prefix_len=0, so a single running `prev` reconstructs every term correctly + // even as the scan crosses segment boundaries. + const size_t seg_begin = anchor_offsets_[anchor_idx]; + const size_t entries_end = + block_.size() - kNAnchorsBytes - anchor_offsets_.size() * kAnchorOffBytes; + if (entries_end < seg_begin || entries_end > block_.size()) { + return Status::Error( + "dict_block: entries region range invalid"); + } + + ByteSource src(block_.subslice(seg_begin, entries_end - seg_begin)); + std::string prev; // the chosen anchor segment starts at an anchor (prev="") + while (!src.eof()) { + DictEntry e; + size_t body_start = 0; + uint64_t entry_total = 0; + RETURN_IF_ERROR( + decode_dict_entry_key(&src, std::string_view(prev), &e, &body_start, &entry_total)); + const std::string_view t(e.term); + if (t < prefix) { + // Still before the range (only reachable inside the anchor segment + // that straddles prefix): skip the body, keep the front-coding base. + RETURN_IF_ERROR(skip_dict_entry_body(&src, body_start, entry_total)); + prev = std::move(e.term); + continue; + } + const bool has_prefix = t.size() >= prefix.size() && t.starts_with(prefix); + if (!has_prefix) { + *prefix_exhausted = true; // sorted: no further matches here or later + return Status::OK(); + } + if (accept_key && !accept_key(t)) { + // Key-only rejection: never pay for this entry's body. + RETURN_IF_ERROR(skip_dict_entry_body(&src, body_start, entry_total)); + prev = std::move(e.term); + continue; + } + // Accepted: materialize the body and hand the entry to the visitor. + RETURN_IF_ERROR( + decode_dict_entry_rest(&src, tier_, body_start, entry_total, &e, term_stats_)); + prev = e.term; // copy the key before the entry is moved into on_hit + bool stop = false; + RETURN_IF_ERROR(on_hit(std::move(e), &stop)); + if (stop) { + return Status::OK(); + } + } + return Status::OK(); +} + +Status DictBlockReader::visit_term_range(std::string_view lower_inclusive, + std::optional upper_exclusive, + const std::function& accept_key, + const std::function& on_hit, + bool* range_exhausted) const { + if (!on_hit || range_exhausted == nullptr) { + return Status::Error( + "dict_block: null visit_term_range args"); + } + *range_exhausted = false; + if (upper_exclusive.has_value() && *upper_exclusive <= lower_inclusive) { + *range_exhausted = true; + return Status::OK(); + } + if (anchor_offsets_.empty()) { + return Status::OK(); + } + + size_t anchor_idx = 0; + if (!lower_inclusive.empty()) { + locate_anchor(lower_inclusive, &anchor_idx); + } + const size_t seg_begin = anchor_offsets_[anchor_idx]; + const size_t entries_end = + block_.size() - kNAnchorsBytes - anchor_offsets_.size() * kAnchorOffBytes; + if (entries_end < seg_begin || entries_end > block_.size()) { + return Status::Error( + "dict_block: entries region range invalid"); + } + + ByteSource src(block_.subslice(seg_begin, entries_end - seg_begin)); + std::string prev; + while (!src.eof()) { + DictEntry entry; + size_t body_start = 0; + uint64_t entry_total = 0; + RETURN_IF_ERROR(decode_dict_entry_key(&src, std::string_view(prev), &entry, &body_start, + &entry_total)); + const std::string_view term(entry.term); + if (term < lower_inclusive) { + RETURN_IF_ERROR(skip_dict_entry_body(&src, body_start, entry_total)); + prev = std::move(entry.term); + continue; + } + if (upper_exclusive.has_value() && term >= *upper_exclusive) { + *range_exhausted = true; + return Status::OK(); + } + if (accept_key && !accept_key(term)) { + RETURN_IF_ERROR(skip_dict_entry_body(&src, body_start, entry_total)); + prev = std::move(entry.term); + continue; + } + RETURN_IF_ERROR( + decode_dict_entry_rest(&src, tier_, body_start, entry_total, &entry, term_stats_)); + prev = entry.term; + bool stop = false; + RETURN_IF_ERROR(on_hit(std::move(entry), &stop)); + if (stop) { + return Status::OK(); + } + } + return Status::OK(); +} + +} // namespace doris::snii::format + +namespace doris::snii::testing { +namespace { +std::atomic& dict_decode_atomic() { + static std::atomic counter {0}; + return counter; +} +} // namespace + +uint64_t dict_decode_counter() { + return dict_decode_atomic().load(std::memory_order_relaxed); +} + +void reset_dict_decode_counter() { + dict_decode_atomic().store(0, std::memory_order_relaxed); +} + +void note_dict_block_decode() { + dict_decode_atomic().fetch_add(1, std::memory_order_relaxed); +} + +} // namespace doris::snii::testing diff --git a/be/src/storage/index/snii/format/dict_block.h b/be/src/storage/index/snii/format/dict_block.h new file mode 100644 index 00000000000000..7a299a1f162f8e --- /dev/null +++ b/be/src/storage/index/snii/format/dict_block.h @@ -0,0 +1,232 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/encoding/byte_sink.h" +#include "storage/index/snii/format/dict_entry.h" +#include "storage/index/snii/format/format_constants.h" + +// DICT block —— a positioning unit mapping term → postings read plan, and also +// the unit for remote on-demand fetching, caching, and CRC checksum +// verification (see docs/design/SNII-design-spec.source.md "DICT block" and +// "dict lookup flow summary" sections). +// +// Byte layout (strictly implemented; multi-byte fixed-width fields are +// little-endian, variable-length integers use LEB128): +// header: +// n_entries varint +// entry_format_ver u8 # = kDictBlockFormatVer +// block_flags u8 # bit0 = has_positions (consistency check +// against the value passed to reader) frq_base varint64 prx_base +// varint64 # present only when has_positions is set +// entries[n_entries] # variable-length DictEntry, front-coded in +// lexicographic order anchor_offsets[n_anchors] # u32 * n_anchors, byte +// offset of each anchor entry within the block n_anchors u32 crc32c +// u32 # covers [header .. n_anchors], detects corruption (sole CRC +// layer) +// +// Anchor rule: every anchor_interval entries, one "term anchor" is forced — +// that entry is encoded with prev_term="" (prefix_len=0, storing the full +// term), and its byte offset is recorded in anchor_offsets; non-anchor entries +// use the preceding entry's term as prev_term for front coding. The reader can +// start from any anchor and scan independently without needing earlier terms, +// enabling anchor binary search + local scan for exact term lookup. +namespace doris::snii::format { + +// DICT block entry_format_ver: self-describing version of the DictEntry +// encoding. Reader rejects a mismatch so a query-only run cannot silently read +// an older dict-entry layout as the current one. +inline constexpr uint8_t kDictBlockFormatVer = 2; + +// block_flags bit definitions. +namespace dict_block_flags { +inline constexpr uint8_t kHasPositions = 1U << 0; // whether to write prx_base / .prx fields +// G16-f: entries omit the ttf_delta/max_freq varints (freq-dropped index -- +// the stats serve only BM25 scoring). Self-describing per block; absent on +// pre-G16-f blocks, whose entries always carry the stats on tier>=T2. +inline constexpr uint8_t kNoTermStats = 1U << 1; +// bit1-7 reserved +} // namespace dict_block_flags + +// DICT block writer: entries are added in lexicographic order via add_entry; +// internally determines anchors and accumulates size estimates, and on finish +// serializes header + entries + anchor table + CRC in one pass. The front-coding +// base is rebuilt from a local prev inside finish(), so no prev_term is retained +// as builder state. +class DictBlockBuilder { +public: + DictBlockBuilder(IndexTier tier, bool has_positions, uint64_t frq_base, uint64_t prx_base, + uint32_t anchor_interval = 16, bool term_stats = true); + + // Append one entry (caller must guarantee lexicographic term order). + // Internally decides whether it becomes an anchor. The copy overload is kept + // for callers that must retain their entry afterwards (materialized fallback, + // tests); the move overload avoids the per-term DictEntry copy -- which for an + // inline entry is two std::vector heap allocations plus the term + // copy -- on the SPIMI build path. + void add_entry(const DictEntry& entry); + void add_entry(DictEntry&& entry); + + // Upper-bound estimate of the serialized size of the current block (including + // header + entries + anchor table + CRC footer), used by the upper layer to + // decide when to cut a new block based on target_dict_block_bytes. + size_t estimated_bytes() const; + + // Number of entries. + uint32_t n_entries() const { return n_entries_; } + + // Serialize the entire block and append it to sink. + void finish(ByteSink* sink) const; + + // Serialize the entire block into an owned buffer. This avoids copying the + // CRC-covered bytes when the caller needs ownership of the complete block. + std::vector finish_owned() const; + +private: + bool is_anchor(uint32_t index) const { return index % anchor_interval_ == 0; } + void encode_covered(ByteSink* sink) const; + + IndexTier tier_; + bool has_positions_; + bool term_stats_ = true; // false: entries omit ttf/max_freq (kNoTermStats) + uint64_t frq_base_; + uint64_t prx_base_; + uint32_t anchor_interval_; + + uint32_t n_entries_ = 0; + std::vector entries_; + size_t entries_est_ = 0; // accumulated byte estimate for the entries section + size_t n_anchors_ = 0; // number of anchors +}; + +// DICT block reader: on open, verifies the CRC and parses the header / anchor +// table; find_term uses anchor binary search + local scan to locate a +// DictEntry. Holds a byte view of the block (non-owning); lifetime is managed +// by the caller. +class DictBlockReader { +public: + DictBlockReader() = default; + + // Parse and verify the entire block. CRC mismatch / truncation / invalid + // structure → Corruption; has_positions in the header inconsistent with the + // supplied argument → InvalidArgument. + static Status open(Slice block, IndexTier tier, bool has_positions, DictBlockReader* out); + + // Anchor binary search + local scan to locate target. Hit → *found=true and + // *out is filled; miss (including out-of-range, gap) → *found=false. + // Structural error → non-OK Status. + Status find_term(std::string_view target, bool* found, DictEntry* out) const; + + // Decodes EVERY entry in the block in lexicographic order into *out (each a + // self-contained DictEntry, owning its term). Used for ordered term + // enumeration (prefix / range scans). Resets the front-coding base at each + // anchor segment. Retained as the golden reference; the prefix path now + // streams via visit_prefix_range. + Status decode_all(std::vector* out) const; + + // Streams the entries of this block whose term lies in [prefix, prefix+) in + // lexicographic order, materializing only the bodies a caller keeps (T07): + // 1) anchor-jump to the anchor segment containing prefix (segments whose + // terms are all < prefix are skipped without any decode); an empty + // prefix or one before the first anchor starts at anchor 0; + // 2) within range, decode each entry's term key only; term < prefix skips + // the body and continues; a term that leaves the prefix range sets + // *prefix_exhausted=true and ends the scan (sorted order guarantees no + // further matches here or in later blocks); + // 3) accept_key(term)==false skips the body (lets callers push a key-only + // predicate down so a non-match never pays for its body); + // 4) only accepted entries have their body decoded and are handed to + // on_hit, which may request an early stop via *stop. + // accept_key may be empty (treated as accept-all). prefix_exhausted is set + // false on entry and true only when a term past the range is seen. + Status visit_prefix_range(std::string_view prefix, + const std::function& accept_key, + const std::function& on_hit, + bool* prefix_exhausted) const; + + // Streams entries in [lower_inclusive, upper_exclusive). A missing upper + // bound scans to the end of the block. Like visit_prefix_range, this starts + // from the nearest anchor and only decodes accepted entry bodies. + Status visit_term_range(std::string_view lower_inclusive, + std::optional upper_exclusive, + const std::function& accept_key, + const std::function& on_hit, + bool* range_exhausted) const; + + uint64_t frq_base() const { return frq_base_; } + uint64_t prx_base() const { return prx_base_; } + uint32_t n_entries() const { return n_entries_; } + + // Resident heap held beyond sizeof(*this): the anchor_offsets_ / anchor_terms_ + // vector buffers plus each non-SSO anchor term's heap allocation. block_ is a + // NON-owning view and is deliberately NOT counted here -- its owning buffer is + // charged by the caller (the resident block's `bytes` vector, or a + // request-scoped decoded block). Summed into + // LogicalIndexReader::memory_usage() per resident block. + size_t heap_bytes() const; + +private: + // Sequentially scan from anchor anchor_idx to the end of that anchor segment, + // searching for target. + Status scan_from_anchor(size_t anchor_idx, std::string_view target, bool* found, + DictEntry* out) const; + + // Find the last anchor index where first_term(anchor) <= target; return false + // if none exists. + bool locate_anchor(std::string_view target, size_t* anchor_idx) const; + + Slice block_; // [header .. crc) full block view + IndexTier tier_ = IndexTier::kT1; + bool has_positions_ = false; + bool term_stats_ = true; // from block flags (kNoTermStats absent => true) + uint64_t frq_base_ = 0; + uint64_t prx_base_ = 0; + uint32_t n_entries_ = 0; + + size_t entries_begin_ = 0; // absolute offset of the start of the entries section + std::vector anchor_offsets_; // byte offset within the block for each anchor entry + std::vector + anchor_terms_; // full term of each anchor entry (used for binary search) +}; + +} // namespace doris::snii::format + +// Test-only instrumentation seam. dict_decode_counter() returns a process-global +// count of DICT block decodes performed by DictBlockReader::open -- i.e. the +// optional zstd decompress + CRC verify + anchor parse that turns on-disk block +// bytes into a usable reader. This is precisely the unit a dict-block cache +// eliminates on repeat, so tests assert dict_decode_counter() == unique_blocks. +// In production DICT blocks are zstd-compressed, so this equals the zstd +// decompress count. Counters use relaxed atomics; reset between tests. +namespace doris::snii::testing { + +uint64_t dict_decode_counter(); +void reset_dict_decode_counter(); +void note_dict_block_decode(); + +} // namespace doris::snii::testing diff --git a/be/src/storage/index/snii/format/dict_block_directory.cpp b/be/src/storage/index/snii/format/dict_block_directory.cpp new file mode 100644 index 00000000000000..8801418f8a3c6e --- /dev/null +++ b/be/src/storage/index/snii/format/dict_block_directory.cpp @@ -0,0 +1,131 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/format/dict_block_directory.h" + +#include + +#include "storage/index/snii/encoding/byte_source.h" +#include "storage/index/snii/encoding/section_framer.h" +#include "storage/index/snii/format/format_constants.h" + +namespace doris::snii::format { + +namespace { + +// Each block_ref has a fixed field order; reuse ByteSink varint/fixed primitives — do not hand-craft bytes manually. +// uncomp_len trails only when the kZstd flag is set, so uncompressed-block +// directories keep their compact (v1-identical) per-ref byte layout. +void encode_ref(const BlockRef& ref, ByteSink* payload) { + payload->put_varint64(ref.offset); + payload->put_varint64(ref.length); + payload->put_varint32(ref.n_entries); + payload->put_u8(ref.flags); + payload->put_fixed32(ref.checksum); + if (ref.flags & block_ref_flags::kZstd) payload->put_varint64(ref.uncomp_len); +} + +Status decode_ref(ByteSource* ps, BlockRef* ref) { + RETURN_IF_ERROR(ps->get_varint64(&ref->offset)); + RETURN_IF_ERROR(ps->get_varint64(&ref->length)); + RETURN_IF_ERROR(ps->get_varint32(&ref->n_entries)); + RETURN_IF_ERROR(ps->get_u8(&ref->flags)); + RETURN_IF_ERROR(ps->get_fixed32(&ref->checksum)); + if (ref->flags & block_ref_flags::kZstd) { + RETURN_IF_ERROR(ps->get_varint64(&ref->uncomp_len)); + } + return Status::OK(); +} + +Status decode_payload(Slice payload, std::vector* refs) { + ByteSource ps(payload); + uint32_t n_blocks = 0; + RETURN_IF_ERROR(ps.get_varint32(&n_blocks)); + // Guard against a corrupted, inflated count from untrusted bytes: each BlockRef + // needs >= 8 bytes (flags u8 + checksum u32 + >= 1 byte for each of 3 varints), + // so cap before reserve to avoid a huge allocation. + constexpr size_t kMinRefBytes = 8; + if (n_blocks > ps.remaining() / kMinRefBytes) { + return Status::Error( + "dict_block_directory: n_blocks exceeds payload capacity"); + } + refs->clear(); + refs->reserve(n_blocks); + uint64_t previous_end = 0; + for (uint32_t i = 0; i < n_blocks; ++i) { + BlockRef ref {}; + RETURN_IF_ERROR(decode_ref(&ps, &ref)); + if (ref.length == 0) { + return Status::Error( + "dict_block_directory: zero-length block"); + } + if (ref.length > std::numeric_limits::max() - ref.offset) { + return Status::Error( + "dict_block_directory: block range end overflow"); + } + const uint64_t block_end = ref.offset + ref.length; + if (i != 0 && ref.offset < previous_end) { + return Status::Error( + "dict_block_directory: blocks overlap or are out of physical order"); + } + previous_end = block_end; + refs->push_back(ref); + } + if (!ps.eof()) { + return Status::Error( + "dict_block_directory: trailing bytes in payload"); + } + return Status::OK(); +} + +} // namespace + +void DictBlockDirectoryBuilder::finish(ByteSink* sink) const { + ByteSink payload; + payload.put_varint32(static_cast(refs_.size())); + for (const auto& ref : refs_) { + encode_ref(ref, &payload); + } + SectionFramer::write(*sink, static_cast(SectionType::kDictBlockDirectory), + payload.view()); +} + +Status DictBlockDirectoryReader::open(Slice section, DictBlockDirectoryReader* out) { + ByteSource src(section); + FramedSection sec; + RETURN_IF_ERROR(SectionFramer::read(src, &sec)); + if (!src.eof()) { + return Status::Error( + "dict_block_directory: trailing framed section bytes"); + } + if (sec.type != static_cast(SectionType::kDictBlockDirectory)) { + return Status::Error( + "dict_block_directory: unexpected section type"); + } + return decode_payload(sec.payload, &out->refs_); +} + +Status DictBlockDirectoryReader::get(uint32_t ordinal, BlockRef* out) const { + if (ordinal >= refs_.size()) { + return Status::Error( + "dict_block_directory: ordinal out of range"); + } + *out = refs_[ordinal]; + return Status::OK(); +} + +} // namespace doris::snii::format diff --git a/be/src/storage/index/snii/format/dict_block_directory.h b/be/src/storage/index/snii/format/dict_block_directory.h new file mode 100644 index 00000000000000..909b32f29d1374 --- /dev/null +++ b/be/src/storage/index/snii/format/dict_block_directory.h @@ -0,0 +1,95 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/encoding/byte_sink.h" + +namespace doris::snii::format { + +// BlockRef.flags bit definitions. +namespace block_ref_flags { +// bit0: the on-disk block bytes are zstd(uncompressed_block). When set, the +// directory also stores uncomp_len, and the reader zstd-decompresses the fetched +// [offset, offset+length) range to uncomp_len before parsing the dict block. The +// block-level crc32c (and BlockRef.checksum) cover the UNCOMPRESSED bytes, so a +// zstd block shrinks the bytes fetched from S3 while keeping the same integrity +// guarantees after decompression in RAM. +inline constexpr uint8_t kZstd = 1u << 0; +} // namespace block_ref_flags + +// Physical location and checksum info for a single DICT block. Aligned with SampledTermIndex by ordinal: +// SampledTermIndex[i]'s first_term corresponds to DictBlockDirectory[i] (see design spec +// "sampled dict index"). The read path issues a single range read over [offset, offset+length). +struct BlockRef { + uint64_t offset = 0; // absolute byte offset of the block within the container + uint64_t length = 0; // ON-DISK byte length of the block (compressed when kZstd) + uint32_t n_entries = 0; // number of DictEntry records within this block + uint8_t flags = 0; // block-level flags (block_ref_flags::*) + uint32_t checksum = 0; // crc32c of the block's UNCOMPRESSED content (verified after read) + uint64_t uncomp_len = 0; // uncompressed block byte length (stored only when kZstd set) +}; + +// DICT block directory: block ordinal → physical location mapping. +// +// on-disk layout (framed by SectionFramer with a unified type+len+crc32c wrapper): +// [u8 type=kDictBlockDirectory][varint64 payload_len][payload][fixed32 crc32c] +// payload = varint32 n_blocks +// then n_blocks × block_ref{ +// varint64 offset, varint64 length, varint32 n_entries, +// u8 flags, fixed32 checksum } +// Section-level crc detects truncation/corruption; block_ref.checksum is the per-block crc. +class DictBlockDirectoryBuilder { +public: + void add(const BlockRef& ref) { refs_.push_back(ref); } + + // Encodes as a kDictBlockDirectory framed section (with embedded crc32c) and appends to sink. + void finish(ByteSink* sink) const; + +private: + std::vector refs_; +}; + +// Reads and verifies a kDictBlockDirectory framed section; provides ordinal → BlockRef lookup. +// After parsing, all block_refs reside in the reader (entering the searcher cache along with meta). +class DictBlockDirectoryReader { +public: + // Verifies the section crc and deserializes all block_refs. + // crc mismatch / truncation / trailing bytes → kCorruption; wrong section type → kInvalidArgument. + static Status open(Slice section, DictBlockDirectoryReader* out); + + uint32_t n_blocks() const { return static_cast(refs_.size()); } + + // Resident heap held beyond sizeof(*this): the refs_ vector buffer. BlockRef + // is trivially copyable (no per-element heap), so the vector buffer is the + // whole charge. Summed into LogicalIndexReader::memory_usage(). + size_t heap_bytes() const { return refs_.capacity() * sizeof(BlockRef); } + + // Returns the ordinal-th block_ref; ordinal >= n_blocks → kNotFound. + Status get(uint32_t ordinal, BlockRef* out) const; + +private: + std::vector refs_; +}; + +} // namespace doris::snii::format diff --git a/be/src/storage/index/snii/format/dict_entry.cpp b/be/src/storage/index/snii/format/dict_entry.cpp new file mode 100644 index 00000000000000..e9ee72912f461b --- /dev/null +++ b/be/src/storage/index/snii/format/dict_entry.cpp @@ -0,0 +1,383 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/format/dict_entry.h" + +#include +#include + +#include "common/check.h" +#include "storage/index/snii/common/slice.h" + +namespace doris::snii::format { + +namespace { + +// Body-decode counter seam (T07). Incremented once per decode_dict_entry_rest +// call so tests can assert a key-first scan only materializes the bodies it +// actually produces. Relaxed atomic: read by tests only, never a sync point. +std::atomic& body_decode_atomic() { + static std::atomic counter {0}; + return counter; +} + +// Pure-function assembly / parsing of flags bits; avoids a long inline if-else +// chain. +uint8_t pack_flags(const DictEntry& e) { + uint8_t f = 0; + if (e.kind == DictEntryKind::kInline) f |= dict_flags::kKind; + if (e.enc == DictEntryEnc::kWindowed) f |= dict_flags::kEnc; + if (e.has_sb) f |= dict_flags::kHasSb; + // bit3 has_champion / bit4 offsets_ref are always 0 in v1. + return f; +} + +void apply_flags(uint8_t f, DictEntry* e) { + e->kind = (f & dict_flags::kKind) ? DictEntryKind::kInline : DictEntryKind::kPodRef; + e->enc = (f & dict_flags::kEnc) ? DictEntryEnc::kWindowed : DictEntryEnc::kSlim; + e->has_sb = (f & dict_flags::kHasSb) != 0; +} + +// Length of the longest common prefix between term and prev_term. +uint32_t common_prefix_len(std::string_view term, std::string_view prev) { + uint32_t n = 0; + const uint32_t lim = static_cast(std::min(term.size(), prev.size())); + while (n < lim && term[n] == prev[n]) ++n; + return n; +} + +bool tier_has_stats(IndexTier tier) { + return tier >= IndexTier::kT2; +} + +// ---- Encode entry body (excluding entry_len and trailing crc) ---- + +void write_term_key(const DictEntry& e, std::string_view prev, ByteSink* sink) { + const uint32_t prefix = common_prefix_len(e.term, prev); + const std::string_view suffix = std::string_view(e.term).substr(prefix); + sink->put_varint32(prefix); + sink->put_varint32(static_cast(suffix.size())); + sink->put_bytes(Slice(suffix)); +} + +void write_stats(const DictEntry& e, IndexTier tier, bool term_stats, ByteSink* sink) { + sink->put_varint32(e.df); + // G16-f: term_stats == false (freq-dropped index, block header flag) omits + // the ttf_delta/max_freq varints -- BM25-only fields, dead without freq. + if (!tier_has_stats(tier) || !term_stats) return; + sink->put_varint64(e.ttf_delta); + sink->put_varint64(e.max_freq); +} + +// Per-window codec mode byte shared by slim/inline single-window regions. +uint8_t pack_win_mode(const DictEntry& e) { + uint8_t mode = 0; + if (e.dd_meta.zstd) mode |= 1u << 0; // dd_zstd + if (e.freq_meta.zstd) mode |= 1u << 1; // freq_zstd + return mode; +} + +// Writes the slim/inline region codec metadata (dd always; freq when tier>=T2). +// store_crc=false (INLINE entries, format v2) omits the redundant per-region +// crc32c: the inline bytes already sit inside the dict block, whose own +// block-level crc32c covers them. POD-ref entries pass store_crc=true (their +// regions live in the separately-fetched .frq POD, uncovered by the block crc). +void write_region_meta(const DictEntry& e, IndexTier tier, bool store_crc, ByteSink* sink) { + sink->put_u8(pack_win_mode(e)); + sink->put_varint64(e.dd_meta.uncomp_len); + if (store_crc) sink->put_fixed32(e.dd_meta.crc); + if (!tier_has_stats(tier)) return; + sink->put_varint64(e.freq_meta.uncomp_len); + if (store_crc) sink->put_fixed32(e.freq_meta.crc); +} + +void write_pod_ref(const DictEntry& e, IndexTier tier, ByteSink* sink) { + sink->put_varint64(e.frq_off_delta); + sink->put_varint64(e.frq_len); + if (e.enc == DictEntryEnc::kWindowed) { + sink->put_varint64(e.prelude_len); + sink->put_varint64(e.frq_docs_len); + } else { + sink->put_varint64(e.frq_docs_len); // slim pod_ref: dd region on-disk length + // POD-ref regions live in the .frq POD (not covered by the block crc): keep + // crc. + write_region_meta(e, tier, /*store_crc=*/true, sink); + } + if (!tier_has_stats(tier)) return; + sink->put_varint64(e.prx_off_delta); + sink->put_varint64(e.prx_len); +} + +void write_inline(const DictEntry& e, IndexTier tier, ByteSink* sink) { + sink->put_varint64(static_cast(e.frq_bytes.size())); + sink->put_bytes(Slice(e.frq_bytes)); + sink->put_varint64(e.inline_dd_disk_len); + // INLINE bytes are covered by the dict block crc32c: omit the redundant + // per-region crc. + write_region_meta(e, tier, /*store_crc=*/false, sink); + if (!tier_has_stats(tier)) return; + sink->put_varint64(static_cast(e.prx_bytes.size())); + sink->put_bytes(Slice(e.prx_bytes)); +} + +void write_body(const DictEntry& e, std::string_view prev, IndexTier tier, bool term_stats, + ByteSink* sink) { + write_term_key(e, prev, sink); + sink->put_u8(pack_flags(e)); + write_stats(e, tier, term_stats, sink); + if (e.kind == DictEntryKind::kInline) { + write_inline(e, tier, sink); + } else { + write_pod_ref(e, tier, sink); + } +} + +// ---- Decode entry body ---- + +Status read_term_key(ByteSource* src, std::string_view prev, DictEntry* out) { + uint32_t prefix = 0; + uint32_t suffix_len = 0; + RETURN_IF_ERROR(src->get_varint32(&prefix)); + RETURN_IF_ERROR(src->get_varint32(&suffix_len)); + if (prefix > prev.size()) { + return Status::Error( + "dict_entry: prefix_len exceeds prev_term length"); + } + Slice suffix; + RETURN_IF_ERROR(src->get_bytes(suffix_len, &suffix)); + out->term.assign(prev.substr(0, prefix)); + out->term.append(reinterpret_cast(suffix.data()), suffix.size()); + return Status::OK(); +} + +Status read_stats(ByteSource* src, IndexTier tier, bool term_stats, DictEntry* out) { + RETURN_IF_ERROR(src->get_varint32(&out->df)); + out->term_stats_present = tier_has_stats(tier) && term_stats; + if (!out->term_stats_present) return Status::OK(); + RETURN_IF_ERROR(src->get_varint64(&out->ttf_delta)); + RETURN_IF_ERROR(src->get_varint64(&out->max_freq)); + return Status::OK(); +} + +// Reads the slim/inline region codec metadata (mode/uncomp/[crc]) and fills the +// dd/freq region disk_len from the supplied total/split lengths. has_crc=false +// (INLINE entries, format v2) means no per-region crc was stored: the on-disk +// crc field is absent and region decode must skip crc verification (verify_crc= +// false) since the dict block's own crc32c already covers the inline bytes. +Status read_region_meta(ByteSource* src, IndexTier tier, bool has_crc, uint64_t dd_disk_len, + uint64_t freq_disk_len, DictEntry* out) { + uint8_t mode = 0; + RETURN_IF_ERROR(src->get_u8(&mode)); + if ((mode & ~0x3u) != 0) { + return Status::Error( + "dict_entry: unknown win_mode bits"); + } + out->dd_meta.zstd = (mode & (1u << 0)) != 0; + out->dd_meta.disk_len = dd_disk_len; + out->dd_meta.verify_crc = has_crc; + RETURN_IF_ERROR(src->get_varint64(&out->dd_meta.uncomp_len)); + if (has_crc) RETURN_IF_ERROR(src->get_fixed32(&out->dd_meta.crc)); + if (!tier_has_stats(tier)) { + if (mode & (1u << 1)) { + return Status::Error( + "dict_entry: freq mode set without freq tier"); + } + return Status::OK(); + } + out->freq_meta.zstd = (mode & (1u << 1)) != 0; + out->freq_meta.disk_len = freq_disk_len; + out->freq_meta.verify_crc = has_crc; + RETURN_IF_ERROR(src->get_varint64(&out->freq_meta.uncomp_len)); + if (has_crc) RETURN_IF_ERROR(src->get_fixed32(&out->freq_meta.crc)); + return Status::OK(); +} + +Status read_pod_ref(ByteSource* src, IndexTier tier, DictEntry* out) { + RETURN_IF_ERROR(src->get_varint64(&out->frq_off_delta)); + RETURN_IF_ERROR(src->get_varint64(&out->frq_len)); + if (out->enc == DictEntryEnc::kWindowed) { + RETURN_IF_ERROR(src->get_varint64(&out->prelude_len)); + RETURN_IF_ERROR(src->get_varint64(&out->frq_docs_len)); + if (out->prelude_len == 0 || out->prelude_len > out->frq_docs_len || + out->frq_docs_len > out->frq_len) { + return Status::Error( + "dict_entry: invalid windowed docs prefix"); + } + } else { + RETURN_IF_ERROR(src->get_varint64(&out->frq_docs_len)); + if (out->frq_docs_len > out->frq_len) { + return Status::Error( + "dict_entry: frq_docs_len exceeds frq_len"); + } + RETURN_IF_ERROR(read_region_meta(src, tier, /*has_crc=*/true, out->frq_docs_len, + out->frq_len - out->frq_docs_len, out)); + } + if (!tier_has_stats(tier)) return Status::OK(); + RETURN_IF_ERROR(src->get_varint64(&out->prx_off_delta)); + RETURN_IF_ERROR(src->get_varint64(&out->prx_len)); + return Status::OK(); +} + +Status read_byte_blob(ByteSource* src, std::vector* out) { + uint64_t len = 0; + RETURN_IF_ERROR(src->get_varint64(&len)); + Slice bytes; + RETURN_IF_ERROR(src->get_bytes(static_cast(len), &bytes)); + out->assign(bytes.data(), bytes.data() + bytes.size()); + return Status::OK(); +} + +Status read_inline(ByteSource* src, IndexTier tier, DictEntry* out) { + RETURN_IF_ERROR(read_byte_blob(src, &out->frq_bytes)); + RETURN_IF_ERROR(src->get_varint64(&out->inline_dd_disk_len)); + if (out->inline_dd_disk_len > out->frq_bytes.size()) { + return Status::Error( + "dict_entry: inline_dd_disk_len exceeds frq_bytes"); + } + const uint64_t freq_disk_len = + static_cast(out->frq_bytes.size()) - out->inline_dd_disk_len; + // INLINE entries store no per-region crc (covered by the block crc): + // has_crc=false. + RETURN_IF_ERROR(read_region_meta(src, tier, /*has_crc=*/false, out->inline_dd_disk_len, + freq_disk_len, out)); + if (!tier_has_stats(tier)) return Status::OK(); + RETURN_IF_ERROR(read_byte_blob(src, &out->prx_bytes)); + return Status::OK(); +} + +Status read_locator(ByteSource* src, IndexTier tier, DictEntry* out) { + if (out->kind == DictEntryKind::kInline) return read_inline(src, tier, out); + return read_pod_ref(src, tier, out); +} + +// Read entry_len (= body length) and verify that src has enough remaining +// bytes. +Status read_entry_len(ByteSource* src, uint64_t* total) { + RETURN_IF_ERROR(src->get_varint64(total)); + if (*total > src->remaining()) { + return Status::Error( + "dict_entry: entry_len out of range"); + } + return Status::OK(); +} + +} // namespace + +Status encode_dict_entry(const DictEntry& entry, std::string_view prev_term, IndexTier tier, + ByteSink* sink, bool term_stats) { + ByteSink body_scratch; + return encode_dict_entry(entry, prev_term, tier, sink, term_stats, &body_scratch); +} + +Status encode_dict_entry(const DictEntry& entry, std::string_view prev_term, IndexTier tier, + ByteSink* sink, bool term_stats, ByteSink* body_scratch) { + if (sink == nullptr || body_scratch == nullptr || sink == body_scratch) { + return Status::Error( + "dict_entry: sink and body_scratch must be non-null and distinct"); + } + + // Serialize the body into a temporary buffer first to obtain the exact + // length, then write entry_len + body. CRC verification is done uniformly at + // the DICT block level (covering block header + all entries + anchor table); + // CRC is not repeated at the entry level, to keep slim/inline low-frequency + // terms maximally compact (spec §DICT block/§dict entry). + body_scratch->clear(); + write_body(entry, prev_term, tier, term_stats, body_scratch); + sink->put_varint64(static_cast(body_scratch->size())); + sink->put_bytes(body_scratch->view()); + return Status::OK(); +} + +Status decode_dict_entry_key(ByteSource* src, std::string_view prev_term, DictEntry* out, + size_t* body_start, uint64_t* entry_total) { + if (src == nullptr || out == nullptr) { + return Status::Error("dict_entry: src / out is null"); + } + *out = DictEntry {}; + + uint64_t total = 0; + RETURN_IF_ERROR(read_entry_len(src, &total)); + *body_start = src->position(); + *entry_total = total; + + return read_term_key(src, prev_term, out); +} + +Status decode_dict_entry_rest(ByteSource* src, IndexTier tier, size_t body_start, + uint64_t entry_total, DictEntry* out, bool term_stats) { + if (src == nullptr || out == nullptr) { + return Status::Error("dict_entry: src / out is null"); + } + // Body-decode seam: count exactly the entries whose body we materialize. + body_decode_atomic().fetch_add(1, std::memory_order_relaxed); + + uint8_t flags = 0; + RETURN_IF_ERROR(src->get_u8(&flags)); + apply_flags(flags, out); + RETURN_IF_ERROR(read_stats(src, tier, term_stats, out)); + RETURN_IF_ERROR(read_locator(src, tier, out)); + + // The body must consume exactly entry_len bytes; otherwise the structure is + // inconsistent with the tier. + const size_t consumed = src->position() - body_start; + if (consumed != static_cast(entry_total)) { + return Status::Error( + "dict_entry: body length does not match entry_len"); + } + return Status::OK(); +} + +Status skip_dict_entry_body(ByteSource* src, size_t body_start, uint64_t entry_total) { + if (src == nullptr) { + return Status::Error("dict_entry: src is null"); + } + const size_t consumed = src->position() - body_start; + if (consumed > static_cast(entry_total)) { + return Status::Error( + "dict_entry: term key overruns entry body"); + } + const size_t advance = static_cast(entry_total) - consumed; + Slice unused; + return src->get_bytes(advance, &unused); +} + +Status decode_dict_entry(ByteSource* src, std::string_view prev_term, IndexTier tier, + DictEntry* out, bool term_stats) { + size_t body_start = 0; + uint64_t entry_total = 0; + RETURN_IF_ERROR(decode_dict_entry_key(src, prev_term, out, &body_start, &entry_total)); + return decode_dict_entry_rest(src, tier, body_start, entry_total, out, term_stats); +} + +Status skip_dict_entry(ByteSource* src) { + if (src == nullptr) + return Status::Error("dict_entry: src is null"); + uint64_t total = 0; + RETURN_IF_ERROR(read_entry_len(src, &total)); + Slice unused; + return src->get_bytes(static_cast(total), &unused); +} + +uint64_t dict_entry_body_decode_count() { + return body_decode_atomic().load(std::memory_order_relaxed); +} + +void reset_dict_entry_counters() { + body_decode_atomic().store(0, std::memory_order_relaxed); +} + +} // namespace doris::snii::format diff --git a/be/src/storage/index/snii/format/dict_entry.h b/be/src/storage/index/snii/format/dict_entry.h new file mode 100644 index 00000000000000..df427080067f01 --- /dev/null +++ b/be/src/storage/index/snii/format/dict_entry.h @@ -0,0 +1,184 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/encoding/byte_sink.h" +#include "storage/index/snii/encoding/byte_source.h" +#include "storage/index/snii/format/format_constants.h" +#include "storage/index/snii/format/frq_pod.h" + +// DictEntry —— on-disk encoding/decoding of a dict entry. +// +// Byte layout (see docs/design/SNII-design-spec.source.md "dict entry" +// section): +// entry_len varint # byte length of entry body, allowing reader to skip +// unknown extensions or fast-skip entries +// --- entry body begins here, covered by entry_len --- +// prefix_len varint # length of shared prefix with prev_term +// suffix_len varint # number of suffix bytes +// suffix u8[] # suffix bytes that differ from prev_term +// flags u8 # bit0 kind / bit1 enc / bit2 has_sb / bit3 +// has_champion(=0) / bit4 offsets_ref(=0) df varint ttf_delta varint +// # only when tier>=T2 max_freq varint # only when tier>=T2 locator: +// pod_ref: frq_off_delta varint, frq_len varint, +// [prelude_len varint, frq_docs_len varint when enc=windowed] +// # docs-only prefix [prelude][dd-block]; windowed entries +// carry # per-window region metadata in the prelude. +// [frq_docs_len varint, slim region meta when enc=slim]: +// # frq_docs_len == dd region on-disk length; the docs-only +// prefix # [frq_off, frq_off+frq_docs_len) a docid-only reader +// fetches # without the freq region. win_mode u8 (bit0 +// dd_zstd, bit1 freq_zstd) dd_uncomp_len varint, crc_dd u32 +// [freq_uncomp_len varint, crc_freq u32 when tier>=T2] +// # The single slim window is [dd_region][freq_region]; +// dd_disk_len # = frq_docs_len, freq_disk_len = frq_len - +// frq_docs_len. +// [prx_off_delta varint, prx_len varint when tier>=T2] +// inline: frq_len varint, frq_bytes u8[], # frq_bytes = +// [dd_region][freq_region] +// slim region meta (as above, sans frq_docs_len which == dd disk +// len +// carried as inline_dd_disk_len varint), +// [prx_len varint, prx_bytes u8[] when tier>=T2] +// --- entry body ends --- +// +// CRC verification is performed at the DICT block level (covering block header +// + all entries + anchor offset table), no per-entry CRC to keep slim/inline +// low-frequency terms compact (spec §DICT block line 330/348). tier and +// positions capability are provided by Core metadata (not stored redundantly +// inside entries): when tier>=T2, ttf_delta / max_freq and .prx locator/bytes +// are written. +namespace doris::snii::format { + +// Dict entry: inline or pod-ref (two states), self-described length, supports +// intra-block front coding. +struct DictEntry { + // term key (front coding relative to prev_term is applied during + // encode/decode; full term stored here). + std::string term; + + // flags. + DictEntryKind kind = DictEntryKind::kPodRef; + DictEntryEnc enc = DictEntryEnc::kSlim; + bool has_sb = false; + + // term stats. + uint32_t df = 0; + uint64_t ttf_delta = 0; // only when tier>=T2 AND the block carries term stats + // G16-f: false when decoded from a kNoTermStats block (freq-dropped index): + // ttf_delta/max_freq above are then meaningless defaults, NOT real zeros. + // Consumers that need them (stats provider / BM25) must check this flag. + bool term_stats_present = true; + uint64_t max_freq = 0; // only when tier>=T2 + + // pod_ref locator. + uint64_t frq_off_delta = 0; + uint64_t frq_len = 0; + uint64_t prelude_len = 0; // only when enc=windowed + uint64_t frq_docs_len = 0; // pod_ref docs-only prefix length + uint64_t prx_off_delta = 0; // only when tier>=T2 + uint64_t prx_len = 0; // only when tier>=T2 + + // slim/inline single-window region codecs. The window is + // [dd_region][freq_region] (no self-describing header). dd_meta drives the + // docs-only decode; freq_meta the scoring decode (only when tier>=T2). For + // slim pod_ref dd_meta.disk_len == frq_docs_len; for inline it is stored as + // inline_dd_disk_len. + FrqRegionMeta dd_meta; + FrqRegionMeta freq_meta; // only when tier>=T2 + uint64_t inline_dd_disk_len = 0; // only for inline: dd region on-disk length + + // inline payload. + std::vector frq_bytes; // = [dd_region][freq_region] + std::vector prx_bytes; // only when tier>=T2 +}; + +// Encodes an entry into sink (appending) using the layout above, with front +// coding relative to prev_term. tier determines whether optional fields are +// written. term_stats == false (G16-f: freq-dropped indexes, declared by the +// block header's kNoTermStats flag) omits the ttf_delta/max_freq varints -- +// they serve only BM25 scoring, dead on an index that dropped freq; df stays. +// Region metadata (freq/prx locators) remains tier-conditioned. +Status encode_dict_entry(const DictEntry& entry, std::string_view prev_term, IndexTier tier, + ByteSink* sink, bool term_stats = true); + +// Same encoding with caller-owned body scratch. The scratch is cleared before +// use while retaining capacity, allowing block builders to avoid one allocation +// per dictionary entry. sink and body_scratch must be distinct objects. +Status encode_dict_entry(const DictEntry& entry, std::string_view prev_term, IndexTier tier, + ByteSink* sink, bool term_stats, ByteSink* body_scratch); + +// Decodes one entry from the current position of src; term is reconstructed +// from prev_term + suffix. Verifies the trailing CRC; out-of-range / CRC +// mismatch / invalid prefix_len all return Corruption. term_stats must match +// the writer's choice (the dict block header flag carries it). +Status decode_dict_entry(ByteSource* src, std::string_view prev_term, IndexTier tier, + DictEntry* out, bool term_stats = true); + +// Skips one entry using only entry_len (does not parse internal fields or +// verify CRC). +Status skip_dict_entry(ByteSource* src); + +// ---- Key-first decode primitives (T07) ---- +// +// decode_dict_entry is split into a "key" stage and a "rest" (body) stage so a +// caller scanning many entries can decide on the (front-coded) term key alone +// whether the entry's body is worth materializing. decode_dict_entry below is +// re-expressed as key + rest and stays byte-for-byte identical in output. + +// Reads only entry_len + the front-coded term key. On return src is positioned +// at the start of the entry body (the flags byte). out is reset to defaults and +// out->term is reconstructed from prev_term + suffix; no other field is touched. +// *body_start receives the absolute src position of the body (right after the +// entry_len varint) and *entry_total the body byte length (already bounds-checked +// against src->remaining()). Used to drive key-first scans (find_term, prefix +// streaming) that skip non-matching bodies. +Status decode_dict_entry_key(ByteSource* src, std::string_view prev_term, DictEntry* out, + size_t* body_start, uint64_t* entry_total); + +// Continues from the position decode_dict_entry_key left at: reads +// flags/stats/locator into *out and verifies the body consumed exactly +// entry_total bytes (body_start anchors the consumed count). Increments the +// body-decode counter seam (see dict_entry_body_decode_count) at its top so +// tests can assert how many entry bodies a scan actually materialized. +Status decode_dict_entry_rest(ByteSource* src, IndexTier tier, size_t body_start, + uint64_t entry_total, DictEntry* out, bool term_stats = true); + +// Skips the remaining body bytes after decode_dict_entry_key, advancing src to +// the next entry without parsing flags/stats/locator. advance = entry_total - +// (src.position() - body_start); the term key already consumed by the key stage +// is accounted for. +Status skip_dict_entry_body(ByteSource* src, size_t body_start, uint64_t entry_total); + +// Test-only instrumentation seam: dict_entry_body_decode_count() returns a +// process-global count of decode_dict_entry_rest calls -- i.e. how many entry +// bodies (flags/stats/locator + any inline byte copies) a scan materialized. +// Key-first find_term / prefix streaming drive this toward the number of +// produced hits instead of the number of entries walked. Counters use relaxed +// atomics; reset between tests. +uint64_t dict_entry_body_decode_count(); +void reset_dict_entry_counters(); + +} // namespace doris::snii::format diff --git a/be/src/storage/index/snii/format/format_constants.h b/be/src/storage/index/snii/format/format_constants.h new file mode 100644 index 00000000000000..fa8401defb4fa3 --- /dev/null +++ b/be/src/storage/index/snii/format/format_constants.h @@ -0,0 +1,137 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +// SNII container and per-section on-disk contract constants. +// Once published, these values are format semantics; changes require bumping +// format_version and maintaining a compatibility policy. All multi-byte +// fixed-width fields are little-endian; variable-length integers use LEB128 +// (see snii/encoding/varint.h). +namespace doris::snii::format { + +// ---- Container-level magic / version ---- +// "SNII" reads as 0x49494E53 in little-endian. +inline constexpr uint32_t kContainerMagic = 0x49494E53u; // 'S''N''I''I' +inline constexpr uint32_t kTailMagic = 0x4C494154u; // 'T''A''I''L' +inline constexpr uint16_t kFormatVersion = 1; +inline constexpr uint16_t kMinReaderVersion = 1; + +// ---- SectionFramer type ids for standalone metadata blobs ---- +enum class SectionType : uint8_t { + kSampledTermIndex = 2, + kDictBlockDirectory = 3, + kCoreMetadataPB = 6, + // G13: zstd-compressed carriers for the two large metadata blobs + // (they are highly compressible sorted string/offset tables and dominate the + // metadata group fetched serially at open). Payload = varint64 + // uncomp_len followed by zstd(original full frame), where "original full + // frame" is the byte-exact kSampledTermIndex / kDictBlockDirectory frame + // (type+len+payload+crc32c) used by the raw layout. Decompression + // therefore reproduces the raw frame verbatim and the sub-module readers + // (which re-verify the inner crc) stay unchanged. The writer emits these ONLY + // when the raw frame reaches kMetaSectionCompressMinBytes AND compression + // shrinks it; otherwise it emits the raw frame. + kSampledTermIndexZstd = 11, + kDictBlockDirectoryZstd = 12, + // Per-document one-byte BM25 norms. This must remain distinct from + // Core metadata so a corrupt section reference cannot reinterpret valid + // collection statistics as document norms. + kNormsPod = 14, +}; + +// ---- Logical index postings storage content configuration (fixed per logical +// index, not per-term) ---- Determines whether to write freq / positions / +// norms+stats. +enum class IndexConfig : uint8_t { + kDocsOnly = 0, // docid only: term/match filtering + kDocsPositions = 1, // docid+positions (+freq only when the caller keeps + // it -- SniiIndexInput::write_freq, G16-c): MATCH_PHRASE + kDocsPositionsScoring = 2, // + norms + stats: phrase + BM25 + kPositionsOffsets = 3, // reserved (highlight/RAG), not implemented in this release +}; + +// term stats / postings capability tiers: only tier>=kT2 writes +// ttf_delta/max_freq and .prx. +enum class IndexTier : uint8_t { + kT1 = 1, // docs-only + kT2 = 2, // docs-positions + kT3 = 3, // docs-positions-scoring +}; + +inline constexpr IndexTier tier_of(IndexConfig cfg) { + return cfg == IndexConfig::kDocsOnly ? IndexTier::kT1 + : cfg == IndexConfig::kDocsPositions ? IndexTier::kT2 + : IndexTier::kT3; // scoring / offsets +} +inline constexpr bool has_positions(IndexConfig cfg) { + return cfg != IndexConfig::kDocsOnly; +} +inline constexpr bool has_scoring(IndexConfig cfg) { + return cfg == IndexConfig::kDocsPositionsScoring; +} + +// ---- DictEntry flags bit definitions ---- +namespace dict_flags { +inline constexpr uint8_t kKind = 1u << 0; // 0=pod_ref / 1=inline +inline constexpr uint8_t kEnc = 1u << 1; // 0=slim / 1=windowed +inline constexpr uint8_t kHasSb = 1u << 2; // posting prelude includes sub-block directory +inline constexpr uint8_t kHasChampion = 1u << 3; // v1 always 0 +inline constexpr uint8_t kOffsetsRef = 1u << 4; // v1 always 0 +// bit5-7 reserved +} // namespace dict_flags + +enum class DictEntryKind : uint8_t { kPodRef = 0, kInline = 1 }; +enum class DictEntryEnc : uint8_t { kSlim = 0, kWindowed = 1 }; + +// ---- .prx window codec (codec byte bit0-5) ---- +// kRaw : plaintext varint payload (doc_count, per-doc pos_count + position +// deltas). kZstd : zstd-compressed plaintext payload (legacy reader still +// supported). kPfor : doc_count + per-doc pos_count (varint), then position +// deltas bit-packed +// as PFOR runs (kFrqBaseUnit each). No entropy coding -> far cheaper +// build CPU than zstd while staying competitive on size for ascending +// deltas. +enum class PrxCodec : uint8_t { + kRaw = 0, + kZstd = 1, + kPfor = 2 /* bit7 cont-reserved */ +}; + +// ---- Build-time parameters (not format semantics; may be tuned against real +// metrics) ---- +inline constexpr uint32_t kFrqBaseUnit = 256; // window base unit +inline constexpr uint32_t kSlimDfThreshold = 512; // df < this → slim +inline constexpr uint32_t kDefaultInlineThreshold = 256; // slim encoded bytes ≤ this → inline +// Adaptive window sizing (design #4): high-df windowed terms use larger windows +// to cut prelude rows + per-window header/crc overhead. Windows remain a whole +// multiple of kFrqBaseUnit so .prx alignment and win_base/last_docid semantics +// are preserved. A term whose df >= kAdaptiveWindowDfThreshold splits into +// kAdaptiveWindowDocs-sized windows instead of kFrqBaseUnit-sized ones. +inline constexpr uint32_t kAdaptiveWindowDfThreshold = 8192; // df >= this -> larger windows +inline constexpr uint32_t kAdaptiveWindowDocs = 1024; // larger window size (4 * base unit) +inline constexpr uint32_t kDefaultTargetDictBlockBytes = 64 * 1024; +// G13: SampledTermIndex / DictBlockDirectory metadata frames +// at or above this raw size are emitted zstd-compressed (kSampledTermIndexZstd / +// kDictBlockDirectoryZstd); smaller ones stay raw -- compression overhead is not +// worth it below a few KB. A build-time parameter, not format semantics: readers +// accept both layouts regardless of the value. +inline constexpr size_t kMetaSectionCompressMinBytes = 4 * 1024; + +} // namespace doris::snii::format diff --git a/be/src/storage/index/snii/format/frq_pod.cpp b/be/src/storage/index/snii/format/frq_pod.cpp new file mode 100644 index 00000000000000..f2d835eb019f36 --- /dev/null +++ b/be/src/storage/index/snii/format/frq_pod.cpp @@ -0,0 +1,439 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/format/frq_pod.h" + +#include +#include +#include +#include +#include +#include +#include + +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/common/uninitialized_buffer.h" +#include "storage/index/snii/encoding/byte_source.h" +#include "storage/index/snii/encoding/crc32c.h" +#include "storage/index/snii/encoding/pfor.h" +#include "storage/index/snii/encoding/zstd_codec.h" +#include "storage/index/snii/format/format_constants.h" +#include "storage/index/snii/format/prx_decode_stats.h" + +namespace doris::snii::testing { +void note_frq_dd_validation_doc_visits(uint64_t count); +void note_frq_dd_materialized_values(uint64_t count); +void note_frq_raw_region_copy_bytes(uint64_t count); +} // namespace doris::snii::testing + +namespace doris::snii::format { + +namespace { + +// Auto-compression threshold: use raw when a region is smaller than this byte +// count (zstd gain is negligible and metadata overhead is relatively large). +inline constexpr size_t kAutoZstdMinBytes = 512; +// Default zstd level for auto mode. +inline constexpr int kDefaultZstdLevel = 3; +// Maximum decompressed byte size for a single region. Guards against a +// corrupted uncomp_len read from S3 that inflated to a huge value: sanity-check +// before allocating/decompressing to avoid GB-scale allocations. Windows are +// 256-doc aligned and normally far smaller than this. +inline constexpr uint32_t kMaxRegionUncompBytes = 256u * 1024 * 1024; +// Maximum doc count per .frq window (guards against a corrupted n). Window +// baseline is 256, practical combined cap is 2048, so this is a loose but +// astronomically-large-number-blocking upper bound. +inline constexpr uint32_t kMaxWindowDocs = 1u << 24; + +// Encode a uint32 array into multiple PFOR runs, each of 256 (kFrqBaseUnit) +// elements. n / run count is not written: the number of runs is derived from +// total length n and kFrqBaseUnit, and the decoder computes it the same way. +void encode_pfor_runs(std::span values, ByteSink* out) { + size_t n = values.size(); + for (size_t off = 0; off < n; off += kFrqBaseUnit) { + size_t run = (n - off < kFrqBaseUnit) ? (n - off) : kFrqBaseUnit; + pfor_encode(values.data() + off, run, out); + } +} + +// Decode n uint32 values from source (multiple PFOR runs of 256 each). +Status decode_pfor_runs(ByteSource* src, size_t n, std::vector* out) { + // Sized then fully overwritten by pfor_decode below; no zero-fill needed. + resize_uninitialized(*out, n); + for (size_t off = 0; off < n; off += kFrqBaseUnit) { + size_t run = (n - off < kFrqBaseUnit) ? (n - off) : kFrqBaseUnit; + RETURN_IF_ERROR(pfor_decode(src, run, out->data() + off)); + } + return Status::OK(); +} + +// Verifies docids are ascending and the first entry is not below win_base. +Status validate_docs(std::span docs, uint64_t win_base) { + if (docs.empty()) return Status::OK(); +#ifdef BE_TEST + ::doris::snii::testing::note_frq_dd_validation_doc_visits(1); +#endif + if (static_cast(docs.front()) < win_base) { + return Status::Error("frq: first docid below win_base"); + } + for (size_t i = 1; i < docs.size(); ++i) { +#ifdef BE_TEST + ::doris::snii::testing::note_frq_dd_validation_doc_visits(1); +#endif + if (docs[i] < docs[i - 1]) { + return Status::Error( + "frq: docids must be ascending"); + } + } + return Status::OK(); +} + +// Decision: given level and plaintext length, determine whether to compress. +bool should_compress(int level, size_t plain_len) { + if (level == 0) return false; // force raw + if (level > 0) return true; // force zstd + return plain_len >= kAutoZstdMinBytes; // auto +} + +// Encodes one region's plaintext into raw or zstd, appends the on-disk bytes to +// out, and fills meta (mode/uncomp_len/disk_len/crc). The region carries no +// header. +Status emit_region(Slice plain, int level, ByteSink* out, FrqRegionMeta* meta) { + if (out == nullptr || meta == nullptr) { + return Status::Error("frq: null region out"); + } + meta->uncomp_len = plain.size(); + if (should_compress(level, plain.size())) { + // zstd needs its own buffer: the compressed bytes differ from `plain`. + std::vector disk; + meta->zstd = true; + RETURN_IF_ERROR(zstd_compress(plain, level > 0 ? level : kDefaultZstdLevel, &disk)); + meta->disk_len = static_cast(disk.size()); + meta->crc = crc32c(Slice(disk)); + out->put_bytes(Slice(disk)); + return Status::OK(); + } + // Raw: the on-disk bytes ARE `plain` (a view over the caller's contiguous + // ByteSink), so crc and emit straight from it -- no temp `disk` alloc/copy. + // disk_len MUST stay == plain.size(): open_region enforces uncomp_len == + // disk_len for raw regions. Byte-identical to the former disk.assign() path + // (disk == plain, so crc32c(disk) == crc32c(plain), put_bytes(disk) == same + // bytes). + meta->zstd = false; + meta->disk_len = static_cast(plain.size()); + meta->crc = crc32c(plain); +#ifdef BE_TEST + ::doris::snii::testing::note_frq_raw_region_copy_bytes(plain.size()); +#endif + out->put_bytes(plain); + return Status::OK(); +} + +void finish_raw_region(ByteSink* out, size_t begin, FrqRegionMeta* meta) { + const size_t length = out->size() - begin; + const Slice appended = length == 0 ? Slice() : Slice(out->buffer().data() + begin, length); + meta->zstd = false; + meta->uncomp_len = static_cast(length); + meta->disk_len = static_cast(length); + meta->crc = crc32c(appended); +} + +void rollback_raw_region(ByteSink* out, size_t begin) { + ByteSink restored; + if (begin != 0) { + restored.put_bytes(Slice(out->buffer().data(), begin)); + } + *out = std::move(restored); +} + +Status append_raw_dd_region(std::span docs, uint64_t win_base, ByteSink* out, + FrqRegionMeta* meta) { + if (!docs.empty() && static_cast(docs.front()) < win_base) { + return Status::Error("frq: first docid below win_base"); + } + + const size_t begin = out->size(); + out->put_varint32(static_cast(docs.size())); + std::array deltas; + uint64_t previous = win_base; + for (size_t offset = 0; offset < docs.size(); offset += kFrqBaseUnit) { + const size_t count = std::min(docs.size() - offset, static_cast(kFrqBaseUnit)); + for (size_t i = 0; i < count; ++i) { + const uint32_t doc = docs[offset + i]; + if (static_cast(doc) < previous) { + rollback_raw_region(out, begin); + return Status::Error( + "frq: docids must be ascending"); + } + deltas[i] = static_cast(static_cast(doc) - previous); + previous = doc; + } + pfor_encode(deltas.data(), count, out); + } + finish_raw_region(out, begin, meta); + return Status::OK(); +} + +// Materializes a region's plaintext (raw borrows the view; zstd decompresses) +// and verifies its crc + slice length against meta. +Status open_region(Slice disk, const FrqRegionMeta& meta, std::vector* holder, + PrxCsrAllocationGate* allocation_gate, Slice* plain) { + if (disk.size() != static_cast(meta.disk_len)) { + return Status::Error( + "frq: region slice length mismatch"); + } + if (meta.uncomp_len > kMaxRegionUncompBytes) { + return Status::Error( + "frq: region uncomp_len exceeds sane cap"); + } + // Inline entries (verify_crc=false) carry no per-region crc: their on-disk + // bytes are covered by the enclosing dict block's block-level crc32c, so the + // region crc would be redundant. POD-ref regions keep their own crc check. + if (meta.verify_crc && crc32c(disk) != meta.crc) { + return Status::Error( + "frq: region crc mismatch"); + } + if (!meta.zstd) { + if (meta.uncomp_len != meta.disk_len) { + return Status::Error( + "frq: raw region length inconsistent"); + } + *plain = disk; + return Status::OK(); + } + if (allocation_gate != nullptr) { + RETURN_IF_ERROR(allocation_gate->reserve_decompression(static_cast(meta.uncomp_len), + &holder)); + DCHECK(holder != nullptr); + } + RETURN_IF_ERROR(zstd_decompress(disk, static_cast(meta.uncomp_len), holder)); + *plain = Slice(*holder); + return Status::OK(); +} + +} // namespace + +Status build_dd_region(std::span docids_ascending, uint64_t win_base, + int zstd_level_or_neg_for_auto, ByteSink* out, FrqRegionMeta* meta) { + if (out == nullptr || meta == nullptr) { + return Status::Error("frq: null dd region out"); + } + if (zstd_level_or_neg_for_auto == 0) { + return append_raw_dd_region(docids_ascending, win_base, out, meta); + } + RETURN_IF_ERROR(validate_docs(docids_ascending, win_base)); + ByteSink plain; // VInt n ++ PFOR_runs(doc_delta) + std::vector dd(docids_ascending.size()); +#ifdef BE_TEST + ::doris::snii::testing::note_frq_dd_materialized_values(dd.size()); +#endif + uint64_t prev = win_base; + for (size_t i = 0; i < docids_ascending.size(); ++i) { + dd[i] = static_cast(static_cast(docids_ascending[i]) - prev); + prev = docids_ascending[i]; + } + plain.put_varint32(static_cast(docids_ascending.size())); + encode_pfor_runs(dd, &plain); + return emit_region(plain.view(), zstd_level_or_neg_for_auto, out, meta); +} + +Status build_dd_region_from_deltas(std::span doc_deltas, + int zstd_level_or_neg_for_auto, ByteSink* out, + FrqRegionMeta* meta) { + if (out == nullptr || meta == nullptr) { + return Status::Error("frq: null dd region out"); + } + if (zstd_level_or_neg_for_auto != 0) { + return Status::Error( + "frq: direct doc-delta encoding requires raw level 0"); + } + + const size_t begin = out->size(); + out->put_varint32(static_cast(doc_deltas.size())); + encode_pfor_runs(doc_deltas, out); + finish_raw_region(out, begin, meta); + return Status::OK(); +} + +} // namespace doris::snii::format + +namespace doris::snii::testing { +#ifdef BE_TEST +namespace { +std::atomic validation_visits {0}; +std::atomic materialized_values {0}; +std::atomic raw_copy_bytes {0}; +} // namespace +#endif + +void note_frq_dd_validation_doc_visits(uint64_t count) { +#ifdef BE_TEST + validation_visits.fetch_add(count, std::memory_order_relaxed); +#endif +} +void note_frq_dd_materialized_values(uint64_t count) { +#ifdef BE_TEST + materialized_values.fetch_add(count, std::memory_order_relaxed); +#endif +} +void note_frq_raw_region_copy_bytes(uint64_t count) { +#ifdef BE_TEST + raw_copy_bytes.fetch_add(count, std::memory_order_relaxed); +#endif +} +void reset_frq_raw_encode_work() { +#ifdef BE_TEST + validation_visits.store(0, std::memory_order_relaxed); + materialized_values.store(0, std::memory_order_relaxed); + raw_copy_bytes.store(0, std::memory_order_relaxed); +#endif +} +uint64_t frq_dd_validation_doc_visits() { +#ifdef BE_TEST + return validation_visits.load(std::memory_order_relaxed); +#else + return 0; +#endif +} +uint64_t frq_dd_materialized_values() { +#ifdef BE_TEST + return materialized_values.load(std::memory_order_relaxed); +#else + return 0; +#endif +} +uint64_t frq_raw_region_copy_bytes() { +#ifdef BE_TEST + return raw_copy_bytes.load(std::memory_order_relaxed); +#else + return 0; +#endif +} +} // namespace doris::snii::testing + +namespace doris::snii::format { + +Status build_freq_region(std::span freqs, int zstd_level_or_neg_for_auto, + ByteSink* out, FrqRegionMeta* meta) { + if (out == nullptr || meta == nullptr) { + return Status::Error("frq: null freq region out"); + } + if (zstd_level_or_neg_for_auto == 0) { + const size_t begin = out->size(); + encode_pfor_runs(freqs, out); + finish_raw_region(out, begin, meta); + return Status::OK(); + } + ByteSink plain; + encode_pfor_runs(freqs, &plain); + return emit_region(plain.view(), zstd_level_or_neg_for_auto, out, meta); +} + +namespace { + +Status decode_dd_region_impl(Slice dd_disk, const FrqRegionMeta& meta, uint64_t win_base, + const uint32_t* expected_doc_count, + PrxCsrAllocationGate* allocation_gate, std::vector* docids) { + if (docids == nullptr) + return Status::Error("frq: null docids out"); + std::vector holder; + Slice plain; + RETURN_IF_ERROR(open_region(dd_disk, meta, &holder, allocation_gate, &plain)); + ByteSource src(plain); + uint32_t n = 0; + RETURN_IF_ERROR(src.get_varint32(&n)); + if (n > kMaxWindowDocs) + return Status::Error( + "frq: doc count exceeds sane cap"); + if (expected_doc_count != nullptr && n != *expected_doc_count) { + return Status::Error( + "frq: encoded doc count differs from metadata"); + } + RETURN_IF_ERROR(decode_pfor_runs(&src, n, docids)); + if (!src.eof()) { + return Status::Error( + "frq: trailing bytes after dd region payload"); + } + uint64_t cur = win_base; + if (n != 0 && cur > std::numeric_limits::max()) { + return Status::Error( + "frq: window base exceeds uint32 docid range"); + } + for (uint32_t i = 0; i < n; ++i) { + const uint32_t delta = (*docids)[i]; + if (i != 0 && delta == 0) { + return Status::Error( + "frq: zero docid delta"); + } + if (delta > std::numeric_limits::max() - cur) { + return Status::Error( + "frq: docid accumulation overflow"); + } + cur += delta; + (*docids)[i] = static_cast(cur); + } + return Status::OK(); +} + +} // namespace + +Status decode_dd_region(Slice dd_disk, const FrqRegionMeta& meta, uint64_t win_base, + std::vector* docids) { + return decode_dd_region_impl(dd_disk, meta, win_base, nullptr, nullptr, docids); +} + +Status decode_dd_region(Slice dd_disk, const FrqRegionMeta& meta, uint64_t win_base, + uint32_t expected_doc_count, std::vector* docids) { + return decode_dd_region_impl(dd_disk, meta, win_base, &expected_doc_count, nullptr, docids); +} + +Status decode_dd_region(Slice dd_disk, const FrqRegionMeta& meta, uint64_t win_base, + uint32_t expected_doc_count, PrxCsrAllocationGate* allocation_gate, + std::vector* docids) { + if (allocation_gate == nullptr) { + return Status::Error( + "frq: allocation gate must be non-null"); + } + return decode_dd_region_impl(dd_disk, meta, win_base, &expected_doc_count, allocation_gate, + docids); +} + +Status decode_freq_region(Slice freq_disk, const FrqRegionMeta& meta, size_t doc_count, + std::vector* freqs) { + if (freqs == nullptr) + return Status::Error("frq: null freqs out"); + std::vector holder; + Slice plain; + RETURN_IF_ERROR(open_region(freq_disk, meta, &holder, nullptr, &plain)); + if (doc_count == 0) { + if (meta.uncomp_len != 0) { + return Status::Error( + "frq: empty freq region expected"); + } + freqs->clear(); + return Status::OK(); + } + ByteSource src(plain); + RETURN_IF_ERROR(decode_pfor_runs(&src, doc_count, freqs)); + if (!src.eof()) { + return Status::Error( + "frq: trailing bytes after freq region payload"); + } + return Status::OK(); +} + +} // namespace doris::snii::format diff --git a/be/src/storage/index/snii/format/frq_pod.h b/be/src/storage/index/snii/format/frq_pod.h new file mode 100644 index 00000000000000..4a3ca5d95b137c --- /dev/null +++ b/be/src/storage/index/snii/format/frq_pod.h @@ -0,0 +1,150 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/encoding/byte_sink.h" + +// .frq region codec (FrqPod): doc-delta (dd) and freq postings, columnar + PFOR +// (see docs/design SNII "frq design" and the read-byte-optimizations +// design 1.6). +// +// PHASE D (posting-level dd/freq grouping): windows are NO LONGER +// self-describing. A windowed .frq payload is laid out as +// [prelude][dd-block][freq-block] +// where the dd-block concatenates every window's dd_region and the freq-block +// concatenates every window's freq_region. Each region is independently encoded +// (raw or zstd, chosen by size) and the per-window codec metadata (mode, +// lengths, crc, offsets) is hoisted into the frq_prelude rows -- the region +// bytes carry NO header. This makes the docs-only prefix ([prelude][dd-block]) +// ONE contiguous run a docid-only / phrase reader can fetch in a single range, +// skipping the freq-block entirely. +// +// dd_region plaintext = VInt n ++ PFOR_runs(doc_delta) # n = doc count +// dd[0] = first_docid - win_base; dd[i] = docid[i] - docid[i-1]; win_base is +// the previous window's last docid (first window = 0). +// freq_region plaintext = PFOR_runs(freq) # present iff +// has_freq PFOR runs are segmented at 256 docs (kFrqBaseUnit); a partial +// segment writes the remainder. Variable-length integers reuse +// snii/encoding/varint; PFOR reuses snii/encoding/pfor; crc32c covers each +// region's ON-DISK bytes. +namespace doris::snii::format { + +class PrxCsrAllocationGate; + +// Codec metadata for ONE encoded region (dd or freq), hoisted into the prelude. +// The region's on-disk bytes are pure payload (no header); these fields drive +// the decode. crc covers the on-disk (disk_len) bytes. +struct FrqRegionMeta { + bool zstd = false; // true => disk bytes are zstd(plaintext); false => raw + uint64_t uncomp_len = 0; // plaintext byte length (== disk_len when raw) + uint64_t disk_len = 0; // on-disk byte length of this region + uint32_t crc = 0; // crc32c of the on-disk (disk_len) bytes + // When false, decode_*_region SKIPS the per-region crc check (and the writer + // omits the 4-byte crc from the dict entry). Set false for INLINE entries: + // their region bytes live inside the dict block, whose own block-level crc32c + // already covers them, so a per-region crc is fully redundant. POD-ref + // regions (slim/windowed) live in the separately-fetched .frq POD -- their + // crc stays. + bool verify_crc = true; +}; + +// Encodes a window's dd_region plaintext (VInt n ++ PFOR_runs(doc_delta)) into +// raw or zstd (per zstd_level_or_neg_for_auto), APPENDS the on-disk bytes to +// out, and fills meta (mode/uncomp_len/disk_len/crc). The region carries no +// header. docids_ascending: ascending docids in this window (single doc or +// empty allowed). win_base: previous window's last docid (first window = 0); +// requires docids[0] >= win_base. zstd_level_or_neg_for_auto: <0 auto (zstd +// when large enough, else raw); 0 force +// raw; >0 force zstd at that level. +// Non-ascending docids / first_docid < win_base / null out returns +// InvalidArgument. +Status build_dd_region(std::span docids_ascending, uint64_t win_base, + int zstd_level_or_neg_for_auto, ByteSink* out, FrqRegionMeta* meta); + +// Trusted writer fast path for a window whose doc deltas have already been +// produced and validated by the posting stream. Appends the same +// VInt n ++ PFOR_runs(doc_delta) bytes as build_dd_region(..., level=0), without +// reconstructing absolute docids or scanning the deltas. Only raw level 0 is +// supported; any other level returns InvalidArgument without modifying out or +// meta. +Status build_dd_region_from_deltas(std::span doc_deltas, + int zstd_level_or_neg_for_auto, ByteSink* out, + FrqRegionMeta* meta); + +// Vector convenience overload (forwards a span view; no copy of the elements). +inline Status build_dd_region(const std::vector& docids_ascending, uint64_t win_base, + int zstd_level_or_neg_for_auto, ByteSink* out, FrqRegionMeta* meta) { + return build_dd_region(std::span(docids_ascending), win_base, + zstd_level_or_neg_for_auto, out, meta); +} + +// Encodes a window's freq_region plaintext (PFOR_runs(freq)) into raw or zstd, +// APPENDS the on-disk bytes to out, and fills meta. Empty freqs yields a +// zero-length region. Null out returns InvalidArgument. +Status build_freq_region(std::span freqs, int zstd_level_or_neg_for_auto, + ByteSink* out, FrqRegionMeta* meta); + +// Vector convenience overload (forwards a span view; no copy of the elements). +inline Status build_freq_region(const std::vector& freqs, int zstd_level_or_neg_for_auto, + ByteSink* out, FrqRegionMeta* meta) { + return build_freq_region(std::span(freqs), zstd_level_or_neg_for_auto, out, + meta); +} + +// Decodes a dd_region from its on-disk slice (exactly disk_len bytes) + meta + +// win_base, reconstructing ascending docids. Verifies meta.crc against the +// slice. crc mismatch / wrong slice length / truncation / decompression / +// oversized count all return a non-OK Status. The freq region is irrelevant +// here (docs-only path). +Status decode_dd_region(Slice dd_disk, const FrqRegionMeta& meta, uint64_t win_base, + std::vector* docids); +// Compaction variant: validates the encoded count before resizing `docids`, so +// a corrupt frame cannot grow a pre-reserved destination buffer past its +// metadata-derived hard budget. +Status decode_dd_region(Slice dd_disk, const FrqRegionMeta& meta, uint64_t win_base, + uint32_t expected_doc_count, std::vector* docids); +// Compaction variant that charges a zstd decompression buffer before allocation +// and may reuse the caller's bounded decoder workspace. +Status decode_dd_region(Slice dd_disk, const FrqRegionMeta& meta, uint64_t win_base, + uint32_t expected_doc_count, PrxCsrAllocationGate* allocation_gate, + std::vector* docids); + +// Decodes a freq_region from its on-disk slice (exactly disk_len bytes) + meta, +// producing doc_count freqs. Verifies meta.crc. doc_count == 0 yields empty +// freqs (and requires a zero-length region). crc mismatch / wrong slice length +// / etc. return a non-OK Status. +Status decode_freq_region(Slice freq_disk, const FrqRegionMeta& meta, size_t doc_count, + std::vector* freqs); + +} // namespace doris::snii::format + +// Test-only work counters for the level-0 writer path. They expose redundant +// work that a direct raw encoder must eliminate without making it part of the +// production codec API. +namespace doris::snii::testing { +void reset_frq_raw_encode_work(); +uint64_t frq_dd_validation_doc_visits(); +uint64_t frq_dd_materialized_values(); +uint64_t frq_raw_region_copy_bytes(); +} // namespace doris::snii::testing diff --git a/be/src/storage/index/snii/format/frq_prelude.cpp b/be/src/storage/index/snii/format/frq_prelude.cpp new file mode 100644 index 00000000000000..9cec931244d7d4 --- /dev/null +++ b/be/src/storage/index/snii/format/frq_prelude.cpp @@ -0,0 +1,642 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/format/frq_prelude.h" + +#include +#include +#include + +#include "storage/index/snii/encoding/byte_source.h" +#include "storage/index/snii/encoding/crc32c.h" + +namespace doris::snii::format { + +namespace { + +// Anti-DoS: a segment holds at most ~15M docs (>=1 doc/window), so 1<<24 +// windows is a generous ceiling that still prevents multi-GB allocations from a +// crafted N. (crc32c is not a MAC and cannot defend a re-stamped inflated count.) +constexpr uint64_t kMaxWindows = 1ull << 24; + +uint64_t ceil_div(uint64_t a, uint64_t b) { + return (a + b - 1) / b; +} + +uint8_t make_flags(const FrqPreludeColumns& cols) { + uint8_t flags = 0; + if (cols.has_freq) flags |= frq_prelude_flags::kHasFreq; + if (cols.has_prx) flags |= frq_prelude_flags::kHasPrx; + return flags; +} + +uint8_t make_win_mode(const WindowMeta& m, bool has_freq) { + uint8_t mode = 0; + if (m.dd_zstd) mode |= frq_win_mode::kDdZstd; + if (has_freq && m.freq_zstd) mode |= frq_win_mode::kFreqZstd; + return mode; +} + +Status checked_add_u64(uint64_t lhs, uint64_t rhs, const char* message, uint64_t* out) { + if (rhs > std::numeric_limits::max() - lhs) { + return Status::Error(message); + } + *out = lhs + rhs; + return Status::OK(); +} + +Status checked_u32(uint64_t value, const char* message, uint32_t* out) { + if (value > std::numeric_limits::max()) { + return Status::Error(message); + } + *out = static_cast(value); + return Status::OK(); +} + +Status validate_window_doc_count(bool first_window, uint64_t win_base, uint64_t last_docid, + uint64_t doc_count) { + uint64_t first_docid = 0; + if (!first_window) { + RETURN_IF_ERROR(checked_add_u64(win_base, 1, "frq_prelude: window base exceeds docid range", + &first_docid)); + } + if (last_docid < first_docid) { + return Status::Error( + "frq_prelude: invalid window docid range"); + } + const uint64_t width = last_docid - first_docid + 1; + if (doc_count > width) { + return Status::Error( + "frq_prelude: doc_count exceeds window width"); + } + return Status::OK(); +} + +// Validates builder input: non-null sink, group_size>=1, sane count, and +// non-decreasing absolute last_docid across windows. +Status validate_input(const FrqPreludeColumns& cols, ByteSink* out) { + if (out == nullptr) + return Status::Error("frq_prelude: null sink"); + if (cols.group_size == 0) { + return Status::Error( + "frq_prelude: group_size must be >= 1"); + } + if (cols.windows.size() > kMaxWindows) { + return Status::Error( + "frq_prelude: window count exceeds cap"); + } + for (size_t w = 1; w < cols.windows.size(); ++w) { + if (cols.windows[w].last_docid < cols.windows[w - 1].last_docid) { + return Status::Error( + "frq_prelude: last_docid not monotonic"); + } + } + return Status::OK(); +} + +// Encodes one window row into a per-block sink. last_docid_delta is the row's +// absolute last_docid minus prev_last (the previous window's absolute last). +// +// dd_off/freq_off/prx_off are NOT serialized: each was a pure running prefix sum +// of the per-window disk/prx lengths, which the reader reconstructs (see +// decode_window_row / RunningOffsets). dd_uncomp_len/freq_uncomp_len are written +// ONLY when the region's zstd win_mode bit is set; a raw region's uncomp_len is +// defined to equal its disk_len (open_region enforces uncomp_len == disk_len on +// raw region bytes), so the reader derives it without a stored field. +void encode_window_row(const WindowMeta& m, bool has_freq, bool has_prx, uint64_t prev_last, + ByteSink* block) { + const uint8_t win_mode = make_win_mode(m, has_freq); + block->put_varint64(static_cast(m.last_docid) - prev_last); + block->put_varint64(m.doc_count); + block->put_u8(win_mode); + block->put_varint64(m.dd_disk_len); + if ((win_mode & frq_win_mode::kDdZstd) != 0) { + block->put_varint64(m.dd_uncomp_len); + } + block->put_fixed32(m.crc_dd); + if (has_freq) { + block->put_varint64(m.freq_disk_len); + if ((win_mode & frq_win_mode::kFreqZstd) != 0) { + block->put_varint64(m.freq_uncomp_len); + } + block->put_fixed32(m.crc_freq); + } + if (has_prx) { + block->put_varint64(m.prx_len); + } + block->put_varint64(m.max_freq); + block->put_u8(m.max_norm); +} + +// One super-block's serialized window block plus its directory fields. +struct SuperBlock { + ByteSink block; + uint64_t last_docid = 0; // absolute last docid of this super-block's last window +}; + +// Builds every super-block's window block (row-encoded) and records the running +// absolute last docid at each super-block boundary. +std::vector encode_super_blocks(const FrqPreludeColumns& cols) { + const uint32_t g = cols.group_size; + const size_t n = cols.windows.size(); + std::vector blocks; + blocks.reserve(static_cast(ceil_div(n, g))); + uint64_t prev_last = 0; // previous window's absolute last docid (chains across blocks) + for (size_t start = 0; start < n; start += g) { + const size_t end = std::min(n, start + g); + SuperBlock sb; + for (size_t w = start; w < end; ++w) { + encode_window_row(cols.windows[w], cols.has_freq, cols.has_prx, prev_last, &sb.block); + prev_last = cols.windows[w].last_docid; + } + sb.last_docid = prev_last; + blocks.push_back(std::move(sb)); + } + return blocks; +} + +// Serializes the super_block_dir (one row per super-block) into dir_sink, using +// each block's byte length to compute its offset within the window_dir region. +void encode_super_block_dir(const std::vector& blocks, ByteSink* dir_sink) { + uint64_t prev_last = 0; + uint64_t block_off = 0; + for (const SuperBlock& sb : blocks) { + dir_sink->put_varint64(sb.last_docid - prev_last); + dir_sink->put_varint64(block_off); + dir_sink->put_varint64(sb.block.size()); + prev_last = sb.last_docid; + block_off += sb.block.size(); + } +} + +} // namespace + +Status build_frq_prelude(const FrqPreludeColumns& cols, ByteSink* out) { + RETURN_IF_ERROR(validate_input(cols, out)); + + const std::vector blocks = encode_super_blocks(cols); + ByteSink dir_sink; + encode_super_block_dir(blocks, &dir_sink); + + // covered = header + super_block_dir (the crc covers exactly this region). + ByteSink covered; + covered.put_u8(make_flags(cols)); + covered.put_varint64(cols.windows.size()); + covered.put_varint64(cols.group_size); + covered.put_varint64(blocks.size()); + covered.put_varint64(dir_sink.size()); + covered.put_bytes(dir_sink.view()); + + out->put_bytes(covered.view()); + out->put_fixed32(crc32c(covered.view())); + for (const SuperBlock& sb : blocks) out->put_bytes(sb.block.view()); + return Status::OK(); +} + +namespace { + +// Decoded header fields shared between parse phases. +struct Header { + bool has_freq = false; + bool has_prx = false; + uint64_t n = 0; + uint64_t group_size = 0; + uint64_t n_super = 0; + uint64_t sbdir_len = 0; +}; + +// Verifies the trailing crc covers [start of buffer .. end of super_block_dir]. +// covered_len = header bytes (up to and including sbdir_len) + sbdir_len. +Status verify_covered_crc(Slice prelude, size_t header_end, uint64_t sbdir_len) { + const size_t covered = header_end + static_cast(sbdir_len); + if (covered + sizeof(uint32_t) > prelude.size()) { + return Status::Error( + "frq_prelude: buffer too short for crc region"); + } + uint32_t stored = 0; + ByteSource crc_src(prelude.subslice(covered, sizeof(uint32_t))); + RETURN_IF_ERROR(crc_src.get_fixed32(&stored)); + if (crc32c(prelude.subslice(0, covered)) != stored) { + return Status::Error( + "frq_prelude: crc32c mismatch"); + } + return Status::OK(); +} + +// Parses + validates the header (counts capped before any later reserve). +Status parse_header(ByteSource* src, Header* h) { + uint8_t flags = 0; + RETURN_IF_ERROR(src->get_u8(&flags)); + h->has_freq = (flags & frq_prelude_flags::kHasFreq) != 0; + h->has_prx = (flags & frq_prelude_flags::kHasPrx) != 0; + RETURN_IF_ERROR(src->get_varint64(&h->n)); + RETURN_IF_ERROR(src->get_varint64(&h->group_size)); + RETURN_IF_ERROR(src->get_varint64(&h->n_super)); + RETURN_IF_ERROR(src->get_varint64(&h->sbdir_len)); + if (h->n > kMaxWindows || h->n_super > kMaxWindows) { + return Status::Error( + "frq_prelude: window count exceeds sane cap"); + } + if (h->group_size == 0) { + return Status::Error( + "frq_prelude: group_size is zero"); + } + if (h->n_super != ceil_div(h->n, h->group_size)) { + return Status::Error( + "frq_prelude: n_super inconsistent with N/G"); + } + return Status::OK(); +} + +// One super-block directory row. +struct SbDirRow { + uint64_t last_docid = 0; + uint64_t block_off = 0; + uint64_t block_len = 0; +}; + +// Decodes the super_block_dir region into absolute-last-docid rows, validating +// monotonic last docids and contiguous, in-bounds block offsets. +Status decode_super_block_dir(Slice dir, const Header& h, std::vector* rows, + uint64_t* window_region_len) { + ByteSource src(dir); + rows->clear(); + rows->reserve(static_cast(h.n_super)); + uint64_t prev_last = 0; + uint64_t expect_off = 0; + for (uint64_t s = 0; s < h.n_super; ++s) { + SbDirRow r; + uint64_t ldd = 0; + RETURN_IF_ERROR(src.get_varint64(&ldd)); + RETURN_IF_ERROR(src.get_varint64(&r.block_off)); + RETURN_IF_ERROR(src.get_varint64(&r.block_len)); + RETURN_IF_ERROR(checked_add_u64( + prev_last, ldd, "frq_prelude: super-block last_docid overflow", &r.last_docid)); + uint32_t checked_last = 0; + RETURN_IF_ERROR(checked_u32(r.last_docid, "frq_prelude: super-block last_docid exceeds u32", + &checked_last)); + if (r.last_docid < prev_last || r.block_off != expect_off) { + return Status::Error( + "frq_prelude: super-block dir inconsistent"); + } + expect_off += r.block_len; + prev_last = r.last_docid; + rows->push_back(r); + } + if (!src.eof()) { + return Status::Error( + "frq_prelude: super-block dir has trailing bytes"); + } + *window_region_len = expect_off; + return Status::OK(); +} + +// Validates a per-window codec mode byte against the known bits. +Status check_win_mode(uint8_t mode, bool has_freq) { + if ((mode & ~frq_win_mode::kKnownBits) != 0) { + return Status::Error( + "frq_prelude: unknown win_mode bits"); + } + if (!has_freq && (mode & frq_win_mode::kFreqZstd) != 0) { + return Status::Error( + "frq_prelude: freq mode set without has_freq"); + } + return Status::OK(); +} + +// Running per-block byte offsets, chained across windows AND across super-blocks +// with the same lifetime as prev_last: dd/freq are prefix sums of the per-window +// on-disk region lengths, prx the prefix sum of prx lengths. They replace the +// three offset columns the row used to serialize (each was exactly this sum), so +// the reader reproduces dd_off/freq_off/prx_off bit-identically to the old +// explicit fields. +struct RunningOffsets { + uint64_t dd = 0; + uint64_t freq = 0; + uint64_t prx = 0; +}; + +// Decodes one window row, advancing prev_last to this window's absolute last and +// the running offsets past this window's dd/freq/prx regions. +Status decode_window_row(ByteSource* src, bool has_freq, bool has_prx, bool first_window, + uint64_t* prev_last, RunningOffsets* run, WindowMeta* m) { + uint64_t ldd = 0, doc_count = 0; + RETURN_IF_ERROR(src->get_varint64(&ldd)); + RETURN_IF_ERROR(src->get_varint64(&doc_count)); + uint8_t mode = 0; + RETURN_IF_ERROR(src->get_u8(&mode)); + RETURN_IF_ERROR(check_win_mode(mode, has_freq)); + m->dd_zstd = (mode & frq_win_mode::kDdZstd) != 0; + m->freq_zstd = has_freq && (mode & frq_win_mode::kFreqZstd) != 0; + + // dd region: read disk_len, derive dd_off from the running dd-block offset, + // then advance it (overflow-guarded). uncomp_len is stored only for a zstd + // region; a raw region's uncomp_len == disk_len by contract. + RETURN_IF_ERROR(src->get_varint64(&m->dd_disk_len)); + m->dd_off = run->dd; + RETURN_IF_ERROR(checked_add_u64(run->dd, m->dd_disk_len, + "frq_prelude: dd-block offset overflow", &run->dd)); + if (m->dd_zstd) { + RETURN_IF_ERROR(src->get_varint64(&m->dd_uncomp_len)); + } else { + m->dd_uncomp_len = m->dd_disk_len; + } + RETURN_IF_ERROR(src->get_fixed32(&m->crc_dd)); + if (has_freq) { + RETURN_IF_ERROR(src->get_varint64(&m->freq_disk_len)); + m->freq_off = run->freq; + RETURN_IF_ERROR(checked_add_u64(run->freq, m->freq_disk_len, + "frq_prelude: freq-block offset overflow", &run->freq)); + if (m->freq_zstd) { + RETURN_IF_ERROR(src->get_varint64(&m->freq_uncomp_len)); + } else { + m->freq_uncomp_len = m->freq_disk_len; + } + RETURN_IF_ERROR(src->get_fixed32(&m->crc_freq)); + } + if (has_prx) { + RETURN_IF_ERROR(src->get_varint64(&m->prx_len)); + m->prx_off = run->prx; + RETURN_IF_ERROR(checked_add_u64(run->prx, m->prx_len, "frq_prelude: prx offset overflow", + &run->prx)); + } + uint64_t max_freq = 0; + RETURN_IF_ERROR(src->get_varint64(&max_freq)); + RETURN_IF_ERROR(src->get_u8(&m->max_norm)); + uint64_t last_docid = 0; + RETURN_IF_ERROR(checked_add_u64(*prev_last, ldd, "frq_prelude: window last_docid overflow", + &last_docid)); + RETURN_IF_ERROR(validate_window_doc_count(first_window, *prev_last, last_docid, doc_count)); + m->win_base = *prev_last; + RETURN_IF_ERROR( + checked_u32(last_docid, "frq_prelude: window last_docid exceeds u32", &m->last_docid)); + RETURN_IF_ERROR( + checked_u32(doc_count, "frq_prelude: window doc_count exceeds u32", &m->doc_count)); + RETURN_IF_ERROR( + checked_u32(max_freq, "frq_prelude: window max_freq exceeds u32", &m->max_freq)); + *prev_last = last_docid; + return Status::OK(); +} + +// Decodes one super-block's window block (<=G rows) into the global window list, +// seeding win_base from prev_last and re-checking the recorded sb last docid. +Status decode_one_block(Slice block, const Header& h, uint64_t sb_last_docid, size_t row_count, + uint64_t* prev_last, RunningOffsets* run, + std::vector* windows) { + ByteSource src(block); + for (size_t i = 0; i < row_count; ++i) { + WindowMeta m; + RETURN_IF_ERROR(decode_window_row(&src, h.has_freq, h.has_prx, windows->empty(), prev_last, + run, &m)); + windows->push_back(m); + } + if (!src.eof()) { + return Status::Error( + "frq_prelude: window block has trailing bytes"); + } + if (*prev_last != sb_last_docid) { + return Status::Error( + "frq_prelude: window block last docid mismatch"); + } + return Status::OK(); +} + +// Decodes all window blocks pointed to by the super_block_dir. +Status decode_all_blocks(Slice window_region, const Header& h, const std::vector& dir, + std::vector* windows) { + windows->clear(); + windows->reserve(static_cast(h.n)); + uint64_t prev_last = 0; + // dd/freq/prx running offsets chain across ALL super-blocks (not reset per + // block), the same lifetime as prev_last, so the derived per-window offsets are + // continuous over the whole dd-block / freq-block / prx span. + RunningOffsets run; + for (size_t s = 0; s < dir.size(); ++s) { + const SbDirRow& r = dir[s]; + if (r.block_off + r.block_len > window_region.size() || + r.block_off + r.block_len < r.block_off) { + return Status::Error( + "frq_prelude: window block out of region"); + } + const uint64_t already = static_cast(windows->size()); + const uint64_t rows = std::min(h.group_size, h.n - already); + Slice block = window_region.subslice(static_cast(r.block_off), + static_cast(r.block_len)); + RETURN_IF_ERROR(decode_one_block(block, h, r.last_docid, static_cast(rows), + &prev_last, &run, windows)); + } + if (windows->size() != h.n) { + return Status::Error( + "frq_prelude: decoded window count mismatch"); + } + return Status::OK(); +} + +// Sums the per-window dd/freq on-disk lengths into the dd-block / freq-block +// lengths, guarding each running sum against u64 overflow. The dd_off/freq_off +// contiguity cross-checks the old prelude ran here are now tautological -- the +// reader DERIVES dd_off/freq_off as these very prefix sums (decode_window_row), +// so `m.dd_off == running-sum` holds by construction and is dropped. The +// length-overflow guards and the returned block lengths are retained: they still +// bound the dd-block/freq-block range the callers' in_bounds checks fetch against, +// and mirror the same checked_add_u64 the offset derivation uses. +Status validate_region_layout(const Header& h, const std::vector& windows, + uint64_t* dd_block_len, uint64_t* freq_block_len) { + uint64_t dd_expect = 0; + uint64_t freq_expect = 0; + for (const WindowMeta& m : windows) { + // Raw regions carry uncomp_len == disk_len (derived, not stored); this + // guard stays as defensive documentation of that invariant. + if (m.dd_disk_len > m.dd_uncomp_len && !m.dd_zstd) { + return Status::Error( + "frq_prelude: raw dd region length inconsistent"); + } + if (dd_expect + m.dd_disk_len < dd_expect) { + return Status::Error( + "frq_prelude: dd block length overflow"); + } + dd_expect += m.dd_disk_len; + if (h.has_freq) { + if (freq_expect + m.freq_disk_len < freq_expect) { + return Status::Error( + "frq_prelude: freq block length overflow"); + } + freq_expect += m.freq_disk_len; + } + } + *dd_block_len = dd_expect; + *freq_block_len = freq_expect; + return Status::OK(); +} + +} // namespace + +namespace { +// TEST-ONLY seam backing testing::window_probe_count(): one increment per window +// last_docid comparison, in select_covering_windows_cursor and in locate_window's +// level-2 scan. thread_local => race-free under the shared const reader and free of +// atomic cost in the production cursor's hot loop; tests reset/read on their own thread. +thread_local uint64_t g_window_probes = 0; +inline void note_window_probe() { + ++g_window_probes; +} +} // namespace + +Status FrqPreludeReader::open(Slice prelude, FrqPreludeReader* out) { + ByteSource src(prelude); + Header h; + RETURN_IF_ERROR(parse_header(&src, &h)); + const size_t header_end = src.position(); + RETURN_IF_ERROR(verify_covered_crc(prelude, header_end, h.sbdir_len)); + + if (header_end + static_cast(h.sbdir_len) > prelude.size()) { + return Status::Error( + "frq_prelude: sbdir_len past buffer"); + } + Slice dir = prelude.subslice(header_end, static_cast(h.sbdir_len)); + std::vector rows; + uint64_t window_region_len = 0; + RETURN_IF_ERROR(decode_super_block_dir(dir, h, &rows, &window_region_len)); + + const size_t region_start = header_end + static_cast(h.sbdir_len) + sizeof(uint32_t); + if (region_start + static_cast(window_region_len) > prelude.size()) { + return Status::Error( + "frq_prelude: window region past buffer"); + } + Slice window_region = prelude.subslice(region_start, static_cast(window_region_len)); + + out->has_freq_ = h.has_freq; + out->has_prx_ = h.has_prx; + out->group_size_ = static_cast(h.group_size); + out->n_super_ = static_cast(h.n_super); + out->sb_last_docid_.clear(); + out->sb_last_docid_.reserve(rows.size()); + for (const SbDirRow& r : rows) out->sb_last_docid_.push_back(r.last_docid); + RETURN_IF_ERROR(decode_all_blocks(window_region, h, rows, &out->windows_)); + // Packed last_docid catalogue for the covering-window cursor (in-memory only; + // byte-identical to each windows_[w].last_docid, never serialized). + out->win_last_docid_.clear(); + out->win_last_docid_.reserve(out->windows_.size()); + for (const WindowMeta& m : out->windows_) { + out->win_last_docid_.push_back(m.last_docid); + } + return validate_region_layout(h, out->windows_, &out->dd_block_len_, &out->freq_block_len_); +} + +Status FrqPreludeReader::window(uint32_t w, WindowMeta* out) const { + if (out == nullptr) + return Status::Error("frq_prelude: null window out"); + if (w >= windows_.size()) { + return Status::Error( + "frq_prelude: window index out of range"); + } + *out = windows_[w]; + return Status::OK(); +} + +Status FrqPreludeReader::locate_window(uint32_t docid, bool* found, uint32_t* w) const { + if (found == nullptr || w == nullptr) { + return Status::Error("frq_prelude: null locate out"); + } + *found = false; + if (windows_.empty()) return Status::OK(); + if (docid > windows_.back().last_docid) return Status::OK(); + + // Level 1: first super-block whose absolute last docid >= docid. + const auto sb_it = std::lower_bound(sb_last_docid_.begin(), sb_last_docid_.end(), + static_cast(docid)); + const size_t sb = static_cast(sb_it - sb_last_docid_.begin()); + // Level 2: window binary search within [sb*G, min((sb+1)*G, N)). + const size_t lo = sb * group_size_; + const size_t hi = std::min(lo + group_size_, windows_.size()); + for (size_t i = lo; i < hi; ++i) { + note_window_probe(); + if (docid <= windows_[i].last_docid) { + *found = true; + *w = static_cast(i); + return Status::OK(); + } + } + return Status::OK(); // unreachable when invariants hold; defensive miss. +} + +void select_covering_windows_cursor(const uint32_t* win_last_docid, uint32_t n_windows, + const uint64_t* sb_last_docid, uint32_t n_super, + uint32_t group_size, const std::vector& candidates, + std::vector* windows) { + windows->clear(); + if (n_windows == 0) { + return; // empty-windows guard (mirrors locate_window's windows_.empty() early-out). + } + uint32_t sb = 0; // monotonic super-block cursor + uint32_t w = 0; // monotonic window cursor + uint32_t last_emitted = UINT32_MAX; // last window pushed (run-collapse dedup) + for (uint32_t d : candidates) { + const uint64_t target = d; // widen once; keeps the comparisons template-bracket free + // Level 1: first super-block whose absolute last docid >= d. sb only advances + // forward, so across the whole call it steps at most n_super times. + while (sb < n_super && sb_last_docid[sb] < target) { + ++sb; + } + if (sb == n_super) { + break; // d past the term's last docid -> this and every later candidate miss. + } + // Boundary jump: never scan windows below the current super-block's first window. + if (w < sb * group_size) { + w = sb * group_size; + } + // Level 2: first window whose absolute last docid >= d. w only advances forward, + // so across the whole call the comparisons below total at most n_windows misses + // plus one hit per candidate => probe_count <= candidates + n_windows. + while (w < n_windows) { + note_window_probe(); + if (d <= win_last_docid[w]) { + break; + } + ++w; + } + if (w == n_windows) { + break; // defensive: invariants guarantee a hit once sb < n_super. + } + if (w != last_emitted) { + windows->push_back(w); + last_emitted = w; + } + } +} + +void FrqPreludeReader::select_covering_windows(const std::vector& candidates, + std::vector* windows) const { + select_covering_windows_cursor( + win_last_docid_.data(), static_cast(win_last_docid_.size()), + sb_last_docid_.data(), static_cast(sb_last_docid_.size()), group_size_, + candidates, windows); +} + +} // namespace doris::snii::format + +namespace doris::snii::format::testing { + +uint64_t window_probe_count() { + return g_window_probes; +} + +void reset_window_probe_count() { + g_window_probes = 0; +} + +} // namespace doris::snii::format::testing diff --git a/be/src/storage/index/snii/format/frq_prelude.h b/be/src/storage/index/snii/format/frq_prelude.h new file mode 100644 index 00000000000000..642f3aed96b71c --- /dev/null +++ b/be/src/storage/index/snii/format/frq_prelude.h @@ -0,0 +1,256 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include "common/status.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/encoding/byte_sink.h" + +// FrqPrelude: a TWO-LEVEL (super-block -> window) skippable directory that +// precedes a windowed .frq posting whose payload is laid out (PHASE D, design +// 1.6) with dd and freq regions GROUPED at posting level: +// windowed .frq payload = [prelude][dd-block][freq-block] +// dd-block = dd_region_0 ++ dd_region_1 ++ ... ++ dd_region_{N-1} +// freq-block = freq_region_0 ++ ... ++ freq_region_{N-1} (iff has_freq) +// Windows are NOT self-describing: each window's full codec metadata (region +// offsets, on-disk/uncompressed lengths, modes, crcs) lives in the prelude rows. +// The docs-only prefix [prelude][dd-block] is therefore ONE contiguous run a +// docid-only / phrase reader fetches in a single range, skipping the freq-block. +// +// DictEntry records prelude_len, frq_len (whole payload) and frq_docs_len +// (= prelude_len + dd_block_len) so a reader can range-fetch the prelude first, +// then fetch either the contiguous dd-block (docs-only) or both blocks (scoring). +// +// On-disk layout (strict; all multi-byte fixed fields little-endian, VInt = +// LEB128 via snii/encoding): +// header: +// u8 flags # bit0 has_freq, bit1 has_prx +// VInt N # number of .frq windows +// VInt G # windows per super-block (group_size; >=1) +// VInt n_super # = ceil(N / G); 0 when N==0 +// VInt sbdir_len # byte length of the super_block_dir region +// u32 crc32c # covers header + super_block_dir (NOT the window blocks) +// super_block_dir[n_super]: # small, resident: one row per super-block +// VInt sb_last_docid_delta # cumulative across super-blocks => absolute last +// # docid of the super-block's last window +// VInt sb_block_off # byte offset of this super-block's window block, +// # measured from the start of the window_dir region +// VInt sb_block_len # byte length of this super-block's window block +// window_dir: n_super self-contained blocks, each holding <=G window rows. +// per window row (T18 slim layout -- dd_off/freq_off/prx_off are NOT stored; +// the reader derives them as running prefix sums of the disk/prx lengths): +// VInt last_docid_delta # cumulative WITHIN the block => absolute last docid +// # (previous window's absolute last docid = win_base; +// # first window of first block: win_base = 0) +// VInt doc_count # number of docs in the window (frq_pod needs it) +// u8 win_mode # bit0 dd_zstd, bit1 freq_zstd +// VInt dd_disk_len # dd_region on-disk byte length +// [VInt dd_uncomp_len] # dd_region plaintext length; present ONLY when +// # win_mode & kDdZstd. A raw region's uncomp_len +// # == dd_disk_len (derived, not stored). +// u32 crc_dd # crc32c of the dd_region on-disk bytes +// VInt freq_disk_len # freq_region on-disk byte length (has_freq) +// [VInt freq_uncomp_len] # freq_region plaintext length; present ONLY when +// # has_freq && win_mode & kFreqZstd (raw: derived +// # == freq_disk_len). +// u32 crc_freq # crc32c of the freq_region on-disk bytes (has_freq) +// VInt prx_len # .prx payload byte length (present iff has_prx) +// VInt max_freq # window max term frequency (WAND block-max) +// u8 max_norm # window score-max norm (WAND); 0 acceptable +// +// The reader reconstructs each window's dd_off / freq_off (byte offset within the +// dd-block / freq-block) and prx_off (offset within the entry's .prx span) as the +// running prefix sums of dd_disk_len / freq_disk_len / prx_len over all windows, +// chained across super-blocks; WindowMeta still exposes those offsets, now derived. +// +// Reconstructing win_base / absolute last_docid (READER CONTRACT) is unchanged: +// the writer chains absolute last docids across windows; each row stores the delta +// of its absolute last docid from the previous window, and sb_last_docid seeds +// each block, so super-block binary search then in-block window binary search +// locate the window covering any docid without decoding the .frq blocks. +// +// The trailing crc32c covers only header + super_block_dir; every region carries +// its own crc (crc_dd / crc_freq) in the row. +namespace doris::snii::format { + +namespace frq_prelude_flags { +inline constexpr uint8_t kHasFreq = 1u << 0; +inline constexpr uint8_t kHasPrx = 1u << 1; +// Reserved extension point (T18): kSlimRows = 1u << 2 would gate the trimmed +// window-row layout (no stored dd_off/freq_off/prx_off, conditional uncomp_len) +// as a distinct on-disk path. It is NOT emitted today: the trim folds into the +// single pre-launch v1 encoding (writer/reader symmetric, no dual decode path). +// If a `lifecycle: launched` index appears before this lands, set this bit on the +// slim writer and branch the reader on it instead of unconditionally decoding slim. +} // namespace frq_prelude_flags + +// Per-window codec mode bits (win_mode byte). +namespace frq_win_mode { +inline constexpr uint8_t kDdZstd = 1u << 0; +inline constexpr uint8_t kFreqZstd = 1u << 1; +inline constexpr uint8_t kKnownBits = kDdZstd | kFreqZstd; +} // namespace frq_win_mode + +// Absolute, decoded metadata for one window (as the reader exposes it). The dd / +// freq region locators are offsets WITHIN the dd-block / freq-block respectively +// (both blocks follow the prelude). dd_off/freq_off/prx_off are DERIVED by the +// reader as running prefix sums of the disk/prx lengths (they are no longer stored +// per row; see the header layout note) -- these public members are unchanged and +// still populated, just by derivation. The reader derives the dd-block length from +// the last window's dd_off + dd_disk_len. +struct WindowMeta { + uint32_t last_docid = 0; // absolute last docid in the window + uint64_t win_base = 0; // absolute last docid of the previous window (0 for w==0) + uint32_t doc_count = 0; + + // dd_region locator (within the dd-block). + bool dd_zstd = false; + uint64_t dd_off = 0; // DERIVED: running sum of prior windows' dd_disk_len + uint64_t dd_disk_len = 0; + uint64_t dd_uncomp_len = 0; // DERIVED == dd_disk_len for raw; stored only when dd_zstd + uint32_t crc_dd = 0; + + // freq_region locator (within the freq-block); valid only when has_freq. + bool freq_zstd = false; + uint64_t freq_off = 0; // DERIVED: running sum of prior windows' freq_disk_len + uint64_t freq_disk_len = 0; + uint64_t freq_uncomp_len = 0; // DERIVED == freq_disk_len for raw; stored only when freq_zstd + uint32_t crc_freq = 0; + + uint64_t prx_off = 0; // valid only when has_prx; DERIVED: running sum of prior prx_len + uint64_t prx_len = 0; // valid only when has_prx + uint32_t max_freq = 0; + uint8_t max_norm = 0; + + // In-memory only (NOT serialized in the prelude row). When false, the dd/freq + // region decode skips crc verification -- used when these region bytes are + // covered by an enclosing crc (e.g. an INLINE entry inside its dict block). + // Windowed/slim POD-ref rows leave this true (their regions carry a crc). + bool verify_crc = true; +}; + +// Builder input: one fully-computed WindowMeta per window, in term order, plus the +// super-block grouping factor. The writer fills last_docid (absolute), doc_count, +// the region locators/crcs, prx locator, max_freq and max_norm; win_base is derived +// during build (so callers may leave it 0). group_size must be >= 1. +struct FrqPreludeColumns { + bool has_freq = true; + bool has_prx = false; + uint32_t group_size = 64; // windows per super-block (G) + std::vector windows; +}; + +// Builds the prelude bytes and appends them to out. +// Returns InvalidArgument when out is null, group_size is 0, or the windows are +// not in non-decreasing last_docid order (a window's absolute last docid must be +// >= the previous window's). +Status build_frq_prelude(const FrqPreludeColumns& cols, ByteSink* out); + +// Reads and verifies a prelude buffer, exposing two-level skip access. The reader +// parses the header + super_block_dir on open (verifying the trailing crc) and +// eagerly decodes every window block into owned WindowMeta rows (the prelude is +// small relative to the postings). It does not retain the input. +class FrqPreludeReader { +public: + // Parses + verifies the prelude. crc mismatch / truncation / inconsistent + // offsets-or-lengths / oversized counts => kCorruption. + static Status open(Slice prelude, FrqPreludeReader* out); + + uint32_t n_windows() const { return static_cast(windows_.size()); } + uint32_t n_super_blocks() const { return n_super_; } + bool has_freq() const { return has_freq_; } + bool has_prx() const { return has_prx_; } + + // Total on-disk byte length of the dd-block (== sum of dd_disk_len; the docs-only + // prefix after the prelude). 0 when there are no windows. + uint64_t dd_block_len() const { return dd_block_len_; } + // Total on-disk byte length of the freq-block (== sum of freq_disk_len). 0 when + // !has_freq or no windows. + uint64_t freq_block_len() const { return freq_block_len_; } + + // Returns the absolute WindowMeta for window w. Out-of-range => InvalidArgument. + Status window(uint32_t w, WindowMeta* out) const; + + // Locates the window covering docid via super-block binary search then window + // binary search. *found=false (with OK) when docid is past the term's last + // docid; otherwise *w is the index of the covering window (the first window + // whose absolute last_docid >= docid). + Status locate_window(uint32_t docid, bool* found, uint32_t* w) const; + + // Selects, as a monotonic two-pointer cursor, the ascending de-duplicated set of + // windows covering the ascending `candidates` (each window covering its + // (win_base, last_docid] span). Writes them to *windows (cleared first). The + // result is element-for-element identical to calling locate_window per candidate + // and collapsing equal runs, but uses O(C + N) window last_docid comparisons + // (C = candidates, N = windows) instead of O(C * group_size). Pure in-memory over + // the decoded directory; never fails. + void select_covering_windows(const std::vector& candidates, + std::vector* windows) const; + + // Packed absolute last_docid of window w (byte-identical to window(w).last_docid), + // exposed for the covering-window cursor's contiguous scan and equivalence tests. + uint32_t window_last_docid(uint32_t w) const { + DCHECK_LT(w, win_last_docid_.size()); + return win_last_docid_[w]; + } + +private: + bool has_freq_ = false; + bool has_prx_ = false; + uint32_t group_size_ = 1; + uint32_t n_super_ = 0; + uint64_t dd_block_len_ = 0; + uint64_t freq_block_len_ = 0; + // Absolute last docid at each super-block boundary (size n_super_). + std::vector sb_last_docid_; + // All windows decoded with absolute fields, in term order (size N). + std::vector windows_; + // Packed copy of each window's absolute last_docid (size N; win_last_docid_[w] == + // windows_[w].last_docid). Built in open() so the covering-window cursor scans a + // contiguous 4B/window array rather than the ~104B WindowMeta rows. In-memory only: + // never serialized; immutable after open() (same lifetime as windows_). + std::vector win_last_docid_; +}; + +// Pure cursor core (no FrqPreludeReader / IO): selects into *windows the ascending, +// de-duplicated indices of the windows covering the ascending `candidates`, given the +// packed window last_docid array (size n_windows), the super-block last_docid directory +// (size n_super) and group_size. A super-block cursor does boundary jumps while a window +// cursor advances forward only => O(C + N) window comparisons, element-for-element equal +// to per-candidate locate_window + run collapse. *windows is cleared first; n_windows == 0 +// yields an empty result. Exposed for isolated equivalence / complexity tests. +void select_covering_windows_cursor(const uint32_t* win_last_docid, uint32_t n_windows, + const uint64_t* sb_last_docid, uint32_t n_super, + uint32_t group_size, const std::vector& candidates, + std::vector* windows); + +// TEST-ONLY observability seam (mirrors the format dict-block decode counter). Counts the +// window last_docid comparisons performed by select_covering_windows_cursor and by +// locate_window's level-2 scan, so tests can assert the cursor stays O(C + N) and bounded +// by C + N regardless of group_size, while the legacy per-candidate scan grows with G. The +// counter is thread-local: race-free under the shared const reader and free of atomic cost +// in the production cursor loop; reset and read on the thread that ran the cursor. +namespace testing { +uint64_t window_probe_count(); +void reset_window_probe_count(); +} // namespace testing + +} // namespace doris::snii::format diff --git a/be/src/storage/index/snii/format/metadata_blob.cpp b/be/src/storage/index/snii/format/metadata_blob.cpp new file mode 100644 index 00000000000000..a8d7a0f4fd938a --- /dev/null +++ b/be/src/storage/index/snii/format/metadata_blob.cpp @@ -0,0 +1,121 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/format/metadata_blob.h" + +#include + +#include "storage/index/snii/encoding/byte_source.h" +#include "storage/index/snii/encoding/section_framer.h" +#include "storage/index/snii/encoding/zstd_codec.h" + +namespace doris::snii::format { + +namespace { + +constexpr int kMetaSectionZstdLevel = 3; +constexpr uint64_t kMaxMetaSectionUncompBytes = 256ULL * 1024 * 1024; + +size_t meta_compress_min_bytes() { + const char* s = std::getenv("SNII_META_COMPRESS_MIN"); + if (s != nullptr) { + char* end = nullptr; + const unsigned long long v = std::strtoull(s, &end, 10); + if (end != s) { + return v; + } + } + return kMetaSectionCompressMinBytes; +} + +} // namespace + +Status encode_metadata_blob(Slice raw_frame, SectionType raw_type, SectionType compressed_type, + ByteSink* out) { + if (out == nullptr) { + return Status::Error("metadata_blob: null sink"); + } + + ByteSource source(raw_frame); + FramedSection raw_section; + RETURN_IF_ERROR(SectionFramer::read(source, &raw_section)); + if (!source.eof() || raw_section.type != static_cast(raw_type)) { + return Status::Error( + "metadata_blob: raw input is not exactly one frame of the expected type"); + } + + if (raw_frame.size() >= meta_compress_min_bytes()) { + std::vector compressed; + if (zstd_compress(raw_frame, kMetaSectionZstdLevel, &compressed).ok()) { + ByteSink payload; + payload.put_varint64(raw_frame.size()); + payload.put_bytes(Slice(compressed)); + if (payload.size() + 16 < raw_frame.size()) { + SectionFramer::write(*out, static_cast(compressed_type), payload.view()); + return Status::OK(); + } + } + } + out->put_bytes(raw_frame); + return Status::OK(); +} + +Status materialize_metadata_blob(Slice stored_frame, SectionType raw_type, + SectionType compressed_type, std::vector* scratch, + Slice* raw_frame) { + if (scratch == nullptr || raw_frame == nullptr) { + return Status::Error("metadata_blob: null frame out"); + } + + ByteSource source(stored_frame); + FramedSection stored_section; + RETURN_IF_ERROR(SectionFramer::read(source, &stored_section)); + if (!source.eof()) { + return Status::Error( + "metadata_blob: trailing stored frame bytes"); + } + if (stored_section.type == static_cast(raw_type)) { + *raw_frame = stored_frame; + return Status::OK(); + } + if (stored_section.type != static_cast(compressed_type)) { + return Status::Error( + "metadata_blob: unexpected stored frame type"); + } + + ByteSource payload(stored_section.payload); + uint64_t uncomp_len = 0; + RETURN_IF_ERROR(payload.get_varint64(&uncomp_len)); + if (uncomp_len == 0 || uncomp_len > kMaxMetaSectionUncompBytes) { + return Status::Error( + "metadata_blob: zstd uncomp_len out of range"); + } + Slice compressed; + RETURN_IF_ERROR(payload.get_bytes(payload.remaining(), &compressed)); + RETURN_IF_ERROR(zstd_decompress(compressed, static_cast(uncomp_len), scratch)); + ByteSource raw_source {Slice(*scratch)}; + FramedSection raw_section; + RETURN_IF_ERROR(SectionFramer::read(raw_source, &raw_section)); + if (!raw_source.eof() || raw_section.type != static_cast(raw_type)) { + return Status::Error( + "metadata_blob: decompressed bytes are not exactly one expected raw frame"); + } + *raw_frame = Slice(*scratch); + return Status::OK(); +} + +} // namespace doris::snii::format diff --git a/be/src/storage/index/snii/format/metadata_blob.h b/be/src/storage/index/snii/format/metadata_blob.h new file mode 100644 index 00000000000000..5e098a797e08d2 --- /dev/null +++ b/be/src/storage/index/snii/format/metadata_blob.h @@ -0,0 +1,40 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include "common/status.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/encoding/byte_sink.h" +#include "storage/index/snii/format/format_constants.h" + +namespace doris::snii::format { + +// Appends a raw metadata frame or a zstd carrier for it. The carrier payload is +// varint64 raw-frame length followed by zstd(raw frame). +Status encode_metadata_blob(Slice raw_frame, SectionType raw_type, SectionType compressed_type, + ByteSink* out); + +// Returns a raw metadata frame from a raw frame or zstd carrier. A raw frame +// remains a view into stored_frame; a materialized carrier is a view into scratch. +Status materialize_metadata_blob(Slice stored_frame, SectionType raw_type, + SectionType compressed_type, std::vector* scratch, + Slice* raw_frame); + +} // namespace doris::snii::format diff --git a/be/src/storage/index/snii/format/metadata_directory.cpp b/be/src/storage/index/snii/format/metadata_directory.cpp new file mode 100644 index 00000000000000..404afe22b6fbde --- /dev/null +++ b/be/src/storage/index/snii/format/metadata_directory.cpp @@ -0,0 +1,150 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/format/metadata_directory.h" + +#include +#include +#include +#include +#include + +#include "gen_cpp/snii.pb.h" + +namespace doris::snii::format { +namespace { + +Status corrupted(std::string_view message) { + return Status::Error(message); +} + +Status unsupported(std::string_view message) { + return Status::Error(message); +} + +Status decode_blob_ref(const doris::snii::SniiBlobRefPB& input, MetadataBlobRef* out) { + if (!input.has_offset() || !input.has_length()) { + return corrupted("metadata directory: missing blob reference field"); + } + if (input.length() == 0) { + return corrupted("metadata directory: empty mandatory blob reference"); + } + *out = {.offset = input.offset(), .length = input.length()}; + return Status::OK(); +} + +Status decode_directory_pb(const doris::snii::SniiMetadataDirectoryPB& input, + std::vector* out) { + if (input.required_features_size() != 0) { + return unsupported("metadata directory: required feature is not supported"); + } + + std::vector entries; + entries.reserve(input.indexes_size()); + for (const auto& index : input.indexes()) { + if (!index.has_index_id() || !index.has_index_suffix() || !index.has_core_metadata() || + !index.has_sampled_term_index() || !index.has_dict_block_directory()) { + return corrupted("metadata directory: missing required logical field"); + } + + LogicalIndexMetadataRef entry; + entry.index_id = index.index_id(); + entry.index_suffix = index.index_suffix(); + RETURN_IF_ERROR(decode_blob_ref(index.core_metadata(), &entry.core_metadata)); + RETURN_IF_ERROR(decode_blob_ref(index.sampled_term_index(), &entry.sampled_term_index)); + RETURN_IF_ERROR(decode_blob_ref(index.dict_block_directory(), &entry.dict_block_directory)); + + for (const auto& existing : entries) { + if (existing.index_id == entry.index_id && + existing.index_suffix == entry.index_suffix) { + return corrupted("metadata directory: duplicate logical index key"); + } + } + entries.push_back(std::move(entry)); + } + *out = std::move(entries); + return Status::OK(); +} + +void encode_blob_ref(const MetadataBlobRef& input, doris::snii::SniiBlobRefPB* out) { + out->set_offset(input.offset); + out->set_length(input.length); +} + +} // namespace + +Status MetadataDirectory::decode(Slice bytes, MetadataDirectory* out) { + if (out == nullptr) { + return Status::Error("metadata directory: null output"); + } + out->entries_.clear(); + if (bytes.size() > static_cast(std::numeric_limits::max())) { + return corrupted("metadata directory: protobuf payload exceeds INT_MAX"); + } + + doris::snii::SniiMetadataDirectoryPB directory; + if (!directory.ParseFromArray(bytes.data(), static_cast(bytes.size()))) { + return corrupted("metadata directory: protobuf parsing failed"); + } + + std::vector entries; + RETURN_IF_ERROR(decode_directory_pb(directory, &entries)); + out->entries_ = std::move(entries); + return Status::OK(); +} + +const LogicalIndexMetadataRef* MetadataDirectory::find(uint64_t index_id, + std::string_view suffix) const { + for (const auto& entry : entries_) { + if (entry.index_id == index_id && entry.index_suffix == suffix) { + return &entry; + } + } + return nullptr; +} + +Status encode_metadata_directory(const std::vector& entries, + ByteSink* out) { + if (out == nullptr) { + return Status::Error("metadata directory: null output"); + } + + doris::snii::SniiMetadataDirectoryPB directory; + for (const auto& entry : entries) { + auto* index = directory.add_indexes(); + index->set_index_id(entry.index_id); + index->set_index_suffix(entry.index_suffix); + encode_blob_ref(entry.core_metadata, index->mutable_core_metadata()); + encode_blob_ref(entry.sampled_term_index, index->mutable_sampled_term_index()); + encode_blob_ref(entry.dict_block_directory, index->mutable_dict_block_directory()); + } + + std::vector validated; + RETURN_IF_ERROR(decode_directory_pb(directory, &validated)); + const size_t size = directory.ByteSizeLong(); + if (size > static_cast(std::numeric_limits::max())) { + return corrupted("metadata directory: protobuf payload exceeds INT_MAX"); + } + std::string payload(size, '\0'); + if (!directory.SerializeToArray(payload.data(), static_cast(size))) { + return corrupted("metadata directory: protobuf serialization failed"); + } + out->put_bytes(Slice(payload)); + return Status::OK(); +} + +} // namespace doris::snii::format diff --git a/be/src/storage/index/snii/format/metadata_directory.h b/be/src/storage/index/snii/format/metadata_directory.h new file mode 100644 index 00000000000000..a25fdcfc668a14 --- /dev/null +++ b/be/src/storage/index/snii/format/metadata_directory.h @@ -0,0 +1,60 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/encoding/byte_sink.h" + +namespace doris::snii::format { + +struct MetadataBlobRef { + uint64_t offset = 0; + uint64_t length = 0; +}; + +struct LogicalIndexMetadataRef { + uint64_t index_id = 0; + std::string index_suffix; + MetadataBlobRef core_metadata; + MetadataBlobRef sampled_term_index; + MetadataBlobRef dict_block_directory; +}; + +class MetadataDirectory { +public: + static Status decode(Slice bytes, MetadataDirectory* out); + + const LogicalIndexMetadataRef* find(uint64_t index_id, std::string_view suffix) const; + size_t size() const { return entries_.size(); } + const std::vector& entries() const { return entries_; } + +private: + std::vector entries_; +}; + +Status encode_metadata_directory(const std::vector& entries, + ByteSink* out); + +} // namespace doris::snii::format diff --git a/be/src/storage/index/snii/format/norms_pod.cpp b/be/src/storage/index/snii/format/norms_pod.cpp new file mode 100644 index 00000000000000..b3da050c17726f --- /dev/null +++ b/be/src/storage/index/snii/format/norms_pod.cpp @@ -0,0 +1,85 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/format/norms_pod.h" + +#include + +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/encoding/byte_source.h" +#include "storage/index/snii/encoding/section_framer.h" +#include "storage/index/snii/encoding/varint.h" +#include "storage/index/snii/format/format_constants.h" + +namespace doris::snii::format { + +void NormsPodWriter::finish(ByteSink* sink) const { + finish(norms_, sink); +} + +void NormsPodWriter::finish(std::span norms, ByteSink* sink) { + // Build inner payload: [varint64 doc_count][raw norm bytes]. + ByteSink payload; + const size_t payload_size = varint_len(norms.size()) + norms.size(); + payload.reserve(payload_size); + payload.put_varint64(norms.size()); + payload.put_bytes(Slice(norms.data(), norms.size())); + // Delegate outer framing to SectionFramer to append type+len+crc32c, avoiding manual checksum assembly. + sink->reserve(1 + varint_len(payload_size) + payload_size + sizeof(uint32_t)); + SectionFramer::write(*sink, static_cast(SectionType::kNormsPod), payload.view()); +} + +Status NormsPodReader::open(Slice framed, NormsPodReader* out) { + // framer handles CRC verify, truncation detection, and payload slicing. + ByteSource src(framed); + FramedSection sec; + RETURN_IF_ERROR(SectionFramer::read(src, &sec)); + if (sec.type != static_cast(SectionType::kNormsPod)) { + return Status::Error( + "norms POD section type mismatch"); + } + if (!src.eof()) { + return Status::Error( + "norms POD trailing framed bytes"); + } + + // Parse inner payload: [varint64 doc_count][bytes]. + ByteSource payload(sec.payload); + uint64_t doc_count = 0; + RETURN_IF_ERROR(payload.get_varint64(&doc_count)); + if (payload.position() != varint_len(doc_count)) { + return Status::Error( + "norms POD non-canonical doc_count"); + } + if (doc_count > std::numeric_limits::max()) { + return Status::Error( + "norms POD doc_count overflows uint32"); + } + // doc_count must exactly equal the remaining byte count (1 byte per doc). + if (payload.remaining() != doc_count) { + return Status::Error( + "norms POD length mismatch"); + } + + Slice bytes; + RETURN_IF_ERROR(payload.get_bytes(static_cast(doc_count), &bytes)); + out->doc_count_ = static_cast(doc_count); + out->norms_ = bytes.data(); + return Status::OK(); +} + +} // namespace doris::snii::format diff --git a/be/src/storage/index/snii/format/norms_pod.h b/be/src/storage/index/snii/format/norms_pod.h new file mode 100644 index 00000000000000..11791240615a65 --- /dev/null +++ b/be/src/storage/index/snii/format/norms_pod.h @@ -0,0 +1,89 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/encoding/byte_sink.h" + +namespace doris::snii::format { + +// norms POD: per logical index / field stores 1-byte encoded doc length per doc, +// used by BM25 length normalization (SniiStatsProvider::encoded_norm) for per-docid lookup. +// +// On-disk layout (the whole section is framed by SectionFramer, which adds a type+len+crc32c envelope): +// framer payload = [varint64 doc_count][bytes encoded_norm[doc_count]] +// framer envelope = [u8 type][varint64 payload_len][payload][fixed32 crc32c] +// The encoding of encoded_norm (length -> 1B) is out of scope for this module; here we only handle raw byte storage and retrieval. +class NormsPodWriter { +public: + // Appends the encoded_norm for the next docid (docid is implicit, assigned in append order starting from 0). + void add(uint8_t encoded_norm) { norms_.push_back(encoded_norm); } + + // Number of docs accumulated so far (i.e., the next docid to be assigned). + size_t count() const { return norms_.size(); } + + // Writes [doc_count][bytes] framed by SectionFramer into sink (appends; does not clear sink). + void finish(ByteSink* sink) const; + // Zero-copy source overload used by streamed compaction. + static void finish(std::span norms, ByteSink* sink); + +private: + std::vector norms_; +}; + +// Read-only view: on open, verifies the framer CRC and checks that doc_count/payload length are consistent, +// afterwards encoded_norm(docid) is O(1) direct indexing (zero-copy, borrows the underlying buffer). +class NormsPodReader { +public: + NormsPodReader() = default; + + // Parses the entire section (including the framer envelope). Returns Corruption on CRC mismatch, truncation, or length inconsistency. + // On success, *out borrows the memory pointed to by framer_payload; the caller must ensure its lifetime. + static Status open(Slice framed, NormsPodReader* out); + + uint32_t doc_count() const { return doc_count_; } + + // Precondition (hard contract): docid < doc_count(). Semantics match std::vector::operator[]: + // the caller is responsible for guaranteeing this (docid comes from trusted postings decoded internally by SNII). Asserts in debug builds; + // no check in Release (NDEBUG). Use try_encoded_norm when the docid is untrusted and needs validation. + uint8_t encoded_norm(uint32_t docid) const { + assert(docid < doc_count_); + return norms_[docid]; + } + + // Checked access: returns InvalidArgument if docid is out of range; never reads out-of-range memory. + Status try_encoded_norm(uint32_t docid, uint8_t* out) const { + if (docid >= doc_count_) + return Status::Error("norms: docid out of range"); + *out = norms_[docid]; + return Status::OK(); + } + +private: + const uint8_t* norms_ = nullptr; + uint32_t doc_count_ = 0; +}; + +} // namespace doris::snii::format diff --git a/be/src/storage/index/snii/format/null_bitmap.cpp b/be/src/storage/index/snii/format/null_bitmap.cpp new file mode 100644 index 00000000000000..9e7d0d298f99f8 --- /dev/null +++ b/be/src/storage/index/snii/format/null_bitmap.cpp @@ -0,0 +1,295 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/format/null_bitmap.h" + +#include +#include +#include +#include + +#include "common/check.h" +// clang-format off +// CRoaring's public header defines ROARING_CONTAINER_T; its internal headers undefine it. +#include "roaring/roaring.hh" +#include "roaring/containers/array.h" +#include "roaring/containers/bitset.h" +#include "roaring/containers/run.h" +// clang-format on +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/encoding/byte_source.h" +#include "storage/index/snii/encoding/section_framer.h" +#include "storage/index/snii/encoding/varint.h" + +namespace doris::snii::format { + +namespace { + +constexpr uint32_t kPortableCookieNoRun = 12346; +constexpr uint16_t kPortableCookieRun = 12347; +constexpr uint32_t kMaxPortableContainers = uint32_t {1} << 16; + +struct ParsedNullBitmap { + uint32_t doc_count = 0; + Slice roaring_bytes; + uint32_t container_count = 0; +}; + +Status parse_null_bitmap(Slice framed, ParsedNullBitmap* out) { + ByteSource src(framed); + FramedSection sec; + RETURN_IF_ERROR(SectionFramer::read(src, &sec)); + if (sec.type != kNullBitmapSectionType || src.remaining() != 0) { + return Status::Error( + "null bitmap: invalid framed section"); + } + + ByteSource payload(sec.payload); + uint64_t doc_count = 0; + RETURN_IF_ERROR(payload.get_varint64(&doc_count)); + if (doc_count > std::numeric_limits::max()) { + return Status::Error( + "null bitmap doc_count overflows uint32"); + } + + uint64_t roaring_size = 0; + RETURN_IF_ERROR(payload.get_varint64(&roaring_size)); + if (roaring_size != payload.remaining()) { + return Status::Error( + "null bitmap roaring_size differs from payload"); + } + RETURN_IF_ERROR(payload.get_bytes(static_cast(roaring_size), &out->roaring_bytes)); + + ByteSource portable(out->roaring_bytes); + uint32_t cookie = 0; + RETURN_IF_ERROR(portable.get_fixed32(&cookie)); + if (static_cast(cookie) == kPortableCookieRun) { + out->container_count = (cookie >> 16) + 1; + } else if (cookie == kPortableCookieNoRun) { + RETURN_IF_ERROR(portable.get_fixed32(&out->container_count)); + if (out->container_count > kMaxPortableContainers) { + return Status::Error( + "null bitmap: portable container count out of range"); + } + } else { + return Status::Error( + "null bitmap: invalid portable cookie"); + } + + const char* data = reinterpret_cast(out->roaring_bytes.data()); + const size_t size = out->roaring_bytes.size(); + const size_t probed = roaring::api::roaring_bitmap_portable_deserialize_size(data, size); + if (probed == 0 || probed != size) { + return Status::Error( + "null bitmap: malformed roaring container"); + } + out->doc_count = static_cast(doc_count); + return Status::OK(); +} + +} // namespace + +NullBitmapWriter:: + NullBitmapWriter() // NOLINT(modernize-use-equals-default): roaring type is incomplete in the header. + : bitmap_(std::make_unique()) {} + +NullBitmapWriter::~NullBitmapWriter() = default; + +void NullBitmapWriter::add_null(uint32_t docid) { + bitmap_->add(docid); +} + +void NullBitmapWriter::add_many(std::span docids) { + bitmap_->addMany(docids.size(), docids.data()); +} + +uint32_t NullBitmapWriter::null_count() const { + return static_cast(bitmap_->cardinality()); +} + +uint64_t NullBitmapWriter::build_memory_upper_bound(std::span sorted_docids) { + if (sorted_docids.empty()) { + return 0; + } + + uint64_t container_count = 0; + uint64_t sparse_container_count = 0; + uint64_t sparse_value_count = 0; + uint64_t dense_container_count = 0; + size_t begin = 0; + while (begin < sorted_docids.size()) { + const uint32_t key = sorted_docids[begin] >> 16; + size_t end = begin + 1; + while (end < sorted_docids.size() && sorted_docids[end] >> 16 == key) { + DCHECK_GT(sorted_docids[end], sorted_docids[end - 1]); + ++end; + } + const uint64_t cardinality = end - begin; + ++container_count; + if (cardinality <= roaring::internal::DEFAULT_MAX_SIZE) { + ++sparse_container_count; + sparse_value_count += cardinality; + } else { + ++dense_container_count; + } + begin = end; + } + + // roaring_array_t uses three parallel arrays. Account a minimum allocation + // of four slots and old+replacement overlap at geometric growth. + constexpr uint64_t kTopEntryBytes = sizeof(void*) + sizeof(uint16_t) + sizeof(uint8_t); + const uint64_t top_capacity = std::max(container_count, 4); + const uint64_t top_array_peak = 3 * top_capacity * kTopEntryBytes; + + // Sparse array growth can retain the old uint16 array while allocating its + // replacement. Eight bytes per live value covers both capacity slack and + // replacement overlap. A dense container additionally covers the largest + // geometric array capacity immediately before conversion and the new 8 KiB + // bitset while both allocations are live. + constexpr uint64_t kSparseValuePeakBytes = 8; + constexpr uint64_t kBitsetBytes = + roaring::internal::BITSET_CONTAINER_SIZE_IN_WORDS * sizeof(uint64_t); + constexpr uint64_t kDenseArrayConversionCapacity = roaring::internal::DEFAULT_MAX_SIZE * 5 / 4; + constexpr uint64_t kDenseArrayConversionBytes = + kDenseArrayConversionCapacity * sizeof(uint16_t); + const uint64_t sparse_peak = + sparse_value_count * kSparseValuePeakBytes + + sparse_container_count * sizeof(roaring::internal::array_container_t); + const uint64_t dense_peak = + dense_container_count * (kDenseArrayConversionBytes + kBitsetBytes + + sizeof(roaring::internal::array_container_t) + + sizeof(roaring::internal::bitset_container_t)); + return sizeof(roaring::Roaring) + top_array_peak + sparse_peak + dense_peak; +} + +Status NullBitmapWriter::serialization_sizes(uint32_t doc_count, + NullBitmapSerializationSizes* out) const { + if (out == nullptr) { + return Status::Error( + "null bitmap: null serialization size output"); + } + const size_t roaring_bytes = bitmap_->getSizeInBytes(); + const size_t prefix_bytes = varint_len(doc_count) + varint_len(roaring_bytes); + if (roaring_bytes > std::numeric_limits::max() - prefix_bytes) { + return Status::Error( + "null bitmap: payload size overflows"); + } + const size_t payload_bytes = prefix_bytes + roaring_bytes; + const size_t envelope_bytes = 1 + varint_len(payload_bytes) + sizeof(uint32_t); + if (payload_bytes > std::numeric_limits::max() - envelope_bytes) { + return Status::Error( + "null bitmap: framed size overflows"); + } + *out = {.roaring_bytes = roaring_bytes, + .payload_bytes = payload_bytes, + .framed_bytes = envelope_bytes + payload_bytes}; + return Status::OK(); +} + +Status NullBitmapWriter::finish(uint32_t doc_count, ByteSink* sink) const { + if (sink == nullptr) { + return Status::Error("null bitmap: null output sink"); + } + NullBitmapSerializationSizes sizes; + RETURN_IF_ERROR(serialization_sizes(doc_count, &sizes)); + + // Serialize the Roaring bitmap to its portable on-disk form. + std::vector roaring_buf(sizes.roaring_bytes); + bitmap_->write(roaring_buf.data()); + + // Build inner payload: [varint64 doc_count][varint64 roaring_size][bytes]. + ByteSink payload; + payload.reserve(sizes.payload_bytes); + payload.put_varint64(doc_count); + payload.put_varint64(sizes.roaring_bytes); + payload.put_bytes( + Slice(reinterpret_cast(roaring_buf.data()), sizes.roaring_bytes)); + DORIS_CHECK_EQ(payload.size(), sizes.payload_bytes); + + // Delegate the type + len + crc32c envelope to SectionFramer. + const size_t start = sink->size(); + sink->reserve(sizes.framed_bytes); + SectionFramer::write(*sink, kNullBitmapSectionType, payload.view()); + DORIS_CHECK_EQ(sink->size() - start, sizes.framed_bytes); + return Status::OK(); +} + +NullBitmapReader:: + NullBitmapReader() // NOLINT(modernize-use-equals-default): roaring type is incomplete in the header. + : bitmap_(std::make_unique()) {} + +NullBitmapReader::~NullBitmapReader() = default; + +NullBitmapReader::NullBitmapReader(NullBitmapReader&&) noexcept = default; +NullBitmapReader& NullBitmapReader::operator=(NullBitmapReader&&) noexcept = default; + +Status NullBitmapReader::open(Slice framed, NullBitmapReader* out) { + ParsedNullBitmap parsed; + RETURN_IF_ERROR(parse_null_bitmap(framed, &parsed)); + *out->bitmap_ = + roaring::Roaring::readSafe(reinterpret_cast(parsed.roaring_bytes.data()), + parsed.roaring_bytes.size()); + out->doc_count_ = parsed.doc_count; + return Status::OK(); +} + +Status NullBitmapReader::decoded_memory_bytes(Slice framed, uint64_t* out) { + if (out == nullptr) { + return Status::Error( + "null bitmap: null decoded memory output"); + } + ParsedNullBitmap parsed; + RETURN_IF_ERROR(parse_null_bitmap(framed, &parsed)); + + constexpr uint64_t kContainerObjectBytes = + std::max({sizeof(roaring::internal::array_container_t), + sizeof(roaring::internal::bitset_container_t), + sizeof(roaring::internal::run_container_t)}); + constexpr uint64_t kContainerMetadataBytes = + sizeof(void*) + sizeof(uint16_t) + sizeof(uint8_t) + kContainerObjectBytes; + constexpr uint64_t kFixedBytes = + sizeof(roaring::Roaring) + sizeof(roaring::api::roaring_bitmap_t); + const uint64_t container_bytes = + static_cast(parsed.container_count) * kContainerMetadataBytes; + if (parsed.roaring_bytes.size() > + std::numeric_limits::max() - container_bytes - kFixedBytes) { + return Status::Error( + "null bitmap: decoded memory size overflows"); + } + *out = parsed.roaring_bytes.size() + container_bytes + kFixedBytes; + return Status::OK(); +} + +bool NullBitmapReader::is_null(uint32_t docid) const { + return bitmap_->contains(docid); +} + +uint32_t NullBitmapReader::null_count() const { + return static_cast(bitmap_->cardinality()); +} + +void NullBitmapReader::copy_to(roaring::Roaring* out) const { + *out = *bitmap_; +} + +void NullBitmapReader::append_docids(std::vector& out) const { + for (uint32_t docid : *bitmap_) { + out.push_back(docid); + } +} + +} // namespace doris::snii::format diff --git a/be/src/storage/index/snii/format/null_bitmap.h b/be/src/storage/index/snii/format/null_bitmap.h new file mode 100644 index 00000000000000..a975a76d399d5d --- /dev/null +++ b/be/src/storage/index/snii/format/null_bitmap.h @@ -0,0 +1,131 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/encoding/byte_sink.h" + +// Forward-declare the CRoaring C++ bitmap so this header stays free of the +// (large) roaring include; the concrete type is only needed in the .cpp. +namespace roaring { +class Roaring; +} // namespace roaring + +namespace doris::snii::format { + +// SectionFramer type byte for the null-bitmap POD. There is no dedicated +// SectionType enum value yet, so we use a documented literal (0x20) outside the +// currently allocated enum range (1..9) to avoid colliding with existing types. +inline constexpr uint8_t kNullBitmapSectionType = 0x20; + +struct NullBitmapSerializationSizes { + size_t roaring_bytes = 0; + size_t payload_bytes = 0; + size_t framed_bytes = 0; +}; + +// NullBitmap POD: per logical index, a Roaring bitmap of null docids (docs whose +// value is NULL / not indexed). It decouples per-doc NULL information from the +// per-term dictionary / postings so NULL handling can pull only this side POD. +// +// On-disk layout (the whole section is framed by SectionFramer, which adds a +// type + varint64 len + payload + fixed32 crc32c envelope): +// framer payload = [varint64 doc_count][varint64 roaring_size][roaring_bytes] +// roaring_bytes is the portable CRoaring serialization (Roaring::write). +class NullBitmapWriter { +public: + NullBitmapWriter(); + ~NullBitmapWriter(); + + NullBitmapWriter(const NullBitmapWriter&) = delete; + NullBitmapWriter& operator=(const NullBitmapWriter&) = delete; + + // Marks docid as NULL (adding the same docid twice is idempotent). + void add_null(uint32_t docid); + void add_many(std::span docids); + + // Number of distinct null docids accumulated so far. + uint32_t null_count() const; + + // Conservative pre-allocation charge for constructing CRoaring from sorted + // docids. It includes top-level array growth and array-to-bitset conversion + // overlap, so the caller must retain this charge until the bitmap is destroyed. + static uint64_t build_memory_upper_bound(std::span sorted_docids); + + Status serialization_sizes(uint32_t doc_count, NullBitmapSerializationSizes* out) const; + + // Serializes [doc_count][roaring_size][roaring_bytes] framed by SectionFramer + // and appends it to sink (does not clear sink). doc_count is the total number + // of docs in the logical index (recorded so the reader can round-trip it). + Status finish(uint32_t doc_count, ByteSink* sink) const; + +private: + std::unique_ptr bitmap_; +}; + +// Read-only view: on open, SectionFramer verifies the CRC and truncation; this +// class then guards roaring_size against the remaining payload bytes before +// deserializing the Roaring bitmap (anti-DoS), so a corrupt size cannot trigger +// an oversized allocation/read. is_null() is then an O(1) membership test. +class NullBitmapReader { +public: + NullBitmapReader(); + ~NullBitmapReader(); + + NullBitmapReader(const NullBitmapReader&) = delete; + NullBitmapReader& operator=(const NullBitmapReader&) = delete; + NullBitmapReader(NullBitmapReader&&) noexcept; + NullBitmapReader& operator=(NullBitmapReader&&) noexcept; + + // Parses the entire section (framer envelope + payload). Returns Corruption on + // CRC mismatch, truncation, doc_count overflow, or an oversized roaring_size. + static Status open(Slice framed, NullBitmapReader* out); + + // Exact pre-allocation charge for CRoaring::readSafe, excluding the caller's + // framed input bytes and decoded docid output vector. + static Status decoded_memory_bytes(Slice framed, uint64_t* out); + + // True iff docid was marked NULL. docids outside the null set (including those + // >= doc_count) return false. + bool is_null(uint32_t docid) const; + + // Number of distinct null docids in the bitmap. + uint32_t null_count() const; + + // Copies the decoded bitmap into the caller-owned Roaring object. + void copy_to(roaring::Roaring* out) const; + + // Appends decoded docids in ascending order without cloning the bitmap. + void append_docids(std::vector& out) const; + + // Total doc count of the logical index, as recorded by the writer. + uint32_t doc_count() const { return doc_count_; } + +private: + std::unique_ptr bitmap_; + uint32_t doc_count_ = 0; +}; + +} // namespace doris::snii::format diff --git a/be/src/storage/index/snii/format/phrase_bigram.h b/be/src/storage/index/snii/format/phrase_bigram.h new file mode 100644 index 00000000000000..3d88f897e9bd6b --- /dev/null +++ b/be/src/storage/index/snii/format/phrase_bigram.h @@ -0,0 +1,33 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +namespace doris::snii::format { + +inline constexpr std::string_view kPhraseBigramTermMarker = + "\x1F" + "SNII_PHRASE_BIGRAM" + "\x1F"; + +inline bool is_phrase_bigram_term(std::string_view term) { + return term.starts_with(kPhraseBigramTermMarker); +} + +} // namespace doris::snii::format diff --git a/be/src/storage/index/snii/format/prx_decode_stats.h b/be/src/storage/index/snii/format/prx_decode_stats.h new file mode 100644 index 00000000000000..f4c3bb093cba88 --- /dev/null +++ b/be/src/storage/index/snii/format/prx_decode_stats.h @@ -0,0 +1,106 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "common/status.h" + +namespace doris::snii::format { + +// Optional allocation seam for full CSR decode callers that must gate every +// retained/output allocation before touching the physical buffers. Query +// decoders leave this null; native compaction supplies its shared-budget +// workspace. Implementations must leave both CSR buffers unchanged when a +// reservation fails. +class PrxCsrAllocationGate { +public: + virtual ~PrxCsrAllocationGate() = default; + + virtual Status reserve_csr(std::vector* pos_flat, size_t position_count, + std::vector* pos_off, size_t offset_count) = 0; + virtual Status reserve_decompression(size_t bytes, std::vector** buffer) = 0; +}; + +struct PrxDecodeStats { + uint64_t raw_frames = 0; + uint64_t zstd_frames = 0; + uint64_t pfor_frames = 0; + uint64_t plaintext_bytes = 0; + uint64_t total_docs = 0; + uint64_t selected_docs = 0; + uint64_t total_positions = 0; + uint64_t selected_positions = 0; + uint64_t fetch_ns = 0; + // Inclusive successful-frame time: header/CRC validation, optional + // decompression, and payload decode. + uint64_t decode_ns = 0; + // Phrase verification excluding only the inclusive decode_ns delta. + uint64_t phrase_verify_ns = 0; + + void merge(const PrxDecodeStats& other); + [[nodiscard]] uint64_t frame_count() const { return raw_frames + zstd_frames + pfor_frames; } + [[nodiscard]] bool is_valid() const { + return selected_docs <= total_docs && selected_positions <= total_positions; + } + bool operator==(const PrxDecodeStats&) const = default; +}; + +struct PrxDecodedShape { + uint32_t total_docs = 0; + uint64_t total_positions = 0; + uint32_t max_frequency = 0; + bool has_zero_frequency = false; +}; + +// Query-plan and matcher calibration inputs are deliberately separate from PrxDecodeStats: the +// latter's 11 production counters remain a stable decode contract. +struct PhraseQueryExecutionStats { + uint64_t exact_candidate_docs = 0; + uint64_t exact_candidate_visits = 0; + uint64_t prefix_leading_candidate_docs = 0; + uint64_t prefix_tail_candidate_visits = 0; + uint64_t common_grams_candidate_queries = 0; + uint64_t common_grams_plain_plans = 0; + uint64_t common_grams_gram_plans = 0; + uint64_t common_grams_fallback_no_gram = 0; + uint64_t common_grams_fallback_incompatible = 0; + uint64_t common_grams_fallback_kill_switch = 0; + uint64_t common_grams_fallback_cost = 0; + uint64_t common_grams_fallback_base_analyzer_mismatch = 0; + uint64_t common_grams_fallback_prefix_tail_empty = 0; + uint64_t common_grams_authoritative_empty = 0; + uint64_t common_grams_plain_posting_bytes = 0; + uint64_t common_grams_gram_posting_bytes = 0; + uint64_t common_grams_plain_estimated_candidate_df = 0; + uint64_t common_grams_gram_estimated_candidate_df = 0; + uint64_t common_grams_plain_estimated_cost = 0; + uint64_t common_grams_gram_estimated_cost = 0; + uint64_t common_grams_planning_ns = 0; +}; + +struct PrxDecodeContext { + PrxDecodeStats* stats = nullptr; + PrxDecodedShape* shape = nullptr; + PhraseQueryExecutionStats* query_stats = nullptr; + PrxCsrAllocationGate* allocation_gate = nullptr; +}; + +} // namespace doris::snii::format diff --git a/be/src/storage/index/snii/format/prx_pod.cpp b/be/src/storage/index/snii/format/prx_pod.cpp new file mode 100644 index 00000000000000..a336bfc2b833db --- /dev/null +++ b/be/src/storage/index/snii/format/prx_pod.cpp @@ -0,0 +1,1459 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/format/prx_pod.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/common/uninitialized_buffer.h" +#include "storage/index/snii/encoding/byte_source.h" +#include "storage/index/snii/encoding/crc32c.h" +#include "storage/index/snii/encoding/pfor.h" +#include "storage/index/snii/encoding/zstd_codec.h" +#include "storage/index/snii/format/format_constants.h" + +namespace doris::snii::format { + +void PrxDecodeStats::merge(const PrxDecodeStats& other) { + raw_frames += other.raw_frames; + zstd_frames += other.zstd_frames; + pfor_frames += other.pfor_frames; + plaintext_bytes += other.plaintext_bytes; + total_docs += other.total_docs; + selected_docs += other.selected_docs; + total_positions += other.total_positions; + selected_positions += other.selected_positions; + fetch_ns += other.fetch_ns; + decode_ns += other.decode_ns; + phrase_verify_ns += other.phrase_verify_ns; +} + +Status validate_prx_window_limits(const PrxWindowLimits& limits) { + if (limits.max_docs == 0 || limits.max_positions == 0 || limits.max_uncomp_bytes == 0) { + return Status::Error( + "prx: window limits must be non-zero"); + } + if (limits.max_docs > kReaderPrxWindowLimits.max_docs || + limits.max_positions > kReaderPrxWindowLimits.max_positions || + limits.max_uncomp_bytes > kReaderPrxWindowLimits.max_uncomp_bytes) { + return Status::Error( + "prx: writer window limits exceed reader limits"); + } + return Status::OK(); +} + +namespace { + +using PrxClock = std::chrono::steady_clock; + +PrxClock::time_point prx_clock_now() { +#ifdef BE_TEST + testing::note_prx_clock_read(); +#endif + return PrxClock::now(); +} + +uint64_t elapsed_ns(PrxClock::time_point start) { + const auto elapsed = + std::chrono::duration_cast(prx_clock_now() - start).count(); + return std::max(1, static_cast(elapsed)); +} + +// Auto-compression threshold: use raw when payload is smaller than this (zstd +// gain is negligible and metadata overhead is relatively large). +inline constexpr size_t kAutoZstdMinBytes = 512; +// Default zstd level in auto mode. +inline constexpr int kDefaultZstdLevel = 3; +// Maximum decompressed byte size for a single .prx window. Guards against a +// corrupted uncomp_len read from S3 inflated to a huge value: sanity-check +// before allocating/decompressing to avoid GB-scale allocations. Windows are +// 256-doc aligned and normally far below this limit. +inline constexpr uint32_t kMaxWindowUncompBytes = kReaderPrxWindowLimits.max_uncomp_bytes; +// Anti-DoS cap on position count decoded from a single window before +// allocation. +inline constexpr uint32_t kMaxWindowPositions = + kReaderPrxWindowLimits.max_positions; // 64M positions/window +// Anti-DoS cap on doc count decoded from a single window before allocation. A +// corrupt doc_count is otherwise fed straight to assign()/reserve() -> +// bad_alloc. +inline constexpr uint32_t kMaxWindowDocs = kReaderPrxWindowLimits.max_docs; // 16M docs/window + +// Writer-side precondition for the FLAT builders: the per-doc partition `freqs` +// must address exactly the positions present in `flat`. If sum(freqs) overruns +// flat.size() a (positions_flat, freqs) mismatch would index flat[off+i] past +// the span end -- an out-of-bounds read on caller-supplied data. Reject it as +// InvalidArgument BEFORE any indexing so the bug surfaces as a clean Status, +// never UB. (sum < size leaves trailing positions unused, which is also a +// writer bug, so we require exact equality.) Uint64 accumulation cannot +// overflow for uint32 freqs. +Status check_flat_partition(std::span flat, std::span freqs) { + size_t sum = 0; + for (uint32_t fc : freqs) { + if (fc > flat.size() - sum) { + return Status::Error( + "prx: sum(freqs) exceeds positions_flat size"); + } + sum += fc; + } + if (sum != flat.size()) { + return Status::Error( + "prx: sum(freqs) does not match positions_flat size"); + } + return Status::OK(); +} + +Status validate_flat_positions(std::span flat, std::span freqs) { + size_t off = 0; + for (uint32_t fc : freqs) { + uint32_t previous = 0; + for (uint32_t i = 0; i < fc; ++i) { + const uint32_t position = flat[off + i]; + if (i != 0 && position < previous) { + return Status::Error( + "prx: positions within a doc must be ascending"); + } + previous = position; + } + off += fc; + } + return Status::OK(); +} + +Status validate_per_doc_window(std::span> per_doc, + const PrxWindowLimits& limits, size_t* total_positions) { + RETURN_IF_ERROR(validate_prx_window_limits(limits)); + if (per_doc.size() > limits.max_docs) { + return Status::Error( + "prx: doc count exceeds writer window limit"); + } + uint64_t total = 0; + for (const auto& positions : per_doc) { + total += positions.size(); + if (total > limits.max_positions) { + return Status::Error( + "prx: position count exceeds writer window limit"); + } + } + *total_positions = static_cast(total); + return Status::OK(); +} + +// Encode per-doc position lists into a self-describing plain payload (doc_count +// + per-doc delta stream). +Status encode_payload(std::span> per_doc, ByteSink* out) { + out->put_varint32(static_cast(per_doc.size())); + for (const auto& doc : per_doc) { + out->put_varint32(static_cast(doc.size())); + uint32_t prev = 0; + for (size_t i = 0; i < doc.size(); ++i) { + uint32_t pos = doc[i]; + if (i > 0 && pos < prev) { + return Status::Error( + "prx: positions within a doc must be ascending"); + } + out->put_varint32(i == 0 ? pos : pos - prev); + prev = pos; + } + } + return Status::OK(); +} + +// FLAT-positions encoder: identical wire output to encode_payload above, but +// reads positions from a single flat span partitioned per-doc by `freqs` (doc d +// owns the next freqs[d] entries). The public entry point has already validated +// that sum(freqs) == flat.size(). This avoids materializing a vector-of-vectors +// for the window. +Status encode_payload_flat(std::span flat, std::span freqs, + ByteSink* out) { + out->put_varint32(static_cast(freqs.size())); + size_t off = 0; + for (uint32_t fc : freqs) { + out->put_varint32(fc); + uint32_t prev = 0; + for (uint32_t i = 0; i < fc; ++i) { + const uint32_t pos = flat[off + i]; + if (i > 0 && pos < prev) { + return Status::Error( + "prx: positions within a doc must be ascending"); + } + out->put_varint32(i == 0 ? pos : pos - prev); + prev = pos; + } + off += fc; + } + return Status::OK(); +} + +// Encode a uint32 array into PFOR runs of kFrqBaseUnit (256) elements each. The +// run count is derived by the decoder from the total length, so it is not +// stored. +void encode_pfor_runs(std::span values, ByteSink* out) { + const size_t n = values.size(); + for (size_t off = 0; off < n; off += kFrqBaseUnit) { + const size_t run = (n - off < kFrqBaseUnit) ? (n - off) : kFrqBaseUnit; + pfor_encode(values.data() + off, run, out); + } +} + +// Decode n uint32 values (multiple PFOR runs of kFrqBaseUnit each) into out. +Status decode_pfor_runs(ByteSource* src, size_t n, std::vector* out) { + // Sized then fully overwritten by pfor_decode below (every [0, n) slot is + // written); no zero-fill needed beyond what std::vector mandates. + resize_uninitialized(*out, n); + for (size_t off = 0; off < n; off += kFrqBaseUnit) { + const size_t run = (n - off < kFrqBaseUnit) ? (n - off) : kFrqBaseUnit; + RETURN_IF_ERROR(pfor_decode(src, run, out->data() + off)); + } + return Status::OK(); +} + +size_t varint32_size(uint32_t value) { + size_t bytes = 1; + while (value >= 128) { + value >>= 7; + ++bytes; + } + return bytes; +} + +// Derive the per-doc position deltas ONCE into `deltas` (flat, in doc order: the +// first position of each doc is absolute, the rest are deltas within the doc), +// enforcing the ascending-position precondition after the public entry point +// validated the exact (flat, freqs) partition. The loop is identical to the delta +// derivation the old encode_pfor_payload_flat ran inline, lifted out so the auto +// path can feed BOTH the PFOR payload and (only when needed) the raw plaintext +// payload from one buffer instead of walking `flat` twice. Accumulate the exact +// raw payload size in the same pass so codec selection does not rescan deltas. +Status compute_flat_deltas(std::span flat, std::span freqs, + std::vector* const deltas, size_t* const plain_size) { +#ifdef BE_TEST + testing::note_prx_delta_materialization(); +#endif + deltas->clear(); + deltas->reserve(flat.size()); + *plain_size = varint32_size(static_cast(freqs.size())); + size_t off = 0; + for (uint32_t fc : freqs) { + *plain_size += varint32_size(fc); + uint32_t prev = 0; + for (uint32_t i = 0; i < fc; ++i) { + const uint32_t pos = flat[off + i]; + if (i > 0 && pos < prev) { + return Status::Error( + "prx: positions within a doc must be ascending"); + } + const uint32_t delta = i == 0 ? pos : pos - prev; + deltas->push_back(delta); + *plain_size += varint32_size(delta); + prev = pos; + } + off += fc; + } + return Status::OK(); +} + +// PFOR window payload (self-describing; no entropy coding): +// VInt doc_count +// VInt total_pos # sum of all pos_counts +// PFOR_runs(pos_counts) # doc_count values (bit-packed; mostly 1 -> ~1 +// bit) PFOR_runs(position_deltas) # total_pos deltas, flat across docs (first +// per +// # doc absolute, rest delta-within-doc) +// Bit-packing the per-doc pos_counts (vs one varint each) is the size win: in a +// uniform corpus most docs have freq 1, so the count column packs to ~1 bit/doc. +// Emits byte-for-byte the same payload the old encode_pfor_payload_flat produced +// (doc_count == freqs.size(), total_pos == deltas.size() == sum(freqs)), but +// reads the already-derived `deltas` instead of re-walking the positions. +void encode_pfor_payload_from_deltas(std::span freqs, + std::span deltas, ByteSink* out) { + out->put_varint32(static_cast(freqs.size())); + out->put_varint32(static_cast(deltas.size())); + encode_pfor_runs(freqs, out); + encode_pfor_runs(deltas, out); +} + +// Raw plaintext payload (self-describing per-doc boundaries): +// VInt doc_count +// per doc: VInt pos_count, then pos_count position deltas (VInt) +// Emits byte-for-byte the same payload the old encode_payload_flat produced, but +// reads the already-derived `deltas` instead of re-walking the positions and +// re-running the partition/ascending checks. +void encode_payload_from_deltas(std::span freqs, std::span deltas, + ByteSink* out) { + out->put_varint32(static_cast(freqs.size())); + size_t off = 0; + for (uint32_t fc : freqs) { + out->put_varint32(fc); + for (uint32_t i = 0; i < fc; ++i) { + out->put_varint32(deltas[off + i]); + } + off += fc; + } +} + +// Decode per-doc position lists from a PFOR payload. +Status decode_pfor_payload(Slice plain, std::vector>* out) { + ByteSource src(plain); + uint32_t doc_count = 0, total_pos = 0; + RETURN_IF_ERROR(src.get_varint32(&doc_count)); + RETURN_IF_ERROR(src.get_varint32(&total_pos)); + if (total_pos > kMaxWindowPositions) { + return Status::Error( + "prx: position count exceeds sane cap"); + } + if (doc_count > kMaxWindowDocs) { + return Status::Error( + "prx: doc count exceeds sane cap"); + } + std::vector pos_counts; + RETURN_IF_ERROR(decode_pfor_runs(&src, doc_count, &pos_counts)); + uint64_t sum = 0; + for (uint32_t d = 0; d < doc_count; ++d) sum += pos_counts[d]; + if (sum != total_pos) { + return Status::Error( + "prx: pos_count sum mismatch"); + } + std::vector deltas; + RETURN_IF_ERROR(decode_pfor_runs(&src, total_pos, &deltas)); + out->clear(); + out->reserve(doc_count); + size_t off = 0; + for (uint32_t d = 0; d < doc_count; ++d) { + std::vector doc; + doc.reserve(pos_counts[d]); + uint32_t prev = 0; + for (uint32_t i = 0; i < pos_counts[d]; ++i) { + prev = (i == 0) ? deltas[off + i] : prev + deltas[off + i]; + doc.push_back(prev); + } + off += pos_counts[d]; + out->push_back(std::move(doc)); + } + if (!src.eof()) + return Status::Error( + "prx: trailing bytes after pfor payload"); + return Status::OK(); +} + +// Writes a PFOR window: codec=pfor, payload, crc(header+payload). +void write_pfor(Slice payload, ByteSink* sink) { + // Single-copy framing: write [codec][varint len][payload] straight into the + // caller's sink, then crc exactly those bytes. view() is taken AFTER the + // payload and BEFORE the crc, so subslice([start, framed_len)) is over a + // settled, contiguous buffer with no pending realloc/aliasing. Byte-identical + // to the former temp-ByteSink assembly, minus one heap alloc + one payload copy. + const size_t start = sink->size(); + sink->put_u8(static_cast(PrxCodec::kPfor)); + sink->put_varint32(static_cast(payload.size())); + sink->put_bytes(payload); + const size_t framed_len = sink->size() - start; + const uint32_t crc = crc32c(sink->view().subslice(start, framed_len)); + sink->put_fixed32(crc); +} + +void write_raw(Slice plain, ByteSink* sink); + +// Emit a RAW frame directly from the already-derived deltas. This is used by +// the singleton fast path so choosing RAW does not allocate a temporary plain +// payload or encode/copy a losing PFOR payload first. +void write_raw_from_deltas(std::span freqs, std::span deltas, + size_t plain_size, ByteSink* sink) { + const size_t start = sink->size(); + sink->put_u8(static_cast(PrxCodec::kRaw)); + sink->put_varint32(static_cast(plain_size)); + const size_t payload_start = sink->size(); + encode_payload_from_deltas(freqs, deltas, sink); + DCHECK_EQ(sink->size() - payload_start, plain_size); + const size_t framed_len = sink->size() - start; + const uint32_t crc = crc32c(sink->view().subslice(start, framed_len)); + sink->put_fixed32(crc); +} + +Status validate_single_doc_byte_limits(std::span positions, + std::span freqs, bool auto_codec, + uint32_t max_uncomp_bytes) { + size_t offset = 0; + std::vector one_doc_deltas; + for (size_t doc = 0; doc < freqs.size(); ++doc) { + const uint32_t frequency = freqs[doc]; + const auto one_freq = freqs.subspan(doc, 1); + const auto one_doc_positions = positions.subspan(offset, frequency); + size_t plain_size = varint32_size(1) + varint32_size(frequency); + uint32_t previous = 0; + for (size_t i = 0; i < one_doc_positions.size(); ++i) { + const uint32_t position = one_doc_positions[i]; + plain_size += varint32_size(i == 0 ? position : position - previous); + previous = position; + } + if (plain_size > max_uncomp_bytes) { + if (!auto_codec) { + return Status::Error( + "prx: one document exceeds the writer window byte limit"); + } + if (one_doc_deltas.capacity() < frequency) { + std::vector().swap(one_doc_deltas); + one_doc_deltas.reserve(frequency); + } else { + one_doc_deltas.clear(); + } + previous = 0; + for (size_t i = 0; i < one_doc_positions.size(); ++i) { + const uint32_t position = one_doc_positions[i]; + one_doc_deltas.push_back(i == 0 ? position : position - previous); + previous = position; + } + ByteSink pfor_payload; + encode_pfor_payload_from_deltas(one_freq, one_doc_deltas, &pfor_payload); + if (pfor_payload.size() > max_uncomp_bytes) { + return Status::Error( + "prx: one document exceeds the writer window byte limit"); + } + } + offset += frequency; + } + return Status::OK(); +} + +size_t pfor_frame_size(size_t payload_size) { + return 1 + varint32_size(static_cast(payload_size)) + payload_size + sizeof(uint32_t); +} + +size_t raw_frame_size(size_t plain_size) { + return 1 + varint32_size(static_cast(plain_size)) + plain_size + sizeof(uint32_t); +} + +size_t zstd_frame_size(size_t plain_size, size_t compressed_size) { + return 1 + varint32_size(static_cast(plain_size)) + + varint32_size(static_cast(compressed_size)) + compressed_size + + sizeof(uint32_t); +} + +struct AutoPrxCodecChoice { + PrxCodec codec = PrxCodec::kPfor; + bool readable = false; +}; + +// Select the smallest complete reader-safe frame among the candidates already +// materialized by the zstd/fallback path. Preserve the existing codec on equal +// sizes by considering PFOR, then ZSTD, then RAW and replacing the winner only +// on a strict size reduction. Sub-threshold singleton RAW selection happens +// before PFOR materialization in build_prx_window_auto_from_flat. +AutoPrxCodecChoice select_auto_prx_codec(size_t pfor_payload_size, size_t plain_payload_size, + size_t compressed_payload_size, bool has_compressed, + uint32_t max_uncomp_bytes) { + const bool pfor_readable = pfor_payload_size <= max_uncomp_bytes; + const bool plain_readable = plain_payload_size <= max_uncomp_bytes; + AutoPrxCodecChoice choice; + size_t selected_frame_size = 0; + if (pfor_readable) { + choice = {.codec = PrxCodec::kPfor, .readable = true}; + selected_frame_size = pfor_frame_size(pfor_payload_size); + } + if (has_compressed && plain_readable) { + const size_t frame_size = zstd_frame_size(plain_payload_size, compressed_payload_size); + if (!choice.readable || frame_size < selected_frame_size) { + choice = {.codec = PrxCodec::kZstd, .readable = true}; + selected_frame_size = frame_size; + } + } + if (plain_readable) { + const size_t frame_size = raw_frame_size(plain_payload_size); + if (!choice.readable || frame_size < selected_frame_size) { + choice = {.codec = PrxCodec::kRaw, .readable = true}; + } + } + return choice; +} + +void write_zstd_compressed(Slice plain, Slice compressed, ByteSink* sink) { + // Single-copy framing (see write_pfor): assemble [codec][uncomp_len][comp_len] + // [compressed] in the caller's sink and crc that span before appending the crc. + const size_t start = sink->size(); + sink->put_u8(static_cast(PrxCodec::kZstd)); + sink->put_varint32(static_cast(plain.size())); + sink->put_varint32(static_cast(compressed.size())); + sink->put_bytes(compressed); + const size_t framed_len = sink->size() - start; + const uint32_t crc = crc32c(sink->view().subslice(start, framed_len)); + sink->put_fixed32(crc); +} + +// Shared auto-mode path for BOTH .prx builders. A single-doc/freq=1 window has a +// provably smaller RAW frame: RAW stores three payload varints, while PFOR adds +// two run headers and is always 4-5 bytes larger. Emit that RAW frame directly +// before any PFOR work. Other sub-threshold windows retain the existing PFOR +// policy so an unmeasured second payload encode cannot regress import CPU. At +// and above the zstd threshold the raw plaintext is already required for +// compression, so all materialized candidates participate in the exact complete +// frame-size comparison at no additional encoding cost. +Status build_prx_window_auto_from_flat(std::span positions_flat, + std::span freqs, int zstd_level, + const PrxWindowLimits& limits, ByteSink* sink, + PrxWindowBuildOutcome* outcome) { + if (freqs.size() == 1 && freqs.front() == 1) { + const size_t plain_size = + varint32_size(1) + varint32_size(1) + varint32_size(positions_flat.front()); + if (plain_size > limits.max_uncomp_bytes) { + return Status::Error( + "prx: one document exceeds the writer window byte limit"); + } + write_raw_from_deltas(freqs, positions_flat, plain_size, sink); + *outcome = PrxWindowBuildOutcome::kBuilt; + return Status::OK(); + } + + std::vector deltas; + size_t plain_size = 0; + RETURN_IF_ERROR(compute_flat_deltas(positions_flat, freqs, &deltas, &plain_size)); + const bool plain_readable = plain_size <= limits.max_uncomp_bytes; + + ByteSink payload; + encode_pfor_payload_from_deltas(freqs, deltas, &payload); + const bool pfor_readable = payload.size() <= limits.max_uncomp_bytes; + if (!pfor_readable && !plain_readable) { + if (freqs.size() <= 1) { + return Status::Error( + "prx: one document exceeds the writer window byte limit"); + } + RETURN_IF_ERROR(validate_single_doc_byte_limits(positions_flat, freqs, true, + limits.max_uncomp_bytes)); + *outcome = PrxWindowBuildOutcome::kNeedsSplit; + return Status::OK(); + } + if (plain_readable && (plain_size >= kAutoZstdMinBytes || !pfor_readable)) { + ByteSink plain; + encode_payload_from_deltas(freqs, deltas, &plain); + DCHECK_EQ(plain.size(), plain_size); + std::vector compressed; + const bool has_compressed = plain_size >= kAutoZstdMinBytes; + if (plain_size >= kAutoZstdMinBytes) { + testing::note_prx_raw_build(); + RETURN_IF_ERROR(zstd_compress(plain.view(), zstd_level, &compressed)); + } + const AutoPrxCodecChoice choice = + select_auto_prx_codec(payload.size(), plain.size(), compressed.size(), + has_compressed, limits.max_uncomp_bytes); + DCHECK(choice.readable); + if (choice.codec == PrxCodec::kZstd) { + write_zstd_compressed(plain.view(), Slice(compressed), sink); + } else if (choice.codec == PrxCodec::kRaw) { + write_raw(plain.view(), sink); + } else { + write_pfor(payload.view(), sink); + } + *outcome = PrxWindowBuildOutcome::kBuilt; + return Status::OK(); + } + DCHECK(pfor_readable); + write_pfor(payload.view(), sink); + *outcome = PrxWindowBuildOutcome::kBuilt; + return Status::OK(); +} + +// Decode per-doc position lists from a plain payload. +Status decode_payload(Slice plain, std::vector>* out) { + ByteSource src(plain); + uint32_t doc_count = 0; + RETURN_IF_ERROR(src.get_varint32(&doc_count)); + if (doc_count > kMaxWindowDocs) { + return Status::Error( + "prx: doc count exceeds sane cap"); + } + out->clear(); + out->reserve(doc_count); + for (uint32_t d = 0; d < doc_count; ++d) { + uint32_t pos_count = 0; + RETURN_IF_ERROR(src.get_varint32_fast(&pos_count)); + std::vector doc; + doc.reserve(pos_count); + uint32_t prev = 0; + for (uint32_t i = 0; i < pos_count; ++i) { + uint32_t delta = 0; + RETURN_IF_ERROR(src.get_varint32(&delta)); + if (i != 0 && delta > std::numeric_limits::max() - prev) { + return Status::Error( + "prx: position accumulation overflow"); + } + prev = (i == 0) ? delta : prev + delta; + doc.push_back(prev); + } + out->push_back(std::move(doc)); + } + if (!src.eof()) + return Status::Error( + "prx: trailing bytes after payload"); + return Status::OK(); +} + +// CSR decode of a PFOR payload: all docs' positions into one flat buffer + +// per-doc offsets, with NO per-doc std::vector allocation. `pos_off` has +// doc_count+1 entries (pos_off[0]==0); doc d's positions are +// pos_flat[pos_off[d] .. pos_off[d+1]). +Status decode_pfor_payload_csr(Slice plain, std::vector* pos_flat, + std::vector* pos_off, + PrxCsrAllocationGate* allocation_gate, uint32_t* max_frequency, + bool* has_zero_frequency) { + ByteSource src(plain); + uint32_t doc_count = 0, total_pos = 0; + RETURN_IF_ERROR(src.get_varint32(&doc_count)); + RETURN_IF_ERROR(src.get_varint32(&total_pos)); + if (total_pos > kMaxWindowPositions) { + return Status::Error( + "prx: position count exceeds sane cap"); + } + if (doc_count > kMaxWindowDocs) { + return Status::Error( + "prx: doc count exceeds sane cap"); + } + if (allocation_gate != nullptr) { + RETURN_IF_ERROR(allocation_gate->reserve_csr(pos_flat, total_pos, pos_off, + static_cast(doc_count) + 1)); + } + pos_off->clear(); + pos_off->reserve(static_cast(doc_count) + 1); + RETURN_IF_ERROR(decode_pfor_runs(&src, doc_count, pos_off)); + uint64_t sum = 0; + uint32_t decoded_max_frequency = 0; + bool decoded_zero_frequency = false; + for (uint32_t d = 0; d < doc_count; ++d) { + sum += (*pos_off)[d]; + decoded_max_frequency = std::max(decoded_max_frequency, (*pos_off)[d]); + decoded_zero_frequency |= (*pos_off)[d] == 0; + } + if (sum != total_pos) + return Status::Error( + "prx: pos_count sum mismatch"); + // pos_flat is sized to total_pos by decode_pfor_runs (resize_uninitialized); + // a separate reserve is redundant. pos_off keeps its reserve (push_back below). + RETURN_IF_ERROR(decode_pfor_runs(&src, total_pos, pos_flat)); + size_t off = 0; + uint32_t next_off = 0; + for (uint32_t d = 0; d < doc_count; ++d) { + const uint32_t pos_count = (*pos_off)[d]; + (*pos_off)[d] = next_off; + uint32_t prev = 0; + for (uint32_t i = 0; i < pos_count; ++i) { + uint32_t& value = (*pos_flat)[off + i]; + if (i != 0 && value > std::numeric_limits::max() - prev) { + return Status::Error( + "prx: position accumulation overflow"); + } + prev = (i == 0) ? value : prev + value; + value = prev; + } + off += pos_count; + next_off += pos_count; + } + pos_off->push_back(next_off); + if (!src.eof()) + return Status::Error( + "prx: trailing bytes after pfor payload"); + *max_frequency = decoded_max_frequency; + *has_zero_frequency = decoded_zero_frequency; + return Status::OK(); +} + +Status validate_doc_ordinals(std::span doc_ordinals, uint32_t doc_count) { + uint32_t prev = 0; + for (size_t i = 0; i < doc_ordinals.size(); ++i) { + const uint32_t doc = doc_ordinals[i]; + if (doc >= doc_count) { + return Status::Error( + "prx: selected doc ordinal out of range"); + } + if (i != 0 && doc <= prev) { + return Status::Error( + "prx: selected doc ordinals must be strictly ascending"); + } + prev = doc; + } + return Status::OK(); +} + +struct SelectedRange { + SelectedRange(uint32_t begin_, uint32_t end_, uint32_t out_begin_) + : begin(begin_), end(end_), out_begin(out_begin_) {} + + uint32_t begin; + uint32_t end; + uint32_t out_begin; +}; + +uint32_t count_covered_pfor_runs(std::span selected, uint32_t total_pos) { + if (selected.empty() || total_pos == 0) { + return 0; + } + uint32_t runs = 0; + uint32_t next_run = 0; + for (const SelectedRange& range : selected) { + if (range.begin == range.end) { + continue; + } + const uint32_t first_run = range.begin / kFrqBaseUnit; + const uint32_t last_run = (range.end - 1) / kFrqBaseUnit; + const uint32_t counted_first = std::max(first_run, next_run); + if (counted_first <= last_run) { + runs += last_run - counted_first + 1; + next_run = last_run + 1; + } + } + return runs; +} + +bool should_decode_full_prx_positions(std::span selected, + uint32_t selected_pos_count, uint32_t total_pos) { + if (selected.empty() || total_pos == 0) { + return false; + } + if (selected_pos_count * 2 >= total_pos) { + return true; + } + const uint32_t total_runs = (total_pos + kFrqBaseUnit - 1) / kFrqBaseUnit; + const uint32_t covered_runs = count_covered_pfor_runs(selected, total_pos); + return covered_runs * 4 >= total_runs * 3; +} + +Status decode_selected_pfor_count_ranges(ByteSource* src, uint32_t doc_count, + std::span doc_ordinals, + std::vector& selected, + std::vector& pos_off, uint64_t* total_pos_count, + uint32_t* selected_pos_count, uint32_t* max_frequency, + bool* has_zero_frequency) { + selected.clear(); + selected.reserve(doc_ordinals.size()); + pos_off.clear(); + pos_off.reserve(doc_ordinals.size() + 1); + pos_off.push_back(0); + + *selected_pos_count = 0; + uint32_t delta_begin = 0; + size_t next_doc = 0; + *total_pos_count = 0; + *max_frequency = 0; + *has_zero_frequency = false; + std::array run_buf {}; + for (uint32_t run_begin = 0; run_begin < doc_count; run_begin += kFrqBaseUnit) { + const uint32_t run_len = std::min(kFrqBaseUnit, doc_count - run_begin); + RETURN_IF_ERROR(pfor_decode(src, run_len, run_buf.data())); + for (uint32_t i = 0; i < run_len; ++i) { + const uint32_t d = run_begin + i; + const uint32_t count = run_buf[i]; + *max_frequency = std::max(*max_frequency, count); + *has_zero_frequency |= count == 0; + *total_pos_count += count; + if (*total_pos_count > kMaxWindowPositions) { + return Status::Error( + "prx: pos_count sum exceeds sane cap"); + } + if (next_doc < doc_ordinals.size() && doc_ordinals[next_doc] == d) { + selected.emplace_back(delta_begin, delta_begin + count, *selected_pos_count); + *selected_pos_count += count; + pos_off.push_back(*selected_pos_count); + ++next_doc; + } + delta_begin += count; + } + } + if (next_doc != doc_ordinals.size()) { + return Status::Error( + "prx: selected doc ordinal was not decoded"); + } + return Status::OK(); +} + +Status decode_selected_pfor_positions(ByteSource* src, uint32_t total_pos, + std::span selected, bool decode_all_runs, + std::span pos_flat) { + std::array run_buf {}; + size_t range_idx = 0; + uint32_t prev = 0; + for (uint32_t run_begin = 0; run_begin < total_pos; run_begin += kFrqBaseUnit) { + const uint32_t run_len = std::min(kFrqBaseUnit, total_pos - run_begin); + const uint32_t run_end = run_begin + run_len; + while (range_idx < selected.size() && selected[range_idx].end <= run_begin) { + ++range_idx; + prev = 0; + } + if (!decode_all_runs && + (range_idx == selected.size() || selected[range_idx].begin >= run_end)) { + RETURN_IF_ERROR(pfor_skip(src, run_len)); + continue; + } + + RETURN_IF_ERROR(pfor_decode(src, run_len, run_buf.data())); + while (range_idx < selected.size() && selected[range_idx].begin < run_end) { + const SelectedRange& range = selected[range_idx]; + const uint32_t copy_begin = std::max(range.begin, run_begin); + const uint32_t copy_end = std::min(range.end, run_end); + if (copy_begin == range.begin) { + prev = 0; + } + uint32_t dst = range.out_begin + copy_begin - range.begin; + for (uint32_t off = copy_begin; off < copy_end; ++off) { + const uint32_t delta = run_buf[off - run_begin]; + if (off != range.begin && delta > std::numeric_limits::max() - prev) { + return Status::Error( + "prx: position accumulation overflow"); + } + prev = (off == range.begin) ? delta : prev + delta; + pos_flat[dst++] = prev; + } + if (copy_end < range.end) { + break; + } + ++range_idx; + prev = 0; + } + } + return Status::OK(); +} + +Status decode_pfor_payload_csr_selective(Slice plain, std::span doc_ordinals, + std::vector* pos_flat, + std::vector* pos_off, + uint32_t* decoded_doc_count, + uint64_t* decoded_total_positions, uint32_t* max_frequency, + bool* has_zero_frequency) { + ByteSource src(plain); + uint32_t doc_count = 0, total_pos = 0; + RETURN_IF_ERROR(src.get_varint32(&doc_count)); + RETURN_IF_ERROR(src.get_varint32(&total_pos)); + if (total_pos > kMaxWindowPositions) { + return Status::Error( + "prx: position count exceeds sane cap"); + } + if (doc_count > kMaxWindowDocs) { + return Status::Error( + "prx: doc count exceeds sane cap"); + } + RETURN_IF_ERROR(validate_doc_ordinals(doc_ordinals, doc_count)); + + pos_flat->clear(); + + std::vector selected; + uint64_t sum = 0; + uint32_t selected_pos_count = 0; + RETURN_IF_ERROR(decode_selected_pfor_count_ranges(&src, doc_count, doc_ordinals, selected, + *pos_off, &sum, &selected_pos_count, + max_frequency, has_zero_frequency)); + if (sum != total_pos) { + return Status::Error( + "prx: pos_count sum mismatch"); + } + + const bool decode_all_runs = + should_decode_full_prx_positions(selected, selected_pos_count, total_pos); + pos_flat->resize(selected_pos_count); + RETURN_IF_ERROR(decode_selected_pfor_positions( + &src, total_pos, selected, decode_all_runs, + std::span(pos_flat->data(), pos_flat->size()))); + if (!src.eof()) { + return Status::Error( + "prx: trailing bytes after pfor payload"); + } + *decoded_doc_count = doc_count; + *decoded_total_positions = total_pos; + return Status::OK(); +} + +// CSR decode of a plain (raw) payload. See decode_pfor_payload_csr. +Status scan_payload_csr_shape(Slice plain, uint32_t* doc_count, uint32_t* total_positions) { + ByteSource src(plain); + RETURN_IF_ERROR(src.get_varint32(doc_count)); + if (*doc_count > kMaxWindowDocs) { + return Status::Error( + "prx: doc count exceeds sane cap"); + } + uint64_t total_pos = 0; + for (uint32_t d = 0; d < *doc_count; ++d) { + uint32_t pos_count = 0; + RETURN_IF_ERROR(src.get_varint32_fast(&pos_count)); + total_pos += pos_count; + if (total_pos > kMaxWindowPositions) { + return Status::Error( + "prx: position count exceeds sane cap"); + } + RETURN_IF_ERROR(src.skip_varints(pos_count)); + } + if (!src.eof()) { + return Status::Error( + "prx: trailing bytes after payload"); + } + *total_positions = static_cast(total_pos); + return Status::OK(); +} + +Status decode_payload_csr(Slice plain, std::vector* pos_flat, + std::vector* pos_off, PrxCsrAllocationGate* allocation_gate, + uint32_t* max_frequency, bool* has_zero_frequency) { + if (allocation_gate != nullptr) { + uint32_t preflight_doc_count = 0; + uint32_t preflight_total_positions = 0; + RETURN_IF_ERROR( + scan_payload_csr_shape(plain, &preflight_doc_count, &preflight_total_positions)); + RETURN_IF_ERROR(allocation_gate->reserve_csr(pos_flat, preflight_total_positions, pos_off, + static_cast(preflight_doc_count) + 1)); + } + ByteSource src(plain); + uint32_t doc_count = 0; + RETURN_IF_ERROR(src.get_varint32(&doc_count)); + if (doc_count > kMaxWindowDocs) { + return Status::Error( + "prx: doc count exceeds sane cap"); + } + pos_flat->clear(); + pos_off->clear(); + pos_off->reserve(static_cast(doc_count) + 1); + pos_off->push_back(0); + uint64_t total_pos = 0; + uint32_t decoded_max_frequency = 0; + bool decoded_zero_frequency = false; + for (uint32_t d = 0; d < doc_count; ++d) { + uint32_t pos_count = 0; + RETURN_IF_ERROR(src.get_varint32_fast(&pos_count)); + decoded_max_frequency = std::max(decoded_max_frequency, pos_count); + decoded_zero_frequency |= pos_count == 0; + total_pos += pos_count; + if (total_pos > kMaxWindowPositions) { + return Status::Error( + "prx: position count exceeds sane cap"); + } + // Tight inline prefix-sum decode (single-byte fast path) -- see + // decode_delta_run. Shared with the selective reader below. + RETURN_IF_ERROR(src.decode_delta_run(pos_count, pos_flat)); + pos_off->push_back(static_cast(pos_flat->size())); + } + if (!src.eof()) + return Status::Error( + "prx: trailing bytes after payload"); + *max_frequency = decoded_max_frequency; + *has_zero_frequency = decoded_zero_frequency; + return Status::OK(); +} + +Status decode_payload_csr_selective(Slice plain, std::span doc_ordinals, + std::vector* pos_flat, std::vector* pos_off, + uint32_t* decoded_doc_count, uint64_t* decoded_total_positions, + uint32_t* max_frequency, bool* has_zero_frequency) { + ByteSource src(plain); + uint32_t doc_count = 0; + RETURN_IF_ERROR(src.get_varint32(&doc_count)); + if (doc_count > kMaxWindowDocs) { + return Status::Error( + "prx: doc count exceeds sane cap"); + } + RETURN_IF_ERROR(validate_doc_ordinals(doc_ordinals, doc_count)); + pos_flat->clear(); + pos_off->clear(); + pos_off->reserve(doc_ordinals.size() + 1); + pos_off->push_back(0); + size_t next_doc = 0; + uint64_t total_pos = 0; + uint32_t decoded_max_frequency = 0; + bool decoded_zero_frequency = false; + for (uint32_t d = 0; d < doc_count; ++d) { + uint32_t pos_count = 0; + RETURN_IF_ERROR(src.get_varint32_fast(&pos_count)); + decoded_max_frequency = std::max(decoded_max_frequency, pos_count); + decoded_zero_frequency |= pos_count == 0; + total_pos += pos_count; + if (total_pos > kMaxWindowPositions) { + return Status::Error( + "prx: position count exceeds sane cap"); + } + const bool selected = next_doc < doc_ordinals.size() && doc_ordinals[next_doc] == d; + if (!selected) { + // Skip this doc's position deltas without decoding them -- the CSR + // layout is sequential so we must advance past them, but only the + // candidate (selected) docs' positions are ever used. With a sparse + // candidate set (the common phrase / phrase-prefix case after docid + // narrowing) most docs in a window are skipped, so this avoids the + // dominant varint-decode cost. + RETURN_IF_ERROR(src.skip_varints(pos_count)); + continue; + } + // Selected doc: decode its `pos_count` ascending position deltas with a + // tight inline prefix-sum decoder (single-byte fast path, no per-value + // get_varint32/Status call chain). This is the CPU hotspot for narrowed + // phrase/phrase-prefix candidate sets, where each selected doc's varint + // run dominates after non-selected docs are skipped. + RETURN_IF_ERROR(src.decode_delta_run(pos_count, pos_flat)); + pos_off->push_back(static_cast(pos_flat->size())); + ++next_doc; + } + if (!src.eof()) + return Status::Error( + "prx: trailing bytes after payload"); + *decoded_doc_count = doc_count; + *decoded_total_positions = total_pos; + *max_frequency = decoded_max_frequency; + *has_zero_frequency = decoded_zero_frequency; + return Status::OK(); +} + +// Decision: given level and plain length, determine whether to compress. +bool should_compress(int level, size_t plain_len) { + if (level == 0) return false; // force raw + if (level > 0) return true; // force zstd + return plain_len >= kAutoZstdMinBytes; // auto +} + +// Write a raw window: codec=raw, uncomp_len, crc(header+payload), payload. +void write_raw(Slice plain, ByteSink* sink) { + // Single-copy framing (see write_pfor): assemble [codec][uncomp_len][payload] + // in the caller's sink and crc that span before appending the crc. + const size_t start = sink->size(); + sink->put_u8(static_cast(PrxCodec::kRaw)); + sink->put_varint32(static_cast(plain.size())); + sink->put_bytes(plain); + const size_t framed_len = sink->size() - start; + const uint32_t crc = crc32c(sink->view().subslice(start, framed_len)); + sink->put_fixed32(crc); +} + +// Write a zstd window: codec=zstd, uncomp_len, comp_len, crc(header+payload), +// payload. +Status write_zstd(Slice plain, int level, ByteSink* sink) { + std::vector comp; + RETURN_IF_ERROR(zstd_compress(plain, level > 0 ? level : kDefaultZstdLevel, &comp)); + write_zstd_compressed(plain, Slice(comp), sink); + return Status::OK(); +} + +struct FramedPrxWindow { + uint8_t codec = 0; + uint32_t uncomp_len = 0; + Slice payload; +}; + +// Read header + payload and verify the frame CRC. +Status read_framed(ByteSource* src, FramedPrxWindow* frame) { + const size_t start = src->position(); + RETURN_IF_ERROR(src->get_u8(&frame->codec)); + if (frame->codec != static_cast(PrxCodec::kRaw) && + frame->codec != static_cast(PrxCodec::kZstd) && + frame->codec != static_cast(PrxCodec::kPfor)) { + return Status::Error("prx: unknown codec"); + } + RETURN_IF_ERROR(src->get_varint32(&frame->uncomp_len)); + if (frame->uncomp_len > kMaxWindowUncompBytes) { + return Status::Error( + "prx: uncomp_len exceeds sane window cap"); + } + size_t payload_len = frame->uncomp_len; + if (frame->codec == static_cast(PrxCodec::kZstd)) { + uint32_t comp_len = 0; + RETURN_IF_ERROR(src->get_varint32(&comp_len)); + payload_len = comp_len; + } + RETURN_IF_ERROR(src->get_bytes(payload_len, &frame->payload)); + const size_t framed_len = src->position() - start; + uint32_t stored = 0; + RETURN_IF_ERROR(src->get_fixed32(&stored)); + if (crc32c(src->slice_from(start, framed_len)) != stored) { + return Status::Error( + "prx: window crc mismatch"); + } + return Status::OK(); +} + +void initialize_frame_stats(const FramedPrxWindow& encoded, PrxDecodeStats* stats) { + if (encoded.codec == static_cast(PrxCodec::kRaw)) { + stats->raw_frames = 1; + stats->plaintext_bytes = encoded.payload.size(); + } else if (encoded.codec == static_cast(PrxCodec::kZstd)) { + stats->zstd_frames = 1; + stats->plaintext_bytes = encoded.uncomp_len; + } else { + stats->pfor_frames = 1; + stats->plaintext_bytes = encoded.uncomp_len; + } +} + +Status decode_csr_frame(const FramedPrxWindow& encoded, std::span doc_ordinals, + bool decode_all_docs, bool all_docs_selected, + std::vector* pos_flat, std::vector* pos_off, + PrxDecodeStats* stats, PrxDecodedShape* shape, + PrxCsrAllocationGate* allocation_gate) { + if (!decode_all_docs && allocation_gate != nullptr) { + return Status::Error( + "prx: selective decode cannot use an allocation gate"); + } + if (stats != nullptr) { + initialize_frame_stats(encoded, stats); + } + uint32_t total_docs = 0; + uint64_t total_positions = 0; + uint32_t max_frequency = 0; + bool has_zero_frequency = false; + + std::vector local_decompressed; + Slice plain = encoded.payload; + if (encoded.codec == static_cast(PrxCodec::kZstd)) { + std::vector* decompressed = &local_decompressed; + if (allocation_gate != nullptr) { + RETURN_IF_ERROR( + allocation_gate->reserve_decompression(encoded.uncomp_len, &decompressed)); + DCHECK(decompressed != nullptr); + DCHECK_GE(decompressed->capacity(), encoded.uncomp_len); + } + RETURN_IF_ERROR(zstd_decompress(encoded.payload, encoded.uncomp_len, decompressed)); + plain = Slice(*decompressed); + } + + if (decode_all_docs) { + if (encoded.codec == static_cast(PrxCodec::kPfor)) { + RETURN_IF_ERROR(decode_pfor_payload_csr(plain, pos_flat, pos_off, allocation_gate, + &max_frequency, &has_zero_frequency)); + } else { + RETURN_IF_ERROR(decode_payload_csr(plain, pos_flat, pos_off, allocation_gate, + &max_frequency, &has_zero_frequency)); + } + total_docs = static_cast(pos_off->size() - 1); + total_positions = pos_flat->size(); + if (!all_docs_selected) { + RETURN_IF_ERROR(validate_doc_ordinals(doc_ordinals, total_docs)); + } + } else if (encoded.codec == static_cast(PrxCodec::kPfor)) { + RETURN_IF_ERROR(decode_pfor_payload_csr_selective(plain, doc_ordinals, pos_flat, pos_off, + &total_docs, &total_positions, + &max_frequency, &has_zero_frequency)); + } else { + RETURN_IF_ERROR(decode_payload_csr_selective(plain, doc_ordinals, pos_flat, pos_off, + &total_docs, &total_positions, &max_frequency, + &has_zero_frequency)); + } + if (shape != nullptr) { + shape->total_docs = total_docs; + shape->total_positions = total_positions; + shape->max_frequency = max_frequency; + shape->has_zero_frequency = has_zero_frequency; + } + if (stats != nullptr) { + stats->total_docs = total_docs; + stats->total_positions = total_positions; + if (all_docs_selected) { + stats->selected_docs = total_docs; + stats->selected_positions = total_positions; + } else if (!decode_all_docs) { + stats->selected_docs = doc_ordinals.size(); + stats->selected_positions = pos_flat->size(); + } + } + return Status::OK(); +} + +Status read_prx_window_csr_impl(ByteSource* source, std::span doc_ordinals, + bool decode_all_docs, bool all_docs_selected, + std::vector* pos_flat, std::vector* pos_off, + PrxDecodeContext* context) { + if (source == nullptr || pos_flat == nullptr || pos_off == nullptr) { + return Status::Error("prx: null arg"); + } + const bool collect_stats = context != nullptr && context->stats != nullptr; + PrxDecodeStats frame_stats; + PrxDecodeStats* stats = collect_stats ? &frame_stats : nullptr; + + PrxClock::time_point decode_start; + if (collect_stats) { + decode_start = prx_clock_now(); + } + FramedPrxWindow encoded; + RETURN_IF_ERROR(read_framed(source, &encoded)); + RETURN_IF_ERROR(decode_csr_frame(encoded, doc_ordinals, decode_all_docs, all_docs_selected, + pos_flat, pos_off, stats, + context == nullptr ? nullptr : context->shape, + context == nullptr ? nullptr : context->allocation_gate)); + if (collect_stats) { + // Stop inclusive decode timing before the logical-selection scan. Phrase + // execution wraps this call in PhraseVerifyTimer, so that scan remains + // part of verification rather than format decode. + frame_stats.decode_ns = elapsed_ns(decode_start); + if (decode_all_docs && !all_docs_selected) { + uint64_t selected_positions = 0; + for (uint32_t ordinal : doc_ordinals) { + DCHECK_LT(static_cast(ordinal) + 1, pos_off->size()); + selected_positions += (*pos_off)[ordinal + 1] - (*pos_off)[ordinal]; + } + frame_stats.selected_docs = doc_ordinals.size(); + frame_stats.selected_positions = selected_positions; + } + // Format decode is complete. Caller-level CSR invariants are validated + // afterwards, so a later phrase error intentionally retains this work. + context->stats->merge(frame_stats); + } + return Status::OK(); +} + +} // namespace + +Status build_prx_window(std::span> per_doc_positions, + int zstd_level_or_negative_for_auto, ByteSink* sink) { + return build_prx_window(per_doc_positions, zstd_level_or_negative_for_auto, + kReaderPrxWindowLimits, sink); +} + +Status build_prx_window(std::span> per_doc_positions, + int zstd_level_or_negative_for_auto, const PrxWindowLimits& limits, + ByteSink* sink) { + if (sink == nullptr) return Status::Error("prx: null sink"); + size_t total_positions = 0; + RETURN_IF_ERROR(validate_per_doc_window(per_doc_positions, limits, &total_positions)); + // Forced legacy codecs (level 0 = raw varint, level > 0 = zstd) are kept so + // the test/legacy paths still exercise them; the auto path (< 0) now emits + // PFOR bit-packed deltas -- no entropy coding, far cheaper build CPU than + // zstd-3. + if (zstd_level_or_negative_for_auto >= 0) { + ByteSink plain; + RETURN_IF_ERROR(encode_payload(per_doc_positions, &plain)); + if (plain.size() > limits.max_uncomp_bytes) { + return Status::Error( + "prx: encoded payload exceeds writer window byte limit"); + } + Slice plain_view = plain.view(); + if (!should_compress(zstd_level_or_negative_for_auto, plain_view.size())) { + write_raw(plain_view, sink); + return Status::OK(); + } + return write_zstd(plain_view, zstd_level_or_negative_for_auto, sink); + } + // Auto mode: flatten the per-doc lists into (positions_flat, freqs) exactly as + // the former encode_pfor_payload did, then run the shared single-encode path so + // this builder stays byte-identical to build_prx_window_flat. + std::vector flat, freqs; + freqs.reserve(per_doc_positions.size()); + flat.reserve(total_positions); + for (const auto& doc : per_doc_positions) { + freqs.push_back(static_cast(doc.size())); + flat.insert(flat.end(), doc.begin(), doc.end()); + } + // G16-h: level < -1 is auto mode at zstd level |level| (-1 stays the default). + const int auto_level = zstd_level_or_negative_for_auto == -1 ? kDefaultZstdLevel + : -zstd_level_or_negative_for_auto; + PrxWindowBuildOutcome outcome = PrxWindowBuildOutcome::kBuilt; + RETURN_IF_ERROR( + build_prx_window_auto_from_flat(flat, freqs, auto_level, limits, sink, &outcome)); + if (outcome == PrxWindowBuildOutcome::kNeedsSplit) { + return Status::Error( + "prx: encoded payload exceeds writer window byte limit"); + } + return Status::OK(); +} + +Status build_prx_window_flat(std::span positions_flat, + std::span freqs, int zstd_level_or_negative_for_auto, + ByteSink* sink) { + return build_prx_window_flat(positions_flat, freqs, zstd_level_or_negative_for_auto, + kReaderPrxWindowLimits, sink); +} + +Status build_prx_window_flat(std::span positions_flat, + std::span freqs, int zstd_level_or_negative_for_auto, + const PrxWindowLimits& limits, ByteSink* sink) { + PrxWindowBuildOutcome outcome = PrxWindowBuildOutcome::kBuilt; + RETURN_IF_ERROR(try_build_prx_window_flat( + positions_flat, freqs, zstd_level_or_negative_for_auto, limits, sink, &outcome)); + if (outcome == PrxWindowBuildOutcome::kNeedsSplit) { + return Status::Error( + "prx: window exceeds writer limits"); + } + return Status::OK(); +} + +Status try_build_prx_window_flat(std::span positions_flat, + std::span freqs, + int zstd_level_or_negative_for_auto, const PrxWindowLimits& limits, + ByteSink* sink, PrxWindowBuildOutcome* outcome) { + if (sink == nullptr || outcome == nullptr) { + return Status::Error("prx: null arg"); + } + RETURN_IF_ERROR(validate_prx_window_limits(limits)); + RETURN_IF_ERROR(check_flat_partition(positions_flat, freqs)); + if (freqs.size() > limits.max_docs || positions_flat.size() > limits.max_positions) { + RETURN_IF_ERROR(validate_flat_positions(positions_flat, freqs)); + for (uint32_t frequency : freqs) { + if (frequency > limits.max_positions) { + return Status::Error( + "prx: one document exceeds the writer window position limit"); + } + } + if (freqs.size() <= 1) { + return Status::Error( + "prx: one document exceeds the writer window shape limit"); + } + RETURN_IF_ERROR(validate_single_doc_byte_limits(positions_flat, freqs, + zstd_level_or_negative_for_auto < 0, + limits.max_uncomp_bytes)); + *outcome = PrxWindowBuildOutcome::kNeedsSplit; + return Status::OK(); + } + if (zstd_level_or_negative_for_auto >= 0) { + ByteSink plain; + RETURN_IF_ERROR(encode_payload_flat(positions_flat, freqs, &plain)); + if (plain.size() > limits.max_uncomp_bytes) { + if (freqs.size() <= 1) { + return Status::Error( + "prx: one document exceeds the writer window byte limit"); + } + RETURN_IF_ERROR(validate_single_doc_byte_limits(positions_flat, freqs, false, + limits.max_uncomp_bytes)); + *outcome = PrxWindowBuildOutcome::kNeedsSplit; + return Status::OK(); + } + Slice plain_view = plain.view(); + if (!should_compress(zstd_level_or_negative_for_auto, plain_view.size())) { + write_raw(plain_view, sink); + *outcome = PrxWindowBuildOutcome::kBuilt; + return Status::OK(); + } + RETURN_IF_ERROR(write_zstd(plain_view, zstd_level_or_negative_for_auto, sink)); + *outcome = PrxWindowBuildOutcome::kBuilt; + return Status::OK(); + } + // Auto mode: shared path with a direct singleton RAW fast path, then PFOR, + // with raw plaintext materialized only for zstd or a tightened-limit fallback. + // G16-h: level < -1 is auto mode at zstd level |level| (-1 stays the default). + const int auto_level = zstd_level_or_negative_for_auto == -1 ? kDefaultZstdLevel + : -zstd_level_or_negative_for_auto; + return build_prx_window_auto_from_flat(positions_flat, freqs, auto_level, limits, sink, + outcome); +} + +Status read_prx_window(ByteSource* source, std::vector>* per_doc_positions) { + if (source == nullptr || per_doc_positions == nullptr) { + return Status::Error("prx: null arg"); + } + FramedPrxWindow frame; + RETURN_IF_ERROR(read_framed(source, &frame)); + if (frame.codec == static_cast(PrxCodec::kPfor)) { + return decode_pfor_payload(frame.payload, per_doc_positions); + } + if (frame.codec == static_cast(PrxCodec::kRaw)) { + return decode_payload(frame.payload, per_doc_positions); + } + std::vector plain; + RETURN_IF_ERROR(zstd_decompress(frame.payload, frame.uncomp_len, &plain)); + return decode_payload(Slice(plain), per_doc_positions); +} + +Status read_prx_window_csr(ByteSource* source, std::vector* pos_flat, + std::vector* pos_off) { + return read_prx_window_csr_impl(source, {}, true, true, pos_flat, pos_off, nullptr); +} + +Status read_prx_window_csr(ByteSource* source, std::vector* pos_flat, + std::vector* pos_off, PrxDecodeContext* context) { + return read_prx_window_csr_impl(source, {}, true, true, pos_flat, pos_off, context); +} + +Status read_prx_window_csr_for_selection(ByteSource* source, std::span doc_ordinals, + std::vector* pos_flat, + std::vector* pos_off, + PrxDecodeContext* context) { + return read_prx_window_csr_impl(source, doc_ordinals, true, false, pos_flat, pos_off, context); +} + +Status read_prx_window_csr_selective(ByteSource* source, std::span doc_ordinals, + std::vector* pos_flat, + std::vector* pos_off) { + return read_prx_window_csr_impl(source, doc_ordinals, false, false, pos_flat, pos_off, nullptr); +} + +Status read_prx_window_csr_selective(ByteSource* source, std::span doc_ordinals, + std::vector* pos_flat, + std::vector* pos_off, PrxDecodeContext* context) { + return read_prx_window_csr_impl(source, doc_ordinals, false, false, pos_flat, pos_off, context); +} + +} // namespace doris::snii::format + +namespace doris::snii::format::testing { +namespace { +std::atomic& prx_raw_build_atomic() { + static std::atomic counter {0}; + return counter; +} + +#ifdef BE_TEST +std::atomic& prx_clock_read_atomic() { + static std::atomic counter {0}; + return counter; +} + +std::atomic& prx_delta_materialization_atomic() { + static std::atomic counter {0}; + return counter; +} +#endif +} // namespace + +uint64_t prx_raw_build_count() { + return prx_raw_build_atomic().load(std::memory_order_relaxed); +} + +void reset_prx_raw_build_count() { + prx_raw_build_atomic().store(0, std::memory_order_relaxed); +} + +void note_prx_raw_build() { + prx_raw_build_atomic().fetch_add(1, std::memory_order_relaxed); +} + +#ifdef BE_TEST +uint64_t prx_clock_read_count() { + return prx_clock_read_atomic().load(std::memory_order_relaxed); +} + +void reset_prx_clock_read_count() { + prx_clock_read_atomic().store(0, std::memory_order_relaxed); +} + +void note_prx_clock_read() { + prx_clock_read_atomic().fetch_add(1, std::memory_order_relaxed); +} + +uint64_t prx_delta_materialization_count() { + return prx_delta_materialization_atomic().load(std::memory_order_relaxed); +} + +void reset_prx_delta_materialization_count() { + prx_delta_materialization_atomic().store(0, std::memory_order_relaxed); +} + +void note_prx_delta_materialization() { + prx_delta_materialization_atomic().fetch_add(1, std::memory_order_relaxed); +} + +uint8_t select_auto_prx_codec_for_test(size_t pfor_payload_size, size_t plain_payload_size, + size_t compressed_payload_size, uint32_t max_uncomp_bytes) { + const AutoPrxCodecChoice choice = + select_auto_prx_codec(pfor_payload_size, plain_payload_size, compressed_payload_size, + plain_payload_size >= kAutoZstdMinBytes, max_uncomp_bytes); + DCHECK(choice.readable); + return static_cast(choice.codec); +} +#endif + +} // namespace doris::snii::format::testing diff --git a/be/src/storage/index/snii/format/prx_pod.h b/be/src/storage/index/snii/format/prx_pod.h new file mode 100644 index 00000000000000..fd96136d3ffe9f --- /dev/null +++ b/be/src/storage/index/snii/format/prx_pod.h @@ -0,0 +1,189 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/encoding/byte_sink.h" +#include "storage/index/snii/encoding/byte_source.h" +#include "storage/index/snii/format/prx_decode_stats.h" + +// .prx position window (PrxPod): stores term position information for several +// docs within one window. +// +// Single-window on-disk byte layout (see docs/design SNII "prx design"): +// u8 codec # PrxCodec: 0=raw / 1=zstd / 2=pfor (bit7 cont-reserved) +// VInt uncomp_len # payload length (raw/pfor: on-disk payload bytes; zstd: +// plaintext) VInt comp_len # present only when codec==zstd u32 crc32c # +// covers header (codec..comp_len) + payload bytes payload # raw: varint +// plaintext; zstd: compressed; pfor: bit-packed +// +// raw/zstd plaintext payload (self-describing per-doc boundaries): +// VInt doc_count +// per doc: VInt pos_count, followed by pos_count position deltas (VInt) +// positions within a doc are ascending, stored as deltas (first absolute). +// +// pfor payload (one auto-build candidate; no entropy coding): +// VInt doc_count +// VInt total_pos # sum of all pos_counts +// PFOR_runs(pos_counts) # doc_count values +// PFOR_runs(position_deltas) # total_pos deltas, kFrqBaseUnit per run, +// # flat doc order (first per doc +// absolute) +// +// Multi-byte fixed-length fields are little-endian; variable-length integers +// reuse snii/encoding/varint. crc32c checksum at window tail detects +// corruption. +namespace doris::snii::format { + +struct PrxWindowLimits { + uint32_t max_docs; + uint32_t max_positions; + uint32_t max_uncomp_bytes; +}; + +enum class PrxWindowBuildOutcome : uint8_t { + kBuilt = 0, + kNeedsSplit = 1, +}; + +inline constexpr PrxWindowLimits kReaderPrxWindowLimits { + .max_docs = 1U << 24, + .max_positions = 1U << 26, + .max_uncomp_bytes = 256U * 1024 * 1024, +}; + +// Writer policies may tighten, but never exceed, the reader's allocation +// guards. Keeping this validation in the format layer makes it impossible for +// a caller to configure a writer that emits frames the current reader rejects. +Status validate_prx_window_limits(const PrxWindowLimits& limits); + +// Build a .prx window and append it to sink. +// per_doc_positions[d] is the position list for the d-th doc within this +// window; must be ascending (duplicates allowed). +// zstd_level_or_negative_for_auto: +// <0 → auto: use direct RAW for a single-doc/freq=1 window; otherwise use +// PFOR below the compression threshold and choose the smallest +// PFOR/ZSTD/RAW frame once raw plaintext is already required (default +// zstd level). +// 0 → force raw varint payload. +// >0 → force ZSTD with the given level. +// Non-ascending positions within a doc return InvalidArgument. +Status build_prx_window(std::span> per_doc_positions, + int zstd_level_or_negative_for_auto, ByteSink* sink); +Status build_prx_window(std::span> per_doc_positions, + int zstd_level_or_negative_for_auto, const PrxWindowLimits& limits, + ByteSink* sink); + +// Vector convenience overload (forwards a span view over the window's per-doc +// lists; the writer can pass a slice of its flat positions WITHOUT deep-copying +// the inner vectors into a fresh std::vector> per +// window). +inline Status build_prx_window(const std::vector>& per_doc_positions, + int zstd_level_or_negative_for_auto, ByteSink* sink) { + return build_prx_window(std::span>(per_doc_positions), + zstd_level_or_negative_for_auto, sink); +} + +// FLAT-positions builder: byte-identical output to build_prx_window above, but +// reads the window's positions from a single flat span partitioned per-doc by +// `freqs` (doc d owns the next freqs[d] entries; freqs.size() == doc count and +// sum(freqs) == positions_flat.size()). Lets the writer pass a subspan of the +// term's flat positions/freqs with NO vector-of-vectors materialization. +Status build_prx_window_flat(std::span positions_flat, + std::span freqs, int zstd_level_or_negative_for_auto, + ByteSink* sink); +Status build_prx_window_flat(std::span positions_flat, + std::span freqs, int zstd_level_or_negative_for_auto, + const PrxWindowLimits& limits, ByteSink* sink); + +// Writer-facing exact attempt. kNeedsSplit is returned only when the current +// multi-document window exceeds a shape or encoded-byte limit and every +// standalone document is representable, so retrying at document boundaries is +// guaranteed to resolve the limit. An unsplittable document, malformed input +// and codec failures remain errors. The sink is unchanged unless the outcome is +// kBuilt. +Status try_build_prx_window_flat(std::span positions_flat, + std::span freqs, + int zstd_level_or_negative_for_auto, const PrxWindowLimits& limits, + ByteSink* sink, PrxWindowBuildOutcome* outcome); + +// Read and verify a .prx window from source, reconstructing the per-doc +// position list. CRC mismatch / invalid codec / truncation / decompression +// failure all return a non-OK Status. +Status read_prx_window(ByteSource* source, std::vector>* per_doc_positions); + +// CSR variant of read_prx_window: decodes ALL docs' positions into one flat +// buffer `pos_flat` with per-doc offsets `pos_off` (size doc_count+1, +// pos_off[0]==0), so doc d's positions are pos_flat[pos_off[d] .. +// pos_off[d+1]). Avoids the per-doc std::vector allocation of read_prx_window +// -- both output vectors are flat uint32 buffers whose capacity a caller can +// retain (clear()) across windows/queries. +Status read_prx_window_csr(ByteSource* source, std::vector* pos_flat, + std::vector* pos_off); +Status read_prx_window_csr(ByteSource* source, std::vector* pos_flat, + std::vector* pos_off, PrxDecodeContext* context); + +// Decode every physical document while reporting the supplied ordinals as the +// logical selection. Used by the density fallback after docid narrowing. +Status read_prx_window_csr_for_selection(ByteSource* source, std::span doc_ordinals, + std::vector* pos_flat, + std::vector* pos_off, PrxDecodeContext* context); + +// Selective CSR variant: decodes positions only for the requested local doc +// ordinals within this PRX window. `doc_ordinals` must be strictly ascending. +// The output uses the same CSR shape, but has doc_ordinals.size()+1 offsets. +Status read_prx_window_csr_selective(ByteSource* source, std::span doc_ordinals, + std::vector* pos_flat, + std::vector* pos_off); +Status read_prx_window_csr_selective(ByteSource* source, std::span doc_ordinals, + std::vector* pos_flat, + std::vector* pos_off, PrxDecodeContext* context); + +} // namespace doris::snii::format + +// Test-only instrumentation seam. prx_raw_build_count() returns a process-global +// count of zstd-candidate plaintext payloads materialized by the auto-mode (.prx) +// window builders. Sub-threshold RAW fallback after an unreadable PFOR is not a +// zstd candidate and is intentionally not counted. In ordinary auto mode the +// small windows that dominate a Zipfian corpus skip the throwaway plaintext and +// second delta walk entirely, so tests assert the count is 0 for sub-threshold +// PFOR windows and exactly 1 per zstd candidate. +// Counters use relaxed atomics and are reset between tests; the writer's segment +// build is single-threaded, so the atomic adds introduce no data race. +namespace doris::snii::format::testing { + +uint64_t prx_raw_build_count(); +void reset_prx_raw_build_count(); +void note_prx_raw_build(); +#ifdef BE_TEST +uint64_t prx_clock_read_count(); +void reset_prx_clock_read_count(); +void note_prx_clock_read(); +uint64_t prx_delta_materialization_count(); +void reset_prx_delta_materialization_count(); +void note_prx_delta_materialization(); +uint8_t select_auto_prx_codec_for_test(size_t pfor_payload_size, size_t plain_payload_size, + size_t compressed_payload_size, uint32_t max_uncomp_bytes); +#endif + +} // namespace doris::snii::format::testing diff --git a/be/src/storage/index/snii/format/sampled_term_index.cpp b/be/src/storage/index/snii/format/sampled_term_index.cpp new file mode 100644 index 00000000000000..e60832132a1a51 --- /dev/null +++ b/be/src/storage/index/snii/format/sampled_term_index.cpp @@ -0,0 +1,189 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/format/sampled_term_index.h" + +#include + +#include "storage/index/snii/encoding/byte_source.h" +#include "storage/index/snii/encoding/section_framer.h" + +namespace doris::snii::format { + +namespace { + +// Longest common prefix length of term and prev (front coding primitive, consistent with dict_entry). +uint32_t common_prefix_len(std::string_view term, std::string_view prev) { + uint32_t n = 0; + const uint32_t lim = static_cast(std::min(term.size(), prev.size())); + while (n < lim && term[n] == prev[n]) ++n; + return n; +} + +// Write a front-coded term key (prefix_len + suffix_len + suffix). +void write_term_key(std::string_view term, std::string_view prev, ByteSink* sink) { + const uint32_t prefix = common_prefix_len(term, prev); + const std::string_view suffix = term.substr(prefix); + sink->put_varint32(prefix); + sink->put_varint32(static_cast(suffix.size())); + sink->put_bytes(Slice(suffix)); +} + +// Read a front-coded term key and reconstruct it into out from prev + suffix. +Status read_term_key(ByteSource* src, std::string_view prev, std::string* out) { + uint32_t prefix = 0; + uint32_t suffix_len = 0; + RETURN_IF_ERROR(src->get_varint32(&prefix)); + RETURN_IF_ERROR(src->get_varint32(&suffix_len)); + if (prefix > prev.size()) { + return Status::Error( + "sampled_term_index: prefix_len exceeds prev_term length"); + } + Slice suffix; + RETURN_IF_ERROR(src->get_bytes(suffix_len, &suffix)); + out->assign(prev.substr(0, prefix)); + out->append(reinterpret_cast(suffix.data()), suffix.size()); + return Status::OK(); +} + +} // namespace + +void SampledTermIndexBuilder::add_block_first_term(std::string_view first_term) { + first_terms_.emplace_back(first_term); +} + +void SampledTermIndexBuilder::finish(ByteSink* sink) { + ByteSink payload; + payload.put_varint32(static_cast(first_terms_.size())); + // min_term / max_term are written only when non-empty (== first/last sample_term). + if (!first_terms_.empty()) { + write_term_key(first_terms_.front(), std::string_view {}, &payload); + write_term_key(first_terms_.back(), std::string_view {}, &payload); + std::string_view prev {}; + for (const auto& t : first_terms_) { + write_term_key(t, prev, &payload); + prev = t; + } + } + SectionFramer::write(*sink, static_cast(SectionType::kSampledTermIndex), + payload.view()); +} + +namespace { + +// Parse n_blocks, min/max (not used directly; consumed for checksum alignment), and all sample_terms from payload. +Status parse_payload(Slice payload, std::vector* terms) { + ByteSource src(payload); + uint32_t n_blocks = 0; + RETURN_IF_ERROR(src.get_varint32(&n_blocks)); + if (n_blocks == 0) { + if (!src.eof()) { + return Status::Error( + "sampled_term_index: empty index contains trailing bytes"); + } + terms->clear(); + return Status::OK(); + } + + // min_term / max_term (do not drive binary search directly; must be consumed to verify structural alignment). + std::string min_term; + std::string max_term; + RETURN_IF_ERROR(read_term_key(&src, std::string_view {}, &min_term)); + RETURN_IF_ERROR(read_term_key(&src, std::string_view {}, &max_term)); + + std::vector out; + out.reserve(n_blocks); + std::string prev; + for (uint32_t i = 0; i < n_blocks; ++i) { + std::string term; + RETURN_IF_ERROR(read_term_key(&src, prev, &term)); + prev = term; + out.push_back(std::move(term)); + } + if (!src.eof()) { + return Status::Error( + "sampled_term_index: payload contains trailing bytes"); + } + if (out.front() != min_term || out.back() != max_term) { + return Status::Error( + "sampled_term_index: min/max inconsistent with sample_terms"); + } + *terms = std::move(out); + return Status::OK(); +} + +} // namespace + +Status SampledTermIndexReader::open(Slice section, SampledTermIndexReader* out) { + if (out == nullptr) { + return Status::Error("sampled_term_index: out is null"); + } + ByteSource src(section); + FramedSection sec; + RETURN_IF_ERROR(SectionFramer::read(src, &sec)); + if (!src.eof()) { + return Status::Error( + "sampled_term_index: trailing framed section bytes"); + } + if (sec.type != static_cast(SectionType::kSampledTermIndex)) { + return Status::Error( + "sampled_term_index: not a kSampledTermIndex section"); + } + *out = SampledTermIndexReader {}; + return parse_payload(sec.payload, &out->sample_terms_); +} + +size_t SampledTermIndexReader::heap_bytes() const { + size_t bytes = sample_terms_.capacity() * sizeof(std::string); + for (const auto& term : sample_terms_) { + bytes += std_string_heap_bytes(term); + } + return bytes; +} + +Status SampledTermIndexReader::locate(std::string_view target, bool* maybe_present, + uint32_t* block_ordinal) const { + if (maybe_present == nullptr || block_ordinal == nullptr) { + return Status::Error( + "sampled_term_index: output pointer is null"); + } + *maybe_present = false; + *block_ordinal = 0; + if (sample_terms_.empty()) { + return Status::OK(); // empty index: always out of range. + } + // target < min_term (first block's first term) -> before the first block, so it + // cannot exist in any block. NOTE: a target GREATER than the last sample term is + // NOT out of range -- sample_terms_ holds each block's FIRST term, so the LAST + // block can contain terms greater than its first term. Such a target routes to + // the last block (upper_bound -> end()), where find_term confirms presence. + if (target < std::string_view(sample_terms_.front())) { + return Status::OK(); + } + // Last sample_term <= target: step back one position after upper_bound. For a + // target past every sample term, upper_bound returns end() and idx = n-1 (the + // last block), which is correct. + auto it = std::upper_bound( + sample_terms_.begin(), sample_terms_.end(), target, + [](std::string_view t, const std::string& s) { return t < std::string_view(s); }); + const auto idx = (it - sample_terms_.begin()) - 1; // it > begin (< min excluded). + *maybe_present = true; + *block_ordinal = static_cast(idx); + return Status::OK(); +} + +} // namespace doris::snii::format diff --git a/be/src/storage/index/snii/format/sampled_term_index.h b/be/src/storage/index/snii/format/sampled_term_index.h new file mode 100644 index 00000000000000..f2d63af88334e8 --- /dev/null +++ b/be/src/storage/index/snii/format/sampled_term_index.h @@ -0,0 +1,103 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/encoding/byte_sink.h" +#include "storage/index/snii/format/format_constants.h" + +// SampledTermIndex -- resident metadata for locating a query term to a candidate DICT block. +// +// Sampling granularity is per DICT block (not a fixed term count): each time the writer produces a DICT block, +// it writes the block's first_term into this index. Size grows proportionally to block count. At read time it is +// loaded into the searcher cache together with SniiLogicalIndexReader. See design spec "Sampled Term Index". +// +// On-disk layout (framed by SectionFramer, uniform type+len+crc32c): +// [u8 type=kSampledTermIndex][varint64 payload_len][payload][fixed32 crc32c] +// payload = +// n_blocks varint32 +// min_term len(varint32) + bytes # == sample_terms[0], omitted when n_blocks=0 +// max_term len(varint32) + bytes # == sample_terms[n-1], omitted when n_blocks=0 +// sample_terms[n_blocks]: # first_term of each block, in ascending order +// prefix_len varint32 # shared prefix length with the previous sample_term +// suffix_len varint32 +// suffix u8[suffix_len] +// +// Term bytes are compared as unsigned byte order (UTF-8 friendly, binary-safe). Front coding reuses +// the same prefix/suffix primitives as DictEntry; do not reimplement. +namespace doris::snii::format { + +// SSO-aware heap-byte accounting for a std::string. libstdc++ keeps up to 15 +// chars inline (SSO), so only capacity() > 15 implies a separate heap buffer of +// capacity()+1 bytes (the +1 is the NUL terminator); an SSO string owns no heap +// and contributes 0. Shared by the resident format readers' heap_bytes() charge +// helpers, which back LogicalIndexReader::memory_usage() (the searcher-cache +// charge). NOTE: the threshold 15 is libstdc++-specific; a different standard +// library needs a different SSO bound here. +inline size_t std_string_heap_bytes(const std::string& s) { + return s.capacity() > 15 ? s.capacity() + 1 : 0; +} + +// Builder: appends the first_term of each DICT block in block ordinal order (must be strictly ascending), +// and serializes the entire set into a single kSampledTermIndex framed section on finish. +class SampledTermIndexBuilder { +public: + // Appends the first_term of the next DICT block. Call order determines block ordinal order. + void add_block_first_term(std::string_view first_term); + + // Serializes and appends to sink. An empty collection (no blocks) is valid; n_blocks=0. + void finish(ByteSink* sink); + +private: + std::vector first_terms_; +}; + +// Reader: verifies the checksum and materializes all sample_terms on open; subsequent locate calls are pure in-memory binary search. +class SampledTermIndexReader { +public: + SampledTermIndexReader() = default; + + // Parses a kSampledTermIndex framed section. + // CRC mismatch / truncation / field overrun → kCorruption; type != kSampledTermIndex → kInvalidArgument. + static Status open(Slice section, SampledTermIndexReader* out); + + // Binary-search locate: returns the block ordinal of the last sample_term <= target. + // target < min_term or target > max_term (including empty index) → *maybe_present=false (out of range, term is definitely absent). + // Otherwise *maybe_present=true and *block_ordinal is the ordinal of the matching block. + Status locate(std::string_view target, bool* maybe_present, uint32_t* block_ordinal) const; + + uint32_t n_blocks() const { return static_cast(sample_terms_.size()); } + + // Resident heap held beyond sizeof(*this): the sample_terms_ vector buffer + // plus each non-SSO term's heap allocation. Summed into + // LogicalIndexReader::memory_usage() so the searcher-cache charge reflects the + // decoded sampled index (previously omitted -> under-charge -> over-commit). + size_t heap_bytes() const; + +private: + std::vector sample_terms_; +}; + +} // namespace doris::snii::format diff --git a/be/src/storage/index/snii/format/stats_block.h b/be/src/storage/index/snii/format/stats_block.h new file mode 100644 index 00000000000000..d1b07283ae437b --- /dev/null +++ b/be/src/storage/index/snii/format/stats_block.h @@ -0,0 +1,34 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +namespace doris::snii::format { + +// Runtime counting statistics used for query planning and BM25. Core protobuf +// metadata owns their on-disk representation. +struct StatsBlock { + uint64_t doc_count = 0; // total doc count at segment level (including unindexed/NULL) + uint64_t indexed_doc_count = 0; // number of docs actually indexed (denominator for avgdl) + uint64_t term_count = 0; // number of unique terms in this index + uint64_t sum_total_term_freq = 0; // total token count across all indexed docs + uint64_t null_count = 0; // number of NULL / not-indexed docs +}; + +} // namespace doris::snii::format diff --git a/be/src/storage/index/snii/format/tail_pointer.cpp b/be/src/storage/index/snii/format/tail_pointer.cpp new file mode 100644 index 00000000000000..69bf45c376072d --- /dev/null +++ b/be/src/storage/index/snii/format/tail_pointer.cpp @@ -0,0 +1,122 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/format/tail_pointer.h" + +#include "storage/index/snii/encoding/byte_source.h" +#include "storage/index/snii/encoding/crc32c.h" +#include "storage/index/snii/format/format_constants.h" + +namespace doris::snii::format { + +namespace { + +// Byte widths of every fixed field, used to derive the constant on-disk size: +// u32 magic + u16 version + 2*u64 + u32 directory crc + u8 size + u32 tail crc. +constexpr size_t kMagicBytes = 4; +constexpr size_t kVersionBytes = 2; +constexpr size_t kU64Bytes = 8; +constexpr size_t kU32Bytes = 4; +constexpr size_t kSizeByteBytes = 1; + +constexpr size_t kFixedSize = + kMagicBytes + kVersionBytes + 2 * kU64Bytes + kU32Bytes + kSizeByteBytes + kU32Bytes; +// tail_checksum is the trailing u32 and covers every byte before it. +constexpr size_t kChecksumCoverage = kFixedSize - kU32Bytes; + +// Serializes the checksum-covered region in fixed field order into covered. +void serialize_covered(const TailPointer& tp, ByteSink* covered) { + covered->put_fixed32(kTailMagic); + covered->put_fixed16(kFormatVersion); + covered->put_fixed64(tp.directory_offset); + covered->put_fixed64(tp.directory_length); + covered->put_fixed32(tp.directory_crc32c); + covered->put_u8(static_cast(kFixedSize)); +} + +} // namespace + +size_t tail_pointer_size() { + return kFixedSize; +} + +Status encode_tail_pointer(const TailPointer& tp, ByteSink* sink) { + if (sink == nullptr) { + return Status::Error("tail_pointer: null sink"); + } + ByteSink covered; + serialize_covered(tp, &covered); + DORIS_CHECK_EQ(covered.size(), kChecksumCoverage); + const uint32_t tail_checksum = crc32c(covered.view()); + sink->put_bytes(covered.view()); + sink->put_fixed32(tail_checksum); + return Status::OK(); +} + +Status decode_tail_pointer(Slice last_bytes, TailPointer* out) { + if (out == nullptr) { + return Status::Error("tail_pointer: null output"); + } + // Anti-DoS / framing: the tail pointer is a fixed-size footer, so reject any + // input that is not exactly the fixed size before touching its contents. + if (last_bytes.size() != kFixedSize) { + return Status::Error( + "tail_pointer: input is not the fixed size"); + } + const Slice covered = last_bytes.subslice(0, kChecksumCoverage); + DORIS_CHECK_EQ(covered.size(), kChecksumCoverage); + ByteSource checksum_source(last_bytes.subslice(kChecksumCoverage, kU32Bytes)); + uint32_t tail_checksum = 0; + RETURN_IF_ERROR(checksum_source.get_fixed32(&tail_checksum)); + DORIS_CHECK(checksum_source.eof()); + if (tail_checksum != crc32c(covered)) { + return Status::Error( + "tail_pointer: tail_checksum mismatch"); + } + + // Only interpret fields after authenticating the complete covered region. + ByteSource src(covered); + + uint32_t magic = 0; + RETURN_IF_ERROR(src.get_fixed32(&magic)); + if (magic != kTailMagic) { + return Status::Error( + "tail_pointer: bad magic"); + } + + uint16_t tail_format_version = 0; + RETURN_IF_ERROR(src.get_fixed16(&tail_format_version)); + if (tail_format_version != kFormatVersion) { + return Status::Error( + "tail_pointer: unsupported container format_version"); + } + RETURN_IF_ERROR(src.get_fixed64(&out->directory_offset)); + RETURN_IF_ERROR(src.get_fixed64(&out->directory_length)); + RETURN_IF_ERROR(src.get_fixed32(&out->directory_crc32c)); + + uint8_t on_disk_size = 0; + RETURN_IF_ERROR(src.get_u8(&on_disk_size)); + if (on_disk_size != kFixedSize) { + return Status::Error( + "tail_pointer: embedded size mismatch"); + } + + DORIS_CHECK(src.eof()); + return Status::OK(); +} + +} // namespace doris::snii::format diff --git a/be/src/storage/index/snii/format/tail_pointer.h b/be/src/storage/index/snii/format/tail_pointer.h new file mode 100644 index 00000000000000..fe2eb6e0bd19fc --- /dev/null +++ b/be/src/storage/index/snii/format/tail_pointer.h @@ -0,0 +1,67 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include "common/status.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/encoding/byte_sink.h" + +namespace doris::snii::format { + +// Fixed-size entry written at the very end of a segment's .idx file. It lets a +// reader locate the raw metadata directory with a single read of the trailing +// tail_pointer_size() bytes (see design spec "fixed tail pointer"). +// +// On-disk layout (all multi-byte fields little-endian, FIXED total size so the +// reader can read exactly the last tail_pointer_size() bytes): +// [u32 magic = kTailMagic] +// [u16 format_version = kFormatVersion] +// [u64 directory_offset] +// [u64 directory_length] +// [u32 directory_crc32c] +// [u8 tail_pointer_size] (== tail_pointer_size()) +// [u32 tail_checksum] (crc32c over all preceding tail-pointer bytes) +// +// The fixed layout deliberately does NOT use the SectionFramer (which is +// variable-length): a footer needs a constant trailing size the reader knows up +// front. +struct TailPointer { + uint64_t directory_offset = 0; + uint64_t directory_length = 0; + uint32_t directory_crc32c = 0; +}; + +// Constant on-disk size of the tail pointer, so the reader knows how many +// trailing bytes to read. +size_t tail_pointer_size(); + +// Appends the fixed-layout tail-pointer bytes (magic / version / fields / size / +// tail_checksum) to sink. Returns Internal if the encoded size would not fit the +// fixed-size contract (a programming error, never expected at runtime). +Status encode_tail_pointer(const TailPointer& tp, ByteSink* sink); + +// Parses the trailing tail-pointer bytes. last_bytes must be exactly +// tail_pointer_size() bytes long. Verifies magic and tail_checksum, then fills +// out with the parsed fields. Wrong magic / checksum mismatch / wrong length -> +// Corruption. +Status decode_tail_pointer(Slice last_bytes, TailPointer* out); + +} // namespace doris::snii::format diff --git a/be/src/storage/index/snii/io/batch_range_fetcher.cpp b/be/src/storage/index/snii/io/batch_range_fetcher.cpp new file mode 100644 index 00000000000000..762c01d1c78024 --- /dev/null +++ b/be/src/storage/index/snii/io/batch_range_fetcher.cpp @@ -0,0 +1,102 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/io/batch_range_fetcher.h" + +#include +#include + +namespace doris::snii::io { +namespace { + +Status checked_end(uint64_t offset, uint64_t len, uint64_t* out) { + if (len > std::numeric_limits::max() - offset) { + return Status::Error( + "batch_range_fetcher: range end overflow"); + } + *out = offset + len; + return Status::OK(); +} + +Status checked_size(uint64_t len, size_t* out) { + if (len > static_cast(std::numeric_limits::max())) { + return Status::Error( + "batch_range_fetcher: physical range too large"); + } + *out = static_cast(len); + return Status::OK(); +} + +} // namespace + +BatchRangeFetcher::BatchRangeFetcher(FileReader* reader, uint64_t coalesce_gap) + : reader_(reader), coalesce_gap_(coalesce_gap) {} + +size_t BatchRangeFetcher::add(uint64_t offset, uint64_t len) { + reqs_.push_back(Req {offset, len}); + return reqs_.size() - 1; +} + +void BatchRangeFetcher::clear() { + reqs_.clear(); + phys_.clear(); +} + +Status BatchRangeFetcher::fetch() { + if (reader_ == nullptr) + return Status::Error( + "batch_range_fetcher: null reader"); + phys_.clear(); + if (reqs_.empty()) return Status::OK(); + + std::vector order(reqs_.size()); + for (size_t i = 0; i < order.size(); ++i) order[i] = i; + std::sort(order.begin(), order.end(), + [&](size_t a, size_t b) { return reqs_[a].offset < reqs_[b].offset; }); + + // Sweep in offset order, merging requests into physical segments. + std::vector segs; + uint64_t cur_start = 0; + uint64_t cur_end = 0; + for (size_t k = 0; k < order.size(); ++k) { + Req& r = reqs_[order[k]]; + uint64_t r_end = 0; + RETURN_IF_ERROR(checked_end(r.offset, r.len, &r_end)); + RETURN_IF_ERROR(checked_size(r.len, &r.len_size)); + const bool disjoint = r.offset > cur_end && r.offset - cur_end > coalesce_gap_; + if (segs.empty() || disjoint) { + segs.push_back(Range {r.offset, 0}); // length finalized below + cur_start = r.offset; + cur_end = r_end; + } else { + cur_end = std::max(cur_end, r_end); + } + r.phys_idx = segs.size() - 1; + RETURN_IF_ERROR(checked_size(r.offset - cur_start, &r.sub_offset)); + RETURN_IF_ERROR(checked_size(cur_end - cur_start, &segs.back().len)); + } + + return reader_->read_batch(segs, &phys_); +} + +Slice BatchRangeFetcher::get(size_t h) const { + const Req& r = reqs_[h]; + const std::vector& buf = phys_[r.phys_idx]; + return Slice(buf.data() + r.sub_offset, r.len_size); +} + +} // namespace doris::snii::io diff --git a/be/src/storage/index/snii/io/batch_range_fetcher.h b/be/src/storage/index/snii/io/batch_range_fetcher.h new file mode 100644 index 00000000000000..1ef41c2fdc75d0 --- /dev/null +++ b/be/src/storage/index/snii/io/batch_range_fetcher.h @@ -0,0 +1,70 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/io/file_reader.h" + +namespace doris::snii::io { + +// Collects the byte ranges a query plan needs, coalesces overlapping/adjacent +// ranges into physical reads, and fetches them in a single batch (one serial +// I/O round as the test-side MeteredFileReader counts it). Callers retrieve each requested range by +// the handle returned from add(). This is the SNII read path's batching layer: +// it front-loads range planning so reads are issued concurrently rather than +// cursor-by-cursor. +class BatchRangeFetcher { +public: + // coalesce_gap: requests separated by a gap <= this many bytes are merged into + // one physical read (reads a few extra bytes to save a request). 0 merges only + // overlapping/adjacent ranges. + explicit BatchRangeFetcher(FileReader* reader, uint64_t coalesce_gap = 0); + + // Registers a desired range; returns a handle usable with get() after fetch(). + size_t add(uint64_t offset, uint64_t len); + + // Coalesces and issues one batched read; fills internal buffers. + Status fetch(); + + // Bytes for handle h (valid only after a successful fetch(), until clear()). + Slice get(size_t h) const; + + size_t pending() const { return reqs_.size(); } + void clear(); + +private: + struct Req { + uint64_t offset; + uint64_t len; + size_t len_size = 0; // validated size_t length after successful fetch() + size_t phys_idx = 0; // index into phys_ after fetch + size_t sub_offset = 0; // byte offset of this req within its physical read + }; + + FileReader* reader_; + uint64_t coalesce_gap_; + std::vector reqs_; + std::vector> phys_; // physical read buffers after fetch +}; + +} // namespace doris::snii::io diff --git a/be/src/storage/index/snii/io/file_reader.h b/be/src/storage/index/snii/io/file_reader.h new file mode 100644 index 00000000000000..46608aea602197 --- /dev/null +++ b/be/src/storage/index/snii/io/file_reader.h @@ -0,0 +1,67 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/io/io_metrics.h" + +namespace doris::snii::io { + +// One logical read request (offset, length). +struct Range { + uint64_t offset = 0; + size_t len = 0; +}; + +// The single physical-read primitive (a BE-internal read_at). All higher layers +// route reads through this so I/O can be accounted and backed by local files or +// object storage interchangeably. +class FileReader { +public: + virtual ~FileReader() = default; + + // Reads exactly len bytes starting at offset into *out (which is resized to + // len). Reading past EOF is an error (Corruption/IoError). + virtual Status read_at(uint64_t offset, size_t len, std::vector* out) = 0; + + // Reads a batch of ranges that may be served concurrently. The default is a + // sequential loop; backends that model concurrency (the test-side + // MeteredFileReader) or perform real parallel fetches (object storage) + // override this. + virtual Status read_batch(const std::vector& ranges, + std::vector>* outs) { + outs->resize(ranges.size()); + for (size_t i = 0; i < ranges.size(); ++i) { + RETURN_IF_ERROR(read_at(ranges[i].offset, ranges[i].len, &(*outs)[i])); + } + return Status::OK(); + } + + // Total size of the underlying object in bytes. + virtual uint64_t size() const = 0; + + // Optional live metrics. Readers that do not account I/O return nullptr. + virtual const IoMetrics* io_metrics() const { return nullptr; } +}; + +} // namespace doris::snii::io diff --git a/be/src/storage/index/snii/io/file_writer.h b/be/src/storage/index/snii/io/file_writer.h new file mode 100644 index 00000000000000..c14c61d2beaefd --- /dev/null +++ b/be/src/storage/index/snii/io/file_writer.h @@ -0,0 +1,40 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include "common/status.h" +#include "storage/index/snii/common/slice.h" + +namespace doris::snii::io { + +// Append-only writer (no seek-back), so the format can be produced in a single +// streaming pass compatible with S3FileWriter / StreamSinkFileWriter / packed +// writer. All container bytes are written front-to-back; back-references are +// resolved by writing metadata last. +class FileWriter { +public: + virtual ~FileWriter() = default; + + virtual Status append(Slice data) = 0; + virtual Status finalize() = 0; + virtual uint64_t bytes_written() const = 0; +}; + +} // namespace doris::snii::io diff --git a/be/src/storage/index/snii/io/io_metrics.h b/be/src/storage/index/snii/io/io_metrics.h new file mode 100644 index 00000000000000..0e1c628ede0137 --- /dev/null +++ b/be/src/storage/index/snii/io/io_metrics.h @@ -0,0 +1,43 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +namespace doris::snii::io { + +// Object-storage access metrics collected at FileReader boundaries. +struct IoMetrics { + uint64_t read_at_calls = 0; // BE-internal logical read requests issued + uint64_t serial_rounds = 0; // dependent serial I/O rounds + uint64_t range_gets = 0; // remote range GETs after cache coalescing + uint64_t remote_bytes = 0; // bytes fetched from remote + uint64_t total_request_bytes = 0; // sum of requested lengths before cache +}; + +inline IoMetrics delta(const IoMetrics& after, const IoMetrics& before) { + IoMetrics out; + out.read_at_calls = after.read_at_calls - before.read_at_calls; + out.serial_rounds = after.serial_rounds - before.serial_rounds; + out.range_gets = after.range_gets - before.range_gets; + out.remote_bytes = after.remote_bytes - before.remote_bytes; + out.total_request_bytes = after.total_request_bytes - before.total_request_bytes; + return out; +} + +} // namespace doris::snii::io diff --git a/be/src/storage/index/snii/query/bm25_scorer.cpp b/be/src/storage/index/snii/query/bm25_scorer.cpp new file mode 100644 index 00000000000000..dc3e82ad42706d --- /dev/null +++ b/be/src/storage/index/snii/query/bm25_scorer.cpp @@ -0,0 +1,64 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/query/bm25_scorer.h" + +#include +#include + +namespace doris::snii::query { + +double decode_norm(uint8_t encoded) { + return encoded == 0 ? 1.0 : static_cast(encoded); +} + +uint8_t encode_norm(uint64_t doc_length) { + const uint64_t clamped = std::clamp(doc_length, 1, 255); + return static_cast(clamped); +} + +ScorerContext ScorerContext::make(uint64_t n, uint64_t df) { + ScorerContext ctx; + ctx.df_ = df; + const double nn = static_cast(n); + const double dff = static_cast(df); + // idf = log(1 + (N - df + 0.5) / (df + 0.5)); always positive for df <= N. + ctx.idf_ = std::log(1.0 + (nn - dff + 0.5) / (dff + 0.5)); + return ctx; +} + +ScorerContext ScorerContext::from_idf(double idf) { + ScorerContext ctx; + ctx.idf_ = idf; + return ctx; +} + +double ScorerContext::score(double tf, uint8_t encoded_norm, double avgdl, + const Bm25Params& params) const { + const double dl = decode_norm(encoded_norm); + const double denom = tf + params.k1 * (1.0 - params.b + params.b * dl / avgdl); + return idf_ * (tf * (params.k1 + 1.0)) / denom; +} + +double ScorerContext::max_score(uint32_t max_freq, uint8_t min_norm, double avgdl, + const Bm25Params& params) const { + // The score grows monotonically with tf and shrinks with dl, so the per-window + // upper bound uses the window's largest tf and smallest dl (min encoded norm). + return score(max_freq, min_norm, avgdl, params); +} + +} // namespace doris::snii::query diff --git a/be/src/storage/index/snii/query/bm25_scorer.h b/be/src/storage/index/snii/query/bm25_scorer.h new file mode 100644 index 00000000000000..841edb171f07f5 --- /dev/null +++ b/be/src/storage/index/snii/query/bm25_scorer.h @@ -0,0 +1,85 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +// Bm25Scorer -- classic Okapi BM25 relevance scoring over SNII native stats. +// +// Per query term, idf is precomputed once from the collection statistics: +// idf = log(1 + (N - df + 0.5) / (df + 0.5)) +// where N = indexed doc count and df = the term's document frequency. The +// per-document contribution of a term then is: +// score = idf * (tf * (k1 + 1)) / (tf + k1 * (1 - b + b * dl / avgdl)) +// where tf is the in-doc term frequency, dl the document length decoded from the +// 1-byte encoded norm, and avgdl the average document length. +// +// Norm encode/decode (DOCUMENTED CONTRACT): the writer stores doc length as a +// byte-quantized value floor-clamped to [1, 255]; decode is the identity map +// back to a double length. encode_norm(len) = clamp(len, 1, 255); +// decode_norm(b) = (b == 0 ? 1.0 : (double)b). This keeps short docs (len <= 255) +// exact and saturates longer docs at 255, matching the reference oracle. +namespace doris::snii::query { + +// BM25 free parameters. Defaults are the classic Lucene/Elasticsearch values. +struct Bm25Params { + double k1 = 1.2; + double b = 0.75; +}; + +// Decodes a 1-byte encoded norm into a document length. byte 0 maps to 1.0 to +// avoid a zero-length divisor; otherwise it is the byte value itself. +double decode_norm(uint8_t encoded); + +// Encodes a document length into a 1-byte norm (clamped to [1, 255]). Provided +// so writers and test oracles share one quantization. +uint8_t encode_norm(uint64_t doc_length); + +// Per-term scoring context: the precomputed idf and the term's df. Built once per +// query term, then reused for every candidate document of that term. +class ScorerContext { +public: + // Builds the context from collection size n (indexed doc count) and the term's + // document frequency df. avgdl and params are supplied per score call. + static ScorerContext make(uint64_t n, uint64_t df); + + // Builds a context from a collection-scoped IDF that was computed outside + // the segment reader. This keeps segment-local TF/norm decoding separate + // from scanner-collection N/DF aggregation. + static ScorerContext from_idf(double idf); + + double idf() const { return idf_; } + uint64_t df() const { return df_; } + + // Scores one document occurrence: tf is the in-doc term frequency, encoded_norm + // the doc's 1-byte length norm, avgdl the collection average length. + double score(double tf, uint8_t encoded_norm, double avgdl, const Bm25Params& params) const; + + // Upper bound on score() over any document, given a window's maximum tf and the + // shortest doc length in the window (smallest dl maximizes the score). Used by + // the WAND-style block-max pruner. max_freq is the window's max tf; min_norm is + // the smallest encoded norm (=> smallest dl => largest score). + double max_score(uint32_t max_freq, uint8_t min_norm, double avgdl, + const Bm25Params& params) const; + +private: + double idf_ = 0.0; + uint64_t df_ = 0; +}; + +} // namespace doris::snii::query diff --git a/be/src/storage/index/snii/query/boolean_query.cpp b/be/src/storage/index/snii/query/boolean_query.cpp new file mode 100644 index 00000000000000..ab26dee1005efb --- /dev/null +++ b/be/src/storage/index/snii/query/boolean_query.cpp @@ -0,0 +1,124 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/query/boolean_query.h" + +#include +#include +#include +#include + +#include "storage/index/snii/format/dict_entry.h" +#include "storage/index/snii/query/docid_sink.h" +#include "storage/index/snii/query/internal/docid_conjunction.h" +#include "storage/index/snii/query/internal/docid_posting_reader.h" +#include "storage/index/snii/query/internal/docid_union.h" +#include "storage/index/snii/reader/dict_block_cache.h" + +namespace doris::snii::query { + +namespace { + +std::vector unique_terms(const std::vector& terms) { + std::vector out; + out.reserve(terms.size()); + for (const std::string& term : terms) out.emplace_back(term); + std::sort(out.begin(), out.end()); + out.erase(std::unique(out.begin(), out.end()), out.end()); + return out; +} + +Status resolve_or_postings(const reader::LogicalIndexReader& idx, + const std::vector& terms, + std::vector* postings) { + postings->clear(); + // Request-scoped (stack-local, single-threaded) cache: OR terms that fall in the + // same on-demand DICT block read + zstd-decode + CRC-verify that block once + // instead of once per term. The resolved DictEntry is copied out, so the cache + // (and any pin it holds) can die when this returns. The shared reader stays const + // and lock-free -- no lock is ever held across the decode/IO. + reader::DictBlockCache dict_cache; + for (std::string_view term : unique_terms(terms)) { + bool found = false; + format::DictEntry entry; + uint64_t frq_base = 0; + uint64_t prx_base = 0; + RETURN_IF_ERROR(idx.lookup(term, &found, &entry, &frq_base, &prx_base, &dict_cache)); + if (!found) continue; + + postings->push_back({std::move(entry), frq_base, prx_base}); + } + return Status::OK(); +} + +} // namespace + +Status boolean_or(const reader::LogicalIndexReader& idx, const std::vector& terms, + std::vector* docids) { + if (docids == nullptr) + return Status::Error("boolean_or: null out"); + docids->clear(); + if (terms.empty()) return Status::OK(); + + std::vector postings; + RETURN_IF_ERROR(resolve_or_postings(idx, terms, &postings)); + return internal::build_docid_union(idx, postings, docids); +} + +Status boolean_or(const reader::LogicalIndexReader& idx, const std::vector& terms, + std::vector* docids, QueryProfile* profile) { + QueryProfileScope profile_scope(idx.reader(), profile); + return boolean_or(idx, terms, docids); +} + +Status boolean_or(const reader::LogicalIndexReader& idx, const std::vector& terms, + DocIdSink* sink) { + if (sink == nullptr) + return Status::Error("boolean_or: null sink"); + if (terms.empty()) return Status::OK(); + + std::vector postings; + RETURN_IF_ERROR(resolve_or_postings(idx, terms, &postings)); + return internal::emit_docid_union(idx, postings, sink); +} + +Status boolean_and(const reader::LogicalIndexReader& idx, const std::vector& terms, + std::vector* docids) { + if (docids == nullptr) + return Status::Error("boolean_and: null out"); + docids->clear(); + if (terms.empty()) return Status::OK(); + + io::BatchRangeFetcher round1(idx.reader()); + std::vector plans; + bool all_present = false; + RETURN_IF_ERROR(internal::plan_terms(idx, terms, &round1, &plans, &all_present, + /*need_positions=*/false)); + if (!all_present) return Status::OK(); + if (round1.pending() > 0) RETURN_IF_ERROR(round1.fetch()); + RETURN_IF_ERROR(internal::open_preludes(round1, &plans, + /*need_positions=*/false)); + return internal::build_docid_only_conjunction(idx, round1, plans, docids); +} + +Status boolean_and(const reader::LogicalIndexReader& idx, const std::vector& terms, + std::vector* docids, QueryProfile* profile) { + QueryProfileScope profile_scope(idx.reader(), profile); + return boolean_and(idx, terms, docids); +} + +} // namespace doris::snii::query diff --git a/be/src/storage/index/snii/query/boolean_query.h b/be/src/storage/index/snii/query/boolean_query.h new file mode 100644 index 00000000000000..40cbb25644fdc1 --- /dev/null +++ b/be/src/storage/index/snii/query/boolean_query.h @@ -0,0 +1,50 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/query/docid_sink.h" +#include "storage/index/snii/query/query_profile.h" +#include "storage/index/snii/reader/logical_index_reader.h" + +// boolean_or -- MATCH_ANY semantics: return the sorted docid set containing at +// least one query term. Empty terms or all-absent terms produce an empty +// result. Duplicate input terms are ignored semantically and do not duplicate +// output docids. +namespace doris::snii::query { + +Status boolean_or(const reader::LogicalIndexReader& idx, const std::vector& terms, + std::vector* docids); +Status boolean_or(const reader::LogicalIndexReader& idx, const std::vector& terms, + std::vector* docids, QueryProfile* profile); +Status boolean_or(const reader::LogicalIndexReader& idx, const std::vector& terms, + DocIdSink* sink); + +// boolean_and (MATCH all-terms): sorted docid set of docs containing EVERY +// term, no positional constraint. Valid on docs-only indexes. Empty terms or +// any absent term -> empty result. +Status boolean_and(const reader::LogicalIndexReader& idx, const std::vector& terms, + std::vector* docids); +Status boolean_and(const reader::LogicalIndexReader& idx, const std::vector& terms, + std::vector* docids, QueryProfile* profile); + +} // namespace doris::snii::query diff --git a/be/src/storage/index/snii/query/count_query.cpp b/be/src/storage/index/snii/query/count_query.cpp new file mode 100644 index 00000000000000..3cc77ff9605e88 --- /dev/null +++ b/be/src/storage/index/snii/query/count_query.cpp @@ -0,0 +1,80 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/query/count_query.h" + +#include +#include + +#include "roaring/roaring.hh" +#include "storage/index/snii/format/dict_entry.h" +#include "storage/index/snii/query/internal/query_test_counters.h" + +namespace doris::snii::query { + +using format::DictEntry; +using reader::LogicalIndexReader; + +Status count_only_term_df(const LogicalIndexReader& idx, std::string_view term, uint64_t* count) { + if (count == nullptr) { + return Status::Error("count_only_term_df: null out"); + } + bool found = false; + DictEntry entry; + uint64_t frq_base = 0; + uint64_t prx_base = 0; + RETURN_IF_ERROR(idx.lookup(term, &found, &entry, &frq_base, &prx_base)); + *count = found ? entry.df : 0; + SNII_QUERY_COUNT(count_fastpath_hits); + return Status::OK(); +} + +Status fabricate_null_disjoint_count_bitmap(uint64_t count, const roaring::Roaring& nulls, + roaring::Roaring* out) { + if (out == nullptr) { + return Status::Error( + "fabricate_null_disjoint_count_bitmap: null out"); + } + roaring::Roaring result; + if (count > 0) { + // [0, count + |nulls|) holds at least `count` non-null ids: at most + // |nulls| of its members are null. count counts only non-null docs, so + // the window end never exceeds the segment doc count (row space). + const uint64_t window_end = count + nulls.cardinality(); + if (window_end > uint64_t(std::numeric_limits::max()) + 1) { + return Status::Error( + "fabricate_null_disjoint_count_bitmap: count {} + null count {} exceeds the " + "uint32 docid domain (corrupt df or null bitmap)", + count, nulls.cardinality()); + } + result.addRange(0, window_end); + result -= nulls; + uint32_t last_kept = 0; + // Keep exactly the first `count` survivors (select ranks are 0-based). + if (!result.select(static_cast(count - 1), &last_kept)) { + return Status::Error( + "fabricate_null_disjoint_count_bitmap: window [0, {}) holds fewer than {} " + "non-null ids (corrupt df or null bitmap)", + window_end, count); + } + result.removeRange(uint64_t(last_kept) + 1, window_end); + } + *out = std::move(result); + return Status::OK(); +} + +} // namespace doris::snii::query diff --git a/be/src/storage/index/snii/query/count_query.h b/be/src/storage/index/snii/query/count_query.h new file mode 100644 index 00000000000000..bae76b4d91eed8 --- /dev/null +++ b/be/src/storage/index/snii/query/count_query.h @@ -0,0 +1,69 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include "common/status.h" +#include "storage/index/snii/reader/logical_index_reader.h" + +// Forward-declare the CRoaring C++ bitmap so this header stays free of the +// (large) roaring include; the concrete type is only needed in the .cpp. +namespace roaring { +class Roaring; +} // namespace roaring + +// count_query -- G02 single-term count-only fast path primitives. They answer +// "how many docs match" from a dict entry alone, without reading .frq bytes. +// Multi-term queries, prefix/regexp/wildcard expansion, and phrases execute the +// normal query path. Deletes and extra predicates are a caller responsibility; +// see SniiIndexReader::_try_count_only_fastpath and the SegmentIterator guards +// in count_on_index_fastpath.h. +namespace doris::snii::query { + +// df of `term` in this segment without decoding postings. An absent term is a +// deterministic answer too: *count = 0 (mirrors term_query's empty result). +// Increments the count_fastpath_hits test seam. +Status count_only_term_df(const reader::LogicalIndexReader& idx, std::string_view term, + uint64_t* count); + +// Builds the fabricated count bitmap for a segment WITH a null bitmap: exactly +// `count` row ids DISJOINT from `nulls` (the first `count` non-null row ids, +// all < count + |nulls|). Why disjoint: the MATCH machinery unconditionally +// subtracts the segment null bitmap from every index result +// (FunctionMatchBase -> InvertedIndexResultBitmap::mask_out_null). Real +// postings never contain null docs -- the writer adds NO tokens for a null doc +// (scalar add_nulls) and a NULL array row is stored as an empty range (zero +// tokens) -- so that subtraction is a no-op on true results and df already IS +// the exact match count regardless of nulls. A naive [0, df) range however MAY +// collide with null row ids and be shrunk by mask_out_null; picking the ids +// from the non-null space makes the subtraction provably a no-op, preserving +// cardinality == df end to end. +// +// The window bound is doc-count-free: count counts only non-null docs, so +// count + |nulls| <= segment doc count and [0, count + |nulls|) always holds +// >= count non-null ids; every fabricated id therefore stays inside the +// segment's [0, num_rows) row space. Errors (id space would exceed the uint32 +// docid domain, or the window unexpectedly holds fewer than `count` survivors) +// only occur on a corrupt index; callers treat them as "fall through to the +// decode path", never as a fabricated answer. +Status fabricate_null_disjoint_count_bitmap(uint64_t count, const roaring::Roaring& nulls, + roaring::Roaring* out); + +} // namespace doris::snii::query diff --git a/be/src/storage/index/snii/query/docid_conjunction.cpp b/be/src/storage/index/snii/query/docid_conjunction.cpp new file mode 100644 index 00000000000000..1e3b892b4abfac --- /dev/null +++ b/be/src/storage/index/snii/query/docid_conjunction.cpp @@ -0,0 +1,929 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/query/internal/docid_conjunction.h" + +#include +#include +#include +#include +#include + +#include "common/check.h" +#include "storage/index/snii/format/frq_pod.h" +#include "storage/index/snii/query/internal/docid_set_ops.h" +#include "storage/index/snii/query/internal/query_test_counters.h" +#include "storage/index/snii/reader/windowed_posting.h" + +namespace doris::snii::query::internal { + +using format::DictEntry; +using format::DictEntryEnc; +using format::DictEntryKind; +using format::FrqPreludeReader; +using format::WindowMeta; +using reader::LogicalIndexReader; + +namespace { + +using CandidateIt = std::vector::const_iterator; + +constexpr uint32_t kBoundedSpanBitsetDocs = 16 * 1024; +constexpr size_t kBoundedSpanBitsetWords = kBoundedSpanBitsetDocs / 64; +constexpr size_t kBoundedSpanBitsetMinInput = 32; + +struct CandidateRange { + size_t begin = 0; + size_t end = 0; +}; + +Status slim_frq_docs_len(const DictEntry& entry, uint64_t win_len, uint64_t* out) { + if (entry.frq_docs_len > win_len) { + return Status::Error( + "docid_conjunction: slim frq_docs_len exceeds frq window"); + } + *out = entry.frq_docs_len > 0 ? entry.frq_docs_len : win_len; + return Status::OK(); +} + +Status add_u64(uint64_t lhs, uint64_t rhs, const char* message, uint64_t* out) { + if (rhs > std::numeric_limits::max() - lhs) { + return Status::Error(message); + } + *out = lhs + rhs; + return Status::OK(); +} + +Status posting_abs_offset(const LogicalIndexReader& idx, uint64_t base, uint64_t delta, + const char* message, uint64_t* out) { + uint64_t with_base = 0; + RETURN_IF_ERROR(add_u64(idx.section_refs().posting_region.offset, base, message, &with_base)); + return add_u64(with_base, delta, message, out); +} + +Status configure_term_plan(const LogicalIndexReader& idx, bool need_positions, + io::BatchRangeFetcher* fetcher, TermPlan* p) { + p->df = p->entry.df; + p->pod_ref = (p->entry.kind == DictEntryKind::kPodRef); + p->windowed = p->pod_ref && p->entry.enc == DictEntryEnc::kWindowed; + if (p->windowed) { + uint64_t prelude_abs = 0; + RETURN_IF_ERROR(posting_abs_offset(idx, p->frq_base, p->entry.frq_off_delta, + "docid_conjunction: prelude offset overflow", + &prelude_abs)); + p->prelude_handle = fetcher->add(prelude_abs, p->entry.prelude_len); + } else if (p->pod_ref) { + uint64_t foff = 0; + uint64_t flen = 0; + uint64_t poff = 0; + uint64_t plen = 0; + RETURN_IF_ERROR(idx.resolve_frq_window(p->entry, p->frq_base, &foff, &flen)); + uint64_t frq_fetch = flen; + RETURN_IF_ERROR(slim_frq_docs_len(p->entry, flen, &frq_fetch)); + p->frq_handle = fetcher->add(foff, frq_fetch); + if (need_positions) { + RETURN_IF_ERROR(idx.resolve_prx_window(p->entry, p->prx_base, &poff, &plen)); + p->prx_handle = fetcher->add(poff, plen); + } + } + return Status::OK(); +} + +std::vector all_windows(const FrqPreludeReader& prelude) { + std::vector ws(prelude.n_windows()); + for (uint32_t i = 0; i < prelude.n_windows(); ++i) ws[i] = i; + return ws; +} + +std::vector ascending_df_order(const std::vector& plans) { + std::vector order(plans.size()); + for (size_t i = 0; i < plans.size(); ++i) order[i] = i; + std::sort(order.begin(), order.end(), + [&](size_t a, size_t b) { return plans[a].df < plans[b].df; }); + return order; +} + +Status first_docid_in_window(const WindowMeta& meta, uint32_t window_ordinal, uint32_t* first) { + if (window_ordinal == 0) { + *first = 0; + return Status::OK(); + } + if (meta.win_base >= std::numeric_limits::max()) { + return Status::Error( + "docid_conjunction: window base exceeds docid range"); + } + *first = static_cast(meta.win_base + 1); + if (*first > meta.last_docid) { + return Status::Error( + "docid_conjunction: invalid window docid range"); + } + return Status::OK(); +} + +Status is_dense_full_window(const WindowMeta& meta, uint32_t window_ordinal, bool* full) { + uint32_t first = 0; + RETURN_IF_ERROR(first_docid_in_window(meta, window_ordinal, &first)); + const uint64_t width = static_cast(meta.last_docid) - first + 1; + *full = meta.doc_count == width; + return Status::OK(); +} + +Status append_docid_range(uint32_t first, uint32_t last, std::vector* out) { + if (last < first) { + return Status::Error( + "docid_conjunction: invalid dense docid range"); + } + const uint64_t count64 = static_cast(last) - first + 1; + if (count64 > static_cast(std::numeric_limits::max() - out->size())) { + return Status::Error( + "docid_conjunction: dense docid range too large"); + } + out->reserve(out->size() + static_cast(count64)); + uint32_t docid = first; + while (true) { + out->push_back(docid); + if (docid == last) break; + ++docid; + } + return Status::OK(); +} + +CandidateRange find_candidate_range(const std::vector& candidates, size_t* search_begin, + uint32_t first, uint32_t last) { + const auto from = candidates.begin() + *search_begin; + const auto begin = std::lower_bound(from, candidates.end(), first); + const auto end = std::upper_bound(begin, candidates.end(), last); + *search_begin = static_cast(end - candidates.begin()); + return {.begin = static_cast(begin - candidates.begin()), + .end = static_cast(end - candidates.begin())}; +} + +void append_candidate_range(CandidateIt begin, CandidateIt end, std::vector* out) { + out->insert(out->end(), begin, end); +} + +void append_new_chunk_docids_to_out(const DocidChunk& chunk, size_t chunk_docids_begin, + std::vector* out) { + out->insert(out->end(), chunk.docids.begin() + chunk_docids_begin, chunk.docids.end()); +} + +void clear_ordinals_if_all_term_docs_selected(const std::vector& term_docids, + DocidChunk* chunk) { + if (chunk->docids.size() == term_docids.size() && !chunk->docids.empty() && + chunk->docids.front() == term_docids.front() && + chunk->docids.back() == term_docids.back()) { + chunk->prx_doc_ordinals.clear(); + } +} + +bool append_term_docs_if_candidates_cover_span(CandidateIt begin, CandidateIt end, + const std::vector& term_docids, + std::vector* out, DocidChunk* chunk) { + const uint32_t first = term_docids.front(); + const uint32_t last = term_docids.back(); + const uint64_t width = static_cast(last) - first + 1; + const size_t candidate_count = static_cast(end - begin); + if (width > candidate_count) { + return false; + } + + const auto span_begin = *begin == first ? begin : std::lower_bound(begin, end, first); + if (span_begin == end || *span_begin != first) { + return false; + } + if (static_cast(end - span_begin) < width) { + return false; + } + + const auto span_last = span_begin + static_cast(width) - 1; + if (*span_last != last) { + return false; + } + + const size_t chunk_docids_begin = chunk->docids.size(); + chunk->docids.insert(chunk->docids.end(), term_docids.begin(), term_docids.end()); + append_new_chunk_docids_to_out(*chunk, chunk_docids_begin, out); + return true; +} + +Status append_candidate_range_with_ordinals(CandidateIt begin, CandidateIt end, uint32_t first, + uint32_t last, std::vector* out, + DocidChunk* chunk) { + const size_t candidate_count = static_cast(end - begin); + chunk->docids.reserve(candidate_count); + const uint64_t width = static_cast(last) - first + 1; + if (width > std::numeric_limits::max()) { + return Status::Error( + "docid_conjunction: dense window exceeds doc count range"); + } + chunk->prx_doc_count = static_cast(width); + const bool full_dense_range = + candidate_count == width && begin != end && *begin == first && *(end - 1) == last; + const size_t chunk_docids_begin = chunk->docids.size(); + if (full_dense_range) { + chunk->docids.insert(chunk->docids.end(), begin, end); + append_new_chunk_docids_to_out(*chunk, chunk_docids_begin, out); + return Status::OK(); + } + chunk->prx_doc_ordinals.reserve(chunk->prx_doc_ordinals.size() + candidate_count); + for (auto it = begin; it != end; ++it) { + chunk->docids.push_back(*it); + chunk->prx_doc_ordinals.push_back(*it - first); + } + append_new_chunk_docids_to_out(*chunk, chunk_docids_begin, out); + return Status::OK(); +} + +bool intersect_dense_term_span_with_ordinals(CandidateIt begin, CandidateIt end, + const std::vector& term_docids, + size_t candidate_count, std::vector* out, + DocidChunk* chunk) { + const uint32_t first = term_docids.front(); + const uint32_t last = term_docids.back(); + const uint64_t width = static_cast(last) - first + 1; + if (term_docids.size() > width) { + return false; + } + const uint64_t missing_count = width - term_docids.size(); + if (missing_count != 0 && + (missing_count * 8 > width || missing_count >= candidate_count || + missing_count > static_cast(std::numeric_limits::max()))) { + return false; + } + + const size_t chunk_docids_begin = chunk->docids.size(); + if (missing_count == 0) { + for (auto it = begin; it != end; ++it) { + if (*it < first) { + continue; + } + if (*it > last) { + break; + } + chunk->docids.push_back(*it); + chunk->prx_doc_ordinals.push_back(*it - first); + } + append_new_chunk_docids_to_out(*chunk, chunk_docids_begin, out); + clear_ordinals_if_all_term_docs_selected(term_docids, chunk); + return true; + } + + std::vector missing; + missing.reserve(static_cast(missing_count)); + uint32_t expect = first; + for (uint32_t docid : term_docids) { + while (expect < docid) { + missing.push_back(expect); + ++expect; + } + if (docid < std::numeric_limits::max()) { + expect = docid + 1; + } + } + while (expect <= last) { + missing.push_back(expect); + if (expect == std::numeric_limits::max()) { + break; + } + ++expect; + } + + size_t miss = 0; + for (auto it = begin; it != end; ++it) { + if (*it < first) { + continue; + } + if (*it > last) { + break; + } + while (miss < missing.size() && missing[miss] < *it) { + ++miss; + } + if (miss < missing.size() && missing[miss] == *it) { + continue; + } + chunk->docids.push_back(*it); + chunk->prx_doc_ordinals.push_back(static_cast(*it - first - miss)); + } + append_new_chunk_docids_to_out(*chunk, chunk_docids_begin, out); + clear_ordinals_if_all_term_docs_selected(term_docids, chunk); + return true; +} + +bool intersect_bounded_span_with_ordinals(CandidateIt begin, CandidateIt end, + const std::vector& term_docids, + size_t candidate_count, std::vector* out, + DocidChunk* chunk) { + if (candidate_count < kBoundedSpanBitsetMinInput || + term_docids.size() < kBoundedSpanBitsetMinInput) { + return false; + } + + const uint32_t first = std::min(*begin, term_docids.front()); + const uint32_t last = std::max(*(end - 1), term_docids.back()); + const uint64_t width = static_cast(last) - first + 1; + if (width > kBoundedSpanBitsetDocs || term_docids.size() > width) { + return false; + } + + const auto word_count = static_cast((width + 63) >> 6); + std::array bits; + std::fill_n(bits.begin(), word_count, 0); + for (uint32_t docid : term_docids) { + const uint32_t off = docid - first; + bits[off >> 6] |= 1ULL << (off & 63); + } + + std::array ordinal_base; + uint32_t ordinal = 0; + for (size_t word = 0; word < word_count; ++word) { + ordinal_base[word] = ordinal; + ordinal += static_cast(__builtin_popcountll(bits[word])); + } + + const size_t chunk_docids_begin = chunk->docids.size(); + for (auto it = begin; it != end; ++it) { + const uint32_t off = *it - first; + const size_t word = off >> 6; + const uint64_t mask = 1ULL << (off & 63); + if ((bits[word] & mask) == 0) { + continue; + } + chunk->docids.push_back(*it); + chunk->prx_doc_ordinals.push_back( + ordinal_base[word] + + static_cast(__builtin_popcountll(bits[word] & (mask - 1)))); + } + append_new_chunk_docids_to_out(*chunk, chunk_docids_begin, out); + clear_ordinals_if_all_term_docs_selected(term_docids, chunk); + return true; +} + +size_t log2_ceil(size_t n) { + if (n <= 1) return 1; + --n; + size_t bits = 0; + while (n != 0) { + ++bits; + n >>= 1; + } + return bits; +} + +void intersect_window_candidate_range(CandidateIt begin, CandidateIt end, + const std::vector& term_docids, uint32_t first, + uint32_t last, std::vector* out) { + const size_t candidate_count = static_cast(end - begin); + if (candidate_count == 0 || term_docids.empty()) return; + + const uint64_t width = static_cast(last) - first + 1; + const uint64_t missing_count = term_docids.size() <= width ? width - term_docids.size() : width; + if (term_docids.size() <= width && missing_count != 0 && missing_count * 8 <= width && + missing_count < candidate_count) { + std::vector missing; + missing.reserve(static_cast(missing_count)); + uint32_t expect = first; + for (uint32_t docid : term_docids) { + while (expect < docid) { + missing.push_back(expect); + ++expect; + } + if (docid < std::numeric_limits::max()) expect = docid + 1; + } + while (expect <= last) { + missing.push_back(expect); + if (expect == std::numeric_limits::max()) break; + ++expect; + } + size_t miss = 0; + for (auto it = begin; it != end; ++it) { + while (miss < missing.size() && missing[miss] < *it) ++miss; + if (miss == missing.size() || missing[miss] != *it) out->push_back(*it); + } + return; + } + + const size_t probes_per_candidate = log2_ceil(term_docids.size()) + 1; + if (candidate_count < term_docids.size() / probes_per_candidate) { + for (auto it = begin; it != end; ++it) { + if (std::binary_search(term_docids.begin(), term_docids.end(), *it)) { + out->push_back(*it); + } + } + return; + } + std::set_intersection(begin, end, term_docids.begin(), term_docids.end(), + std::back_inserter(*out)); +} + +Status intersect_window_candidate_range_with_ordinals(CandidateIt begin, CandidateIt end, + const std::vector& term_docids, + std::vector* out, + DocidChunk* chunk) { + if (term_docids.size() > std::numeric_limits::max()) { + return Status::Error( + "docid_conjunction: prx doc count exceeds u32"); + } + chunk->prx_doc_count = static_cast(term_docids.size()); + if (begin == end || term_docids.empty()) return Status::OK(); + + const size_t candidate_count = static_cast(end - begin); + const size_t max_matches = std::min(candidate_count, term_docids.size()); + out->reserve(out->size() + max_matches); + chunk->docids.reserve(chunk->docids.size() + max_matches); + if (candidate_count == term_docids.size() && *begin == term_docids.front() && + *(end - 1) == term_docids.back() && std::equal(begin, end, term_docids.begin())) { + const size_t chunk_docids_begin = chunk->docids.size(); + chunk->docids.insert(chunk->docids.end(), begin, end); + append_new_chunk_docids_to_out(*chunk, chunk_docids_begin, out); + return Status::OK(); + } + if (append_term_docs_if_candidates_cover_span(begin, end, term_docids, out, chunk)) { + return Status::OK(); + } + + chunk->prx_doc_ordinals.reserve(chunk->prx_doc_ordinals.size() + max_matches); + if (intersect_dense_term_span_with_ordinals(begin, end, term_docids, candidate_count, out, + chunk)) { + return Status::OK(); + } + if (intersect_bounded_span_with_ordinals(begin, end, term_docids, candidate_count, out, + chunk)) { + return Status::OK(); + } + + const size_t probes_per_candidate = log2_ceil(term_docids.size()) + 1; + if (candidate_count < term_docids.size() / probes_per_candidate) { + const size_t chunk_docids_begin = chunk->docids.size(); + size_t doc_index = 0; + for (auto it = begin; it != end; ++it) { + const auto found = + std::lower_bound(term_docids.begin() + doc_index, term_docids.end(), *it); + if (found == term_docids.end()) break; + doc_index = static_cast(found - term_docids.begin()); + if (*found != *it) continue; + chunk->docids.push_back(*it); + chunk->prx_doc_ordinals.push_back(static_cast(doc_index)); + ++doc_index; + } + append_new_chunk_docids_to_out(*chunk, chunk_docids_begin, out); + clear_ordinals_if_all_term_docs_selected(term_docids, chunk); + return Status::OK(); + } + + const size_t probes_per_term_doc = log2_ceil(candidate_count) + 1; + if (term_docids.size() < candidate_count / probes_per_term_doc) { + const size_t chunk_docids_begin = chunk->docids.size(); + auto candidate_it = begin; + for (size_t doc_index = 0; doc_index < term_docids.size(); ++doc_index) { + const uint32_t docid = term_docids[doc_index]; + candidate_it = std::lower_bound(candidate_it, end, docid); + if (candidate_it == end) break; + if (*candidate_it != docid) continue; + chunk->docids.push_back(docid); + chunk->prx_doc_ordinals.push_back(static_cast(doc_index)); + ++candidate_it; + } + append_new_chunk_docids_to_out(*chunk, chunk_docids_begin, out); + clear_ordinals_if_all_term_docs_selected(term_docids, chunk); + return Status::OK(); + } + + const size_t chunk_docids_begin = chunk->docids.size(); + size_t doc_index = 0; + for (auto it = begin; it != end; ++it) { + while (doc_index < term_docids.size() && term_docids[doc_index] < *it) { + ++doc_index; + } + if (doc_index == term_docids.size()) break; + if (term_docids[doc_index] != *it) continue; + chunk->docids.push_back(*it); + chunk->prx_doc_ordinals.push_back(static_cast(doc_index)); + ++doc_index; + } + append_new_chunk_docids_to_out(*chunk, chunk_docids_begin, out); + clear_ordinals_if_all_term_docs_selected(term_docids, chunk); + return Status::OK(); +} + +bool should_scan_all_windows(const LogicalIndexReader& idx, const TermPlan& p, + size_t candidate_count) { + const size_t window_count = p.prelude.n_windows(); + if (candidate_count > window_count * 64) return true; + + const uint64_t doc_count = idx.stats().doc_count; + const bool near_full = doc_count != 0 && static_cast(p.df) * 10 >= doc_count * 9; + return near_full && candidate_count > window_count * 4; +} + +Status decode_flat_docids_only(const io::BatchRangeFetcher& round1, const TermPlan& p, + std::vector* docids) { + Slice dd; + if (p.pod_ref) { + dd = round1.get(p.frq_handle); + } else { + RETURN_IF_ERROR(inline_dd_region(p.entry, &dd)); + } + return format::decode_dd_region(dd, p.entry.dd_meta, /*win_base=*/0, docids); +} + +struct WindowWork { + uint32_t ordinal = 0; + WindowMeta meta; + CandidateRange candidates; + size_t handle = 0; + bool dense_full = false; +}; + +Status emit_dense_full_window_docids(const WindowWork& f, const std::vector* candidates, + std::vector& out, DocidSource* source) { + uint32_t first = 0; + RETURN_IF_ERROR(first_docid_in_window(f.meta, f.ordinal, &first)); + if (source != nullptr) { + DocidChunk chunk; + chunk.windowed = true; + chunk.window = f.ordinal; + chunk.prx_doc_count = f.meta.doc_count; + if (candidates == nullptr) { + RETURN_IF_ERROR(append_docid_range(first, f.meta.last_docid, &chunk.docids)); + } else { + const auto begin = candidates->begin() + f.candidates.begin; + const auto end = candidates->begin() + f.candidates.end; + RETURN_IF_ERROR(append_candidate_range_with_ordinals(begin, end, first, + f.meta.last_docid, &out, &chunk)); + } + source->chunks.push_back(std::move(chunk)); + } + if (candidates == nullptr) { + RETURN_IF_ERROR(append_docid_range(first, f.meta.last_docid, &out)); + } else if (source == nullptr) { + append_candidate_range(candidates->begin() + f.candidates.begin, + candidates->begin() + f.candidates.end, &out); + } + return Status::OK(); +} + +Status emit_decoded_window_docids(const WindowWork& f, const io::BatchRangeFetcher& fetcher, + const std::vector* candidates, + std::vector& out, DocidSource* source, + std::vector& docs, std::vector& freqs, + std::vector>& positions) { + docs.clear(); + freqs.clear(); + positions.clear(); + RETURN_IF_ERROR(reader::decode_window_slices(f.meta, fetcher.get(f.handle), Slice(), Slice(), + /*want_positions=*/false, /*want_freq=*/false, + &docs, &freqs, &positions)); + if (source != nullptr) { + DocidChunk chunk; + chunk.windowed = true; + chunk.window = f.ordinal; + if (candidates == nullptr) { + chunk.docids = docs; + if (docs.size() > std::numeric_limits::max()) { + return Status::Error( + "docid_conjunction: prx doc count exceeds u32"); + } + chunk.prx_doc_count = static_cast(docs.size()); + source->chunks.push_back(std::move(chunk)); + } else { + const auto begin = candidates->begin() + f.candidates.begin; + const auto end = candidates->begin() + f.candidates.end; + RETURN_IF_ERROR( + intersect_window_candidate_range_with_ordinals(begin, end, docs, &out, &chunk)); + if (!chunk.docids.empty()) { + source->chunks.push_back(std::move(chunk)); + } + } + } + if (candidates == nullptr) { + out.insert(out.end(), docs.begin(), docs.end()); + return Status::OK(); + } + if (source != nullptr) { + return Status::OK(); + } + uint32_t first = 0; + RETURN_IF_ERROR(first_docid_in_window(f.meta, f.ordinal, &first)); + intersect_window_candidate_range(candidates->begin() + f.candidates.begin, + candidates->begin() + f.candidates.end, docs, first, + f.meta.last_docid, &out); + return Status::OK(); +} + +Status collect_windowed_docids_only(const LogicalIndexReader& idx, const TermPlan& p, + const std::vector& windows, + const std::vector* candidates, + std::vector* out, DocidSource* source) { + io::BatchRangeFetcher fetcher(idx.reader(), reader::kSameTermCoalesceGap); + std::vector work; + work.reserve(windows.size()); + out->reserve(candidates == nullptr ? p.entry.df : candidates->size()); + size_t candidate_search_begin = 0; + for (uint32_t w : windows) { + WindowMeta meta; + RETURN_IF_ERROR(p.prelude.window(w, &meta)); + uint32_t first = 0; + RETURN_IF_ERROR(first_docid_in_window(meta, w, &first)); + CandidateRange candidate_range; + if (candidates != nullptr) { + candidate_range = find_candidate_range(*candidates, &candidate_search_begin, first, + meta.last_docid); + if (candidate_range.begin == candidate_range.end) { + continue; + } + } + bool dense_full = false; + RETURN_IF_ERROR(is_dense_full_window(meta, w, &dense_full)); + if (dense_full) { + work.push_back(WindowWork { + .ordinal = w, .meta = meta, .candidates = candidate_range, .dense_full = true}); + continue; + } + + reader::WindowAbsRange range; + RETURN_IF_ERROR(reader::windowed_window_range( + idx, p.entry, p.frq_base, p.prx_base, p.prelude, w, + /*want_positions=*/false, /*want_freq=*/false, &range)); + WindowWork f; + f.ordinal = w; + f.meta = meta; + f.candidates = candidate_range; + f.handle = fetcher.add(range.dd_off, range.dd_len); + work.push_back(f); + } + if (fetcher.pending() > 0) { + RETURN_IF_ERROR(fetcher.fetch()); + } + + std::vector docs; + std::vector freqs; + std::vector> positions; + for (const WindowWork& f : work) { + if (f.dense_full) { + RETURN_IF_ERROR(emit_dense_full_window_docids(f, candidates, *out, source)); + continue; + } + RETURN_IF_ERROR(emit_decoded_window_docids(f, fetcher, candidates, *out, source, docs, + freqs, positions)); + } + return Status::OK(); +} + +Status collect_docids_only(const LogicalIndexReader& idx, const io::BatchRangeFetcher& round1, + const TermPlan& p, const std::vector* candidates, + std::vector* out, DocidSource* source) { + if (p.windowed) { + std::vector windows; + if (candidates == nullptr) { + windows = all_windows(p.prelude); + } else if (should_scan_all_windows(idx, p, candidates->size())) { + // Dense candidate sets cover most windows; for near-full terms this also + // avoids a thousands-to-millions probe covering-window cursor pass with no + // byte win. + windows = all_windows(p.prelude); + } else { + p.prelude.select_covering_windows(*candidates, &windows); + } + return collect_windowed_docids_only(idx, p, windows, candidates, out, source); + } + + std::vector term_docids; + RETURN_IF_ERROR(decode_flat_docids_only(round1, p, &term_docids)); + if (source != nullptr) { + DocidChunk chunk; + if (term_docids.size() > std::numeric_limits::max()) { + return Status::Error( + "docid_conjunction: prx doc count exceeds u32"); + } + chunk.prx_doc_count = static_cast(term_docids.size()); + if (candidates == nullptr) { + chunk.docids = term_docids; + } else if (!term_docids.empty()) { + const auto begin = std::ranges::lower_bound(*candidates, term_docids.front()); + const auto end = std::upper_bound(begin, candidates->end(), term_docids.back()); + RETURN_IF_ERROR(intersect_window_candidate_range_with_ordinals(begin, end, term_docids, + out, &chunk)); + } + if (candidates == nullptr || !chunk.docids.empty()) { + source->chunks.push_back(std::move(chunk)); + } + } + if (candidates == nullptr) { + *out = std::move(term_docids); + return Status::OK(); + } + if (source != nullptr) { + return Status::OK(); + } + *out = intersect_sorted(*candidates, term_docids); + return Status::OK(); +} + +Status run_docid_only_conjunction_impl(const LogicalIndexReader& idx, + const io::BatchRangeFetcher& round1, + const std::vector& plans, + const std::vector* initial_candidates, + std::vector* candidates, + std::vector* sources) { + if (sources != nullptr) { + sources->assign(plans.size(), DocidSource {}); + } + candidates->clear(); + if (plans.empty()) { + // No terms: the result is the initial candidate set verbatim (or empty). + if (initial_candidates != nullptr) { + *candidates = *initial_candidates; + } + return Status::OK(); + } + if (initial_candidates != nullptr && initial_candidates->empty()) { + return Status::OK(); + } + const std::vector order = ascending_df_order(plans); + for (size_t k = 0; k < order.size(); ++k) { + const size_t ti = order[k]; + std::vector next; + DocidSource* source = sources == nullptr ? nullptr : &(*sources)[ti]; + // k == 0 intersects against the (const) initial_candidates DIRECTLY, so + // the whole set is never copied once per plan -- the previous code seeded + // *candidates = *initial_candidates before the loop, which for a single + // plan (e.g. one phrase-prefix tail verified against the leading-term + // expected docids) was an O(|initial|) copy per call with no benefit. + // k > 0 chains on the previous term's already-whittled result. + const std::vector* input_candidates = k == 0 ? initial_candidates : candidates; + RETURN_IF_ERROR( + collect_docids_only(idx, round1, plans[ti], input_candidates, &next, source)); + if (source != nullptr && k + 1 == order.size()) { + source->docids_are_final_candidates = true; + } + *candidates = std::move(next); + if (candidates->empty()) { + return Status::OK(); + } + } + return Status::OK(); +} + +} // namespace + +Status resolve_query_term(const LogicalIndexReader& idx, std::string_view term, + ResolvedQueryTerm* resolved, bool* found) { + *found = false; + RETURN_IF_ERROR( + idx.lookup(term, found, &resolved->entry, &resolved->frq_base, &resolved->prx_base)); + return Status::OK(); +} + +Status resolve_query_terms_batch(const LogicalIndexReader& idx, + const std::vector& terms, + std::vector* resolved, + std::vector* found) { + DCHECK(std::ranges::is_sorted(terms)); + DCHECK(std::adjacent_find(terms.begin(), terms.end()) == terms.end()); + resolved->assign(terms.size(), ResolvedQueryTerm {}); + found->assign(terms.size(), 0); + std::vector lookup_results; + RETURN_IF_ERROR(idx.lookup_batch(terms, &lookup_results)); + for (size_t i = 0; i < terms.size(); ++i) { + (*found)[i] = lookup_results[i].found; + if (lookup_results[i].found) { + (*resolved)[i].entry = std::move(lookup_results[i].entry); + (*resolved)[i].frq_base = lookup_results[i].frq_base; + (*resolved)[i].prx_base = lookup_results[i].prx_base; + } + } + return Status::OK(); +} + +Status plan_terms(const LogicalIndexReader& idx, const std::vector& terms, + io::BatchRangeFetcher* fetcher, std::vector* plans, bool* all_present, + bool need_positions) { + *all_present = true; + plans->resize(terms.size()); + for (size_t i = 0; i < terms.size(); ++i) { + ResolvedQueryTerm resolved; + bool found = false; + RETURN_IF_ERROR(resolve_query_term(idx, terms[i], &resolved, &found)); + if (!found) { + *all_present = false; + return Status::OK(); + } + TermPlan& p = (*plans)[i]; + p.order = i; + p.entry = std::move(resolved.entry); + p.frq_base = resolved.frq_base; + p.prx_base = resolved.prx_base; + RETURN_IF_ERROR(configure_term_plan(idx, need_positions, fetcher, &p)); + } + return Status::OK(); +} + +Status plan_resolved_terms(const LogicalIndexReader& idx, + const std::vector& terms, + io::BatchRangeFetcher* fetcher, std::vector* plans, + bool need_positions) { + plans->resize(terms.size()); + for (size_t i = 0; i < terms.size(); ++i) { + TermPlan& p = (*plans)[i]; + p.order = i; + p.entry = terms[i].entry; + SNII_QUERY_COUNT(resolved_term_entry_copies); + p.frq_base = terms[i].frq_base; + p.prx_base = terms[i].prx_base; + RETURN_IF_ERROR(configure_term_plan(idx, need_positions, fetcher, &p)); + } + return Status::OK(); +} + +Status plan_resolved_terms(const LogicalIndexReader& idx, std::vector&& terms, + io::BatchRangeFetcher* fetcher, std::vector* plans, + bool need_positions) { + plans->resize(terms.size()); + for (size_t i = 0; i < terms.size(); ++i) { + TermPlan& p = (*plans)[i]; + p.order = i; +#ifdef BE_TEST + const bool has_frq_payload = !terms[i].entry.frq_bytes.empty(); + const bool has_prx_payload = !terms[i].entry.prx_bytes.empty(); + const uint8_t* const frq_payload = terms[i].entry.frq_bytes.data(); + const uint8_t* const prx_payload = terms[i].entry.prx_bytes.data(); +#endif + p.entry = std::move(terms[i].entry); + SNII_QUERY_COUNT(resolved_term_entry_moves); +#ifdef BE_TEST + SNII_QUERY_ADD( + resolved_term_payload_pointer_reuses, + static_cast(has_frq_payload && p.entry.frq_bytes.data() == frq_payload) + + static_cast(has_prx_payload && + p.entry.prx_bytes.data() == prx_payload)); +#endif + p.frq_base = terms[i].frq_base; + p.prx_base = terms[i].prx_base; + RETURN_IF_ERROR(configure_term_plan(idx, need_positions, fetcher, &p)); + } + return Status::OK(); +} + +Status open_preludes(const io::BatchRangeFetcher& fetcher, std::vector* plans, + bool need_positions) { + for (TermPlan& p : *plans) { + if (!p.windowed) continue; + RETURN_IF_ERROR(FrqPreludeReader::open(fetcher.get(p.prelude_handle), &p.prelude)); + if (need_positions && !p.prelude.has_prx()) { + return Status::Error( + "docid_conjunction: windowed prelude has no positions"); + } + } + return Status::OK(); +} + +Status inline_dd_region(const DictEntry& entry, Slice* out) { + if (entry.dd_meta.disk_len > entry.frq_bytes.size()) { + return Status::Error( + "docid_conjunction: inline dd region exceeds frq bytes"); + } + *out = Slice(entry.frq_bytes.data(), static_cast(entry.dd_meta.disk_len)); + return Status::OK(); +} + +Status build_docid_only_conjunction(const LogicalIndexReader& idx, + const io::BatchRangeFetcher& round1, + const std::vector& plans, + std::vector* candidates) { + return run_docid_only_conjunction_impl(idx, round1, plans, nullptr, candidates, nullptr); +} + +Status build_docid_only_conjunction(const LogicalIndexReader& idx, + const io::BatchRangeFetcher& round1, + const std::vector& plans, + std::vector* candidates, + std::vector* sources) { + return run_docid_only_conjunction_impl(idx, round1, plans, nullptr, candidates, sources); +} + +Status filter_docids_by_conjunction(const LogicalIndexReader& idx, + const io::BatchRangeFetcher& round1, + const std::vector& plans, + const std::vector& initial_candidates, + std::vector* candidates, + std::vector* sources) { + return run_docid_only_conjunction_impl(idx, round1, plans, &initial_candidates, candidates, + sources); +} + +} // namespace doris::snii::query::internal diff --git a/be/src/storage/index/snii/query/docid_posting_reader.cpp b/be/src/storage/index/snii/query/docid_posting_reader.cpp new file mode 100644 index 00000000000000..be58d24a88c869 --- /dev/null +++ b/be/src/storage/index/snii/query/docid_posting_reader.cpp @@ -0,0 +1,380 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/query/internal/docid_posting_reader.h" + +#include +#include + +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/format/dict_entry.h" +#include "storage/index/snii/format/frq_pod.h" +#include "storage/index/snii/format/frq_prelude.h" +#include "storage/index/snii/io/batch_range_fetcher.h" +#include "storage/index/snii/reader/windowed_posting.h" + +namespace doris::snii::query::internal { + +using format::DictEntry; +using format::DictEntryEnc; +using format::DictEntryKind; +using format::FrqPreludeReader; +using format::WindowMeta; +using reader::LogicalIndexReader; + +namespace { + +Status decode_flat_docs(const DictEntry& entry, Slice dd_region, std::vector* docids) { + return format::decode_dd_region(dd_region, entry.dd_meta, + /*win_base=*/0, docids); +} + +Status decode_inline_docs(const DictEntry& entry, std::vector* docids) { + if (entry.dd_meta.disk_len > entry.frq_bytes.size()) { + return Status::Error( + "docid_posting_reader: inline dd region exceeds frq bytes"); + } + return decode_flat_docs( + entry, Slice(entry.frq_bytes.data(), static_cast(entry.dd_meta.disk_len)), + docids); +} + +Status slim_docs_fetch_len(const DictEntry& entry, uint64_t win_len, uint64_t* out) { + if (entry.frq_docs_len > win_len) { + return Status::Error( + "docid_posting_reader: slim frq_docs_len exceeds frq window"); + } + *out = entry.frq_docs_len > 0 ? entry.frq_docs_len : win_len; + return Status::OK(); +} + +Status add_u64(uint64_t lhs, uint64_t rhs, const char* message, uint64_t* out) { + if (rhs > std::numeric_limits::max() - lhs) { + return Status::Error(message); + } + *out = lhs + rhs; + return Status::OK(); +} + +Status prelude_abs(const LogicalIndexReader& idx, const DictEntry& entry, uint64_t frq_base, + uint64_t* out) { + uint64_t with_base = 0; + RETURN_IF_ERROR(add_u64(idx.section_refs().posting_region.offset, frq_base, + "docid_posting_reader: prelude offset overflow", &with_base)); + return add_u64(with_base, entry.frq_off_delta, "docid_posting_reader: prelude offset overflow", + out); +} + +Status validate_windowed_docs_prefix(const DictEntry& entry) { + if (entry.prelude_len == 0) { + return Status::Error( + "docid_posting_reader: windowed entry has no prelude"); + } + if (entry.prelude_len > entry.frq_docs_len) { + return Status::Error( + "docid_posting_reader: prelude_len exceeds docs prefix"); + } + if (entry.frq_docs_len > entry.frq_len) { + return Status::Error( + "docid_posting_reader: docs prefix exceeds frq_len"); + } + return Status::OK(); +} + +struct FlatPlan { + size_t out_index = 0; + const DictEntry* entry = nullptr; + size_t handle = 0; +}; + +struct WindowPlan { + size_t out_index = 0; + const ResolvedDocidPosting* posting = nullptr; + size_t prefix_handle = 0; +}; + +Status plan_flat_docs(const LogicalIndexReader& idx, const ResolvedDocidPosting& posting, + io::BatchRangeFetcher* fetcher, FlatPlan* plan) { + uint64_t win_abs = 0; + uint64_t win_len = 0; + RETURN_IF_ERROR(idx.resolve_frq_window(posting.entry, posting.frq_base, &win_abs, &win_len)); + uint64_t docs_len = 0; + RETURN_IF_ERROR(slim_docs_fetch_len(posting.entry, win_len, &docs_len)); + plan->handle = fetcher->add(win_abs, docs_len); + return Status::OK(); +} + +Status plan_window_prefix(const LogicalIndexReader& idx, WindowPlan* plan, + io::BatchRangeFetcher* fetcher) { + const ResolvedDocidPosting& posting = *plan->posting; + RETURN_IF_ERROR(validate_windowed_docs_prefix(posting.entry)); + uint64_t abs = 0; + RETURN_IF_ERROR(prelude_abs(idx, posting.entry, posting.frq_base, &abs)); + plan->prefix_handle = fetcher->add(abs, posting.entry.frq_docs_len); + return Status::OK(); +} + +// Records a non-inline (flat or windowed) posting into the per-encoding plan lists, +// adding the windowed prefix range to the shared fetcher. Flat ranges are added in a +// later pass (after all preludes are registered). Shared by the batched and streamed +// docid readers so both fetch the whole OR in one round. `posting` must outlive the +// plans (its address is captured); callers pass an element of a stable vector. +Status plan_noninline_posting(const LogicalIndexReader& idx, const ResolvedDocidPosting& posting, + size_t out_index, io::BatchRangeFetcher* fetcher, + std::vector* flat_plans, + std::vector* window_plans) { + if (posting.entry.enc == DictEntryEnc::kWindowed) { + WindowPlan plan; + plan.out_index = out_index; + plan.posting = &posting; + RETURN_IF_ERROR(plan_window_prefix(idx, &plan, fetcher)); + window_plans->push_back(std::move(plan)); + return Status::OK(); + } + FlatPlan plan; + plan.out_index = out_index; + plan.entry = &posting.entry; + flat_plans->push_back(plan); + return Status::OK(); +} + +Status window_dd_slice(Slice dd_block, const WindowMeta& meta, Slice* out) { + if (meta.dd_off > dd_block.size() || meta.dd_disk_len > dd_block.size() - meta.dd_off) { + return Status::Error( + "docid_posting_reader: window dd range out of prefix"); + } + *out = dd_block.subslice(static_cast(meta.dd_off), + static_cast(meta.dd_disk_len)); + return Status::OK(); +} + +Status first_docid_in_window(const WindowMeta& meta, uint32_t window_ordinal, uint32_t* first) { + if (window_ordinal == 0) { + *first = 0; + return Status::OK(); + } + if (meta.win_base >= std::numeric_limits::max()) { + return Status::Error( + "docid_posting_reader: window base exceeds docid range"); + } + *first = static_cast(meta.win_base + 1); + if (*first > meta.last_docid) { + return Status::Error( + "docid_posting_reader: invalid window docid range"); + } + return Status::OK(); +} + +Status is_dense_full_window(const WindowMeta& meta, uint32_t window_ordinal, bool* full) { + uint32_t first = 0; + RETURN_IF_ERROR(first_docid_in_window(meta, window_ordinal, &first)); + const uint64_t width = static_cast(meta.last_docid) - first + 1; + *full = meta.doc_count == width; + return Status::OK(); +} + +Status decode_flat_plan(const io::BatchRangeFetcher& fetcher, const FlatPlan& plan, + std::vector* out) { + return decode_flat_docs(*plan.entry, fetcher.get(plan.handle), out); +} + +Status decode_window_prefix_plan(const io::BatchRangeFetcher& fetcher, const WindowPlan& plan, + DocIdSink* sink); + +Status decode_window_prefix_plan(const io::BatchRangeFetcher& fetcher, const WindowPlan& plan, + std::vector* out) { + VectorDocIdSink sink(*out); + return decode_window_prefix_plan(fetcher, plan, &sink); +} + +Status decode_window_prefix_plan(const io::BatchRangeFetcher& fetcher, const WindowPlan& plan, + DocIdSink* sink) { + const DictEntry& entry = plan.posting->entry; + const Slice prefix = fetcher.get(plan.prefix_handle); + if (entry.prelude_len > prefix.size()) { + return Status::Error( + "docid_posting_reader: short docs prefix"); + } + const size_t prelude_len = static_cast(entry.prelude_len); + FrqPreludeReader prelude; + RETURN_IF_ERROR(FrqPreludeReader::open(prefix.subslice(0, prelude_len), &prelude)); + const uint64_t dd_block_len = prelude.dd_block_len(); + if (dd_block_len > static_cast(std::numeric_limits::max()) - prelude_len) { + return Status::Error( + "docid_posting_reader: docs prefix length overflow"); + } + const size_t expected_prefix_len = prelude_len + static_cast(dd_block_len); + if (prefix.size() != expected_prefix_len) { + return Status::Error( + "docid_posting_reader: docs prefix length mismatch"); + } + const Slice dd_block = prefix.subslice(prelude_len, prefix.size() - prelude_len); + std::vector docs; + std::vector freqs; + std::vector> positions; + for (uint32_t w = 0; w < prelude.n_windows(); ++w) { + WindowMeta meta; + Slice dd_region; + RETURN_IF_ERROR(prelude.window(w, &meta)); + RETURN_IF_ERROR(window_dd_slice(dd_block, meta, &dd_region)); + bool dense_full = false; + RETURN_IF_ERROR(is_dense_full_window(meta, w, &dense_full)); + if (dense_full) { + uint32_t first = 0; + RETURN_IF_ERROR(first_docid_in_window(meta, w, &first)); + RETURN_IF_ERROR(sink->append_range(first, static_cast(meta.last_docid) + 1)); + continue; + } + docs.clear(); + freqs.clear(); + positions.clear(); + RETURN_IF_ERROR(reader::decode_window_slices( + meta, dd_region, Slice(), Slice(), /*want_positions=*/false, + /*want_freq=*/false, &docs, &freqs, &positions)); + RETURN_IF_ERROR(sink->append_sorted(docs)); + } + return Status::OK(); +} + +} // namespace + +Status read_docid_posting(const LogicalIndexReader& idx, const DictEntry& entry, uint64_t frq_base, + uint64_t prx_base, std::vector* docids) { + if (docids == nullptr) { + return Status::Error("docid_posting_reader: null out"); + } + docids->clear(); + VectorDocIdSink sink(*docids); + return read_docid_posting(idx, entry, frq_base, prx_base, &sink); +} + +Status read_docid_posting(const LogicalIndexReader& idx, const DictEntry& entry, uint64_t frq_base, + uint64_t prx_base, DocIdSink* sink) { + if (sink == nullptr) { + return Status::Error("docid_posting_reader: null sink"); + } + ResolvedDocidPosting posting {entry, frq_base, prx_base}; + if (posting.entry.kind == DictEntryKind::kInline) { + std::vector docs; + RETURN_IF_ERROR(decode_inline_docs(posting.entry, &docs)); + return sink->append_sorted(docs); + } + + io::BatchRangeFetcher docs_fetcher(idx.reader()); + if (posting.entry.enc == DictEntryEnc::kWindowed) { + WindowPlan plan; + plan.out_index = 0; + plan.posting = &posting; + RETURN_IF_ERROR(plan_window_prefix(idx, &plan, &docs_fetcher)); + if (docs_fetcher.pending() > 0) RETURN_IF_ERROR(docs_fetcher.fetch()); + return decode_window_prefix_plan(docs_fetcher, plan, sink); + } + + FlatPlan plan; + plan.out_index = 0; + plan.entry = &posting.entry; + RETURN_IF_ERROR(plan_flat_docs(idx, posting, &docs_fetcher, &plan)); + if (docs_fetcher.pending() > 0) RETURN_IF_ERROR(docs_fetcher.fetch()); + std::vector docs; + RETURN_IF_ERROR(decode_flat_plan(docs_fetcher, plan, &docs)); + return sink->append_sorted(docs); +} + +Status read_docid_postings_batched(const LogicalIndexReader& idx, + const std::vector& postings, + std::vector>* docids) { + if (docids == nullptr) { + return Status::Error( + "docid_posting_reader: null batched out"); + } + docids->clear(); + docids->resize(postings.size()); + + std::vector flat_plans; + std::vector window_plans; + io::BatchRangeFetcher docs_fetcher(idx.reader()); + + for (size_t i = 0; i < postings.size(); ++i) { + const ResolvedDocidPosting& posting = postings[i]; + if (posting.entry.kind == DictEntryKind::kInline) { + RETURN_IF_ERROR(decode_inline_docs(posting.entry, &(*docids)[i])); + continue; + } + RETURN_IF_ERROR( + plan_noninline_posting(idx, posting, i, &docs_fetcher, &flat_plans, &window_plans)); + } + + for (FlatPlan& plan : flat_plans) { + const ResolvedDocidPosting& posting = postings[plan.out_index]; + RETURN_IF_ERROR(plan_flat_docs(idx, posting, &docs_fetcher, &plan)); + } + if (docs_fetcher.pending() > 0) RETURN_IF_ERROR(docs_fetcher.fetch()); + + for (const FlatPlan& plan : flat_plans) { + RETURN_IF_ERROR(decode_flat_plan(docs_fetcher, plan, &(*docids)[plan.out_index])); + } + for (const WindowPlan& plan : window_plans) { + RETURN_IF_ERROR(decode_window_prefix_plan(docs_fetcher, plan, &(*docids)[plan.out_index])); + } + return Status::OK(); +} + +Status emit_docid_postings_streamed(const LogicalIndexReader& idx, + const std::vector& postings, + DocIdSink* sink) { + if (sink == nullptr) { + return Status::Error( + "docid_posting_reader: null streamed sink"); + } + + std::vector flat_plans; + std::vector window_plans; + io::BatchRangeFetcher docs_fetcher(idx.reader()); + // One scratch buffer reused across flat/inline postings: clear() keeps capacity, + // so at most one growth total instead of a fresh vector per posting. + std::vector scratch; + + for (size_t i = 0; i < postings.size(); ++i) { + const ResolvedDocidPosting& posting = postings[i]; + if (posting.entry.kind == DictEntryKind::kInline) { + scratch.clear(); + RETURN_IF_ERROR(decode_inline_docs(posting.entry, &scratch)); + RETURN_IF_ERROR(sink->append_sorted(scratch)); + continue; + } + RETURN_IF_ERROR( + plan_noninline_posting(idx, posting, i, &docs_fetcher, &flat_plans, &window_plans)); + } + + for (FlatPlan& plan : flat_plans) { + const ResolvedDocidPosting& posting = postings[plan.out_index]; + RETURN_IF_ERROR(plan_flat_docs(idx, posting, &docs_fetcher, &plan)); + } + if (docs_fetcher.pending() > 0) RETURN_IF_ERROR(docs_fetcher.fetch()); + + for (const FlatPlan& plan : flat_plans) { + scratch.clear(); + RETURN_IF_ERROR(decode_flat_plan(docs_fetcher, plan, &scratch)); + RETURN_IF_ERROR(sink->append_sorted(scratch)); + } + for (const WindowPlan& plan : window_plans) { + RETURN_IF_ERROR(decode_window_prefix_plan(docs_fetcher, plan, sink)); + } + return Status::OK(); +} + +} // namespace doris::snii::query::internal diff --git a/be/src/storage/index/snii/query/docid_set_ops.cpp b/be/src/storage/index/snii/query/docid_set_ops.cpp new file mode 100644 index 00000000000000..5f8931f4146e16 --- /dev/null +++ b/be/src/storage/index/snii/query/docid_set_ops.cpp @@ -0,0 +1,127 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/query/internal/docid_set_ops.h" + +#include +#include +#include +#include + +namespace doris::snii::query::internal { + +std::vector intersect_sorted(const std::vector& a, + const std::vector& b) { + std::vector out; + out.reserve(std::min(a.size(), b.size())); + std::set_intersection(a.begin(), a.end(), b.begin(), b.end(), std::back_inserter(out)); + return out; +} + +void union_sorted_into(std::vector* acc, const std::vector& next) { + std::vector merged; + merged.reserve(acc->size() + next.size()); + std::set_union(acc->begin(), acc->end(), next.begin(), next.end(), std::back_inserter(merged)); + *acc = std::move(merged); +} + +std::vector union_sorted_many(const std::vector>& lists, + size_t reserve_cap) { + constexpr size_t kLinearFanInMax = 8; + struct Cursor { + uint32_t docid = 0; + size_t list = 0; + size_t offset = 0; + }; + struct GreaterDocId { + bool operator()(const Cursor& a, const Cursor& b) const { return a.docid > b.docid; } + }; + + size_t non_empty = 0; + size_t total = 0; + std::priority_queue, GreaterDocId> heap; + for (size_t i = 0; i < lists.size(); ++i) { + if (lists[i].empty()) continue; + ++non_empty; + total += lists[i].size(); + heap.push(Cursor {lists[i][0], i, 0}); + } + // The union is at most `total` (exactly that for disjoint inputs); reserve to it + // so the output grows in a single allocation rather than O(log) geometric + // reallocations. Cap guards against over-reserving for heavily-overlapping inputs. + const size_t reserve_hint = std::min(total, reserve_cap); + if (non_empty == 0) return {}; + if (non_empty == 1) { + for (const std::vector& docs : lists) { + if (!docs.empty()) return docs; + } + } + + if (non_empty <= kLinearFanInMax) { + std::vector offsets(lists.size(), 0); + std::vector out; + out.reserve(reserve_hint); + bool has_last = false; + uint32_t last = 0; + for (;;) { + bool found = false; + uint32_t next = 0; + for (size_t i = 0; i < lists.size(); ++i) { + if (offsets[i] >= lists[i].size()) continue; + const uint32_t docid = lists[i][offsets[i]]; + if (!found || docid < next) { + found = true; + next = docid; + } + } + if (!found) break; + if (!has_last || next != last) { + out.push_back(next); + last = next; + has_last = true; + } + for (size_t i = 0; i < lists.size(); ++i) { + while (offsets[i] < lists[i].size() && lists[i][offsets[i]] == next) { + ++offsets[i]; + } + } + } + return out; + } + + std::vector out; + out.reserve(reserve_hint); + bool has_last = false; + uint32_t last = 0; + while (!heap.empty()) { + const Cursor cur = heap.top(); + heap.pop(); + if (!has_last || cur.docid != last) { + out.push_back(cur.docid); + last = cur.docid; + has_last = true; + } + const size_t next_offset = cur.offset + 1; + const std::vector& docs = lists[cur.list]; + if (next_offset < docs.size()) { + heap.push(Cursor {docs[next_offset], cur.list, next_offset}); + } + } + return out; +} + +} // namespace doris::snii::query::internal diff --git a/be/src/storage/index/snii/query/docid_sink.h b/be/src/storage/index/snii/query/docid_sink.h new file mode 100644 index 00000000000000..604d3dde8bdbb8 --- /dev/null +++ b/be/src/storage/index/snii/query/docid_sink.h @@ -0,0 +1,90 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include + +#include "common/status.h" + +namespace doris::snii::query { + +// Bulk docid handoff for query operators. Each span is sorted ascending; callers +// that need a single vector can use VectorDocIdSink. +class DocIdSink { +public: + virtual ~DocIdSink() = default; + virtual Status append_sorted(std::span docids) = 0; + virtual Status append_range(uint32_t first, uint64_t last_exclusive) = 0; + + // True iff the sink deduplicates and globally orders on its own (e.g. a Roaring + // bitmap via addMany/addRange). For such sinks a multi-term OR can stream each + // posting straight in -- skipping the per-term vector materialization plus the + // K-way merge accumulator. Sinks that hand back a single globally-sorted, + // deduplicated vector (VectorDocIdSink) keep the default false, so callers + // materialize + merge before appending. The gate must stay conservative: + // streaming several postings into a non-dedup sink would break that contract. + virtual bool dedups() const { return false; } +}; + +class VectorDocIdSink final : public DocIdSink { +public: + explicit VectorDocIdSink(std::vector& docids) : docids_(docids) {} + + Status append_sorted(std::span docids) override { + docids_.insert(docids_.end(), docids.begin(), docids.end()); + return Status::OK(); + } + + Status append_range(uint32_t first, uint64_t last_exclusive) override { + if (last_exclusive <= first) { + return Status::OK(); + } + if (last_exclusive > static_cast(std::numeric_limits::max()) + 1) { + return Status::Error( + "docid_sink: range exceeds uint32 docid space"); + } + const uint64_t count = last_exclusive - first; + if (count > static_cast(docids_.max_size() - docids_.size())) { + return Status::Error("docid_sink: range too large"); + } + // GEOMETRIC BULK reserve -- never an exact one: append_range can be + // called once per docid run for a query, and an exact + // reserve(size()+count) caps capacity at "just enough" so the next + // append reallocates + memcpys the whole accumulated vector -- + // quadratic total memcpy across runs (same anti-pattern as the writer's + // add_nulls). Doubling on overflow keeps the O(count) amortization AND + // makes one large range pay at most one reallocation. + const size_t need = docids_.size() + static_cast(count); + if (need > docids_.capacity()) { + docids_.reserve(std::max(need, docids_.capacity() * 2)); + } + for (uint64_t docid = first; docid < last_exclusive; ++docid) { + docids_.push_back(static_cast(docid)); + } + return Status::OK(); + } + +private: + std::vector& docids_; +}; + +} // namespace doris::snii::query diff --git a/be/src/storage/index/snii/query/docid_union.cpp b/be/src/storage/index/snii/query/docid_union.cpp new file mode 100644 index 00000000000000..f1296b18cd685e --- /dev/null +++ b/be/src/storage/index/snii/query/docid_union.cpp @@ -0,0 +1,58 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/query/internal/docid_union.h" + +#include + +#include "storage/index/snii/query/internal/docid_set_ops.h" + +namespace doris::snii::query::internal { + +Status build_docid_union(const reader::LogicalIndexReader& idx, + const std::vector& postings, + std::vector* out) { + if (out == nullptr) + return Status::Error("docid_union: null out"); + out->clear(); + if (postings.empty()) return Status::OK(); + + std::vector> docs_by_posting; + RETURN_IF_ERROR(read_docid_postings_batched(idx, postings, &docs_by_posting)); + *out = union_sorted_many(docs_by_posting); + return Status::OK(); +} + +Status emit_docid_union(const reader::LogicalIndexReader& idx, + const std::vector& postings, DocIdSink* sink) { + if (sink == nullptr) + return Status::Error("docid_union: null sink"); + if (postings.empty()) return Status::OK(); + // A dedup-capable sink (Roaring) orders + dedups across postings itself, so stream + // each posting straight in over a single shared fetch round -- no per-term vector + // or K-way merge accumulator. A plain (non-dedup) sink keeps the materialize+merge + // path so its single-span contract (globally sorted, deduplicated) holds. + if (sink->dedups()) { + return emit_docid_postings_streamed(idx, postings, sink); + } + std::vector acc; + RETURN_IF_ERROR(build_docid_union(idx, postings, &acc)); + if (acc.empty()) return Status::OK(); + return sink->append_sorted(acc); +} + +} // namespace doris::snii::query::internal diff --git a/be/src/storage/index/snii/query/internal/docid_conjunction.h b/be/src/storage/index/snii/query/internal/docid_conjunction.h new file mode 100644 index 00000000000000..ea63d9446a0071 --- /dev/null +++ b/be/src/storage/index/snii/query/internal/docid_conjunction.h @@ -0,0 +1,113 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/format/dict_entry.h" +#include "storage/index/snii/format/frq_prelude.h" +#include "storage/index/snii/io/batch_range_fetcher.h" +#include "storage/index/snii/reader/logical_index_reader.h" + +namespace doris::snii::query::internal { + +struct ResolvedQueryTerm { + format::DictEntry entry; + uint64_t frq_base = 0; + uint64_t prx_base = 0; +}; + +struct TermPlan { + format::DictEntry entry; + uint64_t frq_base = 0; + uint64_t prx_base = 0; + uint32_t df = 0; + size_t order = 0; + size_t frq_handle = 0; + size_t prx_handle = 0; + size_t prelude_handle = 0; + bool pod_ref = false; + bool windowed = false; + format::FrqPreludeReader prelude; +}; + +struct DocidChunk { + std::vector docids; + std::vector prx_doc_ordinals; + uint32_t prx_doc_count = 0; + bool windowed = false; + uint32_t window = 0; +}; + +struct DocidSource { + std::vector chunks; + bool docids_are_final_candidates = false; +}; + +Status resolve_query_term(const reader::LogicalIndexReader& idx, std::string_view term, + ResolvedQueryTerm* resolved, bool* found); + +// Resolves one sorted, duplicate-free term batch through bounded physical DICT +// reads. Results stay aligned with `terms`; absent terms have found[i]=0. +Status resolve_query_terms_batch(const reader::LogicalIndexReader& idx, + const std::vector& terms, + std::vector* resolved, + std::vector* found); + +Status plan_terms(const reader::LogicalIndexReader& idx, const std::vector& terms, + io::BatchRangeFetcher* fetcher, std::vector* plans, bool* all_present, + bool need_positions); + +Status plan_resolved_terms(const reader::LogicalIndexReader& idx, + const std::vector& terms, + io::BatchRangeFetcher* fetcher, std::vector* plans, + bool need_positions); + +Status plan_resolved_terms(const reader::LogicalIndexReader& idx, + std::vector&& terms, io::BatchRangeFetcher* fetcher, + std::vector* plans, bool need_positions); + +Status open_preludes(const io::BatchRangeFetcher& fetcher, std::vector* plans, + bool need_positions); + +Status inline_dd_region(const format::DictEntry& entry, Slice* out); + +Status build_docid_only_conjunction(const reader::LogicalIndexReader& idx, + const io::BatchRangeFetcher& round1, + const std::vector& plans, + std::vector* candidates); + +Status build_docid_only_conjunction(const reader::LogicalIndexReader& idx, + const io::BatchRangeFetcher& round1, + const std::vector& plans, + std::vector* candidates, + std::vector* sources); + +Status filter_docids_by_conjunction(const reader::LogicalIndexReader& idx, + const io::BatchRangeFetcher& round1, + const std::vector& plans, + const std::vector& initial_candidates, + std::vector* candidates, + std::vector* sources); + +} // namespace doris::snii::query::internal diff --git a/be/src/storage/index/snii/query/internal/docid_posting_reader.h b/be/src/storage/index/snii/query/internal/docid_posting_reader.h new file mode 100644 index 00000000000000..0230a1f526039a --- /dev/null +++ b/be/src/storage/index/snii/query/internal/docid_posting_reader.h @@ -0,0 +1,62 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include "common/status.h" +#include "storage/index/snii/format/dict_entry.h" +#include "storage/index/snii/query/docid_sink.h" +#include "storage/index/snii/reader/logical_index_reader.h" + +namespace doris::snii::query::internal { + +struct ResolvedDocidPosting { + format::DictEntry entry; + uint64_t frq_base = 0; + uint64_t prx_base = 0; +}; + +// Decodes the docid-only posting for a resolved term. The caller owns term +// lookup and can batch/plan lookups independently; this module owns only the +// three posting encodings (inline, slim pod_ref, windowed pod_ref). +Status read_docid_posting(const reader::LogicalIndexReader& idx, const format::DictEntry& entry, + uint64_t frq_base, uint64_t prx_base, std::vector* docids); + +Status read_docid_posting(const reader::LogicalIndexReader& idx, const format::DictEntry& entry, + uint64_t frq_base, uint64_t prx_base, query::DocIdSink* sink); + +// Batch counterpart for multi-term docid-only operators. Windowed terms share one +// prelude fetch round and one docid fetch round, so OR-style operators pay by +// stage rather than by term. +Status read_docid_postings_batched(const reader::LogicalIndexReader& idx, + const std::vector& postings, + std::vector>* docids); + +// Streaming counterpart of read_docid_postings_batched for a dedup-capable sink +// (DocIdSink::dedups()==true, e.g. a Roaring bitmap). Shares the exact same single +// docid fetch round, but decodes each posting straight into the sink -- dense-full +// windows via append_range (run-preserving), the rest via append_sorted from one +// reused scratch buffer -- so no per-term vector or K-way merge accumulator is +// materialized. The sink dedups/orders across postings. One I/O round is preserved. +Status emit_docid_postings_streamed(const reader::LogicalIndexReader& idx, + const std::vector& postings, + query::DocIdSink* sink); + +} // namespace doris::snii::query::internal diff --git a/be/src/storage/index/snii/query/internal/docid_set_ops.h b/be/src/storage/index/snii/query/internal/docid_set_ops.h new file mode 100644 index 00000000000000..f651e930a1cdb3 --- /dev/null +++ b/be/src/storage/index/snii/query/internal/docid_set_ops.h @@ -0,0 +1,39 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include + +namespace doris::snii::query::internal { + +std::vector intersect_sorted(const std::vector& a, + const std::vector& b); + +void union_sorted_into(std::vector* acc, const std::vector& next); + +// Sorted-deduplicated union of many sorted lists. The output is reserved by the +// summed input size (the union is at most the total of all inputs; for disjoint +// inputs it is exactly that), capped by `reserve_cap` so heavily-overlapping +// inputs (union << total) do not over-reserve. Default cap = no cap. +std::vector union_sorted_many(const std::vector>& lists, + size_t reserve_cap = std::numeric_limits::max()); + +} // namespace doris::snii::query::internal diff --git a/be/src/storage/index/snii/query/internal/docid_union.h b/be/src/storage/index/snii/query/internal/docid_union.h new file mode 100644 index 00000000000000..3243c082bbeec2 --- /dev/null +++ b/be/src/storage/index/snii/query/internal/docid_union.h @@ -0,0 +1,38 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include "common/status.h" +#include "storage/index/snii/query/docid_sink.h" +#include "storage/index/snii/query/internal/docid_posting_reader.h" +#include "storage/index/snii/reader/logical_index_reader.h" + +namespace doris::snii::query::internal { + +// Reads already-resolved docid postings in planned batches, merges them as a +// sorted deduplicated union, then emits one bulk span to the sink. +Status build_docid_union(const reader::LogicalIndexReader& idx, + const std::vector& postings, + std::vector* out); + +Status emit_docid_union(const reader::LogicalIndexReader& idx, + const std::vector& postings, DocIdSink* sink); + +} // namespace doris::snii::query::internal diff --git a/be/src/storage/index/snii/query/internal/phrase_query_split.h b/be/src/storage/index/snii/query/internal/phrase_query_split.h new file mode 100644 index 00000000000000..4aa5e279460cb2 --- /dev/null +++ b/be/src/storage/index/snii/query/internal/phrase_query_split.h @@ -0,0 +1,598 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/check.h" +#include "storage/index/inverted/common_grams/common_grams_key_codec.h" +#include "storage/index/inverted/common_grams/common_grams_query_cost.h" +#include "storage/index/inverted/common_grams/common_grams_segment_metadata.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/encoding/byte_source.h" +#include "storage/index/snii/format/dict_entry.h" +#include "storage/index/snii/format/frq_pod.h" +#include "storage/index/snii/format/frq_prelude.h" +#include "storage/index/snii/format/prx_pod.h" +#include "storage/index/snii/io/batch_range_fetcher.h" +#include "storage/index/snii/query/internal/docid_conjunction.h" +#include "storage/index/snii/query/internal/docid_posting_reader.h" +#include "storage/index/snii/query/internal/docid_set_ops.h" +#include "storage/index/snii/query/internal/docid_union.h" +#include "storage/index/snii/query/internal/plain_term_routing.h" +#include "storage/index/snii/query/internal/position_math.h" +#include "storage/index/snii/query/internal/query_test_counters.h" +#include "storage/index/snii/query/internal/resolved_phrase_plan.h" +#include "storage/index/snii/query/internal/term_expansion.h" +#include "storage/index/snii/query/phrase_prx_validation.h" +#include "storage/index/snii/query/phrase_query.h" +#include "storage/index/snii/query/phrase_verify_timer.h" +#include "storage/index/snii/query/prefix_query.h" +#include "storage/index/snii/query/term_query.h" +#include "storage/index/snii/reader/windowed_posting.h" +#include "util/debug_points.h" + +// phrase_query implements MATCH_PHRASE with WINDOW (sub-block) SKIPPING for +// high-df windowed terms (design spec section 6.2): +// 1. Resolve every term; reject if any is absent. +// 2. Batch-read each windowed term's prelude + each slim/inline term's full +// docid posting in one round; open the two-level prelude readers. +// 3. Pick the DRIVER = smallest-df term; materialize it fully -> the initial +// candidate docid set. +// 4. For every other term in ascending-df order, narrow the candidate set: +// - slim/inline: intersect with its (already decoded) full posting. +// - windowed: locate_window() the CURRENT candidates -> the SET of +// windows covering them; batch-fetch ONLY those windows' +// .frq docid regions; keep candidates present in some +// covering window. A high-df term thus reads +// O(candidates) windows instead of its whole O(df) +// posting. +// 5. Fetch PRX only for retained chunks and run the positional phrase check +// (term[0]@p, term[1]@p+1, ...) on the survivors. +// The result is identical to a full-read intersection; only the bytes read for +// high-df windowed terms shrink. +// +// Internal to the phrase-query implementation, which spans phrase_plan.cpp, +// phrase_position_source.cpp, phrase_emit.cpp, phrase_prefix_exec.cpp, +// phrase_planned_query.cpp and phrase_query.cpp. This header carries the types and +// functions those translation units share; nothing outside query/ may include it. +namespace doris::snii::query::phrase_impl { + +using query::internal::DocidChunk; +using query::internal::DocidSource; +using query::internal::ResolvedQueryTerm; +using query::internal::TermPlan; +using reader::LogicalIndexReader; + +bool apply_common_grams_plan_debug_override(bool cost_prefers_gram, + CommonGramsPlanDebugOverride debug_override); + +class CommonGramsPlanningTimer { +public: + explicit CommonGramsPlanningTimer(format::PhraseQueryExecutionStats* stats) : stats_(stats) { + if (stats_ != nullptr) { + start_ = std::chrono::steady_clock::now(); + } + } + + ~CommonGramsPlanningTimer() { finish(); } + + CommonGramsPlanningTimer(const CommonGramsPlanningTimer&) = delete; + CommonGramsPlanningTimer& operator=(const CommonGramsPlanningTimer&) = delete; + + void finish() { + if (finished_) { + return; + } + finished_ = true; + if (stats_ == nullptr) { + return; + } + const auto elapsed = std::chrono::duration_cast( + std::chrono::steady_clock::now() - start_) + .count(); + stats_->common_grams_planning_ns += static_cast(std::max(1, elapsed)); + } + +private: + format::PhraseQueryExecutionStats* stats_ = nullptr; + std::chrono::steady_clock::time_point start_; + bool finished_ = false; +}; + +size_t position_span_size(std::pair span); + +bool should_use_monotonic_position_scan(std::pair anchor_span, + size_t checked_span_size, uint32_t anchor_offset, + uint32_t checked_offset); + +struct ExpectedTailPositions { + uint32_t docid = 0; + uint32_t phrase_frequency = 0; + size_t positions_begin = 0; + size_t positions_end = 0; +}; + +static_assert(sizeof(ExpectedTailPositions) == 3 * sizeof(uint64_t)); + +struct ExpectedTailPositionSet { + std::vector docs; + std::vector positions; + std::vector position_matched; + size_t matched_count = 0; + + void clear() { + docs.clear(); + positions.clear(); + position_matched.clear(); + matched_count = 0; + } + + void reserve_docs(size_t count) { + docs.reserve(count); + positions.reserve(count); + } +}; + +// One decoded chunk of a term's posting: a windowed term's covering window, or +// a slim/inline term's single posting. `docids` is decoded in the conjunction +// phase (and reused by the streaming cursor -- the dd region is decoded exactly +// once); `prx` is the on-disk positions bytes, decoded lazily by the cursor +// (once per chunk) during phrase verification. + +struct PosChunk { + std::vector docids; // ascending, absolute + // Empty means the chunk keeps every PRX doc in on-disk order. Non-empty means + // `docids[i]` corresponds to on-disk local document ordinal + // `prx_doc_ordinals[i]`, allowing PRX decode to skip positions for docs that + // were removed by the docid-only conjunction. + std::vector prx_doc_ordinals; + uint32_t prx_doc_count = 0; + Slice prx; // .prx window bytes (reference fetcher/round1/entry) + bool windowed = false; + uint32_t window = 0; +}; + +// A term's retained posting as an ordered list of chunks (windowed: covering +// windows in docid order; slim/inline: one). The referenced prx bytes live in +// `round1` / the per-term fetchers kept alive in phrase_query::owners for the +// whole query, so the cursor can decode positions during verification. + +struct PosSource { + std::vector chunks; + format::PrxDecodeContext* observer_context = nullptr; +}; + +struct PhraseExecutionState { + std::vector srcs; + std::vector> owners; + std::vector candidates; +}; + +struct PhraseTermMapping { + std::vector unique_terms; + std::vector phrase_plan_index; +}; + +struct PhysicalPhrasePlan { + std::vector unique_terms; + std::vector phrase_plan_index; + std::vector position_offsets; + std::vector common_gram_clauses; +}; + +bool has_common_grams_capability( + const LogicalIndexReader& idx, + const segment_v2::inverted_index::CommonGramsQueryIdentity* query_identity); + +bool entry_has_positions(const format::DictEntry& entry); + +Status build_physical_phrase_plan_prefix(const LogicalIndexReader& idx, + const segment_v2::InvertedIndexQueryInfo& query_info, + size_t clause_count, bool allow_common_grams, + PhysicalPhrasePlan* plan, bool* all_representable); + +Status build_physical_phrase_plan(const LogicalIndexReader& idx, + const segment_v2::InvertedIndexQueryInfo& query_info, + bool allow_common_grams, PhysicalPhrasePlan* plan, + bool* all_representable); + +size_t resolved_batch_index(const std::vector& batch_terms, std::string_view term); + +bool all_plan_terms_present(const PhysicalPhrasePlan& plan, + const std::vector& batch_terms, + const std::vector& found); + +uint64_t plan_visible_posting_bytes(const format::DictEntry& entry, bool need_positions); + +segment_v2::inverted_index::CommonGramsPlanRawCost phrase_plan_raw_cost( + const PhysicalPhrasePlan& plan, const std::vector& batch_terms, + const std::vector& resolved, const std::vector& found, + bool need_positions); + +segment_v2::inverted_index::CommonGramsPlanRawCost alternative_clause_raw_cost( + const std::vector& terms, bool need_positions); + +void append_alternative_clause_cost( + const segment_v2::inverted_index::CommonGramsPlanRawCost& clause, + segment_v2::inverted_index::CommonGramsPlanRawCost* plan); + +segment_v2::inverted_index::CommonGramsPlanRawCost hybrid_verification_raw_cost( + const segment_v2::inverted_index::CommonGramsPlanRawCost& prefilter_cost, + const segment_v2::inverted_index::CommonGramsPlanRawCost& verification_cost); + +internal::ResolvedPhrasePlan materialize_resolved_phrase_plan( + const PhysicalPhrasePlan& plan, const std::vector& batch_terms, + std::vector* resolved); + +internal::ResolvedPhrasePlan copy_resolved_phrase_plan( + const PhysicalPhrasePlan& plan, const std::vector& batch_terms, + const std::vector& resolved); + +bool physical_phrase_plan_has_docs_only_term(const PhysicalPhrasePlan& plan, + const std::vector& batch_terms, + const std::vector& resolved); + +void append_physical_phrase_clause(const PhysicalPhrasePlan& source, size_t clause, + uint32_t position_offset, PhysicalPhrasePlan* target); + +struct HybridPositionedCover { + PhysicalPhrasePlan candidate_prefilter; + PhysicalPhrasePlan verification; +}; + +struct HybridExactPlanArtifact { + std::optional positioned_cover; +}; + +HybridExactPlanArtifact build_hybrid_exact_plan_artifact( + const PhysicalPhrasePlan& plain_plan, const PhysicalPhrasePlan& gram_plan, + const std::vector& batch_terms, + const std::vector& resolved); + +struct ResolvedMappedTail { + size_t batch_index = 0; + uint32_t expansion_ordinal = 0; +}; + +struct HybridPrefixMappedTails { + std::vector positioned_indices; + std::vector docs_only_indices; + std::vector docs_only_ordinals; +}; + +struct HybridPrefixPlanArtifact { + HybridPositionedCover plain_tail_cover; + HybridPrefixMappedTails mapped_tail_split; + std::optional positioned_tail_verification; + uint32_t plain_tail_position_offset = 0; + bool maps_tail_to_gram = false; +}; + +std::optional try_build_hybrid_prefix_plan_artifact( + const PhysicalPhrasePlan& plain_leading, const PhysicalPhrasePlan& gram_leading, + const std::vector& batch_terms, const std::vector& resolved, + const std::vector& mapped_tails, bool maps_tail_to_gram); + +struct HybridPrefixCandidateSet { + bool active = false; + std::vector docs; +}; + +Status build_hybrid_leading_candidates(const LogicalIndexReader& idx, + const PhysicalPhrasePlan& candidate_prefilter, + const std::vector& batch_terms, + const std::vector& resolved, + HybridPrefixCandidateSet* candidates); + +Status build_hybrid_docs_only_tail_candidates(const LogicalIndexReader& idx, + const std::vector& resolved, + const std::vector& gram_tail_indices, + const HybridPrefixCandidateSet& leading_candidates, + std::vector* candidates); + +Status execute_hybrid_exact_phrase_plan( + const LogicalIndexReader& idx, const PhysicalPhrasePlan& gram_plan, + const std::vector& batch_terms, const HybridExactPlanArtifact& artifact, + std::vector* resolved, std::vector* docids, + format::PrxDecodeContext* decode_context, bool* candidate_intersection_empty = nullptr); + +void append_resolved_phrase_clause(ResolvedQueryTerm term, uint32_t position_offset, + internal::ResolvedPhrasePlan* plan); + +internal::ResolvedPhrasePlan build_resolved_phrase_plan( + std::vector resolved_terms); + +Status resolve_and_execute_physical_phrase_plan(const LogicalIndexReader& idx, + const PhysicalPhrasePlan& plan, + std::vector* docids, + format::PrxDecodeContext* decode_context, + CommonGramsPlanningTimer& planning_timer); + +Status planned_exact_phrase_query_impl( + const LogicalIndexReader& idx, const segment_v2::InvertedIndexQueryInfo& plain_query_info, + const segment_v2::InvertedIndexQueryInfo& gram_query_info, + const segment_v2::inverted_index::CommonGramsQueryIdentity* common_grams_identity, + std::vector* docids, format::PrxDecodeContext* decode_context, + ExactPhrasePlanKind* selected_plan, + segment_v2::inverted_index::CommonGramsPlanCostModel cost_model, + CommonGramsPlanDebugOverride debug_override); + +PhraseTermMapping build_phrase_term_mapping(const std::vector& terms); + +Status build_position_sources_for_candidates( + const LogicalIndexReader& idx, const io::BatchRangeFetcher& round1, + const std::vector& plans, std::vector* doc_sources, + const std::vector& candidates, + std::vector>* owners, std::vector* srcs, + format::PrxDecodeContext* observer_context); + +class PosChunkDecoder { +public: + explicit PosChunkDecoder(format::PrxDecodeContext* observer_context = nullptr) + : observer_context_(observer_context) {} + + void set_decode_state(format::PrxDecodeContext* observer_context) { + observer_context_ = observer_context; + } + + void reset() { + chunk_ = nullptr; + offsets_by_prx_ordinal_ = false; + } + + Status decode(const PosChunk& chunk) { + chunk_ = &chunk; + ByteSource ps(chunk.prx); + const bool selected_all = chunk.prx_doc_ordinals.empty(); + const bool decode_full = selected_all || should_decode_full_prx_window(chunk); + offsets_by_prx_ordinal_ = decode_full && !selected_all; + return internal::decode_and_validate_prx_frame( + &ps, chunk.prx_doc_ordinals, decode_full, selected_all, chunk.prx_doc_count, + chunk.docids.size(), &pflat_, &poff_, observer_context_); + } + + Status positions(size_t doc_index, std::pair* out) const { + if (chunk_ == nullptr || doc_index >= chunk_->docids.size()) { + return Status::Error( + "phrase_query: decoded chunk doc index out of range"); + } + const size_t pos_index = + offsets_by_prx_ordinal_ ? chunk_->prx_doc_ordinals[doc_index] : doc_index; + if (pos_index + 1 >= poff_.size()) { + return Status::Error( + "phrase_query: prx ordinal offset out of range"); + } + const uint32_t begin = poff_[pos_index]; + const uint32_t end = poff_[pos_index + 1]; + if (begin == end) { + *out = {nullptr, nullptr}; + return Status::OK(); + } + if (end > pflat_.size()) { + return Status::Error( + "phrase_query: prx offset out of range"); + } + *out = {pflat_.data() + begin, pflat_.data() + end}; + return Status::OK(); + } + + inline __attribute__((always_inline)) std::pair + positions_unchecked(size_t doc_index) const { + const size_t pos_index = + offsets_by_prx_ordinal_ ? chunk_->prx_doc_ordinals[doc_index] : doc_index; + const uint32_t begin = poff_[pos_index]; + const uint32_t end = poff_[pos_index + 1]; + if (begin == end) { + return {nullptr, nullptr}; + } + return {pflat_.data() + begin, pflat_.data() + end}; + } + +private: + static bool should_decode_full_prx_window(const PosChunk& chunk) { + return chunk.prx_doc_count != 0 && + static_cast(chunk.prx_doc_ordinals.size()) * 2 >= chunk.prx_doc_count; + } + + const PosChunk* chunk_ = nullptr; + bool offsets_by_prx_ordinal_ = false; + std::vector pflat_; + std::vector poff_; + format::PrxDecodeContext* observer_context_ = nullptr; +}; + +// Streaming position cursor over one term's retained chunks. It advances ONLY +// forward (callers seek ascending candidate docids), decodes each chunk's +// docids once (reused from the conjunction phase) and each chunk's positions at +// most once (lazily, into a flat CSR whose capacity is retained across chunks). +// No per-doc allocation, no per-candidate docid binary search: positions are +// addressed by the doc's local index within its chunk. This is the read-side +// dual of the windowed posting layout -- the S3-native batch fetch already +// pulled every needed chunk into memory; the cursor is pure in-memory column +// iteration. + +class PostingCursor { +public: + void init(const PosSource* src) { + src_ = src; + ci_ = 0; + li_ = 0; + decoded_pos_chunk_ = kNoChunk; + decoder_.set_decode_state(src->observer_context); + decoder_.reset(); + } + + // Positions the cursor at `target` (guaranteed present: candidates are the + // intersection of exactly these chunks' docids). Monotonic forward advance. + Status seek(uint32_t target) { + while (ci_ < src_->chunks.size() && + (src_->chunks[ci_].docids.empty() || src_->chunks[ci_].docids.back() < target)) { + ++ci_; + li_ = 0; + } + if (ci_ >= src_->chunks.size()) { + return Status::Error( + "phrase_query: cursor exhausted before target docid"); + } + const std::vector& d = src_->chunks[ci_].docids; + while (li_ < d.size() && d[li_] < target) { + ++li_; + } + if (li_ >= d.size() || d[li_] != target) { + return Status::Error( + "phrase_query: candidate missing from posting chunk"); + } + return Status::OK(); + } + + // [begin,end) of the current doc's positions, decoding the current chunk's + // .prx exactly once (cached). Must follow a seek that landed on a real doc. + Status positions(std::pair* out) { + if (ci_ >= src_->chunks.size() || li_ >= src_->chunks[ci_].docids.size()) { + return Status::Error( + "phrase_query: cursor positions out of range"); + } + if (decoded_pos_chunk_ != ci_) { + RETURN_IF_ERROR(decoder_.decode(src_->chunks[ci_])); + decoded_pos_chunk_ = ci_; + } + return decoder_.positions(li_, out); + } + + Status next(uint32_t* docid, std::pair* out) { + while (ci_ < src_->chunks.size() && + (src_->chunks[ci_].docids.empty() || li_ >= src_->chunks[ci_].docids.size())) { + ++ci_; + li_ = 0; + } + if (ci_ >= src_->chunks.size()) { + return Status::Error( + "phrase_query: cursor exhausted before next docid"); + } + *docid = src_->chunks[ci_].docids[li_]; + RETURN_IF_ERROR(positions(out)); + ++li_; + return Status::OK(); + } + +private: + static constexpr size_t kNoChunk = static_cast(-1); + + const PosSource* src_ = nullptr; + size_t ci_ = 0; // current chunk + size_t li_ = 0; // current local doc index within the chunk + size_t decoded_pos_chunk_ = kNoChunk; // which chunk decoder_ currently holds + PosChunkDecoder decoder_; +}; + +enum class PhraseCandidateMetric : uint8_t { + kExact, + kPrefixLeading, +}; + +Status build_phrase_execution_state(const LogicalIndexReader& idx, io::BatchRangeFetcher* round1, + std::vector* plans, PhraseExecutionState* state, + const std::vector* candidate_prefilter, + format::PrxDecodeContext* observer_context, + PhraseCandidateMetric candidate_metric); + +Status execute_phrase_plans(const LogicalIndexReader& idx, io::BatchRangeFetcher* round1, + std::vector* plans, + const std::vector& phrase_plan_index, + std::vector* docids, + format::PrxDecodeContext* observer_context, + std::vector* matches, const PhraseQueryOptions& options); + +Status execute_resolved_phrase_prefix_terms( + const LogicalIndexReader& idx, internal::ResolvedPhrasePlan exact_plan, + std::vector tail_terms, uint32_t tail_position_offset, + std::vector* docids, format::PrxDecodeContext* decode_context, + std::vector* matches = nullptr, + const std::vector* candidate_prefilter = nullptr); + +Status execute_hybrid_phrase_prefix_plan( + const LogicalIndexReader& idx, const HybridPrefixPlanArtifact& artifact, + const std::vector& batch_terms, const std::vector& resolved, + const std::vector& plain_tail_terms, std::vector* docids, + format::PrxDecodeContext* decode_context, CommonGramsPlanningTimer& planning_timer, + bool* candidate_intersection_empty); + +struct HybridPrefixCostEstimate { + segment_v2::inverted_index::CommonGramsPlanRawCost raw_cost; + uint64_t estimated_cost = 0; +}; + +HybridPrefixCostEstimate estimate_hybrid_prefix_plan_cost( + const HybridPrefixPlanArtifact& artifact, const std::vector& batch_terms, + const std::vector& resolved, const std::vector& found, + const std::vector& plain_tail_terms, uint32_t position_verify_factor); + +Status phrase_query_impl(const LogicalIndexReader& idx, const std::vector& terms, + std::vector* const docids, + format::PrxDecodeContext* decode_context, + std::vector* matches, const PhraseQueryOptions& options); + +Status phrase_prefix_query_impl(const LogicalIndexReader& idx, + const std::vector& terms, + std::vector* const docids, int32_t max_expansions, + format::PrxDecodeContext* decode_context, + CommonGramsPlanningTimer* planning_timer, + std::vector* matches = nullptr); + +Status planned_phrase_prefix_query_impl( + const LogicalIndexReader& idx, const segment_v2::InvertedIndexQueryInfo& plain_query_info, + const segment_v2::InvertedIndexQueryInfo& gram_query_info, + const segment_v2::inverted_index::CommonGramsQueryIdentity* common_grams_identity, + std::vector* docids, int32_t max_expansions, + format::PrxDecodeContext* decode_context, PhrasePrefixPlanKind* selected_plan, + segment_v2::inverted_index::CommonGramsPlanCostModel cost_model, + CommonGramsPlanDebugOverride debug_override); + +template +segment_v2::inverted_index::CommonGramsPlanRawCost alternative_clause_raw_cost( + const std::vector& terms, const std::vector& indices, + bool need_positions) { + segment_v2::inverted_index::CommonGramsPlanRawCost cost; + unsigned __int128 posting_bytes = 0; + unsigned __int128 candidate_df = 0; + for (size_t index : indices) { + DORIS_CHECK_LT(index, terms.size()); + posting_bytes += plan_visible_posting_bytes(terms[index].entry, need_positions); + candidate_df += terms[index].entry.df; + } + cost.posting_bytes_or_df_sum = posting_bytes > std::numeric_limits::max() + ? std::numeric_limits::max() + : static_cast(posting_bytes); + cost.estimated_candidate_df = candidate_df > std::numeric_limits::max() + ? std::numeric_limits::max() + : static_cast(candidate_df); + cost.clause_count = 1; + return cost; +} + +} // namespace doris::snii::query::phrase_impl diff --git a/be/src/storage/index/snii/query/internal/plain_term_routing.h b/be/src/storage/index/snii/query/internal/plain_term_routing.h new file mode 100644 index 00000000000000..787d237ddaf933 --- /dev/null +++ b/be/src/storage/index/snii/query/internal/plain_term_routing.h @@ -0,0 +1,194 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/inverted/common_grams/common_grams_key_codec.h" +#include "storage/index/inverted/query/query_info.h" +#include "storage/index/snii/reader/logical_index_reader.h" + +namespace doris::snii::query::internal { + +inline segment_v2::inverted_index::PlainTermKeyVersion plain_term_key_version( + const reader::LogicalIndexReader& idx) { + const auto* metadata = idx.common_grams_metadata(); + return metadata == nullptr ? segment_v2::inverted_index::PlainTermKeyVersion::kLegacyRaw + : metadata->plain_term_key_version; +} + +inline Status route_plain_query_term_view(const reader::LogicalIndexReader& idx, + std::string_view logical_term, std::string* scratch, + std::string_view* physical_term, bool* representable) { + DORIS_CHECK(scratch != nullptr); + DORIS_CHECK(physical_term != nullptr); + DORIS_CHECK(representable != nullptr); + scratch->clear(); + *physical_term = {}; + *representable = false; + + const auto version = plain_term_key_version(idx); + if (version == segment_v2::inverted_index::PlainTermKeyVersion::kLegacyRaw && + segment_v2::inverted_index::legacy_raw_exact_requires_bypass(logical_term)) { + return Status::Error( + "SNII legacy raw term overlaps an internal term namespace"); + } + auto encoded = + segment_v2::inverted_index::try_encode_plain_term_view(logical_term, version, scratch); + if (!encoded.has_value()) { + return std::move(encoded.error()); + } + if (encoded->has_value()) { + *physical_term = **encoded; + *representable = true; + } + return Status::OK(); +} + +inline Status route_plain_query_term(const reader::LogicalIndexReader& idx, + std::string_view logical_term, std::string* physical_term, + bool* representable) { + DORIS_CHECK(physical_term != nullptr); + std::string scratch; + std::string_view physical_term_view; + RETURN_IF_ERROR(route_plain_query_term_view(idx, logical_term, &scratch, &physical_term_view, + representable)); + physical_term->assign(physical_term_view); + return Status::OK(); +} + +inline Status route_query_term_view(const reader::LogicalIndexReader& idx, + const segment_v2::TermInfo& term_info, std::string* scratch, + std::string_view* physical_term, bool* representable) { + DORIS_CHECK(term_info.is_single_term()); + if (term_info.key_kind == segment_v2::TermKeyKind::kCommonGram) { + return Status::Error( + "CommonGrams query plan requires segment capability validation"); + } + return route_plain_query_term_view(idx, term_info.get_single_term(), scratch, physical_term, + representable); +} + +inline Status route_query_term(const reader::LogicalIndexReader& idx, + const segment_v2::TermInfo& term_info, std::string* physical_term, + bool* representable) { + DORIS_CHECK(term_info.is_single_term()); + if (term_info.key_kind == segment_v2::TermKeyKind::kCommonGram) { + return Status::Error( + "CommonGrams query plan requires segment capability validation"); + } + return route_plain_query_term(idx, term_info.get_single_term(), physical_term, representable); +} + +inline Status route_query_terms(const reader::LogicalIndexReader& idx, + const segment_v2::InvertedIndexQueryInfo& query_info, + std::vector* routed_terms, bool* all_representable) { + DORIS_CHECK(routed_terms != nullptr); + DORIS_CHECK(all_representable != nullptr); + DORIS_CHECK(routed_terms->size() == query_info.term_infos.size()); + *all_representable = true; + const auto version = plain_term_key_version(idx); + size_t output_index = 0; + std::string scratch; + for (size_t i = 0; i < query_info.term_infos.size(); ++i) { + const auto& term_info = query_info.term_infos[i]; + DORIS_CHECK(term_info.is_single_term()); + if (term_info.key_kind == segment_v2::TermKeyKind::kCommonGram) { + return Status::Error( + "CommonGrams query plan requires segment capability validation"); + } + const auto logical_term = std::string_view((*routed_terms)[i]); + if (version == segment_v2::inverted_index::PlainTermKeyVersion::kLegacyRaw && + segment_v2::inverted_index::legacy_raw_exact_requires_bypass(logical_term)) { + return Status::Error( + "SNII legacy raw term overlaps an internal term namespace"); + } + auto physical_term = segment_v2::inverted_index::try_encode_plain_term_view( + logical_term, version, &scratch); + if (!physical_term.has_value()) { + return std::move(physical_term.error()); + } + if (!physical_term->has_value()) { + *all_representable = false; + continue; + } + if (output_index != i) { + if (scratch.empty()) { + (*routed_terms)[output_index] = std::move((*routed_terms)[i]); + } else { + (*routed_terms)[output_index] = std::move(scratch); + } + } else if (!scratch.empty()) { + (*routed_terms)[i] = std::move(scratch); + } + ++output_index; + } + routed_terms->resize(output_index); + return Status::OK(); +} + +inline Status route_plain_query_terms(const reader::LogicalIndexReader& idx, + const std::vector& logical_terms, + std::vector* physical_terms, + bool* all_representable) { + DORIS_CHECK(physical_terms != nullptr); + DORIS_CHECK(all_representable != nullptr); + physical_terms->clear(); + physical_terms->reserve(logical_terms.size()); + *all_representable = true; + for (const std::string& logical_term : logical_terms) { + std::string physical_term; + bool representable = false; + RETURN_IF_ERROR(route_plain_query_term(idx, logical_term, &physical_term, &representable)); + if (representable) { + physical_terms->push_back(std::move(physical_term)); + } else { + *all_representable = false; + } + } + return Status::OK(); +} + +inline Status route_plain_enumeration_prefix(const reader::LogicalIndexReader& idx, + std::string_view logical_prefix, + std::string* physical_prefix, bool* representable) { + DORIS_CHECK(physical_prefix != nullptr); + DORIS_CHECK(representable != nullptr); + physical_prefix->clear(); + *representable = false; + + const auto version = plain_term_key_version(idx); + if (version == segment_v2::inverted_index::PlainTermKeyVersion::kLegacyRaw && + segment_v2::inverted_index::legacy_raw_prefix_requires_bypass(logical_prefix)) { + return Status::Error( + "SNII legacy raw expansion overlaps an internal term namespace"); + } + auto encoded = segment_v2::inverted_index::try_encode_plain_term(logical_prefix, version, + physical_prefix); + if (!encoded.has_value()) { + return std::move(encoded.error()); + } + *representable = encoded.value(); + return Status::OK(); +} + +} // namespace doris::snii::query::internal diff --git a/be/src/storage/index/snii/query/internal/position_math.h b/be/src/storage/index/snii/query/internal/position_math.h new file mode 100644 index 00000000000000..6db2a0ee4b599b --- /dev/null +++ b/be/src/storage/index/snii/query/internal/position_math.h @@ -0,0 +1,47 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include + +namespace doris::snii::query::internal { + +inline bool build_position_offsets(size_t count, std::vector* out) { + if (count >= std::numeric_limits::max()) { + return false; + } + out->clear(); + out->reserve(count); + uint32_t offset = 0; + while (out->size() < count) { + out->push_back(offset); + ++offset; + } + return true; +} + +inline bool add_position_offset(uint32_t start, uint32_t offset, uint32_t* out) { + if (start > std::numeric_limits::max() - offset) return false; + *out = start + offset; + return true; +} + +} // namespace doris::snii::query::internal diff --git a/be/src/storage/index/snii/query/internal/query_test_counters.h b/be/src/storage/index/snii/query/internal/query_test_counters.h new file mode 100644 index 00000000000000..5e6b8acfd84e5d --- /dev/null +++ b/be/src/storage/index/snii/query/internal/query_test_counters.h @@ -0,0 +1,110 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +// Deterministic op-count seam for the phrase-query hot path (T24/G01). The +// counters let the phrase UTs assert routing/complexity directly: +// - expected_docids_build : how many times the multi-tail phrase-prefix branch +// materializes the expected-docid vector for a query. +// Hoisted out of the per-tail loop, so == 1 per query +// (was == tail_hits when rebuilt inside the loop). +// - anchor_iterations : total sparsest-anchor outer-enumeration size summed +// over candidate docs (== docs x min_span). Anchoring +// on the shortest per-doc span instead of the hardcoded +// phrase-position-0 span shrinks this when the leading +// exact term is high-frequency. +// - monotonic_position_scans +// : number of non-anchor spans whose phrase-position +// lookup uses a forward-only scan instead of one +// binary search per anchor position. +// - prefix_expected_doc_visits +// : total expected-doc iterations performed by the +// multi-tail phrase-prefix verification and result +// materialization path. +// - count_fastpath_hits : count-only (G02) answers produced from dict-entry +// df alone for a single term, with NO posting decode +// (count_query.cpp). +// - resolved_term_entry_copies / moves +// : DictEntry ownership transfers performed by the two +// plan_resolved_terms overloads. +// - resolved_term_payload_pointer_reuses +// : non-empty inline FRQ/PRX vectors whose data pointer +// survives an entry move into TermPlan. +// - phrase_position_epoch_cache_hits / misses +// : same-document PhrasePositionLoader lookups served +// from the plan span cache vs loaded from its cursor. +// +// The seam is active only under SNII_QUERY_TEST_COUNTERS, which is auto-enabled by +// the library-wide BE_TEST define (be/CMakeLists.txt `if (MAKE_TEST)`) used to +// build doris_be_test. Because BE_TEST is applied to the whole BE tree, both the +// phrase_query.cpp increments AND the test translation unit that reads them observe +// the SAME process-wide singleton (the inline function below has one instance +// across every including TU). In a release build BE_TEST is undefined, the struct +// and singleton do not exist, and SNII_QUERY_COUNT/SNII_QUERY_ADD expand to +// ((void)0): zero overhead and NO global mutable state on the production query +// path. +// +// CONCURRENCY: the singleton is intentionally unsynchronized. It is a +// single-threaded, test-only seam -- one phrase query at a time -- and is never +// touched on the production path. Do NOT read or write it from concurrent tests. +// Reset it between test cases with `query_test_counters() = {}`. +#if defined(BE_TEST) && !defined(SNII_QUERY_TEST_COUNTERS) +#define SNII_QUERY_TEST_COUNTERS +#endif + +#ifdef SNII_QUERY_TEST_COUNTERS + +namespace doris::snii::query::internal { + +struct QueryTestCounters { + uint64_t expected_docids_build = 0; + uint64_t anchor_iterations = 0; + uint64_t monotonic_position_scans = 0; + uint64_t prefix_expected_doc_visits = 0; + uint64_t count_fastpath_hits = 0; + uint64_t resolved_term_entry_copies = 0; + uint64_t resolved_term_entry_moves = 0; + uint64_t resolved_term_payload_pointer_reuses = 0; + uint64_t phrase_position_epoch_cache_hits = 0; + uint64_t phrase_position_epoch_cache_misses = 0; +}; + +// `inline` gives a single shared instance across all TUs that include this header +// (phrase_query.cpp and the test), so counter increments made in the library are +// visible to the test that reads them. +inline QueryTestCounters& query_test_counters() { + static QueryTestCounters counters; + return counters; +} + +} // namespace doris::snii::query::internal + +// NOLINTBEGIN(clang-diagnostic-unused-macros): expanded by phrase_query.cpp, not by this header's TU +#define SNII_QUERY_COUNT(field) (++::doris::snii::query::internal::query_test_counters().field) +#define SNII_QUERY_ADD(field, n) \ + (::doris::snii::query::internal::query_test_counters().field += (n)) +// NOLINTEND(clang-diagnostic-unused-macros) + +#else + +#define SNII_QUERY_COUNT(field) ((void)0) +#define SNII_QUERY_ADD(field, n) ((void)0) + +#endif // SNII_QUERY_TEST_COUNTERS diff --git a/be/src/storage/index/snii/query/internal/regex_prefix.h b/be/src/storage/index/snii/query/internal/regex_prefix.h new file mode 100644 index 00000000000000..304ab6161b2ddc --- /dev/null +++ b/be/src/storage/index/snii/query/internal/regex_prefix.h @@ -0,0 +1,37 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include +#include + +namespace doris::snii::query::internal { + +// Computes the dictionary-enumeration prefix used to narrow regexp term +// expansion. For left-anchored ("^") patterns it derives a tight common prefix +// from RE2::PossibleMatchRange (e.g. "^(order)" -> "order", where a naive literal +// scan stops at '(' and yields ""); if RE2 cannot compile the pattern, it falls +// back to a conservative literal-prefix scan. Unanchored patterns return no +// prefix because Hyperscan may match after the beginning of a dictionary term. +// The returned prefix only bounds how many terms visit_prefix_terms enumerates; +// final term acceptance is always decided by Hyperscan. +std::string regex_enum_prefix(std::string_view pattern, const re2::RE2& re); + +} // namespace doris::snii::query::internal diff --git a/be/src/storage/index/snii/query/internal/resolved_phrase_plan.h b/be/src/storage/index/snii/query/internal/resolved_phrase_plan.h new file mode 100644 index 00000000000000..9bfd439753fb5b --- /dev/null +++ b/be/src/storage/index/snii/query/internal/resolved_phrase_plan.h @@ -0,0 +1,76 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/format/prx_decode_stats.h" +#include "storage/index/snii/query/internal/docid_conjunction.h" +#include "storage/index/snii/reader/logical_index_reader.h" + +namespace doris::snii::query { +struct PhraseMatch; +} + +namespace doris::snii::query::internal { + +struct ResolvedPhrasePlan { + std::vector unique_terms; + std::vector phrase_plan_index; + std::vector position_offsets; + + [[nodiscard]] bool is_valid() const { + if (phrase_plan_index.size() != position_offsets.size() || + unique_terms.empty() != phrase_plan_index.empty()) { + return false; + } + if (phrase_plan_index.empty()) { + return true; + } + + std::vector referenced(unique_terms.size(), 0); + for (size_t i = 0; i < phrase_plan_index.size(); ++i) { + if ((i == 0 && position_offsets[i] != 0) || + (i != 0 && position_offsets[i - 1] >= position_offsets[i])) { + return false; + } + const size_t plan_index = phrase_plan_index[i]; + if (plan_index >= unique_terms.size()) { + return false; + } + referenced[plan_index] = 1; + } + return std::ranges::all_of(referenced, [](uint8_t used) { return used != 0; }); + } +}; + +// Executes an already-resolved exact phrase plan. Planning policy and term-key +// semantics stay above this boundary; the executor treats terms as opaque and +// consumes the resolved entries so inline posting payloads can move into the +// execution plans without another allocation/copy. +Status execute_resolved_phrase_plan(const reader::LogicalIndexReader& idx, + ResolvedPhrasePlan&& plan, std::vector* docids, + format::PrxDecodeContext* observer_context = nullptr, + std::vector* matches = nullptr, + const std::vector* candidate_prefilter = nullptr); + +} // namespace doris::snii::query::internal diff --git a/be/src/storage/index/snii/query/internal/sloppy_phrase_matcher.h b/be/src/storage/index/snii/query/internal/sloppy_phrase_matcher.h new file mode 100644 index 00000000000000..b96f3ae00eb43d --- /dev/null +++ b/be/src/storage/index/snii/query/internal/sloppy_phrase_matcher.h @@ -0,0 +1,76 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include + +namespace doris::snii::query::internal { + +using PhrasePositionSpan = std::pair; + +// Matches one candidate document at a time against already-decoded position +// spans. Query-shape storage and all scratch buffers are allocated once and +// reused across documents. +class SloppyPhraseMatcher { +public: + SloppyPhraseMatcher(std::span phrase_plan_index, + std::span position_offsets, uint32_t slop, bool ordered); + + // Returns 1 for the first match when frequencies are not requested. For + // scoring, returns the V3 sloppy frequency: sum(1 / (1 + match_width)). + float match(std::span positions, bool collect_frequency); + +private: + struct Clause { + PhrasePositionSpan positions; + const uint32_t* next = nullptr; + uint32_t raw_position = 0; + int64_t adjusted_position = 0; + bool has_position = false; + }; + + bool initialize_unordered(std::span positions); + bool advance_clause(size_t clause, bool update_end); + bool advance_repeat_collisions(size_t clause); + size_t collision(size_t clause) const; + bool clause_less(size_t left, size_t right) const; + bool clause_greater(size_t left, size_t right) const; + void rebuild_heap(); + size_t pop_heap(); + void push_heap(size_t clause); + bool next_unordered_match(uint64_t* match_width); + float match_unordered(std::span positions, bool collect_frequency); + float match_ordered(std::span positions, bool collect_frequency); + bool advance_ordered_to(size_t clause, int64_t target); + + std::vector phrase_plan_index_; + std::vector position_offsets_; + uint32_t slop_ = 0; + bool ordered_ = false; + bool has_repeats_ = false; + bool positioned_ = false; + int64_t end_ = 0; + std::vector clauses_; + std::vector heap_; +}; + +} // namespace doris::snii::query::internal diff --git a/be/src/storage/index/snii/query/internal/term_expansion.h b/be/src/storage/index/snii/query/internal/term_expansion.h new file mode 100644 index 00000000000000..6da0b4c3f8ed3f --- /dev/null +++ b/be/src/storage/index/snii/query/internal/term_expansion.h @@ -0,0 +1,47 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/query/docid_sink.h" +#include "storage/index/snii/reader/logical_index_reader.h" + +namespace doris::snii::query::internal { + +using TermMatcher = std::function; + +// Enumerates logical plain terms while retaining each matching physical +// DictEntry for direct posting resolution. PrefixHit::term is decoded logical +// text; PrefixHit::entry.term remains the physical dictionary key. +Status visit_expanded_plain_terms(const reader::LogicalIndexReader& idx, + std::string_view enum_prefix, const TermMatcher& matches, + const reader::LogicalIndexReader::PrefixHitVisitor& visitor, + int32_t max_expansions = 0); + +// Enumerates dictionary terms from `enum_prefix`, filters them with `matches`, +// and emits the sorted docid union for matching entries. PrefixHit carries the +// DictEntry and block bases, so callers avoid a second lookup per expanded term. +Status emit_expanded_docid_union(const reader::LogicalIndexReader& idx, + std::string_view enum_prefix, const TermMatcher& matches, + DocIdSink* const sink, int32_t max_expansions = 0); + +} // namespace doris::snii::query::internal diff --git a/be/src/storage/index/snii/query/internal/wildcard_matcher.h b/be/src/storage/index/snii/query/internal/wildcard_matcher.h new file mode 100644 index 00000000000000..78eec25097a997 --- /dev/null +++ b/be/src/storage/index/snii/query/internal/wildcard_matcher.h @@ -0,0 +1,81 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace doris::snii::query::internal { + +// Glob matcher with reusable scratch. '*' matches >=0 bytes, '?' matches exactly +// one byte, every other byte is literal; matching is anchored at both ends (full +// match). The matching result is bit-for-bit identical to the former per-call DP +// in wildcard_query.cpp: the only change is that the two DP rows are constructed +// once and reused (assign(), never reallocated once capacity is large enough) +// across every term in a single expansion. A whole-dictionary scan therefore +// performs O(1) heap allocations for scratch instead of O(2N) -- two small +// std::vector constructions per visited term. +// +// The allocator is templated only so deterministic allocation-counting tests can +// inject a CountingAllocator; production constructs WildcardMatcher<> (default +// std::allocator). The matcher is request-scoped (a stack local of the calling +// wildcard_query frame), holds no shared mutable state, and is not thread-safe by +// design: each query owns its own instance. +template > +class WildcardMatcher { +public: + explicit WildcardMatcher(std::string_view pattern) : pattern_(pattern) {} + + bool operator()(std::string_view text) { + const size_t n = text.size() + 1; + prev_.assign(n, 0); // reuses the buffer; no realloc once capacity >= n + curr_.assign(n, 0); + prev_[0] = 1; + for (char p : pattern_) { + std::fill(curr_.begin(), curr_.end(), 0); + if (p == '*') { + curr_[0] = prev_[0]; + for (size_t i = 1; i < n; ++i) { + curr_[i] = prev_[i] || curr_[i - 1]; + } + } else { + for (size_t i = 1; i < n; ++i) { + curr_[i] = prev_[i - 1] && (p == '?' || p == text[i - 1]); + } + } + prev_.swap(curr_); + } + return prev_[text.size()] != 0; + } + + // Test-only debug accessor: the production path never depends on it. Reports + // the larger of the two scratch-row capacities so perf tests can assert the + // buffer stops reallocating after warmup. + size_t scratch_capacity() const { return std::max(prev_.capacity(), curr_.capacity()); } + +private: + std::string_view pattern_; + std::vector prev_; + std::vector curr_; +}; + +} // namespace doris::snii::query::internal diff --git a/be/src/storage/index/snii/query/phrase_cost.cpp b/be/src/storage/index/snii/query/phrase_cost.cpp new file mode 100644 index 00000000000000..fec24172cc5b2e --- /dev/null +++ b/be/src/storage/index/snii/query/phrase_cost.cpp @@ -0,0 +1,276 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/check.h" +#include "storage/index/inverted/common_grams/common_grams_key_codec.h" +#include "storage/index/inverted/common_grams/common_grams_query_cost.h" +#include "storage/index/inverted/common_grams/common_grams_segment_metadata.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/encoding/byte_source.h" +#include "storage/index/snii/format/dict_entry.h" +#include "storage/index/snii/format/frq_pod.h" +#include "storage/index/snii/format/frq_prelude.h" +#include "storage/index/snii/format/prx_pod.h" +#include "storage/index/snii/io/batch_range_fetcher.h" +#include "storage/index/snii/query/internal/docid_conjunction.h" +#include "storage/index/snii/query/internal/docid_posting_reader.h" +#include "storage/index/snii/query/internal/docid_set_ops.h" +#include "storage/index/snii/query/internal/docid_union.h" +#include "storage/index/snii/query/internal/phrase_query_split.h" +#include "storage/index/snii/query/internal/plain_term_routing.h" +#include "storage/index/snii/query/internal/position_math.h" +#include "storage/index/snii/query/internal/query_test_counters.h" +#include "storage/index/snii/query/internal/resolved_phrase_plan.h" +#include "storage/index/snii/query/internal/term_expansion.h" +#include "storage/index/snii/query/phrase_prx_validation.h" +#include "storage/index/snii/query/phrase_query.h" +#include "storage/index/snii/query/phrase_verify_timer.h" +#include "storage/index/snii/query/prefix_query.h" +#include "storage/index/snii/query/term_query.h" +#include "storage/index/snii/reader/windowed_posting.h" +#include "util/debug_points.h" + +namespace doris::snii::query::phrase_impl { + +using query::internal::DocidChunk; +using query::internal::DocidSource; +using query::internal::ResolvedQueryTerm; +using query::internal::TermPlan; +using reader::LogicalIndexReader; +using internal::PhraseVerifyTimer; + +uint64_t plan_visible_posting_bytes(const format::DictEntry& entry, bool need_positions) { + unsigned __int128 bytes = entry.kind == format::DictEntryKind::kInline + ? entry.inline_dd_disk_len + : entry.frq_docs_len; + if (need_positions) { + bytes += entry.kind == format::DictEntryKind::kInline ? entry.prx_bytes.size() + : entry.prx_len; + } + return bytes > std::numeric_limits::max() ? std::numeric_limits::max() + : static_cast(bytes); +} + +segment_v2::inverted_index::CommonGramsPlanRawCost phrase_plan_raw_cost( + const PhysicalPhrasePlan& plan, const std::vector& batch_terms, + const std::vector& resolved, const std::vector& found, + bool need_positions) { + segment_v2::inverted_index::CommonGramsPlanRawCost cost; + unsigned __int128 posting_bytes = 0; + for (const std::string& term : plan.unique_terms) { + const size_t batch_index = resolved_batch_index(batch_terms, term); + if (found[batch_index] != 0) { + posting_bytes += + plan_visible_posting_bytes(resolved[batch_index].entry, need_positions); + } + } + cost.posting_bytes_or_df_sum = posting_bytes > std::numeric_limits::max() + ? std::numeric_limits::max() + : static_cast(posting_bytes); + cost.estimated_candidate_df = std::numeric_limits::max(); + for (size_t plan_index : plan.phrase_plan_index) { + const size_t batch_index = resolved_batch_index(batch_terms, plan.unique_terms[plan_index]); + if (found[batch_index] != 0) { + cost.estimated_candidate_df = + std::min(cost.estimated_candidate_df, resolved[batch_index].entry.df); + } + } + cost.clause_count = static_cast(plan.phrase_plan_index.size()); + if (cost.clause_count == 0) { + cost.estimated_candidate_df = 0; + } + return cost; +} + +segment_v2::inverted_index::CommonGramsPlanRawCost alternative_clause_raw_cost( + const std::vector& terms, bool need_positions) { + segment_v2::inverted_index::CommonGramsPlanRawCost cost; + unsigned __int128 posting_bytes = 0; + unsigned __int128 candidate_df = 0; + for (const auto& term : terms) { + posting_bytes += plan_visible_posting_bytes(term.entry, need_positions); + candidate_df += term.entry.df; + } + cost.posting_bytes_or_df_sum = posting_bytes > std::numeric_limits::max() + ? std::numeric_limits::max() + : static_cast(posting_bytes); + cost.estimated_candidate_df = candidate_df > std::numeric_limits::max() + ? std::numeric_limits::max() + : static_cast(candidate_df); + cost.clause_count = 1; + return cost; +} + +void append_alternative_clause_cost( + const segment_v2::inverted_index::CommonGramsPlanRawCost& clause, + segment_v2::inverted_index::CommonGramsPlanRawCost* plan) { + const unsigned __int128 posting_bytes = + static_cast(plan->posting_bytes_or_df_sum) + + clause.posting_bytes_or_df_sum; + plan->posting_bytes_or_df_sum = posting_bytes > std::numeric_limits::max() + ? std::numeric_limits::max() + : static_cast(posting_bytes); + plan->estimated_candidate_df = + plan->clause_count == 0 + ? clause.estimated_candidate_df + : std::min(plan->estimated_candidate_df, clause.estimated_candidate_df); + ++plan->clause_count; +} + +segment_v2::inverted_index::CommonGramsPlanRawCost hybrid_verification_raw_cost( + const segment_v2::inverted_index::CommonGramsPlanRawCost& prefilter_cost, + const segment_v2::inverted_index::CommonGramsPlanRawCost& verification_cost) { + auto cost = prefilter_cost; + const unsigned __int128 posting_bytes = + static_cast(cost.posting_bytes_or_df_sum) + + verification_cost.posting_bytes_or_df_sum; + cost.posting_bytes_or_df_sum = posting_bytes > std::numeric_limits::max() + ? std::numeric_limits::max() + : static_cast(posting_bytes); + cost.estimated_candidate_df = + std::min(cost.estimated_candidate_df, verification_cost.estimated_candidate_df); + cost.clause_count = verification_cost.clause_count; + return cost; +} + +bool physical_phrase_plan_has_docs_only_term(const PhysicalPhrasePlan& plan, + const std::vector& batch_terms, + const std::vector& resolved) { + for (const std::string& term : plan.unique_terms) { + if (!entry_has_positions(resolved[resolved_batch_index(batch_terms, term)].entry)) { + return true; + } + } + return false; +} + +void append_physical_phrase_clause(const PhysicalPhrasePlan& source, size_t clause, + uint32_t position_offset, PhysicalPhrasePlan* target) { + DORIS_CHECK_LT(clause, source.phrase_plan_index.size()); + DORIS_CHECK_EQ(source.phrase_plan_index.size(), source.position_offsets.size()); + DORIS_CHECK_EQ(source.phrase_plan_index.size(), source.common_gram_clauses.size()); + const size_t source_term = source.phrase_plan_index[clause]; + DORIS_CHECK_LT(source_term, source.unique_terms.size()); + const std::string& physical_term = source.unique_terms[source_term]; + + const auto unique = std::ranges::find(target->unique_terms, physical_term); + if (unique == target->unique_terms.end()) { + target->phrase_plan_index.push_back(target->unique_terms.size()); + target->unique_terms.push_back(physical_term); + } else { + target->phrase_plan_index.push_back( + static_cast(unique - target->unique_terms.begin())); + } + target->position_offsets.push_back(position_offset); + target->common_gram_clauses.push_back(source.common_gram_clauses[clause]); +} + +HybridPrefixCostEstimate estimate_hybrid_prefix_plan_cost( + const HybridPrefixPlanArtifact& artifact, const std::vector& batch_terms, + const std::vector& resolved, const std::vector& found, + const std::vector& plain_tail_terms, uint32_t position_verify_factor) { + const HybridPositionedCover& plain_tail_cover = artifact.plain_tail_cover; + const bool has_leading_prefilter = + !plain_tail_cover.candidate_prefilter.phrase_plan_index.empty(); + const auto leading_prefilter_cost = + phrase_plan_raw_cost(plain_tail_cover.candidate_prefilter, batch_terms, resolved, found, + /*need_positions=*/false); + unsigned __int128 posting_bytes = leading_prefilter_cost.posting_bytes_or_df_sum; + unsigned __int128 candidate_df_sum = 0; + unsigned __int128 position_verify_work = 0; + uint32_t max_clause_count = 0; + + const auto append_branch = [&](const PhysicalPhrasePlan& verification, + const auto& tail_verification_cost, + const auto* candidate_filter_cost) { + auto verification_cost = phrase_plan_raw_cost(verification, batch_terms, resolved, found, + /*need_positions=*/true); + append_alternative_clause_cost(tail_verification_cost, &verification_cost); + posting_bytes += verification_cost.posting_bytes_or_df_sum; + uint64_t candidate_df = verification_cost.estimated_candidate_df; + if (has_leading_prefilter) { + candidate_df = std::min(candidate_df, leading_prefilter_cost.estimated_candidate_df); + } + if (candidate_filter_cost != nullptr) { + posting_bytes += candidate_filter_cost->posting_bytes_or_df_sum; + candidate_df = std::min(candidate_df, candidate_filter_cost->estimated_candidate_df); + } + candidate_df_sum += candidate_df; + position_verify_work += + static_cast(candidate_df) * verification_cost.clause_count; + max_clause_count = std::max(max_clause_count, verification_cost.clause_count); + }; + + if (!artifact.maps_tail_to_gram) { + DORIS_CHECK(has_leading_prefilter); + DORIS_CHECK(artifact.mapped_tail_split.positioned_indices.empty()); + DORIS_CHECK(artifact.mapped_tail_split.docs_only_indices.empty()); + append_branch( + plain_tail_cover.verification, + alternative_clause_raw_cost(plain_tail_terms, /*need_positions=*/true), + static_cast(nullptr)); + } else { + const HybridPrefixMappedTails& split = artifact.mapped_tail_split; + DORIS_CHECK(!split.positioned_indices.empty() || !split.docs_only_indices.empty()); + DORIS_CHECK(has_leading_prefilter || !split.docs_only_indices.empty()); + if (!split.positioned_indices.empty()) { + DORIS_CHECK(artifact.positioned_tail_verification.has_value()); + append_branch(*artifact.positioned_tail_verification, + alternative_clause_raw_cost(resolved, split.positioned_indices, + /*need_positions=*/true), + static_cast( + nullptr)); + } + if (!split.docs_only_indices.empty()) { + const auto docs_only_filter_cost = + alternative_clause_raw_cost(resolved, split.docs_only_indices, + /*need_positions=*/false); + append_branch(plain_tail_cover.verification, + alternative_clause_raw_cost(plain_tail_terms, split.docs_only_ordinals, + /*need_positions=*/true), + &docs_only_filter_cost); + } + } + + HybridPrefixCostEstimate result; + result.raw_cost.posting_bytes_or_df_sum = posting_bytes > std::numeric_limits::max() + ? std::numeric_limits::max() + : static_cast(posting_bytes); + result.raw_cost.estimated_candidate_df = candidate_df_sum > std::numeric_limits::max() + ? std::numeric_limits::max() + : static_cast(candidate_df_sum); + result.raw_cost.clause_count = max_clause_count; + const unsigned __int128 estimated_cost = + posting_bytes + position_verify_work * position_verify_factor; + result.estimated_cost = estimated_cost > std::numeric_limits::max() + ? std::numeric_limits::max() + : static_cast(estimated_cost); + return result; +} + +} // namespace doris::snii::query::phrase_impl diff --git a/be/src/storage/index/snii/query/phrase_emit.cpp b/be/src/storage/index/snii/query/phrase_emit.cpp new file mode 100644 index 00000000000000..92cf47c8d11722 --- /dev/null +++ b/be/src/storage/index/snii/query/phrase_emit.cpp @@ -0,0 +1,718 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/check.h" +#include "storage/index/inverted/common_grams/common_grams_key_codec.h" +#include "storage/index/inverted/common_grams/common_grams_query_cost.h" +#include "storage/index/inverted/common_grams/common_grams_segment_metadata.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/encoding/byte_source.h" +#include "storage/index/snii/format/dict_entry.h" +#include "storage/index/snii/format/frq_pod.h" +#include "storage/index/snii/format/frq_prelude.h" +#include "storage/index/snii/format/prx_pod.h" +#include "storage/index/snii/io/batch_range_fetcher.h" +#include "storage/index/snii/query/internal/docid_conjunction.h" +#include "storage/index/snii/query/internal/docid_posting_reader.h" +#include "storage/index/snii/query/internal/docid_set_ops.h" +#include "storage/index/snii/query/internal/docid_union.h" +#include "storage/index/snii/query/internal/phrase_query_split.h" +#include "storage/index/snii/query/internal/plain_term_routing.h" +#include "storage/index/snii/query/internal/position_math.h" +#include "storage/index/snii/query/internal/query_test_counters.h" +#include "storage/index/snii/query/internal/resolved_phrase_plan.h" +#include "storage/index/snii/query/internal/sloppy_phrase_matcher.h" +#include "storage/index/snii/query/internal/term_expansion.h" +#include "storage/index/snii/query/phrase_prx_validation.h" +#include "storage/index/snii/query/phrase_query.h" +#include "storage/index/snii/query/phrase_verify_timer.h" +#include "storage/index/snii/query/prefix_query.h" +#include "storage/index/snii/query/term_query.h" +#include "storage/index/snii/reader/windowed_posting.h" +#include "util/debug_points.h" + +namespace doris::snii::query::phrase_impl { + +using query::internal::DocidChunk; +using query::internal::DocidSource; +using query::internal::ResolvedQueryTerm; +using query::internal::TermPlan; +using reader::LogicalIndexReader; +using internal::PhraseVerifyTimer; + +namespace { +class PhrasePositionLoader { +public: + PhrasePositionLoader(size_t plan_count, std::vector& srcs) + : cursors_(plan_count), plan_spans_(plan_count), loaded_epoch_(plan_count, 0) { + for (size_t i = 0; i < plan_count; ++i) { + cursors_[i].init(&srcs[i]); + } + } + + void begin_doc(uint32_t docid) { + docid_ = docid; + ++epoch_; + if (epoch_ == 0) { + std::ranges::fill(loaded_epoch_, 0); + epoch_ = 1; + } + } + + Status positions_for_phrase_pos(const std::vector& phrase_plan_index, size_t phrase_pos, + std::pair* out) { + const size_t plan_index = phrase_plan_index[phrase_pos]; + if (loaded_epoch_[plan_index] != epoch_) { + RETURN_IF_ERROR(cursors_[plan_index].seek(docid_)); + RETURN_IF_ERROR(cursors_[plan_index].positions(&plan_spans_[plan_index])); + loaded_epoch_[plan_index] = epoch_; + SNII_QUERY_COUNT(phrase_position_epoch_cache_misses); + } else { + SNII_QUERY_COUNT(phrase_position_epoch_cache_hits); + } + *out = plan_spans_[plan_index]; + return Status::OK(); + } + +private: + std::vector cursors_; + std::vector> plan_spans_; + std::vector loaded_epoch_; + uint32_t docid_ = 0; + uint32_t epoch_ = 0; +}; + +class PhraseMatchCollector { +public: + PhraseMatchCollector(std::vector* docids, std::vector* matches) + : docids_(docids), matches_(matches) { + DCHECK(docids_ != nullptr || matches_ != nullptr); + } + + bool needs_frequency() const { return matches_ != nullptr; } + + void emit(uint32_t docid, uint32_t frequency) { + DCHECK_GT(frequency, 0); + if (docids_ != nullptr) { + docids_->push_back(docid); + } + if (matches_ != nullptr) { + matches_->push_back({.docid = docid, .frequency = static_cast(frequency)}); + } + } + + void emit_sloppy(uint32_t docid, float frequency) { + DCHECK_GT(frequency, 0.0F); + if (docids_ != nullptr) { + docids_->push_back(docid); + } + if (matches_ != nullptr) { + matches_->push_back({.docid = docid, .frequency = frequency}); + } + } + +private: + std::vector* docids_; + std::vector* matches_; +}; + +bool contains_two_term_phrase(std::pair left_span, + std::pair right_span, + uint32_t right_delta) { + const uint32_t* left = left_span.first; + const uint32_t* right = right_span.first; + if (left == left_span.second || right == right_span.second) { + return false; + } + const uint32_t max_start = std::numeric_limits::max() - right_delta; + if (left + 1 == left_span.second && right + 1 == right_span.second) { + return *left <= max_start && *right == *left + right_delta; + } + while (left != left_span.second && right != right_span.second) { + if (*left > max_start) { + return false; + } + const uint32_t want = *left + right_delta; + while (right != right_span.second && *right < want) { + ++right; + } + if (right == right_span.second) { + return false; + } + if (*right == want) { + return true; + } + ++left; + } + return false; +} + +size_t select_phrase_verification_pair(const std::vector& plans, + const std::vector& phrase_plan_index) { + size_t best_left = 0; + uint64_t best_score = std::numeric_limits::max(); + for (size_t left = 0; left + 1 < phrase_plan_index.size(); ++left) { + const uint64_t score = static_cast(plans[phrase_plan_index[left]].df) + + plans[phrase_plan_index[left + 1]].df; + if (score < best_score) { + best_score = score; + best_left = left; + } + } + return best_left; +} + +class TwoTermPhraseStartCursor { +public: + TwoTermPhraseStartCursor(std::pair left_span, + std::pair right_span, + uint32_t right_delta, uint32_t left_offset) + : left_(left_span.first), + left_end_(left_span.second), + right_(right_span.first), + right_end_(right_span.second), + right_delta_(right_delta), + left_offset_(left_offset), + max_left_(std::numeric_limits::max() - right_delta) {} + + bool next(uint32_t* start) { + DCHECK(start != nullptr); + while (left_ != left_end_ && right_ != right_end_) { + if (*left_ > max_left_) { + return false; + } + const uint32_t want = *left_ + right_delta_; + while (right_ != right_end_ && *right_ < want) { + ++right_; + } + if (right_ == right_end_) { + return false; + } + const uint32_t left_position = *left_++; + if (*right_ == want && left_position >= left_offset_) { + *start = left_position - left_offset_; + return true; + } + } + return false; + } + +private: + const uint32_t* left_; + const uint32_t* left_end_; + const uint32_t* right_; + const uint32_t* right_end_; + uint32_t right_delta_; + uint32_t left_offset_; + uint32_t max_left_; +}; + +uint32_t count_two_term_phrase(std::pair left_span, + std::pair right_span, + uint32_t right_delta) { + TwoTermPhraseStartCursor starts(left_span, right_span, right_delta, /*left_offset=*/0); + uint32_t frequency = 0; + uint32_t start = 0; + while (starts.next(&start)) { + DCHECK_NE(frequency, std::numeric_limits::max()); + ++frequency; + } + return frequency; +} + +Status emit_two_term_phrase_streaming(const std::vector& phrase_plan_index, + const std::vector& position_offsets, + std::vector& srcs, + const std::vector& candidates, + PhraseMatchCollector* collector) { + const size_t left_plan = phrase_plan_index[0]; + const size_t right_plan = phrase_plan_index[1]; + const uint32_t right_delta = position_offsets[1] - position_offsets[0]; + + if (left_plan == right_plan) { + PostingCursor cursor; + cursor.init(&srcs[left_plan]); + for (uint32_t expected_docid : candidates) { + uint32_t docid = 0; + std::pair span; + RETURN_IF_ERROR(cursor.next(&docid, &span)); + if (docid != expected_docid) { + return Status::Error( + "phrase_query: repeated-term cursor/docid mismatch"); + } + const uint32_t frequency = collector->needs_frequency() + ? count_two_term_phrase(span, span, right_delta) + : contains_two_term_phrase(span, span, right_delta); + if (frequency != 0) { + collector->emit(docid, frequency); + } + } + return Status::OK(); + } + + PostingCursor left_cursor; + PostingCursor right_cursor; + left_cursor.init(&srcs[left_plan]); + right_cursor.init(&srcs[right_plan]); + for (uint32_t expected_docid : candidates) { + uint32_t left_docid = 0; + uint32_t right_docid = 0; + std::pair left_span; + std::pair right_span; + RETURN_IF_ERROR(left_cursor.next(&left_docid, &left_span)); + RETURN_IF_ERROR(right_cursor.next(&right_docid, &right_span)); + if (left_docid != expected_docid || right_docid != expected_docid) { + return Status::Error( + "phrase_query: two-term cursor/docid mismatch"); + } + const uint32_t frequency = + collector->needs_frequency() + ? count_two_term_phrase(left_span, right_span, right_delta) + : contains_two_term_phrase(left_span, right_span, right_delta); + if (frequency != 0) { + collector->emit(expected_docid, frequency); + } + } + return Status::OK(); +} + +Status emit_sloppy_phrase_streaming(const std::vector& phrase_plan_index, + const std::vector& position_offsets, + std::vector& srcs, + const std::vector& candidates, + const PhraseQueryOptions& options, + PhraseMatchCollector* collector) { + PhrasePositionLoader loader(srcs.size(), srcs); + std::vector spans(phrase_plan_index.size()); + internal::SloppyPhraseMatcher matcher(phrase_plan_index, position_offsets, options.slop, + options.ordered); + for (uint32_t docid : candidates) { + loader.begin_doc(docid); + for (size_t i = 0; i < phrase_plan_index.size(); ++i) { + RETURN_IF_ERROR(loader.positions_for_phrase_pos(phrase_plan_index, i, &spans[i])); + } + const float frequency = matcher.match(spans, collector->needs_frequency()); + if (frequency > 0.0F) { + collector->emit_sloppy(docid, frequency); + } + } + return Status::OK(); +} + +void emit_two_term_phrase_chunk_pair(const PosChunk& left, const PosChunk& right, + const PosChunkDecoder& left_decoder, + const PosChunkDecoder& right_decoder, uint32_t right_delta, + PhraseMatchCollector* collector) { + size_t li = static_cast(std::ranges::lower_bound(left.docids, right.docids.front()) - + left.docids.begin()); + size_t ri = static_cast(std::ranges::lower_bound(right.docids, left.docids.front()) - + right.docids.begin()); + while (li < left.docids.size() && ri < right.docids.size()) { + const uint32_t left_docid = left.docids[li]; + const uint32_t right_docid = right.docids[ri]; + if (left_docid < right_docid) { + ++li; + continue; + } + if (right_docid < left_docid) { + ++ri; + continue; + } + + const std::pair left_span = + left_decoder.positions_unchecked(li); + const std::pair right_span = + right_decoder.positions_unchecked(ri); + const uint32_t frequency = + collector->needs_frequency() + ? count_two_term_phrase(left_span, right_span, right_delta) + : contains_two_term_phrase(left_span, right_span, right_delta); + if (frequency != 0) { + collector->emit(left_docid, frequency); + } + ++li; + ++ri; + } +} + +Status emit_two_term_phrase_chunk_merge(const std::vector& phrase_plan_index, + const std::vector& position_offsets, + std::vector& srcs, + PhraseMatchCollector* collector) { + const size_t left_plan = phrase_plan_index[0]; + const size_t right_plan = phrase_plan_index[1]; + const uint32_t right_delta = position_offsets[1] - position_offsets[0]; + const PosSource& left_src = srcs[left_plan]; + const PosSource& right_src = srcs[right_plan]; + + PosChunkDecoder left_decoder(left_src.observer_context); + PosChunkDecoder right_decoder(right_src.observer_context); + auto decoded_left_chunk = static_cast(-1); + auto decoded_right_chunk = static_cast(-1); + size_t left_chunk = 0; + size_t right_chunk = 0; + while (left_chunk < left_src.chunks.size() && right_chunk < right_src.chunks.size()) { + const PosChunk& left = left_src.chunks[left_chunk]; + const PosChunk& right = right_src.chunks[right_chunk]; + if (left.docids.empty()) { + ++left_chunk; + continue; + } + if (right.docids.empty()) { + ++right_chunk; + continue; + } + if (left.docids.back() < right.docids.front()) { + ++left_chunk; + continue; + } + if (right.docids.back() < left.docids.front()) { + ++right_chunk; + continue; + } + + if (decoded_left_chunk != left_chunk) { + RETURN_IF_ERROR(left_decoder.decode(left)); + decoded_left_chunk = left_chunk; + } + if (decoded_right_chunk != right_chunk) { + RETURN_IF_ERROR(right_decoder.decode(right)); + decoded_right_chunk = right_chunk; + } + + emit_two_term_phrase_chunk_pair(left, right, left_decoder, right_decoder, right_delta, + collector); + + const uint32_t left_last = left.docids.back(); + const uint32_t right_last = right.docids.back(); + if (left_last <= right_last) { + ++left_chunk; + } + if (right_last <= left_last) { + ++right_chunk; + } + } + return Status::OK(); +} + +bool phrase_start_matches_all_terms( + uint32_t start, size_t phrase_len, size_t pair_left, size_t pair_right, + const std::vector& position_offsets, + const std::vector>& span) { + for (size_t t = 0; t < phrase_len; ++t) { + if (t == pair_left || t == pair_right) { + continue; + } + uint32_t want = 0; + if (!internal::add_position_offset(start, position_offsets[t], &want)) { + return false; + } + if (!std::binary_search(span[t].first, span[t].second, want)) { + return false; + } + } + return true; +} + +Status emit_single_term_phrase_streaming(const std::vector& phrase_plan_index, + std::vector& srcs, + const std::vector& candidates, + PhraseMatchCollector* collector) { + PhrasePositionLoader loader(srcs.size(), srcs); + for (uint32_t d : candidates) { + loader.begin_doc(d); + std::pair single_span; + RETURN_IF_ERROR(loader.positions_for_phrase_pos(phrase_plan_index, 0, &single_span)); + if (single_span.first != single_span.second) { + const auto span_size = static_cast(single_span.second - single_span.first); + DCHECK_LE(span_size, std::numeric_limits::max()); + collector->emit(d, collector->needs_frequency() ? static_cast(span_size) : 1); + } + } + return Status::OK(); +} + +Status emit_multi_term_phrase_streaming(const std::vector& plans, + const std::vector& phrase_plan_index, + const std::vector& position_offsets, + std::vector& srcs, + const std::vector& candidates, + PhraseMatchCollector* collector) { + const size_t phrase_len = phrase_plan_index.size(); + PhrasePositionLoader loader(plans.size(), srcs); + std::vector> span(phrase_len); + const size_t pair_left = select_phrase_verification_pair(plans, phrase_plan_index); + const size_t pair_right = pair_left + 1; + for (uint32_t d : candidates) { + loader.begin_doc(d); + std::pair left_span; + std::pair right_span; + RETURN_IF_ERROR(loader.positions_for_phrase_pos(phrase_plan_index, pair_left, &left_span)); + RETURN_IF_ERROR( + loader.positions_for_phrase_pos(phrase_plan_index, pair_right, &right_span)); + + // `starts` retains raw pointers into the selected pair while the remaining + // clause spans are loaded below. Every unique plan owns an independent + // PostingCursor/PosChunkDecoder in PhrasePositionLoader; repeated phrase + // positions map back to one plan and reuse that plan's epoch-cached span. + TwoTermPhraseStartCursor starts(left_span, right_span, + position_offsets[pair_right] - position_offsets[pair_left], + position_offsets[pair_left]); + uint32_t start = 0; + if (!starts.next(&start)) { + continue; + } + + span[pair_left] = left_span; + span[pair_right] = right_span; + for (size_t pp = 0; pp < phrase_len; ++pp) { + if (pp == pair_left || pp == pair_right) { + continue; + } + RETURN_IF_ERROR(loader.positions_for_phrase_pos(phrase_plan_index, pp, &span[pp])); + } + + uint32_t frequency = 0; + bool has_previous_start = false; + uint32_t previous_start = 0; + const uint32_t* first_clause_position = span[0].first; + while (true) { + if (!collector->needs_frequency()) { + if (phrase_start_matches_all_terms(start, phrase_len, pair_left, pair_right, + position_offsets, span)) { + collector->emit(d, 1); + break; + } + } else if (!has_previous_start || start != previous_start) { + has_previous_start = true; + previous_start = start; + if (phrase_start_matches_all_terms(start, phrase_len, pair_left, pair_right, + position_offsets, span)) { + uint32_t first_clause_want = 0; + const bool representable = internal::add_position_offset( + start, position_offsets[0], &first_clause_want); + DCHECK(representable); + while (first_clause_position != span[0].second && + *first_clause_position < first_clause_want) { + ++first_clause_position; + } + const uint32_t* run_end = first_clause_position; + while (run_end != span[0].second && *run_end == first_clause_want) { + ++run_end; + } + const auto multiplicity = + static_cast(run_end - first_clause_position); + DCHECK_NE(multiplicity, 0); + DCHECK_LE(frequency, std::numeric_limits::max() - multiplicity); + frequency += multiplicity; + first_clause_position = run_end; + } + } + if (!starts.next(&start)) { + break; + } + } + if (frequency != 0) { + collector->emit(d, frequency); + } + } + return Status::OK(); +} + +// Single streaming pass over the candidates: for each (ascending) candidate, +// gather positions lazily, and test the consecutive-phrase predicate +// (term[0]@p, term[1]@p+1, ...). Multi-term phrases first test the cheapest +// adjacent pair by df before decoding the remaining terms for that document. +// Cursors decode each retained chunk at most once and address positions by +// local index -- no per-candidate docid binary search, no full-candidate +// position materialization. Candidates are ascending so the emitted docids are +// already sorted. + +Status emit_phrase_streaming(const std::vector& plans, + const std::vector& phrase_plan_index, + const std::vector& position_offsets, + std::vector& srcs, const std::vector& candidates, + PhraseMatchCollector* collector, const PhraseQueryOptions& options) { + const size_t phrase_len = phrase_plan_index.size(); + if (options.slop != 0) { + return emit_sloppy_phrase_streaming(phrase_plan_index, position_offsets, srcs, candidates, + options, collector); + } + if (phrase_len == 1) { + return emit_single_term_phrase_streaming(phrase_plan_index, srcs, candidates, collector); + } + if (phrase_len == 2) { + if (phrase_plan_index[0] != phrase_plan_index[1]) { + return emit_two_term_phrase_chunk_merge(phrase_plan_index, position_offsets, srcs, + collector); + } + return emit_two_term_phrase_streaming(phrase_plan_index, position_offsets, srcs, candidates, + collector); + } + return emit_multi_term_phrase_streaming(plans, phrase_plan_index, position_offsets, srcs, + candidates, collector); +} + +// candidate_prefilter (optional): an ascending docid set the phrase must ALSO +// lie in. When provided, the leading-term conjunction is intersected with it so +// only docs in the prefilter get their positions read. Docs outside the +// prefilter cannot contribute (the caller guarantees the final answer is a +// subset), so this is result-preserving while cutting the position decode -- +// used by phrase-prefix to restrict the huge leading-phrase candidate set to +// the docs that also carry some tail expansion. + +} // namespace +Status build_phrase_execution_state(const LogicalIndexReader& idx, io::BatchRangeFetcher* round1, + std::vector* plans, PhraseExecutionState* state, + const std::vector* candidate_prefilter, + format::PrxDecodeContext* observer_context, + PhraseCandidateMetric candidate_metric) { + if (round1->pending() > 0) { + RETURN_IF_ERROR(round1->fetch()); + } + RETURN_IF_ERROR(internal::open_preludes(*round1, plans, + /*need_positions=*/true)); + + state->owners.clear(); + state->candidates.clear(); + std::vector doc_sources; + if (candidate_prefilter != nullptr) { + if (candidate_prefilter->empty()) { + return Status::OK(); + } + RETURN_IF_ERROR(internal::filter_docids_by_conjunction( + idx, *round1, *plans, *candidate_prefilter, &state->candidates, &doc_sources)); + } else { + RETURN_IF_ERROR(internal::build_docid_only_conjunction(idx, *round1, *plans, + &state->candidates, &doc_sources)); + } + if (observer_context != nullptr && observer_context->query_stats != nullptr) { + if (candidate_metric == PhraseCandidateMetric::kExact) { + observer_context->query_stats->exact_candidate_docs += state->candidates.size(); + observer_context->query_stats->exact_candidate_visits += state->candidates.size(); + } else { + observer_context->query_stats->prefix_leading_candidate_docs += + state->candidates.size(); + } + } + if (state->candidates.empty()) { + return Status::OK(); + } + RETURN_IF_ERROR(build_position_sources_for_candidates(idx, *round1, *plans, &doc_sources, + state->candidates, &state->owners, + &state->srcs, observer_context)); + return Status::OK(); +} + +namespace { +Status execute_phrase_plans_at_offsets( + const LogicalIndexReader& idx, io::BatchRangeFetcher* round1, std::vector* plans, + const std::vector& phrase_plan_index, const std::vector& position_offsets, + std::vector* docids, format::PrxDecodeContext* observer_context, + std::vector* matches, const PhraseQueryOptions& options, + const std::vector* candidate_prefilter = nullptr) { + DCHECK_EQ(phrase_plan_index.size(), position_offsets.size()); + PhraseExecutionState state; + RETURN_IF_ERROR(build_phrase_execution_state(idx, round1, plans, &state, candidate_prefilter, + observer_context, PhraseCandidateMetric::kExact)); + if (state.candidates.empty()) { + return Status::OK(); + } + + PhraseVerifyTimer verify_timer(observer_context); + PhraseMatchCollector collector(docids, matches); + RETURN_IF_ERROR(emit_phrase_streaming(*plans, phrase_plan_index, position_offsets, state.srcs, + state.candidates, &collector, options)); + verify_timer.commit_success(); + return Status::OK(); +} + +} // namespace +Status execute_phrase_plans(const LogicalIndexReader& idx, io::BatchRangeFetcher* round1, + std::vector* plans, + const std::vector& phrase_plan_index, + std::vector* docids, + format::PrxDecodeContext* observer_context, + std::vector* matches, const PhraseQueryOptions& options) { + std::vector position_offsets; + if (!internal::build_position_offsets(phrase_plan_index.size(), &position_offsets)) { + return Status::Error( + "phrase_query: phrase length exceeds doc position range"); + } + return execute_phrase_plans_at_offsets(idx, round1, plans, phrase_plan_index, position_offsets, + docids, observer_context, matches, options); +} + +} // namespace doris::snii::query::phrase_impl + +namespace doris::snii::query { + +using namespace phrase_impl; // NOLINT(google-build-using-namespace): module-internal impl namespace + +Status internal::execute_resolved_phrase_plan(const LogicalIndexReader& idx, + internal::ResolvedPhrasePlan&& plan, + std::vector* docids, + format::PrxDecodeContext* observer_context, + std::vector* matches, + const std::vector* candidate_prefilter) { + if (docids == nullptr && matches == nullptr) { + return Status::Error("resolved_phrase_plan: null out"); + } + if (docids != nullptr) { + docids->clear(); + } + if (matches != nullptr) { + matches->clear(); + } + DORIS_CHECK(plan.is_valid()); + if (plan.phrase_plan_index.empty()) { + return Status::OK(); + } + + if (plan.phrase_plan_index.size() == 1) { + DORIS_CHECK(matches == nullptr); + const internal::ResolvedQueryTerm& term = plan.unique_terms[plan.phrase_plan_index.front()]; + RETURN_IF_ERROR(internal::read_docid_posting(idx, term.entry, term.frq_base, term.prx_base, + docids)); + if (candidate_prefilter != nullptr) { + *docids = internal::intersect_sorted(*docids, *candidate_prefilter); + } + return Status::OK(); + } + + io::BatchRangeFetcher round1(idx.reader()); + std::vector plans; + RETURN_IF_ERROR(internal::plan_resolved_terms(idx, std::move(plan.unique_terms), &round1, + &plans, + /*need_positions=*/false)); + return execute_phrase_plans_at_offsets(idx, &round1, &plans, plan.phrase_plan_index, + plan.position_offsets, docids, observer_context, matches, + {}, candidate_prefilter); +} + +} // namespace doris::snii::query diff --git a/be/src/storage/index/snii/query/phrase_plan.cpp b/be/src/storage/index/snii/query/phrase_plan.cpp new file mode 100644 index 00000000000000..d9facfddcf9b8d --- /dev/null +++ b/be/src/storage/index/snii/query/phrase_plan.cpp @@ -0,0 +1,600 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/check.h" +#include "storage/index/inverted/common_grams/common_grams_key_codec.h" +#include "storage/index/inverted/common_grams/common_grams_query_cost.h" +#include "storage/index/inverted/common_grams/common_grams_segment_metadata.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/encoding/byte_source.h" +#include "storage/index/snii/format/dict_entry.h" +#include "storage/index/snii/format/frq_pod.h" +#include "storage/index/snii/format/frq_prelude.h" +#include "storage/index/snii/format/prx_pod.h" +#include "storage/index/snii/io/batch_range_fetcher.h" +#include "storage/index/snii/query/internal/docid_conjunction.h" +#include "storage/index/snii/query/internal/docid_posting_reader.h" +#include "storage/index/snii/query/internal/docid_set_ops.h" +#include "storage/index/snii/query/internal/docid_union.h" +#include "storage/index/snii/query/internal/phrase_query_split.h" +#include "storage/index/snii/query/internal/plain_term_routing.h" +#include "storage/index/snii/query/internal/position_math.h" +#include "storage/index/snii/query/internal/query_test_counters.h" +#include "storage/index/snii/query/internal/resolved_phrase_plan.h" +#include "storage/index/snii/query/internal/term_expansion.h" +#include "storage/index/snii/query/phrase_prx_validation.h" +#include "storage/index/snii/query/phrase_query.h" +#include "storage/index/snii/query/phrase_verify_timer.h" +#include "storage/index/snii/query/prefix_query.h" +#include "storage/index/snii/query/term_query.h" +#include "storage/index/snii/reader/windowed_posting.h" +#include "util/debug_points.h" + +namespace doris::snii::query::phrase_impl { + +using query::internal::DocidChunk; +using query::internal::DocidSource; +using query::internal::ResolvedQueryTerm; +using query::internal::TermPlan; +using reader::LogicalIndexReader; +using internal::PhraseVerifyTimer; + +bool apply_common_grams_plan_debug_override(bool cost_prefers_gram, + CommonGramsPlanDebugOverride debug_override) { + switch (debug_override) { + case CommonGramsPlanDebugOverride::kNone: + return cost_prefers_gram; + case CommonGramsPlanDebugOverride::kForcePlain: + return false; + case CommonGramsPlanDebugOverride::kForceCommonGrams: + return true; + } + DORIS_CHECK(false); + return cost_prefers_gram; +} + +size_t position_span_size(std::pair span) { + if (span.first == span.second) { + return 0; + } + DCHECK(span.first != nullptr); + DCHECK(span.second != nullptr); + return static_cast(span.second - span.first); +} + +bool should_use_monotonic_position_scan(std::pair anchor_span, + size_t checked_span_size, uint32_t anchor_offset, + uint32_t checked_offset) { + const uint64_t anchor_count = position_span_size(anchor_span); + const uint64_t binary_search_upper_bound = + anchor_count * (static_cast(std::bit_width(checked_span_size)) + 1); + const uint64_t monotonic_scan_upper_bound = checked_span_size + 2 * anchor_count + 2; + + // Require a 2x comparison margin before paying even the O(1) validity + // checks and adding the scan-path branches. This keeps low-TF spans on the + // simpler binary-search path while retaining the high-TF dense case. + if (2 * monotonic_scan_upper_bound > binary_search_upper_bound) { + return false; + } + + // Scanning is considered only when every anchor yields a representable + // phrase start and checked-term position. Endpoint checks are sufficient + // because anchor positions are sorted; invalid boundary shapes stay on the + // existing per-anchor path without extra binary searches in this gate. + if (*anchor_span.first < anchor_offset) { + return false; + } + if (checked_offset <= anchor_offset) { + return true; + } + const uint32_t offset_delta = checked_offset - anchor_offset; + return anchor_span.second[-1] <= std::numeric_limits::max() - offset_delta; +} + +bool has_common_grams_capability( + const LogicalIndexReader& idx, + const segment_v2::inverted_index::CommonGramsQueryIdentity* query_identity) { + const auto* metadata = idx.common_grams_metadata(); + if (metadata == nullptr || query_identity == nullptr) { + return false; + } + if (idx.common_grams_posting_policy() == format::CommonGramsPostingPolicy::kHybridV1) { + return segment_v2::inverted_index::is_common_grams_query_compatible( + *metadata, *query_identity, + segment_v2::inverted_index::CommonGramsCoverage::kMixed); + } + return segment_v2::inverted_index::is_common_grams_query_compatible(*metadata, *query_identity); +} + +bool entry_has_positions(const format::DictEntry& entry) { + return entry.kind == format::DictEntryKind::kInline ? !entry.prx_bytes.empty() + : entry.prx_len != 0; +} + +Status build_physical_phrase_plan_prefix(const LogicalIndexReader& idx, + const segment_v2::InvertedIndexQueryInfo& query_info, + size_t clause_count, bool allow_common_grams, + PhysicalPhrasePlan* plan, bool* all_representable) { + plan->unique_terms.clear(); + plan->phrase_plan_index.clear(); + plan->position_offsets.clear(); + plan->common_gram_clauses.clear(); + *all_representable = true; + DORIS_CHECK_LE(clause_count, query_info.term_infos.size()); + if (clause_count == 0) { + return Status::OK(); + } + + const int32_t first_position = query_info.term_infos.front().position; + DORIS_CHECK_LE(clause_count, static_cast(std::numeric_limits::max())); + plan->phrase_plan_index.reserve(clause_count); + plan->position_offsets.reserve(clause_count); + plan->common_gram_clauses.reserve(clause_count); + for (size_t i = 0; i < clause_count; ++i) { + const segment_v2::TermInfo& term_info = query_info.term_infos[i]; + DORIS_CHECK(term_info.is_single_term()); + DORIS_CHECK_EQ(static_cast(term_info.position), + static_cast(first_position) + static_cast(i)); + + std::string physical_term; + if (term_info.key_kind == segment_v2::TermKeyKind::kCommonGram) { + DORIS_CHECK(allow_common_grams); + DORIS_CHECK(std::string_view(term_info.get_single_term()) + .starts_with(segment_v2::inverted_index::CG_V1_MARKER)); + physical_term = term_info.get_single_term(); + } else { + bool representable = false; + RETURN_IF_ERROR(internal::route_plain_query_term(idx, term_info.get_single_term(), + &physical_term, &representable)); + if (!representable) { + *all_representable = false; + return Status::OK(); + } + } + + auto unique = std::ranges::find(plan->unique_terms, physical_term); + if (unique == plan->unique_terms.end()) { + plan->phrase_plan_index.push_back(plan->unique_terms.size()); + plan->unique_terms.push_back(std::move(physical_term)); + } else { + plan->phrase_plan_index.push_back( + static_cast(unique - plan->unique_terms.begin())); + } + plan->position_offsets.push_back(static_cast(i)); + plan->common_gram_clauses.push_back( + static_cast(term_info.key_kind == segment_v2::TermKeyKind::kCommonGram)); + } + return Status::OK(); +} + +Status build_physical_phrase_plan(const LogicalIndexReader& idx, + const segment_v2::InvertedIndexQueryInfo& query_info, + bool allow_common_grams, PhysicalPhrasePlan* plan, + bool* all_representable) { + return build_physical_phrase_plan_prefix(idx, query_info, query_info.term_infos.size(), + allow_common_grams, plan, all_representable); +} + +size_t resolved_batch_index(const std::vector& batch_terms, std::string_view term) { + const auto it = std::ranges::lower_bound(batch_terms, term); + DORIS_CHECK(it != batch_terms.end()); + DORIS_CHECK_EQ(*it, term); + return static_cast(it - batch_terms.begin()); +} + +bool all_plan_terms_present(const PhysicalPhrasePlan& plan, + const std::vector& batch_terms, + const std::vector& found) { + for (const std::string& term : plan.unique_terms) { + if (found[resolved_batch_index(batch_terms, term)] == 0) { + return false; + } + } + return true; +} + +internal::ResolvedPhrasePlan materialize_resolved_phrase_plan( + const PhysicalPhrasePlan& plan, const std::vector& batch_terms, + std::vector* resolved) { + internal::ResolvedPhrasePlan result; + result.phrase_plan_index = plan.phrase_plan_index; + result.position_offsets = plan.position_offsets; + result.unique_terms.reserve(plan.unique_terms.size()); + for (const std::string& term : plan.unique_terms) { + result.unique_terms.push_back( + std::move((*resolved)[resolved_batch_index(batch_terms, term)])); + } + return result; +} + +internal::ResolvedPhrasePlan copy_resolved_phrase_plan( + const PhysicalPhrasePlan& plan, const std::vector& batch_terms, + const std::vector& resolved) { + internal::ResolvedPhrasePlan result; + result.phrase_plan_index = plan.phrase_plan_index; + result.position_offsets = plan.position_offsets; + result.unique_terms.reserve(plan.unique_terms.size()); + for (const std::string& term : plan.unique_terms) { + result.unique_terms.push_back(resolved[resolved_batch_index(batch_terms, term)]); + } + return result; +} + +namespace { +PhysicalPhrasePlan build_hybrid_positioned_verification( + const PhysicalPhrasePlan& plain_plan, const PhysicalPhrasePlan& gram_plan, + const std::vector& batch_terms, const std::vector& resolved, + bool tail_covers_last_plain_clause, PhysicalPhrasePlan* candidate_prefilter) { + const size_t original_clause_count = plain_plan.phrase_plan_index.size(); + DORIS_CHECK_GT(original_clause_count, 0); + DORIS_CHECK_EQ(plain_plan.position_offsets.size(), original_clause_count); + DORIS_CHECK_EQ(plain_plan.common_gram_clauses.size(), original_clause_count); + DORIS_CHECK_EQ(gram_plan.position_offsets.size(), gram_plan.phrase_plan_index.size()); + DORIS_CHECK_EQ(gram_plan.common_gram_clauses.size(), gram_plan.phrase_plan_index.size()); + + PhysicalPhrasePlan verification; + std::vector positioned_gram_at(original_clause_count, + gram_plan.phrase_plan_index.size()); + for (size_t clause = 0; clause < gram_plan.phrase_plan_index.size(); ++clause) { + if (gram_plan.common_gram_clauses[clause] == 0) { + continue; + } + const size_t gram_term = gram_plan.phrase_plan_index[clause]; + DORIS_CHECK_LT(gram_term, gram_plan.unique_terms.size()); + const size_t batch_index = + resolved_batch_index(batch_terms, gram_plan.unique_terms[gram_term]); + if (!entry_has_positions(resolved[batch_index].entry)) { + if (candidate_prefilter != nullptr) { + append_physical_phrase_clause(gram_plan, clause, gram_plan.position_offsets[clause], + candidate_prefilter); + } + continue; + } + + const size_t original_offset = gram_plan.position_offsets[clause]; + DORIS_CHECK_LT(original_offset + 1, original_clause_count); + DORIS_CHECK_EQ(positioned_gram_at[original_offset], gram_plan.phrase_plan_index.size()); + positioned_gram_at[original_offset] = clause; + } + // Start from every positioned gram edge, then remove an edge only when both + // endpoint tokens remain covered by another positioned edge. The left-to-right + // pass produces a minimum-clause edge cover while retaining grams in preference + // to adding their two plain components during the pass below. + std::vector positioned_coverage(original_clause_count, 0); + if (tail_covers_last_plain_clause) { + ++positioned_coverage.back(); + } + for (size_t original_offset = 0; original_offset < original_clause_count; ++original_offset) { + if (positioned_gram_at[original_offset] == gram_plan.phrase_plan_index.size()) { + continue; + } + ++positioned_coverage[original_offset]; + ++positioned_coverage[original_offset + 1]; + } + for (size_t original_offset = 0; original_offset < original_clause_count; ++original_offset) { + if (positioned_gram_at[original_offset] == gram_plan.phrase_plan_index.size() || + positioned_coverage[original_offset] <= 1 || + positioned_coverage[original_offset + 1] <= 1) { + continue; + } + positioned_gram_at[original_offset] = gram_plan.phrase_plan_index.size(); + --positioned_coverage[original_offset]; + --positioned_coverage[original_offset + 1]; + } + + for (size_t original_offset = 0; original_offset < original_clause_count; ++original_offset) { + const size_t gram_clause = positioned_gram_at[original_offset]; + if (gram_clause != gram_plan.phrase_plan_index.size()) { + append_physical_phrase_clause(gram_plan, gram_clause, + static_cast(original_offset), &verification); + continue; + } + if (positioned_coverage[original_offset] == 0) { + DORIS_CHECK_EQ(plain_plan.position_offsets[original_offset], original_offset); + append_physical_phrase_clause(plain_plan, original_offset, + static_cast(original_offset), &verification); + } + } + DORIS_CHECK(!verification.phrase_plan_index.empty() || + (tail_covers_last_plain_clause && original_clause_count == 1)); + return verification; +} + +HybridPositionedCover build_hybrid_positioned_cover(const PhysicalPhrasePlan& plain_plan, + const PhysicalPhrasePlan& gram_plan, + const std::vector& batch_terms, + const std::vector& resolved, + bool tail_covers_last_plain_clause) { + HybridPositionedCover result; + result.verification = build_hybrid_positioned_verification( + plain_plan, gram_plan, batch_terms, resolved, tail_covers_last_plain_clause, + &result.candidate_prefilter); + return result; +} + +} // namespace +HybridExactPlanArtifact build_hybrid_exact_plan_artifact( + const PhysicalPhrasePlan& plain_plan, const PhysicalPhrasePlan& gram_plan, + const std::vector& batch_terms, + const std::vector& resolved) { + HybridExactPlanArtifact artifact; + if (gram_plan.phrase_plan_index.size() > 1 && + physical_phrase_plan_has_docs_only_term(gram_plan, batch_terms, resolved)) { + artifact.positioned_cover.emplace( + build_hybrid_positioned_cover(plain_plan, gram_plan, batch_terms, resolved, + /*tail_covers_last_plain_clause=*/false)); + DORIS_CHECK(!artifact.positioned_cover->candidate_prefilter.phrase_plan_index.empty()); + } + return artifact; +} + +namespace { +Status build_physical_phrase_plan_candidates(const LogicalIndexReader& idx, + const PhysicalPhrasePlan& plan, + const std::vector& batch_terms, + const std::vector& resolved, + std::vector* candidates) { + std::vector candidate_terms; + candidate_terms.reserve(plan.unique_terms.size()); + for (const std::string& term : plan.unique_terms) { + candidate_terms.push_back(resolved[resolved_batch_index(batch_terms, term)]); + } + + io::BatchRangeFetcher round1(idx.reader()); + std::vector plans; + RETURN_IF_ERROR(internal::plan_resolved_terms(idx, std::move(candidate_terms), &round1, &plans, + /*need_positions=*/false)); + if (round1.pending() > 0) { + RETURN_IF_ERROR(round1.fetch()); + } + RETURN_IF_ERROR(internal::open_preludes(round1, &plans, /*need_positions=*/false)); + return internal::build_docid_only_conjunction(idx, round1, plans, candidates); +} + +HybridPrefixMappedTails split_hybrid_prefix_mapped_tails( + const std::vector& resolved, + const std::vector& mapped_tails) { + HybridPrefixMappedTails result; + for (const ResolvedMappedTail& tail : mapped_tails) { + DORIS_CHECK_LT(tail.batch_index, resolved.size()); + if (entry_has_positions(resolved[tail.batch_index].entry)) { + result.positioned_indices.push_back(tail.batch_index); + } else { + result.docs_only_indices.push_back(tail.batch_index); + result.docs_only_ordinals.push_back(tail.expansion_ordinal); + } + } + return result; +} + +} // namespace +std::optional try_build_hybrid_prefix_plan_artifact( + const PhysicalPhrasePlan& plain_leading, const PhysicalPhrasePlan& gram_leading, + const std::vector& batch_terms, const std::vector& resolved, + const std::vector& mapped_tails, bool maps_tail_to_gram) { + const bool requires_plain_verification = + physical_phrase_plan_has_docs_only_term(gram_leading, batch_terms, resolved) || + std::ranges::any_of(mapped_tails, [&](const ResolvedMappedTail& tail) { + DORIS_CHECK_LT(tail.batch_index, resolved.size()); + return !entry_has_positions(resolved[tail.batch_index].entry); + }); + if (!requires_plain_verification) { + return std::nullopt; + } + + DORIS_CHECK(!plain_leading.phrase_plan_index.empty()); + DORIS_CHECK_LE(plain_leading.phrase_plan_index.size(), + static_cast(std::numeric_limits::max())); + HybridPrefixPlanArtifact artifact; + artifact.plain_tail_cover = + build_hybrid_positioned_cover(plain_leading, gram_leading, batch_terms, resolved, + /*tail_covers_last_plain_clause=*/false); + artifact.plain_tail_position_offset = + static_cast(plain_leading.phrase_plan_index.size()); + artifact.maps_tail_to_gram = maps_tail_to_gram; + if (!maps_tail_to_gram) { + DORIS_CHECK(mapped_tails.empty()); + return artifact; + } + + DORIS_CHECK(!mapped_tails.empty()); + artifact.mapped_tail_split = split_hybrid_prefix_mapped_tails(resolved, mapped_tails); + if (!artifact.mapped_tail_split.positioned_indices.empty()) { + artifact.positioned_tail_verification.emplace(build_hybrid_positioned_verification( + plain_leading, gram_leading, batch_terms, resolved, + /*tail_covers_last_plain_clause=*/true, + /*candidate_prefilter=*/nullptr)); + } + return artifact; +} + +Status build_hybrid_leading_candidates(const LogicalIndexReader& idx, + const PhysicalPhrasePlan& candidate_prefilter, + const std::vector& batch_terms, + const std::vector& resolved, + HybridPrefixCandidateSet* candidates) { + candidates->active = !candidate_prefilter.phrase_plan_index.empty(); + candidates->docs.clear(); + if (!candidates->active) { + return Status::OK(); + } + return build_physical_phrase_plan_candidates(idx, candidate_prefilter, batch_terms, resolved, + &candidates->docs); +} + +namespace { +Status build_tail_candidates_within_leading(const LogicalIndexReader& idx, + const std::vector& resolved, + const std::vector& tail_indices, + const std::vector& leading_candidates, + std::vector* candidates) { + std::vector tail_terms; + tail_terms.reserve(tail_indices.size()); + for (size_t index : tail_indices) { + DORIS_CHECK_LT(index, resolved.size()); + tail_terms.push_back(resolved[index]); + } + + io::BatchRangeFetcher round1(idx.reader()); + std::vector tail_plans; + RETURN_IF_ERROR(internal::plan_resolved_terms(idx, std::move(tail_terms), &round1, &tail_plans, + /*need_positions=*/false)); + if (round1.pending() > 0) { + RETURN_IF_ERROR(round1.fetch()); + } + RETURN_IF_ERROR(internal::open_preludes(round1, &tail_plans, /*need_positions=*/false)); + + candidates->clear(); + std::vector one_tail_plan; + one_tail_plan.reserve(1); + for (auto& tail_plan : tail_plans) { + one_tail_plan.clear(); + one_tail_plan.push_back(std::move(tail_plan)); + std::vector tail_matches; + RETURN_IF_ERROR(internal::filter_docids_by_conjunction( + idx, round1, one_tail_plan, leading_candidates, &tail_matches, nullptr)); + internal::union_sorted_into(candidates, tail_matches); + if (candidates->size() == leading_candidates.size()) { + break; + } + } + return Status::OK(); +} + +} // namespace +Status build_hybrid_docs_only_tail_candidates(const LogicalIndexReader& idx, + const std::vector& resolved, + const std::vector& gram_tail_indices, + const HybridPrefixCandidateSet& leading_candidates, + std::vector* candidates) { + DORIS_CHECK(!gram_tail_indices.empty()); + unsigned __int128 tail_df_sum = 0; + for (size_t index : gram_tail_indices) { + DORIS_CHECK_LT(index, resolved.size()); + tail_df_sum += resolved[index].entry.df; + } + if (leading_candidates.active && + static_cast(leading_candidates.docs.size()) <= tail_df_sum) { + return build_tail_candidates_within_leading(idx, resolved, gram_tail_indices, + leading_candidates.docs, candidates); + } + + std::vector tail_postings; + tail_postings.reserve(gram_tail_indices.size()); + for (size_t index : gram_tail_indices) { + const auto& tail = resolved[index]; + tail_postings.push_back({tail.entry, tail.frq_base, tail.prx_base}); + } + std::vector tail_candidates; + RETURN_IF_ERROR(internal::build_docid_union(idx, tail_postings, &tail_candidates)); + *candidates = leading_candidates.active + ? internal::intersect_sorted(leading_candidates.docs, tail_candidates) + : std::move(tail_candidates); + return Status::OK(); +} + +Status execute_hybrid_exact_phrase_plan( + const LogicalIndexReader& idx, const PhysicalPhrasePlan& gram_plan, + const std::vector& batch_terms, const HybridExactPlanArtifact& artifact, + std::vector* resolved, std::vector* docids, + format::PrxDecodeContext* decode_context, bool* candidate_intersection_empty) { + DORIS_CHECK(idx.common_grams_posting_policy() == format::CommonGramsPostingPolicy::kHybridV1); + if (candidate_intersection_empty != nullptr) { + *candidate_intersection_empty = false; + } + if (!artifact.positioned_cover.has_value()) { + auto resolved_gram = materialize_resolved_phrase_plan(gram_plan, batch_terms, resolved); + return internal::execute_resolved_phrase_plan(idx, std::move(resolved_gram), docids, + decode_context); + } + + const HybridPositionedCover& hybrid_plan = *artifact.positioned_cover; + std::vector gram_candidates; + RETURN_IF_ERROR(build_physical_phrase_plan_candidates( + idx, hybrid_plan.candidate_prefilter, batch_terms, *resolved, &gram_candidates)); + if (gram_candidates.empty()) { + if (candidate_intersection_empty != nullptr) { + *candidate_intersection_empty = true; + } + return Status::OK(); + } + auto resolved_verification = + materialize_resolved_phrase_plan(hybrid_plan.verification, batch_terms, resolved); + return internal::execute_resolved_phrase_plan(idx, std::move(resolved_verification), docids, + decode_context, nullptr, &gram_candidates); +} + +void append_resolved_phrase_clause(ResolvedQueryTerm term, uint32_t position_offset, + internal::ResolvedPhrasePlan* plan) { + const auto unique = std::ranges::find(plan->unique_terms, term.entry.term, + [](const ResolvedQueryTerm& candidate) { + return std::string_view(candidate.entry.term); + }); + if (unique == plan->unique_terms.end()) { + plan->phrase_plan_index.push_back(plan->unique_terms.size()); + plan->unique_terms.push_back(std::move(term)); + } else { + plan->phrase_plan_index.push_back(static_cast(unique - plan->unique_terms.begin())); + } + plan->position_offsets.push_back(position_offset); +} + +internal::ResolvedPhrasePlan build_resolved_phrase_plan( + std::vector resolved_terms) { + internal::ResolvedPhrasePlan plan; + plan.unique_terms.reserve(resolved_terms.size()); + plan.phrase_plan_index.reserve(resolved_terms.size()); + plan.position_offsets.reserve(resolved_terms.size()); + for (size_t i = 0; i < resolved_terms.size(); ++i) { + DORIS_CHECK_LE(i, static_cast(std::numeric_limits::max())); + append_resolved_phrase_clause(std::move(resolved_terms[i]), static_cast(i), + &plan); + } + return plan; +} + +Status resolve_and_execute_physical_phrase_plan(const LogicalIndexReader& idx, + const PhysicalPhrasePlan& plan, + std::vector* docids, + format::PrxDecodeContext* decode_context, + CommonGramsPlanningTimer& planning_timer) { + std::vector batch_terms = plan.unique_terms; + std::ranges::sort(batch_terms); + std::vector resolved; + std::vector found; + RETURN_IF_ERROR(internal::resolve_query_terms_batch(idx, batch_terms, &resolved, &found)); + if (!all_plan_terms_present(plan, batch_terms, found)) { + return Status::OK(); + } + auto resolved_plan = materialize_resolved_phrase_plan(plan, batch_terms, &resolved); + planning_timer.finish(); + return internal::execute_resolved_phrase_plan(idx, std::move(resolved_plan), docids, + decode_context); +} + +} // namespace doris::snii::query::phrase_impl diff --git a/be/src/storage/index/snii/query/phrase_planned_query.cpp b/be/src/storage/index/snii/query/phrase_planned_query.cpp new file mode 100644 index 00000000000000..c5937e71c8122d --- /dev/null +++ b/be/src/storage/index/snii/query/phrase_planned_query.cpp @@ -0,0 +1,743 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/check.h" +#include "storage/index/inverted/common_grams/common_grams_key_codec.h" +#include "storage/index/inverted/common_grams/common_grams_query_cost.h" +#include "storage/index/inverted/common_grams/common_grams_segment_metadata.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/encoding/byte_source.h" +#include "storage/index/snii/format/dict_entry.h" +#include "storage/index/snii/format/frq_pod.h" +#include "storage/index/snii/format/frq_prelude.h" +#include "storage/index/snii/format/prx_pod.h" +#include "storage/index/snii/io/batch_range_fetcher.h" +#include "storage/index/snii/query/internal/docid_conjunction.h" +#include "storage/index/snii/query/internal/docid_posting_reader.h" +#include "storage/index/snii/query/internal/docid_set_ops.h" +#include "storage/index/snii/query/internal/docid_union.h" +#include "storage/index/snii/query/internal/phrase_query_split.h" +#include "storage/index/snii/query/internal/plain_term_routing.h" +#include "storage/index/snii/query/internal/position_math.h" +#include "storage/index/snii/query/internal/query_test_counters.h" +#include "storage/index/snii/query/internal/resolved_phrase_plan.h" +#include "storage/index/snii/query/internal/term_expansion.h" +#include "storage/index/snii/query/phrase_prx_validation.h" +#include "storage/index/snii/query/phrase_query.h" +#include "storage/index/snii/query/phrase_verify_timer.h" +#include "storage/index/snii/query/prefix_query.h" +#include "storage/index/snii/query/term_query.h" +#include "storage/index/snii/reader/windowed_posting.h" +#include "util/debug_points.h" + +namespace doris::snii::query::phrase_impl { + +using query::internal::DocidChunk; +using query::internal::DocidSource; +using query::internal::ResolvedQueryTerm; +using query::internal::TermPlan; +using reader::LogicalIndexReader; +using internal::PhraseVerifyTimer; + +Status planned_exact_phrase_query_impl( + const LogicalIndexReader& idx, const segment_v2::InvertedIndexQueryInfo& plain_query_info, + const segment_v2::InvertedIndexQueryInfo& gram_query_info, + const segment_v2::inverted_index::CommonGramsQueryIdentity* common_grams_identity, + std::vector* docids, format::PrxDecodeContext* decode_context, + ExactPhrasePlanKind* selected_plan, + segment_v2::inverted_index::CommonGramsPlanCostModel cost_model, + CommonGramsPlanDebugOverride debug_override) { + if (docids == nullptr) { + return Status::Error( + "planned_exact_phrase_query: null out"); + } + docids->clear(); + DORIS_CHECK(!plain_query_info.has_common_gram()); + auto* query_stats = decode_context == nullptr ? nullptr : decode_context->query_stats; + CommonGramsPlanningTimer planning_timer(query_stats); + if (query_stats != nullptr) { + ++query_stats->common_grams_candidate_queries; + } + + PhysicalPhrasePlan plain_plan; + bool plain_representable = false; + RETURN_IF_ERROR(build_physical_phrase_plan(idx, plain_query_info, + /*allow_common_grams=*/false, &plain_plan, + &plain_representable)); + if (!plain_representable || plain_plan.phrase_plan_index.empty()) { + if (query_stats != nullptr) { + ++query_stats->common_grams_plain_plans; + } + if (selected_plan != nullptr) { + *selected_plan = ExactPhrasePlanKind::kPlain; + } + return Status::OK(); + } + + const bool index_has_common_grams = has_common_grams_capability(idx, common_grams_identity); + const bool gram_capable = gram_query_info.has_common_gram() && index_has_common_grams; + if (!gram_capable) { + if (query_stats != nullptr) { + ++query_stats->common_grams_plain_plans; + if (gram_query_info.has_common_gram()) { + ++query_stats->common_grams_fallback_incompatible; + } else { + ++query_stats->common_grams_fallback_no_gram; + } + } + if (selected_plan != nullptr) { + *selected_plan = ExactPhrasePlanKind::kPlain; + } + return resolve_and_execute_physical_phrase_plan(idx, plain_plan, docids, decode_context, + planning_timer); + } + + PhysicalPhrasePlan gram_plan; + bool gram_representable = false; + RETURN_IF_ERROR(build_physical_phrase_plan(idx, gram_query_info, + /*allow_common_grams=*/true, &gram_plan, + &gram_representable)); + DORIS_CHECK(gram_representable); + DORIS_CHECK(!gram_plan.phrase_plan_index.empty()); + + std::vector batch_terms = plain_plan.unique_terms; + batch_terms.insert(batch_terms.end(), gram_plan.unique_terms.begin(), + gram_plan.unique_terms.end()); + std::ranges::sort(batch_terms); + batch_terms.erase(std::unique(batch_terms.begin(), batch_terms.end()), batch_terms.end()); + + std::vector resolved; + std::vector found; + RETURN_IF_ERROR(internal::resolve_query_terms_batch(idx, batch_terms, &resolved, &found)); + const bool plain_present = all_plan_terms_present(plain_plan, batch_terms, found); + const bool gram_present = all_plan_terms_present(gram_plan, batch_terms, found); + if (!plain_present || !gram_present) { + if (query_stats != nullptr) { + if (plain_present) { + ++query_stats->common_grams_gram_plans; + ++query_stats->common_grams_authoritative_empty; + } else { + ++query_stats->common_grams_plain_plans; + ++query_stats->common_grams_authoritative_empty; + } + } + if (selected_plan != nullptr) { + *selected_plan = + plain_present ? ExactPhrasePlanKind::kCommonGrams : ExactPhrasePlanKind::kPlain; + } + return Status::OK(); + } + + std::optional hybrid_artifact; + if (idx.common_grams_posting_policy() == format::CommonGramsPostingPolicy::kHybridV1) { + hybrid_artifact.emplace( + build_hybrid_exact_plan_artifact(plain_plan, gram_plan, batch_terms, resolved)); + } + const bool plain_needs_positions = plain_plan.phrase_plan_index.size() > 1; + const bool hybrid_gram_verification = + hybrid_artifact.has_value() && hybrid_artifact->positioned_cover.has_value(); + const bool gram_needs_positions = + gram_plan.phrase_plan_index.size() > 1 && !hybrid_gram_verification; + const auto plain_raw_cost = + phrase_plan_raw_cost(plain_plan, batch_terms, resolved, found, plain_needs_positions); + segment_v2::inverted_index::CommonGramsPlanRawCost gram_raw_cost; + if (hybrid_gram_verification) { + const HybridPositionedCover& hybrid_plan = *hybrid_artifact->positioned_cover; + const auto gram_prefilter_raw_cost = + phrase_plan_raw_cost(hybrid_plan.candidate_prefilter, batch_terms, resolved, found, + /*need_positions=*/false); + const auto gram_verification_raw_cost = + phrase_plan_raw_cost(hybrid_plan.verification, batch_terms, resolved, found, + /*need_positions=*/true); + gram_raw_cost = + hybrid_verification_raw_cost(gram_prefilter_raw_cost, gram_verification_raw_cost); + } else { + gram_raw_cost = + phrase_plan_raw_cost(gram_plan, batch_terms, resolved, found, gram_needs_positions); + } + const uint64_t plain_cost = segment_v2::inverted_index::estimate_common_grams_plan_cost( + plain_raw_cost, plain_needs_positions ? cost_model.position_verify_factor : 0); + const uint64_t gram_cost = segment_v2::inverted_index::estimate_common_grams_plan_cost( + gram_raw_cost, (gram_needs_positions || hybrid_gram_verification) + ? cost_model.position_verify_factor + : 0); + const bool cost_prefers_gram = segment_v2::inverted_index::common_grams_plan_cost_wins( + plain_cost, gram_cost, cost_model.common_grams_cost_ratio_percent); + const bool use_gram = apply_common_grams_plan_debug_override(cost_prefers_gram, debug_override); + if (query_stats != nullptr) { + query_stats->common_grams_plain_posting_bytes += plain_raw_cost.posting_bytes_or_df_sum; + query_stats->common_grams_gram_posting_bytes += gram_raw_cost.posting_bytes_or_df_sum; + query_stats->common_grams_plain_estimated_candidate_df += + plain_raw_cost.estimated_candidate_df; + query_stats->common_grams_gram_estimated_candidate_df += + gram_raw_cost.estimated_candidate_df; + query_stats->common_grams_plain_estimated_cost += plain_cost; + query_stats->common_grams_gram_estimated_cost += gram_cost; + } + const ExactPhrasePlanKind chosen_kind = + use_gram ? ExactPhrasePlanKind::kCommonGrams : ExactPhrasePlanKind::kPlain; + if (query_stats != nullptr) { + if (use_gram) { + ++query_stats->common_grams_gram_plans; + } else { + ++query_stats->common_grams_plain_plans; + if (debug_override == CommonGramsPlanDebugOverride::kNone) { + ++query_stats->common_grams_fallback_cost; + } + } + } + if (selected_plan != nullptr) { + *selected_plan = chosen_kind; + } + planning_timer.finish(); + if (use_gram && + idx.common_grams_posting_policy() == format::CommonGramsPostingPolicy::kHybridV1) { + DORIS_CHECK(hybrid_artifact.has_value()); + bool candidate_intersection_empty = false; + RETURN_IF_ERROR(execute_hybrid_exact_phrase_plan( + idx, gram_plan, batch_terms, *hybrid_artifact, &resolved, docids, decode_context, + &candidate_intersection_empty)); + if (candidate_intersection_empty) { + if (query_stats != nullptr) { + ++query_stats->common_grams_authoritative_empty; + } + } + return Status::OK(); + } + auto resolved_plan = materialize_resolved_phrase_plan(use_gram ? gram_plan : plain_plan, + batch_terms, &resolved); + return internal::execute_resolved_phrase_plan(idx, std::move(resolved_plan), docids, + decode_context); +} + +Status phrase_query_impl(const LogicalIndexReader& idx, const std::vector& terms, + std::vector* const docids, + format::PrxDecodeContext* decode_context, + std::vector* matches, const PhraseQueryOptions& options) { + if (docids == nullptr && matches == nullptr) { + return Status::Error("phrase_query: null out"); + } + if (docids != nullptr) { + docids->clear(); + } + if (matches != nullptr) { + matches->clear(); + } + if (terms.empty()) { + return Status::OK(); + } + if (terms.size() == 1) { + DORIS_CHECK(matches == nullptr); + return term_query(idx, terms.front(), docids); + } + if (!idx.has_positions()) { + return Status::Error( + "phrase_query: index has no positions"); + } + io::BatchRangeFetcher round1(idx.reader()); + const PhraseTermMapping mapping = build_phrase_term_mapping(terms); + std::vector plans; + bool all_present = false; + RETURN_IF_ERROR(internal::plan_terms(idx, mapping.unique_terms, &round1, &plans, &all_present, + /*need_positions=*/false)); + if (!all_present) { + return Status::OK(); + } + return execute_phrase_plans(idx, &round1, &plans, mapping.phrase_plan_index, docids, + decode_context, matches, options); +} + +Status phrase_prefix_query_impl(const LogicalIndexReader& idx, + const std::vector& terms, + std::vector* const docids, int32_t max_expansions, + format::PrxDecodeContext* decode_context, + CommonGramsPlanningTimer* planning_timer, + std::vector* matches) { + if (docids == nullptr && matches == nullptr) { + return Status::Error("phrase_prefix_query: null out"); + } + if (docids != nullptr) { + docids->clear(); + } + if (matches != nullptr) { + matches->clear(); + } + if (terms.empty()) { + return Status::OK(); + } + if (terms.size() == 1) { + DORIS_CHECK(matches == nullptr); + if (planning_timer != nullptr) { + planning_timer->finish(); + } + return prefix_query(idx, terms.front(), docids, max_expansions); + } + std::vector exact_terms; + exact_terms.reserve(terms.size() - 1); + std::string physical_term_scratch; + for (size_t i = 0; i + 1 < terms.size(); ++i) { + std::string_view physical_term; + bool representable = false; + RETURN_IF_ERROR(internal::route_plain_query_term_view(idx, terms[i], &physical_term_scratch, + &physical_term, &representable)); + if (!representable) { + return Status::OK(); + } + ResolvedQueryTerm resolved; + bool found = false; + RETURN_IF_ERROR(internal::resolve_query_term(idx, physical_term, &resolved, &found)); + if (!found) { + return Status::OK(); + } + exact_terms.push_back(std::move(resolved)); + } + + // Expand the tail in the logical plain namespace. The visitor range-seeks + // past typed internal namespaces before counting max_expansions and decodes + // escaped physical keys before applying the logical prefix. + std::vector tail_hits; + RETURN_IF_ERROR(internal::visit_expanded_plain_terms( + idx, terms.back(), [](std::string_view) { return true; }, + [&](LogicalIndexReader::PrefixHit&& hit, bool*) { + tail_hits.push_back(std::move(hit)); + return Status::OK(); + }, + max_expansions)); + if (tail_hits.empty()) { + return Status::OK(); + } + std::vector tail_terms; + tail_terms.reserve(tail_hits.size()); + for (auto& hit : tail_hits) { + tail_terms.push_back(ResolvedQueryTerm { + .entry = std::move(hit.entry), .frq_base = hit.frq_base, .prx_base = hit.prx_base}); + } + auto exact_plan = build_resolved_phrase_plan(std::move(exact_terms)); + if (planning_timer != nullptr) { + planning_timer->finish(); + } + DORIS_CHECK_LE(terms.size() - 1, static_cast(std::numeric_limits::max())); + return execute_resolved_phrase_prefix_terms(idx, std::move(exact_plan), std::move(tail_terms), + static_cast(terms.size() - 1), docids, + decode_context, matches); +} + +Status planned_phrase_prefix_query_impl( + const LogicalIndexReader& idx, const segment_v2::InvertedIndexQueryInfo& plain_query_info, + const segment_v2::InvertedIndexQueryInfo& gram_query_info, + const segment_v2::inverted_index::CommonGramsQueryIdentity* common_grams_identity, + std::vector* docids, int32_t max_expansions, + format::PrxDecodeContext* decode_context, PhrasePrefixPlanKind* selected_plan, + segment_v2::inverted_index::CommonGramsPlanCostModel cost_model, + CommonGramsPlanDebugOverride debug_override) { + if (docids == nullptr) { + return Status::Error( + "planned_phrase_prefix_query: null out"); + } + docids->clear(); + if (selected_plan != nullptr) { + *selected_plan = PhrasePrefixPlanKind::kPlain; + } + DORIS_CHECK(!plain_query_info.has_common_gram()); + auto* query_stats = decode_context == nullptr ? nullptr : decode_context->query_stats; + CommonGramsPlanningTimer planning_timer(query_stats); + if (query_stats != nullptr) { + ++query_stats->common_grams_candidate_queries; + } + if (plain_query_info.term_infos.empty()) { + if (query_stats != nullptr) { + ++query_stats->common_grams_plain_plans; + } + return Status::OK(); + } + for (const auto& term_info : plain_query_info.term_infos) { + DORIS_CHECK(term_info.is_single_term()); + } + + PhysicalPhrasePlan plain_leading; + bool plain_representable = false; + RETURN_IF_ERROR(build_physical_phrase_plan_prefix( + idx, plain_query_info, plain_query_info.term_infos.size() - 1, + /*allow_common_grams=*/false, &plain_leading, &plain_representable)); + if (!plain_representable) { + if (query_stats != nullptr) { + ++query_stats->common_grams_plain_plans; + } + return Status::OK(); + } + + const auto execute_plain = [&]() { + std::vector terms; + terms.reserve(plain_query_info.term_infos.size()); + for (const auto& term_info : plain_query_info.term_infos) { + terms.push_back(term_info.get_single_term()); + } + return phrase_prefix_query_impl(idx, terms, docids, max_expansions, decode_context, + &planning_timer); + }; + + const bool index_has_common_grams = has_common_grams_capability(idx, common_grams_identity); + + const bool can_plan_common_grams = gram_query_info.has_common_gram() && index_has_common_grams; + if (!can_plan_common_grams) { + if (query_stats != nullptr) { + ++query_stats->common_grams_plain_plans; + if (gram_query_info.has_common_gram()) { + ++query_stats->common_grams_fallback_incompatible; + } else { + ++query_stats->common_grams_fallback_no_gram; + } + } + return execute_plain(); + } + DORIS_CHECK(!gram_query_info.term_infos.empty()); + DORIS_CHECK(gram_query_info.term_infos.back().is_single_term()); + + PhysicalPhrasePlan gram_leading; + bool gram_representable = false; + RETURN_IF_ERROR(build_physical_phrase_plan_prefix( + idx, gram_query_info, gram_query_info.term_infos.size() - 1, + /*allow_common_grams=*/true, &gram_leading, &gram_representable)); + if (!gram_representable) { + if (query_stats != nullptr) { + ++query_stats->common_grams_plain_plans; + ++query_stats->common_grams_fallback_incompatible; + } + return execute_plain(); + } + + internal::ResolvedPhrasePlan selected_leading; + std::vector selected_tail_terms; + DORIS_CHECK_LE(plain_leading.phrase_plan_index.size(), + static_cast(std::numeric_limits::max())); + const uint32_t plain_tail_position_offset = + static_cast(plain_leading.phrase_plan_index.size()); + uint32_t selected_tail_position_offset = plain_tail_position_offset; + bool authoritative_empty = false; + bool execute_as_exact_phrase = false; + bool hybrid_executed = false; + const Status planning_status = [&]() -> Status { + const bool maps_tail_to_gram = + gram_query_info.term_infos.back().key_kind == segment_v2::TermKeyKind::kCommonGram; + std::vector logical_tail_terms; + std::vector plain_tail_terms; + std::vector tail_hits; + RETURN_IF_ERROR(internal::visit_expanded_plain_terms( + idx, plain_query_info.term_infos.back().get_single_term(), + [](std::string_view) { return true; }, + [&](LogicalIndexReader::PrefixHit&& hit, bool*) { + tail_hits.push_back(std::move(hit)); + return Status::OK(); + }, + max_expansions)); + if (tail_hits.empty()) { + if (query_stats != nullptr) { + ++query_stats->common_grams_plain_plans; + ++query_stats->common_grams_fallback_prefix_tail_empty; + ++query_stats->common_grams_authoritative_empty; + } + authoritative_empty = true; + return Status::OK(); + } + if (maps_tail_to_gram) { + logical_tail_terms.reserve(tail_hits.size()); + } else { + DORIS_CHECK(gram_query_info.term_infos.back().key_kind == + segment_v2::TermKeyKind::kPlain); + } + plain_tail_terms.reserve(tail_hits.size()); + for (auto& hit : tail_hits) { + if (maps_tail_to_gram) { + logical_tail_terms.push_back(std::move(hit.term)); + } + plain_tail_terms.push_back(ResolvedQueryTerm {.entry = std::move(hit.entry), + .frq_base = hit.frq_base, + .prx_base = hit.prx_base}); + } + + const auto select_plain_plan = [&]() -> Status { + if (query_stats != nullptr) { + ++query_stats->common_grams_plain_plans; + ++query_stats->common_grams_fallback_incompatible; + } + std::vector batch_terms = plain_leading.unique_terms; + std::ranges::sort(batch_terms); + std::vector resolved; + std::vector found; + RETURN_IF_ERROR( + internal::resolve_query_terms_batch(idx, batch_terms, &resolved, &found)); + if (!all_plan_terms_present(plain_leading, batch_terms, found)) { + if (query_stats != nullptr) { + ++query_stats->common_grams_authoritative_empty; + } + authoritative_empty = true; + return Status::OK(); + } + selected_leading = + materialize_resolved_phrase_plan(plain_leading, batch_terms, &resolved); + selected_tail_terms = std::move(plain_tail_terms); + selected_tail_position_offset = plain_tail_position_offset; + return Status::OK(); + }; + + std::vector mapped_tail_terms; + if (maps_tail_to_gram) { + DORIS_CHECK_GE(plain_query_info.term_infos.size(), 2U); + const std::string& left = + plain_query_info.term_infos[plain_query_info.term_infos.size() - 2] + .get_single_term(); + mapped_tail_terms.reserve(logical_tail_terms.size()); + for (const std::string& tail : logical_tail_terms) { + std::string gram; + auto encoded = + segment_v2::inverted_index::try_encode_common_gram(left, tail, &gram); + if (!encoded.has_value()) { + return std::move(encoded.error()); + } + if (!*encoded) { + return select_plain_plan(); + } + mapped_tail_terms.push_back(std::move(gram)); + } + } + + std::vector batch_terms = plain_leading.unique_terms; + batch_terms.insert(batch_terms.end(), gram_leading.unique_terms.begin(), + gram_leading.unique_terms.end()); + batch_terms.insert(batch_terms.end(), mapped_tail_terms.begin(), mapped_tail_terms.end()); + std::ranges::sort(batch_terms); + batch_terms.erase(std::unique(batch_terms.begin(), batch_terms.end()), batch_terms.end()); + + std::vector resolved; + std::vector found; + RETURN_IF_ERROR(internal::resolve_query_terms_batch(idx, batch_terms, &resolved, &found)); + const bool plain_present = all_plan_terms_present(plain_leading, batch_terms, found); + const bool gram_present = all_plan_terms_present(gram_leading, batch_terms, found); + if (!plain_present || !gram_present) { + if (query_stats != nullptr) { + if (plain_present) { + ++query_stats->common_grams_gram_plans; + ++query_stats->common_grams_authoritative_empty; + } else { + ++query_stats->common_grams_plain_plans; + ++query_stats->common_grams_authoritative_empty; + } + } + if (selected_plan != nullptr && plain_present) { + *selected_plan = PhrasePrefixPlanKind::kCommonGrams; + } + authoritative_empty = true; + return Status::OK(); + } + + std::vector present_mapped_tail_indices; + std::vector present_mapped_tails; + std::vector present_gram_tail_ordinals; + if (maps_tail_to_gram) { + present_mapped_tail_indices.reserve(mapped_tail_terms.size()); + present_mapped_tails.reserve(mapped_tail_terms.size()); + present_gram_tail_ordinals.reserve(mapped_tail_terms.size()); + for (size_t ordinal = 0; ordinal < mapped_tail_terms.size(); ++ordinal) { + const std::string& term = mapped_tail_terms[ordinal]; + const size_t batch_index = resolved_batch_index(batch_terms, term); + if (found[batch_index] != 0) { + present_mapped_tail_indices.push_back(batch_index); + DORIS_CHECK_LE(ordinal, + static_cast(std::numeric_limits::max())); + const uint32_t expansion_ordinal = static_cast(ordinal); + present_mapped_tails.push_back(ResolvedMappedTail { + .batch_index = batch_index, .expansion_ordinal = expansion_ordinal}); + present_gram_tail_ordinals.push_back(expansion_ordinal); + } + } + if (present_mapped_tail_indices.empty()) { + if (query_stats != nullptr) { + ++query_stats->common_grams_gram_plans; + ++query_stats->common_grams_authoritative_empty; + } + if (selected_plan != nullptr) { + *selected_plan = PhrasePrefixPlanKind::kCommonGrams; + } + authoritative_empty = true; + return Status::OK(); + } + } + + std::optional hybrid_artifact; + if (idx.common_grams_posting_policy() == format::CommonGramsPostingPolicy::kHybridV1) { + hybrid_artifact = try_build_hybrid_prefix_plan_artifact( + plain_leading, gram_leading, batch_terms, resolved, present_mapped_tails, + maps_tail_to_gram); + } + const bool hybrid_plan_requires_plain_verification = hybrid_artifact.has_value(); + const bool plain_needs_positions = !plain_leading.phrase_plan_index.empty(); + const bool gram_needs_positions = + !gram_leading.phrase_plan_index.empty() && !hybrid_plan_requires_plain_verification; + auto plain_raw_cost = phrase_plan_raw_cost(plain_leading, batch_terms, resolved, found, + plain_needs_positions); + const auto plain_tail_raw_cost = + alternative_clause_raw_cost(plain_tail_terms, plain_needs_positions); + append_alternative_clause_cost(plain_tail_raw_cost, &plain_raw_cost); + const uint64_t plain_cost = segment_v2::inverted_index::estimate_common_grams_plan_cost( + plain_raw_cost, plain_needs_positions ? cost_model.position_verify_factor : 0); + segment_v2::inverted_index::CommonGramsPlanRawCost gram_raw_cost; + uint64_t gram_cost = 0; + if (hybrid_plan_requires_plain_verification) { + const HybridPrefixCostEstimate hybrid_cost = estimate_hybrid_prefix_plan_cost( + *hybrid_artifact, batch_terms, resolved, found, plain_tail_terms, + cost_model.position_verify_factor); + gram_raw_cost = hybrid_cost.raw_cost; + gram_cost = hybrid_cost.estimated_cost; + } else { + gram_raw_cost = phrase_plan_raw_cost(gram_leading, batch_terms, resolved, found, + gram_needs_positions); + if (maps_tail_to_gram) { + append_alternative_clause_cost( + alternative_clause_raw_cost(resolved, present_mapped_tail_indices, + gram_needs_positions), + &gram_raw_cost); + } else { + append_alternative_clause_cost(plain_tail_raw_cost, &gram_raw_cost); + } + gram_cost = segment_v2::inverted_index::estimate_common_grams_plan_cost( + gram_raw_cost, gram_needs_positions ? cost_model.position_verify_factor : 0); + } + const bool cost_prefers_gram = segment_v2::inverted_index::common_grams_plan_cost_wins( + plain_cost, gram_cost, cost_model.common_grams_cost_ratio_percent); + const bool use_gram = + apply_common_grams_plan_debug_override(cost_prefers_gram, debug_override); + if (query_stats != nullptr) { + query_stats->common_grams_plain_posting_bytes += plain_raw_cost.posting_bytes_or_df_sum; + query_stats->common_grams_gram_posting_bytes += gram_raw_cost.posting_bytes_or_df_sum; + query_stats->common_grams_plain_estimated_candidate_df += + plain_raw_cost.estimated_candidate_df; + query_stats->common_grams_gram_estimated_candidate_df += + gram_raw_cost.estimated_candidate_df; + query_stats->common_grams_plain_estimated_cost += plain_cost; + query_stats->common_grams_gram_estimated_cost += gram_cost; + } + if (query_stats != nullptr) { + if (use_gram) { + ++query_stats->common_grams_gram_plans; + } else { + ++query_stats->common_grams_plain_plans; + if (debug_override == CommonGramsPlanDebugOverride::kNone) { + ++query_stats->common_grams_fallback_cost; + } + } + } + if (selected_plan != nullptr) { + *selected_plan = + use_gram ? PhrasePrefixPlanKind::kCommonGrams : PhrasePrefixPlanKind::kPlain; + } + + const bool hybrid_needs_plain_verification = + use_gram && hybrid_plan_requires_plain_verification; + if (hybrid_needs_plain_verification) { + DORIS_CHECK(hybrid_artifact.has_value()); + bool candidate_intersection_empty = false; + RETURN_IF_ERROR(execute_hybrid_phrase_prefix_plan( + idx, *hybrid_artifact, batch_terms, resolved, plain_tail_terms, docids, + decode_context, planning_timer, &candidate_intersection_empty)); + hybrid_executed = true; + if (candidate_intersection_empty) { + if (query_stats != nullptr) { + ++query_stats->common_grams_authoritative_empty; + } + authoritative_empty = true; + return Status::OK(); + } + return Status::OK(); + } + + const PhysicalPhrasePlan& selected_physical_leading = + use_gram ? gram_leading : plain_leading; + selected_leading = + materialize_resolved_phrase_plan(selected_physical_leading, batch_terms, &resolved); + if (!use_gram || !maps_tail_to_gram) { + selected_tail_terms = std::move(plain_tail_terms); + selected_tail_position_offset = plain_tail_position_offset; + return Status::OK(); + } + + if (present_mapped_tail_indices.size() == 1) { + DORIS_CHECK_GT(plain_tail_position_offset, 0U); + const uint32_t mapped_tail_position = plain_tail_position_offset - 1; + const size_t batch_index = present_mapped_tail_indices.front(); + const auto existing = + std::ranges::find(selected_leading.unique_terms, batch_terms[batch_index], + [](const ResolvedQueryTerm& term) { + return std::string_view(term.entry.term); + }); + if (existing == selected_leading.unique_terms.end()) { + append_resolved_phrase_clause(std::move(resolved[batch_index]), + mapped_tail_position, &selected_leading); + } else { + selected_leading.phrase_plan_index.push_back( + static_cast(existing - selected_leading.unique_terms.begin())); + selected_leading.position_offsets.push_back(mapped_tail_position); + } + execute_as_exact_phrase = true; + return Status::OK(); + } + + selected_tail_terms.reserve(present_mapped_tail_indices.size()); + for (size_t batch_index : present_mapped_tail_indices) { + const auto existing = + std::ranges::find(selected_leading.unique_terms, batch_terms[batch_index], + [](const ResolvedQueryTerm& term) { + return std::string_view(term.entry.term); + }); + if (existing == selected_leading.unique_terms.end()) { + selected_tail_terms.push_back(std::move(resolved[batch_index])); + } else { + selected_tail_terms.push_back(*existing); + } + } + DORIS_CHECK_GT(plain_tail_position_offset, 0U); + selected_tail_position_offset = plain_tail_position_offset - 1; + return Status::OK(); + }(); + RETURN_IF_ERROR(planning_status); + planning_timer.finish(); + if (authoritative_empty) { + return Status::OK(); + } + if (hybrid_executed) { + return Status::OK(); + } + if (execute_as_exact_phrase) { + return internal::execute_resolved_phrase_plan(idx, std::move(selected_leading), docids, + decode_context); + } + return execute_resolved_phrase_prefix_terms( + idx, std::move(selected_leading), std::move(selected_tail_terms), + selected_tail_position_offset, docids, decode_context); +} + +} // namespace doris::snii::query::phrase_impl diff --git a/be/src/storage/index/snii/query/phrase_position_source.cpp b/be/src/storage/index/snii/query/phrase_position_source.cpp new file mode 100644 index 00000000000000..d48aaf31a82256 --- /dev/null +++ b/be/src/storage/index/snii/query/phrase_position_source.cpp @@ -0,0 +1,395 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/check.h" +#include "storage/index/inverted/common_grams/common_grams_key_codec.h" +#include "storage/index/inverted/common_grams/common_grams_query_cost.h" +#include "storage/index/inverted/common_grams/common_grams_segment_metadata.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/encoding/byte_source.h" +#include "storage/index/snii/format/dict_entry.h" +#include "storage/index/snii/format/frq_pod.h" +#include "storage/index/snii/format/frq_prelude.h" +#include "storage/index/snii/format/prx_pod.h" +#include "storage/index/snii/io/batch_range_fetcher.h" +#include "storage/index/snii/query/internal/docid_conjunction.h" +#include "storage/index/snii/query/internal/docid_posting_reader.h" +#include "storage/index/snii/query/internal/docid_set_ops.h" +#include "storage/index/snii/query/internal/docid_union.h" +#include "storage/index/snii/query/internal/phrase_query_split.h" +#include "storage/index/snii/query/internal/plain_term_routing.h" +#include "storage/index/snii/query/internal/position_math.h" +#include "storage/index/snii/query/internal/query_test_counters.h" +#include "storage/index/snii/query/internal/resolved_phrase_plan.h" +#include "storage/index/snii/query/internal/term_expansion.h" +#include "storage/index/snii/query/phrase_prx_validation.h" +#include "storage/index/snii/query/phrase_query.h" +#include "storage/index/snii/query/phrase_verify_timer.h" +#include "storage/index/snii/query/prefix_query.h" +#include "storage/index/snii/query/term_query.h" +#include "storage/index/snii/reader/windowed_posting.h" +#include "util/debug_points.h" + +namespace doris::snii::query::phrase_impl { + +using query::internal::DocidChunk; +using query::internal::DocidSource; +using query::internal::ResolvedQueryTerm; +using query::internal::TermPlan; +using reader::LogicalIndexReader; +using internal::PhraseVerifyTimer; + +PhraseTermMapping build_phrase_term_mapping(const std::vector& terms) { + PhraseTermMapping mapping; + mapping.phrase_plan_index.reserve(terms.size()); + for (const std::string& term : terms) { + auto it = std::ranges::find(mapping.unique_terms, term); + if (it == mapping.unique_terms.end()) { + mapping.phrase_plan_index.push_back(mapping.unique_terms.size()); + mapping.unique_terms.push_back(term); + continue; + } + mapping.phrase_plan_index.push_back(static_cast(it - mapping.unique_terms.begin())); + } + return mapping; +} + +namespace { +Status append_prx_doc_ordinal(size_t ordinal, std::vector* out) { + if (ordinal > std::numeric_limits::max()) { + return Status::Error( + "phrase_query: prx doc ordinal exceeds u32"); + } + out->push_back(static_cast(ordinal)); + return Status::OK(); +} + +Status append_selected_ordinal(size_t doc_index, const std::vector& prx_doc_ordinals, + std::vector* selected_ordinals) { + if (!prx_doc_ordinals.empty()) { + selected_ordinals->push_back(prx_doc_ordinals[doc_index]); + return Status::OK(); + } + return append_prx_doc_ordinal(doc_index, selected_ordinals); +} + +Status append_selected_doc(size_t doc_index, uint32_t docid, + const std::vector& prx_doc_ordinals, + std::vector* selected_docids, + std::vector* selected_ordinals) { + selected_docids->push_back(docid); + return append_selected_ordinal(doc_index, prx_doc_ordinals, selected_ordinals); +} + +Status materialize_selected_prefix(size_t count, size_t capacity, + const std::vector& docids, + const std::vector& prx_doc_ordinals, + std::vector* selected_docids, + std::vector* selected_ordinals) { + selected_docids->reserve(capacity); + selected_ordinals->reserve(capacity); + selected_docids->insert(selected_docids->end(), docids.begin(), docids.begin() + count); + for (size_t i = 0; i < count; ++i) { + RETURN_IF_ERROR(append_selected_ordinal(i, prx_doc_ordinals, selected_ordinals)); + } + return Status::OK(); +} + +Status materialize_selected_prefix_if_needed(bool* selected_all, size_t count, size_t capacity, + const std::vector& docids, + const std::vector& prx_doc_ordinals, + std::vector* selected_docids, + std::vector* selected_ordinals) { + if (!*selected_all) { + return Status::OK(); + } + *selected_all = false; + return materialize_selected_prefix(count, capacity, docids, prx_doc_ordinals, selected_docids, + selected_ordinals); +} + +Status select_candidate_docs_for_prx(std::vector* docids, + std::vector* prx_doc_ordinals, + uint32_t prx_doc_count, + const std::vector& candidates, PosChunk* chunk) { + chunk->docids.clear(); + chunk->prx_doc_ordinals.clear(); + if (prx_doc_count == 0 && docids->size() > std::numeric_limits::max()) { + return Status::Error( + "phrase_query: prx doc count exceeds u32"); + } + chunk->prx_doc_count = + prx_doc_count == 0 ? static_cast(docids->size()) : prx_doc_count; + if (docids->empty() || candidates.empty()) { + return Status::OK(); + } + if (!prx_doc_ordinals->empty() && prx_doc_ordinals->size() != docids->size()) { + return Status::Error( + "phrase_query: prx ordinal/docid count mismatch"); + } + + std::vector selected_docids; + std::vector selected_ordinals; + bool selected_all = true; + const size_t selected_capacity = std::min(docids->size(), candidates.size()); + + auto candidate_it = std::ranges::lower_bound(candidates, docids->front()); + size_t candidate_index = static_cast(candidate_it - candidates.begin()); + for (size_t doc_index = 0; doc_index < docids->size(); ++doc_index) { + const uint32_t docid = (*docids)[doc_index]; + while (candidate_index < candidates.size() && candidates[candidate_index] < docid) { + ++candidate_index; + } + if (candidate_index == candidates.size()) { + RETURN_IF_ERROR(materialize_selected_prefix_if_needed( + &selected_all, doc_index, selected_capacity, *docids, *prx_doc_ordinals, + &selected_docids, &selected_ordinals)); + break; + } + if (candidates[candidate_index] != docid) { + RETURN_IF_ERROR(materialize_selected_prefix_if_needed( + &selected_all, doc_index, selected_capacity, *docids, *prx_doc_ordinals, + &selected_docids, &selected_ordinals)); + continue; + } + + if (!selected_all) { + RETURN_IF_ERROR(append_selected_doc(doc_index, docid, *prx_doc_ordinals, + &selected_docids, &selected_ordinals)); + } + ++candidate_index; + } + + if (selected_all) { + chunk->docids = std::move(*docids); + chunk->prx_doc_ordinals = std::move(*prx_doc_ordinals); + docids->clear(); + prx_doc_ordinals->clear(); + return Status::OK(); + } + if (selected_docids.empty()) { + return Status::OK(); + } + chunk->docids = std::move(selected_docids); + chunk->prx_doc_ordinals = std::move(selected_ordinals); + return Status::OK(); +} + +// PRX byte ranges for every candidate-bearing chunk across all phrase terms are +// added to one shared BatchRangeFetcher and fetched in a single batched round +// (T02). Pass 1 records, for each chunk that needs on-disk PRX bytes, where to +// write the fetched slice back: which plan's PosSource, which chunk within it, +// and the fetcher handle. + +struct PrxRangeAssignment { + size_t plan_index; + size_t chunk_index; + size_t handle; +}; + +void record_prx_assignment(std::vector* assignments, size_t plan_index, + size_t chunk_index, size_t handle) { + assignments->push_back(PrxRangeAssignment { + .plan_index = plan_index, .chunk_index = chunk_index, .handle = handle}); +} + +Status build_flat_position_source(const LogicalIndexReader& idx, + const io::BatchRangeFetcher& round1, DocidSource* doc_source, + const TermPlan& p, const std::vector& candidates, + size_t plan_index, io::BatchRangeFetcher* prx_fetcher, + std::vector* assignments, PosSource* src) { + PosChunk chunk; + std::vector docids; + std::vector prx_doc_ordinals; + const bool docids_are_final_candidates = + doc_source->docids_are_final_candidates && !doc_source->chunks.empty(); + if (!doc_source->chunks.empty()) { + DocidChunk& doc_chunk = doc_source->chunks.front(); + docids = std::move(doc_chunk.docids); + prx_doc_ordinals = std::move(doc_chunk.prx_doc_ordinals); + chunk.prx_doc_count = doc_chunk.prx_doc_count; + } + // pod_ref PRX bytes are read from the shared fetcher (one batched round for the + // whole phrase); inline PRX bytes already live in the dict entry. The pod_ref + // range is added unconditionally to keep the bytes read identical to the prior + // per-term fetch(); the handle is only recorded as an assignment when the chunk + // is kept (an empty chunk reads the same bytes but needs no backfill). + bool has_prx_handle = false; + size_t prx_handle = 0; + if (p.pod_ref) { + uint64_t poff = 0; + uint64_t plen = 0; + RETURN_IF_ERROR(idx.resolve_prx_window(p.entry, p.prx_base, &poff, &plen)); + prx_handle = prx_fetcher->add(poff, plen); + has_prx_handle = true; + } else { + chunk.prx = Slice(p.entry.prx_bytes); + } + if (docids.empty()) { + Slice dd; + if (p.pod_ref) { + dd = round1.get(p.frq_handle); + } else { + RETURN_IF_ERROR(internal::inline_dd_region(p.entry, &dd)); + } + RETURN_IF_ERROR(format::decode_dd_region(dd, p.entry.dd_meta, + /*win_base=*/0, &docids)); + if (docids.size() > std::numeric_limits::max()) { + return Status::Error( + "phrase_query: prx doc count exceeds u32"); + } + chunk.prx_doc_count = static_cast(docids.size()); + } + if (docids_are_final_candidates) { + chunk.docids = std::move(docids); + chunk.prx_doc_ordinals = std::move(prx_doc_ordinals); + if (!chunk.docids.empty()) { + if (has_prx_handle) { + record_prx_assignment(assignments, plan_index, src->chunks.size(), prx_handle); + } + src->chunks.push_back(std::move(chunk)); + } + return Status::OK(); + } + RETURN_IF_ERROR(select_candidate_docs_for_prx(&docids, &prx_doc_ordinals, chunk.prx_doc_count, + candidates, &chunk)); + if (!chunk.docids.empty()) { + if (has_prx_handle) { + record_prx_assignment(assignments, plan_index, src->chunks.size(), prx_handle); + } + src->chunks.push_back(std::move(chunk)); + } + return Status::OK(); +} + +bool chunk_may_contain_candidate(const DocidChunk& chunk, const std::vector& candidates) { + if (chunk.docids.empty() || candidates.empty()) { + return false; + } + const auto it = std::ranges::lower_bound(candidates, chunk.docids.front()); + return it != candidates.end() && *it <= chunk.docids.back(); +} + +Status decode_windowed_position_source(const LogicalIndexReader& idx, const TermPlan& p, + DocidSource* doc_source, + const std::vector& candidates, size_t plan_index, + io::BatchRangeFetcher* prx_fetcher, + std::vector* assignments, + PosSource* src) { + for (size_t i = 0; i < doc_source->chunks.size(); ++i) { + DocidChunk& doc_chunk = doc_source->chunks[i]; + if (!doc_source->docids_are_final_candidates && + !chunk_may_contain_candidate(doc_chunk, candidates)) { + continue; + } + if (!doc_chunk.windowed) { + return Status::Error( + "phrase_query: expected windowed doc chunk"); + } + PosChunk chunk; + if (doc_source->docids_are_final_candidates) { + chunk.docids = std::move(doc_chunk.docids); + chunk.prx_doc_ordinals = std::move(doc_chunk.prx_doc_ordinals); + chunk.prx_doc_count = doc_chunk.prx_doc_count; + } else { + RETURN_IF_ERROR( + select_candidate_docs_for_prx(&doc_chunk.docids, &doc_chunk.prx_doc_ordinals, + doc_chunk.prx_doc_count, candidates, &chunk)); + } + if (chunk.docids.empty()) { + continue; + } + + reader::WindowAbsRange range; + RETURN_IF_ERROR(reader::windowed_window_range( + idx, p.entry, p.frq_base, p.prx_base, p.prelude, doc_chunk.window, + /*want_positions=*/true, /*want_freq=*/false, &range)); + chunk.windowed = true; + chunk.window = doc_chunk.window; + const size_t prx_handle = prx_fetcher->add(range.prx_off, range.prx_len); + record_prx_assignment(assignments, plan_index, src->chunks.size(), prx_handle); + src->chunks.push_back(std::move(chunk)); + } + return Status::OK(); +} + +} // namespace +Status build_position_sources_for_candidates( + const LogicalIndexReader& idx, const io::BatchRangeFetcher& round1, + const std::vector& plans, std::vector* doc_sources, + const std::vector& candidates, + std::vector>* owners, std::vector* srcs, + format::PrxDecodeContext* observer_context) { + srcs->assign(plans.size(), PosSource {}); + for (PosSource& source : *srcs) { + source.observer_context = observer_context; + } + // All phrase terms share one PRX fetcher: pass 1 adds every candidate-bearing + // chunk's PRX range and records a backfill assignment; a single fetch() then + // issues one batched read (one serial round on a remote reader); pass 2 fills + // in each chunk's PRX slice. This collapses the prior per-term fetch() -- O(n) + // serial remote rounds for an n-term phrase -- into one. + auto prx_fetcher = + std::make_unique(idx.reader(), reader::kSameTermCoalesceGap); + std::vector assignments; + for (size_t i = 0; i < plans.size(); ++i) { + const TermPlan& p = plans[i]; + if (p.windowed) { + RETURN_IF_ERROR(decode_windowed_position_source(idx, p, &(*doc_sources)[i], candidates, + i, prx_fetcher.get(), &assignments, + &(*srcs)[i])); + continue; + } + RETURN_IF_ERROR(build_flat_position_source(idx, round1, &(*doc_sources)[i], p, candidates, + i, prx_fetcher.get(), &assignments, + &(*srcs)[i])); + } + if (prx_fetcher->pending() > 0) { + if (observer_context == nullptr || observer_context->stats == nullptr) { + RETURN_IF_ERROR(prx_fetcher->fetch()); + } else { + const auto fetch_start = std::chrono::steady_clock::now(); + RETURN_IF_ERROR(prx_fetcher->fetch()); + const auto elapsed = std::chrono::duration_cast( + std::chrono::steady_clock::now() - fetch_start) + .count(); + observer_context->stats->fetch_ns += + std::max(1, static_cast(elapsed)); + } + } + for (const PrxRangeAssignment& a : assignments) { + (*srcs)[a.plan_index].chunks[a.chunk_index].prx = prx_fetcher->get(a.handle); + } + // Keep the fetcher alive only when some chunk slice references its buffers. + if (!assignments.empty()) { + owners->push_back(std::move(prx_fetcher)); + } + return Status::OK(); +} + +} // namespace doris::snii::query::phrase_impl diff --git a/be/src/storage/index/snii/query/phrase_prefix_exec.cpp b/be/src/storage/index/snii/query/phrase_prefix_exec.cpp new file mode 100644 index 00000000000000..2a580715b8ba7d --- /dev/null +++ b/be/src/storage/index/snii/query/phrase_prefix_exec.cpp @@ -0,0 +1,719 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/check.h" +#include "storage/index/inverted/common_grams/common_grams_key_codec.h" +#include "storage/index/inverted/common_grams/common_grams_query_cost.h" +#include "storage/index/inverted/common_grams/common_grams_segment_metadata.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/encoding/byte_source.h" +#include "storage/index/snii/format/dict_entry.h" +#include "storage/index/snii/format/frq_pod.h" +#include "storage/index/snii/format/frq_prelude.h" +#include "storage/index/snii/format/prx_pod.h" +#include "storage/index/snii/io/batch_range_fetcher.h" +#include "storage/index/snii/query/internal/docid_conjunction.h" +#include "storage/index/snii/query/internal/docid_posting_reader.h" +#include "storage/index/snii/query/internal/docid_set_ops.h" +#include "storage/index/snii/query/internal/docid_union.h" +#include "storage/index/snii/query/internal/phrase_query_split.h" +#include "storage/index/snii/query/internal/plain_term_routing.h" +#include "storage/index/snii/query/internal/position_math.h" +#include "storage/index/snii/query/internal/query_test_counters.h" +#include "storage/index/snii/query/internal/resolved_phrase_plan.h" +#include "storage/index/snii/query/internal/term_expansion.h" +#include "storage/index/snii/query/phrase_prx_validation.h" +#include "storage/index/snii/query/phrase_query.h" +#include "storage/index/snii/query/phrase_verify_timer.h" +#include "storage/index/snii/query/prefix_query.h" +#include "storage/index/snii/query/term_query.h" +#include "storage/index/snii/reader/windowed_posting.h" +#include "util/debug_points.h" + +namespace doris::snii::query::phrase_impl { + +using query::internal::DocidChunk; +using query::internal::DocidSource; +using query::internal::ResolvedQueryTerm; +using query::internal::TermPlan; +using reader::LogicalIndexReader; +using internal::PhraseVerifyTimer; + +namespace { +Status collect_expected_tail_positions(const std::vector& plans, + const std::vector& phrase_plan_index, + const std::vector& position_offsets, + std::vector& srcs, + const std::vector& candidates, + ExpectedTailPositionSet* out, + bool preserve_first_clause_multiplicity) { + const size_t n = phrase_plan_index.size(); + DCHECK(n > 1); + DCHECK_EQ(plans.size(), srcs.size()); + std::vector cur(plans.size()); + for (size_t i = 0; i < plans.size(); ++i) { + cur[i].init(&srcs[i]); + } + + std::vector> unique_span(plans.size()); + std::vector> span(n); + for (uint32_t d : candidates) { + for (size_t i = 0; i < plans.size(); ++i) { + RETURN_IF_ERROR(cur[i].seek(d)); + RETURN_IF_ERROR(cur[i].positions(&unique_span[plans[i].order])); + } + for (size_t i = 0; i < n; ++i) { + DCHECK_LT(phrase_plan_index[i], unique_span.size()); + span[i] = unique_span[phrase_plan_index[i]]; + } + + // Anchor the outer enumeration on the SPARSEST exact term (smallest + // per-doc position span), not the hardcoded phrase-position-0 term. The + // set of valid phrase starts is anchor-independent -- each valid start + // maps 1:1 to exactly one anchor position (anchor_pos = start + + // offset[anchor]) -- so enumerating the shortest span and binary-searching + // the others yields the identical result set with the fewest outer + // iterations. A leading high-frequency exact term no longer forces + // O(|span[0]|) work per candidate doc. + size_t anchor = 0; + auto best = position_span_size(span[0]); + if (!preserve_first_clause_multiplicity) { + for (size_t t = 1; t < n; ++t) { + const auto sz = position_span_size(span[t]); + if (sz < best) { + best = sz; + anchor = t; + } + } + } + const uint32_t anchor_off = position_offsets[anchor]; + SNII_QUERY_ADD(anchor_iterations, best); + + // Only the first non-anchor term is probed for every viable anchor. + // Later terms can be skipped after an earlier mismatch, so using the + // total anchor count to choose a forward scan for them could turn one + // binary lookup into a full-span walk. The first term uses a forward + // scan only when all anchors are valid and its conservative comparison + // upper bound is at most half that of repeated binary search. + const size_t first_checked_term = anchor == 0 ? 1 : 0; + size_t monotonic_position_scan_term = n; + if (should_use_monotonic_position_scan(span[anchor], + position_span_size(span[first_checked_term]), + anchor_off, position_offsets[first_checked_term])) { + monotonic_position_scan_term = first_checked_term; + SNII_QUERY_COUNT(monotonic_position_scans); + } + + const size_t expected_begin = out->positions.size(); + for (const uint32_t* p = span[anchor].first; p != span[anchor].second; ++p) { + const uint32_t anchor_pos = *p; + // Underflow guard: a general anchor (offset > 0) can sit at a position + // smaller than its offset, which would wrap `start`. Such a position + // admits no valid phrase start and is skipped. (The old span[0] anchor + // had offset 0 and could never underflow.) + if (anchor_pos < anchor_off) { + continue; + } + const uint32_t start = anchor_pos - anchor_off; + bool ok = true; + for (size_t t = 0; t < n; ++t) { + if (t == anchor) { + continue; // the anchor term's position is satisfied by construction + } + uint32_t want = 0; + if (!internal::add_position_offset(start, position_offsets[t], &want)) { + ok = false; + break; + } + if (t == monotonic_position_scan_term) { + while (span[t].first != span[t].second && *span[t].first < want) { + ++span[t].first; + } + if (span[t].first == span[t].second || *span[t].first != want) { + ok = false; + break; + } + } else if (!std::binary_search(span[t].first, span[t].second, want)) { + ok = false; + break; + } + } + uint32_t tail_pos = 0; + if (ok && internal::add_position_offset(start, position_offsets[n], &tail_pos)) { + out->positions.push_back(tail_pos); + } + } + const size_t expected_end = out->positions.size(); + if (expected_end != expected_begin) { + out->docs.push_back(ExpectedTailPositions { + .docid = d, .positions_begin = expected_begin, .positions_end = expected_end}); + } + } + return Status::OK(); +} + +Status collect_single_term_expected_tail_positions(std::vector& srcs, + const std::vector& candidates, + uint32_t tail_offset, + ExpectedTailPositionSet* out) { + PostingCursor cursor; + cursor.init(srcs.data()); + out->reserve_docs(out->docs.size() + candidates.size()); + + for (uint32_t d : candidates) { + RETURN_IF_ERROR(cursor.seek(d)); + std::pair span; + RETURN_IF_ERROR(cursor.positions(&span)); + + const size_t expected_begin = out->positions.size(); + for (const uint32_t* p = span.first; p != span.second; ++p) { + uint32_t tail_pos = 0; + if (internal::add_position_offset(*p, tail_offset, &tail_pos)) { + out->positions.push_back(tail_pos); + } + } + const size_t expected_end = out->positions.size(); + if (expected_end != expected_begin) { + out->docs.push_back(ExpectedTailPositions { + .docid = d, .positions_begin = expected_begin, .positions_end = expected_end}); + } + } + return Status::OK(); +} + +Status collect_expected_tail_positions(const LogicalIndexReader& idx, + internal::ResolvedPhrasePlan exact_plan, + uint32_t tail_position_offset, ExpectedTailPositionSet* out, + const std::vector* candidate_prefilter, + format::PrxDecodeContext* observer_context, + bool preserve_first_clause_multiplicity) { + out->clear(); + io::BatchRangeFetcher round1(idx.reader()); + std::vector plans; + RETURN_IF_ERROR(internal::plan_resolved_terms(idx, std::move(exact_plan.unique_terms), &round1, + &plans, + /*need_positions=*/false)); + + PhraseExecutionState state; + RETURN_IF_ERROR(build_phrase_execution_state(idx, &round1, &plans, &state, candidate_prefilter, + observer_context, + PhraseCandidateMetric::kPrefixLeading)); + if (state.candidates.empty()) { + return Status::OK(); + } + out->reserve_docs(state.candidates.size()); + DORIS_CHECK(!exact_plan.position_offsets.empty()); + DORIS_CHECK_GT(tail_position_offset, exact_plan.position_offsets.back()); + std::vector position_offsets = std::move(exact_plan.position_offsets); + position_offsets.push_back(tail_position_offset); + PhraseVerifyTimer verify_timer(observer_context); + if (exact_plan.phrase_plan_index.size() == 1) { + DORIS_CHECK_LT(position_offsets[0], position_offsets[1]); + RETURN_IF_ERROR(collect_single_term_expected_tail_positions( + state.srcs, state.candidates, position_offsets[1] - position_offsets[0], out)); + } else { + RETURN_IF_ERROR(collect_expected_tail_positions( + plans, exact_plan.phrase_plan_index, position_offsets, state.srcs, state.candidates, + out, preserve_first_clause_multiplicity)); + } + verify_timer.commit_success(); + return Status::OK(); +} + +bool contains_any_position(const ExpectedTailPositionSet& expected, + const ExpectedTailPositions& wanted, + std::pair actual) { + for (size_t i = wanted.positions_begin; i < wanted.positions_end; ++i) { + if (std::binary_search(actual.first, actual.second, expected.positions[i])) { + return true; + } + } + return false; +} + +uint32_t mark_matching_positions(ExpectedTailPositionSet* expected, + const ExpectedTailPositions& wanted, + std::pair actual) { + DCHECK_EQ(expected->position_matched.size(), expected->positions.size()); + size_t expected_index = wanted.positions_begin; + const uint32_t* actual_position = actual.first; + uint32_t added = 0; + while (expected_index < wanted.positions_end && actual_position != actual.second) { + const uint32_t expected_position = expected->positions[expected_index]; + if (expected_position < *actual_position) { + ++expected_index; + continue; + } + if (*actual_position < expected_position) { + ++actual_position; + continue; + } + const size_t expected_run_end = static_cast( + std::upper_bound(expected->positions.begin() + expected_index, + expected->positions.begin() + wanted.positions_end, + expected_position) - + expected->positions.begin()); + while (expected_index < expected_run_end) { + if (expected->position_matched[expected_index] == 0) { + expected->position_matched[expected_index] = 1; + DCHECK_NE(added, std::numeric_limits::max()); + ++added; + } + ++expected_index; + } + actual_position = std::upper_bound(actual_position, actual.second, expected_position); + } + return added; +} + +// Upper bound on prefix expansions whose position cursors are held resident at +// once. The old per-tail loop verified a single expansion at a time (one +// PosSource + one PRX buffer live); the merged sweep below holds up to this many +// tail PosSources + cursors + PRX buffers simultaneously so it can read every +// tail's docid/prx bytes in ONE batched round and verify them in a single +// forward pass. `max_expansions` may be unbounded (<= 0), so this hard cap keeps +// resident memory bounded independent of the query: expansions beyond the cap +// are processed as additional capped groups (each a fresh single fetch) whose +// matched-doc flags are accumulated. The cap is tightened because each cursor +// here also holds decoded PRX rather than plain docids. +constexpr size_t kMaxTailMergeBatch = 32; + +// Phrase-prefix only reads the residual tails' docid union to prefilter the +// leading-phrase candidate set when the smallest leading term's df reaches this +// -- i.e. when the leading candidate set is large enough that decoding all its +// positions dwarfs an extra docid-only union read. Scale the threshold with the +// segment so a fixed absolute gate does not disable the prefilter after rowset +// segmentation, while retaining the original 1<<16 cap for large segments. +constexpr uint32_t kMinPrefixLeadingPrefilterMinDf = 256; +constexpr uint32_t kMaxPrefixLeadingPrefilterMinDf = 1u << 16; +constexpr uint32_t kPrefixLeadingPrefilterDocFraction = 8; +// The tail union is decoded once for filtering and again for verification. +// Require enough leading PRX work to cover both reads and union overhead. +constexpr uint32_t kPrefixLeadingToTailDfRatio = 8; + +uint32_t prefix_leading_prefilter_min_df(const LogicalIndexReader& idx, + bool allow_segment_relative_gate) { + if (!allow_segment_relative_gate) { + return kMaxPrefixLeadingPrefilterMinDf; + } + const uint64_t segment_relative = + idx.stats().indexed_doc_count / kPrefixLeadingPrefilterDocFraction; + return static_cast(std::clamp( + segment_relative, kMinPrefixLeadingPrefilterMinDf, kMaxPrefixLeadingPrefilterMinDf)); +} + +// Merged multi-tail verification for ONE resident-capped group of prefix +// expansions (`tails`, already truncated by max_expansions upstream). This +// replaces the per-tail verify-then-union loop: instead of re-planning + TWO +// remote rounds (docid, then prx) + a separate doc-walk PER tail and unioning N +// result lists, it plans every tail into ONE shared round1 fetch, intersects +// each tail with `expected_docids` in memory (no I/O), builds every surviving +// tail's position source in ONE batched PRX round, then sweeps the group's tail +// cursors over the ascending `expected` docs a SINGLE time -- marking a doc as +// soon as ANY tail has a position adjacent to a leading match. +// +// The marked set is byte-identical to the per-tail path's +// UNION_{tail in group} { d : d in tail INTERSECT expected AND +// contains_any_position(expected, doc_d, pos_tail(d)) } +// because each tail's PosSource is still built from its OWN final-candidate +// docids (the shared-candidate argument is ignored for final-candidate sources), +// so pos_tail(d) and the per-doc position test are unchanged. Only the I/O rounds +// (2N -> 2) and the N separate unions (-> in-place flags) collapse. Bigram +// postings are NEVER consulted: every tail is verified against its unigram +// positions here. + +Status collect_merged_tail_matches(const LogicalIndexReader& idx, + std::vector tails, + ExpectedTailPositionSet* expected, + const std::vector& expected_docids, bool final_group, + std::vector* final_matches, + format::PrxDecodeContext* observer_context, + std::vector* frequency_matches) { + DCHECK(expected != nullptr); + DCHECK(final_matches != nullptr || frequency_matches != nullptr); + const size_t n = tails.size(); + if (n == 0 || expected->docs.empty()) { + return Status::OK(); + } + + // Plan every tail into ONE fetcher so their docid postings + windowed + // preludes are read in a single batched round (the per-tail path issued one + // round per tail). Each tail keeps its own single-term plan vector so the + // conjunction filter below consumes it directly, without slicing a shared + // plan vector (whose prelude readers own decoded directory buffers). + io::BatchRangeFetcher round1(idx.reader()); + std::vector> tail_plans(n); + for (size_t i = 0; i < n; ++i) { + std::vector one; + one.push_back(std::move(tails[i])); + RETURN_IF_ERROR(internal::plan_resolved_terms(idx, std::move(one), &round1, &tail_plans[i], + /*need_positions=*/false)); + } + if (round1.pending() > 0) { + RETURN_IF_ERROR(round1.fetch()); + } + for (size_t i = 0; i < n; ++i) { + RETURN_IF_ERROR(internal::open_preludes(round1, &tail_plans[i], + /*need_positions=*/true)); + } + + // Per-tail candidate docids (tail posting INTERSECT expected) and the aligned + // final-candidate doc sources feeding the batched position builder. The + // conjunction reads only already-fetched round1 bytes; a single-plan filter + // marks its one source docids_are_final_candidates, so the position builder + // materializes each tail's PosSource directly over its own candidate docs. + // Tails whose intersection is empty are dropped here (exactly as the old + // per-tail early return did), so no dead tail decodes its full posting. + std::vector> tail_candidates(n); + std::vector active_plans; + std::vector active_sources; + std::vector active_index; // active slot -> tail index (into tail_candidates) + for (size_t i = 0; i < n; ++i) { + std::vector tail_source; + RETURN_IF_ERROR(internal::filter_docids_by_conjunction( + idx, round1, tail_plans[i], expected_docids, &tail_candidates[i], &tail_source)); + if (tail_candidates[i].empty()) { + continue; // this expansion has no doc in the expected set: nothing to verify + } + active_plans.push_back(std::move(tail_plans[i].front())); + active_sources.push_back(tail_source.empty() ? DocidSource {} + : std::move(tail_source.front())); + active_index.push_back(i); + } + if (active_plans.empty() && (!final_group || expected->matched_count == 0)) { + return Status::OK(); + } + // An empty final group must still sweep expected->docs and emit matches that + // earlier resident groups recorded in-place. + + // ONE batched PRX round for every retained chunk across all surviving tails + // (vs one round per tail before). `candidates` is intentionally empty: every + // source is a final-candidate source, so the builder addresses positions by + // the source's own docids and never consults the shared candidate list. + std::vector> owners; + std::vector srcs; + const std::vector no_shared_candidates; + if (!active_plans.empty()) { + RETURN_IF_ERROR(build_position_sources_for_candidates(idx, round1, active_plans, + &active_sources, no_shared_candidates, + &owners, &srcs, observer_context)); + } + + // Single forward sweep over the ascending expected docs. For each doc probe + // only the tails that actually posted it (per-tail ascending cursor over + // tail_candidates), decode positions once, and emit the doc the instant one + // tail's positions land adjacent to a leading match. Cursors advance strictly + // forward because expected.docs is strictly ascending and each cursor is + // sought at most once per doc. + std::vector cursors(active_plans.size()); + for (size_t a = 0; a < active_plans.size(); ++a) { + cursors[a].init(&srcs[a]); + } + std::vector tail_pos(active_plans.size(), 0); + PhraseVerifyTimer verify_timer(observer_context); + DCHECK_EQ(expected->docs.size(), expected_docids.size()); + for (size_t expected_index = 0; expected_index < expected->docs.size(); ++expected_index) { + ExpectedTailPositions& doc = expected->docs[expected_index]; + DCHECK_EQ(doc.docid, expected_docids[expected_index]); + SNII_QUERY_COUNT(prefix_expected_doc_visits); + if (observer_context != nullptr && observer_context->query_stats != nullptr) { + ++observer_context->query_stats->prefix_tail_candidate_visits; + } + const uint32_t d = doc.docid; + if (frequency_matches != nullptr || doc.phrase_frequency == 0) { + bool matched = false; + uint32_t added_frequency = 0; + for (size_t a = 0; + a < active_plans.size() && (frequency_matches != nullptr || !matched); ++a) { + std::vector& cand = tail_candidates[active_index[a]]; + size_t& ti = tail_pos[a]; + while (ti < cand.size() && cand[ti] < d) { + ++ti; + } + if (ti >= cand.size() || cand[ti] != d) { + continue; // this expansion has no posting at d + } + RETURN_IF_ERROR(cursors[a].seek(d)); + std::pair actual; + RETURN_IF_ERROR(cursors[a].positions(&actual)); + if (frequency_matches == nullptr && contains_any_position(*expected, doc, actual)) { + matched = true; + } else if (frequency_matches != nullptr) { + const uint32_t added = mark_matching_positions(expected, doc, actual); + DCHECK_LE(added_frequency, std::numeric_limits::max() - added); + added_frequency += added; + } + } + if (matched || added_frequency != 0) { + if (doc.phrase_frequency == 0) { + ++expected->matched_count; + } + if (frequency_matches == nullptr) { + doc.phrase_frequency = 1; + } else { + DCHECK_LE(doc.phrase_frequency, + std::numeric_limits::max() - added_frequency); + doc.phrase_frequency += added_frequency; + } + } + } + if (final_group && doc.phrase_frequency != 0) { + if (final_matches != nullptr) { + final_matches->push_back(d); + } + if (frequency_matches != nullptr) { + frequency_matches->push_back(PhraseMatch { + .docid = d, .frequency = static_cast(doc.phrase_frequency)}); + } + } + } + verify_timer.commit_success(); + return Status::OK(); +} + +} // namespace +Status execute_resolved_phrase_prefix_terms( + const LogicalIndexReader& idx, internal::ResolvedPhrasePlan exact_plan, + std::vector tail_terms, uint32_t tail_position_offset, + std::vector* docids, format::PrxDecodeContext* decode_context, + std::vector* matches, const std::vector* candidate_prefilter) { + DORIS_CHECK(docids != nullptr || matches != nullptr); + if (tail_terms.empty()) { + return Status::OK(); + } + if (exact_plan.phrase_plan_index.empty()) { + DORIS_CHECK(matches == nullptr); + DORIS_CHECK_EQ(tail_position_offset, 0U); + if (tail_terms.size() == 1) { + const auto& tail = tail_terms.front(); + RETURN_IF_ERROR(internal::read_docid_posting(idx, tail.entry, tail.frq_base, + tail.prx_base, docids)); + if (candidate_prefilter != nullptr) { + *docids = internal::intersect_sorted(*docids, *candidate_prefilter); + } + return Status::OK(); + } + std::vector tail_postings; + tail_postings.reserve(tail_terms.size()); + for (const auto& tail : tail_terms) { + tail_postings.push_back({tail.entry, tail.frq_base, tail.prx_base}); + } + RETURN_IF_ERROR(internal::build_docid_union(idx, tail_postings, docids)); + if (candidate_prefilter != nullptr) { + *docids = internal::intersect_sorted(*docids, *candidate_prefilter); + } + return Status::OK(); + } + if (!idx.has_positions()) { + return Status::Error( + "phrase_prefix_query: index has no positions"); + } + DORIS_CHECK(!exact_plan.position_offsets.empty()); + DORIS_CHECK_GT(tail_position_offset, exact_plan.position_offsets.back()); + if (tail_terms.size() == 1) { + append_resolved_phrase_clause(std::move(tail_terms.front()), tail_position_offset, + &exact_plan); + return internal::execute_resolved_phrase_plan(idx, std::move(exact_plan), docids, + decode_context, matches, candidate_prefilter); + } + + uint32_t min_lead_df = std::numeric_limits::max(); + for (const ResolvedQueryTerm& term : exact_plan.unique_terms) { + min_lead_df = std::min(min_lead_df, term.entry.df); + } + uint64_t tail_df_sum = 0; + for (const ResolvedQueryTerm& tail : tail_terms) { + tail_df_sum += tail.entry.df; + } + const std::vector* prefilter = candidate_prefilter; + std::vector tail_union; + std::vector combined_prefilter; + const bool allow_segment_relative_prefilter = + candidate_prefilter == nullptr && exact_plan.phrase_plan_index.size() == 1; + const uint32_t leading_prefilter_min_df = + prefix_leading_prefilter_min_df(idx, allow_segment_relative_prefilter); + if (min_lead_df >= leading_prefilter_min_df && + tail_df_sum <= static_cast(min_lead_df) / kPrefixLeadingToTailDfRatio) { + std::vector tail_postings; + tail_postings.reserve(tail_terms.size()); + for (const ResolvedQueryTerm& tail : tail_terms) { + tail_postings.push_back({tail.entry, tail.frq_base, tail.prx_base}); + } + RETURN_IF_ERROR(internal::build_docid_union(idx, tail_postings, &tail_union)); + if (tail_union.empty()) { + return Status::OK(); + } + if (candidate_prefilter == nullptr) { + prefilter = &tail_union; + } else { + combined_prefilter = internal::intersect_sorted(*candidate_prefilter, tail_union); + if (combined_prefilter.empty()) { + return Status::OK(); + } + prefilter = &combined_prefilter; + } + } + + ExpectedTailPositionSet expected; + RETURN_IF_ERROR(collect_expected_tail_positions(idx, std::move(exact_plan), + tail_position_offset, &expected, prefilter, + decode_context, matches != nullptr)); + if (expected.docs.empty()) { + return Status::OK(); + } + if (matches != nullptr) { + expected.position_matched.assign(expected.positions.size(), 0); + } + + std::vector expected_docids; + expected_docids.reserve(expected.docs.size()); + for (const ExpectedTailPositions& doc : expected.docs) { + expected_docids.push_back(doc.docid); + } + SNII_QUERY_COUNT(expected_docids_build); + + std::vector final_matches; + // Keep the expected-doc set stable across groups. Compacting matched docs + // speculates that later tail groups intersect it; an empty intersection + // skips the sweep entirely, so the compaction work cannot be repaid safely. + for (size_t start = 0; start < tail_terms.size(); start += kMaxTailMergeBatch) { + const size_t end = std::min(start + kMaxTailMergeBatch, tail_terms.size()); + const bool final_group = end == tail_terms.size(); + std::vector group; + group.reserve(end - start); + for (size_t i = start; i < end; ++i) { + group.push_back(std::move(tail_terms[i])); + } + RETURN_IF_ERROR(collect_merged_tail_matches( + idx, std::move(group), &expected, expected_docids, final_group, + docids == nullptr ? nullptr : &final_matches, decode_context, matches)); + if (matches == nullptr && !final_group && expected.matched_count == expected.docs.size()) { + final_matches.reserve(expected.docs.size()); + for (const ExpectedTailPositions& doc : expected.docs) { + DCHECK_NE(doc.phrase_frequency, 0); + final_matches.push_back(doc.docid); + } + break; + } + } + if (docids != nullptr) { + *docids = std::move(final_matches); + } + return Status::OK(); +} + +namespace { +template +std::vector copy_resolved_terms(const std::vector& resolved, + const std::vector& indices) { + std::vector result; + result.reserve(indices.size()); + for (size_t index : indices) { + DORIS_CHECK_LT(index, resolved.size()); + result.push_back(resolved[index]); + } + return result; +} + +} // namespace +Status execute_hybrid_phrase_prefix_plan( + const LogicalIndexReader& idx, const HybridPrefixPlanArtifact& artifact, + const std::vector& batch_terms, const std::vector& resolved, + const std::vector& plain_tail_terms, std::vector* docids, + format::PrxDecodeContext* decode_context, CommonGramsPlanningTimer& planning_timer, + bool* candidate_intersection_empty) { + DORIS_CHECK(idx.common_grams_posting_policy() == format::CommonGramsPostingPolicy::kHybridV1); + DORIS_CHECK(docids != nullptr); + DORIS_CHECK(candidate_intersection_empty != nullptr); + docids->clear(); + *candidate_intersection_empty = false; + + const HybridPositionedCover& plain_tail_cover = artifact.plain_tail_cover; + const uint32_t plain_tail_position_offset = artifact.plain_tail_position_offset; + HybridPrefixCandidateSet leading_candidates; + RETURN_IF_ERROR(build_hybrid_leading_candidates(idx, plain_tail_cover.candidate_prefilter, + batch_terms, resolved, &leading_candidates)); + if (leading_candidates.active && leading_candidates.docs.empty()) { + planning_timer.finish(); + *candidate_intersection_empty = true; + return Status::OK(); + } + + if (!artifact.maps_tail_to_gram) { + DORIS_CHECK(leading_candidates.active); + DORIS_CHECK(artifact.mapped_tail_split.positioned_indices.empty()); + DORIS_CHECK(artifact.mapped_tail_split.docs_only_indices.empty()); + planning_timer.finish(); + RETURN_IF_ERROR(execute_resolved_phrase_prefix_terms( + idx, + copy_resolved_phrase_plan(plain_tail_cover.verification, batch_terms, resolved), + plain_tail_terms, plain_tail_position_offset, docids, decode_context, nullptr, + &leading_candidates.docs)); + *candidate_intersection_empty = docids->empty(); + return Status::OK(); + } + + const HybridPrefixMappedTails& split = artifact.mapped_tail_split; + DORIS_CHECK(!split.positioned_indices.empty() || !split.docs_only_indices.empty()); + DORIS_CHECK(leading_candidates.active || !split.docs_only_indices.empty()); + std::vector docs_only_tail_candidates; + if (!split.docs_only_indices.empty()) { + RETURN_IF_ERROR(build_hybrid_docs_only_tail_candidates( + idx, resolved, split.docs_only_indices, leading_candidates, + &docs_only_tail_candidates)); + } + planning_timer.finish(); + + if (!split.positioned_indices.empty()) { + DORIS_CHECK(artifact.positioned_tail_verification.has_value()); + std::vector positioned_docs; + RETURN_IF_ERROR(execute_resolved_phrase_prefix_terms( + idx, + copy_resolved_phrase_plan(*artifact.positioned_tail_verification, batch_terms, + resolved), + copy_resolved_terms(resolved, split.positioned_indices), + plain_tail_position_offset - 1, &positioned_docs, decode_context, nullptr, + leading_candidates.active ? &leading_candidates.docs : nullptr)); + internal::union_sorted_into(docids, positioned_docs); + } + + if (!split.docs_only_indices.empty() && !docs_only_tail_candidates.empty()) { + std::vector docs_only_docs; + RETURN_IF_ERROR(execute_resolved_phrase_prefix_terms( + idx, + copy_resolved_phrase_plan(plain_tail_cover.verification, batch_terms, resolved), + copy_resolved_terms(plain_tail_terms, split.docs_only_ordinals), + plain_tail_position_offset, &docs_only_docs, decode_context, nullptr, + &docs_only_tail_candidates)); + internal::union_sorted_into(docids, docs_only_docs); + } + *candidate_intersection_empty = docids->empty(); + return Status::OK(); +} + +} // namespace doris::snii::query::phrase_impl diff --git a/be/src/storage/index/snii/query/phrase_prx_validation.h b/be/src/storage/index/snii/query/phrase_prx_validation.h new file mode 100644 index 00000000000000..a4082360affba7 --- /dev/null +++ b/be/src/storage/index/snii/query/phrase_prx_validation.h @@ -0,0 +1,55 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include + +#include "common/status.h" + +namespace doris::snii { +class ByteSource; +namespace format { +struct PrxDecodeContext; +} +} // namespace doris::snii + +namespace doris::snii::query::internal { + +// The production seam shared by PosChunkDecoder and focused shape-validation +// tests. A successful format decode commits its stats before caller-level CSR +// validation runs. +Status decode_and_validate_prx_frame(ByteSource* source, + std::span selected_doc_ordinals, + bool decode_full, bool all_docs_selected, + uint32_t expected_total_docs, size_t expected_selected_docs, + std::vector* pos_flat, + std::vector* pos_offsets, + format::PrxDecodeContext* decode_context); + +// Validates the CSR shape expected by phrase execution. Format-level decode +// statistics have already been committed when this function runs. +Status validate_prx_frame(std::span pos_flat, std::span pos_offsets, + uint32_t actual_total_docs, uint32_t expected_total_docs, + size_t expected_selected_docs, + std::span selected_doc_ordinals, + bool offsets_by_prx_ordinal, bool all_docs_selected); + +} // namespace doris::snii::query::internal diff --git a/be/src/storage/index/snii/query/phrase_query.cpp b/be/src/storage/index/snii/query/phrase_query.cpp new file mode 100644 index 00000000000000..e5fa8a6204ce48 --- /dev/null +++ b/be/src/storage/index/snii/query/phrase_query.cpp @@ -0,0 +1,326 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/query/phrase_query.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/check.h" +#include "storage/index/inverted/common_grams/common_grams_key_codec.h" +#include "storage/index/inverted/common_grams/common_grams_query_cost.h" +#include "storage/index/inverted/common_grams/common_grams_segment_metadata.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/encoding/byte_source.h" +#include "storage/index/snii/format/dict_entry.h" +#include "storage/index/snii/format/frq_pod.h" +#include "storage/index/snii/format/frq_prelude.h" +#include "storage/index/snii/format/prx_pod.h" +#include "storage/index/snii/io/batch_range_fetcher.h" +#include "storage/index/snii/query/internal/docid_conjunction.h" +#include "storage/index/snii/query/internal/docid_posting_reader.h" +#include "storage/index/snii/query/internal/docid_set_ops.h" +#include "storage/index/snii/query/internal/docid_union.h" +#include "storage/index/snii/query/internal/phrase_query_split.h" +#include "storage/index/snii/query/internal/plain_term_routing.h" +#include "storage/index/snii/query/internal/position_math.h" +#include "storage/index/snii/query/internal/query_test_counters.h" +#include "storage/index/snii/query/internal/resolved_phrase_plan.h" +#include "storage/index/snii/query/internal/term_expansion.h" +#include "storage/index/snii/query/phrase_prx_validation.h" +#include "storage/index/snii/query/phrase_verify_timer.h" +#include "storage/index/snii/query/prefix_query.h" +#include "storage/index/snii/query/term_query.h" +#include "storage/index/snii/reader/windowed_posting.h" +#include "util/debug_points.h" + +namespace doris::snii::query { + +using query::internal::DocidChunk; +using query::internal::DocidSource; +using query::internal::ResolvedQueryTerm; +using query::internal::TermPlan; +using reader::LogicalIndexReader; + +CommonGramsPlanDebugOverride common_grams_plan_debug_override() { + CommonGramsPlanDebugOverride result = CommonGramsPlanDebugOverride::kNone; + DBUG_EXECUTE_IF(COMMON_GRAMS_FORCE_PLAIN_PLAN_DEBUG_POINT, + { result = CommonGramsPlanDebugOverride::kForcePlain; }); + DBUG_EXECUTE_IF(COMMON_GRAMS_FORCE_GRAM_PLAN_DEBUG_POINT, { + DORIS_CHECK(result != CommonGramsPlanDebugOverride::kForcePlain); + result = CommonGramsPlanDebugOverride::kForceCommonGrams; + }); + return result; +} + +using namespace phrase_impl; // NOLINT(google-build-using-namespace): module-internal impl namespace + +namespace internal { + +Status validate_prx_frame(std::span pos_flat, std::span pos_offsets, + uint32_t actual_total_docs, uint32_t expected_total_docs, + size_t expected_selected_docs, + std::span selected_doc_ordinals, + bool offsets_by_prx_ordinal, bool all_docs_selected) { + if (!all_docs_selected && expected_selected_docs != selected_doc_ordinals.size()) { + return Status::Error( + "phrase_query: selected prx ordinal-count mismatch"); + } + if (actual_total_docs != expected_total_docs) { + return Status::Error( + "phrase_query: prx total doc-count mismatch"); + } + const size_t expected_offsets = offsets_by_prx_ordinal + ? static_cast(expected_total_docs) + 1 + : expected_selected_docs + 1; + if (pos_offsets.size() != expected_offsets) { + return Status::Error( + offsets_by_prx_ordinal ? "phrase_query: full prx doc-count mismatch" + : "phrase_query: selected prx/doc-count mismatch"); + } + if (pos_offsets.back() != pos_flat.size()) { + return Status::Error( + "phrase_query: prx final offset mismatch"); + } + if (offsets_by_prx_ordinal && !selected_doc_ordinals.empty() && + static_cast(selected_doc_ordinals.back()) + 1 >= pos_offsets.size()) { + return Status::Error( + "phrase_query: prx ordinal offset out of range"); + } + return Status::OK(); +} + +Status decode_and_validate_prx_frame(ByteSource* source, + std::span selected_doc_ordinals, + bool decode_full, bool all_docs_selected, + uint32_t expected_total_docs, size_t expected_selected_docs, + std::vector* pos_flat, + std::vector* pos_offsets, + format::PrxDecodeContext* decode_context) { + DCHECK(decode_full || !all_docs_selected); + format::PrxDecodedShape decoded_shape; + format::PrxDecodeContext frame_context { + .stats = decode_context == nullptr ? nullptr : decode_context->stats, + .shape = &decoded_shape}; + if (decode_full) { + if (all_docs_selected) { + RETURN_IF_ERROR( + format::read_prx_window_csr(source, pos_flat, pos_offsets, &frame_context)); + } else { + RETURN_IF_ERROR(format::read_prx_window_csr_for_selection( + source, selected_doc_ordinals, pos_flat, pos_offsets, &frame_context)); + } + } else { + RETURN_IF_ERROR(format::read_prx_window_csr_selective( + source, selected_doc_ordinals, pos_flat, pos_offsets, &frame_context)); + } + return validate_prx_frame(*pos_flat, *pos_offsets, decoded_shape.total_docs, + expected_total_docs, expected_selected_docs, selected_doc_ordinals, + decode_full && !all_docs_selected, all_docs_selected); +} + +namespace { + +using PhraseVerifyClock = std::chrono::steady_clock; + +PhraseVerifyClock::time_point phrase_verify_clock_now() { +#ifdef BE_TEST + testing::note_phrase_verify_clock_read(); +#endif + return PhraseVerifyClock::now(); +} + +} // namespace + +uint64_t exclusive_phrase_verify_ns(uint64_t elapsed_ns, uint64_t decode_ns_before, + uint64_t decode_ns_after) { + DCHECK_GE(decode_ns_after, decode_ns_before); + const uint64_t decode_delta = decode_ns_after - decode_ns_before; + return elapsed_ns > decode_delta ? elapsed_ns - decode_delta : 0; +} + +PhraseVerifyTimer::PhraseVerifyTimer(format::PrxDecodeContext* decode_context) + : stats_(decode_context == nullptr ? nullptr : decode_context->stats) { + if (stats_ != nullptr) { + decode_ns_before_ = stats_->decode_ns; + start_ = phrase_verify_clock_now(); + } +} + +void PhraseVerifyTimer::commit_success() { + if (stats_ == nullptr) { + return; + } + const auto elapsed = + std::chrono::duration_cast(phrase_verify_clock_now() - start_) + .count(); + stats_->phrase_verify_ns += exclusive_phrase_verify_ns(static_cast(elapsed), + decode_ns_before_, stats_->decode_ns); +} + +} // namespace internal + +Status phrase_query(const LogicalIndexReader& idx, const std::vector& terms, + std::vector* const docids) { + return phrase_query_impl(idx, terms, docids, nullptr, nullptr, {}); +} + +Status phrase_query(const LogicalIndexReader& idx, const std::vector& terms, + std::vector* const docids, QueryProfile* profile) { + return phrase_query(idx, terms, docids, profile, {}); +} + +Status phrase_query(const LogicalIndexReader& idx, const std::vector& terms, + std::vector* const docids, QueryProfile* profile, + const PhraseQueryOptions& options) { + QueryProfileScope profile_scope(idx.reader(), profile); + format::PrxDecodeContext decode_context { + .stats = profile == nullptr ? nullptr : &profile->prx_decode_stats, + .query_stats = profile == nullptr ? nullptr : &profile->phrase_query_stats}; + return phrase_query_impl(idx, terms, docids, profile == nullptr ? nullptr : &decode_context, + nullptr, options); +} + +Status phrase_query_with_frequencies(const LogicalIndexReader& idx, + const std::vector& terms, + std::vector* matches, QueryProfile* profile, + const PhraseQueryOptions& options) { + if (matches == nullptr) { + return Status::Error( + "phrase_query_with_frequencies: null out"); + } + matches->clear(); + if (terms.size() < 2) { + return Status::Error( + "phrase_query_with_frequencies: at least two terms are required"); + } + QueryProfileScope profile_scope(idx.reader(), profile); + format::PrxDecodeContext decode_context { + .stats = profile == nullptr ? nullptr : &profile->prx_decode_stats, + .query_stats = profile == nullptr ? nullptr : &profile->phrase_query_stats}; + return phrase_query_impl(idx, terms, nullptr, profile == nullptr ? nullptr : &decode_context, + matches, options); +} + +Status planned_exact_phrase_query( + const LogicalIndexReader& idx, const segment_v2::InvertedIndexQueryInfo& plain_query_info, + const segment_v2::InvertedIndexQueryInfo& gram_query_info, + const segment_v2::inverted_index::CommonGramsQueryIdentity* common_grams_identity, + std::vector* docids, QueryProfile* profile, ExactPhrasePlanKind* selected_plan, + segment_v2::inverted_index::CommonGramsPlanCostModel cost_model, + std::optional debug_override) { + QueryProfileScope profile_scope(idx.reader(), profile); + format::PrxDecodeContext decode_context { + .stats = profile == nullptr ? nullptr : &profile->prx_decode_stats, + .query_stats = profile == nullptr ? nullptr : &profile->phrase_query_stats}; + return planned_exact_phrase_query_impl( + idx, plain_query_info, gram_query_info, common_grams_identity, docids, + profile == nullptr ? nullptr : &decode_context, selected_plan, cost_model, + debug_override.has_value() ? *debug_override : common_grams_plan_debug_override()); +} + +Status planned_phrase_prefix_query( + const LogicalIndexReader& idx, const segment_v2::InvertedIndexQueryInfo& plain_query_info, + const segment_v2::InvertedIndexQueryInfo& gram_query_info, + const segment_v2::inverted_index::CommonGramsQueryIdentity* common_grams_identity, + std::vector* docids, QueryProfile* profile, int32_t max_expansions, + PhrasePrefixPlanKind* selected_plan, + segment_v2::inverted_index::CommonGramsPlanCostModel cost_model, + std::optional debug_override) { + QueryProfileScope profile_scope(idx.reader(), profile); + format::PrxDecodeContext decode_context { + .stats = profile == nullptr ? nullptr : &profile->prx_decode_stats, + .query_stats = profile == nullptr ? nullptr : &profile->phrase_query_stats}; + return planned_phrase_prefix_query_impl( + idx, plain_query_info, gram_query_info, common_grams_identity, docids, max_expansions, + profile == nullptr ? nullptr : &decode_context, selected_plan, cost_model, + debug_override.has_value() ? *debug_override : common_grams_plan_debug_override()); +} + +Status phrase_prefix_query(const LogicalIndexReader& idx, const std::vector& terms, + std::vector* const docids, int32_t max_expansions) { + return phrase_prefix_query_impl(idx, terms, docids, max_expansions, nullptr, nullptr); +} + +Status phrase_prefix_query(const LogicalIndexReader& idx, const std::vector& terms, + std::vector* const docids, QueryProfile* profile, + int32_t max_expansions) { + QueryProfileScope profile_scope(idx.reader(), profile); + format::PrxDecodeContext decode_context { + .stats = profile == nullptr ? nullptr : &profile->prx_decode_stats, + .query_stats = profile == nullptr ? nullptr : &profile->phrase_query_stats}; + return phrase_prefix_query_impl(idx, terms, docids, max_expansions, + profile == nullptr ? nullptr : &decode_context, nullptr); +} + +Status phrase_prefix_query_with_frequencies(const LogicalIndexReader& idx, + const std::vector& terms, + std::vector* matches, + QueryProfile* profile, int32_t max_expansions) { + if (matches == nullptr) { + return Status::Error( + "phrase_prefix_query_with_frequencies: null out"); + } + matches->clear(); + if (terms.size() < 2) { + return Status::Error( + "phrase_prefix_query_with_frequencies: at least two terms are required"); + } + QueryProfileScope profile_scope(idx.reader(), profile); + format::PrxDecodeContext decode_context { + .stats = profile == nullptr ? nullptr : &profile->prx_decode_stats, + .query_stats = profile == nullptr ? nullptr : &profile->phrase_query_stats}; + return phrase_prefix_query_impl(idx, terms, nullptr, max_expansions, + profile == nullptr ? nullptr : &decode_context, nullptr, + matches); +} + +} // namespace doris::snii::query + +#ifdef BE_TEST +namespace doris::snii::query::internal::testing { +namespace { + +std::atomic& phrase_verify_clock_read_atomic() { + static std::atomic counter {0}; + return counter; +} + +} // namespace + +uint64_t phrase_verify_clock_read_count() { + return phrase_verify_clock_read_atomic().load(std::memory_order_relaxed); +} + +void reset_phrase_verify_clock_read_count() { + phrase_verify_clock_read_atomic().store(0, std::memory_order_relaxed); +} + +void note_phrase_verify_clock_read() { + phrase_verify_clock_read_atomic().fetch_add(1, std::memory_order_relaxed); +} + +} // namespace doris::snii::query::internal::testing +#endif diff --git a/be/src/storage/index/snii/query/phrase_query.h b/be/src/storage/index/snii/query/phrase_query.h new file mode 100644 index 00000000000000..2db4285e1c2bb8 --- /dev/null +++ b/be/src/storage/index/snii/query/phrase_query.h @@ -0,0 +1,141 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/inverted/common_grams/common_grams_query_cost.h" +#include "storage/index/inverted/common_grams/common_grams_segment_metadata.h" +#include "storage/index/inverted/query/query_info.h" +#include "storage/index/snii/format/prx_decode_stats.h" +#include "storage/index/snii/query/query_profile.h" +#include "storage/index/snii/reader/logical_index_reader.h" + +// phrase_query -- MATCH_PHRASE: return the sorted docid set in which the terms +// occur consecutively (for some i, every term k appears at position pos+k in +// the same doc). It first builds the docid conjunction with docs-only posting +// reads, then fetches PRX only for chunks that can contain final candidates: +// 1. read preludes / docs-only posting ranges and intersect per-term docids; +// 2. fetch retained PRX chunks and stream positions for survivors; +// 3. for each surviving doc, check that some position p exists with +// term[0]@p, term[1]@p+1, ... term[n-1]@p+(n-1). +// An empty term list -> empty result. Any term absent -> empty result. +namespace doris::snii::query { + +enum class ExactPhrasePlanKind : uint8_t { + kPlain = 0, + kCommonGrams = 1, +}; + +enum class PhrasePrefixPlanKind : uint8_t { + kPlain = 0, + kCommonGrams = 1, +}; + +// Benchmark-only plan override. The debug points are inert unless the process-wide +// enable_debug_points switch is on, and planners apply them only after both complete plans resolve. +enum class CommonGramsPlanDebugOverride : uint8_t { + kNone = 0, + kForcePlain = 1, + kForceCommonGrams = 2, +}; + +inline constexpr char COMMON_GRAMS_FORCE_PLAIN_PLAN_DEBUG_POINT[] = + "snii.common_grams.force_plain_plan"; +inline constexpr char COMMON_GRAMS_FORCE_GRAM_PLAN_DEBUG_POINT[] = + "snii.common_grams.force_gram_plan"; + +CommonGramsPlanDebugOverride common_grams_plan_debug_override(); + +struct PhraseMatch { + uint32_t docid = 0; + float frequency = 0.0F; + + bool operator==(const PhraseMatch&) const = default; +}; + +struct PhraseQueryOptions { + uint32_t slop = 0; + bool ordered = false; +}; + +Status phrase_query(const reader::LogicalIndexReader& idx, const std::vector& terms, + std::vector* docids); +Status phrase_query(const reader::LogicalIndexReader& idx, const std::vector& terms, + std::vector* docids, QueryProfile* profile); +Status phrase_query(const reader::LogicalIndexReader& idx, const std::vector& terms, + std::vector* docids, QueryProfile* profile, + const PhraseQueryOptions& options); + +// Scoring-only multi-term entry point. It runs the same opaque-term matcher as +// phrase_query() and returns exact occurrence counts or V3-compatible, +// distance-weighted sloppy-phrase frequencies for each matching doc. +Status phrase_query_with_frequencies(const reader::LogicalIndexReader& idx, + const std::vector& terms, + std::vector* matches, + QueryProfile* profile = nullptr, + const PhraseQueryOptions& options = {}); + +// Selects between equivalent plain and CommonGrams exact-phrase plans above +// the existing opaque-term matcher. A missing or mismatched query identity +// forces the plain plan before any gram DICT lookup. +Status planned_exact_phrase_query( + const reader::LogicalIndexReader& idx, + const segment_v2::InvertedIndexQueryInfo& plain_query_info, + const segment_v2::InvertedIndexQueryInfo& gram_query_info, + const segment_v2::inverted_index::CommonGramsQueryIdentity* common_grams_identity, + std::vector* docids, QueryProfile* profile = nullptr, + ExactPhrasePlanKind* selected_plan = nullptr, + segment_v2::inverted_index::CommonGramsPlanCostModel cost_model = {}, + std::optional debug_override = std::nullopt); + +Status planned_phrase_prefix_query( + const reader::LogicalIndexReader& idx, + const segment_v2::InvertedIndexQueryInfo& plain_query_info, + const segment_v2::InvertedIndexQueryInfo& gram_query_info, + const segment_v2::inverted_index::CommonGramsQueryIdentity* common_grams_identity, + std::vector* docids, QueryProfile* profile = nullptr, int32_t max_expansions = 0, + PhrasePrefixPlanKind* selected_plan = nullptr, + segment_v2::inverted_index::CommonGramsPlanCostModel cost_model = {}, + std::optional debug_override = std::nullopt); + +// phrase_prefix_query -- MATCH_PHRASE_PREFIX: the last item in `terms` is a +// term prefix and preceding items are exact terms. For example {"quick", "bro"} +// matches "quick brown" and "quick bronze". Empty terms -> empty result. +Status phrase_prefix_query(const reader::LogicalIndexReader& idx, + const std::vector& terms, + std::vector* const docids, int32_t max_expansions = 0); +Status phrase_prefix_query(const reader::LogicalIndexReader& idx, + const std::vector& terms, + std::vector* const docids, QueryProfile* profile, + int32_t max_expansions = 0); + +// Scoring-only multi-term entry point. Tail expansions are one logical phrase +// clause, so each phrase start contributes at most once even when several +// expanded terms share that position. +Status phrase_prefix_query_with_frequencies(const reader::LogicalIndexReader& idx, + const std::vector& terms, + std::vector* matches, + QueryProfile* profile = nullptr, + int32_t max_expansions = 0); + +} // namespace doris::snii::query diff --git a/be/src/storage/index/snii/query/phrase_verify_timer.h b/be/src/storage/index/snii/query/phrase_verify_timer.h new file mode 100644 index 00000000000000..bb04f33a64abed --- /dev/null +++ b/be/src/storage/index/snii/query/phrase_verify_timer.h @@ -0,0 +1,51 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include "storage/index/snii/format/prx_decode_stats.h" + +namespace doris::snii::query::internal { + +uint64_t exclusive_phrase_verify_ns(uint64_t elapsed_ns, uint64_t decode_ns_before, + uint64_t decode_ns_after); + +class PhraseVerifyTimer { +public: + explicit PhraseVerifyTimer(format::PrxDecodeContext* decode_context); + + void commit_success(); + +private: + format::PrxDecodeStats* stats_ = nullptr; + std::chrono::steady_clock::time_point start_; + uint64_t decode_ns_before_ = 0; +}; + +#ifdef BE_TEST +namespace testing { + +uint64_t phrase_verify_clock_read_count(); +void reset_phrase_verify_clock_read_count(); +void note_phrase_verify_clock_read(); + +} // namespace testing +#endif +} // namespace doris::snii::query::internal diff --git a/be/src/storage/index/snii/query/prefix_query.cpp b/be/src/storage/index/snii/query/prefix_query.cpp new file mode 100644 index 00000000000000..632c80d8b099f8 --- /dev/null +++ b/be/src/storage/index/snii/query/prefix_query.cpp @@ -0,0 +1,56 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/query/prefix_query.h" + +#include +#include + +#include "storage/index/snii/query/internal/term_expansion.h" + +namespace doris::snii::query { + +using reader::LogicalIndexReader; + +Status prefix_query(const LogicalIndexReader& idx, std::string_view prefix, + std::vector* const docids, int32_t max_expansions) { + if (docids == nullptr) { + return Status::Error("prefix_query: null out"); + } + docids->clear(); + VectorDocIdSink sink(*docids); + return prefix_query(idx, prefix, &sink, max_expansions); +} + +Status prefix_query(const LogicalIndexReader& idx, std::string_view prefix, + std::vector* const docids, QueryProfile* profile, + int32_t max_expansions) { + QueryProfileScope profile_scope(idx.reader(), profile); + return prefix_query(idx, prefix, docids, max_expansions); +} + +Status prefix_query(const LogicalIndexReader& idx, std::string_view prefix, DocIdSink* const sink, + int32_t max_expansions) { + if (sink == nullptr) { + return Status::Error("prefix_query: null sink"); + } + + return internal::emit_expanded_docid_union( + idx, prefix, [](std::string_view) { return true; }, sink, max_expansions); +} + +} // namespace doris::snii::query diff --git a/be/src/storage/index/snii/query/prefix_query.h b/be/src/storage/index/snii/query/prefix_query.h new file mode 100644 index 00000000000000..6d17aa7b48b1c6 --- /dev/null +++ b/be/src/storage/index/snii/query/prefix_query.h @@ -0,0 +1,42 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/query/docid_sink.h" +#include "storage/index/snii/query/query_profile.h" +#include "storage/index/snii/reader/logical_index_reader.h" + +// prefix_query -- MATCH_PREFIX semantics: enumerate dictionary terms with the +// requested prefix, then return the sorted docid set containing any enumerated +// term. Empty prefix enumerates all terms. No matching terms -> empty result. +namespace doris::snii::query { + +Status prefix_query(const reader::LogicalIndexReader& idx, std::string_view prefix, + std::vector* const docids, int32_t max_expansions = 0); +Status prefix_query(const reader::LogicalIndexReader& idx, std::string_view prefix, + std::vector* const docids, QueryProfile* profile, + int32_t max_expansions = 0); +Status prefix_query(const reader::LogicalIndexReader& idx, std::string_view prefix, + DocIdSink* const sink, int32_t max_expansions = 0); + +} // namespace doris::snii::query diff --git a/be/src/storage/index/snii/query/query_profile.cpp b/be/src/storage/index/snii/query/query_profile.cpp new file mode 100644 index 00000000000000..025272c421eadd --- /dev/null +++ b/be/src/storage/index/snii/query/query_profile.cpp @@ -0,0 +1,94 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/query/query_profile.h" + +#include +#include +#include + +#include "storage/index/snii/io/file_reader.h" + +namespace doris::snii::query { + +namespace { + +#ifdef BE_TEST +std::atomic query_profile_clock_reads {0}; +#endif + +std::chrono::steady_clock::time_point query_profile_clock_now() { +#ifdef BE_TEST + query_profile_clock_reads.fetch_add(1, std::memory_order_relaxed); +#endif + return std::chrono::steady_clock::now(); +} + +} // namespace + +QueryProfileScope::QueryProfileScope(io::FileReader* reader, QueryProfile* profile) + : reader_(reader), profile_(profile) { + if (profile_ == nullptr) return; + + start_ = query_profile_clock_now(); + *profile_ = QueryProfile {}; + if (reader_ == nullptr) return; + + const io::IoMetrics* metrics = reader_->io_metrics(); + if (metrics == nullptr) return; + + profile_->has_io_metrics = true; + profile_->io_before = *metrics; +} + +QueryProfileScope::~QueryProfileScope() { + finish(); +} + +void QueryProfileScope::finish() { + if (profile_ == nullptr || finished_) return; + finished_ = true; + + const auto end = query_profile_clock_now(); + const auto elapsed = std::chrono::duration_cast(end - start_).count(); + profile_->elapsed_ns = std::max(1, static_cast(elapsed)); + + if (!profile_->has_io_metrics || reader_ == nullptr) return; + const io::IoMetrics* metrics = reader_->io_metrics(); + if (metrics == nullptr) { + profile_->has_io_metrics = false; + return; + } + profile_->io_after = *metrics; + profile_->io_delta = io::delta(profile_->io_after, profile_->io_before); +} + +#ifdef BE_TEST +namespace testing { + +uint64_t query_profile_clock_read_count() { + return query_profile_clock_reads.load(std::memory_order_relaxed); +} + +void reset_query_profile_clock_read_count() { + query_profile_clock_reads.store(0, std::memory_order_relaxed); +} + +} // namespace testing +#endif + +} // namespace doris::snii::query diff --git a/be/src/storage/index/snii/query/query_profile.h b/be/src/storage/index/snii/query/query_profile.h new file mode 100644 index 00000000000000..2598deeef24f85 --- /dev/null +++ b/be/src/storage/index/snii/query/query_profile.h @@ -0,0 +1,67 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include "storage/index/snii/format/prx_decode_stats.h" +#include "storage/index/snii/io/io_metrics.h" + +namespace doris::snii::io { +class FileReader; +} + +namespace doris::snii::query { + +#ifdef BE_TEST +namespace testing { + +uint64_t query_profile_clock_read_count(); +void reset_query_profile_clock_read_count(); + +} // namespace testing +#endif + +struct QueryProfile { + uint64_t elapsed_ns = 0; + bool has_io_metrics = false; + io::IoMetrics io_before; + io::IoMetrics io_after; + io::IoMetrics io_delta; + format::PrxDecodeStats prx_decode_stats; + format::PhraseQueryExecutionStats phrase_query_stats; +}; + +class QueryProfileScope { +public: + QueryProfileScope(io::FileReader* reader, QueryProfile* profile); + ~QueryProfileScope(); + QueryProfileScope(const QueryProfileScope&) = delete; + QueryProfileScope& operator=(const QueryProfileScope&) = delete; + + void finish(); + +private: + io::FileReader* reader_ = nullptr; + QueryProfile* profile_ = nullptr; + std::chrono::steady_clock::time_point start_; + bool finished_ = false; +}; + +} // namespace doris::snii::query diff --git a/be/src/storage/index/snii/query/regexp_query.cpp b/be/src/storage/index/snii/query/regexp_query.cpp new file mode 100644 index 00000000000000..af26fa5878fbdf --- /dev/null +++ b/be/src/storage/index/snii/query/regexp_query.cpp @@ -0,0 +1,192 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/query/regexp_query.h" + +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "common/logging.h" +#include "storage/index/snii/query/internal/regex_prefix.h" +#include "storage/index/snii/query/internal/term_expansion.h" + +namespace doris::snii::query { + +namespace { + +template +struct HyperscanDeleter { + template + void operator()(T* ptr) const { + deleter(ptr); + } +}; + +using HyperscanDatabasePtr = + std::unique_ptr>; +using HyperscanScratchPtr = + std::unique_ptr>; +using HyperscanCompileErrorPtr = + std::unique_ptr>; + +bool is_regex_metachar(char c) { + switch (c) { + case '.': + case '^': + case '$': + case '|': + case '(': + case ')': + case '[': + case ']': + case '*': + case '+': + case '?': + case '{': + case '}': + case '\\': + return true; + default: + return false; + } +} + +std::string literal_prefix_for_regex(std::string_view pattern) { + std::string out; + size_t i = 0; + if (!pattern.empty() && pattern.front() == '^') { + i = 1; + } + for (; i < pattern.size(); ++i) { + const char c = pattern[i]; + if (is_regex_metachar(c)) { + break; + } + out.push_back(c); + } + return out; +} + +} // namespace + +namespace internal { + +std::string regex_enum_prefix(std::string_view pattern, const re2::RE2& re) { + // Left-anchored patterns can yield a tighter enumeration prefix via RE2's + // PossibleMatchRange than the conservative literal scan (which stops at the + // first metacharacter). The prefix only bounds how many dictionary terms are + // enumerated. Unanchored patterns cannot use a prefix because Hyperscan may + // match the pattern after the beginning of a dictionary term. + if (pattern.empty() || pattern.front() != '^') { + return {}; + } + + if (re.ok()) { + std::string min_prefix; + std::string max_prefix; + if (re.PossibleMatchRange(&min_prefix, &max_prefix, 256) && !min_prefix.empty() && + !max_prefix.empty() && min_prefix.front() == max_prefix.front()) { + const auto mismatch_pair = std::ranges::mismatch(min_prefix, max_prefix); + const auto common_len = + static_cast(std::distance(min_prefix.begin(), mismatch_pair.in1)); + if (common_len > 0) { + return min_prefix.substr(0, common_len); + } + } + } + return literal_prefix_for_regex(pattern); +} + +} // namespace internal + +Status regexp_query(const reader::LogicalIndexReader& idx, std::string_view pattern, + std::vector* const docids, int32_t max_expansions) { + if (docids == nullptr) { + return Status::Error("regexp_query: null out"); + } + docids->clear(); + VectorDocIdSink sink(*docids); + return regexp_query(idx, pattern, &sink, max_expansions); +} + +Status regexp_query(const reader::LogicalIndexReader& idx, std::string_view pattern, + std::vector* const docids, QueryProfile* profile, + int32_t max_expansions) { + QueryProfileScope profile_scope(idx.reader(), profile); + return regexp_query(idx, pattern, docids, max_expansions); +} + +Status regexp_query(const reader::LogicalIndexReader& idx, std::string_view pattern, + DocIdSink* const sink, int32_t max_expansions) { + if (sink == nullptr) { + return Status::Error("regexp_query: null sink"); + } + + const std::string compiled_pattern(pattern); + hs_database_t* raw_database = nullptr; + hs_compile_error_t* raw_compile_error = nullptr; + const auto compile_status = + hs_compile(compiled_pattern.c_str(), HS_FLAG_DOTALL | HS_FLAG_ALLOWEMPTY | HS_FLAG_UTF8, + HS_MODE_BLOCK, nullptr, &raw_database, &raw_compile_error); + HyperscanCompileErrorPtr compile_error(raw_compile_error); + if (compile_status != HS_SUCCESS) { + return Status::OK(); + } + HyperscanDatabasePtr database(raw_database); + + hs_scratch_t* raw_scratch = nullptr; + if (hs_alloc_scratch(database.get(), &raw_scratch) != HS_SUCCESS) { + return Status::MemoryAllocFailed("regexp_query: failed to allocate Hyperscan scratch"); + } + HyperscanScratchPtr scratch(raw_scratch); + + std::string enum_prefix; + if (!pattern.empty() && pattern.front() == '^') { + re2::RE2::Options options; + options.set_log_errors(false); + const re2::RE2 re(re2::StringPiece(pattern.data(), pattern.size()), options); + enum_prefix = internal::regex_enum_prefix(pattern, re); + } + const auto on_match = [](unsigned int, unsigned long long, unsigned long long, unsigned int, + void* context) -> int { + *static_cast(context) = true; + return 0; + }; + return internal::emit_expanded_docid_union( + idx, enum_prefix, + [&database, &scratch, on_match](std::string_view term) { + bool matched = false; + const auto scan_status = + hs_scan(database.get(), term.data(), static_cast(term.size()), 0, + scratch.get(), on_match, &matched); + DCHECK_EQ(scan_status, HS_SUCCESS); + return matched; + }, + sink, max_expansions); +} + +} // namespace doris::snii::query diff --git a/be/src/storage/index/snii/query/regexp_query.h b/be/src/storage/index/snii/query/regexp_query.h new file mode 100644 index 00000000000000..d992751e84215d --- /dev/null +++ b/be/src/storage/index/snii/query/regexp_query.h @@ -0,0 +1,43 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/query/docid_sink.h" +#include "storage/index/snii/query/query_profile.h" +#include "storage/index/snii/reader/logical_index_reader.h" + +// regexp_query -- MATCH_REGEXP semantics over dictionary terms. Hyperscan applies +// the same unanchored matching semantics and flags as the V3 inverted index. An +// invalid pattern produces no matches. Matching terms are executed as a sorted +// deduplicated docid union. +namespace doris::snii::query { + +Status regexp_query(const reader::LogicalIndexReader& idx, std::string_view pattern, + std::vector* const docids, int32_t max_expansions = 0); +Status regexp_query(const reader::LogicalIndexReader& idx, std::string_view pattern, + std::vector* const docids, QueryProfile* profile, + int32_t max_expansions = 0); +Status regexp_query(const reader::LogicalIndexReader& idx, std::string_view pattern, + DocIdSink* const sink, int32_t max_expansions = 0); + +} // namespace doris::snii::query diff --git a/be/src/storage/index/snii/query/scoring_query.cpp b/be/src/storage/index/snii/query/scoring_query.cpp new file mode 100644 index 00000000000000..96afa8832b6237 --- /dev/null +++ b/be/src/storage/index/snii/query/scoring_query.cpp @@ -0,0 +1,913 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/query/scoring_query.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/encoding/byte_source.h" +#include "storage/index/snii/format/dict_entry.h" +#include "storage/index/snii/format/format_constants.h" +#include "storage/index/snii/format/frq_pod.h" +#include "storage/index/snii/format/frq_prelude.h" +#include "storage/index/snii/io/batch_range_fetcher.h" +#include "storage/index/snii/reader/dict_block_cache.h" +#include "storage/index/snii/reader/windowed_posting.h" + +namespace doris::snii::query { + +using format::DictEntry; +using format::DictEntryEnc; +using format::DictEntryKind; +using format::FrqPreludeReader; +using format::WindowMeta; +using reader::LogicalIndexReader; + +namespace { + +// One scored posting for one term in one doc. +struct TermPosting { + uint32_t docid = 0; + double score = 0.0; +}; + +// One window's block-max upper bound and the docid range it covers. block_max is +// true when max_score came from the frq_prelude columns (vs the exact-score +// fallback); both are valid upper bounds, so it is informational only. +struct WindowBound { + uint32_t first_docid = 0; // inclusive + uint32_t last_docid = 0; // inclusive + double max_score = 0.0; // block-max upper bound for any doc in this window + bool block_max = false; +}; + +// All scored postings of one query term plus its block-max metadata. +struct TermCursor { + std::vector postings; // ascending docid, exact per-doc scores + std::vector windows; // ascending, covering all postings + size_t pos = 0; // DAAT cursor into postings +}; + +uint32_t current_doc(const TermCursor& c) { + return c.pos < c.postings.size() ? c.postings[c.pos].docid + : std::numeric_limits::max(); +} + +// Reads one slim .frq window's bytes for a slim pod_ref/inline entry (prelude +// stripped). Windowed entries are handled separately via the prelude decode. +Status fetch_slim_window_bytes(const LogicalIndexReader& idx, const DictEntry& entry, + uint64_t frq_base, std::vector* window_owned, + Slice* window) { + if (entry.kind == DictEntryKind::kInline) { + *window = Slice(entry.frq_bytes); + return Status::OK(); + } + uint64_t win_abs = 0; + uint64_t win_len = 0; + RETURN_IF_ERROR(idx.resolve_frq_window(entry, frq_base, &win_abs, &win_len)); + io::BatchRangeFetcher fetcher(idx.reader()); + const size_t h = fetcher.add(win_abs, win_len); + RETURN_IF_ERROR(fetcher.fetch()); + Slice got = fetcher.get(h); + window_owned->assign(got.data(), got.data() + got.size()); + *window = Slice(*window_owned); + return Status::OK(); +} + +// Reads a windowed entry's frq_prelude (block-max columns live here). +Status fetch_prelude(const LogicalIndexReader& idx, const DictEntry& entry, uint64_t frq_base, + FrqPreludeReader* out) { + const auto& region = idx.section_refs().posting_region; + const uint64_t prelude_abs = region.offset + frq_base + entry.frq_off_delta; + io::BatchRangeFetcher fetcher(idx.reader()); + const size_t h = fetcher.add(prelude_abs, entry.prelude_len); + RETURN_IF_ERROR(fetcher.fetch()); + return FrqPreludeReader::open(fetcher.get(h), out); +} + +// Builds per-window block-max bounds from a windowed entry's prelude. Each +// WindowMeta carries the window's max_freq / max_norm and its covered docid +// range (win_base+1 .. last_docid), so bounds come straight from the directory. +Status build_window_bounds(const FrqPreludeReader& prelude, const ScorerContext& ctx, double avgdl, + const Bm25Params& params, std::vector* windows) { + const uint32_t n = prelude.n_windows(); + for (uint32_t w = 0; w < n; ++w) { + WindowMeta m; + RETURN_IF_ERROR(prelude.window(w, &m)); + if (m.doc_count == 0) continue; + WindowBound wb; + wb.first_docid = static_cast(m.win_base) + (w == 0 ? 0u : 1u); + wb.last_docid = m.last_docid; + wb.max_score = ctx.max_score(m.max_freq, m.max_norm, avgdl, params); + wb.block_max = true; + windows->push_back(wb); + } + return Status::OK(); +} + +// Fallback single window covering all postings, bounded by the exact max score +// (always a valid upper bound, so pruning stays correct). +void single_window_fallback(const std::vector& postings, + std::vector* windows) { + if (postings.empty()) return; + WindowBound wb; + wb.first_docid = postings.front().docid; + wb.last_docid = postings.back().docid; + wb.block_max = false; + for (const auto& p : postings) wb.max_score = std::max(wb.max_score, p.score); + windows->push_back(wb); +} + +// Computes exact per-doc BM25 scores from decoded (docid, freq) vectors. +Status score_decoded(const stats::SniiStatsProvider& stats, const ScorerContext& ctx, double avgdl, + const Bm25Params& params, const std::vector& docids, + const std::vector& freqs, std::vector* out) { + out->reserve(docids.size()); + for (size_t i = 0; i < docids.size(); ++i) { + uint8_t norm = 0; + RETURN_IF_ERROR(stats.encoded_norm(docids[i], &norm)); + const uint32_t tf = i < freqs.size() ? freqs[i] : 1; + out->push_back({docids[i], ctx.score(tf, norm, avgdl, params)}); + } + return Status::OK(); +} + +// Decodes a slim/inline term's single .frq window ([dd_region][freq_region]) into +// docids/freqs using the entry's region metadata. +Status decode_slim(const LogicalIndexReader& idx, const DictEntry& entry, uint64_t frq_base, + std::vector* docids, std::vector* freqs) { + std::vector owned; + Slice window; + RETURN_IF_ERROR(fetch_slim_window_bytes(idx, entry, frq_base, &owned, &window)); + const uint64_t dd_len = entry.dd_meta.disk_len; + if (dd_len > window.size()) { + return Status::Error( + "scoring_query: slim dd region exceeds window"); + } + Slice dd_region = window.subslice(0, static_cast(dd_len)); + RETURN_IF_ERROR(format::decode_dd_region(dd_region, entry.dd_meta, + /*win_base=*/0, docids)); + // G16-c freq-dropped segments (write_freq == false) carry a zero-length + // freq region on slim/inline entries. Fail with the SEMANTIC error and NOT + // with INVERTED_INDEX_FILE_CORRUPTED: the Doris segment iterator silently + // downgrades that code to a non-index evaluation, which would mask a + // by-design layout as data corruption once BM25 runs over mixed segments. + if (window.size() == static_cast(dd_len) && !docids->empty()) { + return Status::Error( + "scoring_query: freqs requested but the slim entry has no freq region " + "(freq-dropped positions index)"); + } + Slice freq_region = window.subslice(static_cast(dd_len), + window.size() - static_cast(dd_len)); + return format::decode_freq_region(freq_region, entry.freq_meta, docids->size(), freqs); +} + +// Builds the cursor for a windowed term: tiles all windows for exact scores and +// reads the prelude once for true per-window block-max bounds. +Status build_windowed_cursor(const LogicalIndexReader& idx, const stats::SniiStatsProvider& stats, + const ScorerContext& ctx, const DictEntry& entry, uint64_t frq_base, + uint64_t prx_base, double avgdl, const Bm25Params& params, + TermCursor* cursor) { + reader::DecodedPosting posting; + // Scoring needs freqs for BM25: fetch the FULL windows (want_freq=true). + RETURN_IF_ERROR(reader::read_windowed_posting(idx, entry, frq_base, prx_base, + /*want_positions=*/false, + /*want_freq=*/true, &posting)); + RETURN_IF_ERROR(score_decoded(stats, ctx, avgdl, params, posting.docids, posting.freqs, + &cursor->postings)); + FrqPreludeReader prelude; + if (fetch_prelude(idx, entry, frq_base, &prelude).ok()) { + RETURN_IF_ERROR(build_window_bounds(prelude, ctx, avgdl, params, &cursor->windows)); + } + return Status::OK(); +} + +Status build_resolved_cursor(const LogicalIndexReader& idx, const stats::SniiStatsProvider& stats, + const ScorerContext& ctx, const DictEntry& entry, uint64_t frq_base, + uint64_t prx_base, double avgdl, const Bm25Params& params, + TermCursor* cursor) { + const bool windowed = + entry.kind == DictEntryKind::kPodRef && entry.enc == DictEntryEnc::kWindowed; + if (windowed) { + RETURN_IF_ERROR(build_windowed_cursor(idx, stats, ctx, entry, frq_base, prx_base, avgdl, + params, cursor)); + } else { + std::vector docids; + std::vector freqs; + RETURN_IF_ERROR(decode_slim(idx, entry, frq_base, &docids, &freqs)); + RETURN_IF_ERROR(score_decoded(stats, ctx, avgdl, params, docids, freqs, &cursor->postings)); + } + if (cursor->windows.empty()) { + single_window_fallback(cursor->postings, &cursor->windows); + } + return Status::OK(); +} + +Status accumulate_decoded_candidate_scores(const stats::SniiStatsProvider& stats, + const ScorerContext& scorer, double avgdl, + const Bm25Params& params, + const std::vector& docids, + const std::vector& freqs, + std::span candidates, + std::span scores) { + DCHECK_EQ(candidates.size(), scores.size()); + DCHECK_EQ(docids.size(), freqs.size()); + size_t doc_index = 0; + size_t candidate_index = 0; + while (doc_index < docids.size() && candidate_index < candidates.size()) { + if (docids[doc_index] < candidates[candidate_index]) { + ++doc_index; + continue; + } + if (candidates[candidate_index] < docids[doc_index]) { + ++candidate_index; + continue; + } + uint8_t norm = 0; + RETURN_IF_ERROR(stats.encoded_norm(docids[doc_index], &norm)); + scores[candidate_index] += scorer.score(freqs[doc_index], norm, avgdl, params); + ++doc_index; + ++candidate_index; + } + return Status::OK(); +} + +struct CandidateWindowWork { + WindowMeta meta; + size_t candidate_begin = 0; + size_t candidate_end = 0; + size_t dd_handle = 0; + size_t freq_handle = 0; +}; + +Status accumulate_windowed_candidate_scores(const LogicalIndexReader& idx, + const stats::SniiStatsProvider& stats, + const DictEntry& entry, uint64_t frq_base, + uint64_t prx_base, + const std::vector& candidates, + const ScorerContext& scorer, double avgdl, + const Bm25Params& params, std::vector* scores) { + FrqPreludeReader prelude; + RETURN_IF_ERROR(reader::fetch_windowed_prelude(idx, entry, frq_base, &prelude)); + + std::vector windows; + const uint32_t window_count = prelude.n_windows(); + const bool candidates_cover_segment = + static_cast(candidates.size()) == stats.doc_count() && !candidates.empty() && + candidates.front() == 0 && + static_cast(candidates.back()) + 1 == stats.doc_count(); + bool scan_all = candidates_cover_segment; + if (scan_all) { + windows.resize(window_count); + std::iota(windows.begin(), windows.end(), 0); + } else { + prelude.select_covering_windows(candidates, &windows); + scan_all = windows.size() == window_count; + } + if (windows.empty()) { + return Status::OK(); + } + + // A sparse set must not be re-expanded into a near-full posting by merging + // across unselected windows. Dense/all-window reads retain the existing + // same-term coalescing policy and therefore the existing request shape. + const uint64_t coalesce_gap = scan_all ? reader::kSameTermCoalesceGap : 0; + io::BatchRangeFetcher fetcher(idx.reader(), coalesce_gap); + std::vector work; + work.reserve(windows.size()); + size_t candidate_cursor = 0; + for (uint32_t window : windows) { + CandidateWindowWork item; + RETURN_IF_ERROR(prelude.window(window, &item.meta)); + const uint32_t first_docid = + window == 0 ? 0 : static_cast(item.meta.win_base + 1); + while (candidate_cursor < candidates.size() && candidates[candidate_cursor] < first_docid) { + ++candidate_cursor; + } + item.candidate_begin = candidate_cursor; + while (candidate_cursor < candidates.size() && + candidates[candidate_cursor] <= item.meta.last_docid) { + ++candidate_cursor; + } + item.candidate_end = candidate_cursor; + + reader::WindowAbsRange range; + RETURN_IF_ERROR(reader::windowed_window_range( + idx, entry, frq_base, prx_base, prelude, window, + /*want_positions=*/false, /*want_freq=*/true, &range)); + item.dd_handle = fetcher.add(range.dd_off, range.dd_len); + item.freq_handle = fetcher.add(range.freq_off, range.freq_len); + work.push_back(std::move(item)); + } + RETURN_IF_ERROR(fetcher.fetch()); + + std::vector docids; + std::vector freqs; + std::vector> positions; + for (const auto& item : work) { + docids.clear(); + freqs.clear(); + RETURN_IF_ERROR(reader::decode_window_slices( + item.meta, fetcher.get(item.dd_handle), fetcher.get(item.freq_handle), Slice(), + /*want_positions=*/false, /*want_freq=*/true, &docids, &freqs, &positions)); + const size_t candidate_count = item.candidate_end - item.candidate_begin; + RETURN_IF_ERROR(accumulate_decoded_candidate_scores( + stats, scorer, avgdl, params, docids, freqs, + std::span(candidates) + .subspan(item.candidate_begin, candidate_count), + std::span(*scores).subspan(item.candidate_begin, candidate_count))); + } + return Status::OK(); +} + +Status accumulate_resolved_candidate_scores(const LogicalIndexReader& idx, + const stats::SniiStatsProvider& stats, + const DictEntry& entry, uint64_t frq_base, + uint64_t prx_base, + const std::vector& candidates, + const ScorerContext& scorer, double avgdl, + const Bm25Params& params, std::vector* scores) { + const bool windowed = + entry.kind == DictEntryKind::kPodRef && entry.enc == DictEntryEnc::kWindowed; + if (windowed) { + return accumulate_windowed_candidate_scores(idx, stats, entry, frq_base, prx_base, + candidates, scorer, avgdl, params, scores); + } + + std::vector docids; + std::vector freqs; + RETURN_IF_ERROR(decode_slim(idx, entry, frq_base, &docids, &freqs)); + return accumulate_decoded_candidate_scores(stats, scorer, avgdl, params, docids, freqs, + candidates, *scores); +} + +// Builds the cursor for one term: postings with exact scores + window bounds. +Status build_cursor(const LogicalIndexReader& idx, const stats::SniiStatsProvider& stats, + const std::string& term, const Bm25Params& params, bool* found, + TermCursor* cursor) { + DictEntry entry; + uint64_t frq_base = 0; + uint64_t prx_base = 0; + RETURN_IF_ERROR(idx.lookup(term, found, &entry, &frq_base, &prx_base)); + if (!*found) return Status::OK(); + + const ScorerContext ctx = ScorerContext::make(stats.indexed_doc_count(), entry.df); + return build_resolved_cursor(idx, stats, ctx, entry, frq_base, prx_base, stats.avgdl(), params, + cursor); +} + +// Block-max upper bound for a term at a given docid: the max_score of the window +// covering docid (windows are ascending and contiguous). Beyond the last window +// the bound is 0 (the term cannot contribute). +double term_bound_at(const TermCursor& c, uint32_t docid) { + // Windows are ascending and contiguous; the first window whose last_docid is + // >= docid covers it. Its block-max is a valid upper bound for any contained + // doc, so it also bounds gaps between windows. + for (const auto& w : c.windows) { + if (docid <= w.last_docid) return w.max_score; + } + return 0.0; +} + +// Min-heap keyed on score (smallest at top) maintaining the top-K. +struct TopK { + explicit TopK(uint32_t k) : k_(k) {} + void offer(uint32_t docid, double score) { + if (heap_.size() < k_) { + heap_.push({score, docid}); + return; + } + if (heap_.empty()) return; + const Entry& worst = heap_.top(); // lowest score; ties: largest docid + const bool better = score > worst.first || (score == worst.first && docid < worst.second); + if (better) { + heap_.pop(); + heap_.push({score, docid}); + } + } + double threshold() const { return heap_.size() < k_ ? -1.0 : heap_.top().first; } + + using Entry = std::pair; + struct Cmp { + bool operator()(const Entry& a, const Entry& b) const { + if (a.first != b.first) return a.first > b.first; // min-score at top + return a.second < b.second; // for ties, largest docid at top (evictable) + } + }; + uint32_t k_; + std::priority_queue, Cmp> heap_; +}; + +void drain_sorted(TopK* topk, std::vector* out) { + std::vector all; + while (!topk->heap_.empty()) { + all.push_back({topk->heap_.top().second, topk->heap_.top().first}); + topk->heap_.pop(); + } + std::sort(all.begin(), all.end(), [](const ScoredDoc& a, const ScoredDoc& b) { + if (a.score != b.score) return a.score > b.score; + return a.docid < b.docid; + }); + *out = std::move(all); +} + +Status build_cursors(const LogicalIndexReader& idx, const stats::SniiStatsProvider& stats, + const std::vector& terms, const Bm25Params& params, + std::vector* cursors) { + for (const auto& term : terms) { + bool found = false; + TermCursor c; + RETURN_IF_ERROR(build_cursor(idx, stats, term, params, &found, &c)); + if (found && !c.postings.empty()) cursors->push_back(std::move(c)); + } + return Status::OK(); +} + +} // namespace + +Status scoring_query_candidates(const LogicalIndexReader& idx, + const stats::SniiStatsProvider& segment_stats, + const std::vector& terms, + const roaring::Roaring& final_candidates, double collection_avgdl, + const Bm25Params& params, std::vector* out) { + if (out == nullptr) { + return Status::Error( + "scoring_query_candidates: null out"); + } + out->clear(); + if (final_candidates.isEmpty()) { + return Status::OK(); + } + if (!(collection_avgdl > 0.0)) { + return Status::Error( + "scoring_query_candidates: collection avgdl must be positive"); + } + + std::vector candidate_docids; + candidate_docids.reserve(final_candidates.cardinality()); + for (uint32_t docid : final_candidates) { + candidate_docids.push_back(docid); + } + std::vector candidate_scores(candidate_docids.size(), 0.0); + reader::DictBlockCache dict_block_cache; + + for (const auto& term : terms) { + bool found = false; + DictEntry entry; + uint64_t frq_base = 0; + uint64_t prx_base = 0; + RETURN_IF_ERROR(idx.lookup(term.physical_term, &found, &entry, &frq_base, &prx_base, + &dict_block_cache)); + if (!found) { + continue; + } + + const ScorerContext scorer = ScorerContext::from_idf(term.idf); + RETURN_IF_ERROR(accumulate_resolved_candidate_scores( + idx, segment_stats, entry, frq_base, prx_base, candidate_docids, scorer, + collection_avgdl, params, &candidate_scores)); + } + + std::vector scored_candidates; + scored_candidates.reserve(candidate_docids.size()); + for (size_t i = 0; i < candidate_docids.size(); ++i) { + scored_candidates.push_back({.docid = candidate_docids[i], .score = candidate_scores[i]}); + } + *out = std::move(scored_candidates); + return Status::OK(); +} + +Status scoring_query_exhaustive(const LogicalIndexReader& idx, + const stats::SniiStatsProvider& stats, + const std::vector& terms, uint32_t k, + const Bm25Params& params, std::vector* out) { + if (out == nullptr) + return Status::Error("scoring_query: null out"); + out->clear(); + if (k == 0) return Status::OK(); + + std::vector cursors; + RETURN_IF_ERROR(build_cursors(idx, stats, terms, params, &cursors)); + + std::unordered_map scores; + for (const auto& c : cursors) + for (const auto& p : c.postings) scores[p.docid] += p.score; + + std::vector all; + all.reserve(scores.size()); + for (const auto& [docid, score] : scores) all.push_back({docid, score}); + std::sort(all.begin(), all.end(), [](const ScoredDoc& a, const ScoredDoc& b) { + if (a.score != b.score) return a.score > b.score; + return a.docid < b.docid; + }); + if (all.size() > k) all.resize(k); + *out = std::move(all); + return Status::OK(); +} + +namespace { + +// --- Phase C: selective-fetch (lazy window) WAND ----------------------------- +// +// A LazyTermCursor knows its per-window block-max bounds + docid ranges from the +// frq_prelude WITHOUT fetching any .frq window. Each window's exact (docid,score) +// postings are decoded on first access and cached, so a window is fetched at most +// once and ONLY when the WAND control flow touches a posting in it. Combined with +// window-level SkipTo (advance past whole windows whose last_docid < target via +// the prelude, never fetching them), the offer sequence is byte-identical to the +// eager scoring_query_wand path -- only the bytes read differ. +// +// Soundness: a window is fetched only when lazy_current_doc/lazy_skip_to land the +// cursor inside it, i.e. it covers a candidate the WAND pivot already proved can +// reach the running theta (bound >= theta). lazy_skip_to jumps the cursor to the +// SAME posting (first docid >= target) the eager per-doc walk would, so pivots, +// alignments and offers are identical to the eager path; only windows the eager +// path read-through-but-never-offered-from are skipped. Windows whose block-max +// bound never reaches theta are never the pivot, so never fetched. + +// One query term's lazily-fetched scoring state. +struct LazyTermCursor { + const LogicalIndexReader* idx = nullptr; + const stats::SniiStatsProvider* stats = nullptr; + ScorerContext ctx = ScorerContext::make(1, 1); + Bm25Params params; + DictEntry entry; + uint64_t frq_base = 0; + uint64_t prx_base = 0; + FrqPreludeReader prelude; + bool windowed = false; // false => slim/inline single block already materialized + + std::vector windows; // ascending; from prelude (or slim fallback) + std::vector postings; // sparse: only fetched windows are filled + std::vector win_start; // prefix offsets, size = windows.size()+1 + std::vector fetched; // size = windows.size() + size_t pos = 0; // virtual cursor over all windows' postings +}; + +// Total posting count across all windows (the virtual stream length). +uint32_t total_postings(const LazyTermCursor& c) { + return c.win_start.empty() ? 0 : c.win_start.back(); +} + +// Index of the window whose virtual range contains posting index p (p < total). +uint32_t window_of(const LazyTermCursor& c, uint32_t p) { + const auto it = std::upper_bound(c.win_start.begin(), c.win_start.end(), p); + return static_cast((it - c.win_start.begin()) - 1); +} + +// Fetches + decodes window w into the cursor's posting cache (idempotent). Only +// reached when the WAND proves window w can still contribute to the top-K. +Status materialize_window(LazyTermCursor* c, uint32_t w) { + if (c->fetched[w]) return Status::OK(); + WindowMeta meta; + RETURN_IF_ERROR(c->prelude.window(w, &meta)); + reader::WindowAbsRange r; + RETURN_IF_ERROR(reader::windowed_window_range( + *c->idx, c->entry, c->frq_base, c->prx_base, c->prelude, w, + /*want_positions=*/false, /*want_freq=*/true, &r)); + // Scoring needs docids + freqs: fetch the window's dd sub-range AND freq sub-range. + io::BatchRangeFetcher fetcher(c->idx->reader(), reader::kSameTermCoalesceGap); + const size_t dh = fetcher.add(r.dd_off, r.dd_len); + const size_t fh = fetcher.add(r.freq_off, r.freq_len); + RETURN_IF_ERROR(fetcher.fetch()); + std::vector docids; + std::vector freqs; + std::vector> pos; + RETURN_IF_ERROR(reader::decode_window_slices(meta, fetcher.get(dh), fetcher.get(fh), Slice(), + /*want_positions=*/false, + /*want_freq=*/true, &docids, &freqs, &pos)); + if (docids.size() != c->win_start[w + 1] - c->win_start[w]) { + return Status::Error( + "scoring_query: selective window doc-count drift"); + } + std::vector scored; + RETURN_IF_ERROR( + score_decoded(*c->stats, c->ctx, c->stats->avgdl(), c->params, docids, freqs, &scored)); + std::copy(scored.begin(), scored.end(), c->postings.begin() + c->win_start[w]); + c->fetched[w] = 1; + return Status::OK(); +} + +// Current docid at the cursor, fetching the covering window if needed. Exhausted +// cursor -> UINT32_MAX. +Status lazy_current_doc(LazyTermCursor* c, uint32_t* docid) { + if (c->pos >= total_postings(*c)) { + *docid = std::numeric_limits::max(); + return Status::OK(); + } + const uint32_t w = window_of(*c, static_cast(c->pos)); + RETURN_IF_ERROR(materialize_window(c, w)); + *docid = c->postings[c->pos].docid; + return Status::OK(); +} + +// Advances pos to the first posting with docid >= target, skipping ENTIRE windows +// whose last_docid < target WITHOUT fetching them (prelude-only), then fetching +// just the landing window. Lands on the same posting the eager per-doc walk would. +Status lazy_skip_to(LazyTermCursor* c, uint32_t target) { + const uint32_t total = total_postings(*c); + while (c->pos < total) { + const uint32_t w = window_of(*c, static_cast(c->pos)); + if (c->windows[w].last_docid >= target) break; + c->pos = c->win_start[w + 1]; // skip this window entirely (no fetch) + } + if (c->pos >= total) return Status::OK(); + const uint32_t w = window_of(*c, static_cast(c->pos)); + RETURN_IF_ERROR(materialize_window(c, w)); + while (c->pos < total && c->postings[c->pos].docid < target) ++c->pos; + return Status::OK(); +} + +// Initializes a lazy windowed cursor from the prelude alone: per-window block-max +// bounds + ranges + cache slots, with NO .frq window fetched. +Status build_lazy_windowed(LazyTermCursor* c) { + RETURN_IF_ERROR(reader::fetch_windowed_prelude(*c->idx, c->entry, c->frq_base, &c->prelude)); + RETURN_IF_ERROR( + build_window_bounds(c->prelude, c->ctx, c->stats->avgdl(), c->params, &c->windows)); + // build_window_bounds keeps only non-empty windows, in window order. Build the + // matching prefix-sum of doc_counts over those same non-empty windows so the + // bound list, win_start and fetched stay 1:1. + const uint32_t nb = static_cast(c->windows.size()); + c->win_start.assign(nb + 1, 0); + c->fetched.assign(nb, 0); + uint32_t bi = 0; + uint32_t acc = 0; + for (uint32_t w = 0; w < c->prelude.n_windows() && bi < nb; ++w) { + WindowMeta meta; + RETURN_IF_ERROR(c->prelude.window(w, &meta)); + if (meta.doc_count == 0) continue; + acc += meta.doc_count; + c->win_start[++bi] = acc; + } + c->postings.assign(acc, TermPosting {}); + return Status::OK(); +} + +// Initializes a slim/inline cursor: its single window is small, so fetch + score +// it eagerly (exactly as the existing path). One bound covers all its postings. +Status build_lazy_slim(LazyTermCursor* c) { + std::vector docids; + std::vector freqs; + RETURN_IF_ERROR(decode_slim(*c->idx, c->entry, c->frq_base, &docids, &freqs)); + RETURN_IF_ERROR(score_decoded(*c->stats, c->ctx, c->stats->avgdl(), c->params, docids, freqs, + &c->postings)); + single_window_fallback(c->postings, &c->windows); + c->win_start = {0, static_cast(c->postings.size())}; + c->fetched.assign(1, 1); // already materialized + return Status::OK(); +} + +// Builds a LazyTermCursor for one term: prelude-only for windowed terms (no .frq +// fetched), fully-materialized single window for slim/inline (small). +Status build_lazy_cursor(const LogicalIndexReader& idx, const stats::SniiStatsProvider& stats, + const std::string& term, const Bm25Params& params, bool* found, + LazyTermCursor* c) { + uint64_t prx_base = 0; + RETURN_IF_ERROR(idx.lookup(term, found, &c->entry, &c->frq_base, &prx_base)); + if (!*found) return Status::OK(); + c->idx = &idx; + c->stats = &stats; + c->params = params; + c->prx_base = prx_base; + c->ctx = ScorerContext::make(stats.indexed_doc_count(), c->entry.df); + c->windowed = + c->entry.kind == DictEntryKind::kPodRef && c->entry.enc == DictEntryEnc::kWindowed; + return c->windowed ? build_lazy_windowed(c) : build_lazy_slim(c); +} + +Status selective_build_cursors(const LogicalIndexReader& idx, const stats::SniiStatsProvider& stats, + const std::vector& terms, const Bm25Params& params, + std::vector* cursors) { + for (const auto& term : terms) { + bool found = false; + LazyTermCursor c; + RETURN_IF_ERROR(build_lazy_cursor(idx, stats, term, params, &found, &c)); + if (found && total_postings(c) > 0) cursors->push_back(std::move(c)); + } + return Status::OK(); +} + +// Block-max upper bound for a lazy cursor at docid: block_max of the window +// covering docid (ascending, contiguous). Beyond the last window -> 0. Same +// semantics as term_bound_at over the eager cursor's window list. +double lazy_term_bound_at(const LazyTermCursor& c, uint32_t docid) { + for (const auto& w : c.windows) { + if (docid <= w.last_docid) return w.max_score; + } + return 0.0; +} + +// Sorts cursors ascending by current docid (materializing each cursor's current +// covering window), returning the smallest current docid via *front. +Status selective_sort_by_doc(std::vector* cursors, uint32_t* front) { + std::vector cur(cursors->size()); + for (size_t i = 0; i < cursors->size(); ++i) { + RETURN_IF_ERROR(lazy_current_doc(&(*cursors)[i], &cur[i])); + } + std::vector order(cursors->size()); + for (size_t i = 0; i < order.size(); ++i) order[i] = i; + std::sort(order.begin(), order.end(), [&](size_t a, size_t b) { return cur[a] < cur[b]; }); + std::vector sorted; + sorted.reserve(cursors->size()); + for (size_t i : order) sorted.push_back(std::move((*cursors)[i])); + *cursors = std::move(sorted); + *front = order.empty() ? std::numeric_limits::max() : cur[order.front()]; + return Status::OK(); +} + +// Finds the pivot term: the first cursor (current-docid order) at which the +// accumulated block-max bound reaches theta. >= keeps boundary ties (matching the +// exhaustive total order). *found=false when no remaining doc can beat theta. +Status selective_pivot(std::vector* cursors, double theta, size_t* pivot, + uint32_t* pivot_doc, bool* found) { + double bound = 0.0; + *found = false; + for (size_t i = 0; i < cursors->size(); ++i) { + uint32_t d = 0; + RETURN_IF_ERROR(lazy_current_doc(&(*cursors)[i], &d)); + if (d == std::numeric_limits::max()) break; + bound += lazy_term_bound_at((*cursors)[i], d); + if (bound >= theta) { + *pivot = i; + *pivot_doc = d; + *found = true; + return Status::OK(); + } + } + return Status::OK(); +} + +// Scores the aligned pivot doc exactly (summing all cursors AT pivot_doc) and +// advances those cursors by one posting. +Status selective_score_pivot(std::vector* cursors, uint32_t pivot_doc, TopK* topk) { + double doc_score = 0.0; + for (auto& c : *cursors) { + uint32_t d = 0; + RETURN_IF_ERROR(lazy_current_doc(&c, &d)); + if (d == pivot_doc) { + doc_score += c.postings[c.pos].score; // window already materialized + ++c.pos; + } + } + topk->offer(pivot_doc, doc_score); + return Status::OK(); +} + +// Advances the first lagging cursor (current doc < pivot_doc) up to pivot_doc. +Status selective_advance_lagging(std::vector* cursors, uint32_t pivot_doc) { + for (auto& c : *cursors) { + uint32_t d = 0; + RETURN_IF_ERROR(lazy_current_doc(&c, &d)); + if (d < pivot_doc) { + RETURN_IF_ERROR(lazy_skip_to(&c, pivot_doc)); + return Status::OK(); + } + } + return Status::OK(); +} + +// One WAND iteration body: sort, pick pivot, then either score (aligned) or skip +// a lagging cursor forward. *done=true ends the loop. +Status selective_step(std::vector* cursors, TopK* topk, bool* done) { + uint32_t front = 0; + RETURN_IF_ERROR(selective_sort_by_doc(cursors, &front)); + if (cursors->empty() || front == std::numeric_limits::max()) { + *done = true; + return Status::OK(); + } + size_t pivot = 0; + uint32_t pivot_doc = 0; + bool found_pivot = false; + RETURN_IF_ERROR(selective_pivot(cursors, topk->threshold(), &pivot, &pivot_doc, &found_pivot)); + if (!found_pivot) { + *done = true; + return Status::OK(); + } + if (front == pivot_doc) { + return selective_score_pivot(cursors, pivot_doc, topk); + } + return selective_advance_lagging(cursors, pivot_doc); +} + +Status selective_wand_loop(std::vector* cursors, TopK* topk) { + bool done = false; + while (!done) { + RETURN_IF_ERROR(selective_step(cursors, topk, &done)); + } + return Status::OK(); +} + +} // namespace + +Status scoring_query_wand_selective(const LogicalIndexReader& idx, + const stats::SniiStatsProvider& stats, + const std::vector& terms, uint32_t k, + const Bm25Params& params, std::vector* out) { + if (out == nullptr) + return Status::Error("scoring_query: null out"); + out->clear(); + if (k == 0) return Status::OK(); + + std::vector cursors; + RETURN_IF_ERROR(selective_build_cursors(idx, stats, terms, params, &cursors)); + + TopK topk(k); + RETURN_IF_ERROR(selective_wand_loop(&cursors, &topk)); + drain_sorted(&topk, out); + return Status::OK(); +} + +Status scoring_query_wand(const LogicalIndexReader& idx, const stats::SniiStatsProvider& stats, + const std::vector& terms, uint32_t k, + const Bm25Params& params, std::vector* out) { + if (out == nullptr) + return Status::Error("scoring_query: null out"); + out->clear(); + if (k == 0) return Status::OK(); + + std::vector cursors; + RETURN_IF_ERROR(build_cursors(idx, stats, terms, params, &cursors)); + + TopK topk(k); + // Document-at-a-time WAND with block-max bounds. + while (true) { + // Sort cursors by current docid (ascending; exhausted cursors sink). + std::sort(cursors.begin(), cursors.end(), [](const TermCursor& a, const TermCursor& b) { + return current_doc(a) < current_doc(b); + }); + if (cursors.empty() || + current_doc(cursors.front()) == std::numeric_limits::max()) { + break; + } + + const double theta = topk.threshold(); + // Accumulate block-max upper bounds in docid order to find the pivot term. + double bound = 0.0; + size_t pivot = 0; + bool found_pivot = false; + for (size_t i = 0; i < cursors.size(); ++i) { + const uint32_t d = current_doc(cursors[i]); + if (d == std::numeric_limits::max()) break; + bound += term_bound_at(cursors[i], d); + // Use >= (not >) so a doc whose upper bound only TIES the K-th threshold is + // still explored and exact-scored: under the (score desc, docid asc) total + // order a tie can still evict the current K-th entry (smaller docid wins), + // exactly as the exhaustive path would. Strict > would wrongly prune ties. + if (bound >= theta) { + pivot = i; + found_pivot = true; + break; + } + } + if (!found_pivot) break; // no doc can beat the threshold anymore. + + const uint32_t pivot_doc = current_doc(cursors[pivot]); + if (current_doc(cursors.front()) == pivot_doc) { + // All cursors at the pivot doc are aligned: score it exactly. + double doc_score = 0.0; + for (auto& c : cursors) { + if (current_doc(c) == pivot_doc) { + doc_score += c.postings[c.pos].score; + ++c.pos; + } + } + topk.offer(pivot_doc, doc_score); + } else { + // Advance a lagging cursor toward pivot_doc (skip docs it cannot win on). + for (auto& c : cursors) { + if (current_doc(c) < pivot_doc) { + while (c.pos < c.postings.size() && c.postings[c.pos].docid < pivot_doc) { + ++c.pos; + } + break; + } + } + } + } + drain_sorted(&topk, out); + return Status::OK(); +} + +} // namespace doris::snii::query diff --git a/be/src/storage/index/snii/query/scoring_query.h b/be/src/storage/index/snii/query/scoring_query.h new file mode 100644 index 00000000000000..e8e83a7881efdb --- /dev/null +++ b/be/src/storage/index/snii/query/scoring_query.h @@ -0,0 +1,98 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/query/bm25_scorer.h" +#include "storage/index/snii/reader/logical_index_reader.h" +#include "storage/index/snii/stats/snii_stats_provider.h" + +// scoring_query -- top-K BM25 scored retrieval over one logical index for one or +// more query terms. Two entry points produce IDENTICAL rankings: +// - scoring_query_exhaustive(): scores every candidate document (the baseline +// correctness oracle). +// - scoring_query_wand(): a block-max / WAND-style optimization that uses the +// per-window max_freq / max_norm columns from the frq_prelude to bound each +// window's best possible score and SKIP windows that cannot enter the +// current top-K. A window without block-max stats (slim/inline entries or a +// missing prelude) is never pruned, so the result still equals the +// exhaustive ranking. +// +// Results are sorted by score descending; ties are broken by ascending docid so +// the ordering is deterministic and the two paths compare equal. +namespace doris::snii::query { + +// One scored hit. +struct ScoredDoc { + uint32_t docid = 0; + double score = 0.0; +}; + +// One logical scoring clause after its plain term has been routed to this +// segment's physical key. IDF remains collection-scoped and is never derived +// from the physical term's segment-local dictionary entry. +struct CollectionScoringTerm { + std::string physical_term; + double idf = 0.0; +}; + +// Scores every document in final_candidates using collection-scoped IDF and +// avgdl plus segment-local TF/norm. Results are returned in ascending docid +// order. Repeated terms are repeated scoring clauses. This path deliberately +// does not use the segment-local WAND bounds below. +Status scoring_query_candidates(const reader::LogicalIndexReader& idx, + const stats::SniiStatsProvider& segment_stats, + const std::vector& terms, + const roaring::Roaring& final_candidates, double collection_avgdl, + const Bm25Params& params, std::vector* out); + +// Exhaustive baseline: score every doc that contains any query term, return the +// top-k by score. params controls k1/b. Unknown terms are skipped. +Status scoring_query_exhaustive(const reader::LogicalIndexReader& idx, + const stats::SniiStatsProvider& stats, + const std::vector& terms, uint32_t k, + const Bm25Params& params, std::vector* out); + +// WAND-style block-max pruning. MUST return the same top-k as the exhaustive +// path. Windows whose block-max upper bound cannot beat the current k-th score +// are skipped; windows lacking block-max stats are scored fully. +Status scoring_query_wand(const reader::LogicalIndexReader& idx, + const stats::SniiStatsProvider& stats, + const std::vector& terms, uint32_t k, + const Bm25Params& params, std::vector* out); + +// SELECTIVE-FETCH block-max WAND (design spec section 5, "Phase C"). Same WAND / +// theta / >= tie machinery as scoring_query_wand, but it DEFERS the .frq window +// fetch: for each windowed term it first reads ONLY the frq_prelude (block-max +// columns), then fetches a term's .frq window lazily and at most once -- and ONLY +// when the running block-max bound proves a doc in that window can still reach the +// top-K (bound >= theta). A window the bound rules out is never fetched. The +// result (top-K docids AND scores, INCLUDING ties) is byte-identical to +// scoring_query_exhaustive / scoring_query_wand; only the bytes read differ. +// Slim/inline terms (no prelude) are fetched fully, exactly as today. +Status scoring_query_wand_selective(const reader::LogicalIndexReader& idx, + const stats::SniiStatsProvider& stats, + const std::vector& terms, uint32_t k, + const Bm25Params& params, std::vector* out); + +} // namespace doris::snii::query diff --git a/be/src/storage/index/snii/query/sloppy_phrase_matcher.cpp b/be/src/storage/index/snii/query/sloppy_phrase_matcher.cpp new file mode 100644 index 00000000000000..d7e1ede9261c50 --- /dev/null +++ b/be/src/storage/index/snii/query/sloppy_phrase_matcher.cpp @@ -0,0 +1,266 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/query/internal/sloppy_phrase_matcher.h" + +#include +#include + +#include "common/check.h" + +namespace doris::snii::query::internal { + +SloppyPhraseMatcher::SloppyPhraseMatcher(std::span phrase_plan_index, + std::span position_offsets, uint32_t slop, + bool ordered) + : phrase_plan_index_(phrase_plan_index.begin(), phrase_plan_index.end()), + position_offsets_(position_offsets.begin(), position_offsets.end()), + slop_(slop), + ordered_(ordered), + clauses_(phrase_plan_index.size()) { + DORIS_CHECK_EQ(phrase_plan_index_.size(), position_offsets_.size()); + DORIS_CHECK_GT(phrase_plan_index_.size(), 1); + DORIS_CHECK_GT(slop_, 0); + heap_.reserve(phrase_plan_index_.size()); + for (size_t i = 0; i < phrase_plan_index_.size() && !has_repeats_; ++i) { + for (size_t j = 0; j < i; ++j) { + has_repeats_ = phrase_plan_index_[i] == phrase_plan_index_[j]; + if (has_repeats_) { + break; + } + } + } +} + +float SloppyPhraseMatcher::match(std::span positions, + bool collect_frequency) { + DCHECK_EQ(positions.size(), clauses_.size()); + return ordered_ ? match_ordered(positions, collect_frequency) + : match_unordered(positions, collect_frequency); +} + +bool SloppyPhraseMatcher::initialize_unordered(std::span positions) { + heap_.clear(); + end_ = std::numeric_limits::min(); + for (size_t i = 0; i < clauses_.size(); ++i) { + Clause& clause = clauses_[i]; + clause.positions = positions[i]; + DCHECK(clause.positions.first != clause.positions.second); + clause.raw_position = *clause.positions.first; + clause.next = clause.positions.first + 1; + clause.adjusted_position = static_cast(clause.raw_position) - position_offsets_[i]; + clause.has_position = true; + } + + if (has_repeats_) { + for (size_t i = 0; i < clauses_.size(); ++i) { + size_t preceding_repeats = 0; + for (size_t j = 0; j < i; ++j) { + preceding_repeats += phrase_plan_index_[i] == phrase_plan_index_[j]; + } + for (size_t repeat = 0; repeat < preceding_repeats; ++repeat) { + if (!advance_clause(i, false)) { + positioned_ = false; + return false; + } + } + } + } + + for (size_t i = 0; i < clauses_.size(); ++i) { + end_ = std::max(end_, clauses_[i].adjusted_position); + heap_.push_back(i); + } + rebuild_heap(); + positioned_ = true; + return true; +} + +bool SloppyPhraseMatcher::advance_clause(size_t clause_index, bool update_end) { + Clause& clause = clauses_[clause_index]; + if (clause.next == clause.positions.second) { + return false; + } + clause.raw_position = *clause.next++; + clause.adjusted_position = + static_cast(clause.raw_position) - position_offsets_[clause_index]; + if (update_end) { + end_ = std::max(end_, clause.adjusted_position); + } + return true; +} + +size_t SloppyPhraseMatcher::collision(size_t clause_index) const { + const Clause& clause = clauses_[clause_index]; + for (size_t i = 0; i < clauses_.size(); ++i) { + if (i != clause_index && phrase_plan_index_[i] == phrase_plan_index_[clause_index] && + clauses_[i].raw_position == clause.raw_position) { + return i; + } + } + return clauses_.size(); +} + +bool SloppyPhraseMatcher::clause_less(size_t left, size_t right) const { + const Clause& left_clause = clauses_[left]; + const Clause& right_clause = clauses_[right]; + if (left_clause.adjusted_position != right_clause.adjusted_position) { + return left_clause.adjusted_position < right_clause.adjusted_position; + } + if (position_offsets_[left] != position_offsets_[right]) { + return position_offsets_[left] < position_offsets_[right]; + } + return left < right; +} + +bool SloppyPhraseMatcher::clause_greater(size_t left, size_t right) const { + return left != right && !clause_less(left, right); +} + +bool SloppyPhraseMatcher::advance_repeat_collisions(size_t clause_index) { + size_t current = clause_index; + size_t other = collision(current); + while (other != clauses_.size()) { + current = clause_less(current, other) ? current : other; + if (!advance_clause(current, true)) { + return false; + } + other = collision(current); + } + rebuild_heap(); + return true; +} + +void SloppyPhraseMatcher::rebuild_heap() { + const auto greater = [this](size_t left, size_t right) { return clause_greater(left, right); }; + std::make_heap(heap_.begin(), heap_.end(), greater); +} + +size_t SloppyPhraseMatcher::pop_heap() { + const auto greater = [this](size_t left, size_t right) { return clause_greater(left, right); }; + std::pop_heap(heap_.begin(), heap_.end(), greater); + const size_t result = heap_.back(); + heap_.pop_back(); + return result; +} + +void SloppyPhraseMatcher::push_heap(size_t clause) { + const auto greater = [this](size_t left, size_t right) { return clause_greater(left, right); }; + heap_.push_back(clause); + std::push_heap(heap_.begin(), heap_.end(), greater); +} + +bool SloppyPhraseMatcher::next_unordered_match(uint64_t* match_width) { + if (!positioned_ || heap_.size() < 2) { + return false; + } + size_t clause = pop_heap(); + *match_width = static_cast(end_ - clauses_[clause].adjusted_position); + int64_t next_position = clauses_[heap_.front()].adjusted_position; + while (advance_clause(clause, true)) { + if (has_repeats_ && !advance_repeat_collisions(clause)) { + break; + } + if (clauses_[clause].adjusted_position > next_position) { + push_heap(clause); + if (*match_width <= slop_) { + return true; + } + clause = pop_heap(); + next_position = clauses_[heap_.front()].adjusted_position; + *match_width = static_cast(end_ - clauses_[clause].adjusted_position); + } else { + *match_width = std::min( + *match_width, static_cast(end_ - clauses_[clause].adjusted_position)); + } + } + positioned_ = false; + return *match_width <= slop_; +} + +float SloppyPhraseMatcher::match_unordered(std::span positions, + bool collect_frequency) { + if (!initialize_unordered(positions)) { + return 0.0F; + } + float frequency = 0.0F; + uint64_t match_width = 0; + while (next_unordered_match(&match_width)) { + if (!collect_frequency) { + return 1.0F; + } + frequency += 1.0F / (1.0F + static_cast(match_width)); + } + return frequency; +} + +bool SloppyPhraseMatcher::advance_ordered_to(size_t clause_index, int64_t target) { + Clause& clause = clauses_[clause_index]; + while (!clause.has_position || static_cast(clause.raw_position) < target) { + if (clause.next == clause.positions.second) { + return false; + } + clause.raw_position = *clause.next++; + clause.has_position = true; + } + return true; +} + +float SloppyPhraseMatcher::match_ordered(std::span positions, + bool collect_frequency) { + for (size_t i = 0; i < clauses_.size(); ++i) { + clauses_[i].positions = positions[i]; + clauses_[i].next = positions[i].first; + clauses_[i].has_position = false; + } + + float frequency = 0.0F; + Clause& first = clauses_.front(); + while (first.next != first.positions.second) { + first.raw_position = *first.next++; + first.has_position = true; + int64_t previous_start = + static_cast(first.raw_position) - position_offsets_.front(); + uint64_t match_width = 0; + bool all_terms_positioned = true; + for (size_t i = 1; i < clauses_.size(); ++i) { + const int64_t target = previous_start + position_offsets_[i]; + if (!advance_ordered_to(i, target)) { + all_terms_positioned = false; + break; + } + const int64_t current_start = + static_cast(clauses_[i].raw_position) - position_offsets_[i]; + DCHECK_GE(current_start, previous_start); + match_width += static_cast(current_start - previous_start); + if (match_width > slop_) { + all_terms_positioned = false; + break; + } + previous_start = current_start; + } + if (all_terms_positioned) { + if (!collect_frequency) { + return 1.0F; + } + frequency += 1.0F / (1.0F + static_cast(match_width)); + } + } + return frequency; +} + +} // namespace doris::snii::query::internal diff --git a/be/src/storage/index/snii/query/term_expansion.cpp b/be/src/storage/index/snii/query/term_expansion.cpp new file mode 100644 index 00000000000000..ea4b682df783bd --- /dev/null +++ b/be/src/storage/index/snii/query/term_expansion.cpp @@ -0,0 +1,168 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/query/internal/term_expansion.h" + +#include +#include +#include +#include + +#include "storage/index/inverted/common_grams/common_grams_key_codec.h" +#include "storage/index/snii/format/phrase_bigram.h" +#include "storage/index/snii/query/internal/docid_posting_reader.h" +#include "storage/index/snii/query/internal/docid_union.h" +#include "storage/index/snii/query/internal/plain_term_routing.h" +#include "storage/index/snii/reader/dict_block_cache.h" + +namespace doris::snii::query::internal { +namespace { + +Status legacy_raw_prefix_exists(const reader::LogicalIndexReader& idx, std::string_view prefix, + bool* exists, reader::DictBlockCache* cache) { + DORIS_CHECK(exists != nullptr); + *exists = false; + return idx.visit_prefix_terms( + prefix, + [&](reader::LogicalIndexReader::PrefixHit&&, bool* stop) -> Status { + *exists = true; + *stop = true; + return Status::OK(); + }, + cache); +} + +Status prove_legacy_raw_has_no_reserved_terms(const reader::LogicalIndexReader& idx, + reader::DictBlockCache* cache) { + bool exists = false; + RETURN_IF_ERROR(legacy_raw_prefix_exists(idx, segment_v2::inverted_index::CG_V1_MARKER, &exists, + cache)); + if (!exists) { + RETURN_IF_ERROR( + legacy_raw_prefix_exists(idx, format::kPhraseBigramTermMarker, &exists, cache)); + } + if (exists) { + return Status::Error( + "SNII legacy raw expansion overlaps an existing internal term namespace"); + } + return Status::OK(); +} + +} // namespace + +Status visit_expanded_plain_terms(const reader::LogicalIndexReader& idx, + std::string_view enum_prefix, const TermMatcher& matches, + const reader::LogicalIndexReader::PrefixHitVisitor& visitor, + int32_t max_expansions) { + if (!matches || !visitor) { + return Status::Error( + "term_expansion: null matcher or visitor"); + } + + std::string physical_prefix; + bool representable = false; + reader::DictBlockCache dict_cache(/*max_entries=*/1); + const auto version = plain_term_key_version(idx); + if (version == segment_v2::inverted_index::PlainTermKeyVersion::kLegacyRaw && + enum_prefix.empty()) { + RETURN_IF_ERROR(prove_legacy_raw_has_no_reserved_terms(idx, &dict_cache)); + representable = true; + } else { + RETURN_IF_ERROR( + route_plain_enumeration_prefix(idx, enum_prefix, &physical_prefix, &representable)); + } + if (!representable) { + return Status::OK(); + } + + int32_t count = 0; + bool stop_expansion = false; + std::string decoded_scratch; + const auto visit_hit = [&](reader::LogicalIndexReader::PrefixHit&& hit, bool* stop) -> Status { + std::string_view logical_term; + if (version != segment_v2::inverted_index::PlainTermKeyVersion::kEscapedV1 || + !hit.term.starts_with(segment_v2::inverted_index::PLAIN_ESCAPE_PREFIX)) { + if (version == segment_v2::inverted_index::PlainTermKeyVersion::kEscapedV1) { + DCHECK(!segment_v2::inverted_index::is_internal_term_key(hit.term)); + } + logical_term = hit.term; + decoded_scratch.clear(); + } else { + auto decoded = segment_v2::inverted_index::decode_plain_term_view(hit.term, version, + &decoded_scratch); + if (!decoded.has_value()) { + return std::move(decoded.error()); + } + logical_term = *decoded; + } + if (!matches(logical_term)) { + return Status::OK(); + } + if (!decoded_scratch.empty()) { + hit.term = decoded_scratch; + } + bool visitor_stop = false; + RETURN_IF_ERROR(visitor(std::move(hit), &visitor_stop)); + ++count; + *stop = visitor_stop || (max_expansions > 0 && count >= max_expansions); + stop_expansion = *stop; + return Status::OK(); + }; + + if (version == segment_v2::inverted_index::PlainTermKeyVersion::kEscapedV1 && + physical_prefix.empty()) { + RETURN_IF_ERROR(idx.visit_term_range( + /*lower_inclusive=*/ {}, segment_v2::inverted_index::INTERNAL_TERM_NAMESPACE_BEGIN, + visit_hit, &dict_cache)); + if (!stop_expansion && (max_expansions <= 0 || count < max_expansions)) { + RETURN_IF_ERROR( + idx.visit_term_range(segment_v2::inverted_index::INTERNAL_TERM_NAMESPACE_END, + /*upper_exclusive=*/std::nullopt, visit_hit, &dict_cache)); + } + } else { + RETURN_IF_ERROR(idx.visit_prefix_terms( + physical_prefix, + [&](reader::LogicalIndexReader::PrefixHit&& hit, bool* stop) -> Status { + if (version == segment_v2::inverted_index::PlainTermKeyVersion::kEscapedV1) { + DCHECK(!segment_v2::inverted_index::is_internal_term_key(hit.term)); + } + return visit_hit(std::move(hit), stop); + }, + &dict_cache)); + } + return Status::OK(); +} + +Status emit_expanded_docid_union(const reader::LogicalIndexReader& idx, + std::string_view enum_prefix, const TermMatcher& matches, + DocIdSink* const sink, int32_t max_expansions) { + if (sink == nullptr) { + return Status::Error("term_expansion: null sink"); + } + + std::vector postings; + RETURN_IF_ERROR(visit_expanded_plain_terms( + idx, enum_prefix, matches, + [&](reader::LogicalIndexReader::PrefixHit&& hit, bool*) { + postings.push_back({std::move(hit.entry), hit.frq_base, hit.prx_base}); + return Status::OK(); + }, + max_expansions)); + return emit_docid_union(idx, postings, sink); +} + +} // namespace doris::snii::query::internal diff --git a/be/src/storage/index/snii/query/term_query.cpp b/be/src/storage/index/snii/query/term_query.cpp new file mode 100644 index 00000000000000..633229eab67e97 --- /dev/null +++ b/be/src/storage/index/snii/query/term_query.cpp @@ -0,0 +1,58 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/query/term_query.h" + +#include + +#include "storage/index/snii/format/dict_entry.h" +#include "storage/index/snii/query/internal/docid_posting_reader.h" + +namespace doris::snii::query { + +using format::DictEntry; +using reader::LogicalIndexReader; + +Status term_query(const LogicalIndexReader& idx, std::string_view term, + std::vector* docids) { + if (docids == nullptr) + return Status::Error("term_query: null out"); + docids->clear(); + VectorDocIdSink sink(*docids); + return term_query(idx, term, &sink); +} + +Status term_query(const LogicalIndexReader& idx, std::string_view term, DocIdSink* sink) { + if (sink == nullptr) + return Status::Error("term_query: null sink"); + + bool found = false; + DictEntry entry; + uint64_t frq_base = 0; + uint64_t prx_base = 0; + RETURN_IF_ERROR(idx.lookup(term, &found, &entry, &frq_base, &prx_base)); + if (!found) return Status::OK(); + return internal::read_docid_posting(idx, entry, frq_base, prx_base, sink); +} + +Status term_query(const LogicalIndexReader& idx, std::string_view term, + std::vector* docids, QueryProfile* profile) { + QueryProfileScope profile_scope(idx.reader(), profile); + return term_query(idx, term, docids); +} + +} // namespace doris::snii::query diff --git a/be/src/storage/index/snii/query/term_query.h b/be/src/storage/index/snii/query/term_query.h new file mode 100644 index 00000000000000..7296acd9c60702 --- /dev/null +++ b/be/src/storage/index/snii/query/term_query.h @@ -0,0 +1,41 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/query/docid_sink.h" +#include "storage/index/snii/query/query_profile.h" +#include "storage/index/snii/reader/logical_index_reader.h" + +// term_query -- the simplest SNII query: return the sorted docid set that +// contains term. It runs the term lookup on the logical index, then issues a +// single batched .frq range read (one serial round) to decode the postings. +// Absent term -> empty result (OK status). +namespace doris::snii::query { + +Status term_query(const reader::LogicalIndexReader& idx, std::string_view term, + std::vector* docids); +Status term_query(const reader::LogicalIndexReader& idx, std::string_view term, DocIdSink* sink); +Status term_query(const reader::LogicalIndexReader& idx, std::string_view term, + std::vector* docids, QueryProfile* profile); + +} // namespace doris::snii::query diff --git a/be/src/storage/index/snii/query/wildcard_query.cpp b/be/src/storage/index/snii/query/wildcard_query.cpp new file mode 100644 index 00000000000000..e8313c499797aa --- /dev/null +++ b/be/src/storage/index/snii/query/wildcard_query.cpp @@ -0,0 +1,77 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/query/wildcard_query.h" + +#include +#include +#include +#include + +#include "storage/index/snii/query/internal/term_expansion.h" +#include "storage/index/snii/query/internal/wildcard_matcher.h" + +namespace doris::snii::query { + +namespace { + +std::string literal_prefix_for_wildcard(std::string_view pattern) { + std::string out; + for (char c : pattern) { + if (c == '*' || c == '?') { + break; + } + out.push_back(c); + } + return out; +} + +} // namespace + +Status wildcard_query(const reader::LogicalIndexReader& idx, std::string_view pattern, + std::vector* const docids, int32_t max_expansions) { + if (docids == nullptr) { + return Status::Error("wildcard_query: null out"); + } + docids->clear(); + VectorDocIdSink sink(*docids); + return wildcard_query(idx, pattern, &sink, max_expansions); +} + +Status wildcard_query(const reader::LogicalIndexReader& idx, std::string_view pattern, + std::vector* const docids, QueryProfile* profile, + int32_t max_expansions) { + QueryProfileScope profile_scope(idx.reader(), profile); + return wildcard_query(idx, pattern, docids, max_expansions); +} + +Status wildcard_query(const reader::LogicalIndexReader& idx, std::string_view pattern, + DocIdSink* const sink, int32_t max_expansions) { + if (sink == nullptr) { + return Status::Error("wildcard_query: null sink"); + } + const std::string enum_prefix = literal_prefix_for_wildcard(pattern); + // Request-scoped matcher: its two DP scratch rows are reused across every + // visited dictionary term, so the whole-dictionary scan triggered by a + // leading wildcard performs O(1) scratch allocations instead of O(2N). + internal::WildcardMatcher<> matcher(pattern); + return internal::emit_expanded_docid_union( + idx, enum_prefix, [&matcher](std::string_view term) { return matcher(term); }, sink, + max_expansions); +} + +} // namespace doris::snii::query diff --git a/be/src/storage/index/snii/query/wildcard_query.h b/be/src/storage/index/snii/query/wildcard_query.h new file mode 100644 index 00000000000000..66c08b18ae270e --- /dev/null +++ b/be/src/storage/index/snii/query/wildcard_query.h @@ -0,0 +1,42 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/query/docid_sink.h" +#include "storage/index/snii/query/query_profile.h" +#include "storage/index/snii/reader/logical_index_reader.h" + +// wildcard_query -- MATCH_WILDCARD semantics over dictionary terms. `*` matches +// any byte sequence, `?` matches one byte, and all other bytes match literally. +// Matching terms are executed as a sorted deduplicated docid union. +namespace doris::snii::query { + +Status wildcard_query(const reader::LogicalIndexReader& idx, std::string_view pattern, + std::vector* const docids, int32_t max_expansions = 0); +Status wildcard_query(const reader::LogicalIndexReader& idx, std::string_view pattern, + std::vector* const docids, QueryProfile* profile, + int32_t max_expansions = 0); +Status wildcard_query(const reader::LogicalIndexReader& idx, std::string_view pattern, + DocIdSink* const sink, int32_t max_expansions = 0); + +} // namespace doris::snii::query diff --git a/be/src/storage/index/snii/reader/dict_block_cache.h b/be/src/storage/index/snii/reader/dict_block_cache.h new file mode 100644 index 00000000000000..18c65978ee9263 --- /dev/null +++ b/be/src/storage/index/snii/reader/dict_block_cache.h @@ -0,0 +1,124 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/format/dict_block.h" + +// DictBlockCache -- a REQUEST-SCOPED (per-query) MRU cache of decoded DICT +// blocks, keyed by block ordinal. +// +// Why request-scoped (and not a reader-level shared cache): the same DICT block +// is decoded once per LogicalIndexReader::lookup() today, so a multi-term query +// (phrase / boolean / conjunction) whose terms fall in the same block re-runs +// the zstd decompress + CRC verify + anchor parse for every term. Threading one +// of these caches through a single query's lookups collapses that to a single +// decode per unique block. +// +// CONCURRENCY: this object carries NO shared mutable state and is intentionally +// NOT thread-safe. It is meant to live on one query's stack/context and be used +// by a single thread; concurrent queries each own a separate cache. The shared +// LogicalIndexReader therefore stays const and lock-free -- no lock is ever held +// across a decode/IO. (The cross-query, lock-striped variant that would let +// queries share decoded blocks is deferred to the T26 concurrency work.) +namespace doris::snii::reader { + +// A decoded DICT block with stable backing storage. Heap-allocated and owned by +// a shared_ptr so the embedded DictBlockReader's Slice into `bytes` stays valid +// for the whole lifetime of any pin handed to a caller -- even after the block +// has been evicted from the cache. +struct DecodedDictBlock { + std::vector bytes; // decompressed (or raw) block bytes + format::DictBlockReader reader; // its Slice points into `bytes` +}; + +class DictBlockCache { +public: + // Loads (decodes) the block for an ordinal into a freshly heap-allocated + // DecodedDictBlock. Always invoked OUTSIDE any cache bookkeeping; it performs + // the file read + optional zstd decompress + CRC/anchor parse. + using Loader = std::function*)>; + + // A small fixed bound is enough for a single query: it only needs to keep the + // handful of distinct blocks touched while resolving one query's terms. + static constexpr size_t kDefaultMaxEntries = 8; + + DictBlockCache() = default; + explicit DictBlockCache(size_t max_entries) + : max_entries_(max_entries == 0 ? 1 : max_entries) {} + + // Returns the decoded block for `ordinal`, invoking `loader` only on a miss. + // The returned pin keeps the block alive for the caller's use regardless of + // any later eviction. On a hit, `loader` is not called (no re-decode). + Status get_or_load(uint32_t ordinal, const Loader& loader, + std::shared_ptr* out) { + if (auto it = index_.find(ordinal); it != index_.end()) { + order_.splice(order_.begin(), order_, it->second); // promote to MRU + *out = it->second->block; + return Status::OK(); + } + + std::shared_ptr loaded; + // decode happens here, never under a lock (explicit Status, header-safe: + // RETURN_IF_ERROR would need a bare `Status` in scope). + if (Status st = loader(&loaded); !st.ok()) { + return st; + } + order_.push_front(Entry {.ordinal = ordinal, .block = loaded}); + index_[ordinal] = order_.begin(); + evict_overflow(); + *out = std::move(loaded); + return Status::OK(); + } + + // Number of resident (non-evicted) entries -- bounded by max_entries(). + size_t size() const { return index_.size(); } + size_t max_entries() const { return max_entries_; } + +private: + struct Entry { + uint32_t ordinal = 0; + std::shared_ptr block; + }; + + // Drops least-recently-used entries until the bound holds. Evicting only + // releases the cache's reference; any pin a caller still holds keeps the + // block (and its reader's Slice) alive. + void evict_overflow() { + while (index_.size() > max_entries_) { + const Entry& victim = order_.back(); + index_.erase(victim.ordinal); + order_.pop_back(); + } + } + + size_t max_entries_ = kDefaultMaxEntries; + std::list order_; // front = most recently used + std::unordered_map::iterator> index_; +}; + +} // namespace doris::snii::reader diff --git a/be/src/storage/index/snii/reader/logical_index_reader.cpp b/be/src/storage/index/snii/reader/logical_index_reader.cpp new file mode 100644 index 00000000000000..51a7d89da28b67 --- /dev/null +++ b/be/src/storage/index/snii/reader/logical_index_reader.cpp @@ -0,0 +1,1107 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/reader/logical_index_reader.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "storage/index/snii/encoding/byte_source.h" +#include "storage/index/snii/encoding/crc32c.h" +#include "storage/index/snii/encoding/section_framer.h" +#include "storage/index/snii/encoding/varint.h" +#include "storage/index/snii/encoding/zstd_codec.h" +#include "storage/index/snii/format/dict_block.h" +#include "storage/index/snii/format/dict_block_directory.h" +#include "storage/index/snii/format/metadata_blob.h" +#include "storage/index/snii/format/norms_pod.h" +#include "storage/index/snii/format/null_bitmap.h" +#include "storage/index/snii/io/batch_range_fetcher.h" +#include "storage/index/snii/reader/dict_block_cache.h" + +namespace doris::snii::reader { + +struct LogicalIndexReader::NormsCacheState { + struct Data { + std::vector bytes; + format::NormsPodReader reader; + }; + + std::mutex mutex; + std::unique_ptr ready; + std::shared_future in_flight; +}; + +using format::BlockRef; +using format::bsbf_hash; +using format::DictBlockDirectoryReader; +using format::DictBlockReader; +using format::DictEntry; +using format::IndexTier; +using format::kBsbfBytesPerBlock; +using format::kBsbfHeaderSize; +using format::RegionRef; +using format::SampledTermIndexReader; + +namespace { +constexpr uint64_t kMaxDictBlockUncompBytes = 256ULL * 1024 * 1024; +constexpr uint64_t kDefaultDictResidentMaxBytes = 256ULL * 1024; +constexpr size_t kMaxDictLookupBatchRuns = 16; +constexpr uint64_t kMaxDictLookupBatchBytes = 4ULL * 1024 * 1024; +// Conservatively covers make_shared's control block and allocator bookkeeping +// for the state/data/vector allocations. The framed norms bytes are charged +// separately at their exact validated length. +constexpr size_t kNormsAllocationOverheadCharge = 128; + +// L0/L1 tiering threshold (bytes). Defaults to kBsbfResidentMaxBytes; the env +// SNII_BSBF_RESIDENT_MAX overrides it for tuning and for exercising the +// on-demand L1 path in tests without a 250K-term corpus. Read fresh each open. +uint64_t bsbf_resident_max_bytes() { + const char* s = std::getenv("SNII_BSBF_RESIDENT_MAX"); + if (s != nullptr) { + char* end = nullptr; + const unsigned long long v = std::strtoull(s, &end, 10); + if (end != s) { + return v; + } + } + return format::kBsbfResidentMaxBytes; +} + +uint64_t dict_resident_max_bytes() { + const char* s = std::getenv("SNII_DICT_RESIDENT_MAX"); + if (s != nullptr) { + char* end = nullptr; + const unsigned long long v = std::strtoull(s, &end, 10); + if (end != s) { + return v; + } + } + return kDefaultDictResidentMaxBytes; +} + +Status checked_size(uint64_t value, const char* error, size_t* out) { + if (value > std::numeric_limits::max()) { + return Status::Error(error); + } + *out = static_cast(value); + return Status::OK(); +} + +Status validate_norms_region(io::FileReader* reader, const RegionRef& norms, uint64_t doc_count, + size_t* length) { + *length = 0; + if (norms.length == 0) { + return Status::OK(); + } + const uint64_t file_size = reader->size(); + if (norms.offset > file_size || norms.length > file_size - norms.offset) { + return Status::Error( + "logical_index: norms region past end of file"); + } + if (doc_count > std::numeric_limits::max()) { + return Status::Error( + "logical_index: norms doc count exceeds uint32"); + } + const uint64_t payload_length = varint_len(doc_count) + doc_count; + const uint64_t expected_length = + 1 + varint_len(payload_length) + payload_length + sizeof(uint32_t); + if (norms.length != expected_length) { + return Status::Error( + "logical_index: norms region length mismatch"); + } + if (norms.length > std::numeric_limits::max()) { + return Status::Error( + "logical_index: norms region exceeds cache charge bounds"); + } + *length = static_cast(norms.length); + return Status::OK(); +} + +Status validate_null_bitmap_frame(Slice framed) { + ByteSource source(framed); + FramedSection section; + RETURN_IF_ERROR(SectionFramer::read(source, §ion)); + if (section.type != format::kNullBitmapSectionType) { + return Status::Error( + "logical_index: invalid null bitmap section type"); + } + if (source.remaining() != 0) { + return Status::Error( + "logical_index: trailing null bitmap frame bytes"); + } + + // NullBitmapReader owns the Roaring validation. Parse only the two prefix + // varints here to pin the region to exactly one complete payload; otherwise + // a valid frame could silently carry ignored bytes after roaring_bytes. + ByteSource payload(section.payload); + uint64_t doc_count = 0; + RETURN_IF_ERROR(payload.get_varint64(&doc_count)); + if (doc_count > std::numeric_limits::max()) { + return Status::Error( + "logical_index: null bitmap doc count exceeds uint32"); + } + uint64_t roaring_size = 0; + RETURN_IF_ERROR(payload.get_varint64(&roaring_size)); + if (roaring_size != payload.remaining()) { + return Status::Error( + "logical_index: null bitmap payload length mismatch"); + } + return Status::OK(); +} + +Status dict_block_memory_bytes(const BlockRef& ref, uint64_t* out) { + if ((ref.flags & format::block_ref_flags::kZstd) == 0) { + *out = ref.length; + return Status::OK(); + } + if (ref.uncomp_len == 0 || ref.uncomp_len > kMaxDictBlockUncompBytes) { + return Status::Error( + "dict block: zstd uncomp_len out of range"); + } + *out = ref.uncomp_len; + return Status::OK(); +} + +Status checked_memory_add(uint64_t lhs, uint64_t rhs, const char* message, uint64_t* out) { + if (rhs > std::numeric_limits::max() - lhs) { + return Status::Error(message); + } + *out = lhs + rhs; + return Status::OK(); +} + +Status checked_memory_mul(uint64_t lhs, uint64_t rhs, const char* message, uint64_t* out) { + if (lhs != 0 && rhs > std::numeric_limits::max() / lhs) { + return Status::Error(message); + } + *out = lhs * rhs; + return Status::OK(); +} + +// Decompresses a zstd dict block from its on-disk bytes into *out. The decode +// buffer in zstd_decompress is resize_uninitialized'd (T19) then fully written. +Status zstd_decompress_dict_block(Slice on_disk, const BlockRef& ref, std::vector* out) { + uint64_t memory_bytes = 0; + RETURN_IF_ERROR(dict_block_memory_bytes(ref, &memory_bytes)); + size_t uncomp_len = 0; + RETURN_IF_ERROR( + checked_size(memory_bytes, "dict block: zstd length out of range", &uncomp_len)); + return zstd_decompress(on_disk, uncomp_len, out); +} + +// Materializes the usable (uncompressed) bytes of a dict block from a view over +// its on-disk bytes -- a raw block is copied, a zstd block is decompressed. Used +// by the resident single-range path, where on_disk is a sub-slice of the shared +// region buffer (so a raw block must be copied, not aliased). +Status decompress_dict_block_payload(Slice on_disk, const BlockRef& ref, + std::vector* out) { + if ((ref.flags & format::block_ref_flags::kZstd) == 0) { + out->assign(on_disk.data(), on_disk.data() + on_disk.size()); + return Status::OK(); + } + return zstd_decompress_dict_block(on_disk, ref, out); +} + +Status read_dict_block_bytes(io::FileReader* reader, const BlockRef& ref, + std::vector* out) { + size_t read_len = 0; + RETURN_IF_ERROR(checked_size(ref.length, "dict block: on-disk length out of range", &read_len)); + + std::vector block_bytes; + RETURN_IF_ERROR(reader->read_at(ref.offset, read_len, &block_bytes)); + if (block_bytes.size() != read_len) { + return Status::Error( + "dict block: short read"); + } + + // Raw on-demand block: move the freshly read buffer in (no copy). + if ((ref.flags & format::block_ref_flags::kZstd) == 0) { + *out = std::move(block_bytes); + return Status::OK(); + } + return zstd_decompress_dict_block(Slice(block_bytes), ref, out); +} + +Status open_dict_block(io::FileReader* reader, const BlockRef& ref, IndexTier tier, + bool has_positions, std::vector* bytes, DictBlockReader* out) { + RETURN_IF_ERROR(read_dict_block_bytes(reader, ref, bytes)); + return DictBlockReader::open(Slice(*bytes), tier, has_positions, out); +} + +// Validates that block `ref` lies fully within dict_region and returns its byte +// range relative to the start of the region. Defends the single-range resident +// read against a corrupt directory ref (offset before the region, or a range +// that runs past it) before it is used to index the region buffer. +Status slice_dict_block_in_region(const BlockRef& ref, const RegionRef& dict_region, + size_t region_len, size_t* rel_off, size_t* len) { + if (ref.offset < dict_region.offset) { + return Status::Error( + "dict block: ref before dict region"); + } + size_t rel = 0; + size_t block_len = 0; + RETURN_IF_ERROR( + checked_size(ref.offset - dict_region.offset, "dict block: ref offset OOR", &rel)); + RETURN_IF_ERROR(checked_size(ref.length, "dict block: ref length OOR", &block_len)); + if (rel > region_len || block_len > region_len - rel) { + return Status::Error( + "dict block: ref past dict region"); + } + *rel_off = rel; + *len = block_len; + return Status::OK(); +} +} // namespace + +Status LogicalIndexReader::load_resident_dict_blocks() { + resident_dict_blocks_.clear(); + + const uint64_t max_bytes = dict_resident_max_bytes(); + if (max_bytes == 0 || dbd_.n_blocks() == 0) { + return Status::OK(); + } + + uint64_t total_bytes = 0; + for (uint32_t ord = 0; ord < dbd_.n_blocks(); ++ord) { + BlockRef ref {}; + RETURN_IF_ERROR(dbd_.get(ord, &ref)); + uint64_t block_bytes = 0; + RETURN_IF_ERROR(dict_block_memory_bytes(ref, &block_bytes)); + if (block_bytes > max_bytes - total_bytes) { + return Status::OK(); + } + total_bytes += block_bytes; + } + + // The resident blocks are physically contiguous within dict_region, so read + // the whole region in a SINGLE range read (was one read_at per block -> up to + // ~4 serial S3 rounds on a cold open) and decode each block from a sub-slice. + // The region buffer is <= the resident byte cap (<=256KB) and freed on return; + // each ResidentDictBlock keeps its own decoded copy. + const RegionRef& dict_region = section_refs().dict_region; + size_t region_len = 0; + RETURN_IF_ERROR( + checked_size(dict_region.length, "dict region: length out of range", ®ion_len)); + std::vector region; + RETURN_IF_ERROR(reader_->read_at(dict_region.offset, region_len, ®ion)); + if (region.size() != region_len) { + return Status::Error( + "dict region: short read"); + } + + resident_dict_blocks_.reserve(dbd_.n_blocks()); + for (uint32_t ord = 0; ord < dbd_.n_blocks(); ++ord) { + BlockRef ref {}; + RETURN_IF_ERROR(dbd_.get(ord, &ref)); + size_t rel_off = 0; + size_t block_len = 0; + RETURN_IF_ERROR( + slice_dict_block_in_region(ref, dict_region, region_len, &rel_off, &block_len)); + const Slice on_disk(region.data() + rel_off, block_len); + ResidentDictBlock block; + RETURN_IF_ERROR(decompress_dict_block_payload(on_disk, ref, &block.bytes)); + RETURN_IF_ERROR( + DictBlockReader::open(Slice(block.bytes), tier_, has_positions_, &block.reader)); + resident_dict_blocks_.push_back(std::move(block)); + } + return Status::OK(); +} + +Status LogicalIndexReader::dict_block_reader_for_ordinal( + uint32_t ordinal, DictBlockCache* cache, std::shared_ptr* pin, + const DictBlockReader** out) const { + pin->reset(); + if (!resident_dict_blocks_.empty()) { + if (resident_dict_blocks_.size() != dbd_.n_blocks() || + ordinal >= resident_dict_blocks_.size()) { + return Status::Error( + "logical_index: incomplete resident dict"); + } + // Resident blocks live for the reader lifetime: no pin needed. + *out = &resident_dict_blocks_[ordinal].reader; + return Status::OK(); + } + + // On-demand: decode into a heap-allocated DecodedDictBlock held by *pin so the + // reader's Slice never dangles. The loader (file read + optional zstd + CRC + + // anchor parse) runs OUTSIDE any cache bookkeeping; on a cache hit it is not + // called, so a block shared by several terms of one query decodes only once. + DictBlockCache::Loader loader = [&](std::shared_ptr* slot) -> Status { + BlockRef ref {}; + RETURN_IF_ERROR(dbd_.get(ordinal, &ref)); + auto block = std::make_shared(); + RETURN_IF_ERROR(open_dict_block(reader_, ref, tier_, has_positions_, &block->bytes, + &block->reader)); + *slot = std::move(block); + return Status::OK(); + }; + if (cache != nullptr) { + RETURN_IF_ERROR(cache->get_or_load(ordinal, loader, pin)); + } else { + RETURN_IF_ERROR(loader(pin)); + } + *out = &(*pin)->reader; + return Status::OK(); +} + +Status LogicalIndexReader::load_resident_bsbf() { + // Block-split bloom XFilter -- gated on RESIDENCY (P1 cold-read fix, see + // docs/perf/P1-cold-read-amplification.md). The bloom is set up and used ONLY + // when the whole (small) filter fits under the resident cap: it is read in + // full, verified, and kept in memory so probes are in-memory and enter the + // Doris searcher cache with the rest of the logical-index metadata. + // + // When NON-resident (the common case for a real text column, where the filter + // is many MB) the bloom is skipped ENTIRELY: not even the 28B header is read, + // and has_bsbf_ stays false. Every term then falls through to sti -> dict, + // which yields the true found/absent. At 1 MiB cache-block granularity a + // non-resident bloom never saves a physical block (an absent term still costs + // one dict block either way), so its 28B header + per-term 32B probes were pure + // cold read amplification. + const RegionRef& bsbf = core_.section_refs.bsbf; + if (open_mode_ != LogicalIndexOpenMode::kQuery || bsbf.length == 0 || + bsbf.length > bsbf_resident_max_bytes()) { + return Status::OK(); + } + if (bsbf.length <= kBsbfHeaderSize) { + return Status::Error( + "logical_index: bsbf section too small"); + } + const uint64_t num_bytes = bsbf.length - kBsbfHeaderSize; + std::vector head; + RETURN_IF_ERROR(reader_->read_at(bsbf.offset, bsbf.length, &head)); + if (head.size() < bsbf.length) { + return Status::Error( + "logical_index: short bsbf resident read"); + } + RETURN_IF_ERROR(format::BsbfHeader::parse(Slice(head.data(), kBsbfHeaderSize), bsbf.offset, + &bsbf_header_)); + // Cross-check the header geometry against the section ref. + if (bsbf_header_.num_bytes != num_bytes) { + return Status::Error( + "logical_index: bsbf header/section size mismatch"); + } + const Slice bitset(head.data() + kBsbfHeaderSize, bsbf_header_.num_bytes); + if (crc32c(bitset) != bsbf_header_.bitset_crc) { + return Status::Error( + "logical_index: bsbf bitset crc mismatch"); + } + bsbf_resident_bitset_.assign(bitset.data(), bitset.data() + bitset.size()); + has_bsbf_ = true; + bsbf_resident_ = true; + return Status::OK(); +} + +Status LogicalIndexReader::open(io::FileReader* file_reader, Slice core_frame, Slice sti_blob, + Slice dbd_blob, LogicalIndexReader* out, + LogicalIndexOpenMode open_mode) { + if (out == nullptr) { + return Status::Error("logical_index: null out"); + } + *out = LogicalIndexReader {}; + if (file_reader == nullptr) { + return Status::Error("logical_index: null file reader"); + } + if (core_frame.empty() || sti_blob.empty() || dbd_blob.empty()) { + return Status::Error( + "logical_index: empty mandatory metadata blob"); + } + + LogicalIndexReader candidate; + candidate.reader_ = file_reader; + candidate.open_mode_ = open_mode; + RETURN_IF_ERROR(format::decode_core_metadata(core_frame, &candidate.core_)); + candidate.tier_ = format::tier_of(candidate.core_.index_config); + candidate.has_positions_ = format::has_positions(candidate.core_.index_config); + size_t norms_length = 0; + RETURN_IF_ERROR(validate_norms_region(file_reader, candidate.core_.section_refs.norms, + candidate.core_.stats.doc_count, &norms_length)); + if (norms_length != 0) { + constexpr size_t fixed_charge = sizeof(NormsCacheState) + sizeof(NormsCacheState::Data) + + kNormsAllocationOverheadCharge; + if (norms_length > std::numeric_limits::max() - fixed_charge) { + return Status::Error( + "logical_index: norms cache charge overflow"); + } + candidate.norms_reserved_charge_ = fixed_charge + norms_length; + candidate.norms_cache_ = std::make_shared(); + } + // Raw frames alias the group read and compressed carriers materialize into + // transient scratch. Both readers own their decoded state. + { + std::vector scratch; + Slice frame; + RETURN_IF_ERROR(format::materialize_metadata_blob( + sti_blob, format::SectionType::kSampledTermIndex, + format::SectionType::kSampledTermIndexZstd, &scratch, &frame)); + RETURN_IF_ERROR(SampledTermIndexReader::open(frame, &candidate.sti_)); + } + { + std::vector scratch; + Slice frame; + RETURN_IF_ERROR(format::materialize_metadata_blob( + dbd_blob, format::SectionType::kDictBlockDirectory, + format::SectionType::kDictBlockDirectoryZstd, &scratch, &frame)); + RETURN_IF_ERROR(DictBlockDirectoryReader::open(frame, &candidate.dbd_)); + } + if (candidate.sti_.n_blocks() != candidate.dbd_.n_blocks()) { + return Status::Error( + "logical_index: sampled-term index and block directory count mismatch"); + } + if (open_mode == LogicalIndexOpenMode::kQuery) { + RETURN_IF_ERROR(candidate.load_resident_dict_blocks()); + } + RETURN_IF_ERROR(candidate.load_resident_bsbf()); + *out = std::move(candidate); + return Status::OK(); +} + +size_t LogicalIndexReader::memory_usage() const { + size_t bytes = sizeof(*this) + bsbf_resident_bitset_.capacity(); + if (core_.common_grams_metadata) { + const auto& common_grams = *core_.common_grams_metadata; + bytes += format::std_string_heap_bytes(common_grams.common_grams_dictionary_identity); + bytes += format::std_string_heap_bytes(common_grams.base_analyzer_fingerprint); + bytes += format::std_string_heap_bytes(common_grams.common_grams_fingerprint); + } + bytes += sti_.heap_bytes(); + bytes += dbd_.heap_bytes(); + for (const auto& block : resident_dict_blocks_) { + bytes += sizeof(block) + block.bytes.capacity() + block.reader.heap_bytes(); + } + // Norms are loaded lazily, but the searcher-cache charge is fixed when this + // reader is inserted. Reserve their complete framed size and heap state up + // front so later allocation is already accounted for. Saturation prevents + // an already-large resident metadata charge from wrapping around. + if (norms_reserved_charge_ > std::numeric_limits::max() - bytes) { + return std::numeric_limits::max(); + } + bytes += norms_reserved_charge_; + return bytes; +} + +Status LogicalIndexReader::open_norms(format::NormsPodReader* out) const { + if (out == nullptr) { + return Status::Error( + "logical_index: null norms reader"); + } + if (reader_ == nullptr) { + return Status::Error("logical_index: not opened"); + } + const RegionRef& norms = section_refs().norms; + if (norms.length == 0) { + return Status::Error( + "logical_index: index has no norms"); + } + DORIS_CHECK(norms_cache_ != nullptr); + + std::shared_future in_flight; + std::shared_ptr> completion; + { + std::lock_guard lock(norms_cache_->mutex); + if (norms_cache_->ready != nullptr) { + *out = norms_cache_->ready->reader; + return Status::OK(); + } + if (norms_cache_->in_flight.valid()) { + in_flight = norms_cache_->in_flight; + } else { + completion = std::make_shared>(); + in_flight = completion->get_future().share(); + norms_cache_->in_flight = in_flight; + } + } + + if (completion == nullptr) { + const Status status = in_flight.get(); + RETURN_IF_ERROR(status); + std::lock_guard lock(norms_cache_->mutex); + DORIS_CHECK(norms_cache_->ready != nullptr); + *out = norms_cache_->ready->reader; + return Status::OK(); + } + + auto data = std::make_unique(); + const size_t read_len = static_cast(norms.length); + Status status = reader_->read_at(norms.offset, read_len, &data->bytes); + if (status.ok() && data->bytes.size() != read_len) { + status = Status::Error( + "logical_index: short norms read"); + } + if (status.ok()) { + status = format::NormsPodReader::open(Slice(data->bytes), &data->reader); + } + { + std::lock_guard lock(norms_cache_->mutex); + if (status.ok()) { + norms_cache_->ready = std::move(data); + *out = norms_cache_->ready->reader; + } + norms_cache_->in_flight = std::shared_future {}; + } + completion->set_value(status); + return status; +} + +void LogicalIndexReader::release_compaction_norms() const { + DORIS_CHECK(open_mode_ == LogicalIndexOpenMode::kCompaction); + DORIS_CHECK(norms_cache_ != nullptr); + std::lock_guard lock(norms_cache_->mutex); + DORIS_CHECK(!norms_cache_->in_flight.valid()); + norms_cache_->ready.reset(); +} + +Status LogicalIndexReader::read_null_docids( + std::vector* out, const NullDocidsDecodeReservation& reserve_decode) const { + if (out == nullptr) { + return Status::Error( + "logical_index: null null-docids output"); + } + out->clear(); + if (reader_ == nullptr) { + return Status::Error("logical_index: not opened"); + } + + const RegionRef& ref = section_refs().null_bitmap; + if (ref.length == 0) { + if (stats().null_count != 0) { + return Status::Error( + "logical_index: null bitmap section missing"); + } + return Status::OK(); + } + + const uint64_t file_size = reader_->size(); + if (ref.offset > file_size || ref.length > file_size - ref.offset) { + return Status::Error( + "logical_index: null bitmap region past end of file"); + } + size_t read_len = 0; + RETURN_IF_ERROR( + checked_size(ref.length, "logical_index: null bitmap length out of range", &read_len)); + std::vector bytes; + RETURN_IF_ERROR(reader_->read_at(ref.offset, read_len, &bytes)); + if (bytes.size() != read_len) { + return Status::Error( + "logical_index: short null bitmap read"); + } + RETURN_IF_ERROR(validate_null_bitmap_frame(Slice(bytes))); + + uint64_t decoded_memory_bytes = 0; + RETURN_IF_ERROR( + format::NullBitmapReader::decoded_memory_bytes(Slice(bytes), &decoded_memory_bytes)); + if (reserve_decode) { + RETURN_IF_ERROR(reserve_decode(decoded_memory_bytes)); + } + + format::NullBitmapReader null_bitmap_reader; + RETURN_IF_ERROR(format::NullBitmapReader::open(Slice(bytes), &null_bitmap_reader)); + if (null_bitmap_reader.doc_count() != stats().doc_count) { + return Status::Error( + "logical_index: null bitmap doc count mismatch"); + } + if (null_bitmap_reader.null_count() != stats().null_count) { + return Status::Error( + "logical_index: null bitmap cardinality mismatch"); + } + + out->reserve(null_bitmap_reader.null_count()); + null_bitmap_reader.append_docids(*out); + for (uint32_t docid : *out) { + if (docid >= stats().doc_count) { + out->clear(); + return Status::Error( + "logical_index: null docid outside document domain"); + } + } + return Status::OK(); +} + +Status LogicalIndexReader::null_docids_scan_memory(NullDocidsScanMemory* out) const { + if (out == nullptr) { + return Status::Error( + "logical_index: null null-docids scan memory out"); + } + *out = NullDocidsScanMemory {}; + if (reader_ == nullptr) { + return Status::Error("logical_index: not opened"); + } + const RegionRef& ref = section_refs().null_bitmap; + if (ref.length == 0) { + if (stats().null_count != 0) { + return Status::Error( + "logical_index: null bitmap section missing"); + } + return Status::OK(); + } + const uint64_t file_size = reader_->size(); + if (ref.offset > file_size || ref.length > file_size - ref.offset) { + return Status::Error( + "logical_index: null bitmap region past end of file"); + } + size_t frame_bytes = 0; + RETURN_IF_ERROR(checked_size(ref.length, "logical_index: null bitmap length out of range", + &frame_bytes)); + RETURN_IF_ERROR(checked_memory_mul(stats().null_count, sizeof(uint32_t), + "logical_index: null docids output memory overflows", + &out->output_bytes)); + + out->frame_bytes = frame_bytes; + return Status::OK(); +} + +Status LogicalIndexReader::lookup(std::string_view term, bool* found, DictEntry* entry, + uint64_t* frq_base, uint64_t* prx_base, + DictBlockCache* cache) const { + *found = false; + if (reader_ == nullptr) { + return Status::Error("logical_index: not opened"); + } + + bool maybe = false; + uint32_t ordinal = 0; + RETURN_IF_ERROR(locate_candidate_dict_block(term, &maybe, &ordinal)); + if (!maybe) { + return Status::OK(); + } + + // Use a resident small-DICT block when present; otherwise read the DICT + // block on demand and parse it with the same validation path used at open. + // `pin` keeps an on-demand block alive through find_term (resident: null). + const DictBlockReader* br = nullptr; + std::shared_ptr pin; + RETURN_IF_ERROR(dict_block_reader_for_ordinal(ordinal, cache, &pin, &br)); + + bool hit = false; + RETURN_IF_ERROR(br->find_term(term, &hit, entry)); + if (!hit) { + return Status::OK(); + } + + *found = true; + *frq_base = br->frq_base(); + *prx_base = br->prx_base(); + return Status::OK(); +} + +Status LogicalIndexReader::locate_candidate_dict_block(std::string_view term, bool* maybe_present, + uint32_t* ordinal) const { + *maybe_present = false; + // A DEFINITELY-ABSENT term returns without a DICT read. The bloom is + // consulted only when resident; otherwise STI/DICT remains authoritative. + if (has_bsbf_) { + const uint64_t h = bsbf_hash(term); + bool maybe = true; + if (bsbf_resident_) { + const uint32_t blk = format::bsbf_block_index(h, bsbf_header_.num_blocks); + maybe = format::bsbf_block_contains( + h, + bsbf_resident_bitset_.data() + static_cast(blk) * kBsbfBytesPerBlock); + } + if (!maybe) { + return Status::OK(); + } + } + return sti_.locate(term, maybe_present, ordinal); +} + +Status LogicalIndexReader::collect_batch_lookup_groups( + const std::vector& terms, std::vector* candidates, + std::vector* groups) const { + candidates->clear(); + candidates->reserve(terms.size()); + for (size_t i = 0; i < terms.size(); ++i) { + bool maybe = false; + uint32_t ordinal = 0; + RETURN_IF_ERROR(locate_candidate_dict_block(terms[i], &maybe, &ordinal)); + if (maybe) { + candidates->push_back({i, ordinal}); + } + } + + auto by_ordinal = [](const BatchLookupCandidate& lhs, const BatchLookupCandidate& rhs) { + return lhs.ordinal < rhs.ordinal; + }; + if (!std::ranges::is_sorted(*candidates, by_ordinal)) { + std::ranges::sort(*candidates, by_ordinal); + } + + groups->clear(); + for (size_t begin = 0; begin < candidates->size();) { + const uint32_t ordinal = (*candidates)[begin].ordinal; + size_t end = begin + 1; + while (end < candidates->size() && (*candidates)[end].ordinal == ordinal) { + ++end; + } + groups->push_back({ordinal, begin, end}); + begin = end; + } + return Status::OK(); +} + +Status LogicalIndexReader::resolve_batch_lookup_group( + const std::vector& terms, const std::vector& candidates, + const BatchLookupGroup& group, const DictBlockReader& block_reader, + std::vector* results) { + for (size_t i = group.begin; i < group.end; ++i) { + const size_t term_index = candidates[i].term_index; + BatchLookupResult& result = (*results)[term_index]; + RETURN_IF_ERROR(block_reader.find_term(terms[term_index], &result.found, &result.entry)); + if (result.found) { + result.frq_base = block_reader.frq_base(); + result.prx_base = block_reader.prx_base(); + } + } + return Status::OK(); +} + +Status LogicalIndexReader::lookup_batch_on_demand( + const std::vector& terms, const std::vector& candidates, + const std::vector& groups, + std::vector* results) const { + for (size_t wave_begin = 0; wave_begin < groups.size();) { + std::vector pending; + pending.reserve(kMaxDictLookupBatchRuns); + uint64_t pending_bytes = 0; + uint64_t pending_end = 0; + size_t pending_runs = 0; + size_t wave_end = wave_begin; + while (wave_end < groups.size()) { + BlockRef ref {}; + RETURN_IF_ERROR(dbd_.get(groups[wave_end].ordinal, &ref)); + const uint64_t ref_end = ref.offset + ref.length; + const bool starts_new_run = pending.empty() || ref.offset > pending_end; + if (!pending.empty() && ((starts_new_run && pending_runs == kMaxDictLookupBatchRuns) || + ref.length > kMaxDictLookupBatchBytes || + pending_bytes > kMaxDictLookupBatchBytes - ref.length)) { + break; + } + pending.push_back({wave_end, ref, 0}); + pending_bytes += ref.length; + if (starts_new_run) { + ++pending_runs; + } + pending_end = std::max(pending_end, ref_end); + ++wave_end; + } + DORIS_CHECK(!pending.empty()); + io::BatchRangeFetcher fetcher(reader_, /*coalesce_gap=*/0); + for (PendingBatchLookupBlock& block : pending) { + block.handle = fetcher.add(block.ref.offset, block.ref.length); + } + RETURN_IF_ERROR(fetcher.fetch()); + + for (const PendingBatchLookupBlock& block : pending) { + const Slice on_disk = fetcher.get(block.handle); + std::vector decoded; + Slice payload = on_disk; + if ((block.ref.flags & format::block_ref_flags::kZstd) != 0) { + RETURN_IF_ERROR(zstd_decompress_dict_block(on_disk, block.ref, &decoded)); + payload = Slice(decoded); + } + DictBlockReader block_reader; + RETURN_IF_ERROR(DictBlockReader::open(payload, tier_, has_positions_, &block_reader)); + RETURN_IF_ERROR(resolve_batch_lookup_group(terms, candidates, groups[block.group_index], + block_reader, results)); + } + wave_begin = wave_end; + } + return Status::OK(); +} + +Status LogicalIndexReader::lookup_batch(const std::vector& terms, + std::vector* results) const { + DORIS_CHECK(results != nullptr); + DCHECK(std::ranges::is_sorted(terms)); + DCHECK(std::adjacent_find(terms.begin(), terms.end()) == terms.end()); + results->assign(terms.size(), BatchLookupResult {}); + if (reader_ == nullptr) { + return Status::Error("logical_index: not opened"); + } + + std::vector candidates; + std::vector groups; + RETURN_IF_ERROR(collect_batch_lookup_groups(terms, &candidates, &groups)); + if (groups.empty()) { + return Status::OK(); + } + + // Resident dictionaries always stay zero-I/O. One-block batches keep the + // existing synchronous read-through path. + if (!resident_dict_blocks_.empty() || groups.size() == 1) { + for (const BatchLookupGroup& group : groups) { + const DictBlockReader* block_reader = nullptr; + std::shared_ptr pin; + RETURN_IF_ERROR(dict_block_reader_for_ordinal(group.ordinal, /*cache=*/nullptr, &pin, + &block_reader)); + RETURN_IF_ERROR( + resolve_batch_lookup_group(terms, candidates, group, *block_reader, results)); + } + return Status::OK(); + } + + return lookup_batch_on_demand(terms, candidates, groups, results); +} + +Status LogicalIndexReader::decode_dict_block(uint32_t ordinal, std::vector* entries, + uint64_t* frq_base, uint64_t* prx_base) const { + if (entries == nullptr || frq_base == nullptr || prx_base == nullptr) { + return Status::Error( + "logical_index: null decode_dict_block out"); + } + if (reader_ == nullptr) { + return Status::Error("logical_index: not opened"); + } + entries->clear(); + // Same resolution path lookup() uses (resident block or one on-demand range + // read + zstd + CRC); `pin` keeps an on-demand block alive through + // decode_all. No request-scoped cache: a sequential full scan touches each + // block exactly once. + const DictBlockReader* br = nullptr; + std::shared_ptr pin; + RETURN_IF_ERROR(dict_block_reader_for_ordinal(ordinal, /*cache=*/nullptr, &pin, &br)); + RETURN_IF_ERROR(br->decode_all(entries)); + *frq_base = br->frq_base(); + *prx_base = br->prx_base(); + return Status::OK(); +} + +Status LogicalIndexReader::dict_block_scan_memory(uint32_t ordinal, + DictBlockScanMemory* out) const { + if (out == nullptr) { + return Status::Error( + "logical_index: null dict block scan memory out"); + } + *out = DictBlockScanMemory {}; + if (reader_ == nullptr) { + return Status::Error("logical_index: not opened"); + } + + BlockRef ref {}; + RETURN_IF_ERROR(dbd_.get(ordinal, &ref)); + uint64_t plain_bytes = 0; + RETURN_IF_ERROR(dict_block_memory_bytes(ref, &plain_bytes)); + + // During a compressed decode the fetched bytes coexist with the plain block. + // The reader then owns the plain bytes plus anchor vectors/strings. Counting + // every entry as an anchor is conservative for all valid anchor intervals. + uint64_t anchor_slots = 0; + RETURN_IF_ERROR(checked_memory_mul(ref.n_entries, sizeof(uint32_t) + sizeof(std::string), + "logical_index: dict decode anchor memory overflows", + &anchor_slots)); + uint64_t decode_bytes = 0; + RETURN_IF_ERROR(checked_memory_add(ref.length, plain_bytes, + "logical_index: dict decode bytes overflow", &decode_bytes)); + RETURN_IF_ERROR(checked_memory_add(decode_bytes, anchor_slots, + "logical_index: dict decode slots overflow", &decode_bytes)); + RETURN_IF_ERROR(checked_memory_add(decode_bytes, plain_bytes, + "logical_index: dict decode terms overflow", &decode_bytes)); + + uint64_t entry_slots = 0; + RETURN_IF_ERROR(checked_memory_mul(ref.n_entries, sizeof(DictEntry), + "logical_index: dict entry slots overflow", &entry_slots)); + uint64_t entry_payload = 0; + RETURN_IF_ERROR(checked_memory_mul(plain_bytes, 2, "logical_index: dict entry payload overflow", + &entry_payload)); + RETURN_IF_ERROR(checked_memory_add(entry_slots, entry_payload, + "logical_index: dict entries memory overflow", + &out->entries_bytes)); + out->decode_bytes = decode_bytes; + return Status::OK(); +} + +Status LogicalIndexReader::visit_prefix_terms(std::string_view prefix, + const PrefixHitVisitor& visitor, + DictBlockCache* cache) const { + if (!visitor) { + return Status::Error( + "logical_index: null prefix visitor"); + } + if (reader_ == nullptr) { + return Status::Error("logical_index: not opened"); + } + + // Seek the start block: the SampledTermIndex block whose first term <= prefix + // (terms with `prefix` are >= prefix, so they begin in that block or later). + // If the prefix sorts before every sample (or is empty), start at block 0. + uint32_t start = 0; + if (!prefix.empty()) { + bool maybe = false; + uint32_t ordinal = 0; + RETURN_IF_ERROR(sti_.locate(prefix, &maybe, &ordinal)); + if (maybe) { + start = ordinal; + } + } + + for (uint32_t ord = start; ord < dbd_.n_blocks(); ++ord) { + const DictBlockReader* br = nullptr; + std::shared_ptr pin; + RETURN_IF_ERROR(dict_block_reader_for_ordinal(ord, cache, &pin, &br)); + + // Stream this block's prefix range: anchor-jump past pre-prefix segments, + // decode only the bodies we keep, and stop at the first term past the + // range (decode_all materialized every entry of every scanned block). + // The visitor still owns final term acceptance, so results are identical; + // `br`/`pin` stay alive across this synchronous call. + bool prefix_exhausted = false; + bool visitor_stopped = false; + RETURN_IF_ERROR(br->visit_prefix_range( + prefix, /*accept_key=*/ {}, + [&](DictEntry&& e, bool* stop) -> Status { + PrefixHit hit; + hit.term = e.term; + hit.entry = std::move(e); + hit.frq_base = br->frq_base(); + hit.prx_base = br->prx_base(); + RETURN_IF_ERROR(visitor(std::move(hit), stop)); + visitor_stopped = *stop; + return Status::OK(); + }, + &prefix_exhausted)); + if (visitor_stopped || prefix_exhausted) { + return Status::OK(); + } + } + return Status::OK(); +} + +Status LogicalIndexReader::visit_term_range(std::string_view lower_inclusive, + std::optional upper_exclusive, + const PrefixHitVisitor& visitor, + DictBlockCache* cache) const { + if (!visitor) { + return Status::Error( + "logical_index: null range visitor"); + } + if (reader_ == nullptr) { + return Status::Error("logical_index: not opened"); + } + if (upper_exclusive.has_value() && *upper_exclusive <= lower_inclusive) { + return Status::OK(); + } + if (upper_exclusive.has_value()) { + bool upper_reaches_dictionary = false; + uint32_t upper_ordinal = 0; + RETURN_IF_ERROR(sti_.locate(*upper_exclusive, &upper_reaches_dictionary, &upper_ordinal)); + if (!upper_reaches_dictionary) { + return Status::OK(); + } + } + + uint32_t start = 0; + if (!lower_inclusive.empty()) { + bool maybe = false; + uint32_t ordinal = 0; + RETURN_IF_ERROR(sti_.locate(lower_inclusive, &maybe, &ordinal)); + if (maybe) { + start = ordinal; + } + } + + for (uint32_t ord = start; ord < dbd_.n_blocks(); ++ord) { + const DictBlockReader* block_reader = nullptr; + std::shared_ptr pin; + RETURN_IF_ERROR(dict_block_reader_for_ordinal(ord, cache, &pin, &block_reader)); + + bool range_exhausted = false; + bool visitor_stopped = false; + RETURN_IF_ERROR(block_reader->visit_term_range( + lower_inclusive, upper_exclusive, /*accept_key=*/ {}, + [&](DictEntry&& entry, bool* stop) -> Status { + PrefixHit hit; + hit.term = entry.term; + hit.entry = std::move(entry); + hit.frq_base = block_reader->frq_base(); + hit.prx_base = block_reader->prx_base(); + RETURN_IF_ERROR(visitor(std::move(hit), stop)); + visitor_stopped = *stop; + return Status::OK(); + }, + &range_exhausted)); + if (visitor_stopped || range_exhausted) { + return Status::OK(); + } + } + return Status::OK(); +} + +Status LogicalIndexReader::prefix_terms(std::string_view prefix, std::vector* const out, + int32_t max_terms, DictBlockCache* cache) const { + if (out == nullptr) { + return Status::Error("logical_index: null out"); + } + out->clear(); + return visit_prefix_terms( + prefix, + [&](PrefixHit&& hit, bool* stop) { + out->push_back(std::move(hit)); + *stop = max_terms > 0 && out->size() >= static_cast(max_terms); + return Status::OK(); + }, + cache); +} + +namespace { + +// Validates a pod_ref window locator against the posting region and returns the +// absolute window range (after the prelude). Rejects corrupt locators rather +// than letting size_t underflow / uint64 overflow reach read_at. +Status resolve_window(const format::RegionRef& section, uint64_t base, uint64_t off_delta, + uint64_t total_len, uint64_t prelude_len, uint64_t* abs_off, uint64_t* len) { + if (prelude_len > total_len) { + return Status::Error( + "logical_index: prelude_len exceeds window len"); + } + const uint64_t in_region = base + off_delta; + if (in_region < base) { + return Status::Error( + "logical_index: locator overflow"); + } + if (in_region > section.length || total_len > section.length - in_region) { + return Status::Error( + "logical_index: window past posting region"); + } + *abs_off = section.offset + in_region + prelude_len; + *len = total_len - prelude_len; + return Status::OK(); +} + +} // namespace + +Status LogicalIndexReader::resolve_frq_window(const format::DictEntry& entry, uint64_t frq_base, + uint64_t* abs_off, uint64_t* len) const { + return resolve_window(section_refs().posting_region, frq_base, entry.frq_off_delta, + entry.frq_len, entry.prelude_len, abs_off, len); +} + +Status LogicalIndexReader::resolve_prx_window(const format::DictEntry& entry, uint64_t prx_base, + uint64_t* abs_off, uint64_t* len) const { + // .prx windows carry no prelude (prelude_len = 0); both spans live in the + // same posting region (prx span precedes frq span for the same term). + return resolve_window(section_refs().posting_region, prx_base, entry.prx_off_delta, + entry.prx_len, 0, abs_off, len); +} + +} // namespace doris::snii::reader diff --git a/be/src/storage/index/snii/reader/logical_index_reader.h b/be/src/storage/index/snii/reader/logical_index_reader.h new file mode 100644 index 00000000000000..f7fdee0a902f18 --- /dev/null +++ b/be/src/storage/index/snii/reader/logical_index_reader.h @@ -0,0 +1,285 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/format/bsbf.h" +#include "storage/index/snii/format/core_metadata.h" +#include "storage/index/snii/format/dict_block.h" +#include "storage/index/snii/format/dict_block_directory.h" +#include "storage/index/snii/format/dict_entry.h" +#include "storage/index/snii/format/format_constants.h" +#include "storage/index/snii/format/metadata_blob.h" +#include "storage/index/snii/format/sampled_term_index.h" +#include "storage/index/snii/format/stats_block.h" +#include "storage/index/snii/io/file_reader.h" + +// LogicalIndexReader -- read-side counterpart of LogicalIndexWriter for one +// logical index. It owns decoded Core, SampledTermIndex, and DICT block-directory +// state from one adjacent metadata group, and resolves a query term to its +// DictEntry through the documented lookup flow: +// XFilter (reject absent) -> SampledTermIndex (candidate block ordinal) -> +// DICT block directory (block range) -> resident small-DICT block or one +// range read of the DICT block -> DictBlockReader::find_term. +// +// lookup() also returns the block's frq_base/prx_base (captured by the +// DictBlockReader) so callers can resolve a pod_ref entry's absolute .frq/.prx +// offsets via the writer's contract. Both deltas index into the SAME +// interleaved posting region (prx_base == frq_base; the prx span precedes the +// frq span): +// abs_frq = posting_region.offset + frq_base + entry.frq_off_delta +// abs_prx = posting_region.offset + prx_base + entry.prx_off_delta +// +// The reader retains no raw metadata-group bytes after open. +namespace doris::snii::format { +class NormsPodReader; +} + +namespace doris::snii::reader { + +// Forward-declared: this widely-included header only names DictBlockCache* and +// shared_ptr*; the full definitions are pulled into the +// .cpp and into tests that construct a cache. Keeps the request-scoped cache +// header out of the ~500 TUs that transitively include this one. +struct DecodedDictBlock; +class DictBlockCache; + +enum class LogicalIndexOpenMode : uint8_t { + kQuery, + kCompaction, +}; + +struct DictBlockScanMemory { + uint64_t decode_bytes = 0; + uint64_t entries_bytes = 0; +}; + +struct NullDocidsScanMemory { + uint64_t frame_bytes = 0; + uint64_t output_bytes = 0; +}; + +class LogicalIndexReader { +public: + LogicalIndexReader() = default; + + // Parses one mandatory Core/STI/DBD metadata group and binds the reader to + // file_reader. The reader retains decoded state, not the input byte slices. + static Status open(io::FileReader* file_reader, Slice core_frame, Slice sti_blob, + Slice dbd_blob, LogicalIndexReader* out, + LogicalIndexOpenMode open_mode = LogicalIndexOpenMode::kQuery); + + // Resolves term to a DictEntry. *found=false when the term is absent (XFilter + // rejection, out-of-range sample, or DICT-block miss). On a hit, *entry is + // filled and *frq_base / *prx_base carry the candidate block's bases. + // + // `cache` is an optional REQUEST-SCOPED DictBlockCache: when a single query + // threads one cache through its per-term lookups, an on-demand DICT block hit + // by several terms is decoded once instead of once per term. nullptr keeps the + // pre-existing behavior (each lookup materializes its own block). The cache is + // caller-owned, single-threaded, and never mutates this (const) reader. + Status lookup(std::string_view term, bool* found, format::DictEntry* entry, uint64_t* frq_base, + uint64_t* prx_base, DictBlockCache* cache = nullptr) const; + + struct BatchLookupResult { + bool found = false; + format::DictEntry entry; + uint64_t frq_base = 0; + uint64_t prx_base = 0; + }; + + // Resolves one sorted, duplicate-free term batch. Terms are first mapped to + // candidate DICT ordinals through the same XFilter/STI path as lookup(), then + // distinct on-demand blocks are fetched concurrently in bounded waves. + // Results stay aligned with `terms`; absent terms have found=false. + Status lookup_batch(const std::vector& terms, + std::vector* results) const; + + // One enumerated term whose key has the requested prefix, with its DictEntry + // and the owning DICT block's frq/prx bases (for posting resolution). + struct PrefixHit { + std::string term; + format::DictEntry entry; + uint64_t frq_base = 0; + uint64_t prx_base = 0; + }; + + using PrefixHitVisitor = std::function; + + // Ordered term enumeration: every term with `prefix`, in lexicographic order, + // by seeking the start DICT block via the SampledTermIndex and scanning + // forward across contiguous blocks until the terms pass the prefix range. + // Empty prefix enumerates all terms. This is the contiguous-DICT-block design + // the term-anchor layout was built for (MATCH_PHRASE_PREFIX / prefix / range + // queries). The visitor form avoids materializing all hits when callers only + // need a bounded expansion. + Status visit_prefix_terms(std::string_view prefix, const PrefixHitVisitor& visitor, + DictBlockCache* cache = nullptr) const; + Status visit_term_range(std::string_view lower_inclusive, + std::optional upper_exclusive, + const PrefixHitVisitor& visitor, DictBlockCache* cache = nullptr) const; + Status prefix_terms(std::string_view prefix, std::vector* const out, + int32_t max_terms = 0, DictBlockCache* cache = nullptr) const; + + // ---- Sequential whole-dictionary access (T2.3, compaction index merge) ---- + // Number of DICT blocks in this index (0 for an empty dictionary). + uint32_t n_dict_blocks() const { return dbd_.n_blocks(); } + // Decodes EVERY entry of DICT block `ordinal` in lexicographic order into + // *entries (each self-contained, owning its term and any inline posting + // bytes) and returns the block's frq/prx bases. One block is materialized + // at a time so a full-dictionary scan (SniiSegmentTermCursor) holds a single + // block's entries, never the whole vocabulary. ordinal must be + // < n_dict_blocks(). + Status decode_dict_block(uint32_t ordinal, std::vector* entries, + uint64_t* frq_base, uint64_t* prx_base) const; + // Returns conservative pre-allocation charges for the on-demand block decode + // and its fully materialized DictEntry vector. Compaction cursors reserve both + // before decoding so MEM_LIMIT_EXCEEDED is returned before the normal block + // allocations whenever the shared merge cap cannot admit them. + Status dict_block_scan_memory(uint32_t ordinal, DictBlockScanMemory* out) const; + + // Resolves a pod_ref entry's absolute .frq / .prx window byte range, + // validating the locator against the posting_region length (defends against + // corrupt entries: prelude_len > frq_len underflow, or off_delta+len past the + // region). Both windows resolve against the single posting_region. *abs_off + // is the absolute file offset of the window (after prelude); *len its byte + // length. + Status resolve_frq_window(const format::DictEntry& entry, uint64_t frq_base, uint64_t* abs_off, + uint64_t* len) const; + Status resolve_prx_window(const format::DictEntry& entry, uint64_t prx_base, uint64_t* abs_off, + uint64_t* len) const; + + const format::SectionRefs& section_refs() const { return core_.section_refs; } + const format::StatsBlock& stats() const { return core_.stats; } + format::IndexTier tier() const { return tier_; } + bool has_positions() const { return has_positions_; } + LogicalIndexOpenMode open_mode() const { return open_mode_; } + const segment_v2::inverted_index::CommonGramsSegmentMetadata* common_grams_metadata() const { + return core_.common_grams_metadata ? &*core_.common_grams_metadata : nullptr; + } + format::CommonGramsPostingPolicy common_grams_posting_policy() const { + return core_.common_grams_posting_policy; + } + io::FileReader* reader() const { return reader_; } + + // Returns a reader over the validated norms section. The first call reads + // and validates the section; later calls share the immutable reader-owned + // bytes. The full on-disk section is reserved in memory_usage() before this + // LogicalIndexReader enters the searcher cache, so lazy loading cannot make + // the cache under-report its eventual resident size. + Status open_norms(format::NormsPodReader* out) const; + // Compaction scans one source norm vector at a time. This charge matches the + // reader's full cache accounting; release_compaction_norms() drops the loaded + // frame after its values have been scattered into destination vectors. + size_t compaction_norms_cache_charge() const { return norms_reserved_charge_; } + void release_compaction_norms() const; + + // Reads and validates the sparse null-bitmap side POD, then returns its + // docids in ascending order. Work is O(null_count), not O(doc_count), which + // lets compaction remap NULL rows without scanning the complete document + // domain. A missing section is valid only when StatsBlock::null_count is 0. + using NullDocidsDecodeReservation = std::function; + Status read_null_docids(std::vector* out, + const NullDocidsDecodeReservation& reserve_decode = + NullDocidsDecodeReservation()) const; + Status null_docids_scan_memory(NullDocidsScanMemory* out) const; + size_t memory_usage() const; + +private: + struct NormsCacheState; + struct BatchLookupCandidate { + size_t term_index = 0; + uint32_t ordinal = 0; + }; + struct BatchLookupGroup { + uint32_t ordinal = 0; + size_t begin = 0; + size_t end = 0; + }; + struct PendingBatchLookupBlock { + size_t group_index = 0; + format::BlockRef ref; + size_t handle = 0; + }; + io::FileReader* reader_ = nullptr; + format::IndexTier tier_ = format::IndexTier::kT1; + bool has_positions_ = false; + LogicalIndexOpenMode open_mode_ = LogicalIndexOpenMode::kQuery; + format::CoreMetadata core_; + format::SampledTermIndexReader sti_; + format::DictBlockDirectoryReader dbd_; + format::BsbfHeader bsbf_header_; // resident header (from section ref) + bool has_bsbf_ = false; + // L0 tiering: when the bsbf section is small (<= kBsbfResidentMaxBytes) its + // whole bitset is loaded here at open -> in-memory probe, no per-lookup + // round. Larger filters keep only the parsed header here, so the small + // header enters Doris searcher cache and lookup reads just one 32-byte body + // block for an L1 probe. + bool bsbf_resident_ = false; + std::vector bsbf_resident_bitset_; + + // Small DICT blocks are opened once with the index so exact lookups avoid an + // otherwise serial S3 round for the term dictionary. Empty means the + // dictionary exceeded the resident threshold and lookup/prefix enumeration + // read blocks on demand. Each DictBlockReader holds a Slice into the owning + // bytes. + struct ResidentDictBlock { + std::vector bytes; + format::DictBlockReader reader; + }; + Status load_resident_dict_blocks(); + Status load_resident_bsbf(); + // Resolves the DictBlockReader for `ordinal`. Resident blocks return a pointer + // into the reader-owned resident set with *pin left null (stable for the reader + // lifetime). On-demand blocks are decoded (optionally via the request-scoped + // `cache`) into a heap-allocated DecodedDictBlock; *pin holds it alive so *out + // never dangles under a later cache eviction. Callers must keep *pin alive for + // as long as they use *out. + Status dict_block_reader_for_ordinal(uint32_t ordinal, DictBlockCache* cache, + std::shared_ptr* pin, + const format::DictBlockReader** out) const; + Status locate_candidate_dict_block(std::string_view term, bool* maybe_present, + uint32_t* ordinal) const; + Status collect_batch_lookup_groups(const std::vector& terms, + std::vector* candidates, + std::vector* groups) const; + static Status resolve_batch_lookup_group(const std::vector& terms, + const std::vector& candidates, + const BatchLookupGroup& group, + const format::DictBlockReader& block_reader, + std::vector* results); + Status lookup_batch_on_demand(const std::vector& terms, + const std::vector& candidates, + const std::vector& groups, + std::vector* results) const; + std::vector resident_dict_blocks_; + std::shared_ptr norms_cache_; + size_t norms_reserved_charge_ = 0; +}; + +} // namespace doris::snii::reader diff --git a/be/src/storage/index/snii/reader/snii_segment_reader.cpp b/be/src/storage/index/snii/reader/snii_segment_reader.cpp new file mode 100644 index 00000000000000..e769946b9eecfa --- /dev/null +++ b/be/src/storage/index/snii/reader/snii_segment_reader.cpp @@ -0,0 +1,193 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/reader/snii_segment_reader.h" + +#include +#include +#include +#include + +#include "storage/index/snii/encoding/crc32c.h" +#include "storage/index/snii/format/core_metadata.h" +#include "storage/index/snii/format/tail_pointer.h" + +namespace doris::snii::reader { +namespace { + +Status corrupted(std::string_view message) { + return Status::Error(message); +} + +Status read_tail_pointer(io::FileReader* reader, format::TailPointer* tail, + uint64_t* footer_offset) { + const size_t footer_size = format::tail_pointer_size(); + const uint64_t total = reader->size(); + if (total < footer_size) { + return corrupted("segment: file smaller than tail pointer"); + } + *footer_offset = total - footer_size; + std::vector bytes; + RETURN_IF_ERROR(reader->read_at(*footer_offset, footer_size, &bytes)); + return format::decode_tail_pointer(Slice(bytes), tail); +} + +// Proves Core -> STI -> DBD are adjacent and end at or before directory_offset, so open_index() can +// cover the whole group with one range read. It also bounds core.length + sti.length + dbd.length by +// directory_offset (itself bounded by the file size), which is why open_index() may sum and narrow +// those on-disk 64-bit lengths to size_t without a further overflow check. +Status validate_metadata_group(const format::LogicalIndexMetadataRef& entry, + uint64_t directory_offset) { + const auto& core = entry.core_metadata; + const auto& sti = entry.sampled_term_index; + const auto& dbd = entry.dict_block_directory; + if (core.offset > directory_offset || core.length > directory_offset - core.offset) { + return corrupted("segment: Core metadata reference is outside metadata area"); + } + const uint64_t sti_offset = core.offset + core.length; + if (sti.offset != sti_offset) { + return corrupted("segment: STI metadata is not adjacent to Core metadata"); + } + if (sti.length > directory_offset - sti.offset) { + return corrupted("segment: STI metadata reference is outside metadata area"); + } + const uint64_t dbd_offset = sti.offset + sti.length; + if (dbd.offset != dbd_offset) { + return corrupted("segment: DBD metadata is not adjacent to STI metadata"); + } + if (dbd.length > directory_offset - dbd.offset) { + return corrupted("segment: DBD metadata reference is outside metadata area"); + } + return Status::OK(); +} + +Status find_metadata_ref(const format::MetadataDirectory& directory, uint64_t index_id, + std::string_view suffix, const format::LogicalIndexMetadataRef** out) { + *out = directory.find(index_id, suffix); + if (*out == nullptr) { + return Status::Error( + "segment: logical index not found"); + } + return Status::OK(); +} + +} // namespace + +Status SniiSegmentReader::open(io::FileReader* const reader, SniiSegmentReader* const out) { + if (reader == nullptr) { + return Status::Error("segment: null reader"); + } + if (out == nullptr) { + return Status::Error("segment: null out"); + } + *out = {}; + + // The per-segment bootstrap header at offset zero remains an inspect-tool record and is + // intentionally not read here. The footer validates the exact format version and its own CRC, + // and it locates the metadata directory. Reading only the file tail avoids an otherwise + // redundant offset-zero cache block or remote round trip on cold queries. Future incompatible + // evolution must bump the footer format version rather than rely on a min-reader-version change + // under a stable format version. + format::TailPointer tail; + uint64_t footer_offset = 0; + RETURN_IF_ERROR(read_tail_pointer(reader, &tail, &footer_offset)); + if (tail.directory_offset > footer_offset || + tail.directory_length > footer_offset - tail.directory_offset) { + return corrupted("segment: metadata directory reference overlaps footer or EOF"); + } + if (tail.directory_length > static_cast(std::numeric_limits::max())) { + return corrupted("segment: metadata directory exceeds protobuf parse limit"); + } + // The protobuf parse limit above already bounds the length, so narrowing it is safe. + const auto directory_length = static_cast(tail.directory_length); + + std::vector directory_bytes; + RETURN_IF_ERROR(reader->read_at(tail.directory_offset, directory_length, &directory_bytes)); + if (crc32c(Slice(directory_bytes)) != tail.directory_crc32c) { + return corrupted("segment: metadata directory crc32c mismatch"); + } + + format::MetadataDirectory directory; + RETURN_IF_ERROR(format::MetadataDirectory::decode(Slice(directory_bytes), &directory)); + for (const auto& entry : directory.entries()) { + RETURN_IF_ERROR(validate_metadata_group(entry, tail.directory_offset)); + } + + out->reader_ = reader; + out->directory_ = std::move(directory); + return Status::OK(); +} + +Status SniiSegmentReader::index_exists(uint64_t index_id, std::string_view suffix, + bool* const exists) const { + if (exists == nullptr) { + return Status::Error("segment: null exists out"); + } + if (reader_ == nullptr) { + return Status::Error("segment: not opened"); + } + *exists = directory_.find(index_id, suffix) != nullptr; + return Status::OK(); +} + +Status SniiSegmentReader::open_index(uint64_t index_id, std::string_view suffix, + LogicalIndexReader* const out, + LogicalIndexOpenMode open_mode) const { + if (out == nullptr) { + return Status::Error("segment: null index out"); + } + *out = {}; + if (reader_ == nullptr) { + return Status::Error("segment: not opened"); + } + const format::LogicalIndexMetadataRef* entry = nullptr; + RETURN_IF_ERROR(find_metadata_ref(directory_, index_id, suffix, &entry)); + + // Safe to sum and narrow: open() ran validate_metadata_group on every directory entry. + const auto core_length = static_cast(entry->core_metadata.length); + const auto sti_length = static_cast(entry->sampled_term_index.length); + const auto dbd_length = static_cast(entry->dict_block_directory.length); + const size_t group_length = core_length + sti_length + dbd_length; + std::vector group; + RETURN_IF_ERROR(reader_->read_at(entry->core_metadata.offset, group_length, &group)); + DORIS_CHECK_EQ(group.size(), group_length); + const Slice bytes(group); + return LogicalIndexReader::open( + reader_, bytes.subslice(0, core_length), bytes.subslice(core_length, sti_length), + bytes.subslice(core_length + sti_length, dbd_length), out, open_mode); +} + +Status SniiSegmentReader::section_refs_for_index(uint64_t index_id, std::string_view suffix, + format::SectionRefs* const out) const { + if (out == nullptr) { + return Status::Error("segment: null section refs out"); + } + if (reader_ == nullptr) { + return Status::Error("segment: not opened"); + } + const format::LogicalIndexMetadataRef* entry = nullptr; + RETURN_IF_ERROR(find_metadata_ref(directory_, index_id, suffix, &entry)); + std::vector core_bytes; + RETURN_IF_ERROR(reader_->read_at(entry->core_metadata.offset, entry->core_metadata.length, + &core_bytes)); + format::CoreMetadata core; + RETURN_IF_ERROR(format::decode_core_metadata(Slice(core_bytes), &core)); + *out = core.section_refs; + return Status::OK(); +} + +} // namespace doris::snii::reader diff --git a/be/src/storage/index/snii/reader/snii_segment_reader.h b/be/src/storage/index/snii/reader/snii_segment_reader.h new file mode 100644 index 00000000000000..42c29f867c6e3c --- /dev/null +++ b/be/src/storage/index/snii/reader/snii_segment_reader.h @@ -0,0 +1,79 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/format/core_metadata.h" +#include "storage/index/snii/format/metadata_directory.h" +#include "storage/index/snii/io/file_reader.h" +#include "storage/index/snii/reader/logical_index_reader.h" + +// SniiSegmentReader -- entry point for the SNII segment read path. It opens a +// single .idx container through a (possibly metered) io::FileReader and exposes +// its logical indexes. open() reads only the file tail: +// 1. the fixed tail pointer (last tail_pointer_size() bytes), which also gates +// the container format_version ('TAIL' magic + format_version exact-match + +// tail crc), and +// 2. the raw protobuf logical-index metadata directory. +// The bootstrap header at offset 0 is still WRITTEN on disk (for inspect tooling) +// but is intentionally NOT read at open: its only runtime role (the container +// version gate) is already covered, more strictly, by the tail pointer, so +// skipping it avoids a redundant offset-0 cache block / remote round-trip per +// segment on cold queries. +// Per-index metadata groups are read lazily by open_index() so opening one logical +// index does not read every other logical index's metadata. +// +// open_index() then materializes one LogicalIndexReader from the metadata group +// of a given (index_id, suffix); query functions operate on that reader. +namespace doris::snii::reader { + +class SniiSegmentReader { +public: + SniiSegmentReader() = default; + + // Reads the tail pointer + raw metadata directory from reader (the offset-0 + // bootstrap header is not read; the tail pointer gates the container version). + // reader must outlive the returned SniiSegmentReader and every + // LogicalIndexReader opened from it. reader == nullptr / out == nullptr -> + // InvalidArgument; structural problems -> Corruption / Unsupported. + static Status open(io::FileReader* const reader, SniiSegmentReader* const out); + + uint32_t n_logical_indexes() const { return static_cast(directory_.size()); } + + Status index_exists(uint64_t index_id, std::string_view suffix, bool* const exists) const; + + // Loads the adjacent Core/STI/DBD group for (index_id, suffix) and builds a + // LogicalIndexReader bound to the same FileReader. Absent index -> NotFound. + Status open_index(uint64_t index_id, std::string_view suffix, LogicalIndexReader* const out, + LogicalIndexOpenMode open_mode = LogicalIndexOpenMode::kQuery) const; + Status section_refs_for_index(uint64_t index_id, std::string_view suffix, + format::SectionRefs* const out) const; + + io::FileReader* reader() const { return reader_; } + +private: + io::FileReader* reader_ = nullptr; + format::MetadataDirectory directory_; +}; + +} // namespace doris::snii::reader diff --git a/be/src/storage/index/snii/reader/windowed_posting.cpp b/be/src/storage/index/snii/reader/windowed_posting.cpp new file mode 100644 index 00000000000000..17d931640fb428 --- /dev/null +++ b/be/src/storage/index/snii/reader/windowed_posting.cpp @@ -0,0 +1,308 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/reader/windowed_posting.h" + +#include +#include + +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/encoding/byte_source.h" +#include "storage/index/snii/format/frq_pod.h" +#include "storage/index/snii/format/frq_prelude.h" +#include "storage/index/snii/format/prx_pod.h" +#include "storage/index/snii/io/batch_range_fetcher.h" + +namespace doris::snii::reader { + +using format::DictEntry; +using format::FrqPreludeReader; +using format::FrqRegionMeta; +using format::WindowMeta; + +namespace { + +// Resolves the absolute file offset of the prelude bytes for a windowed entry. +// The frq span lives in the interleaved posting region (after the term's prx span). +uint64_t prelude_abs(const LogicalIndexReader& idx, const DictEntry& entry, uint64_t frq_base) { + const auto& region = idx.section_refs().posting_region; + return region.offset + frq_base + entry.frq_off_delta; +} + +// Validates that [off, off+len) fits within [0, total). +Status in_bounds(uint64_t off, uint64_t len, uint64_t total) { + if (off > total || len > total - off) { + return Status::Error( + "windowed_posting: range out of section"); + } + return Status::OK(); +} + +// Block geometry of a windowed entry's grouped .frq payload (all offsets absolute). +struct BlockGeometry { + uint64_t dd_block_off = 0; // absolute start of the dd-block + uint64_t dd_block_len = 0; + uint64_t freq_block_off = 0; // absolute start of the freq-block + uint64_t freq_block_len = 0; + uint64_t frq_region_len = 0; // entry.frq_len - prelude_len (dd-block + freq-block) +}; + +// Derives the dd-block / freq-block absolute ranges from the entry + prelude, +// validating they tile the post-prelude .frq region exactly. +Status resolve_blocks(const LogicalIndexReader& idx, const DictEntry& entry, uint64_t frq_base, + const FrqPreludeReader& prelude, BlockGeometry* g) { + if (entry.prelude_len > entry.frq_len) { + return Status::Error( + "windowed_posting: prelude_len exceeds frq_len"); + } + const uint64_t frq_window_start = prelude_abs(idx, entry, frq_base) + entry.prelude_len; + g->frq_region_len = entry.frq_len - entry.prelude_len; + g->dd_block_len = prelude.dd_block_len(); + g->freq_block_len = prelude.freq_block_len(); + // dd-block + freq-block must fit exactly within the post-prelude region. + if (g->dd_block_len > g->frq_region_len || + g->freq_block_len > g->frq_region_len - g->dd_block_len) { + return Status::Error( + "windowed_posting: blocks exceed frq region"); + } + g->dd_block_off = frq_window_start; + g->freq_block_off = frq_window_start + g->dd_block_len; + return Status::OK(); +} + +// Per-window decode state for the full-posting path. +struct WindowSlices { + WindowMeta meta; + Slice dd_region; + Slice freq_region; + Slice prx_window; +}; + +// Carves window w's dd (and freq when want_freq) sub-slices out of the fetched +// blocks, validating each locator against its block length. +Status carve_region_slices(const WindowMeta& m, Slice dd_block, Slice freq_block, bool want_freq, + WindowSlices* out) { + RETURN_IF_ERROR(in_bounds(m.dd_off, m.dd_disk_len, dd_block.size())); + out->dd_region = + dd_block.subslice(static_cast(m.dd_off), static_cast(m.dd_disk_len)); + if (!want_freq) { + return Status::OK(); + } + RETURN_IF_ERROR(in_bounds(m.freq_off, m.freq_disk_len, freq_block.size())); + out->freq_region = freq_block.subslice(static_cast(m.freq_off), + static_cast(m.freq_disk_len)); + return Status::OK(); +} + +// Decodes window w from the fetched blocks (+ optional prx slice) and appends to out. +Status append_window(const WindowSlices& ws, bool want_positions, bool want_freq, + DecodedPosting* out) { + std::vector docids, freqs; + std::vector> pos; + RETURN_IF_ERROR(decode_window_slices(ws.meta, ws.dd_region, ws.freq_region, ws.prx_window, + want_positions, want_freq, &docids, &freqs, &pos)); + out->docids.insert(out->docids.end(), docids.begin(), docids.end()); + out->freqs.insert(out->freqs.end(), freqs.begin(), freqs.end()); + if (want_positions) { + for (auto& v : pos) { + out->positions.push_back(std::move(v)); + } + } + return Status::OK(); +} + +} // namespace + +Status fetch_windowed_prelude(const LogicalIndexReader& idx, const DictEntry& entry, + uint64_t frq_base, FrqPreludeReader* prelude) { + if (entry.prelude_len == 0) { + return Status::Error( + "windowed_posting: windowed entry has no prelude"); + } + if (entry.prelude_len > entry.frq_len) { + return Status::Error( + "windowed_posting: prelude_len exceeds frq_len"); + } + const uint64_t prelude_offset = prelude_abs(idx, entry, frq_base); + io::BatchRangeFetcher fetcher(idx.reader()); + const size_t h = fetcher.add(prelude_offset, entry.prelude_len); + RETURN_IF_ERROR(fetcher.fetch()); + return FrqPreludeReader::open(fetcher.get(h), prelude); +} + +Status windowed_window_range(const LogicalIndexReader& idx, const DictEntry& entry, + uint64_t frq_base, uint64_t prx_base, const FrqPreludeReader& prelude, + uint32_t w, bool want_positions, bool want_freq, WindowAbsRange* out) { + if (out == nullptr) { + return Status::Error("windowed_posting: null range"); + } + *out = WindowAbsRange {}; + BlockGeometry g; + RETURN_IF_ERROR(resolve_blocks(idx, entry, frq_base, prelude, &g)); + WindowMeta meta; + RETURN_IF_ERROR(prelude.window(w, &meta)); + + // dd sub-range within the dd-block. + RETURN_IF_ERROR(in_bounds(meta.dd_off, meta.dd_disk_len, g.dd_block_len)); + out->dd_off = g.dd_block_off + meta.dd_off; + out->dd_len = meta.dd_disk_len; + + if (want_freq) { + // Symmetric to the positions guard below: a G16 freq-elided posting + // (freq-dropped index or prune-mode bigram) declares has_freq=false in + // its prelude flags. INVALID_ARGUMENT and not FILE_CORRUPTED: the + // Doris segment iterator silently downgrades the corruption code to a + // non-index evaluation, which would mask this by-design layout. + if (!prelude.has_freq()) { + return Status::Error( + "windowed_posting: freqs requested but prelude has none"); + } + RETURN_IF_ERROR(in_bounds(meta.freq_off, meta.freq_disk_len, g.freq_block_len)); + out->freq_off = g.freq_block_off + meta.freq_off; + out->freq_len = meta.freq_disk_len; + } + + if (!want_positions) { + return Status::OK(); + } + if (!prelude.has_prx()) { + return Status::Error( + "windowed_posting: positions requested but prelude has none"); + } + const uint64_t prx_region_start = + idx.section_refs().posting_region.offset + prx_base + entry.prx_off_delta; + RETURN_IF_ERROR(in_bounds(meta.prx_off, meta.prx_len, entry.prx_len)); + out->prx_off = prx_region_start + meta.prx_off; + out->prx_len = meta.prx_len; + return Status::OK(); +} + +Status decode_window_slices(const WindowMeta& meta, Slice dd_region, Slice freq_region, + Slice prx_window, bool want_positions, bool want_freq, + std::vector* docids, std::vector* freqs, + std::vector>* positions) { + FrqRegionMeta dd_meta; + dd_meta.zstd = meta.dd_zstd; + dd_meta.uncomp_len = meta.dd_uncomp_len; + dd_meta.disk_len = meta.dd_disk_len; + dd_meta.crc = meta.crc_dd; + dd_meta.verify_crc = meta.verify_crc; + RETURN_IF_ERROR(format::decode_dd_region(dd_region, dd_meta, meta.win_base, docids)); + if (docids->size() != meta.doc_count) { + return Status::Error( + "windowed_posting: frq doc_count mismatch"); + } + if (want_freq) { + FrqRegionMeta freq_meta; + freq_meta.zstd = meta.freq_zstd; + freq_meta.uncomp_len = meta.freq_uncomp_len; + freq_meta.disk_len = meta.freq_disk_len; + freq_meta.crc = meta.crc_freq; + freq_meta.verify_crc = meta.verify_crc; + RETURN_IF_ERROR(format::decode_freq_region(freq_region, freq_meta, meta.doc_count, freqs)); + } else { + freqs->clear(); + } + if (!want_positions) { + return Status::OK(); + } + + ByteSource psrc(prx_window); + RETURN_IF_ERROR(format::read_prx_window(&psrc, positions)); + if (!psrc.eof()) { + return Status::Error( + "windowed_posting: trailing bytes after prx frame"); + } + if (positions->size() != docids->size()) { + return Status::Error( + "windowed_posting: prx/frq doc-count mismatch"); + } + return Status::OK(); +} + +namespace { + +// Fetches the dd-block (always), the freq-block (when want_freq) and the whole .prx +// region (when want_positions) of a windowed entry in ONE batch and returns the +// in-memory block slices. The dd-block is a single contiguous range -> the +// docid-only / phrase path reads it as one Range GET (the byte-saving core). +Status fetch_blocks(const LogicalIndexReader& idx, const DictEntry& entry, uint64_t prx_base, + const BlockGeometry& g, bool want_positions, bool want_freq, + io::BatchRangeFetcher* fetcher, size_t* dd_h, size_t* freq_h, size_t* prx_h) { + *dd_h = fetcher->add(g.dd_block_off, g.dd_block_len); + if (want_freq) { + *freq_h = fetcher->add(g.freq_block_off, g.freq_block_len); + } + if (want_positions) { + const uint64_t prx_region_start = + idx.section_refs().posting_region.offset + prx_base + entry.prx_off_delta; + *prx_h = fetcher->add(prx_region_start, entry.prx_len); + } + return fetcher->fetch(); +} + +} // namespace + +Status read_windowed_posting(const LogicalIndexReader& idx, const DictEntry& entry, + uint64_t frq_base, uint64_t prx_base, bool want_positions, + bool want_freq, DecodedPosting* out) { + if (out == nullptr) { + return Status::Error("windowed_posting: null out"); + } + *out = DecodedPosting {}; + + FrqPreludeReader prelude; + RETURN_IF_ERROR(fetch_windowed_prelude(idx, entry, frq_base, &prelude)); + if (want_positions && !prelude.has_prx()) { + return Status::Error( + "windowed_posting: positions requested but prelude has none"); + } + // G16 freq-elided postings (freq-dropped index or prune-mode bigram) + // declare has_freq=false; fail with the semantic error instead of a deep + // region-decode corruption. INVALID_ARGUMENT so the Doris segment + // iterator's corruption downgrade never masks this by-design layout. + if (want_freq && !prelude.has_freq()) { + return Status::Error( + "windowed_posting: freqs requested but prelude has none"); + } + BlockGeometry g; + RETURN_IF_ERROR(resolve_blocks(idx, entry, frq_base, prelude, &g)); + + io::BatchRangeFetcher fetcher(idx.reader()); + size_t dd_h = 0, freq_h = 0, prx_h = 0; + RETURN_IF_ERROR(fetch_blocks(idx, entry, prx_base, g, want_positions, want_freq, &fetcher, + &dd_h, &freq_h, &prx_h)); + const Slice dd_block = fetcher.get(dd_h); + const Slice freq_block = want_freq ? fetcher.get(freq_h) : Slice(); + const Slice prx_region = want_positions ? fetcher.get(prx_h) : Slice(); + + const uint32_t n = prelude.n_windows(); + for (uint32_t w = 0; w < n; ++w) { + WindowSlices ws; + RETURN_IF_ERROR(prelude.window(w, &ws.meta)); + RETURN_IF_ERROR(carve_region_slices(ws.meta, dd_block, freq_block, want_freq, &ws)); + if (want_positions) { + RETURN_IF_ERROR(in_bounds(ws.meta.prx_off, ws.meta.prx_len, prx_region.size())); + ws.prx_window = prx_region.subslice(static_cast(ws.meta.prx_off), + static_cast(ws.meta.prx_len)); + } + RETURN_IF_ERROR(append_window(ws, want_positions, want_freq, out)); + } + return Status::OK(); +} + +} // namespace doris::snii::reader diff --git a/be/src/storage/index/snii/reader/windowed_posting.h b/be/src/storage/index/snii/reader/windowed_posting.h new file mode 100644 index 00000000000000..dee527430aff08 --- /dev/null +++ b/be/src/storage/index/snii/reader/windowed_posting.h @@ -0,0 +1,121 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include "common/status.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/format/dict_entry.h" +#include "storage/index/snii/format/frq_prelude.h" +#include "storage/index/snii/reader/logical_index_reader.h" + +// WindowedPostingReader -- shared read-side decode of a windowed term's posting +// from its two-level frq_prelude + GROUPED dd-block / freq-block (design 1.6). +// +// A windowed pod_ref entry's .frq payload is laid out +// [prelude][dd-block][freq-block] +// where the dd-block concatenates every window's dd_region and the freq-block +// every window's freq_region. The docs-only prefix [prelude][dd-block] is ONE +// contiguous run. This helper: +// 1. range-fetches the prelude (prelude_len bytes) and parses the directory, +// 2. range-fetches the WHOLE dd-block in ONE contiguous range (and, for +// scoring, +// the whole freq-block in one more range), +// 3. decodes each window's dd region (and freq region) from the in-memory +// blocks +// via the prelude metadata (dd_off/dd_disk_len, freq_off/freq_disk_len), +// and concatenates the per-window docids / freqs / positions. +// +// The slim/inline single-window path is handled by the term/phrase/scoring +// callers directly; this helper is for enc=windowed entries only. +namespace doris::snii::reader { + +// Coalesce gap (bytes) used when batch-fetching MULTIPLE dd sub-ranges of the +// SAME term (the phrase window-skip path): dd regions of one term are +// contiguous in the dd-block, so merging reads separated by <= this gap into +// one physical Range GET trades a little over-read for fewer remote GETs (the +// design's higher-priority metric). Only applied to same-term multi-window +// batches, never to cross-term. +inline constexpr uint64_t kSameTermCoalesceGap = 16 * 1024; + +// Full decoded posting for one windowed term (docids ascending across windows). +struct DecodedPosting { + std::vector docids; + std::vector freqs; // aligned with docids + std::vector> positions; // aligned; empty when no prx +}; + +// Decodes the entire windowed posting. want_positions requires the index to +// have positions (and the entry to carry prx). want_freq selects whether the +// freq-block is fetched + decoded: when false ONLY the contiguous +// [prelude][dd-block] prefix is fetched (docid-only / phrase callers) and +// DecodedPosting.freqs stays empty; when true the freq-block is additionally +// fetched (scoring). Returns Corruption on any prelude/block inconsistency +// (doc-count mismatch, out-of-range offsets). +Status read_windowed_posting(const LogicalIndexReader& idx, const format::DictEntry& entry, + uint64_t frq_base, uint64_t prx_base, bool want_positions, + bool want_freq, DecodedPosting* out); + +// --- Sub-block (window) skipping helpers (shared with phrase / selective WAND) +// -- +// +// These expose the per-window dd/freq/prx addressing within the grouped blocks +// so the skip path can fetch ONLY the windows covering candidate docids (their +// dd sub-ranges within the dd-block, near-contiguous and coalesce-friendly) +// instead of the whole posting, without duplicating the offset arithmetic. + +// Absolute file byte ranges of one window's regions. dd is always valid; freq +// is valid only when want_freq; prx is valid only when want_positions (and +// has_prx). +struct WindowAbsRange { + uint64_t dd_off = 0; + uint64_t dd_len = 0; + uint64_t freq_off = 0; + uint64_t freq_len = 0; + uint64_t prx_off = 0; + uint64_t prx_len = 0; +}; + +// Fetches + parses the two-level prelude of a windowed entry (one batched +// read). +Status fetch_windowed_prelude(const LogicalIndexReader& idx, const format::DictEntry& entry, + uint64_t frq_base, format::FrqPreludeReader* prelude); + +// Computes the absolute file ranges of window w's dd region (and freq region +// when want_freq, and .prx window when want_positions), fully validated against +// the POD sections (anti-DoS: rejects out-of-range offsets and overflowing +// locators). +Status windowed_window_range(const LogicalIndexReader& idx, const format::DictEntry& entry, + uint64_t frq_base, uint64_t prx_base, + const format::FrqPreludeReader& prelude, uint32_t w, + bool want_positions, bool want_freq, WindowAbsRange* out); + +// Decodes one window's docids (and per-doc positions when want_positions, and +// per-doc freqs when want_freq) from already-fetched byte slices: dd_region is +// the window's dd sub-slice; freq_region its freq sub-slice (ignored when +// !want_freq); prx_window its .prx bytes. The decoded docids are absolute +// (win_base applied). Returns Corruption on any doc-count mismatch between the +// prelude, dd/freq and prx. +Status decode_window_slices(const format::WindowMeta& meta, Slice dd_region, Slice freq_region, + Slice prx_window, bool want_positions, bool want_freq, + std::vector* docids, std::vector* freqs, + std::vector>* positions); + +} // namespace doris::snii::reader diff --git a/be/src/storage/index/snii/snii_doris_adapter.cpp b/be/src/storage/index/snii/snii_doris_adapter.cpp new file mode 100644 index 00000000000000..e4a5875184535f --- /dev/null +++ b/be/src/storage/index/snii/snii_doris_adapter.cpp @@ -0,0 +1,405 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/snii_doris_adapter.h" + +#include + +#include +#include +#include +#include + +#include "common/cast_set.h" +#include "cpp/sync_point.h" +#include "runtime/exec_env.h" +#include "runtime/thread_context.h" +#include "storage/index/snii/common/uninitialized_buffer.h" +#include "util/countdown_latch.h" +#include "util/threadpool.h" + +namespace doris::segment_v2::snii_doris { +namespace { +// Per-call cap on concurrently dispatched physical segment reads. A coalesced +// batch of at most this many segments is served as a single concurrent round +// (mirrors the "at most one serial round" contract that the test-side +// MeteredFileReader measures, and the S3 standalone reader's 16-way fan-out). +constexpr size_t kMaxConcurrentReads = 16; +} // namespace + +thread_local const io::IOContext* DorisSniiFileReader::_scoped_io_ctx = nullptr; +ThreadPool* DorisSniiFileReader::_io_pool_for_test = nullptr; + +Status DorisSniiFileWriter::append(::doris::snii::Slice data) { + if (_writer == nullptr) { + return Status::Error("doris writer is null"); + } + return _writer->append(Slice(reinterpret_cast(data.data()), data.size())); +} + +Status DorisSniiFileWriter::finalize() { + if (_writer == nullptr) { + return Status::Error("doris writer is null"); + } + return Status::OK(); +} + +uint64_t DorisSniiFileWriter::bytes_written() const { + return _writer == nullptr ? 0 : _writer->bytes_appended(); +} + +DorisSniiFileReader::DorisSniiFileReader(io::FileReaderSPtr reader, const io::IOContext* io_ctx, + bool direct_remote_io) + : _reader(std::move(reader)), + _direct_remote_io(direct_remote_io), + _default_io_ctx(_make_index_io_context(io_ctx)) {} + +io::IOContext DorisSniiFileReader::_make_index_io_context(const io::IOContext* io_ctx) { + io::IOContext index_io_ctx; + if (io_ctx != nullptr) { + index_io_ctx = *io_ctx; + } + index_io_ctx.is_inverted_index = true; + // is_index_data is inherited from io_ctx: META scopes set it true at the source + // (index_file_reader), non-meta reads default to false. + return index_io_ctx; +} + +DorisSniiFileReader::ScopedIOContext::ScopedIOContext(const io::IOContext* io_ctx) + : _previous(_scoped_io_ctx), _io_ctx(DorisSniiFileReader::_make_index_io_context(io_ctx)) { + _scoped_io_ctx = &_io_ctx; +} + +DorisSniiFileReader::ScopedIOContext::~ScopedIOContext() { + _scoped_io_ctx = _previous; +} + +Status DorisSniiFileReader::read_at(uint64_t offset, size_t len, std::vector* out) { + if (out == nullptr) { + return Status::Error("output buffer is null"); + } + RETURN_IF_ERROR(_check_read_range(offset, len)); + RETURN_IF_ERROR(_read_at(offset, len, out, _current_io_ctx())); + if (len > 0) { + _record_read_stats(cast_set(len), cast_set(len), 1, 1); + } + return Status::OK(); +} + +// NOLINTNEXTLINE(readability-non-const-parameter): out is the SNII read output buffer. +Status DorisSniiFileReader::_read_at(uint64_t offset, size_t len, std::vector* out, + const io::IOContext* io_ctx) const { + TEST_SYNC_POINT_RETURN_WITH_VALUE("DorisSniiFileReader::_read_at", + Status::IOError("injected SNII read failure"), offset, len); + DCHECK(_reader != nullptr); + DCHECK(out != nullptr); + DCHECK(_check_read_range(offset, len).ok()); + if (len == 0) { + out->clear(); + return Status::OK(); + } + out->resize(len); + size_t bytes_read = 0; + auto status = _reader->read_at(offset, Slice(out->data(), len), &bytes_read, io_ctx); + if (!status.ok()) { + return status; + } + if (bytes_read != len) { + return Status::Error( + fmt::format("short read at offset {}, expect {}, got {}", offset, len, bytes_read)); + } + return Status::OK(); +} + +// NOLINTBEGIN(readability-non-const-parameter): outs is the SNII batch read output buffer. +Status DorisSniiFileReader::read_batch(const std::vector<::doris::snii::io::Range>& ranges, + std::vector>* outs) { + if (outs == nullptr) { + return Status::Error("output buffers is null"); + } + outs->clear(); + outs->resize(ranges.size()); + if (ranges.empty()) { + return Status::OK(); + } + + // ----- Phase 1: plan (serial, lock-free) ----- + // No section-classification lock exists on this reader, so the whole plan is + // a plain in-memory scan; the NO-IO-UNDER-LOCK red line holds trivially (no + // lock is taken anywhere in read_batch). + struct IndexedRange { + uint64_t offset = 0; + size_t len = 0; + size_t index = 0; + }; + int64_t request_bytes = 0; + std::vector sorted; + sorted.reserve(ranges.size()); + for (size_t i = 0; i < ranges.size(); ++i) { + RETURN_IF_ERROR(_check_read_range(ranges[i].offset, ranges[i].len)); + request_bytes += cast_set(ranges[i].len); + if (ranges[i].len == 0) { + continue; + } + sorted.push_back({ranges[i].offset, ranges[i].len, i}); + } + if (sorted.empty()) { + return Status::OK(); + } + // F27: callers (BatchRangeFetcher::fetch) already pass offset-sorted ranges; + // only pay for a sort when the input is actually out of order. + auto by_offset = [](const IndexedRange& lhs, const IndexedRange& rhs) { + return lhs.offset < rhs.offset; + }; + if (!std::ranges::is_sorted(sorted, by_offset)) { + std::ranges::sort(sorted, by_offset); + } + + // Coalesce the sorted ranges into disjoint physical segments. + struct Seg { + uint64_t offset = 0; + size_t len = 0; + size_t begin = 0; // first index into `sorted` covered by this segment + size_t end = 0; // one-past-last index into `sorted` + bool single = false; + }; + constexpr uint64_t max_coalesced_gap = 4096; + constexpr uint64_t max_coalesced_read = 1ULL << 20; + std::vector segs; + for (size_t begin = 0; begin < sorted.size();) { + uint64_t read_offset = sorted[begin].offset; + uint64_t read_end = sorted[begin].offset + sorted[begin].len; + size_t end = begin + 1; + while (end < sorted.size()) { + const uint64_t next_end = sorted[end].offset + sorted[end].len; + if ((sorted[end].offset > read_end && + sorted[end].offset - read_end > max_coalesced_gap) || + next_end - read_offset > max_coalesced_read) { + break; + } + read_end = std::max(read_end, next_end); + ++end; + } + Seg seg; + seg.offset = read_offset; + seg.len = cast_set(read_end - read_offset); + seg.begin = begin; + seg.end = end; + // A single-range group exactly covers its segment, so it can be read + // straight into the caller's output slot with no temp + no second copy. + seg.single = (end == begin + 1); + segs.push_back(seg); + begin = end; + } + + // Resolve per-segment target buffers, io contexts and the shared sink on the + // calling thread: workers (which run on tracker-less pool threads) must not + // allocate, and per-segment private cache-stat slots keep disjoint physical + // reads from racing on the shared FileCacheStatistics. + const size_t num_segs = segs.size(); + const io::IOContext* base_io_ctx = _current_io_ctx(); + std::vector> tmp_bufs(num_segs); + std::vector*> targets(num_segs); + std::vector seg_stats(num_segs); + std::vector seg_io_ctx(num_segs); + std::vector seg_status(num_segs); + int64_t read_bytes = 0; + for (size_t s = 0; s < num_segs; ++s) { + const Seg& seg = segs[s]; + std::vector* target = + seg.single ? &(*outs)[sorted[seg.begin].index] : &tmp_bufs[s]; + ::doris::snii::resize_uninitialized(*target, seg.len); + targets[s] = target; + seg_io_ctx[s] = *base_io_ctx; + seg_io_ctx[s].file_cache_stats = + base_io_ctx->file_cache_stats != nullptr ? &seg_stats[s] : nullptr; + read_bytes += cast_set(seg.len); + } + + // ----- Phase 2: physical reads (lock-free; concurrent when a pool exists) ----- + auto run_segment = [&](size_t s) { + seg_status[s] = _read_at(segs[s].offset, segs[s].len, targets[s], &seg_io_ctx[s]); + }; + ThreadPool* pool = _select_io_pool(); + if (pool != nullptr && num_segs > 1) { + // Carried onto the pool threads below. They are "Orphan" threads with no MemTrackerLimiter + // of their own (buffered_reader.cpp:426), and the read does allocate down there: in cloud + // mode _read_at reaches CachedRemoteFileReader::read_at_impl -> + // _read_from_indirect_cache -> _read_remote_blocks_into_cache -> _execute_remote_read -> + // _execute_s3_fallback, which does `new char[span_size]`. Without this the span is charged + // to no tracker at all, which memory_orphan_check() (on by default) treats as a bug. + // Same shape as the hedged-read path in cached_remote_file_reader.cpp:500-505. + const std::shared_ptr parent_resource_ctx = + thread_context()->resource_ctx(); + for (size_t base = 0; base < num_segs; base += kMaxConcurrentReads) { + const size_t wave_end = std::min(base + kMaxConcurrentReads, num_segs); + ::doris::CountDownLatch latch(cast_set(wave_end - base)); + for (size_t s = base; s < wave_end; ++s) { + Status submit_st = + pool->submit_func([&run_segment, &latch, s, parent_resource_ctx]() { + std::unique_ptr attach_task; + if (parent_resource_ctx != nullptr) { + attach_task = std::make_unique(parent_resource_ctx); + } + run_segment(s); + latch.count_down(); + }); + if (!submit_st.ok()) { + // Pool full/shut down: read this segment inline; never skip + // the count_down or the latch would not drain. + run_segment(s); + latch.count_down(); + } + } + latch.wait(); + } + } else { + // Serial fallback: no executor (e.g. tools without ExecEnv) or a single + // segment (avoids micro-batch scheduling overhead). + for (size_t s = 0; s < num_segs; ++s) { + run_segment(s); + } + } + + // ----- Phase 3: merge stats, first-error, scatter, account (serial) ----- + // Fold every segment's private stats back FIRST: physical IO that already + // happened (including partial work inside a segment that then failed) must + // reach the query profile even when another segment of this batch errors. + if (base_io_ctx->file_cache_stats != nullptr) { + for (size_t s = 0; s < num_segs; ++s) { + _merge_file_cache_statistics(base_io_ctx->file_cache_stats, seg_stats[s]); + } + } + Status first_error = Status::OK(); + int64_t completed_request_bytes = 0; + int64_t completed_read_bytes = 0; + for (size_t s = 0; s < num_segs; ++s) { + if (!seg_status[s].ok()) { + if (first_error.ok()) { + first_error = seg_status[s]; + } + continue; + } + completed_read_bytes += cast_set(segs[s].len); + for (size_t i = segs[s].begin; i < segs[s].end; ++i) { + completed_request_bytes += cast_set(sorted[i].len); + } + } + if (!first_error.ok()) { + // Record only what actually completed so the logical counters stay + // truthful for the failed batch; ranges/rounds reflect what was issued. + _record_read_stats(completed_request_bytes, completed_read_bytes, + cast_set(num_segs), + cast_set(_compute_num_waves(num_segs))); + return first_error; + } + for (size_t s = 0; s < num_segs; ++s) { + const Seg& seg = segs[s]; + if (seg.single) { + continue; // already read in place + } + const std::vector& bytes = tmp_bufs[s]; + for (size_t i = seg.begin; i < seg.end; ++i) { + const uint64_t pos = sorted[i].offset - seg.offset; + auto& out = (*outs)[sorted[i].index]; + out.assign(bytes.begin() + cast_set(pos), + bytes.begin() + cast_set(pos + sorted[i].len)); + } + } + _record_read_stats(request_bytes, read_bytes, cast_set(num_segs), + cast_set(_compute_num_waves(num_segs))); + return Status::OK(); +} +// NOLINTEND(readability-non-const-parameter) + +uint64_t DorisSniiFileReader::size() const { + return _reader == nullptr ? 0 : _reader->size(); +} + +const io::IOContext* DorisSniiFileReader::_current_io_ctx() const { + return _scoped_io_ctx != nullptr ? _scoped_io_ctx : &_default_io_ctx; +} + +void DorisSniiFileReader::_record_read_stats(int64_t request_bytes, int64_t read_bytes, + int64_t range_read_count, + int64_t serial_read_rounds) const { + const auto* io_ctx = _current_io_ctx(); + if (io_ctx->file_cache_stats == nullptr) { + return; + } + auto* stats = io_ctx->file_cache_stats; + stats->inverted_index_request_bytes += request_bytes; + stats->inverted_index_read_bytes += read_bytes; + stats->inverted_index_range_read_count += range_read_count; + stats->inverted_index_serial_read_rounds += serial_read_rounds; + if (_direct_remote_io) { + // No CachedRemoteFileReader below us: every byte this reader fetched was a + // direct remote GET, so account it as physical remote IO here. + stats->inverted_index_remote_physical_read_bytes += read_bytes; + } +} + +void DorisSniiFileReader::set_io_thread_pool_for_test(ThreadPool* pool) { + _io_pool_for_test = pool; +} + +ThreadPool* DorisSniiFileReader::_select_io_pool() { + if (_io_pool_for_test != nullptr) { + return _io_pool_for_test; + } + if (ExecEnv::ready()) { + return ExecEnv::GetInstance()->buffered_reader_prefetch_thread_pool(); + } + return nullptr; +} + +size_t DorisSniiFileReader::_compute_num_waves(size_t seg_count) { + if (seg_count == 0) { + return 0; + } + return (seg_count + kMaxConcurrentReads - 1) / kMaxConcurrentReads; +} + +void DorisSniiFileReader::_merge_file_cache_statistics(io::FileCacheStatistics* dst, + const io::FileCacheStatistics& src) { + if (dst == nullptr) { + return; + } + // Delegate to the canonical field list so per-wave private stats can never + // silently drop fields the general layer adds (e.g. write_cache_io_timer, + // remote_only_on_miss_*), which a hand-rolled copy here used to do. + dst->merge_from(src); +} + +Status DorisSniiFileReader::_check_read_range(uint64_t offset, size_t len) const { + if (_reader == nullptr) { + return Status::Error("doris reader is null"); + } + if (offset > std::numeric_limits::max() - len) { + return Status::Error( + fmt::format("read range overflows: offset {}, len {}", offset, len)); + } + const uint64_t end = offset + len; + if (end > _reader->size()) { + return Status::Error( + fmt::format("read range exceeds file size: offset {}, len {}, file size {}", offset, + len, _reader->size())); + } + return Status::OK(); +} + +} // namespace doris::segment_v2::snii_doris diff --git a/be/src/storage/index/snii/snii_doris_adapter.h b/be/src/storage/index/snii/snii_doris_adapter.h new file mode 100644 index 00000000000000..8d5966e0f7a90d --- /dev/null +++ b/be/src/storage/index/snii/snii_doris_adapter.h @@ -0,0 +1,114 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include "common/status.h" +#include "io/fs/file_reader.h" +#include "io/fs/file_writer.h" +#include "io/io_common.h" +#include "storage/index/snii/io/file_reader.h" +#include "storage/index/snii/io/file_writer.h" +#include "util/slice.h" + +namespace doris { +class ThreadPool; +} // namespace doris + +namespace doris::segment_v2::snii_doris { + +class DorisSniiFileWriter final : public ::doris::snii::io::FileWriter { +public: + explicit DorisSniiFileWriter(io::FileWriter* writer) : _writer(writer) {} + + Status append(::doris::snii::Slice data) override; + Status finalize() override; + uint64_t bytes_written() const override; + +private: + io::FileWriter* _writer = nullptr; +}; + +class DorisSniiFileReader final : public ::doris::snii::io::FileReader { +public: + class ScopedIOContext { + public: + explicit ScopedIOContext(const io::IOContext* io_ctx); + ~ScopedIOContext(); + + ScopedIOContext(const ScopedIOContext&) = delete; + ScopedIOContext& operator=(const ScopedIOContext&) = delete; + + private: + const io::IOContext* _previous = nullptr; + io::IOContext _io_ctx; + }; + + // `direct_remote_io` marks a reader whose byte ranges are served straight from + // remote storage with no CachedRemoteFileReader in between (the NO_CACHE + // diagnostic bypass on a non-local filesystem). Only that layer would normally + // account physical remote bytes, so this reader then counts its own reads as + // physical remote IO. Never set it for cached or local readers: the former + // double-counts, the latter reports local disk as remote fetch volume. + explicit DorisSniiFileReader(io::FileReaderSPtr reader, const io::IOContext* io_ctx = nullptr, + bool direct_remote_io = false); + + Status read_at(uint64_t offset, size_t len, std::vector* out) override; + Status read_batch(const std::vector<::doris::snii::io::Range>& ranges, + std::vector>* outs) override; + uint64_t size() const override; + + // Test-only: inject (or clear with nullptr) the thread pool used to fan out + // batch segment reads. nullptr (default) routes to the BE buffered-reader + // prefetch pool when ExecEnv is ready, otherwise a serial fallback. + static void set_io_thread_pool_for_test(ThreadPool* pool); + +private: + static io::IOContext _make_index_io_context(const io::IOContext* io_ctx); + Status _check_read_range(uint64_t offset, size_t len) const; + Status _read_at(uint64_t offset, size_t len, std::vector* out, + const io::IOContext* io_ctx) const; + const io::IOContext* _current_io_ctx() const; + void _record_read_stats(int64_t request_bytes, int64_t read_bytes, int64_t range_read_count, + int64_t serial_read_rounds) const; + + // Selects the executor for parallel batch segment reads: the test seam if + // set, else the BE prefetch pool when ExecEnv is ready, else nullptr (the + // caller then reads segments serially). + static ThreadPool* _select_io_pool(); + // Number of dependent serial dispatch waves for `seg_count` physical + // segments given the per-call concurrency cap. This is the F19 metric: + // a coalesced batch of <= cap segments is a single concurrent round. + static size_t _compute_num_waves(size_t seg_count); + // Folds a per-segment private stats slot back into the shared sink (disjoint + // physical reads never race on it). Delegates to FileCacheStatistics::merge_from + // so the field list can never drift from io_common.h. + static void _merge_file_cache_statistics(io::FileCacheStatistics* dst, + const io::FileCacheStatistics& src); + + io::FileReaderSPtr _reader; + const bool _direct_remote_io = false; + io::IOContext _default_io_ctx; + static thread_local const io::IOContext* _scoped_io_ctx; + // Test seam for the batch-read executor; see set_io_thread_pool_for_test. + static ThreadPool* _io_pool_for_test; +}; + +} // namespace doris::segment_v2::snii_doris diff --git a/be/src/storage/index/snii/snii_index_reader.cpp b/be/src/storage/index/snii/snii_index_reader.cpp new file mode 100644 index 00000000000000..3b09fb5dddaa3f --- /dev/null +++ b/be/src/storage/index/snii/snii_index_reader.cpp @@ -0,0 +1,1183 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/snii_index_reader.h" + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/config.h" +#include "runtime/exec_env.h" +#include "runtime/query_context.h" +#include "runtime/runtime_profile.h" +#include "runtime/runtime_state.h" +#include "storage/index/index_file_reader.h" +#include "storage/index/index_reader_helper.h" +#include "storage/index/inverted/analyzer/analyzer.h" +#include "storage/index/inverted/analyzer/segment_analyzer_context.h" +#include "storage/index/inverted/common/single_flight.h" +#include "storage/index/inverted/inverted_index_cache.h" +#include "storage/index/inverted/inverted_index_iterator.h" +#include "storage/index/inverted/token_filter/common_grams_filter.h" +#include "storage/index/snii/format/null_bitmap.h" +#include "storage/index/snii/query/boolean_query.h" +#include "storage/index/snii/query/count_query.h" +#include "storage/index/snii/query/docid_sink.h" +#include "storage/index/snii/query/internal/plain_term_routing.h" +#include "storage/index/snii/query/phrase_query.h" +#include "storage/index/snii/query/prefix_query.h" +#include "storage/index/snii/query/regexp_query.h" +#include "storage/index/snii/query/scoring_query.h" +#include "storage/index/snii/query/term_query.h" +#include "storage/index/snii/query/wildcard_query.h" +#include "storage/index/snii/reader/logical_index_reader.h" +#include "storage/index/snii/snii_doris_adapter.h" +#include "storage/index/snii/snii_prx_profile.h" +#include "storage/index/snii/stats/snii_stats_provider.h" +#include "util/defer_op.h" +#include "util/time.h" + +#ifdef BE_TEST +namespace doris::snii::testing { +namespace { + +std::atomic prx_execution_profile_scope_constructions {0}; +std::atomic prx_execution_profile_scope_flushes {0}; + +} // namespace + +void record_prx_execution_profile_scope_construction() { + prx_execution_profile_scope_constructions.fetch_add(1, std::memory_order_relaxed); +} + +void record_prx_execution_profile_scope_flush() { + prx_execution_profile_scope_flushes.fetch_add(1, std::memory_order_relaxed); +} + +void reset_prx_execution_profile_scope_counters() { + prx_execution_profile_scope_constructions.store(0, std::memory_order_relaxed); + prx_execution_profile_scope_flushes.store(0, std::memory_order_relaxed); +} + +uint64_t prx_execution_profile_scope_construction_count() { + return prx_execution_profile_scope_constructions.load(std::memory_order_relaxed); +} + +uint64_t prx_execution_profile_scope_flush_count() { + return prx_execution_profile_scope_flushes.load(std::memory_order_relaxed); +} + +} // namespace doris::snii::testing +#endif + +namespace doris::segment_v2 { + +namespace { + +class RoaringDocIdSink final : public ::doris::snii::query::DocIdSink { +public: + explicit RoaringDocIdSink(roaring::Roaring* bitmap) : _bitmap(bitmap) { + DCHECK(_bitmap != nullptr); + } + + Status append_sorted(std::span docids) override { + if (!docids.empty()) { + _bitmap->addMany(docids.size(), docids.data()); + } + return Status::OK(); + } + + Status append_range(uint32_t first, uint64_t last_exclusive) override { + if (last_exclusive > first) { + _bitmap->addRange(first, last_exclusive); + } + return Status::OK(); + } + + // Roaring addMany/addRange deduplicate and order natively, so multi-term OR + // can stream each posting straight into the bitmap (no per-term vector + merge). + bool dedups() const override { return true; } + +private: + roaring::Roaring* _bitmap; +}; + +struct SniiQueryExecutionResult { + std::shared_ptr bitmap; + std::vector<::doris::snii::query::PhraseMatch> phrase_matches; +}; + +std::vector to_terms(const InvertedIndexQueryInfo& query_info) { + std::vector terms; + terms.reserve(query_info.term_infos.size()); + for (const auto& term_info : query_info.term_infos) { + DCHECK(term_info.is_single_term()); + terms.push_back(term_info.get_single_term()); + } + return terms; +} + +bool uses_plain_term_frequency_scoring(InvertedIndexQueryType query_type, + const InvertedIndexQueryInfo& query_info) { + return query_type == InvertedIndexQueryType::MATCH_ANY_QUERY || + query_type == InvertedIndexQueryType::MATCH_ALL_QUERY || + (query_type == InvertedIndexQueryType::MATCH_PHRASE_QUERY && + query_info.term_infos.size() == 1); +} + +bool uses_phrase_frequency_scoring(InvertedIndexQueryType query_type, + const InvertedIndexQueryInfo& query_info) { + return query_info.term_infos.size() > 1 && + (query_type == InvertedIndexQueryType::MATCH_PHRASE_QUERY || + query_type == InvertedIndexQueryType::MATCH_PHRASE_PREFIX_QUERY); +} + +Status score_plain_term_candidates(const IndexQueryContextPtr& context, + std::string_view column_name, + const InvertedIndexQueryInfo& query_info, + const ::doris::snii::reader::LogicalIndexReader& logical_reader, + const ::doris::snii::stats::SniiStatsProvider& segment_stats, + const roaring::Roaring& final_candidates) { + DORIS_CHECK(context->collection_statistics != nullptr); + DORIS_CHECK(context->collection_similarity != nullptr); + + const std::wstring field_name = StringUtil::string_to_wstring(std::string(column_name)); + const double collection_avgdl = + context->collection_statistics->get_or_calculate_avg_dl(field_name); + std::vector<::doris::snii::query::CollectionScoringTerm> scoring_terms; + scoring_terms.reserve(query_info.term_infos.size()); + for (const auto& term_info : query_info.term_infos) { + DORIS_CHECK(term_info.is_single_term()); + std::string physical_term; + bool representable = false; + RETURN_IF_ERROR(::doris::snii::query::internal::route_query_term( + logical_reader, term_info, &physical_term, &representable)); + if (!representable) { + continue; + } + const std::string& logical_term = term_info.get_single_term(); + const double idf = context->collection_statistics->get_or_calculate_idf( + field_name, StringUtil::string_to_wstring(logical_term)); + scoring_terms.push_back({.physical_term = std::move(physical_term), .idf = idf}); + } + DORIS_CHECK(final_candidates.isEmpty() || !scoring_terms.empty()); + + std::vector<::doris::snii::query::ScoredDoc> scored_docs; + RETURN_IF_ERROR(::doris::snii::query::scoring_query_candidates( + logical_reader, segment_stats, scoring_terms, final_candidates, collection_avgdl, + ::doris::snii::query::Bm25Params {}, &scored_docs)); + for (const auto& scored_doc : scored_docs) { + context->collection_similarity->collect(scored_doc.docid, + static_cast(scored_doc.score)); + } + return Status::OK(); +} + +Status score_phrase_matches(const IndexQueryContextPtr& context, std::string_view column_name, + InvertedIndexQueryType query_type, + const InvertedIndexQueryInfo& query_info, + const ::doris::snii::reader::LogicalIndexReader& logical_reader, + const ::doris::snii::stats::SniiStatsProvider& segment_stats, + const roaring::Roaring& final_candidates, + const std::vector<::doris::snii::query::PhraseMatch>& matches) { + DORIS_CHECK(context->collection_statistics != nullptr); + DORIS_CHECK(context->collection_similarity != nullptr); + DORIS_CHECK(uses_phrase_frequency_scoring(query_type, query_info)); + DORIS_CHECK_EQ(final_candidates.cardinality(), matches.size()); + + const std::wstring field_name = StringUtil::string_to_wstring(std::string(column_name)); + const double collection_avgdl = + context->collection_statistics->get_or_calculate_avg_dl(field_name); + const size_t idf_term_count = query_type == InvertedIndexQueryType::MATCH_PHRASE_PREFIX_QUERY + ? query_info.term_infos.size() - 1 + : query_info.term_infos.size(); + double idf_sum = 0.0; + for (size_t i = 0; i < idf_term_count; ++i) { + const auto& term_info = query_info.term_infos[i]; + DORIS_CHECK(term_info.is_single_term()); + idf_sum += context->collection_statistics->get_or_calculate_idf( + field_name, StringUtil::string_to_wstring(term_info.get_single_term())); + } + + const auto scorer = ::doris::snii::query::ScorerContext::from_idf(idf_sum); + std::vector<::doris::snii::query::ScoredDoc> scored_docs; + scored_docs.reserve(matches.size()); + for (const auto& match : matches) { + DCHECK(final_candidates.contains(match.docid)); + DCHECK_NE(match.frequency, 0); + uint8_t norm = 0; + RETURN_IF_ERROR(segment_stats.encoded_norm(match.docid, &norm)); + scored_docs.push_back({.docid = match.docid, + .score = scorer.score(match.frequency, norm, collection_avgdl, + ::doris::snii::query::Bm25Params {})}); + } + for (const auto& scored_doc : scored_docs) { + context->collection_similarity->collect(scored_doc.docid, + static_cast(scored_doc.score)); + } + return Status::OK(); +} + +void parse_phrase_slop(std::string* query, InvertedIndexQueryInfo* query_info) { + DCHECK(query != nullptr); + DCHECK(query_info != nullptr); + const auto is_digits = [](std::string_view str) { + return std::all_of(str.begin(), str.end(), [](unsigned char c) { return std::isdigit(c); }); + }; + + const size_t last_space_pos = query->find_last_of(' '); + if (last_space_pos == std::string::npos) { + return; + } + const size_t tilde_pos = last_space_pos + 1; + if (tilde_pos >= query->size() - 1 || (*query)[tilde_pos] != '~') { + return; + } + + const size_t slop_pos = tilde_pos + 1; + std::string_view slop_str(query->data() + slop_pos, query->size() - slop_pos); + if (slop_str.empty()) { + return; + } + + bool ordered = false; + if (slop_str.size() == 1) { + if (!std::isdigit(static_cast(slop_str[0]))) { + return; + } + } else if (slop_str.back() == '+') { + ordered = true; + slop_str.remove_suffix(1); + } + + if (!is_digits(slop_str)) { + return; + } + auto result = std::from_chars(slop_str.begin(), slop_str.end(), query_info->slop); + if (result.ec != std::errc()) { + return; + } + query_info->ordered = ordered; + *query = query->substr(0, last_space_pos); +} + +std::shared_ptr docids_to_bitmap(const std::vector& docids) { + auto result = std::make_shared(); + if (!docids.empty()) { + result->addMany(docids.size(), docids.data()); + } + result->runOptimize(); + return result; +} + +// Runs `compute` under single-flight keyed by `key`: concurrent identical queries collapse to a +// single execution and the followers reuse the leader's bitmap. `compute(out)` fills *out and +// returns its Status; on overall success *result receives the bitmap. See SingleFlight for why +// this matters under a cold cache with parallel scanners hitting the same segment. +template +Status run_query_single_flight( + ::doris::segment_v2::inverted_index::SingleFlight< + std::pair>>& flight, + const std::string& key, std::shared_ptr* result, +#ifdef BE_TEST + SniiIndexReader::SingleFlightFollowerJoinedObserver follower_joined_observer, + void* follower_joined_opaque, + SniiIndexReader::SingleFlightLeaderBeforeComputeObserver leader_before_compute_observer, + void* leader_before_compute_opaque, +#endif + Compute&& compute) { + auto follower = flight.join_or_lead(key); + if (follower.has_value()) { +#ifdef BE_TEST + if (follower_joined_observer != nullptr) { + follower_joined_observer(follower_joined_opaque); + } +#endif + auto [leader_status, leader_bitmap] = follower->get(); + if (leader_status.ok() && leader_bitmap != nullptr) { + *result = std::move(leader_bitmap); + return Status::OK(); + } + // Leader failed; fall through and compute independently (rare error path). + } + const bool is_leader = !follower.has_value(); +#ifdef BE_TEST + if (is_leader && leader_before_compute_observer != nullptr) { + leader_before_compute_observer(leader_before_compute_opaque); + } +#endif + + Status status = Status::OK(); + std::shared_ptr bitmap; + { + // Publish to any waiting followers on every exit path (including errors). + DEFER(if (is_leader) { flight.publish(key, std::make_pair(status, bitmap)); }); + status = compute(&bitmap); + } + RETURN_IF_ERROR(status); + *result = std::move(bitmap); + return Status::OK(); +} + +Status execute_snii_query(const ::doris::snii::reader::LogicalIndexReader& logical_reader, + InvertedIndexQueryType query_type, + const InvertedIndexQueryInfo& query_info, std::string_view search_str, + const std::vector& terms, int32_t max_expansions, + bool collect_phrase_frequency, SniiQueryExecutionResult* result, + ::doris::snii::query::QueryProfile* profile) { + result->bitmap = std::make_shared(); + result->phrase_matches.clear(); + DORIS_CHECK(!collect_phrase_frequency || uses_phrase_frequency_scoring(query_type, query_info)); + RoaringDocIdSink sink(result->bitmap.get()); + std::vector docids; + bool emitted_to_sink = false; + Status status; + switch (query_type) { + case InvertedIndexQueryType::EQUAL_QUERY: + case InvertedIndexQueryType::MATCH_ANY_QUERY: + status = terms.size() == 1 + ? ::doris::snii::query::term_query(logical_reader, terms.front(), &sink) + : ::doris::snii::query::boolean_or(logical_reader, terms, &sink); + emitted_to_sink = true; + break; + case InvertedIndexQueryType::MATCH_ALL_QUERY: + if (terms.size() == 1) { + status = ::doris::snii::query::term_query(logical_reader, terms.front(), &sink); + emitted_to_sink = true; + } else { + status = ::doris::snii::query::boolean_and(logical_reader, terms, &docids); + } + break; + case InvertedIndexQueryType::MATCH_PHRASE_QUERY: + if (terms.size() == 1) { + status = ::doris::snii::query::term_query(logical_reader, terms.front(), &sink); + emitted_to_sink = true; + } else { + status = collect_phrase_frequency + ? ::doris::snii::query::phrase_query_with_frequencies( + logical_reader, terms, &result->phrase_matches, profile, + {.slop = static_cast(query_info.slop), + .ordered = query_info.ordered}) + : ::doris::snii::query::phrase_query( + logical_reader, terms, &docids, profile, + {.slop = static_cast(query_info.slop), + .ordered = query_info.ordered}); + } + break; + case InvertedIndexQueryType::MATCH_PHRASE_PREFIX_QUERY: + if (terms.size() == 1) { + status = ::doris::snii::query::prefix_query(logical_reader, terms.front(), &sink, + max_expansions); + emitted_to_sink = true; + } else { + status = collect_phrase_frequency + ? ::doris::snii::query::phrase_prefix_query_with_frequencies( + logical_reader, terms, &result->phrase_matches, profile, + max_expansions) + : ::doris::snii::query::phrase_prefix_query( + logical_reader, terms, &docids, profile, max_expansions); + } + break; + case InvertedIndexQueryType::MATCH_REGEXP_QUERY: + status = ::doris::snii::query::regexp_query(logical_reader, search_str, &sink, + max_expansions); + emitted_to_sink = true; + break; + case InvertedIndexQueryType::WILDCARD_QUERY: + status = ::doris::snii::query::wildcard_query(logical_reader, search_str, &sink, + max_expansions); + emitted_to_sink = true; + break; + case InvertedIndexQueryType::LESS_THAN_QUERY: + case InvertedIndexQueryType::LESS_EQUAL_QUERY: + case InvertedIndexQueryType::GREATER_THAN_QUERY: + case InvertedIndexQueryType::GREATER_EQUAL_QUERY: + case InvertedIndexQueryType::RANGE_QUERY: + return Status::Error( + "SNII inverted index storage format does not support BKD/range query"); + default: + return Status::Error( + "SNII unsupported inverted index query type {}", query_type_to_string(query_type)); + } + RETURN_IF_ERROR(status); + if (collect_phrase_frequency) { + for (const auto& match : result->phrase_matches) { + result->bitmap->add(match.docid); + } + result->bitmap->runOptimize(); + } else if (emitted_to_sink) { + result->bitmap->runOptimize(); + } else { + result->bitmap = docids_to_bitmap(docids); + } + return Status::OK(); +} + +} // namespace + +Status SniiIndexReader::new_iterator(std::unique_ptr* iterator) { + if (*iterator == nullptr) { + *iterator = InvertedIndexIterator::create_unique(); + } + dynamic_cast(iterator->get()) + ->add_reader(_reader_type, + dynamic_pointer_cast(shared_from_this())); + return Status::OK(); +} + +Status SniiIndexReader::_parse_query_terms( + const IndexQueryContextPtr& context, std::string search_str, + InvertedIndexQueryType query_type, const InvertedIndexAnalyzerCtx* analyzer_ctx, + InvertedIndexQueryInfo* query_info, + std::optional purpose_override) { + DCHECK(query_info != nullptr); + if (query_type == InvertedIndexQueryType::MATCH_REGEXP_QUERY || + query_type == InvertedIndexQueryType::WILDCARD_QUERY) { + query_info->term_infos.emplace_back(search_str, 0); + return Status::OK(); + } + if (query_type == InvertedIndexQueryType::MATCH_PHRASE_QUERY) { + parse_phrase_slop(&search_str, query_info); + } + + const bool actual_similarity = + context->collection_similarity && + IndexReaderHelper::is_need_similarity_score(query_type, &_index_meta); + const auto purpose = purpose_override.value_or(inverted_index::select_analysis_purpose( + query_type, query_info->slop, actual_similarity)); + SCOPED_RAW_TIMER(&context->stats->inverted_index_analyzer_timer); + try { + if (analyzer_ctx != nullptr && !analyzer_ctx->should_tokenize()) { + query_info->term_infos.emplace_back(search_str); + } else { + auto analyzer = analyzer_ctx == nullptr ? nullptr : analyzer_ctx->get_analyzer(purpose); + if (analyzer != nullptr) { + auto reader = inverted_index::InvertedIndexAnalyzer::create_reader( + analyzer_ctx->char_filter_map); + reader->init(search_str.data(), static_cast(search_str.size()), true); + query_info->term_infos = inverted_index::InvertedIndexAnalyzer::get_analyse_result( + reader, analyzer.get()); + } else { + query_info->term_infos = inverted_index::InvertedIndexAnalyzer::get_analyse_result( + search_str, _index_meta.properties(), purpose); + } + } + } catch (const CLuceneError& e) { + return Status::Error( + "SNII analyze query failed: {}", e.what()); + } catch (const Exception& e) { + return Status::Error( + "SNII analyze query failed: {}", e.what()); + } + return Status::OK(); +} + +Status SniiIndexReader::_get_logical_reader( + const IndexQueryContextPtr& context, InvertedIndexCacheHandle* searcher_cache_handle, + std::unique_ptr<::doris::snii::reader::LogicalIndexReader>* uncached_reader, + const ::doris::snii::reader::LogicalIndexReader** logical_reader) { + DCHECK(searcher_cache_handle != nullptr); + DCHECK(uncached_reader != nullptr); + DCHECK(logical_reader != nullptr); + + const bool enable_searcher_cache = + context->runtime_state != nullptr && + context->runtime_state->query_options().enable_inverted_index_searcher_cache; + const auto index_file_key = _index_file_reader->get_index_file_cache_key(&_index_meta); + InvertedIndexSearcherCache::CacheKey searcher_cache_key(index_file_key); + + bool cache_hit = false; + if (enable_searcher_cache) { + SCOPED_RAW_TIMER(&context->stats->inverted_index_lookup_timer); + cache_hit = InvertedIndexSearcherCache::instance()->lookup(searcher_cache_key, + searcher_cache_handle); + } + + if (cache_hit) { + context->stats->inverted_index_searcher_cache_hit++; + *logical_reader = searcher_cache_handle->get_snii_logical_reader(); + if (*logical_reader == nullptr) { + return Status::InternalError("SNII searcher cache entry has no logical reader"); + } + return Status::OK(); + } + + SCOPED_RAW_TIMER(&context->stats->inverted_index_searcher_open_timer); + context->stats->inverted_index_searcher_cache_miss++; +#ifdef BE_TEST + if (_searcher_open_observer != nullptr) { + _searcher_open_observer(_searcher_open_opaque); + } +#endif + RETURN_IF_ERROR( + _index_file_reader->init(config::inverted_index_read_buffer_size, context->io_ctx)); + auto opened_reader = + DORIS_TRY(_index_file_reader->open_snii_index(&_index_meta, context->io_ctx)); + + if (!enable_searcher_cache) { + *logical_reader = opened_reader.get(); + *uncached_reader = std::move(opened_reader); + return Status::OK(); + } + + const size_t reader_size = std::max(opened_reader->memory_usage(), 1); + auto* cache_value = new InvertedIndexSearcherCache::CacheValue( + std::move(opened_reader), reader_size, UnixMillis(), _index_file_reader); + InvertedIndexSearcherCache::instance()->insert(searcher_cache_key, cache_value, + searcher_cache_handle); + *logical_reader = searcher_cache_handle->get_snii_logical_reader(); + if (*logical_reader == nullptr) { + return Status::InternalError("SNII searcher cache insert produced empty logical reader"); + } + return Status::OK(); +} + +Status SniiIndexReader::query(const IndexQueryContextPtr& context, const std::string& column_name, + const Field& query_value, InvertedIndexQueryType query_type, + std::shared_ptr& bit_map, + const InvertedIndexAnalyzerCtx* analyzer_ctx) { + return _query(context, column_name, query_value, query_type, bit_map, nullptr, analyzer_ctx); +} + +Status SniiIndexReader::query_with_null_bitmap( + const IndexQueryContextPtr& context, const std::string& column_name, + const Field& query_value, InvertedIndexQueryType query_type, + std::shared_ptr& bit_map, + InvertedIndexQueryCacheHandle* null_bitmap_cache_handle, + const InvertedIndexAnalyzerCtx* analyzer_ctx) { + DORIS_CHECK(null_bitmap_cache_handle != nullptr); + return _query(context, column_name, query_value, query_type, bit_map, null_bitmap_cache_handle, + analyzer_ctx); +} + +Status SniiIndexReader::_query(const IndexQueryContextPtr& context, const std::string& column_name, + const Field& query_value, InvertedIndexQueryType query_type, + std::shared_ptr& bit_map, + InvertedIndexQueryCacheHandle* null_bitmap_cache_handle, + const InvertedIndexAnalyzerCtx* analyzer_ctx) { + const bool track_requested_null_time = null_bitmap_cache_handle != nullptr; + const int64_t query_ns_before = + track_requested_null_time ? context->stats->inverted_index_query_timer : 0; + int64_t requested_null_ns = 0; + DEFER({ + if (!track_requested_null_time) { + return; + } + const int64_t inclusive_query_ns = + context->stats->inverted_index_query_timer - query_ns_before; + DORIS_CHECK_GE(inclusive_query_ns, 0); + const int64_t exclusive_query_ns = + inclusive_query_ns > requested_null_ns ? inclusive_query_ns - requested_null_ns : 0; + context->stats->inverted_index_query_timer = query_ns_before + exclusive_query_ns; + }); + SCOPED_RAW_TIMER(&context->stats->inverted_index_query_timer); + const std::string search_str = query_value.get(); + const auto finish_query = + [&](const ::doris::snii::reader::LogicalIndexReader* reader) -> Status { + if (null_bitmap_cache_handle == nullptr) { + return Status::OK(); + } + const int64_t null_ns_before = context->stats->inverted_index_query_null_bitmap_timer; + Status status = _read_null_bitmap(context, null_bitmap_cache_handle, reader); + const int64_t null_ns_after = context->stats->inverted_index_query_null_bitmap_timer; + DORIS_CHECK_GE(null_ns_after, null_ns_before); + requested_null_ns += null_ns_after - null_ns_before; + return status; + }; + + if (int ignore_above = + std::stoi(get_parser_ignore_above_value_from_properties(_index_meta.properties())); + _reader_type == InvertedIndexReaderType::STRING_TYPE && search_str.size() > ignore_above) { + return Status::Error( + "query value is too long, evaluate skipped."); + } + + const bool actual_similarity = + context->collection_similarity && + IndexReaderHelper::is_need_similarity_score(query_type, &_index_meta); + const int32_t max_expansions = + context->runtime_state == nullptr + ? 50 + : context->runtime_state->query_options().inverted_index_max_expansions; + InvertedIndexQueryInfo query_info; + std::string plain_analysis_str = search_str; + if (query_type == InvertedIndexQueryType::MATCH_PHRASE_QUERY) { + parse_phrase_slop(&plain_analysis_str, &query_info); + } + const bool common_grams_phrase_shape = + (query_type == InvertedIndexQueryType::MATCH_PHRASE_QUERY && query_info.slop == 0) || + query_type == InvertedIndexQueryType::MATCH_PHRASE_PREFIX_QUERY; + const bool common_grams_query_eligible = common_grams_phrase_shape && !actual_similarity; + const bool raw_pattern_query = query_type == InvertedIndexQueryType::MATCH_REGEXP_QUERY || + query_type == InvertedIndexQueryType::WILDCARD_QUERY; + // Lucene-style CommonGrams: the plan decision is purely local -- the segment's own analyzer + // identity says whether gram terms can exist, and the mutable BE config + // enable_common_grams_query_plan is the only runtime switch (its cache generation bumps on + // every flip, keeping result-cache entries from crossing plan modes). + const auto common_grams_safety = config::common_grams_query_plan_config_snapshot(); + const inverted_index::CommonGramsPlanCostModel common_grams_cost_model { + .position_verify_factor = common_grams_safety.position_verify_factor, + .common_grams_cost_ratio_percent = common_grams_safety.plan_cost_ratio_percent, + .generation = common_grams_safety.cost_model_generation}; + const auto has_common_grams_analyzer = [](const InvertedIndexAnalyzerCtx* ctx) { + return ctx != nullptr && ctx->analyzer_provider != nullptr && + ctx->analyzer_provider->uses_common_grams() && + ctx->has_complete_common_grams_identity(); + }; + const bool safety_requires_plain = !common_grams_safety.enabled; + // The raw cache key cannot prove whether the immutable segment analyzer has CommonGrams until + // its metadata is open. Delay every eligible forced-plain lookup, then restore ordinary cache + // access below only for a segment that cannot contain gram terms. + const bool initial_force_plain = common_grams_query_eligible && safety_requires_plain; + const bool initial_allow_result_cache = !actual_similarity && !initial_force_plain; + const bool defer_result_cache_lookup = !actual_similarity && !initial_allow_result_cache; + const InvertedIndexRawQuerySemantic raw_semantic { + .raw_query_bytes = search_str, + .query_type = query_type, + .slop = query_info.slop, + .ordered = query_info.ordered, + .max_expansions = max_expansions, + .common_grams_cache_generation = common_grams_safety.cache_generation}; + const auto index_file_key = _index_file_reader->get_index_file_cache_key(&_index_meta); + InvertedIndexQueryCache::CacheKey cache_key {index_file_key, column_name, query_type, + raw_semantic.encode()}; + std::string single_flight_key = cache_key.encode(); + auto* cache = InvertedIndexQueryCache::instance(); + InvertedIndexQueryCacheHandle cache_handler; + bool allow_result_cache = initial_allow_result_cache; + if (handle_query_cache(context, cache, cache_key, &cache_handler, bit_map, + allow_result_cache)) { + return finish_query(nullptr); + } + + snii_doris::DorisSniiFileReader::ScopedIOContext io_context_scope(context->io_ctx); + InvertedIndexCacheHandle searcher_cache_handle; + std::unique_ptr<::doris::snii::reader::LogicalIndexReader> uncached_reader; + const ::doris::snii::reader::LogicalIndexReader* logical_reader = nullptr; + RETURN_IF_ERROR(_get_logical_reader(context, &searcher_cache_handle, &uncached_reader, + &logical_reader)); + + std::optional rebuilt_analyzer_context; + const auto* common_grams_metadata = logical_reader->common_grams_metadata(); + if (!raw_pattern_query) { + auto rebuilt_result = inverted_index::maybe_rebuild_segment_analyzer_context( + analyzer_ctx, common_grams_metadata, _index_meta.properties(), + ExecEnv::GetInstance()->index_policy_mgr()); + if (!rebuilt_result.has_value()) { + if (common_grams_query_eligible) { + ++context->stats->snii_stats.common_grams_fallback_base_analyzer_mismatch; + } + return std::move(rebuilt_result.error()); + } + rebuilt_analyzer_context = std::move(*rebuilt_result); + } + const InvertedIndexAnalyzerCtx* effective_analyzer_context = + rebuilt_analyzer_context ? &*rebuilt_analyzer_context : analyzer_ctx; + const bool effective_common_grams_configured = + common_grams_query_eligible && has_common_grams_analyzer(effective_analyzer_context); + const bool segment_may_contain_common_grams = + common_grams_metadata != nullptr && common_grams_metadata->common_grams_coverage != + inverted_index::CommonGramsCoverage::kNone; + const bool force_plain = + common_grams_query_eligible && safety_requires_plain && + (effective_common_grams_configured || segment_may_contain_common_grams); + allow_result_cache = !actual_similarity && !force_plain; + if (defer_result_cache_lookup && allow_result_cache && + handle_query_cache(context, cache, cache_key, &cache_handler, bit_map, + allow_result_cache)) { + return finish_query(logical_reader); + } + InvertedIndexQueryInfo execution_query_info = query_info; + const auto plain_purpose = common_grams_query_eligible + ? std::optional(inverted_index::AnalysisPurpose::kPlainQuery) + : std::nullopt; + RETURN_IF_ERROR(_parse_query_terms(context, plain_analysis_str, query_type, + effective_analyzer_context, &execution_query_info, + plain_purpose)); + if (execution_query_info.term_infos.empty()) { + auto msg = fmt::format("token parser result is empty for SNII query '{}'", search_str); + if (is_match_query(query_type)) { + LOG(WARNING) << msg; + bit_map = std::make_shared(); + insert_query_cache(context, cache, cache_key, bit_map, &cache_handler, + allow_result_cache); + return finish_query(logical_reader); + } + return Status::Error(msg); + } + if (execution_query_info.has_common_gram()) { + return Status::Error( + "CommonGrams term escaped the plain query analyzer"); + } + if (actual_similarity && query_type == InvertedIndexQueryType::MATCH_PHRASE_PREFIX_QUERY && + execution_query_info.term_infos.size() == 1) { + return Status::Error( + "SNII scoring does not support a single-token phrase-prefix query"); + } + std::vector terms = to_terms(execution_query_info); + + // G02 count-only fast path: the SegmentIterator asserted (via the context + // flag) that only the match COUNT of this predicate matters, so eligible + // shapes are answered from dict-entry df without decoding postings. Placed + // AFTER the query-cache lookup (a cached row-accurate bitmap is free and + // counts correctly) and BEFORE single-flight; the fabricated [0, df) bitmap + // is returned early and NEVER inserted into the query cache or published to + // single-flight followers -- both are keyed identically to row-accurate + // queries and must only ever serve real row ids. + if (context->count_on_index_fastpath) { + bool count_handled = false; + std::shared_ptr count_bitmap; + RETURN_IF_ERROR(_try_count_only_fastpath(context, query_type, execution_query_info, terms, + &count_handled, &count_bitmap, logical_reader)); + if (count_handled) { + bit_map = std::move(count_bitmap); + RETURN_IF_ERROR(finish_query(logical_reader)); + // G03 reply: tell the SegmentIterator the bitmap is count-shaped + // (cardinality exact, row ids fabricated) so it may short-circuit + // row emission. Deliberately NOT set on the cache-hit return above + // or on the decode path below -- those bitmaps are row-accurate + // and keep today's emission. + context->count_on_index_fastpath_hit = true; + return Status::OK(); + } + } + + // Under a cold cache, parallel scanners _lazy_init the same segment concurrently and each + // would otherwise miss the searcher/query caches and redundantly open + decode this segment's + // index. Collapse identical concurrent queries into one shared execution (see SingleFlight). + static ::doris::segment_v2::inverted_index::SingleFlight< + std::pair>> + query_single_flight; + std::shared_ptr result_bitmap; + std::vector<::doris::snii::query::PhraseMatch> phrase_matches; + auto* phrase_matches_out = + actual_similarity && uses_phrase_frequency_scoring(query_type, execution_query_info) + ? &phrase_matches + : nullptr; + Status single_flight_status; + if (!allow_result_cache) { + single_flight_status = + _compute_query_bitmap(context, + {.query_type = query_type, + .query_info = execution_query_info, + .search_str = search_str, + .max_expansions = max_expansions, + .common_grams_query_shape = common_grams_query_eligible, + .force_plain = force_plain, + .common_grams_cost_model = common_grams_cost_model, + .analyzer_ctx = effective_analyzer_context, + .physical_raw_query_key = single_flight_key, + .logical_reader = logical_reader}, + &terms, &result_bitmap, phrase_matches_out); + } else { + DORIS_CHECK(phrase_matches_out == nullptr); + single_flight_status = run_query_single_flight( + query_single_flight, single_flight_key, &result_bitmap, +#ifdef BE_TEST + _single_flight_follower_joined_observer, _single_flight_follower_joined_opaque, + _single_flight_leader_before_compute_observer, + _single_flight_leader_before_compute_opaque, +#endif + [&](std::shared_ptr* out) { + auto status = _compute_query_bitmap( + context, + {.query_type = query_type, + .query_info = execution_query_info, + .search_str = search_str, + .max_expansions = max_expansions, + .common_grams_query_shape = common_grams_query_eligible, + .force_plain = force_plain, + .common_grams_cost_model = common_grams_cost_model, + .analyzer_ctx = effective_analyzer_context, + .physical_raw_query_key = single_flight_key, + .logical_reader = logical_reader}, + &terms, out, nullptr); + if (status.ok()) { + insert_query_cache(context, cache, cache_key, *out, &cache_handler, + allow_result_cache); + } + return status; + }); + } + RETURN_IF_ERROR(single_flight_status); + DORIS_CHECK(result_bitmap != nullptr); + if (actual_similarity && !result_bitmap->isEmpty()) { + ::doris::snii::stats::SniiStatsProvider segment_stats; + RETURN_IF_ERROR( + ::doris::snii::stats::SniiStatsProvider::open(logical_reader, &segment_stats)); + if (phrase_matches_out != nullptr) { + RETURN_IF_ERROR(score_phrase_matches(context, column_name, query_type, + execution_query_info, *logical_reader, + segment_stats, *result_bitmap, phrase_matches)); + } else if (uses_plain_term_frequency_scoring(query_type, execution_query_info)) { + RETURN_IF_ERROR(score_plain_term_candidates(context, column_name, execution_query_info, + *logical_reader, segment_stats, + *result_bitmap)); + } + } + bit_map = result_bitmap; + return finish_query(logical_reader); +} + +Status SniiIndexReader::_compute_query_bitmap( + const IndexQueryContextPtr& context, const SniiQueryBitmapRequest& request, + std::vector* preanalyzed_terms, std::shared_ptr* out, + std::vector<::doris::snii::query::PhraseMatch>* phrase_matches) { + // Bound once so the body below reads the same as before the request object was introduced; + // renaming 71 uses would have buried the actual change. + const InvertedIndexQueryType query_type = request.query_type; + const InvertedIndexQueryInfo& request_query_info = request.query_info; + const bool common_grams_query_shape = request.common_grams_query_shape; + const bool force_plain = request.force_plain; + const inverted_index::CommonGramsPlanCostModel common_grams_cost_model = + request.common_grams_cost_model; + const InvertedIndexAnalyzerCtx* analyzer_ctx = request.analyzer_ctx; + const std::string_view physical_raw_query_key = request.physical_raw_query_key; + const std::string_view search_str = request.search_str; + const int32_t max_expansions = request.max_expansions; + const ::doris::snii::reader::LogicalIndexReader* logical_reader = request.logical_reader; + + DORIS_CHECK(preanalyzed_terms != nullptr); + DORIS_CHECK(logical_reader != nullptr); + DORIS_CHECK(request_query_info.term_infos.size() == preanalyzed_terms->size()); + if (phrase_matches != nullptr) { + phrase_matches->clear(); + } + const auto* common_grams_metadata = logical_reader->common_grams_metadata(); + InvertedIndexQueryInfo query_info = request_query_info; + std::vector routed_terms = *preanalyzed_terms; + auto* terms = &routed_terms; + + const auto* common_grams_identity = + analyzer_ctx == nullptr ? nullptr : analyzer_ctx->get_common_grams_identity(); + const bool common_grams_configured = common_grams_query_shape && analyzer_ctx != nullptr && + analyzer_ctx->analyzer_provider != nullptr && + analyzer_ctx->analyzer_provider->uses_common_grams() && + analyzer_ctx->has_complete_common_grams_identity(); + const bool common_grams_candidate = + common_grams_configured && query_info.term_infos.size() >= 2 && !force_plain; + const bool common_grams_forced_plain = + common_grams_configured && query_info.term_infos.size() >= 2 && force_plain; + enum class CommonGramsPlainFallback : uint8_t { kNone, kNoGram, kIncompatible, kKillSwitch }; + CommonGramsPlainFallback common_grams_plain_fallback = + common_grams_forced_plain ? CommonGramsPlainFallback::kKillSwitch + : CommonGramsPlainFallback::kNone; + const bool common_grams_compatible = + common_grams_metadata != nullptr && common_grams_identity != nullptr && + (logical_reader->common_grams_posting_policy() == + ::doris::snii::format::CommonGramsPostingPolicy::kHybridV1 + ? inverted_index::is_common_grams_query_compatible( + *common_grams_metadata, *common_grams_identity, + inverted_index::CommonGramsCoverage::kMixed) + : inverted_index::is_common_grams_query_compatible(*common_grams_metadata, + *common_grams_identity)); + const auto* common_grams_word_set = + common_grams_configured ? analyzer_ctx->analyzer_provider->common_grams_word_set() + : nullptr; + const auto common_grams_query_mode = + query_type == InvertedIndexQueryType::MATCH_PHRASE_QUERY + ? inverted_index::CommonGramsQueryMode::kExact + : inverted_index::CommonGramsQueryMode::kPhrasePrefix; + const bool proven_no_common_gram = + common_grams_candidate && common_grams_compatible && common_grams_word_set != nullptr && + !inverted_index::common_grams_query_may_use_gram( + *preanalyzed_terms, common_grams_query_mode, *common_grams_word_set); + if (proven_no_common_gram && query_type == InvertedIndexQueryType::MATCH_PHRASE_QUERY) { + DORIS_CHECK(phrase_matches == nullptr); + ::doris::snii::SniiPrxExecutionProfileScope execution_profile(*context->stats); + InvertedIndexQueryInfo empty_gram_query_info; + std::vector docids; + RETURN_IF_ERROR(::doris::snii::query::planned_exact_phrase_query( + *logical_reader, query_info, empty_gram_query_info, common_grams_identity, &docids, + execution_profile.profile(), nullptr, common_grams_cost_model, + ::doris::snii::query::CommonGramsPlanDebugOverride::kNone)); + *out = docids_to_bitmap(docids); + return Status::OK(); + } + if (proven_no_common_gram) { + common_grams_plain_fallback = CommonGramsPlainFallback::kNoGram; + } else if (common_grams_candidate && common_grams_compatible) { + DORIS_CHECK(phrase_matches == nullptr); + DORIS_CHECK(!physical_raw_query_key.empty()); + const auto debug_override = ::doris::snii::query::common_grams_plan_debug_override(); + ::doris::snii::SniiPrxExecutionProfileScope execution_profile(*context->stats); + + // The gram-side analysis always runs. Without a memoized plan choice nothing can tell us + // in advance that the plain plan wins, and one analyzer pass over a phrase string is noise + // next to the posting decode that follows it. + InvertedIndexQueryInfo gram_query_info; + const auto gram_purpose = query_type == InvertedIndexQueryType::MATCH_PHRASE_QUERY + ? inverted_index::AnalysisPurpose::kExactPhraseQuery + : inverted_index::AnalysisPurpose::kPhrasePrefixQuery; + RETURN_IF_ERROR(_parse_query_terms(context, std::string(search_str), query_type, + analyzer_ctx, &gram_query_info, gram_purpose)); + + std::vector docids; + if (query_type == InvertedIndexQueryType::MATCH_PHRASE_QUERY) { + RETURN_IF_ERROR(::doris::snii::query::planned_exact_phrase_query( + *logical_reader, query_info, gram_query_info, common_grams_identity, &docids, + execution_profile.profile(), nullptr, common_grams_cost_model, debug_override)); + } else { + DORIS_CHECK(query_type == InvertedIndexQueryType::MATCH_PHRASE_PREFIX_QUERY); + RETURN_IF_ERROR(::doris::snii::query::planned_phrase_prefix_query( + *logical_reader, query_info, gram_query_info, common_grams_identity, &docids, + execution_profile.profile(), max_expansions, nullptr, common_grams_cost_model, + debug_override)); + } + *out = docids_to_bitmap(docids); + return Status::OK(); + } else if (common_grams_candidate) { + common_grams_plain_fallback = CommonGramsPlainFallback::kIncompatible; + } + switch (query_type) { + case InvertedIndexQueryType::EQUAL_QUERY: + case InvertedIndexQueryType::MATCH_ANY_QUERY: + case InvertedIndexQueryType::MATCH_ALL_QUERY: + case InvertedIndexQueryType::MATCH_PHRASE_QUERY: { + bool all_representable = false; + RETURN_IF_ERROR(::doris::snii::query::internal::route_query_terms( + *logical_reader, query_info, terms, &all_representable)); + if (terms->empty() && (query_type == InvertedIndexQueryType::EQUAL_QUERY || + query_type == InvertedIndexQueryType::MATCH_ANY_QUERY)) { + *out = std::make_shared(); + return Status::OK(); + } + if (!all_representable && (query_type == InvertedIndexQueryType::MATCH_ALL_QUERY || + query_type == InvertedIndexQueryType::MATCH_PHRASE_QUERY)) { + *out = std::make_shared(); + return Status::OK(); + } + break; + } + default: + break; + } + SniiQueryExecutionResult query_result; + const bool phrase_can_decode_prx = query_type == InvertedIndexQueryType::MATCH_PHRASE_QUERY; + const bool needs_prx_profile = + terms->size() > 1 && (phrase_can_decode_prx || + query_type == InvertedIndexQueryType::MATCH_PHRASE_PREFIX_QUERY); + if (needs_prx_profile) { + ::doris::snii::SniiPrxExecutionProfileScope execution_profile(*context->stats); + const Status execution_status = execute_snii_query( + *logical_reader, query_type, query_info, search_str, *terms, max_expansions, + phrase_matches != nullptr, &query_result, execution_profile.profile()); + if (common_grams_plain_fallback != CommonGramsPlainFallback::kNone) { + auto& plan_stats = execution_profile.profile()->phrase_query_stats; + ++plan_stats.common_grams_candidate_queries; + ++plan_stats.common_grams_plain_plans; + switch (common_grams_plain_fallback) { + case CommonGramsPlainFallback::kNoGram: + ++plan_stats.common_grams_fallback_no_gram; + break; + case CommonGramsPlainFallback::kIncompatible: + ++plan_stats.common_grams_fallback_incompatible; + break; + case CommonGramsPlainFallback::kKillSwitch: + ++plan_stats.common_grams_fallback_kill_switch; + break; + case CommonGramsPlainFallback::kNone: + break; + } + } + RETURN_IF_ERROR(execution_status); + } else { + RETURN_IF_ERROR(execute_snii_query(*logical_reader, query_type, query_info, search_str, + *terms, max_expansions, phrase_matches != nullptr, + &query_result, nullptr)); + } + *out = std::move(query_result.bitmap); + if (phrase_matches != nullptr) { + *phrase_matches = std::move(query_result.phrase_matches); + } + return Status::OK(); +} + +#ifdef BE_TEST +Status SniiIndexReader::_compute_query_bitmap(const IndexQueryContextPtr& context, + InvertedIndexQueryType query_type, + const InvertedIndexQueryInfo& query_info, + std::string_view search_str, + std::vector* terms, + int32_t max_expansions, + std::shared_ptr* out) { + snii_doris::DorisSniiFileReader::ScopedIOContext io_context_scope(context->io_ctx); + InvertedIndexCacheHandle searcher_cache_handle; + std::unique_ptr<::doris::snii::reader::LogicalIndexReader> uncached_reader; + const ::doris::snii::reader::LogicalIndexReader* logical_reader = nullptr; + RETURN_IF_ERROR(_get_logical_reader(context, &searcher_cache_handle, &uncached_reader, + &logical_reader)); + return _compute_query_bitmap(context, + {.query_type = query_type, + .query_info = query_info, + .search_str = search_str, + .max_expansions = max_expansions, + .logical_reader = logical_reader}, + terms, out, nullptr); +} +#endif + +Status SniiIndexReader::_try_count_only_fastpath( + const IndexQueryContextPtr& context, InvertedIndexQueryType query_type, + const InvertedIndexQueryInfo& query_info, const std::vector& terms, + bool* handled, std::shared_ptr* out, + const ::doris::snii::reader::LogicalIndexReader* preopened_reader) { + *handled = false; + // Shape guard: only exact-term query types. Prefix/regexp/wildcard/ + // phrase-prefix expand the term set, so no single dict entry carries the + // count; range types never reach SNII anyway. + switch (query_type) { + case InvertedIndexQueryType::EQUAL_QUERY: + case InvertedIndexQueryType::MATCH_ANY_QUERY: + case InvertedIndexQueryType::MATCH_ALL_QUERY: + case InvertedIndexQueryType::MATCH_PHRASE_QUERY: + break; + default: + return Status::OK(); + } + if (terms.size() != 1) { + // Multi-term MATCH_ANY (OR) / MATCH_ALL (AND) counts are not derivable + // from per-term dfs (overlap unknown), and phrases require positional + // verification, so execute the normal query path. + return Status::OK(); + } + + snii_doris::DorisSniiFileReader::ScopedIOContext io_context_scope(context->io_ctx); + InvertedIndexCacheHandle searcher_cache_handle; + std::unique_ptr<::doris::snii::reader::LogicalIndexReader> uncached_reader; + const ::doris::snii::reader::LogicalIndexReader* logical_reader = preopened_reader; + if (logical_reader == nullptr) { + RETURN_IF_ERROR(_get_logical_reader(context, &searcher_cache_handle, &uncached_reader, + &logical_reader)); + } + + std::string physical_term_scratch; + std::string_view physical_term; + bool representable = false; + DORIS_CHECK(query_info.term_infos.size() == 1); + RETURN_IF_ERROR(::doris::snii::query::internal::route_query_term_view( + *logical_reader, query_info.term_infos.front(), &physical_term_scratch, &physical_term, + &representable)); + uint64_t count = 0; + if (representable) { + RETURN_IF_ERROR( + ::doris::snii::query::count_only_term_df(*logical_reader, physical_term, &count)); + } + + // Null handling. df is the exact match count REGARDLESS of nulls: the + // writer adds no tokens for a null doc (scalar add_nulls; a NULL array row + // is an empty range), so postings -- and therefore df -- never include + // null rows, exactly matching MATCH's "null never matches" semantics. The + // fabricated bitmap however flows through FunctionMatchBase -> + // InvertedIndexResultBitmap::mask_out_null, which subtracts the segment's + // REAL null bitmap from it; a dense [0, df) range colliding with null row + // ids would be shrunk below df. So on a segment WITH a null bitmap, load + // it (query-cache backed, the same read the normal MATCH path performs) + // and fabricate df ids DISJOINT from it, making that subtraction a + // provable no-op. Segments without a null section (the writer omits it + // when no row is null) keep the trivial [0, df) range. + auto result = std::make_shared(); + if (count > 0 && logical_reader->section_refs().null_bitmap.length > 0) { + InvertedIndexQueryCacheHandle null_bitmap_cache_handle; + RETURN_IF_ERROR(_read_null_bitmap(context, &null_bitmap_cache_handle, logical_reader)); + std::shared_ptr nulls = null_bitmap_cache_handle.get_bitmap(); + // Fall through on a missing bitmap behind the cache handle or a + // fabrication failure (df + null count breaching the docid domain): + // both mean a corrupt index, and the row-accurate decode -- which + // intersects real ids -- must own the answer rather than a blind + // fabrication. The count_fastpath_hits test seam already counted the + // dict lookup above; production correctness is unaffected. + if (nulls == nullptr) { + return Status::OK(); + } + if (!nulls->isEmpty()) { + if (!::doris::snii::query::fabricate_null_disjoint_count_bitmap(count, *nulls, + result.get()) + .ok()) { + return Status::OK(); + } + } else { + result->addRange(0, count); + } + } else if (count > 0) { + result->addRange(0, count); + } + *out = std::move(result); + *handled = true; + return Status::OK(); +} + +Status SniiIndexReader::try_query(const IndexQueryContextPtr& /*context*/, + const std::string& /*column_name*/, const Field& /*query_value*/, + InvertedIndexQueryType /*query_type*/, size_t* /*count*/) { + return Status::Error("SNII does not support try_query"); +} + +Status SniiIndexReader::read_null_bitmap(const IndexQueryContextPtr& context, + InvertedIndexQueryCacheHandle* cache_handle, + lucene::store::Directory* /*dir*/) { + return _read_null_bitmap(context, cache_handle, nullptr); +} + +Status SniiIndexReader::_read_null_bitmap( + const IndexQueryContextPtr& context, InvertedIndexQueryCacheHandle* cache_handle, + const ::doris::snii::reader::LogicalIndexReader* preopened_reader) { + SCOPED_RAW_TIMER(&context->stats->inverted_index_query_null_bitmap_timer); + auto index_file_key = _index_file_reader->get_index_file_cache_key(&_index_meta); + InvertedIndexQueryCache::CacheKey cache_key { + index_file_key, "", InvertedIndexQueryType::UNKNOWN_QUERY, "null_bitmap"}; + auto* cache = InvertedIndexQueryCache::instance(); + if (cache->lookup(cache_key, cache_handle)) { + return Status::OK(); + } + + snii_doris::DorisSniiFileReader::ScopedIOContext io_context_scope(context->io_ctx); + InvertedIndexCacheHandle searcher_cache_handle; + std::unique_ptr<::doris::snii::reader::LogicalIndexReader> uncached_reader; + const ::doris::snii::reader::LogicalIndexReader* logical_reader = preopened_reader; + if (logical_reader == nullptr) { + RETURN_IF_ERROR(_get_logical_reader(context, &searcher_cache_handle, &uncached_reader, + &logical_reader)); + } + auto null_bitmap = std::make_shared(); + const auto& ref = logical_reader->section_refs().null_bitmap; + if (ref.length > 0) { + std::vector bytes; + RETURN_IF_ERROR(logical_reader->reader()->read_at(ref.offset, ref.length, &bytes)); + ::doris::snii::format::NullBitmapReader reader; + RETURN_IF_ERROR(::doris::snii::format::NullBitmapReader::open(::doris::snii::Slice(bytes), + &reader)); + reader.copy_to(null_bitmap.get()); + null_bitmap->runOptimize(); + } + cache->insert(cache_key, null_bitmap, cache_handle); + return Status::OK(); +} + +} // namespace doris::segment_v2 diff --git a/be/src/storage/index/snii/snii_index_reader.h b/be/src/storage/index/snii/snii_index_reader.h new file mode 100644 index 00000000000000..aef3d12d904e68 --- /dev/null +++ b/be/src/storage/index/snii/snii_index_reader.h @@ -0,0 +1,180 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "storage/index/inverted/common_grams/common_grams_query_cost.h" +#include "storage/index/inverted/inverted_index_query_type.h" +#include "storage/index/inverted/inverted_index_reader.h" + +namespace doris::snii::reader { +class LogicalIndexReader; +} // namespace doris::snii::reader + +namespace doris::snii::query { +struct PhraseMatch; +} // namespace doris::snii::query + +namespace doris::segment_v2 { + +// One query plus the plan the caller chose for it. This is a parameter object rather than a +// parameter list because _compute_query_bitmap() took fourteen positional arguments, two of them +// adjacent bools (common_grams_query_shape, force_plain) that no call site could tell apart +// without counting commas. +struct SniiQueryBitmapRequest { + InvertedIndexQueryType query_type; + const InvertedIndexQueryInfo& query_info; + std::string_view search_str; + int32_t max_expansions = 0; + + // Plan decisions the caller has already made. Both bools are false on the plain path. + bool common_grams_query_shape = false; + bool force_plain = false; + inverted_index::CommonGramsPlanCostModel common_grams_cost_model {}; + const InvertedIndexAnalyzerCtx* analyzer_ctx = nullptr; + // Identifies the physical query for the single-flight key; empty when unused. + std::string_view physical_raw_query_key {}; + + const ::doris::snii::reader::LogicalIndexReader* logical_reader = nullptr; +}; + +class SniiIndexReader final : public InvertedIndexReader { + ENABLE_FACTORY_CREATOR(SniiIndexReader); + +public: +#ifdef BE_TEST + using SingleFlightFollowerJoinedObserver = void (*)(void*) noexcept; + using SingleFlightLeaderBeforeComputeObserver = void (*)(void*) noexcept; + using SearcherOpenObserver = void (*)(void*) noexcept; +#endif + + SniiIndexReader(const TabletIndex* index_meta, + const std::shared_ptr& index_file_reader, + InvertedIndexReaderType reader_type) + : InvertedIndexReader(index_meta, index_file_reader), _reader_type(reader_type) {} + + Status new_iterator(std::unique_ptr* iterator) override; + Status query(const IndexQueryContextPtr& context, const std::string& column_name, + const Field& query_value, InvertedIndexQueryType query_type, + std::shared_ptr& bit_map, + const InvertedIndexAnalyzerCtx* analyzer_ctx = nullptr) override; + Status query_with_null_bitmap(const IndexQueryContextPtr& context, + const std::string& column_name, const Field& query_value, + InvertedIndexQueryType query_type, + std::shared_ptr& bit_map, + InvertedIndexQueryCacheHandle* null_bitmap_cache_handle, + const InvertedIndexAnalyzerCtx* analyzer_ctx = nullptr) override; + Status try_query(const IndexQueryContextPtr& context, const std::string& column_name, + const Field& query_value, InvertedIndexQueryType query_type, + size_t* count) override; + Status read_null_bitmap(const IndexQueryContextPtr& context, + InvertedIndexQueryCacheHandle* cache_handle, + lucene::store::Directory* dir = nullptr) override; + InvertedIndexReaderType type() override { return _reader_type; } + +#ifdef BE_TEST + void set_single_flight_follower_joined_observer_for_test( + SingleFlightFollowerJoinedObserver observer, void* opaque) { + _single_flight_follower_joined_observer = observer; + _single_flight_follower_joined_opaque = opaque; + } + void set_single_flight_leader_before_compute_observer_for_test( + SingleFlightLeaderBeforeComputeObserver observer, void* opaque) { + _single_flight_leader_before_compute_observer = observer; + _single_flight_leader_before_compute_opaque = opaque; + } + void set_searcher_open_observer_for_test(SearcherOpenObserver observer, void* opaque) { + _searcher_open_observer = observer; + _searcher_open_opaque = opaque; + } +#endif + +private: + Status _query(const IndexQueryContextPtr& context, const std::string& column_name, + const Field& query_value, InvertedIndexQueryType query_type, + std::shared_ptr& bit_map, + InvertedIndexQueryCacheHandle* null_bitmap_cache_handle, + const InvertedIndexAnalyzerCtx* analyzer_ctx); + Status _parse_query_terms( + const IndexQueryContextPtr& context, std::string search_str, + InvertedIndexQueryType query_type, const InvertedIndexAnalyzerCtx* analyzer_ctx, + InvertedIndexQueryInfo* query_info, + std::optional purpose_override = std::nullopt); + Status _get_logical_reader( + const IndexQueryContextPtr& context, InvertedIndexCacheHandle* searcher_cache_handle, + std::unique_ptr<::doris::snii::reader::LogicalIndexReader>* uncached_reader, + const ::doris::snii::reader::LogicalIndexReader** logical_reader); + Status _read_null_bitmap(const IndexQueryContextPtr& context, + InvertedIndexQueryCacheHandle* cache_handle, + const ::doris::snii::reader::LogicalIndexReader* preopened_reader); + // Opens the segment index and runs the query, producing the result bitmap. Invoked as the + // single-flight "compute" step by query(); see SingleFlight for the concurrency rationale. + Status _compute_query_bitmap(const IndexQueryContextPtr& context, + const SniiQueryBitmapRequest& request, + std::vector* terms, + std::shared_ptr* out, + std::vector<::doris::snii::query::PhraseMatch>* phrase_matches); +#ifdef BE_TEST + Status _compute_query_bitmap(const IndexQueryContextPtr& context, + InvertedIndexQueryType query_type, + const InvertedIndexQueryInfo& query_info, + std::string_view search_str, std::vector* terms, + int32_t max_expansions, std::shared_ptr* out); +#endif + // G02 count-only fast path. Only called when the caller (SegmentIterator) + // set context->count_on_index_fastpath, i.e. the match count alone decides + // the scan result. On *handled = true, *out is a bitmap of cardinality df + // (row ids NOT real) built from a single exact term's dict-entry df WITHOUT + // decoding postings. On a segment without a null bitmap the fabricated ids + // are the dense range [0, df); on a segment WITH one they are the first df + // NON-NULL row ids (see + // fabricate_null_disjoint_count_bitmap) so that the unconditional + // FunctionMatchBase -> mask_out_null subtraction of the real null bitmap + // is a no-op and the cardinality stays df -- which is already the exact + // match count, because postings never contain null docs. Falls through + // (*handled = false) for every other shape: every multi-term query + // (including phrase and OR/AND), prefix/regexp/wildcard/phrase-prefix + // expansion. Multi-term sloppy phrases fall through with every other + // multi-term shape; a single-term phrase remains exactly one posting df. + // On *handled = true, query() also raises + // context->count_on_index_fastpath_hit (G03) so the SegmentIterator may + // short-circuit row emission for the count-shaped bitmap. + Status _try_count_only_fastpath( + const IndexQueryContextPtr& context, InvertedIndexQueryType query_type, + const InvertedIndexQueryInfo& query_info, const std::vector& terms, + bool* handled, std::shared_ptr* out, + const ::doris::snii::reader::LogicalIndexReader* preopened_reader = nullptr); + + InvertedIndexReaderType _reader_type; +#ifdef BE_TEST + SingleFlightFollowerJoinedObserver _single_flight_follower_joined_observer = nullptr; + void* _single_flight_follower_joined_opaque = nullptr; + SingleFlightLeaderBeforeComputeObserver _single_flight_leader_before_compute_observer = nullptr; + void* _single_flight_leader_before_compute_opaque = nullptr; + SearcherOpenObserver _searcher_open_observer = nullptr; + void* _searcher_open_opaque = nullptr; +#endif +}; + +} // namespace doris::segment_v2 diff --git a/be/src/storage/index/snii/snii_index_writer.cpp b/be/src/storage/index/snii/snii_index_writer.cpp new file mode 100644 index 00000000000000..d730ed7dd457f0 --- /dev/null +++ b/be/src/storage/index/snii/snii_index_writer.cpp @@ -0,0 +1,508 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/snii_index_writer.h" + +#include + +#include +#include +#include +#include + +#include "common/cast_set.h" +#include "common/config.h" +#include "common/logging.h" +#include "storage/index/index_file_writer.h" +#include "storage/index/inverted/analyzer/analyzer.h" +#include "storage/index/inverted/common_grams/common_grams_key_codec.h" +#include "storage/index/inverted/query/query_info.h" +#include "storage/index/inverted/token_filter/common_grams_filter.h" +#include "storage/index/snii/query/bm25_scorer.h" +#include "storage/index/snii/writer/global_memory_limiter.h" +#include "storage/tablet/tablet_schema.h" + +namespace doris::segment_v2 { +namespace { + +Status validate_common_grams_metadata_seed( + const inverted_index::CommonGramsSegmentMetadata& metadata) { + using namespace inverted_index; + auto status = validate_common_grams_segment_metadata(metadata); + if (!status.ok() || metadata.plain_term_key_version != PlainTermKeyVersion::kEscapedV1 || + metadata.common_grams_coverage != CommonGramsCoverage::kComplete || + metadata.common_grams_semantics_version != COMMON_GRAMS_SEMANTICS_VERSION_V1 || + metadata.common_grams_key_version != COMMON_GRAMS_KEY_VERSION_V1 || + metadata.scoring_coverage != ScoringCoverage::kComplete || + metadata.scoring_stats_version != COMMON_GRAMS_SCORING_STATS_VERSION_V1 || + metadata.norm_semantics_version != COMMON_GRAMS_NORM_SEMANTICS_VERSION_V1) { + return Status::Error( + "SNII CommonGrams metadata identity seed is incomplete or incompatible"); + } + return Status::OK(); +} + +} // namespace + +SniiIndexColumnWriter::SniiIndexColumnWriter( + IndexFileWriter* index_file_writer, const TabletIndex* index_meta, FieldType value_type, + std::optional common_grams_metadata_seed) + : _index_file_writer(index_file_writer), + _index_meta(index_meta), + _is_char(value_type == FieldType::OLAP_FIELD_TYPE_CHAR), + _common_grams_build_enabled(config::enable_common_grams_index_build), + _common_grams_metadata_seed(std::move(common_grams_metadata_seed)) {} + +Status SniiIndexColumnWriter::init() { + _should_analyzer = + inverted_index::InvertedIndexAnalyzer::should_analyzer(_index_meta->properties()); + _has_positions = get_parser_phrase_support_string_from_properties(_index_meta->properties()) == + INVERTED_INDEX_PARSER_PHRASE_SUPPORT_YES; + _config = _has_positions ? ::doris::snii::format::IndexConfig::kDocsPositions + : ::doris::snii::format::IndexConfig::kDocsOnly; + auto ignore_above_value = + get_parser_ignore_above_value_from_properties(_index_meta->properties()); + _ignore_above = cast_set(std::stoul(ignore_above_value)); + const auto spill_threshold = + static_cast(config::inverted_index_ram_buffer_size * 1024 * 1024); + _memory_reporter = std::make_unique<::doris::snii::writer::MemoryReporter>( + nullptr, spill_threshold, + ::doris::snii::writer::MemoryReporter::CapPolicy::kSpillThreshold); + _term_buffer = std::make_unique<::doris::snii::writer::SpimiTermBuffer>( + _has_positions, spill_threshold, _memory_reporter.get()); + // G09: join the PROCESS-WIDE build-RAM limiter. The per-writer spill threshold above + // bounds one writer; a load keeps (tablets x concurrency) writers alive at + // once, none of which may ever reach it -- the global registry bounds their + // SUM by asking the largest buffers to spill early (advisory flags honored + // on each writer's own thread; byte-identical output). Budget refreshed + // from the mutable config at every writer init; 0 disables (no + // registration, zero per-token overhead beyond the G08 path). + // G09 anti-storm knobs (see the config comments): the forced-spill floor + // gates both the owner-side honor (a request is a pending no-op until the + // reclaimable arena regrows past it) and the limiter's victim eligibility, + // and the run-file cap merge-compacts a writer's spill runs so the final + // k-way merge's fd fan-in stays bounded. Applied unconditionally -- the + // floor also protects test-seam requests, and the cap also bounds + // per-writer gate-2 runs when the global limiter is off. + _term_buffer->set_forced_spill_min_arena_bytes( + static_cast(std::max(config::snii_forced_spill_min_arena_bytes, 0))); + _term_buffer->set_max_run_files( + static_cast(std::max(config::snii_spill_max_run_files_per_buffer, 0))); + const int64_t global_budget = config::snii_index_writer_global_memory_bytes; + if (global_budget > 0) { + auto* global_limiter = ::doris::snii::writer::GlobalMemoryLimiter::instance(); + global_limiter->set_budget_bytes(global_budget); + global_limiter->set_min_victim_arena_bytes(config::snii_forced_spill_min_arena_bytes); + _term_buffer->attach_global_limiter(global_limiter); + } + _analyzer_config.analyzer_name = get_analyzer_name_from_properties(_index_meta->properties()); + _analyzer_config.parser_type = get_inverted_index_parser_type_from_string( + get_parser_string_from_properties(_index_meta->properties())); + _analyzer_config.parser_mode = + get_parser_mode_string_from_properties(_index_meta->properties()); + _analyzer_config.char_filter_map = + get_parser_char_filter_map_from_properties(_index_meta->properties()); + _analyzer_config.lower_case = + get_parser_lowercase_from_properties(_index_meta->properties()); + _analyzer_config.stop_words = get_parser_stopwords_from_properties(_index_meta->properties()); + try { + _char_string_reader = inverted_index::InvertedIndexAnalyzer::create_reader( + _analyzer_config.char_filter_map); + if (_should_analyzer) { + auto analyzer_provider = + inverted_index::InvertedIndexAnalyzer::create_analyzer_provider( + &_analyzer_config); + const bool policy_uses_common_grams = analyzer_provider->uses_common_grams(); + _uses_common_grams = _common_grams_build_enabled && policy_uses_common_grams; + if (policy_uses_common_grams && !_uses_common_grams) { + _common_grams_metadata_seed.reset(); + } + if (_uses_common_grams) { + if (!_has_positions) { + close_on_error(); + return Status::Error( + "SNII CommonGrams requires phrase positions"); + } + const auto base_analyzer_fingerprint = + analyzer_provider->base_analyzer_fingerprint(); + if (base_analyzer_fingerprint.empty()) { + close_on_error(); + return Status::Error( + "SNII CommonGrams analyzer has no immutable base fingerprint"); + } + const auto* identity = analyzer_provider->common_grams_identity(); + if (identity == nullptr) { + close_on_error(); + return Status::Error( + "SNII CommonGrams analyzer has no immutable dictionary identity"); + } + if (_common_grams_metadata_seed.has_value()) { + if (!inverted_index::common_grams_identity_matches(*_common_grams_metadata_seed, + *identity)) { + close_on_error(); + return Status::Error( + "SNII CommonGrams metadata seed does not match the analyzer " + "identity"); + } + } else { + _common_grams_metadata_seed = + inverted_index::make_common_grams_segment_metadata(*identity); + } + auto status = validate_common_grams_metadata_seed(*_common_grams_metadata_seed); + if (!status.ok()) { + close_on_error(); + return status; + } + _config = ::doris::snii::format::IndexConfig::kDocsPositionsScoring; + } else if (_common_grams_metadata_seed.has_value()) { + close_on_error(); + return Status::Error( + "SNII CommonGrams metadata cannot be attached to a plain analyzer"); + } + _analyzer = analyzer_provider->get_analyzer( + _uses_common_grams ? inverted_index::AnalysisPurpose::kSniiTransientIndex + : inverted_index::AnalysisPurpose::kPlainQuery); + } else if (_common_grams_metadata_seed.has_value()) { + close_on_error(); + return Status::Error( + "SNII CommonGrams metadata cannot be attached to a keyword analyzer"); + } + } catch (const CLuceneError& e) { + return Status::Error( + "SNII create analyzer failed: {}", e.what()); + } catch (const Exception& e) { + return Status::Error( + "SNII create analyzer failed: {}", e.what()); + } + if (_uses_common_grams) { + _term_buffer->enable_common_gram_pair_keys(); + } + return Status::OK(); +} + +void SniiIndexColumnWriter::set_direct_load(bool is_direct_load) { + // The PRX compression-tier hint must be stable for one index. The first + // pre-row call wins; repeat or late calls are ignored and logged. + DCHECK(!_direct_load_marked && _rid == 0); + if (_direct_load_marked || _rid != 0) { + LOG_EVERY_N(WARNING, 100) << "SNII set_direct_load(" << is_direct_load + << ") ignored (already_marked=" << _direct_load_marked + << ", rows_fed=" << _rid << ") for index " + << (_index_meta != nullptr ? _index_meta->index_id() : -1) + << "; keeping the first-captured PRX tier decision"; + return; + } + _direct_load_marked = true; + _is_direct_load = is_direct_load; +} + +Status SniiIndexColumnWriter::_add_value_tokens(const Slice& value, uint32_t docid, + uint32_t position_base, uint32_t* max_position, + uint32_t* semantic_length) { + DCHECK(max_position != nullptr); + DCHECK(semantic_length != nullptr); + *max_position = position_base; + *semantic_length = 0; + const size_t logical_size = _is_char ? strnlen(value.data, value.size) : value.size; + const std::string_view logical_value(value.data, logical_size); + if ((!_should_analyzer && logical_value.size() > _ignore_above) || + (_should_analyzer && logical_value.empty())) { + return Status::OK(); + } + + // T1a: tokens STREAM from the analyzer straight into the SPIMI buffer as + // string_views (the buffer interns the bytes into its own storage) -- no + // per-row vector and no per-token std::string materialization + // (the old get_analyse_result lane; profile: 3.4-4.7% of import CPU burned + // in token realloc). Golden-byte pins (snii_writer_golden_bytes_test.cpp) + // hold this path byte-identical to the materializing one it replaced. + auto consume_token = [&](std::string_view term, int32_t token_position, bool retain_positions) { + const uint32_t position = + _has_positions ? position_base + cast_set(token_position) : 0; + _term_buffer->add_token(term, docid, position, retain_positions); + *max_position = std::max(*max_position, position); + }; + + if (!_should_analyzer) { + // Keyword lane: the whole value is one exact-match token at position 0 + // (an EMPTY value is a valid keyword token, mirrored from the old lane). + consume_token(logical_value, 0, _has_positions); + } else { + try { + _char_string_reader->init(logical_value.data(), cast_set(logical_value.size()), + false); + if (_uses_common_grams) { + auto* token_stream = _analyzer->reusableTokenStream(L"", _char_string_reader); + if (_common_grams_filter == nullptr) { + _common_grams_filter = + dynamic_cast(token_stream); + DORIS_CHECK(_common_grams_filter != nullptr); + } else { + DCHECK_EQ(static_cast(_common_grams_filter), + token_stream); + } + _common_grams_filter->reset(); + + int32_t position = 0; + std::optional<::doris::snii::writer::ClassifiedPlainTerm> previous; + size_t previous_logical_term_size = 0; + inverted_index::SniiCommonGramsIndexEvent event; + while (_common_grams_filter->next_snii_index_event(&event)) { + const auto current = _term_buffer->intern_classified_plain_term( + event.plain_term, event.logical_term, + _common_grams_filter->common_words()); + const bool has_preceding_gram = + previous.has_value() && (previous->is_common || current.is_common) && + inverted_index::common_gram_component_sizes_encodable( + previous_logical_term_size, event.logical_term.size()); + const uint32_t gram_position = position_base + cast_set(position); + const uint32_t physical_position = + position_base + cast_set(position + 1); + if (has_preceding_gram) { + DCHECK(previous.has_value()); + _term_buffer->add_common_gram_and_plain( + previous->id, current.id, docid, gram_position, physical_position, + previous->is_common && current.is_common); + } else { + _term_buffer->add_plain_token(current.id, docid, physical_position); + } + ++position; + ++*semantic_length; + *max_position = std::max(*max_position, physical_position); + previous = current; + previous_logical_term_size = event.logical_term.size(); + } + } else { + std::unique_ptr owned_token_stream( + _analyzer->tokenStream(L"", _char_string_reader)); + auto* token_stream = owned_token_stream.get(); + // EXACT InvertedIndexAnalyzer::get_analyse_result semantics, + // including the subtle one: an empty token's position increment is + // dropped WITH the token (not accumulated into the next). + lucene::analysis::Token token; + int32_t position = 0; + while (token_stream->next(&token)) { + if (token.termLength() != 0) { + const std::string_view term(token.termBuffer(), + token.termLength()); + position += token.getPositionIncrement(); + consume_token(term, position, _has_positions); + } + } + token_stream->close(); + } + } catch (const CLuceneError& e) { + return _latch_analysis_failure(Status::Error( + "SNII analyze value failed: {}", e.what())); + } catch (const Exception& e) { + return _latch_analysis_failure(Status::Error( + "SNII analyze value failed: {}", e.what())); + } + } + return Status::OK(); +} + +Status SniiIndexColumnWriter::add_values(const std::string /*name*/, const void* values, + size_t count) { + if (!_failure_status.ok()) { + return _failure_status; + } + const auto* v = reinterpret_cast(values); + for (size_t i = 0; i < count; ++i) { + uint32_t max_position = 0; + uint32_t semantic_length = 0; + RETURN_IF_ERROR(_add_value_tokens(*v, _rid, 0, &max_position, &semantic_length)); + if (_uses_common_grams) { + _encoded_norms.push_back(::doris::snii::query::encode_norm(semantic_length)); + _report_encoded_norms_capacity(); + _scoring_token_count += semantic_length; + } + ++v; + ++_rid; + } + return Status::OK(); +} + +Status SniiIndexColumnWriter::add_array_values(size_t field_size, const void* value_ptr, + const uint8_t* nested_null_map, + const uint8_t* offsets_ptr, size_t count) { + if (!_failure_status.ok()) { + return _failure_status; + } + if (_uses_common_grams) { + return _latch_analysis_failure(Status::Error( + "SNII CommonGrams does not support ARRAY fields")); + } + if (count == 0) { + return Status::OK(); + } + const auto* offsets = reinterpret_cast(offsets_ptr); + size_t start_off = 0; + for (size_t i = 0; i < count; ++i) { + auto array_elem_size = offsets[i + 1] - offsets[i]; + uint32_t position_base = 0; + for (auto j = start_off; j < start_off + array_elem_size; ++j) { + if (nested_null_map != nullptr && nested_null_map[j] == 1) { + continue; + } + const auto* value = reinterpret_cast( + reinterpret_cast(value_ptr) + j * field_size); + uint32_t max_position = position_base; + uint32_t semantic_length = 0; + RETURN_IF_ERROR(_add_value_tokens(*value, _rid, position_base, &max_position, + &semantic_length)); + position_base = max_position + 1; + } + start_off += array_elem_size; + ++_rid; + } + return Status::OK(); +} + +void SniiIndexColumnWriter::_report_null_docids_capacity(bool release_all) { + if (_memory_reporter == nullptr) { + return; + } + const int64_t now = + release_all ? 0 : static_cast(_null_docids.capacity() * sizeof(uint32_t)); + if (now != _null_docids_charged_bytes) { + _memory_reporter->report(now - _null_docids_charged_bytes); + _null_docids_charged_bytes = now; + } +} + +void SniiIndexColumnWriter::_report_encoded_norms_capacity(bool release_all) { + if (_memory_reporter == nullptr) { + return; + } + const int64_t now = release_all ? 0 : static_cast(_encoded_norms.capacity()); + if (now != _encoded_norms_charged_bytes) { + _memory_reporter->report(now - _encoded_norms_charged_bytes); + _encoded_norms_charged_bytes = now; + } +} + +Status SniiIndexColumnWriter::add_nulls(uint32_t count) { + if (!_failure_status.ok()) { + return _failure_status; + } + // GEOMETRIC BULK reserve -- never an exact one: append_nullable calls + // add_nulls once per NULL RUN (thousands to millions of calls on a large + // interleaved-null segment), and an exact reserve(size()+count) caps + // capacity at "just enough" -- the NEXT call then reallocates and memcpys + // the WHOLE array, defeating geometric growth and turning total memcpy + // quadratic: O(runs x array_bytes). On an agentlogs full-compaction segment + // (12.4M rows, 22% interleaved nulls) that was TBs of memcpy per tablet -- + // the compaction ran 8+x slower than V3 (whose add_nulls is a roaring + // addRange). Doubling on overflow keeps the O(count) amortization AND makes + // one large run pay at most one reallocation. + const size_t need = _null_docids.size() + count; + if (need > _null_docids.capacity()) { + _null_docids.reserve(std::max(need, _null_docids.capacity() * 2)); + } + for (uint32_t i = 0; i < count; ++i) { + _null_docids.push_back(_rid + i); + } + _rid += count; + if (_uses_common_grams) { + _encoded_norms.insert(_encoded_norms.end(), count, ::doris::snii::query::encode_norm(0)); + _report_encoded_norms_capacity(); + } + _report_null_docids_capacity(); + return Status::OK(); +} + +Status SniiIndexColumnWriter::add_array_nulls(const uint8_t* null_map, size_t num_rows) { + if (!_failure_status.ok()) { + return _failure_status; + } + DCHECK(_rid >= num_rows); + if (num_rows == 0 || null_map == nullptr) { + return Status::OK(); + } + const auto first_row = _rid - num_rows; + for (size_t i = 0; i < num_rows; ++i) { + if (null_map[i] == 1) { + _null_docids.push_back(cast_set(first_row + i)); + } + } + _report_null_docids_capacity(); + return Status::OK(); +} + +Status SniiIndexColumnWriter::finish() { + if (!_failure_status.ok()) { + return _failure_status; + } + DCHECK(_term_buffer != nullptr); + auto status = _term_buffer->status(); + if (!status.ok()) { + return Status::InternalError("SNII term buffer error: {}", status.to_string()); + } + // Ownership of _null_docids hands off to the flush below (transient, + // flush-scoped); release the accumulation-phase charge so the retained + // reporter (and the LOAD MemTracker behind it) balances to zero. + _report_null_docids_capacity(/*release_all=*/true); + IndexFileWriter::SniiAddIndexOptions options {}; + options.is_direct_load = _is_direct_load; + if (_uses_common_grams) { + options.encoded_norms = std::move(_encoded_norms); + options.common_grams_metadata = _build_common_grams_metadata(); + options.common_grams_posting_policy = + ::doris::snii::format::CommonGramsPostingPolicy::kHybridV1; + } + status = _index_file_writer->add_snii_index( + _index_meta, cast_set(_rid), std::move(_null_docids), _term_buffer.get(), + _config, std::move(options), _memory_reporter.get()); + _report_encoded_norms_capacity(/*release_all=*/true); + RETURN_IF_ERROR(status); + _index_file_writer->retain_snii_memory_reporter(std::move(_memory_reporter)); + _term_buffer.reset(); + return Status::OK(); +} + +inverted_index::CommonGramsSegmentMetadata SniiIndexColumnWriter::_build_common_grams_metadata() + const { + DORIS_CHECK(_uses_common_grams); + DORIS_CHECK(_common_grams_metadata_seed.has_value()); + auto metadata = *_common_grams_metadata_seed; + metadata.common_grams_coverage = inverted_index::CommonGramsCoverage::kMixed; + metadata.scoring_doc_count = _rid; + metadata.scoring_token_count = _scoring_token_count; + return metadata; +} + +Status SniiIndexColumnWriter::_latch_analysis_failure(Status status) { + DORIS_CHECK(!status.ok()); + DORIS_CHECK(_failure_status.ok()); + _failure_status = std::move(status); + close_on_error(); + return _failure_status; +} + +void SniiIndexColumnWriter::close_on_error() { + _term_buffer.reset(); + // Balance the LOAD MemTracker mirror before dropping the reporter. + _report_null_docids_capacity(/*release_all=*/true); + _report_encoded_norms_capacity(/*release_all=*/true); + _memory_reporter.reset(); + _null_docids.clear(); + std::vector().swap(_encoded_norms); + _scoring_token_count = 0; +} + +} // namespace doris::segment_v2 diff --git a/be/src/storage/index/snii/snii_index_writer.h b/be/src/storage/index/snii/snii_index_writer.h new file mode 100644 index 00000000000000..72a7bb56669dff --- /dev/null +++ b/be/src/storage/index/snii/snii_index_writer.h @@ -0,0 +1,144 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include + +#include "storage/index/index_writer.h" +#include "storage/index/inverted/common_grams/common_grams_segment_metadata.h" +#include "storage/index/inverted/inverted_index_parser.h" +#include "storage/index/inverted/query/query_info.h" +#include "storage/index/inverted/util/reader.h" +#include "storage/index/snii/format/format_constants.h" +#include "storage/index/snii/writer/memory_reporter.h" +#include "storage/index/snii/writer/spimi_term_buffer.h" +#include "util/slice.h" + +namespace lucene::analysis { +class Analyzer; +} + +namespace doris::segment_v2::inverted_index { +class CommonGramsFilter; +} + +namespace doris::segment_v2 { + +class SniiIndexColumnWriter final : public IndexColumnWriter { +public: + SniiIndexColumnWriter(IndexFileWriter* index_file_writer, const TabletIndex* index_meta, + FieldType value_type, + std::optional + common_grams_metadata_seed = std::nullopt); + ~SniiIndexColumnWriter() override = default; + + Status init() override; + void set_direct_load(bool is_direct_load) override; + Status add_values(const std::string name, const void* values, size_t count) override; + Status add_array_values(size_t field_size, const void* value_ptr, + const uint8_t* nested_null_map, const uint8_t* offsets_ptr, + size_t count) override; + Status add_nulls(uint32_t count) override; + Status add_array_nulls(const uint8_t* null_map, size_t num_rows) override; + Status finish() override; + int64_t size() const override { return 0; } + void close_on_error() override; + +#ifdef BE_TEST + // TEST-ONLY view of the accumulated null docids: the growth-policy + // regression pin asserts add_nulls keeps geometric growth (an exact + // reserve(size+count) per null RUN made total memcpy quadratic -- the + // agentlogs full-compaction pathology). + const std::vector& null_docids_for_test() const { return _null_docids; } + ::doris::snii::writer::SpimiTermBuffer* term_buffer_for_test() const { + return _term_buffer.get(); + } + ::doris::snii::writer::MemoryReporter* memory_reporter_for_test() const { + return _memory_reporter.get(); + } + const std::vector& encoded_norms_for_test() const { return _encoded_norms; } + uint64_t scoring_token_count_for_test() const { return _scoring_token_count; } + ::doris::snii::format::IndexConfig config_for_test() const { return _config; } + bool has_common_grams_metadata_seed_for_test() const { + return _common_grams_metadata_seed.has_value(); + } + inverted_index::CommonGramsSegmentMetadata common_grams_metadata_for_test() const { + return _build_common_grams_metadata(); + } + void set_analysis_for_test(inverted_index::ReaderPtr reader, + std::shared_ptr analyzer) { + _should_analyzer = true; + _char_string_reader = std::move(reader); + _analyzer = std::move(analyzer); + } +#endif + +private: + Status _add_value_tokens(const Slice& value, uint32_t docid, uint32_t position_base, + uint32_t* max_position, uint32_t* semantic_length); + inverted_index::CommonGramsSegmentMetadata _build_common_grams_metadata() const; + // Mirrors _null_docids' capacity into _memory_reporter (delta-charged); + // release_all zeroes the charge (finish() handoff / close_on_error()). + void _report_null_docids_capacity(bool release_all = false); + void _report_encoded_norms_capacity(bool release_all = false); + Status _latch_analysis_failure(Status status); + + IndexFileWriter* _index_file_writer = nullptr; + const TabletIndex* _index_meta = nullptr; + bool _should_analyzer = false; + bool _has_positions = false; + const bool _is_char; + const bool _common_grams_build_enabled; + bool _uses_common_grams = false; + // Latch: set_direct_load() ran. The first call wins; a repeat or late call + // is ignored (and logged) so one index keeps one stable compression-tier + // decision. + bool _direct_load_marked = false; + // Captured by set_direct_load() under the same latch: this writer serves a + // stream/broker load (DataWriteType::TYPE_DIRECT). Consumed at finish() to + // route the prx region to the load-tier zstd level (patch C, + // config::snii_prx_zstd_level_direct_load). + bool _is_direct_load = false; + uint32_t _ignore_above = 0; + uint32_t _rid = 0; + ::doris::snii::format::IndexConfig _config = ::doris::snii::format::IndexConfig::kDocsOnly; + InvertedIndexAnalyzerConfig _analyzer_config; + inverted_index::ReaderPtr _char_string_reader; + std::shared_ptr _analyzer; + inverted_index::CommonGramsFilter* _common_grams_filter = nullptr; + std::unique_ptr<::doris::snii::writer::MemoryReporter> _memory_reporter; + std::unique_ptr<::doris::snii::writer::SpimiTermBuffer> _term_buffer; + std::vector _null_docids; + std::vector _encoded_norms; + uint64_t _scoring_token_count = 0; + std::optional _common_grams_metadata_seed; + // Bytes of _null_docids capacity currently mirrored into _memory_reporter + // (and through it Doris's LOAD MemTracker). Re-charged on growth in + // add_nulls / add_array_nulls, released in finish() / close_on_error() -- + // without it a large interleaved-null segment accumulates untracked RSS the + // G09 limiter cannot see. + int64_t _null_docids_charged_bytes = 0; + int64_t _encoded_norms_charged_bytes = 0; + Status _failure_status = Status::OK(); +}; + +} // namespace doris::segment_v2 diff --git a/be/src/storage/index/snii/snii_prx_profile.h b/be/src/storage/index/snii/snii_prx_profile.h new file mode 100644 index 00000000000000..c343c5b54fa076 --- /dev/null +++ b/be/src/storage/index/snii/snii_prx_profile.h @@ -0,0 +1,308 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include "runtime/runtime_profile.h" +#include "storage/index/snii/format/prx_decode_stats.h" +#include "storage/index/snii/query/query_profile.h" +#include "storage/olap_common.h" + +namespace doris::snii { + +#ifdef BE_TEST +namespace testing { + +void record_prx_execution_profile_scope_construction(); +void record_prx_execution_profile_scope_flush(); +void reset_prx_execution_profile_scope_counters(); +uint64_t prx_execution_profile_scope_construction_count(); +uint64_t prx_execution_profile_scope_flush_count(); + +} // namespace testing +#endif + +inline void add_prx_decode_stats(OlapReaderStatistics* target, + const format::PrxDecodeStats& delta) { + SniiQueryStats& stats = target->snii_stats; + stats.prx_raw_frames += static_cast(delta.raw_frames); + stats.prx_zstd_frames += static_cast(delta.zstd_frames); + stats.prx_pfor_frames += static_cast(delta.pfor_frames); + stats.prx_plaintext_bytes += static_cast(delta.plaintext_bytes); + stats.prx_total_docs += static_cast(delta.total_docs); + stats.prx_selected_docs += static_cast(delta.selected_docs); + stats.prx_total_positions += static_cast(delta.total_positions); + stats.prx_selected_positions += static_cast(delta.selected_positions); + stats.prx_fetch_ns += static_cast(delta.fetch_ns); + stats.prx_decode_ns += static_cast(delta.decode_ns); + stats.prx_phrase_verify_ns += static_cast(delta.phrase_verify_ns); +} + +inline void add_phrase_query_stats(OlapReaderStatistics* target, + const format::PhraseQueryExecutionStats& delta) { + SniiQueryStats& stats = target->snii_stats; + stats.phrase_candidate_docs += static_cast(delta.exact_candidate_docs); + stats.phrase_candidate_visits += static_cast(delta.exact_candidate_visits); + stats.phrase_prefix_leading_candidate_docs += + static_cast(delta.prefix_leading_candidate_docs); + stats.phrase_prefix_tail_candidate_visits += + static_cast(delta.prefix_tail_candidate_visits); + stats.common_grams_candidate_queries += + static_cast(delta.common_grams_candidate_queries); + stats.common_grams_plain_plans += static_cast(delta.common_grams_plain_plans); + stats.common_grams_gram_plans += static_cast(delta.common_grams_gram_plans); + stats.common_grams_fallback_no_gram += + static_cast(delta.common_grams_fallback_no_gram); + stats.common_grams_fallback_incompatible += + static_cast(delta.common_grams_fallback_incompatible); + stats.common_grams_fallback_kill_switch += + static_cast(delta.common_grams_fallback_kill_switch); + stats.common_grams_fallback_cost += static_cast(delta.common_grams_fallback_cost); + stats.common_grams_fallback_base_analyzer_mismatch += + static_cast(delta.common_grams_fallback_base_analyzer_mismatch); + stats.common_grams_fallback_prefix_tail_empty += + static_cast(delta.common_grams_fallback_prefix_tail_empty); + stats.common_grams_authoritative_empty += + static_cast(delta.common_grams_authoritative_empty); + stats.common_grams_plain_posting_bytes += + static_cast(delta.common_grams_plain_posting_bytes); + stats.common_grams_gram_posting_bytes += + static_cast(delta.common_grams_gram_posting_bytes); + stats.common_grams_plain_estimated_candidate_df += + static_cast(delta.common_grams_plain_estimated_candidate_df); + stats.common_grams_gram_estimated_candidate_df += + static_cast(delta.common_grams_gram_estimated_candidate_df); + stats.common_grams_plain_estimated_cost += + static_cast(delta.common_grams_plain_estimated_cost); + stats.common_grams_gram_estimated_cost += + static_cast(delta.common_grams_gram_estimated_cost); + stats.common_grams_planning_ns += static_cast(delta.common_grams_planning_ns); +} + +// Exists only around an actual SNII compute execution. Query-cache hits, +// count-only fast paths, and single-flight followers never construct this +// scope, so they cannot contribute another execution's PRX work. The destructor +// flushes already-committed frames on both success and early error returns. +class SniiPrxExecutionProfileScope { +public: + explicit SniiPrxExecutionProfileScope(OlapReaderStatistics& target) : target_(target) { +#ifdef BE_TEST + testing::record_prx_execution_profile_scope_construction(); +#endif + } + ~SniiPrxExecutionProfileScope() { + add_prx_decode_stats(&target_, profile_.prx_decode_stats); + add_phrase_query_stats(&target_, profile_.phrase_query_stats); +#ifdef BE_TEST + testing::record_prx_execution_profile_scope_flush(); +#endif + } + + SniiPrxExecutionProfileScope(const SniiPrxExecutionProfileScope&) = delete; + SniiPrxExecutionProfileScope& operator=(const SniiPrxExecutionProfileScope&) = delete; + SniiPrxExecutionProfileScope(SniiPrxExecutionProfileScope&&) = delete; + SniiPrxExecutionProfileScope& operator=(SniiPrxExecutionProfileScope&&) = delete; + + query::QueryProfile* profile() { return &profile_; } + +private: + OlapReaderStatistics& target_; + query::QueryProfile profile_; +}; + +class SniiPrxRuntimeProfileCounters { +public: + static constexpr std::array counter_names() { + return {"SniiPrxRawFrames", + "SniiPrxZstdFrames", + "SniiPrxPforFrames", + "SniiPrxPlaintextBytes", + "SniiPrxTotalDocs", + "SniiPrxSelectedDocs", + "SniiPrxTotalPositions", + "SniiPrxSelectedPositions", + "SniiPrxFetchTime", + "SniiPrxInclusiveDecodeTime", + "SniiPrxExclusivePhraseVerifyTime"}; + } + + void initialize(RuntimeProfile* profile) { + raw_frames_ = profile->add_nonzero_counter("SniiPrxRawFrames", TUnit::UNIT, + RuntimeProfile::ROOT_COUNTER, 1); + zstd_frames_ = profile->add_nonzero_counter("SniiPrxZstdFrames", TUnit::UNIT, + RuntimeProfile::ROOT_COUNTER, 1); + pfor_frames_ = profile->add_nonzero_counter("SniiPrxPforFrames", TUnit::UNIT, + RuntimeProfile::ROOT_COUNTER, 1); + plaintext_bytes_ = profile->add_nonzero_counter("SniiPrxPlaintextBytes", TUnit::BYTES, + RuntimeProfile::ROOT_COUNTER, 1); + total_docs_ = profile->add_nonzero_counter("SniiPrxTotalDocs", TUnit::UNIT, + RuntimeProfile::ROOT_COUNTER, 1); + selected_docs_ = profile->add_nonzero_counter("SniiPrxSelectedDocs", TUnit::UNIT, + RuntimeProfile::ROOT_COUNTER, 1); + total_positions_ = profile->add_nonzero_counter("SniiPrxTotalPositions", TUnit::UNIT, + RuntimeProfile::ROOT_COUNTER, 1); + selected_positions_ = profile->add_nonzero_counter("SniiPrxSelectedPositions", TUnit::UNIT, + RuntimeProfile::ROOT_COUNTER, 1); + fetch_ns_ = profile->add_nonzero_counter("SniiPrxFetchTime", TUnit::TIME_NS, + RuntimeProfile::ROOT_COUNTER, 1); + decode_ns_ = profile->add_nonzero_counter("SniiPrxInclusiveDecodeTime", TUnit::TIME_NS, + RuntimeProfile::ROOT_COUNTER, 1); + phrase_verify_ns_ = + profile->add_nonzero_counter("SniiPrxExclusivePhraseVerifyTime", TUnit::TIME_NS, + RuntimeProfile::ROOT_COUNTER, 1); + } + + void update(const OlapReaderStatistics& stats) const { + const SniiQueryStats& s = stats.snii_stats; + COUNTER_UPDATE(raw_frames_, s.prx_raw_frames); + COUNTER_UPDATE(zstd_frames_, s.prx_zstd_frames); + COUNTER_UPDATE(pfor_frames_, s.prx_pfor_frames); + COUNTER_UPDATE(plaintext_bytes_, s.prx_plaintext_bytes); + COUNTER_UPDATE(total_docs_, s.prx_total_docs); + COUNTER_UPDATE(selected_docs_, s.prx_selected_docs); + COUNTER_UPDATE(total_positions_, s.prx_total_positions); + COUNTER_UPDATE(selected_positions_, s.prx_selected_positions); + COUNTER_UPDATE(fetch_ns_, s.prx_fetch_ns); + COUNTER_UPDATE(decode_ns_, s.prx_decode_ns); + COUNTER_UPDATE(phrase_verify_ns_, s.prx_phrase_verify_ns); + } + +private: + RuntimeProfile::Counter* raw_frames_ = nullptr; + RuntimeProfile::Counter* zstd_frames_ = nullptr; + RuntimeProfile::Counter* pfor_frames_ = nullptr; + RuntimeProfile::Counter* plaintext_bytes_ = nullptr; + RuntimeProfile::Counter* total_docs_ = nullptr; + RuntimeProfile::Counter* selected_docs_ = nullptr; + RuntimeProfile::Counter* total_positions_ = nullptr; + RuntimeProfile::Counter* selected_positions_ = nullptr; + RuntimeProfile::Counter* fetch_ns_ = nullptr; + RuntimeProfile::Counter* decode_ns_ = nullptr; + RuntimeProfile::Counter* phrase_verify_ns_ = nullptr; +}; + +class SniiPhraseRuntimeProfileCounters { +public: + void initialize(RuntimeProfile* profile) { + candidate_docs_ = profile->add_nonzero_counter("SniiPhraseCandidateDocs", TUnit::UNIT, + RuntimeProfile::ROOT_COUNTER, 1); + candidate_visits_ = profile->add_nonzero_counter("SniiPhraseCandidateVisits", TUnit::UNIT, + RuntimeProfile::ROOT_COUNTER, 1); + prefix_leading_candidate_docs_ = + profile->add_nonzero_counter("SniiPhrasePrefixLeadingCandidateDocs", TUnit::UNIT, + RuntimeProfile::ROOT_COUNTER, 1); + prefix_tail_candidate_visits_ = + profile->add_nonzero_counter("SniiPhrasePrefixTailCandidateVisits", TUnit::UNIT, + RuntimeProfile::ROOT_COUNTER, 1); + common_grams_candidate_queries_ = profile->add_nonzero_counter( + "SniiCommonGramsCandidateQueries", TUnit::UNIT, RuntimeProfile::ROOT_COUNTER, 1); + common_grams_plain_plans_ = profile->add_nonzero_counter( + "SniiCommonGramsPlainPlans", TUnit::UNIT, RuntimeProfile::ROOT_COUNTER, 1); + common_grams_gram_plans_ = profile->add_nonzero_counter( + "SniiCommonGramsGramPlans", TUnit::UNIT, RuntimeProfile::ROOT_COUNTER, 1); + common_grams_fallback_no_gram_ = profile->add_nonzero_counter( + "SniiCommonGramsFallbackNoGram", TUnit::UNIT, RuntimeProfile::ROOT_COUNTER, 1); + common_grams_fallback_incompatible_ = + profile->add_nonzero_counter("SniiCommonGramsFallbackIncompatible", TUnit::UNIT, + RuntimeProfile::ROOT_COUNTER, 1); + common_grams_fallback_kill_switch_ = profile->add_nonzero_counter( + "SniiCommonGramsFallbackKillSwitch", TUnit::UNIT, RuntimeProfile::ROOT_COUNTER, 1); + common_grams_fallback_cost_ = profile->add_nonzero_counter( + "SniiCommonGramsFallbackCost", TUnit::UNIT, RuntimeProfile::ROOT_COUNTER, 1); + common_grams_fallback_base_analyzer_mismatch_ = + profile->add_nonzero_counter("SniiCommonGramsFallbackBaseAnalyzerMismatch", + TUnit::UNIT, RuntimeProfile::ROOT_COUNTER, 1); + common_grams_fallback_prefix_tail_empty_ = + profile->add_nonzero_counter("SniiCommonGramsFallbackPrefixTailEmpty", TUnit::UNIT, + RuntimeProfile::ROOT_COUNTER, 1); + common_grams_authoritative_empty_ = profile->add_nonzero_counter( + "SniiCommonGramsAuthoritativeEmpty", TUnit::UNIT, RuntimeProfile::ROOT_COUNTER, 1); + common_grams_plain_posting_bytes_ = profile->add_nonzero_counter( + "SniiCommonGramsPlainPostingBytes", TUnit::BYTES, RuntimeProfile::ROOT_COUNTER, 1); + common_grams_gram_posting_bytes_ = profile->add_nonzero_counter( + "SniiCommonGramsGramPostingBytes", TUnit::BYTES, RuntimeProfile::ROOT_COUNTER, 1); + common_grams_plain_estimated_candidate_df_ = + profile->add_nonzero_counter("SniiCommonGramsPlainEstimatedCandidateDf", + TUnit::UNIT, RuntimeProfile::ROOT_COUNTER, 1); + common_grams_gram_estimated_candidate_df_ = + profile->add_nonzero_counter("SniiCommonGramsGramEstimatedCandidateDf", TUnit::UNIT, + RuntimeProfile::ROOT_COUNTER, 1); + common_grams_plain_estimated_cost_ = profile->add_nonzero_counter( + "SniiCommonGramsPlainEstimatedCost", TUnit::UNIT, RuntimeProfile::ROOT_COUNTER, 1); + common_grams_gram_estimated_cost_ = profile->add_nonzero_counter( + "SniiCommonGramsGramEstimatedCost", TUnit::UNIT, RuntimeProfile::ROOT_COUNTER, 1); + common_grams_planning_ns_ = profile->add_nonzero_counter( + "SniiCommonGramsPlanningTime", TUnit::TIME_NS, RuntimeProfile::ROOT_COUNTER, 1); + } + + void update(const OlapReaderStatistics& stats) const { + const SniiQueryStats& s = stats.snii_stats; + COUNTER_UPDATE(candidate_docs_, s.phrase_candidate_docs); + COUNTER_UPDATE(candidate_visits_, s.phrase_candidate_visits); + COUNTER_UPDATE(prefix_leading_candidate_docs_, s.phrase_prefix_leading_candidate_docs); + COUNTER_UPDATE(prefix_tail_candidate_visits_, s.phrase_prefix_tail_candidate_visits); + COUNTER_UPDATE(common_grams_candidate_queries_, s.common_grams_candidate_queries); + COUNTER_UPDATE(common_grams_plain_plans_, s.common_grams_plain_plans); + COUNTER_UPDATE(common_grams_gram_plans_, s.common_grams_gram_plans); + COUNTER_UPDATE(common_grams_fallback_no_gram_, s.common_grams_fallback_no_gram); + COUNTER_UPDATE(common_grams_fallback_incompatible_, s.common_grams_fallback_incompatible); + COUNTER_UPDATE(common_grams_fallback_kill_switch_, s.common_grams_fallback_kill_switch); + COUNTER_UPDATE(common_grams_fallback_cost_, s.common_grams_fallback_cost); + COUNTER_UPDATE(common_grams_fallback_base_analyzer_mismatch_, + s.common_grams_fallback_base_analyzer_mismatch); + COUNTER_UPDATE(common_grams_fallback_prefix_tail_empty_, + s.common_grams_fallback_prefix_tail_empty); + COUNTER_UPDATE(common_grams_authoritative_empty_, s.common_grams_authoritative_empty); + COUNTER_UPDATE(common_grams_plain_posting_bytes_, s.common_grams_plain_posting_bytes); + COUNTER_UPDATE(common_grams_gram_posting_bytes_, s.common_grams_gram_posting_bytes); + COUNTER_UPDATE(common_grams_plain_estimated_candidate_df_, + s.common_grams_plain_estimated_candidate_df); + COUNTER_UPDATE(common_grams_gram_estimated_candidate_df_, + s.common_grams_gram_estimated_candidate_df); + COUNTER_UPDATE(common_grams_plain_estimated_cost_, s.common_grams_plain_estimated_cost); + COUNTER_UPDATE(common_grams_gram_estimated_cost_, s.common_grams_gram_estimated_cost); + COUNTER_UPDATE(common_grams_planning_ns_, s.common_grams_planning_ns); + } + +private: + RuntimeProfile::Counter* candidate_docs_ = nullptr; + RuntimeProfile::Counter* candidate_visits_ = nullptr; + RuntimeProfile::Counter* prefix_leading_candidate_docs_ = nullptr; + RuntimeProfile::Counter* prefix_tail_candidate_visits_ = nullptr; + RuntimeProfile::Counter* common_grams_candidate_queries_ = nullptr; + RuntimeProfile::Counter* common_grams_plain_plans_ = nullptr; + RuntimeProfile::Counter* common_grams_gram_plans_ = nullptr; + RuntimeProfile::Counter* common_grams_fallback_no_gram_ = nullptr; + RuntimeProfile::Counter* common_grams_fallback_incompatible_ = nullptr; + RuntimeProfile::Counter* common_grams_fallback_kill_switch_ = nullptr; + RuntimeProfile::Counter* common_grams_fallback_cost_ = nullptr; + RuntimeProfile::Counter* common_grams_fallback_base_analyzer_mismatch_ = nullptr; + RuntimeProfile::Counter* common_grams_fallback_prefix_tail_empty_ = nullptr; + RuntimeProfile::Counter* common_grams_authoritative_empty_ = nullptr; + RuntimeProfile::Counter* common_grams_plain_posting_bytes_ = nullptr; + RuntimeProfile::Counter* common_grams_gram_posting_bytes_ = nullptr; + RuntimeProfile::Counter* common_grams_plain_estimated_candidate_df_ = nullptr; + RuntimeProfile::Counter* common_grams_gram_estimated_candidate_df_ = nullptr; + RuntimeProfile::Counter* common_grams_plain_estimated_cost_ = nullptr; + RuntimeProfile::Counter* common_grams_gram_estimated_cost_ = nullptr; + RuntimeProfile::Counter* common_grams_planning_ns_ = nullptr; +}; + +} // namespace doris::snii diff --git a/be/src/storage/index/snii/snii_query_stats.h b/be/src/storage/index/snii/snii_query_stats.h new file mode 100644 index 00000000000000..a7466af8d7c8de --- /dev/null +++ b/be/src/storage/index/snii/snii_query_stats.h @@ -0,0 +1,66 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +namespace doris::snii { + +// Populated by snii_prx_profile.h from a per-execution query::QueryProfile, and read back out by +// SniiPrxRuntimeProfileCounters / SniiPhraseRuntimeProfileCounters into the scan node's +// RuntimeProfile. Lives in its own header, mirroring InvertedIndexStatistics +// (storage/index/inverted/inverted_index_stats.h), so OlapReaderStatistics -- shared by every +// storage format -- holds one named field here instead of one field per counter. +struct SniiQueryStats { + int64_t prx_raw_frames = 0; + int64_t prx_zstd_frames = 0; + int64_t prx_pfor_frames = 0; + int64_t prx_plaintext_bytes = 0; + int64_t prx_total_docs = 0; + int64_t prx_selected_docs = 0; + int64_t prx_total_positions = 0; + int64_t prx_selected_positions = 0; + int64_t prx_fetch_ns = 0; + int64_t prx_decode_ns = 0; + int64_t prx_phrase_verify_ns = 0; + + int64_t phrase_candidate_docs = 0; + int64_t phrase_candidate_visits = 0; + int64_t phrase_prefix_leading_candidate_docs = 0; + int64_t phrase_prefix_tail_candidate_visits = 0; + + int64_t common_grams_candidate_queries = 0; + int64_t common_grams_plain_plans = 0; + int64_t common_grams_gram_plans = 0; + int64_t common_grams_fallback_no_gram = 0; + int64_t common_grams_fallback_incompatible = 0; + int64_t common_grams_fallback_kill_switch = 0; + int64_t common_grams_fallback_cost = 0; + int64_t common_grams_fallback_base_analyzer_mismatch = 0; + int64_t common_grams_fallback_prefix_tail_empty = 0; + int64_t common_grams_authoritative_empty = 0; + int64_t common_grams_plain_posting_bytes = 0; + int64_t common_grams_gram_posting_bytes = 0; + int64_t common_grams_plain_estimated_candidate_df = 0; + int64_t common_grams_gram_estimated_candidate_df = 0; + int64_t common_grams_plain_estimated_cost = 0; + int64_t common_grams_gram_estimated_cost = 0; + int64_t common_grams_planning_ns = 0; +}; + +} // namespace doris::snii diff --git a/be/src/storage/index/snii/stats/snii_stats_provider.cpp b/be/src/storage/index/snii/stats/snii_stats_provider.cpp new file mode 100644 index 00000000000000..e12fe6d5f73d80 --- /dev/null +++ b/be/src/storage/index/snii/stats/snii_stats_provider.cpp @@ -0,0 +1,145 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/stats/snii_stats_provider.h" + +#include + +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/format/dict_entry.h" +#include "storage/index/snii/format/format_constants.h" +#include "storage/index/snii/format/stats_block.h" + +namespace doris::snii::stats { + +using format::DictEntry; +using format::NormsPodReader; +using format::RegionRef; + +namespace { + +// Resolves a term's DictEntry. *found=false for an absent term (OK status). +Status lookup_entry(const reader::LogicalIndexReader& idx, std::string_view term, bool* found, + DictEntry* entry) { + uint64_t frq_base = 0; + uint64_t prx_base = 0; + return idx.lookup(term, found, entry, &frq_base, &prx_base); +} + +} // namespace + +Status SniiStatsProvider::open(const reader::LogicalIndexReader* idx, SniiStatsProvider* out) { + return open_impl(idx, out, true); +} + +#ifdef BE_TEST +Status SniiStatsProvider::open_legacy_for_test(const reader::LogicalIndexReader* idx, + SniiStatsProvider* out) { + return open_impl(idx, out, false); +} +#endif + +Status SniiStatsProvider::open_impl(const reader::LogicalIndexReader* idx, SniiStatsProvider* out, + bool require_semantic_metadata) { + if (idx == nullptr || out == nullptr) { + return Status::Error("stats_provider: null argument"); + } + out->idx_ = idx; + const auto& sb = idx->stats(); + out->doc_count_ = sb.doc_count; + out->indexed_doc_count_ = sb.indexed_doc_count; + out->sum_total_term_freq_ = sb.sum_total_term_freq; + + const RegionRef& norms = idx->section_refs().norms; + const auto* metadata = idx->common_grams_metadata(); + if (metadata != nullptr) { + using namespace segment_v2::inverted_index; + RETURN_IF_ERROR(validate_snii_scoring_metadata( + metadata, sb.doc_count, sb.sum_total_term_freq, + idx->tier() == format::IndexTier::kT3, idx->has_positions(), norms.length != 0)); + out->doc_count_ = metadata->scoring_doc_count; + out->indexed_doc_count_ = metadata->scoring_doc_count; + out->sum_total_term_freq_ = metadata->scoring_token_count; + } else if (require_semantic_metadata) { + RETURN_IF_ERROR(segment_v2::inverted_index::validate_snii_scoring_metadata( + nullptr, sb.doc_count, sb.sum_total_term_freq, + idx->tier() == format::IndexTier::kT3, idx->has_positions(), norms.length != 0)); + } + if (norms.length == 0) { + out->has_norms_ = false; + return Status::OK(); + } + + RETURN_IF_ERROR(idx->open_norms(&out->norms_reader_)); + if (metadata != nullptr && out->norms_reader_.doc_count() != metadata->scoring_doc_count) { + return Status::Error( + "snii_stats: semantic norms doc count {} differs from scoring doc count {}", + out->norms_reader_.doc_count(), metadata->scoring_doc_count); + } + out->has_norms_ = true; + return Status::OK(); +} + +double SniiStatsProvider::avgdl() const { + const uint64_t denom = std::max(1, indexed_doc_count_); + return static_cast(sum_total_term_freq_) / static_cast(denom); +} + +Status SniiStatsProvider::doc_freq(std::string_view term, uint64_t* df) const { + if (df == nullptr) + return Status::Error("stats_provider: null df"); + *df = 0; + bool found = false; + DictEntry entry; + RETURN_IF_ERROR(lookup_entry(*idx_, term, &found, &entry)); + if (found) *df = entry.df; + return Status::OK(); +} + +Status SniiStatsProvider::total_term_freq(std::string_view term, uint64_t* ttf) const { + if (ttf == nullptr) + return Status::Error("stats_provider: null ttf"); + *ttf = 0; + bool found = false; + DictEntry entry; + RETURN_IF_ERROR(lookup_entry(*idx_, term, &found, &entry)); + if (!found) return Status::OK(); + // tier>=T2 entries carry the total term frequency directly in ttf_delta (the + // LogicalIndexWriter stores ttf there, not a delta from df). G16-f blocks + // (kNoTermStats: freq-dropped index) omit it -- fail with the semantic + // error instead of silently returning the default 0 (mixed old/new + // segments would otherwise disagree on the same term). + if (!entry.term_stats_present) { + return Status::Error( + "snii_stats: ttf requested but the dict block carries no term stats " + "(freq-dropped index)"); + } + *ttf = entry.ttf_delta; + return Status::OK(); +} + +Status SniiStatsProvider::encoded_norm(uint32_t docid, uint8_t* out) const { + if (out == nullptr) + return Status::Error("stats_provider: null out"); + if (!has_norms_) { + return Status::Error( + "stats_provider: index has no norms"); + } + return norms_reader_.try_encoded_norm(docid, out); +} + +} // namespace doris::snii::stats diff --git a/be/src/storage/index/snii/stats/snii_stats_provider.h b/be/src/storage/index/snii/stats/snii_stats_provider.h new file mode 100644 index 00000000000000..284a6ecc07cb71 --- /dev/null +++ b/be/src/storage/index/snii/stats/snii_stats_provider.h @@ -0,0 +1,92 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include "common/status.h" +#include "storage/index/snii/format/norms_pod.h" +#include "storage/index/snii/reader/logical_index_reader.h" + +// SniiStatsProvider -- exposes the native SNII scoring statistics required by +// BM25, sourced directly from the on-disk structures of one logical index: +// - semantic segment-level counts from CommonGrams scoring metadata. +// - per-term df / ttf from the term's DictEntry (resolved through the reader's +// lookup flow). The LogicalIndexWriter stores ttf directly in ttf_delta for +// tier>=T2 entries, so total_term_freq returns entry.ttf_delta. +// - per-doc length normalization byte (encoded_norm) from the norms POD, +// lazily loaded and validated once by LogicalIndexReader, then shared by +// every stats provider for that cached logical index. +// +// avgdl() = sum_total_term_freq / max(1, indexed_doc_count): the average document +// length used by BM25 length normalization. The provider performs no scoring; it +// only surfaces the statistics so query::Bm25Scorer can combine them. +namespace doris::snii::stats { + +class SniiStatsProvider { +public: + SniiStatsProvider() = default; + + // Binds to idx and acquires its shared validated norms view when the index + // carries scoring norms. idx must outlive this provider. Complete CommonGrams + // scoring metadata requires compatible semantic statistics and norms. + static Status open(const reader::LogicalIndexReader* idx, SniiStatsProvider* out); + +#ifdef BE_TEST + // Legacy scorer fixtures predate semantic scoring metadata. Production + // callers must use open() and fail closed when the proof is absent. + static Status open_legacy_for_test(const reader::LogicalIndexReader* idx, + SniiStatsProvider* out); +#endif + + // Segment-level semantic counts persisted by the SNII writer. + uint64_t doc_count() const { return doc_count_; } + uint64_t indexed_doc_count() const { return indexed_doc_count_; } + uint64_t sum_total_term_freq() const { return sum_total_term_freq_; } + + // Average document length: sum_total_term_freq / max(1, indexed_doc_count). + double avgdl() const; + + // Per-term document frequency. Absent term -> *df = 0 (OK status). + Status doc_freq(std::string_view term, uint64_t* df) const; + + // Per-term total term frequency (ttf = df + ttf_delta at tier>=T2). Absent + // term -> *ttf = 0 (OK status). + Status total_term_freq(std::string_view term, uint64_t* ttf) const; + + // 1-byte encoded doc-length norm for docid (raw byte from the norms POD). + // Out-of-range docid -> InvalidArgument; index without norms -> InvalidArgument. + Status encoded_norm(uint32_t docid, uint8_t* out) const; + + bool has_norms() const { return has_norms_; } + +private: + static Status open_impl(const reader::LogicalIndexReader* idx, SniiStatsProvider* out, + bool require_semantic_metadata); + + const reader::LogicalIndexReader* idx_ = nullptr; + uint64_t doc_count_ = 0; + uint64_t indexed_doc_count_ = 0; + uint64_t sum_total_term_freq_ = 0; + bool has_norms_ = false; + // Zero-copy view into the immutable bytes owned by idx_ after open_norms(). + format::NormsPodReader norms_reader_; +}; + +} // namespace doris::snii::stats diff --git a/be/src/storage/index/snii/writer/compact_posting_pool.cpp b/be/src/storage/index/snii/writer/compact_posting_pool.cpp new file mode 100644 index 00000000000000..9cef1a38b9f572 --- /dev/null +++ b/be/src/storage/index/snii/writer/compact_posting_pool.cpp @@ -0,0 +1,120 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/writer/compact_posting_pool.h" + +#include +#include +#include + +namespace doris::snii::writer { + +// Gentle (~1.5x) many-level payload-capacity schedule. Starting at 5 bytes with a +// slow ramp keeps the over-allocated FINAL slice small for the millions of low-df +// terms (the dominant arena-overhead source) while still reaching multi-KiB slices +// for high-df chains in a bounded number of hops (so the per-slice 4-byte forward +// pointer stays a small fraction of a large chain's bytes). +const uint32_t CompactPostingPool::kSliceSizes[kLevelCount] = { + 5, 8, 12, 18, 27, 40, 60, 90, 135, 202, 303, 455, 683, 1024, 1536, 2304}; +const uint8_t CompactPostingPool::kNextLevel[kLevelCount] = {1, 2, 3, 4, 5, 6, 7, 8, + 9, 10, 11, 12, 13, 14, 15, 15}; + +CompactPostingPool::CompactPostingPool() = default; + +uint32_t CompactPostingPool::kSliceSizes_level0() { + return kSliceSizes[0]; +} + +uint32_t CompactPostingPool::kSliceSize_at(int level) { + return kSliceSizes[level]; +} + +uint8_t CompactPostingPool::kNextLevel_at(int level) { + return kNextLevel[level]; +} + +void CompactPostingPool::reset() { + std::vector>().swap(blocks_); + next_offset_ = 0; + payload_bytes_ = 0; +} + +uint32_t CompactPostingPool::alloc_run(uint32_t bytes) { + const uint32_t in_block = next_offset_ & kBlockMask; + // A fresh block is needed when (a) there is no tail block yet, (b) the run does + // not fit in the current tail block's remaining space, or (c) next_offset_ sits + // exactly on a block boundary whose block has not been allocated (a previous run + // that exactly filled the tail leaves next_offset_ == blocks_.size()*kBlockSize, + // so in_block == 0 must NOT be mistaken for an empty fresh block). + const bool tail_exists = (next_offset_ >> kBlockShift) < blocks_.size(); + const bool need_block = !tail_exists || in_block + bytes > kBlockSize; + // Hard invariant (see arena_bytes()): the uint32 offset must never wrap. The spimi + // accumulator force-spills below 4 GiB, but enforce it here too -- in release as + // well as debug -- so any direct user of the pool fails loudly instead of silently + // aliasing block 0. We are a library: throw and let the caller decide how to + // handle it, rather than aborting the process. The run starts either in the + // current tail or at a new block's base; compute that start in 64 bits before the + // uint32 arithmetic can wrap. + const uint64_t run_start = + need_block ? static_cast(blocks_.size()) * kBlockSize : next_offset_; + if (run_start + bytes > UINT32_MAX) { + throw std::overflow_error( + "snii: CompactPostingPool arena exceeded the 4 GiB uint32 offset limit; " + "the caller must spill before this point"); + } + if (need_block) { + blocks_.emplace_back(kBlockSize, 0); + next_offset_ = static_cast((blocks_.size() - 1) * kBlockSize); + } + const uint32_t off = next_offset_; + next_offset_ += bytes; + return off; +} + +uint32_t CompactPostingPool::alloc_slice(int level, uint32_t* slice_end) { + const uint32_t cap = kSliceSizes[level]; + const uint32_t first = alloc_run(cap + kPtrBytes); + *slice_end = first + cap; + // Zero the forward pointer so a not-yet-extended tail slice reads next_head == 0. + std::memset(at(*slice_end), 0, kPtrBytes); + return first; +} + +uint32_t CompactPostingPool::read_ptr(uint32_t slice_end) const { + uint32_t v; + std::memcpy(&v, at(slice_end), sizeof(v)); + return v; +} + +void CompactPostingPool::write_ptr(uint32_t slice_end, uint32_t next_head) { + std::memcpy(at(slice_end), &next_head, sizeof(next_head)); +} + +uint32_t CompactPostingPool::start_chain(SliceWriter* w, uint8_t* level) { + *level = 0; + const uint32_t head = alloc_slice(0, &w->slice_end); + w->cur = head; + return head; +} + +CompactPostingPool::Cursor::Cursor(const CompactPostingPool* pool, uint32_t head, uint64_t budget) + : pool_(pool), cur_(head), level_(0), budget_(budget) { + // The first slice is level 0; its payload region ends kSliceSizes[0] bytes in. + slice_end_ = head + CompactPostingPool::kSliceSizes[0]; +} + +} // namespace doris::snii::writer diff --git a/be/src/storage/index/snii/writer/compact_posting_pool.h b/be/src/storage/index/snii/writer/compact_posting_pool.h new file mode 100644 index 00000000000000..38adf4363b6528 --- /dev/null +++ b/be/src/storage/index/snii/writer/compact_posting_pool.h @@ -0,0 +1,333 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace doris::snii::writer { + +// SEGMENTED BYTE ARENA with per-term SLICED runs (a ByteBlockPool, after Lucene). +// +// WHY: the SPIMI accumulator's bulk memory is the per-term posting bytes. Backing +// each term with its own std::vector pays two taxes that dominate peak +// RSS at scale: (1) geometric-growth doubling slack (~1.17x of the live payload), +// and (2) a 24-32 B vector/struct header per term (hundreds of thousands of +// terms). This pool removes both: all term bytes live in a few large fixed-size +// blocks (so slack is ~one block, amortized to ~1.05x), and a term needs only two +// 32-bit cursors of live state (chain head for reads + write head for appends). +// +// HOW (slices): a term's bytes are not stored contiguously. They live in a chain +// of SLICES of geometrically growing payload capacity (the kSliceSizes schedule: +// 4, 8, 16, ... bytes of payload). Each slice is laid out as +// [ payload bytes ... ][ 4-byte forward pointer ] +// The forward pointer holds the absolute offset of the next slice's first payload +// byte (0 while the slice is still the tail of the chain). When a slice's payload +// region fills, the writer allocates a larger slice, stores its head into the old +// slice's 4 pointer bytes, and keeps appending. A reader walks the chain by +// reading payload bytes until a slice boundary, then following the pointer. +// +// Both writer and reader recompute each slice's capacity from the chain's slice +// INDEX (0, 1, 2, ...) via the deterministic schedule, so neither needs to store +// per-slice sizes. The writer carries the current slice's end offset in its +// SliceWriter handle; the reader recomputes capacities as it advances. +// +// Offsets are GLOBAL absolute byte indices into the logical concatenation of all +// blocks: offset = block_index * kBlockSize + byte_in_block. kBlockSize is a power +// of two, so offset -> (block, byte) is a shift/mask. +class CompactPostingPool { +public: + // Block size (power of two). 32 KiB blocks keep per-block tail waste tiny (it + // matters at the smaller 1M scale where the whole arena is only tens of MiB) and + // bound the outer vector header cost; at the 5M scale a few thousand + // blocks is still cheap. Empirically the lowest peak across both scales. + static constexpr uint32_t kBlockShift = 15; + static constexpr uint32_t kBlockSize = 1u << kBlockShift; // 32 KiB + static constexpr uint32_t kBlockMask = kBlockSize - 1; + + // Per-slice forward-pointer width (absolute uint32 next-slice offset). + static constexpr uint32_t kPtrBytes = 4; + + // Geometric slice payload-capacity schedule and the level transition. Level i + // slices hold kSliceSizes[i] payload bytes; on overflow the chain advances to + // kNextLevel[i] (capping at the largest level). A GENTLE (~1.5x) many-level + // schedule starting small minimizes the over-allocated final slice (the + // dominant arena overhead) while keeping the per-slice forward-pointer count + // bounded for high-df chains. + static constexpr int kLevelCount = 16; + + CompactPostingPool(); + + CompactPostingPool(const CompactPostingPool&) = delete; + CompactPostingPool& operator=(const CompactPostingPool&) = delete; + + // Payload capacity (bytes) of a fresh level-0 slice. Exposed for tests that need + // to fill exactly one slice without hardcoding the schedule. + static uint32_t kSliceSizes_level0(); + + // Payload capacity of the slice at `level`, and the level a chain advances to when + // that slice overflows. Exposed (like kSliceSizes_level0) so tests can simulate the + // arena's bump allocator exactly -- e.g. to construct an EXACT block-boundary fill -- + // without hardcoding the private schedule. `level` must be in [0, kLevelCount). + static uint32_t kSliceSize_at(int level); + static uint8_t kNextLevel_at(int level); + + // Live append handle for one term's chain. POD, 8 bytes: the absolute write + // cursor and the absolute end of the current slice's payload region. The chain's + // current slice LEVEL is kept by the caller (a uint8, packed alongside its other + // flags) so this handle stays 8 bytes -- shaving the per-term accumulator. `head` + // (the chain's first payload offset) is also stored by the CALLER (the read entry + // point); start_chain returns it. + struct SliceWriter { + uint32_t cur = 0; // next byte to write (absolute) + uint32_t slice_end = 0; // one-past-last payload byte of the current slice + }; + + // Begins a fresh chain, initializing `w` to its first (level-0) slice and + // *level to 0, and returns the chain head (absolute first payload offset). + uint32_t start_chain(SliceWriter* w, uint8_t* level); + + // Appends one payload byte to the chain described by `w` / `*level`, growing the + // chain with a new linked slice (and advancing *level) when the current slice's + // payload region is exhausted. + void append_byte(SliceWriter* w, uint8_t* level, uint8_t value); + // Encodes and appends one unsigned LEB128 value. The common case copies all + // encoded bytes into the current slice in one operation; only a slice-boundary + // value enters the cold allocation loop. + void append_varint(SliceWriter* w, uint8_t* level, uint64_t value); + + // Total live payload bytes ever written across all chains (excludes slice + // forward-pointer overhead). Drives the spill-threshold estimate only. + uint64_t payload_bytes() const { return payload_bytes_; } + + // Bytes the arena currently occupies (block_count * kBlockSize). The pool + // addresses bytes with a uint32 offset (next_offset_), so the arena MUST stay + // below 4 GiB or alloc_run wraps and silently aliases block 0. The accumulator + // watches this to force a safety spill before the wrap; alloc_run also enforces it + // directly (throws std::overflow_error on a would-be wrap) so a direct user of the + // pool fails loudly rather than silently corrupting. + // Hard invariant: a single CompactPostingPool never exceeds UINT32_MAX bytes. + uint64_t arena_bytes() const { return static_cast(blocks_.size()) << kBlockShift; } + + // Releases ALL blocks back to the OS. Called after the accumulator is fully + // drained (or before a spill's next fill) so no input-side bytes stay resident. + void reset(); + + // ---- Reader ---------------------------------------------------------------- + // Forward cursor over one term's chain, yielding its payload bytes in write + // order by walking the slice forward pointers. + // + // CONTRACT of the `budget` ctor argument (single, unambiguous meaning): + // `budget` is an UPPER BOUND on the number of bytes this cursor may yield. It + // is NOT required to equal the exact payload length: passing the exact length + // is fine, and so is passing any value >= it (the production caller passes the + // chain's write-head offset, which always bounds the payload from above). The + // cursor is SELF-TERMINATING: once it walks off the last written byte it sees + // the tail slice's zero forward pointer and stops, regardless of how much + // budget remains. So an over-large budget can never make next() read past the + // chain (no aliasing of block 0, no off-chain access) -- the budget is purely a + // secondary cap. has_next() is therefore a reliable "more bytes remain" + // predicate for ANY budget >= the true length: it becomes false at the smaller + // of (budget exhausted, chain tail reached). + class Cursor { + public: + Cursor(const CompactPostingPool* pool, uint32_t head, uint64_t budget); + + // True while the cursor can still yield a REAL payload byte: the budget is not + // spent AND the cursor has not reached the chain tail. It peeks the tail forward + // pointer at a slice boundary so it never reports a phantom trailing byte, making + // has_next()/next() a safe loop for any budget >= the true payload length. + bool has_next() const; + // Yields the next payload byte. Returns 0 (and yields no more) once the chain + // tail is reached or the budget is spent -- never reads past the chain. + uint8_t next(); + // Decodes one unsigned LEB128 value directly from the current slice, following + // a slice pointer only when the encoded value straddles a boundary. + uint64_t read_varint(); + + private: + const CompactPostingPool* pool_; + uint32_t cur_; // absolute read cursor + uint32_t slice_end_; // one-past-last payload byte of the current slice + uint32_t level_; // current slice level + uint64_t budget_; // remaining byte budget (upper bound on bytes to yield) + }; + + // Builds a cursor over the chain at `head`. `budget` is an UPPER BOUND on bytes to + // read (see Cursor's contract): the exact payload length or anything larger. The + // production caller passes the write-head offset, which always bounds the payload + // from above; the cursor self-terminates at the chain tail regardless. + Cursor cursor(uint32_t head, uint64_t budget) const { return Cursor(this, head, budget); } + +private: + static const uint32_t kSliceSizes[kLevelCount]; + static const uint8_t kNextLevel[kLevelCount]; + + uint8_t* at(uint32_t off) { return &blocks_[off >> kBlockShift][off & kBlockMask]; } + const uint8_t* at(uint32_t off) const { return &blocks_[off >> kBlockShift][off & kBlockMask]; } + + // Reads/writes the 4-byte forward pointer at the END of a slice whose payload + // region ends at `slice_end` (pointer occupies [slice_end, slice_end+4)). + uint32_t read_ptr(uint32_t slice_end) const; + void write_ptr(uint32_t slice_end, uint32_t next_head); + + // Reserves `bytes` contiguous bytes from the arena tail (a fresh block if the + // current tail cannot hold them) and returns the first reserved absolute offset. + // `bytes` must be <= kBlockSize. + uint32_t alloc_run(uint32_t bytes); + + // Allocates a slice at `level` (payload region + 4 pointer bytes), zeroes its + // forward pointer, and returns the first payload offset; sets *slice_end. + uint32_t alloc_slice(int level, uint32_t* slice_end); + + std::vector> blocks_; // fixed kBlockSize blocks + uint32_t next_offset_ = 0; // global bump pointer (absolute) into the tail block + uint64_t payload_bytes_ = 0; +}; + +// ---- Inlined per-byte hot paths -------------------------------------------- +// append_byte (one call per encoded payload byte during SPIMI ingest) and +// Cursor::has_next/next (one call per arena byte during finalize drain/merge) +// are the writer's two hottest per-byte loops. Their callers live in a DIFFERENT +// translation unit (spimi_term_buffer.cpp), and the build has no LTO/IPO/unity +// build, so an out-of-line .cpp definition forces a non-inlinable cross-TU call +// (plus stack spill of cur_/slice_end_/budget_) on every single byte. Defining +// the bodies inline HERE lets the caller's TU inline them, keeping the cursor +// state in registers and eliminating the per-byte call/ret. They are placed +// AFTER the class so every private member they touch (at, read_ptr, alloc_slice, +// write_ptr, kSliceSizes, kNextLevel, payload_bytes_) is already declared and +// visible; Cursor is a nested class, so it may use the enclosing pool's privates +// (read_ptr/at) through pool_. The cold slice-overflow helpers (alloc_slice, +// write_ptr) stay out-of-line in the .cpp -- only the per-byte work is inlined. +// Pure code move from the .cpp: behavior and produced bytes are identical. + +inline void CompactPostingPool::append_byte(SliceWriter* w, uint8_t* level, uint8_t value) { + if (w->cur == w->slice_end) { + // Current slice payload region is full: grow the chain with a larger slice and + // record the link in the old slice's trailing pointer bytes. + const uint8_t next_level = kNextLevel[*level]; + uint32_t new_end = 0; + const uint32_t new_head = alloc_slice(next_level, &new_end); + write_ptr(w->slice_end, new_head); + *level = next_level; + w->cur = new_head; + w->slice_end = new_end; + } + *at(w->cur) = value; + ++w->cur; + ++payload_bytes_; +} + +inline void CompactPostingPool::append_varint(SliceWriter* w, uint8_t* level, uint64_t value) { + std::array encoded {}; + size_t size = 0; + do { + uint8_t byte = static_cast(value & 0x7fU); + value >>= 7; + if (value != 0) { + byte |= 0x80U; + } + encoded[size++] = byte; + } while (value != 0); + + size_t copied = 0; + while (copied < size) { + if (w->cur == w->slice_end) { + const uint8_t next_level = kNextLevel[*level]; + uint32_t new_end = 0; + const uint32_t new_head = alloc_slice(next_level, &new_end); + write_ptr(w->slice_end, new_head); + *level = next_level; + w->cur = new_head; + w->slice_end = new_end; + } + const size_t available = w->slice_end - w->cur; + const size_t count = std::min(size - copied, available); + std::memcpy(at(w->cur), encoded.data() + copied, count); + w->cur += static_cast(count); + copied += count; + payload_bytes_ += count; + } +} + +inline bool CompactPostingPool::Cursor::has_next() const { + if (budget_ == 0) { + return false; + } + // At a slice boundary, the chain continues only if the forward pointer is non-zero; + // a zero pointer is the tail marker (offset 0 is never a valid next-slice head). Peek + // it so has_next() never reports a phantom byte that next() would have to fabricate. + if (cur_ == slice_end_) { + return pool_->read_ptr(slice_end_) != 0; + } + return true; +} + +inline uint8_t CompactPostingPool::Cursor::next() { + // Budget guard: the caller's stated upper bound is spent -- yield nothing more. + if (budget_ == 0) { + return 0; + } + if (cur_ == slice_end_) { + // Reached this slice's payload boundary. Follow the forward pointer to the next + // slice -- UNLESS it is zero, which marks the CHAIN TAIL (offset 0 is always the + // pool's very first slice, never a valid *next*-slice head, so a zero pointer is + // unambiguously "no more slices"). Without this tail check, an over-reading caller + // would follow the zero pointer to offset 0 and alias block 0's bytes (or read an + // unallocated block) -- UB. Stopping here makes the cursor self-terminating and + // safe regardless of how large a budget the caller passed. + const uint32_t next_head = pool_->read_ptr(slice_end_); + if (next_head == 0) { + budget_ = 0; // chain exhausted: no further bytes exist + return 0; + } + level_ = CompactPostingPool::kNextLevel[level_]; + cur_ = next_head; + slice_end_ = next_head + CompactPostingPool::kSliceSizes[level_]; + } + const uint8_t v = *pool_->at(cur_); + ++cur_; + --budget_; + return v; +} + +inline uint64_t CompactPostingPool::Cursor::read_varint() { + uint64_t result = 0; + uint32_t shift = 0; + for (;;) { + uint8_t byte; + if (budget_ != 0 && cur_ != slice_end_) { + byte = *pool_->at(cur_); + ++cur_; + --budget_; + } else { + byte = next(); + } + result |= static_cast(byte & 0x7fU) << shift; + if ((byte & 0x80U) == 0) { + return result; + } + shift += 7; + } +} + +} // namespace doris::snii::writer diff --git a/be/src/storage/index/snii/writer/global_memory_limiter.cpp b/be/src/storage/index/snii/writer/global_memory_limiter.cpp new file mode 100644 index 00000000000000..ed8038b03d3b49 --- /dev/null +++ b/be/src/storage/index/snii/writer/global_memory_limiter.cpp @@ -0,0 +1,166 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/writer/global_memory_limiter.h" + +#include +#include +#include + +#include "common/logging.h" + +namespace doris::snii::writer { + +GlobalMemoryLimiter* GlobalMemoryLimiter::instance() { + // Intentionally leaked (never destroyed): buffers un-register from their + // destructors, which may run during static teardown at process exit -- a + // destroyed registry there would be use-after-free. A leaked singleton + // makes the "limiter outlives every attached buffer" contract + // unconditional. + static auto* g_instance = new GlobalMemoryLimiter(); + return g_instance; +} + +void GlobalMemoryLimiter::register_buffer(std::atomic* spill_flag, int64_t resident_bytes, + int64_t arena_bytes) { + std::lock_guard guard(mutex_); + Entry& slot = entries_[spill_flag]; // zero-initialized on first insert + total_bytes_locked_ += resident_bytes - slot.resident; + slot.resident = resident_bytes; + slot.arena = arena_bytes; +} + +void GlobalMemoryLimiter::report(std::atomic* spill_flag, int64_t resident_bytes, + int64_t arena_bytes) { + std::lock_guard guard(mutex_); + auto it = entries_.find(spill_flag); + if (it == entries_.end()) { + // Not registered (or already unregistered): ignore rather than + // resurrect an entry nobody will remove. + return; + } + total_bytes_locked_ += resident_bytes - it->second.resident; + it->second.resident = resident_bytes; + it->second.arena = arena_bytes; + request_spills_locked(); +} + +void GlobalMemoryLimiter::unregister_buffer(std::atomic* spill_flag) { + std::lock_guard guard(mutex_); + auto it = entries_.find(spill_flag); + if (it == entries_.end()) { + return; + } + total_bytes_locked_ -= it->second.resident; + entries_.erase(it); +} + +int64_t GlobalMemoryLimiter::total_bytes() const { + std::lock_guard guard(mutex_); + return total_bytes_locked_; +} + +size_t GlobalMemoryLimiter::registered_count() const { + std::lock_guard guard(mutex_); + return entries_.size(); +} + +bool GlobalMemoryLimiter::budget_degraded() const { + std::lock_guard guard(mutex_); + return degraded_; +} + +void GlobalMemoryLimiter::request_spills_locked() { + const int64_t budget = budget_bytes_.load(std::memory_order_relaxed); + if (budget <= 0 || total_bytes_locked_ <= budget) { + return; + } + // BUDGET SANITY (graceful degradation): the budget bounds SPILLABLE + // memory only -- a forced spill reclaims a buffer's posting arena, never + // its persistent vocab / pair-map structures. When the per-writer budget + // share falls below the minimum useful working set, those persistent + // bytes alone exceed the budget and no amount of forced spilling can get + // under it; flagging would only manufacture a storm of floor-sized runs + // for zero net relief (the conc=16 wikipedia field failure). Log ONCE per + // degradation episode and stop flagging until the ratio recovers (writers + // draining/unregistering, or a raised budget). + const auto writer_count = static_cast(entries_.size()); + const int64_t min_useful = min_useful_budget_per_writer_bytes_.load(std::memory_order_relaxed); + if (writer_count > 0 && min_useful > 0 && budget / writer_count < min_useful) { + if (!degraded_) { + degraded_ = true; + LOG(WARNING) << "SNII global index-writer memory budget cannot be met by spilling: " + << "budget=" << budget << " B across " << writer_count + << " registered writers is below the minimum useful " << min_useful + << " B/writer (persistent per-writer structures dominate; the budget " + << "bounds SPILLABLE memory, not persistent memory). Forced spilling is " + << "suspended until writers drain or the budget " + << "(snii_index_writer_global_memory_bytes) is raised."; + } + return; + } + if (degraded_) { + degraded_ = false; // episode over; a relapse will log once again + LOG(INFO) << "SNII global index-writer memory budget recovered (" << budget << " B / " + << writer_count << " writers); forced spilling resumes."; + } + const int64_t overage = total_bytes_locked_ - budget; + // FORCED-SPILL FLOOR + PER-BUFFER COOLDOWN: only buffers holding at least + // the floor of RECLAIMABLE arena are eligible victims -- flagging a + // smaller arena would cut a tiny run and reclaim next to nothing, and a + // buffer that just honored a forced spill (arena ~0) stays exempt until + // its arena regrows past the floor. Never below one byte: an empty arena + // has nothing to write to a run. + const int64_t victim_floor = + std::max(min_victim_arena_bytes_.load(std::memory_order_relaxed), 1); + // Largest RECLAIMABLE consumers first: victims are ranked by their + // spillable ARENA (what the forced spill frees), not the + // persistent-dominated resident total. n is the live writer count of the + // process (at most a few hundred), and this only runs while over budget, + // so the sort under the mutex is bounded, allocation-light work. + std::vector*>> by_arena; + by_arena.reserve(entries_.size()); + for (const auto& [flag, entry] : entries_) { + if (entry.arena >= victim_floor) { + by_arena.emplace_back(entry.arena, flag); + } + } + std::sort(by_arena.begin(), by_arena.end(), + [](const auto& a, const auto& b) { return a.first > b.first; }); + int64_t covered = 0; + for (const auto& [arena, flag] : by_arena) { + if (covered >= overage) { + break; + } + // An ALREADY-pending flag (set by an earlier report, owner not yet at + // its next token) counts toward the covered sum without a fresh store: + // re-flagging it would be a no-op, and skipping the store avoids + // dirtying the owner's cache line every over-budget report. + if (!flag->load(std::memory_order_relaxed)) { + flag->store(true, std::memory_order_relaxed); + } + // Count the ARENA toward coverage: it is all a forced spill of this + // victim can actually reclaim. When the persistent remainder keeps the + // sum over budget even after every eligible arena is flagged, the loop + // simply flags them all -- each still cuts a >= floor-sized run, and + // the cooldown above keeps any one buffer from being re-victimized + // before it has a floor's worth of arena again. + covered += arena; + } +} + +} // namespace doris::snii::writer diff --git a/be/src/storage/index/snii/writer/global_memory_limiter.h b/be/src/storage/index/snii/writer/global_memory_limiter.h new file mode 100644 index 00000000000000..2f19168fc3a4a8 --- /dev/null +++ b/be/src/storage/index/snii/writer/global_memory_limiter.h @@ -0,0 +1,201 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include +#include +#include +#include + +namespace doris::snii::writer { + +// Process-wide SNII build-RAM limiter (G09) -- the index-build analogue of +// Doris's MemTableMemoryLimiter. Every live SPIMI accumulator registers here +// and forwards its ACCURATE resident-byte total (G08) plus its SPILLABLE +// arena bytes through its existing debounced report path; when the resident +// sum across ALL writers of the process (tablets x segments x concurrent +// loads) exceeds the configured budget, the limiter requests spills from the +// largest-ARENA eligible buffers until the flagged (reclaimable) arena sum +// covers the overage. +// +// WHY: the per-writer gate-2 cap (e.g. 512 MiB) bounds ONE writer, but a load +// keeps (tablets x concurrency) writers alive at once -- wikipedia at +// concurrency 16 held 100+ writers at 300-500 MB each (~41 GiB), none of which +// ever reached its own cap, so per-writer spilling never fired. This registry +// bounds the SUM. +// +// ASYNC-SAFE REQUESTS: the SPIMI structures are single-threaded, so the +// limiter must never spill on the reporting thread. A request is a relaxed +// atomic FLAG on the target buffer (SpimiTermBuffer::global_spill_requested_) +// that the OWNER's next add_token / maybe_spill_after_token observes and +// honors on its own thread (bypassing the G08 per-writer anti-churn floor but +// still requiring the FORCED-SPILL FLOOR of reclaimable arena -- see below -- +// so every forced run is worth its fixed costs). Flags are ADVISORY: the +// owner may have just spilled or drained -- the flag is then a (harmless) +// no-op or one extra floor-sized run. The limiter itself only ever takes its +// registry mutex and flips atomics; it never blocks a reporting thread beyond +// that mutex and never calls back into a buffer. +// +// LIFETIME: buffers un-register in their destructor. register / report / +// unregister all serialize on the registry mutex, and flags are only ever set +// UNDER that mutex, so once unregister_buffer returns no thread can touch the +// (about-to-die) flag again. The limiter must outlive every attached buffer +// (trivial for the process singleton; test-local instances are declared before +// the buffers they serve). +// +// THE BUDGET BOUNDS SPILLABLE MEMORY, NOT PERSISTENT MEMORY: a forced spill +// releases only the buffer's posting ARENA; the persistent vocab / pair-map / +// slot structures (~100-500 MB per wikipedia writer) survive it. If the +// persistent bytes of all writers alone exceed the budget, NO amount of +// spilling can reach the budget -- the budget is a back-pressure valve over +// the reclaimable arenas, not a hard cap on resident RSS. Three defenses keep +// an unreachable budget from degenerating into a forced-spill storm (the +// conc=16 wikipedia field failure: every report re-flagged every buffer, each +// honoring with one 32 KiB arena block -> thousands of tiny runs per buffer -> +// EMFILE re-opening them for the k-way merge -> failed loads): +// * VICTIMS BY ARENA: victims are selected by their reported SPILLABLE arena +// bytes -- the only bytes a forced spill can actually reclaim -- never by +// the persistent-dominated resident total, and only buffers whose arena is +// at least min_victim_arena_bytes (config snii_forced_spill_min_arena_bytes) +// are eligible. Every forced run is therefore at least floor-sized. +// * PER-BUFFER COOLDOWN: right after a buffer honors a forced spill its +// arena is ~0, below the victim floor, so it is EXEMPT from new flags +// until the arena regrows past the floor. No timer state: the eligibility +// rule IS the cooldown. +// * BUDGET SANITY: when budget / registered_count falls below a per-writer +// minimum useful share (kMinUsefulBudgetPerWriterBytes), persistent +// structures alone dominate and the budget is provably unreachable by +// spilling; the limiter LOGS ONCE (per degradation episode) and stops +// flagging until the ratio recovers (writers finishing / budget raised). +class GlobalMemoryLimiter { +public: + // Victim floor default (mirrors config snii_forced_spill_min_arena_bytes): + // a buffer is only ever asked to force-spill once its reclaimable arena + // holds at least this much, so no forced run is smaller than this. + static constexpr int64_t kDefaultMinVictimArenaBytes = 64LL << 20; // 64 MiB + // Minimum useful per-writer budget share: below budget/registered_count == + // this, spilling cannot meet the budget (persistent per-writer structures + // alone exceed the share) -- the limiter degrades to log-once-and-stop. + static constexpr int64_t kMinUsefulBudgetPerWriterBytes = 96LL << 20; // 96 MiB + + // Local instances are constructible for unit tests; production code uses + // the process singleton below. + GlobalMemoryLimiter() = default; + GlobalMemoryLimiter(const GlobalMemoryLimiter&) = delete; + GlobalMemoryLimiter& operator=(const GlobalMemoryLimiter&) = delete; + + // Process singleton (never destroyed before the writers that use it). + static GlobalMemoryLimiter* instance(); + + // Budget in bytes across every registered buffer; <= 0 disables the + // limiter (registration still tracked, but no flag is ever set). Refreshed + // from the mutable BE config at each writer init, so a config change takes + // effect for the whole registry at the next writer creation. + void set_budget_bytes(int64_t budget_bytes) { + budget_bytes_.store(budget_bytes, std::memory_order_relaxed); + } + int64_t budget_bytes() const { return budget_bytes_.load(std::memory_order_relaxed); } + + // Victim-eligibility floor over a buffer's reported SPILLABLE arena bytes + // (see the class comment). Refreshed from the mutable BE config alongside + // the budget at each writer init. Values < 1 behave as 1 (an empty arena + // is never a victim -- there would be nothing to write to the run). + void set_min_victim_arena_bytes(int64_t bytes) { + min_victim_arena_bytes_.store(bytes, std::memory_order_relaxed); + } + int64_t min_victim_arena_bytes() const { + return min_victim_arena_bytes_.load(std::memory_order_relaxed); + } + + // Per-writer minimum useful budget share for the degradation check. + // Production keeps the default (kMinUsefulBudgetPerWriterBytes); tests + // lower it to exercise selection with synthetic byte scales. + void set_min_useful_budget_per_writer_bytes(int64_t bytes) { + min_useful_budget_per_writer_bytes_.store(bytes, std::memory_order_relaxed); + } + + // True while the limiter is in the degraded log-once-and-stop state (the + // per-writer budget share fell below the useful minimum at the last + // over-budget report). Observability / tests. + bool budget_degraded() const; + + // Adds `spill_flag` (the owning buffer's advisory request flag; also the + // entry's identity) with its current resident bytes and its SPILLABLE + // arena bytes (<= resident; what a forced spill can reclaim, the victim + // selection key). Re-registering an already-registered flag just updates + // its bytes. Never sets flags itself: a single registration cannot create + // NEW overage worth reacting to before the buffer's first report. + void register_buffer(std::atomic* spill_flag, int64_t resident_bytes, + int64_t arena_bytes); + + // Updates the entry's resident and spillable-arena bytes (ABSOLUTE totals, + // not deltas -- self-healing across any missed report). When the + // registered RESIDENT sum exceeds the budget, sets the request flags of + // the largest-ARENA eligible entries (arena >= the victim floor; see the + // class comment) -- counting entries whose flag is ALREADY pending toward + // the covered sum, so an in-flight request is not amplified -- until the + // flagged ARENA bytes cover the overage or eligible entries run out. A + // report for a flag that is not registered is ignored. + void report(std::atomic* spill_flag, int64_t resident_bytes, int64_t arena_bytes); + + // Removes the entry (subtracting its bytes from the total). After this + // returns, the limiter never touches `spill_flag` again -- safe to destroy + // the owning buffer. + void unregister_buffer(std::atomic* spill_flag); + + // Registered sum / entry count, for tests and observability. + int64_t total_bytes() const; + size_t registered_count() const; + +private: + // One registered buffer's last reported byte totals. `resident` feeds the + // over-budget decision (the sum the budget bounds); `arena` -- the + // SPILLABLE subset a forced spill can actually reclaim -- feeds victim + // selection and eligibility. + struct Entry { + int64_t resident = 0; + int64_t arena = 0; + }; + + // Called with mutex_ held whenever the total may exceed the budget: after + // the budget-sanity check (degrade to log-once-and-stop when the + // per-writer share is below the useful minimum), sorts the ELIGIBLE + // entries (arena >= victim floor) by ARENA descending and flags from the + // top until the flagged arena sum covers (total - budget) or eligible + // entries run out. O(n log n) over the live writer count (at most a few + // hundred) -- bounded work under the mutex, no I/O, no callbacks. + void request_spills_locked(); + + mutable std::mutex mutex_; + std::atomic budget_bytes_ {0}; + std::atomic min_victim_arena_bytes_ {kDefaultMinVictimArenaBytes}; + std::atomic min_useful_budget_per_writer_bytes_ {kMinUsefulBudgetPerWriterBytes}; + // All below guarded by mutex_. total_bytes_locked_ is the maintained sum + // of entries_ resident values (avoids an O(n) walk per report). + int64_t total_bytes_locked_ = 0; + // Degradation episode latch: set (with ONE warning log) when the budget + // sanity check fails, cleared when a later over-budget report finds the + // ratio recovered -- so a relapse logs again, but a sustained episode + // logs exactly once. + bool degraded_ = false; + phmap::flat_hash_map*, Entry> entries_; +}; + +} // namespace doris::snii::writer diff --git a/be/src/storage/index/snii/writer/logical_index_writer.cpp b/be/src/storage/index/snii/writer/logical_index_writer.cpp new file mode 100644 index 00000000000000..63cd04bd3914c9 --- /dev/null +++ b/be/src/storage/index/snii/writer/logical_index_writer.cpp @@ -0,0 +1,952 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/writer/logical_index_writer.h" + +#include +#include +#include +#include +#include +#include + +#include "storage/index/inverted/common_grams/common_grams_key_codec.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/encoding/crc32c.h" +#include "storage/index/snii/encoding/varint.h" +#include "storage/index/snii/encoding/zstd_codec.h" +#include "storage/index/snii/format/bsbf.h" +#include "storage/index/snii/format/dict_block.h" +#include "storage/index/snii/format/dict_block_directory.h" +#include "storage/index/snii/format/frq_pod.h" +#include "storage/index/snii/format/norms_pod.h" +#include "storage/index/snii/format/null_bitmap.h" +#include "storage/index/snii/format/prx_pod.h" +#include "storage/index/snii/writer/posting_window_emitter.h" + +namespace doris::snii::writer { + +using format::BlockRef; +using format::DictBlockBuilder; +using format::DictBlockDirectoryBuilder; +using format::DictEntry; +using format::DictEntryEnc; +using format::DictEntryKind; +using format::SampledTermIndexBuilder; +using format::SectionRefs; +using segment_v2::inverted_index::CG_V1_MARKER; +using segment_v2::inverted_index::CommonGramsCoverage; +using segment_v2::inverted_index::ScoringCoverage; +using segment_v2::inverted_index::validate_common_grams_segment_metadata; + +namespace { + +// Target false-positive probability for the block-split bloom XFilter. Sizes +// the filter via Parquet OptimalNumOfBytes; L0 keeps the probe in memory and L1 +// keeps the per-query cost at one 32-byte block. +constexpr double kBsbfFpp = 0.01; +// Force-raw level for .frq dd/freq regions. Their plaintext is PFOR-bit-packed +// doc-deltas/freqs -- already high-entropy, so zstd shrinks ~30 MB of input by +// <0.1 MiB while burning ~0.4s CPU (and an extra crc pass over the compressed +// bytes) at 5M. We force raw here and keep zstd only on .prx (which compresses +// ~77%). Output stays self-describing: the region meta records zstd=false. +constexpr int kRawFrqRegion = 0; +// zstd level for whole-DICT-block compression comes from +// SniiIndexInput::dict_block_zstd_level (default 3: ~40% on the 64KiB +// front-coded blocks at ~120 MiB/s encode / ~600 MiB/s decode; higher levels +// trade import CPU for size, decode speed unchanged). G16-h made it (and the +// .prx auto level) caller-tunable. + +using format::FrqRegionMeta; + +// Fused single-pass term-level freq statistics: total_freq (running sum) and +// max_freq (running max) in ONE scan, reused by validate_term (has_prx +// position-count budget), stats_.sum_total_term_freq, and the DictEntry +// ttf_delta/max_freq. Byte-identical to the former separate SumOf/MaxOf scans: +// same left-to-right accumulation order and the same max init of 0, so a freq of +// 0 never lowers the max. Complete CommonGrams entries bypass this helper: +// their ttf is the already-known PRX position count and max_freq is not stored. +FreqStats fuse_freq_stats(const std::vector& freqs) { +#ifdef BE_TEST + testing::note_term_freq_scan(); +#endif + FreqStats fs; + for (uint32_t f : freqs) { + fs.total_freq += f; + fs.max_freq = std::max(f, fs.max_freq); + } + return fs; +} + +// Default window doc count by df: high-df windowed terms combine kFrqBaseUnit +// units into larger (kAdaptiveWindowDocs) windows. PRX limits may subsequently +// recut one of these default windows at document boundaries. +uint32_t adaptive_window_docs(uint32_t df) { + return df >= format::kAdaptiveWindowDfThreshold ? format::kAdaptiveWindowDocs + : format::kFrqBaseUnit; +} + +bool fits_prx_window_shape(uint64_t doc_count, uint64_t position_count, + const format::PrxWindowLimits& limits) { + return doc_count <= limits.max_docs && position_count <= limits.max_positions; +} + +} // namespace + +// The only encoder for TermPostingSource input. It borrows the writer's reusable +// posting buffer, streams PRX windows directly to the final sink, and stages grouped DD +// and frequency regions without retaining the complete term. +class StreamingTermEncoder { +public: + StreamingTermEncoder(LogicalIndexWriter* writer, StreamedTermPostings* postings, + bool declared_common_gram, bool term_has_freq, bool term_has_prx, + TermPostingBuffer* buffer, uint64_t frq_base, uint64_t prx_base) + : writer_(writer), + postings_(postings), + declared_common_gram_(declared_common_gram), + term_has_freq_(term_has_freq), + term_has_prx_(term_has_prx), + buffer_(buffer), + frq_base_(frq_base), + prx_base_(prx_base), + emitter_(WindowEmitterOptions { + .posting_out = writer->posting_out_, + .posting_region_offset = writer->posting_off0_, + .frq_base = frq_base, + .prx_base = prx_base, + .encoded_norms = writer->encoded_norms_, + .has_freq = term_has_freq, + .has_prx = term_has_prx, + .prx_zstd_level = writer->prx_zstd_level_, + .prx_window_limits = writer->prx_window_limits_, + .term_frequency_source = + declared_common_gram ? (postings->retain_positions + ? TermFrequencySource::kPositions + : TermFrequencySource::kDocuments) + : TermFrequencySource::kFrequenciesOrDocuments, + .memory_reporter = writer->memory_reporter_, + }) { + DCHECK(buffer_ != nullptr); + DCHECK(buffer_->empty()); + } + + ~StreamingTermEncoder() { + buffer_->clear_reuse_and_release_excess(format::kAdaptiveWindowDfThreshold); + } + + Status encode(DictEntry* entry, FreqStats* stats) { + if (postings_->source == nullptr) { + return Status::Error( + "logical_index: streamed term has a null posting source"); + } + if (writer_->has_prx_ && !postings_->retain_positions && !declared_common_gram_) { + return Status::Error( + "logical_index: only a declared CommonGrams term may omit positions"); + } +#ifdef BE_TEST + if (!declared_common_gram_) { + testing::note_term_freq_scan(); + } +#endif + bool exhausted = false; + RETURN_IF_ERROR(fill(format::kAdaptiveWindowDfThreshold, &exhausted)); + if (exhausted && buffer_->document_count() < format::kAdaptiveWindowDfThreshold) { + entry->term = std::move(postings_->term); + entry->df = total_docs_; + entry->ttf_delta = stats_.total_freq; + entry->max_freq = stats_.max_freq; + RETURN_IF_ERROR(encode_small(entry)); + *stats = stats_; + return Status::OK(); + } + + if (!buffer_->empty()) { + RETURN_IF_ERROR(encode_windowed_buffer(format::kAdaptiveWindowDocs)); + } + while (!exhausted) { + RETURN_IF_ERROR(fill(format::kAdaptiveWindowDocs, &exhausted)); + if (!buffer_->empty()) { + RETURN_IF_ERROR(encode_windowed_buffer(format::kAdaptiveWindowDocs)); + } + } + + entry->term = std::move(postings_->term); + entry->df = total_docs_; + entry->ttf_delta = stats_.total_freq; + entry->max_freq = stats_.max_freq; + RETURN_IF_ERROR(finish_windowed(entry)); + *stats = stats_; + return Status::OK(); + } + +private: + Status fill(uint32_t target_docs, bool* exhausted) { + buffer_->clear_reuse(); + position_offsets_.clear(); + *exhausted = false; + RETURN_IF_ERROR(postings_->source->fill(target_docs, buffer_, exhausted)); + const size_t count = buffer_->document_count(); + if (count > target_docs) { + return Status::Error( + "logical_index: posting source exceeded target_docs"); + } + if (!*exhausted && count != target_docs) { + return Status::Error( + "logical_index: posting source returned a short non-terminal fill"); + } + if (count == 0 && !*exhausted) { + return Status::Error( + "logical_index: posting source returned empty before EOF"); + } + return validate_and_accumulate(); + } + + Status validate_and_accumulate() { + const auto docids = buffer_->docids(); + const auto freqs = buffer_->freqs(); + const auto positions = buffer_->positions_flat(); + if (postings_->retain_positions && freqs.size() != docids.size()) { + return Status::Error( + "logical_index: positioned source must provide one freq per docid"); + } + if (!postings_->retain_positions && !freqs.empty() && freqs.size() != docids.size()) { + return Status::Error( + "logical_index: docs-only source freqs must be empty or parallel"); + } + if (postings_->retain_positions && + positions.size() > std::numeric_limits::max()) { + return Status::Error( + "logical_index: one source fill exceeds uint32 position offsets"); + } + if (postings_->retain_positions) { + position_offsets_.resize(freqs.size() + 1); + } + // One fused pass serves both the positions-count validation and the + // frequency statistics below. fill() has already capped the buffer at + // target_docs (at most the adaptive window sizes), so a uint64 sum of + // uint32 frequencies cannot overflow within one fill; the per-term + // accumulation below keeps its overflow check. + uint64_t fill_freq_sum = 0; + uint32_t fill_max_freq = 0; + for (size_t doc = 0; doc < freqs.size(); ++doc) { + const uint32_t freq = freqs[doc]; + fill_freq_sum += freq; + fill_max_freq = std::max(fill_max_freq, freq); + if (postings_->retain_positions) { + position_offsets_[doc + 1] = static_cast(fill_freq_sum); + } + } + if (postings_->retain_positions) { + if (fill_freq_sum != positions.size()) { + return Status::Error( + "logical_index: source positions count must equal sum(freqs)"); + } + } else { + if (!positions.empty()) { + return Status::Error( + "logical_index: docs-only source must not provide positions"); + } + } + + for (uint32_t docid : docids) { + if (last_docid_.has_value() && docid <= *last_docid_) { + return Status::Error( + "logical_index: source docids must be strictly ascending"); + } + if (docid >= writer_->doc_count_) { + return Status::Error( + "logical_index: source docid must be less than doc_count"); + } + last_docid_ = docid; + } + if (docids.size() > std::numeric_limits::max() - total_docs_) { + return Status::Error( + "logical_index: source document count exceeds uint32"); + } + total_docs_ += static_cast(docids.size()); + + if (declared_common_gram_) { + const uint64_t increment = + postings_->retain_positions ? positions.size() : docids.size(); + if (increment > std::numeric_limits::max() - stats_.total_freq) { + return Status::Error( + "logical_index: source total frequency overflow"); + } + stats_.total_freq += increment; + } else if (freqs.empty()) { + if (docids.size() > std::numeric_limits::max() - stats_.total_freq) { + return Status::Error( + "logical_index: source total frequency overflow"); + } + stats_.total_freq += docids.size(); + } else { + if (fill_freq_sum > std::numeric_limits::max() - stats_.total_freq) { + return Status::Error( + "logical_index: source total frequency overflow"); + } + stats_.total_freq += fill_freq_sum; + stats_.max_freq = std::max(stats_.max_freq, fill_max_freq); + } + return Status::OK(); + } + + Status encode_small(DictEntry* entry) { + if (total_docs_ >= format::kSlimDfThreshold || + (term_has_prx_ && + !fits_prx_window_shape(total_docs_, stats_.total_freq, writer_->prx_window_limits_))) { + RETURN_IF_ERROR(encode_windowed_buffer(adaptive_window_docs(total_docs_))); + return finish_windowed(entry); + } + + std::vector prx_window; + if (term_has_prx_) { + ByteSink sink; + format::PrxWindowBuildOutcome outcome = format::PrxWindowBuildOutcome::kBuilt; + RETURN_IF_ERROR(format::try_build_prx_window_flat( + buffer_->positions_flat(), buffer_->freqs(), -writer_->prx_zstd_level_, + writer_->prx_window_limits_, &sink, &outcome)); + if (outcome == format::PrxWindowBuildOutcome::kNeedsSplit) { + RETURN_IF_ERROR(encode_windowed_buffer(format::kFrqBaseUnit)); + return finish_windowed(entry); + } + prx_window = sink.take(); + } + + ByteSink frq_sink; + FrqRegionMeta dd_meta; + FrqRegionMeta freq_meta {}; + RETURN_IF_ERROR(format::build_dd_region(buffer_->docids(), /*win_base=*/0, kRawFrqRegion, + &frq_sink, &dd_meta)); + if (term_has_freq_) { + RETURN_IF_ERROR(format::build_freq_region(buffer_->freqs(), kRawFrqRegion, &frq_sink, + &freq_meta)); + } + std::vector frq_window = frq_sink.take(); + entry->enc = DictEntryEnc::kSlim; + entry->dd_meta = dd_meta; + entry->freq_meta = freq_meta; + if (frq_window.size() <= format::kDefaultInlineThreshold) { + entry->kind = DictEntryKind::kInline; + entry->inline_dd_disk_len = dd_meta.disk_len; + entry->frq_bytes = std::move(frq_window); + if (term_has_prx_) entry->prx_bytes = std::move(prx_window); + return Status::OK(); + } + + entry->kind = DictEntryKind::kPodRef; + entry->frq_docs_len = dd_meta.disk_len; + if (term_has_prx_) { + const uint64_t prx_off = writer_->posting_size(); + RETURN_IF_ERROR(writer_->posting_out_->append(Slice(prx_window))); + entry->prx_off_delta = prx_off - prx_base_; + entry->prx_len = writer_->posting_size() - prx_off; + } + const uint64_t frq_off = writer_->posting_size(); + RETURN_IF_ERROR(writer_->posting_out_->append(Slice(frq_window))); + entry->frq_off_delta = frq_off - frq_base_; + entry->frq_len = writer_->posting_size() - frq_off; + return Status::OK(); + } + + Status encode_windowed_buffer(uint32_t unit) { + DCHECK_GT(unit, 0); + for (size_t begin = 0; begin < buffer_->document_count(); begin += unit) { + const size_t count = + std::min(buffer_->document_count() - begin, static_cast(unit)); + const auto offsets = + term_has_prx_ + ? std::span(position_offsets_).subspan(begin, count + 1) + : std::span {}; + const auto positions = + term_has_prx_ ? buffer_->positions_flat().subspan( + offsets.front(), + static_cast(offsets.back() - offsets.front())) + : std::span {}; + RETURN_IF_ERROR(emitter_.emit_window(PostingRunView { + .docids = buffer_->docids().subspan(begin, count), + .freqs = buffer_->freqs().empty() ? std::span {} + : buffer_->freqs().subspan(begin, count), + .position_offsets = offsets, + .positions_flat = positions, + })); + } + return Status::OK(); + } + + Status finish_windowed(DictEntry* entry) { + TermAggregateStats emitted_stats; + RETURN_IF_ERROR(emitter_.finish_term(entry, &emitted_stats)); + DCHECK_EQ(emitted_stats.df, total_docs_); + DCHECK_EQ(emitted_stats.total_freq, stats_.total_freq); + DCHECK_EQ(emitted_stats.max_freq, stats_.max_freq); + return Status::OK(); + } + + LogicalIndexWriter* writer_; + StreamedTermPostings* postings_; + bool declared_common_gram_ = false; + bool term_has_freq_ = false; + bool term_has_prx_ = false; + TermPostingBuffer* buffer_ = nullptr; + uint64_t frq_base_ = 0; + uint64_t prx_base_ = 0; + WindowEmitter emitter_; + std::vector position_offsets_; + FreqStats stats_; + std::optional last_docid_; + uint32_t total_docs_ = 0; +}; + +namespace testing { +#ifdef BE_TEST +namespace { +// Function-local-static op-count seam backing term_freq_scans(). One atomic, +// relaxed: the writer build path is single-threaded, so only the COUNT matters, +// not ordering (the atomic keeps it race-clean if a test ever parallelizes). +std::atomic& term_freq_scan_counter() { + static std::atomic counter {0}; + return counter; +} +} // namespace +#endif + +void note_term_freq_scan() { +#ifdef BE_TEST + term_freq_scan_counter().fetch_add(1, std::memory_order_relaxed); +#endif +} +uint64_t term_freq_scans() { +#ifdef BE_TEST + return term_freq_scan_counter().load(std::memory_order_relaxed); +#else + return 0; +#endif +} +void reset_term_freq_scans() { +#ifdef BE_TEST + term_freq_scan_counter().store(0, std::memory_order_relaxed); +#endif +} + +// Forwards to the real fused helper so pure boundary tests exercise production +// code (not a test-local re-implementation). +FreqStats fuse_freq_stats_for_test(const std::vector& freqs) { + return fuse_freq_stats(freqs); +} +} // namespace testing + +LogicalIndexWriter::LogicalIndexWriter(const SniiIndexInput& in) + : LogicalIndexWriter(in, TrackedNullDocids(std::vector(in.null_docids))) {} + +LogicalIndexWriter::LogicalIndexWriter(const SniiIndexInput& in, TrackedNullDocids null_docids) + : index_id_(in.index_id), + index_suffix_(in.index_suffix), + index_config_(in.config), + tier_(format::tier_of(in.config)), + has_prx_(format::has_positions(in.config)), + // G16-c: the caller can drop freq layout entirely (in.write_freq == + // false) on a freq-capable tier -- see SniiIndexInput::write_freq. + has_freq_(format::tier_of(in.config) >= format::IndexTier::kT2 && in.write_freq), + has_norms_(format::has_scoring(in.config)), + doc_count_(in.doc_count), + null_docids_(std::move(null_docids)), + terms_(in.terms), + term_source_(in.term_source), + encoded_norms_(in.encoded_norms), + common_grams_metadata_(in.common_grams_metadata), + common_grams_posting_policy_(in.common_grams_posting_policy), + target_dict_block_bytes_(in.target_dict_block_bytes != 0 + ? in.target_dict_block_bytes + : format::kDefaultTargetDictBlockBytes), + dict_block_zstd_level_(in.dict_block_zstd_level), + prx_zstd_level_(in.prx_zstd_level), + prx_window_limits_(in.prx_window_limits), + memory_reporter_(in.mem_reporter), + dict_buf_(in.dict_resident_cap_bytes, "dict", in.mem_reporter), + norms_section_reservation_(in.mem_reporter == nullptr + ? MemoryReporter::Reservation() + : in.mem_reporter->make_reservation()), + null_bitmap_section_reservation_(in.mem_reporter == nullptr + ? MemoryReporter::Reservation() + : in.mem_reporter->make_reservation()), + term_hashes_reservation_(in.mem_reporter == nullptr + ? MemoryReporter::Reservation() + : in.mem_reporter->make_reservation()), + bsbf_bytes_reservation_(in.mem_reporter == nullptr + ? MemoryReporter::Reservation() + : in.mem_reporter->make_reservation()) {} + +Status LogicalIndexWriter::reserve_term_hash_for_append() { + if (memory_reporter_ == nullptr || term_hashes_.size() < term_hashes_.capacity()) { + return Status::OK(); + } + size_t target_capacity = term_hashes_.capacity() == 0 ? 1 : term_hashes_.capacity() * 2; + if (target_capacity < term_hashes_.capacity() || + target_capacity > std::numeric_limits::max() / sizeof(uint64_t)) { + return Status::Error( + "logical_index: term hash capacity overflow"); + } + MemoryReporter::Reservation replacement; + RETURN_IF_ERROR(term_hashes_reservation_.prepare_replacement(target_capacity * sizeof(uint64_t), + &replacement)); + term_hashes_.reserve(target_capacity); + DCHECK_EQ(term_hashes_.capacity(), target_capacity); + term_hashes_reservation_ = std::move(replacement); + return Status::OK(); +} + +// Serializes the current open block, zstd-compresses it (the dict region is the +// single largest section -- term keys + entry meta + inline postings -- and the +// 64KiB blocks compress ~40%), streams the compressed bytes into the dict +// scratch file, and records a directory entry. The block-level crc32c +// (rec.checksum) covers the UNCOMPRESSED bytes, so DictBlockReader::open +// verifies integrity after the reader decompresses. A compressed block also +// shrinks the bytes a term lookup fetches from S3 -- aligning with the +// read-byte thesis. If zstd does not shrink a (tiny) block, it is stored raw so +// a lookup never pays a pointless decompress. +Status LogicalIndexWriter::flush_block(DictBlockBuilder* block, std::string first_term) { + std::vector plain_bytes = block->finish_owned(); + const Slice plain(plain_bytes); + BlockRecord rec; + rec.rel_offset = dict_buf_.size(); + rec.n_entries = block->n_entries(); + rec.checksum = crc32c(plain); // crc over UNCOMPRESSED block bytes + rec.first_term = std::move(first_term); + + std::vector comp; + Status zs = zstd_compress(plain, dict_block_zstd_level_, &comp); + if (zs.ok() && comp.size() < plain.size()) { + rec.flags = format::block_ref_flags::kZstd; + rec.uncomp_len = static_cast(plain.size()); + rec.length = static_cast(comp.size()); + RETURN_IF_ERROR(dict_buf_.append_move(std::move(comp))); + } else { + rec.flags = 0; + rec.uncomp_len = 0; + rec.length = static_cast(plain.size()); + RETURN_IF_ERROR(dict_buf_.append_move(std::move(plain_bytes))); + } + blocks_.push_back(std::move(rec)); + return Status::OK(); +} + +// Running state for the in-flight DICT block while terms stream past. +struct LogicalIndexWriter::BlockState { + explicit BlockState(MemoryReporter* memory_reporter) : transfer_buffer(memory_reporter) {} + + std::unique_ptr block; + std::string block_first_term; + uint64_t frq_base = 0; + uint64_t prx_base = 0; + bool term_stats = true; + TermPostingBuffer transfer_buffer; +}; + +// Out-of-line so unique_ptr sees the complete type (see header). +LogicalIndexWriter::~LogicalIndexWriter() = default; + +Status LogicalIndexWriter::process_term(StreamedTermPostings& tp, BlockState* st) { + const bool is_declared_common_gram = + common_grams_metadata_.has_value() && tp.term.starts_with(CG_V1_MARKER) && + (common_grams_metadata_->common_grams_coverage == CommonGramsCoverage::kComplete || + common_grams_posting_policy_ == format::CommonGramsPostingPolicy::kHybridV1); + const bool term_has_prx = has_prx_ && tp.retain_positions; + const bool term_has_freq = has_freq_ && !is_declared_common_gram; + + if (st->block && st->term_stats != term_has_freq) { + RETURN_IF_ERROR(flush_block(st->block.get(), st->block_first_term)); + st->block.reset(); + } + if (!st->block) { + const uint64_t base = posting_size(); + st->frq_base = base; + st->prx_base = base; + st->term_stats = term_has_freq; + st->block = std::make_unique(tier_, has_prx_, st->frq_base, st->prx_base, + /*anchor_interval=*/16, + /*term_stats=*/term_has_freq); + st->block_first_term = tp.term; + } + + RETURN_IF_ERROR(reserve_term_hash_for_append()); + const uint64_t term_hash = format::bsbf_hash(tp.term); + DictEntry entry; + FreqStats stats; + StreamingTermEncoder encoder(this, &tp, is_declared_common_gram, term_has_freq, term_has_prx, + &st->transfer_buffer, st->frq_base, st->prx_base); + RETURN_IF_ERROR(encoder.encode(&entry, &stats)); + + term_hashes_.push_back(term_hash); + ++term_count_; + stats_.sum_total_term_freq += stats.total_freq; + st->block->add_entry(std::move(entry)); + if (st->block->estimated_bytes() >= target_dict_block_bytes_) { + RETURN_IF_ERROR(flush_block(st->block.get(), st->block_first_term)); + st->block.reset(); + } + return Status::OK(); +} + +Status LogicalIndexWriter::build_blocks() { + BlockState st(memory_reporter_); + if (term_source_ != nullptr) { + RETURN_IF_ERROR(term_source_->for_each_term_sorted( + [&](StreamedTermPostings&& tp) { return process_term(tp, &st); })); + } else { + for (const auto& tp : terms_) { + SpanTermPostingSource source(tp.docids, tp.freqs, tp.positions_flat); + StreamedTermPostings streamed { + .term = tp.term, .retain_positions = tp.retain_positions, .source = &source}; + RETURN_IF_ERROR(process_term(streamed, &st)); + } + } + if (st.block) RETURN_IF_ERROR(flush_block(st.block.get(), st.block_first_term)); + return Status::OK(); +} + +Status LogicalIndexWriter::prepare_build(io::FileWriter* posting_out) { + if (posting_out == nullptr) { + return Status::Error( + "logical_index: null posting sink"); + } + RETURN_IF_ERROR(format::validate_prx_window_limits(prx_window_limits_)); + if (has_norms_ && encoded_norms_.size() != doc_count_) { + return Status::Error( + "logical_index: norms length must equal doc_count"); + } + for (size_t i = 0; i < null_docids_.size(); ++i) { + if (null_docids_[i] >= doc_count_) { + return Status::Error( + "logical_index: null docid must be less than doc_count"); + } + if (i != 0 && null_docids_[i] <= null_docids_[i - 1]) { + return Status::Error( + "logical_index: null docids must be strictly ascending"); + } + } + if (common_grams_metadata_) { + RETURN_IF_ERROR(validate_common_grams_segment_metadata(*common_grams_metadata_)); + if (common_grams_metadata_->common_grams_coverage == CommonGramsCoverage::kComplete && + !has_prx_) { + return Status::Error( + "logical_index: complete CommonGrams metadata requires positions"); + } + if (common_grams_metadata_->scoring_coverage == ScoringCoverage::kComplete) { + if (!has_norms_ || !has_freq_) { + return Status::Error( + "logical_index: complete scoring metadata requires frequencies and " + "semantic norms"); + } + if (common_grams_metadata_->scoring_doc_count != doc_count_) { + return Status::Error( + "logical_index: scoring doc count must equal doc_count"); + } + } + } + if (common_grams_posting_policy_ == format::CommonGramsPostingPolicy::kHybridV1 && + (!common_grams_metadata_ || + common_grams_metadata_->common_grams_coverage != CommonGramsCoverage::kMixed || + !has_prx_)) { + return Status::Error( + "logical_index: hybrid CommonGrams postings require mixed metadata and positions"); + } + // The interleaved posting region streams STRAIGHT into the container output + // (no temp round-trip): posting_size() is the region-relative byte count, + // derived from the output offset advanced since this index's region began. + // The DICT region is staged in dict_buf_ (tiered: RAM under the cap = + // spill-only; spills above it) since it must land contiguously after the + // concurrently-streamed posting region. + posting_out_ = posting_out; + posting_off0_ = posting_out->bytes_written(); + return Status::OK(); +} + +Status LogicalIndexWriter::finalize_build() { + if (common_grams_metadata_ && + common_grams_metadata_->scoring_coverage == ScoringCoverage::kComplete) { + if (common_grams_metadata_->scoring_token_count > stats_.sum_total_term_freq) { + return Status::Error( + "logical_index: semantic scoring token count exceeds physical term frequency"); + } + if (stats_.sum_total_term_freq != 0 && common_grams_metadata_->scoring_token_count == 0) { + return Status::Error( + "logical_index: non-empty physical postings have zero semantic scoring tokens"); + } + if (common_grams_metadata_->plain_term_key_version == + ::doris::segment_v2::inverted_index::PlainTermKeyVersion::kRawNoInternal && + common_grams_metadata_->common_grams_coverage == CommonGramsCoverage::kNone && + common_grams_metadata_->scoring_token_count != stats_.sum_total_term_freq) { + return Status::Error( + "logical_index: semantic plain token count must equal physical term frequency"); + } + } + // Seal the dict buffer so a spilled temp is flushed before + // stream_dict_region_into reads it back. A no-op for a RAM-resident dict. + RETURN_IF_ERROR(dict_buf_.seal()); + + stats_.doc_count = doc_count_; + stats_.indexed_doc_count = doc_count_ - static_cast(null_docids_.size()); + stats_.term_count = term_count_; + stats_.null_count = static_cast(null_docids_.size()); + + if (has_norms_) { + const size_t payload_size = varint_len(encoded_norms_.size()) + encoded_norms_.size(); + const size_t section_size = 1 + varint_len(payload_size) + payload_size + sizeof(uint32_t); + MemoryReporter::Reservation build_reservation = + memory_reporter_ == nullptr ? MemoryReporter::Reservation() + : memory_reporter_->make_reservation(); + if (memory_reporter_ != nullptr) { + RETURN_IF_ERROR(build_reservation.set_bytes(payload_size)); + RETURN_IF_ERROR(norms_section_reservation_.set_bytes(section_size)); + } + ByteSink nsink; + format::NormsPodWriter::finish(encoded_norms_, &nsink); + norms_section_ = nsink.take(); + DORIS_CHECK_EQ(norms_section_.capacity(), section_size); + if (memory_reporter_ != nullptr) { + DORIS_CHECK_EQ(norms_section_reservation_.bytes(), norms_section_.capacity()); + } + } + + if (!null_docids_.empty()) { + MemoryReporter::Reservation bitmap_build_reservation = + memory_reporter_ == nullptr ? MemoryReporter::Reservation() + : memory_reporter_->make_reservation(); + if (memory_reporter_ != nullptr) { + RETURN_IF_ERROR(bitmap_build_reservation.set_bytes( + format::NullBitmapWriter::build_memory_upper_bound( + std::span(null_docids_.data(), null_docids_.size())))); + } + format::NullBitmapWriter null_writer; + null_writer.add_many(std::span(null_docids_.data(), null_docids_.size())); + null_docids_.release(); + + format::NullBitmapSerializationSizes sizes; + RETURN_IF_ERROR(null_writer.serialization_sizes(doc_count_, &sizes)); + if (sizes.roaring_bytes > std::numeric_limits::max() - sizes.payload_bytes) { + return Status::Error( + "logical_index: null bitmap scratch size overflows"); + } + MemoryReporter::Reservation scratch_reservation = + memory_reporter_ == nullptr ? MemoryReporter::Reservation() + : memory_reporter_->make_reservation(); + if (memory_reporter_ != nullptr) { + RETURN_IF_ERROR( + scratch_reservation.set_bytes(sizes.roaring_bytes + sizes.payload_bytes)); + RETURN_IF_ERROR(null_bitmap_section_reservation_.set_bytes(sizes.framed_bytes)); + } + ByteSink null_sink; + RETURN_IF_ERROR(null_writer.finish(doc_count_, &null_sink)); + null_bitmap_section_ = null_sink.take(); + DORIS_CHECK_EQ(null_bitmap_section_.size(), sizes.framed_bytes); + if (memory_reporter_ != nullptr) { + DORIS_CHECK_LE(null_bitmap_section_.capacity(), + null_bitmap_section_reservation_.bytes()); + } + } + null_docids_.release(); + + // Build the absent-term filter (block-split bloom, Parquet-canonical) from + // the per-term keys (no retained strings) as a [28B header][bitset] blob; the + // compound writer places it as a PHYSICAL section probed one 32-byte block on + // demand. + bsbf_bytes_.clear(); + bsbf_built_ = false; + if (!term_hashes_.empty()) { + const uint32_t bitset_bytes = format::bsbf_optimal_num_bytes( + static_cast(term_hashes_.size()), kBsbfFpp); + const size_t serialized_bytes = format::kBsbfHeaderSize + bitset_bytes; + MemoryReporter::Reservation builder_reservation = + memory_reporter_ == nullptr ? MemoryReporter::Reservation() + : memory_reporter_->make_reservation(); + if (memory_reporter_ != nullptr) { + RETURN_IF_ERROR(builder_reservation.set_bytes(bitset_bytes)); + RETURN_IF_ERROR(bsbf_bytes_reservation_.set_bytes(serialized_bytes)); + } + format::BsbfBuilder bf; + RETURN_IF_ERROR(format::BsbfBuilder::create(static_cast(term_hashes_.size()), + kBsbfFpp, &bf)); + DCHECK_EQ(bf.resident_capacity_bytes(), bitset_bytes); + for (uint64_t k : term_hashes_) bf.insert(k); + ByteSink bsink; + bsink.reserve(serialized_bytes); + RETURN_IF_ERROR(bf.serialize(&bsink)); + bsbf_bytes_ = bsink.take(); + DCHECK_EQ(bsbf_bytes_.capacity(), serialized_bytes); + bsbf_built_ = true; + } + std::vector().swap(term_hashes_); // release + term_hashes_reservation_.reset(); + + return Status::OK(); +} + +void LogicalIndexWriter::release_bsbf_bytes() { + std::vector().swap(bsbf_bytes_); + bsbf_bytes_reservation_.reset(); +} + +void LogicalIndexWriter::release_null_bitmap_bytes() { + std::vector().swap(null_bitmap_section_); + null_bitmap_section_reservation_.reset(); +} + +void LogicalIndexWriter::release_norms_bytes() { + std::vector().swap(norms_section_); + norms_section_reservation_.reset(); +} + +Status LogicalIndexWriter::build(io::FileWriter* posting_out) { + // Single-session invariant: a writer that ran (or is running) a streamed + // session must not also build() -- the posting sink anchor and the dict + // buffer are one-shot. + if (stream_phase_ != StreamPhase::kIdle) { + return Status::Error( + "logical_index: build() on a writer with a streamed session"); + } + // prepare_build is pure entry validation up to its final sink-anchor + // assignments, so a failure there leaves the writer clean (still kIdle). + RETURN_IF_ERROR(prepare_build(posting_out)); + // Poison-by-default past this point: build_blocks/finalize_build may fail + // AFTER posting bytes hit the sink or term state advanced, and a dirty + // writer must never accept a later begin_streamed/build (single-session + + // crash-safety invariant 6). Only full success seals to kFinished. + stream_phase_ = StreamPhase::kFailed; + RETURN_IF_ERROR(build_blocks()); + RETURN_IF_ERROR(finalize_build()); + // Claim the (only) session so a later begin_streamed/push_term errors out. + stream_phase_ = StreamPhase::kFinished; + return Status::OK(); +} + +Status LogicalIndexWriter::begin_streamed(io::FileWriter* posting_out) { + if (stream_phase_ == StreamPhase::kFailed) { + return Status::Error( + "logical_index: begin_streamed on a failed writer (a prior session error left " + "partial state; allocate a fresh writer)"); + } + if (stream_phase_ != StreamPhase::kIdle) { + return Status::Error( + "logical_index: begin_streamed on an already-claimed writer session"); + } + RETURN_IF_ERROR(prepare_build(posting_out)); + stream_state_ = std::make_unique(memory_reporter_); + stream_phase_ = StreamPhase::kActive; + return Status::OK(); +} + +Status LogicalIndexWriter::push_term(StreamedTermPostings&& tp) { + if (stream_phase_ != StreamPhase::kActive) { + return Status::Error( + "logical_index: push_term without an active streamed session"); + } + if (has_pushed_term_ && tp.term <= last_pushed_term_) { + return Status::Error( + "logical_index: pushed terms must be strictly increasing ('{}' after '{}')", + tp.term, last_pushed_term_); + } + last_pushed_term_ = tp.term; + has_pushed_term_ = true; + Status status = process_term(tp, stream_state_.get()); + if (!status.ok()) { + stream_phase_ = StreamPhase::kFailed; + stream_state_.reset(); + } + return status; +} + +Status LogicalIndexWriter::finish_streamed() { + if (stream_phase_ == StreamPhase::kFinished) { + return Status::Error( + "logical_index: finish_streamed on an already-finished session"); + } + if (stream_phase_ == StreamPhase::kFailed) { + return Status::Error( + "logical_index: finish_streamed on a failed session (a prior push/finish error " + "poisoned the writer; the partial output must be discarded)"); + } + if (stream_phase_ != StreamPhase::kActive) { + return Status::Error( + "logical_index: finish_streamed without begin_streamed"); + } + // Poison-by-default: a failed trailing flush or finalize leaves partial + // output, so only full success below seals the session to kFinished. + stream_phase_ = StreamPhase::kFailed; + // Trailing-block flush mirrors the tail of build_blocks(); the shared + // finalize_build() then seals the dict buffer and materializes the + // stats/norms/null-bitmap/BSBF sections. + if (stream_state_->block) { + RETURN_IF_ERROR(flush_block(stream_state_->block.get(), stream_state_->block_first_term)); + } + stream_state_.reset(); + RETURN_IF_ERROR(finalize_build()); + stream_phase_ = StreamPhase::kFinished; + return Status::OK(); +} + +Status LogicalIndexWriter::finish_metadata(const SectionRefs& abs_refs, uint64_t dict_region_offset, + SerializedMetadataGroup* out) const { + if (out == nullptr) { + return Status::Error( + "logical_index: null metadata output"); + } + *out = {}; + + SampledTermIndexBuilder sti; + for (const auto& b : blocks_) sti.add_block_first_term(b.first_term); + ByteSink sti_sink; + sti.finish(&sti_sink); + + DictBlockDirectoryBuilder dir; + for (const auto& b : blocks_) { + BlockRef ref; + ref.offset = dict_region_offset + b.rel_offset; + ref.length = b.length; + ref.n_entries = b.n_entries; + ref.flags = b.flags; + ref.checksum = b.checksum; + ref.uncomp_len = b.uncomp_len; + dir.add(ref); + } + ByteSink dir_sink; + dir.finish(&dir_sink); + + ByteSink sti_blob; + RETURN_IF_ERROR( + format::encode_metadata_blob(sti_sink.view(), format::SectionType::kSampledTermIndex, + format::SectionType::kSampledTermIndexZstd, &sti_blob)); + out->sampled_term_index = sti_blob.take(); + + ByteSink dbd_blob; + RETURN_IF_ERROR( + format::encode_metadata_blob(dir_sink.view(), format::SectionType::kDictBlockDirectory, + format::SectionType::kDictBlockDirectoryZstd, &dbd_blob)); + out->dict_block_directory = dbd_blob.take(); + + format::CoreMetadata core; + core.index_config = index_config_; + core.stats = stats_; + core.section_refs = abs_refs; + core.common_grams_metadata = common_grams_metadata_; + core.common_grams_posting_policy = common_grams_posting_policy_; + ByteSink core_sink; + RETURN_IF_ERROR(format::encode_core_metadata(core, &core_sink)); + out->core = core_sink.take(); + return Status::OK(); +} + +} // namespace doris::snii::writer diff --git a/be/src/storage/index/snii/writer/logical_index_writer.h b/be/src/storage/index/snii/writer/logical_index_writer.h new file mode 100644 index 00000000000000..c93c58d8b722ac --- /dev/null +++ b/be/src/storage/index/snii/writer/logical_index_writer.h @@ -0,0 +1,459 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/inverted/common_grams/common_grams_segment_metadata.h" +#include "storage/index/snii/encoding/byte_sink.h" +#include "storage/index/snii/format/core_metadata.h" +#include "storage/index/snii/format/dict_block.h" +#include "storage/index/snii/format/dict_block_directory.h" +#include "storage/index/snii/format/dict_entry.h" +#include "storage/index/snii/format/format_constants.h" +#include "storage/index/snii/format/metadata_blob.h" +#include "storage/index/snii/format/prx_pod.h" +#include "storage/index/snii/format/sampled_term_index.h" +#include "storage/index/snii/format/stats_block.h" +#include "storage/index/snii/io/file_writer.h" +#include "storage/index/snii/writer/memory_reporter.h" +#include "storage/index/snii/writer/spillable_byte_buffer.h" +#include "storage/index/snii/writer/spimi_term_buffer.h" +#include "storage/index/snii/writer/term_posting_source.h" + +// LogicalIndexWriter -- builds the per-logical-index section bytes (interleaved +// posting region + DICT block region) plus the SampledTermIndex and DICT block +// directory metadata for ONE logical index. It owns the in-memory section bytes, +// runtime statistics, and references needed by the container orchestrator +// (SniiCompoundWriter) to resolve absolute offsets and emit the Core/STI/DBD +// metadata group. +// +// This module deliberately produces ONLY relative bytes/structures: it has no +// knowledge of the absolute file position where the sections will land. The +// orchestrator stitches the absolute offsets in afterward (append-only, no +// seek-back). See snii_compound_writer.h for the precise offset contract. +// +// POSTING REGION (single interleaved sink): the former separate .frq POD and .prx +// POD are merged into ONE posting region. For each pod_ref term, in term order, the +// writer appends its prx span FIRST then its frq span, contiguously: +// posting region = concat over pod_ref terms of [prx span][frq span]. +// The prx span is empty when !has_prx (docs-only / keyword tier). INLINE terms +// append NOTHING to the posting region. +// +// Per-term encoding policy (v1): +// df >= kSlimDfThreshold (512), or a lower-df term whose positions cannot fit +// one configured reader-safe PRX window: WINDOWED pod_ref. The term's [prx +// windows] are appended to the posting region first, then its +// [prelude][dd-block][freq-block] frq span. The DictEntry records frq/prx +// off_delta+len relative to frq_base/prx_base (see below). +// Other df < kSlimDfThreshold terms: SLIM. The postings are encoded as a +// single .frq window (and .prx window). If the encoded .frq bytes are small +// (<= kDefaultInlineThreshold), they are stored INLINE inside the DictEntry +// (kind=inline); otherwise the term's [prx][frq] spans are appended to the +// posting region as a slim pod_ref (kind=pod_ref, enc=slim, no prelude). +// +// frq_base / prx_base convention (DOCUMENTED CONTRACT): +// For each DICT block, frq_base == prx_base == the running byte offset into THIS +// index's posting region at the moment the block opens (the posting-region size +// when the block's first POD-backed entry is appended). A windowed/slim pod_ref +// entry then sets frq_off_delta = (offset of its frq span within the posting +// region) - frq_base, so the reader computes the absolute file offset as +// section_refs.posting_region.offset + frq_base + frq_off_delta. +// prx_base / prx_off_delta follow the identical rule against the SAME region. +// Because [prx][frq] are written contiguously per term, a writer-side property +// holds when has_prx: frq_off_delta == prx_off_delta + prx_len. The reader does +// NOT rely on it -- each delta is resolved independently. +// Inline entries carry no off_delta (bytes live in the entry). +namespace doris::snii::writer { + +class SniiStreamedIndexSession; +class StreamingTermEncoder; + +struct SerializedMetadataGroup { + std::vector core; + std::vector sampled_term_index; + std::vector dict_block_directory; +}; + +// Inputs describing one logical index to be written. +struct SniiIndexInput { + uint64_t index_id = 0; + std::string index_suffix; + format::IndexConfig config = format::IndexConfig::kDocsPositions; + uint32_t doc_count = 0; + std::vector null_docids; + // Per-doc 1-byte encoded norm (length doc_count); only consumed when the + // config has scoring. May be empty otherwise. + std::vector encoded_norms; + // G16-h: zstd levels for the dict-block whole-block compression and the + // .prx window auto mode (both default 3 == the historical constants). + // Higher levels trade import CPU for size; decode speed is unaffected. + int dict_block_zstd_level = 3; + int prx_zstd_level = 3; + // Internal writer policy. Production callers keep the reader limits; unit + // tests may only tighten them to exercise extreme-window behavior without + // allocating hundreds of MiB. + format::PrxWindowLimits prx_window_limits = format::kReaderPrxWindowLimits; + // G16-c: whether freq-capable (tier>=T2) postings lay out freq regions at + // all. Freq bytes serve ONLY BM25 scoring (want_freq=true lives solely in + // scoring_query), so the CALLER resolves the policy -- the Doris adapter + // passes has_scoring(config) || config::snii_positions_index_write_freq, + // i.e. plain kDocsPositions indexes drop freq unless the escape hatch is + // set. Defaults to true so the core library and existing callers keep the + // full T2 layout unless they opt out. The drop is value-driven on disk + // (windowed prelude flags bit0; slim/inline zero-length freq regions), so + // readers need no index-level flag. Ignored for docs-only configs. + bool write_freq = true; + // Lexicographically sorted terms with ascending-docid postings. Used when + // `term_source` is null (callers that already hold a materialized vector, + // e.g. unit tests). The writer reads but does not retain these. + std::vector terms; + // Optional streaming term source. When non-null, the writer DRAINS it via + // SpimiTermBuffer::for_each_term_sorted so that only one term's postings is + // materialized at a time (avoiding the full TermPostings vector and its + // second-copy peak). `terms` is ignored when this is set. The buffer is + // consumed (emptied) by build(); the caller must keep it alive until build() + // returns and must not reuse it afterwards. + SpimiTermBuffer* term_source = nullptr; + // Target DICT block size in bytes; a block is cut once its estimate reaches + // this. 0 uses kDefaultTargetDictBlockBytes. Smaller values yield more blocks + // (and a finer-grained sampled-term index). + uint32_t target_dict_block_bytes = 0; + // Maximum resident capacity of the staged DICT region before it spills. + // Ordinary builds keep the default unlimited local cap and use their shared + // spill-threshold reporter. Streamed compaction sets a bounded watermark so + // reclaimable DICT blocks cannot consume the hard-capped term workspace. + uint64_t dict_resident_cap_bytes = std::numeric_limits::max(); + // Optional writer-level build-RAM reporter (one per SniiCompoundWriter = one + // segment inverted index). When non-null, the dict buffer reports its REAL + // resident-byte deltas (positive on grow, negative on spill). The SPIMI side + // (arena + slot index) reports through the SAME reporter, injected directly at + // the term_source's construction by the caller. null in bench / unit tests -> no + // reporting. NEVER report live_bytes_ (a gated estimate); report + // arena_bytes()+slot_of_+dict ram_bytes_. + MemoryReporter* mem_reporter = nullptr; + // Optional persisted CommonGrams capability and semantic scoring stats. + // Missing metadata preserves the legacy SNII image and cannot be treated as + // compatibility proof by readers. + std::optional common_grams_metadata; + // Optional per-term CommonGrams postings shape. HybridV1 requires Mixed + // coverage metadata and a positions-capable logical index. + format::CommonGramsPostingPolicy common_grams_posting_policy = + format::CommonGramsPostingPolicy::kNone; +}; + +// Move-only ownership of a NULL-docid allocation and its precharged bytes. +// Reservation is declared first so destruction always frees the vector before +// returning its charge. Move assignment is intentionally forbidden because its +// default member order would release the destination charge before its vector. +class TrackedNullDocids { +public: + explicit TrackedNullDocids(std::vector&& docids) : docids_(std::move(docids)) {} + TrackedNullDocids(MemoryReporter::Reservation&& reservation, std::vector&& docids) + : reservation_(std::move(reservation)), docids_(std::move(docids)) {} + + TrackedNullDocids(const TrackedNullDocids&) = delete; + TrackedNullDocids& operator=(const TrackedNullDocids&) = delete; + TrackedNullDocids(TrackedNullDocids&&) noexcept = default; + TrackedNullDocids& operator=(TrackedNullDocids&&) = delete; + + bool empty() const { return docids_.empty(); } + size_t size() const { return docids_.size(); } + const uint32_t* data() const { return docids_.data(); } + uint32_t operator[](size_t index) const { return docids_[index]; } + auto begin() const { return docids_.begin(); } + auto end() const { return docids_.end(); } + + void release() { + std::vector().swap(docids_); + reservation_.reset(); + } + +private: + MemoryReporter::Reservation reservation_; + std::vector docids_; +}; + +// Move-only ownership of a destination norms allocation and its precharged +// bytes. Streamed sessions adopt both together so the vector remains accounted +// for until the writer has materialized the norms section. +class TrackedEncodedNorms { +public: + explicit TrackedEncodedNorms(std::vector&& norms) : norms_(std::move(norms)) {} + TrackedEncodedNorms(MemoryReporter::Reservation&& reservation, std::vector&& norms) + : reservation_(std::move(reservation)), norms_(std::move(norms)) {} + + TrackedEncodedNorms(const TrackedEncodedNorms&) = delete; + TrackedEncodedNorms& operator=(const TrackedEncodedNorms&) = delete; + TrackedEncodedNorms(TrackedEncodedNorms&&) noexcept = default; + TrackedEncodedNorms& operator=(TrackedEncodedNorms&&) = delete; + + bool empty() const { return norms_.empty(); } + size_t size() const { return norms_.size(); } + uint8_t operator[](size_t index) const { return norms_[index]; } + auto begin() const { return norms_.begin(); } + auto end() const { return norms_.end(); } + + void release() { + std::vector().swap(norms_); + reservation_.reset(); + } + +private: + friend class SniiStreamedIndexSession; + MemoryReporter::Reservation reservation_; + std::vector norms_; +}; + +// Term-level frequency statistics. Ordinary terms compute sum(freqs) and +// max(freqs) in one fused scan. Complete CommonGrams entries derive total_freq +// from their required PRX position count and leave max_freq at 0 because their +// statless DICT block does not serialize it. +struct FreqStats { + uint64_t total_freq = 0; + uint32_t max_freq = 0; +}; + +// Builds and holds the section bytes + meta sub-sections for one logical index. +class LogicalIndexWriter { +public: + explicit LogicalIndexWriter(const SniiIndexInput& in); + // Out-of-line: stream_state_ points at the private nested BlockState, which + // is incomplete here (unique_ptr needs the complete type at destruction). + ~LogicalIndexWriter(); + + // Builds DICT blocks, the interleaved posting region, sampled-term index, dict + // directory, stats and bsbf. The posting region is written STRAIGHT into + // `posting_out` as terms are produced (no temp round-trip for the bulk); the + // orchestrator captures its absolute offset/length from posting_out->bytes_written() + // around this call. Must be called once before the accessors below. Returns + // InvalidArgument on a null sink or inconsistent input (e.g. norms/doc_count + // mismatch when scoring is enabled, or non-ascending docids). + Status build(io::FileWriter* posting_out); + + // Streamed three-phase alternative to build() (T2.1, the compaction index + // merge fast path): the CALLER produces terms one at a time (k-way merge + // over source segments) instead of handing the writer a term source. + // begin_streamed(sink) -> push_term(tp) x N -> finish_streamed() + // push_term funnels through the SAME process_term choke point build() + // drains through (bigram prune gates, shape validation, encode), and the + // setup/finalize steps are shared with build() -- so the produced bytes are + // IDENTICAL to a build() fed the same terms in the same order (the T2 + // byte-golden invariant). A writer instance runs EXACTLY ONE session: + // build() and begin_streamed are mutually exclusive, push_term after + // finish_streamed and a second finish_streamed are errors -- a half-fed + // session can never masquerade as a sealed index (crash-safety invariant 6). + // Failure poisons the session: any push_term/finish_streamed/build failure + // past entry validation (e.g. a posting-sink append error mid-encode) moves + // the writer to a terminal failed state where every subsequent + // push_term/finish_streamed/build/begin_streamed is rejected, so a caller + // that swallows an error can never seal (or re-claim) a corrupt index. + // Entry rejections themselves (term-order / postings-shape / phase checks) + // do not poison an active session EXCEPT postings-shape violations, which + // fail inside the shared process_term and conservatively poison too; only + // the term-order guard is explicitly recoverable (see the UT contract). + Status begin_streamed(io::FileWriter* posting_out); + // Consumes tp synchronously. Entry validation: terms must arrive in STRICTLY + // increasing lexicographic order (the one invariant process_term cannot see + // -- DICT blocks, the sampled term index and the reader's binary search all + // assume it; equal terms are rejected too, the upstream merge must have + // combined duplicates). The streaming encoder rejects invalid per-term + // posting shapes. Returns InvalidArgument on any violation. + Status push_term(StreamedTermPostings&& tp); + Status finish_streamed(); + + // DICT region byte length (relative; orchestrator decides its absolute offset). The + // DICT region (zstd-compressed blocks) is built into a tiered buffer during build() + // -- it must land contiguously AFTER the posting region (streamed concurrently), so + // it cannot stream directly. The buffer stays in RAM while small (spill-only build) + // and spills to a temp once it crosses the RAM cap (bounded peak RSS for a huge + // dict). Its bytes are emitted via stream_dict_region_into below. The posting region + // went straight to the output during build(), so it has no length accessor here -- + // the orchestrator measures it directly. norms stays in RAM (1 byte/doc). + uint64_t dict_region_size() const { return dict_buf_.size(); } + const std::vector& norms_bytes() const { return norms_section_; } + const std::vector& null_bitmap_bytes() const { return null_bitmap_section_; } + // Block-split bloom XFilter blob ([28B header][bitset]); empty when no terms. + const std::vector& bsbf_bytes() const { return bsbf_bytes_; } + bool has_bsbf() const { return bsbf_built_; } + void release_bsbf_bytes(); + void release_null_bitmap_bytes(); + void release_norms_bytes(); + bool has_null_bitmap() const { return !null_bitmap_section_.empty(); } + + // Streams the DICT region (RAM or spilled temp) into the append-only container + // after its posting region. + Status stream_dict_region_into(io::FileWriter* out) { + return dict_buf_.stream_into_and_release(out); + } + + bool has_prx() const { return has_prx_; } + bool has_norms() const { return has_norms_; } + format::IndexTier tier() const { return tier_; } + uint64_t index_id() const { return index_id_; } + const std::string& index_suffix() const { return index_suffix_; } + + // Builds the three mandatory v1 metadata blobs. The orchestrator writes them + // contiguously as Core -> STI -> DBD and publishes their absolute references + // only after all three appends succeed. + Status finish_metadata(const format::SectionRefs& abs_refs, uint64_t dict_region_offset, + SerializedMetadataGroup* out) const; + +private: + friend class SniiStreamedIndexSession; + LogicalIndexWriter(const SniiIndexInput& in, TrackedNullDocids null_docids); + + // One DICT block's directory record. The block's serialized bytes are appended to + // the in-RAM dict buffer as soon as the block is cut; only this compact summary + // (offset within the dict region + length + entry count + checksum) is kept to + // build the DICT block directory at finish_metadata time. The absolute file offset is + // computed as dict_region_offset + rel_offset. + struct BlockRecord { + uint64_t rel_offset = 0; // byte offset of this block within the dict region + uint64_t length = 0; // ON-DISK block length (compressed when flags&kZstd) + uint32_t n_entries = 0; + uint32_t checksum = 0; // crc32c of the UNCOMPRESSED block bytes + uint8_t flags = 0; // block_ref_flags::* (kZstd when block is compressed) + uint64_t uncomp_len = 0; // uncompressed block length (when flags&kZstd) + std::string first_term; + }; + + // Shared entry/exit of build() and the streamed session, extracted so the + // two paths are byte-identical BY CONSTRUCTION (not by parallel-maintained + // copies). prepare_build validates the sink/norms/CommonGrams identity and + // anchors the posting region; finalize_build re-checks the stats-dependent + // CommonGrams invariants, seals the dict buffer and materializes + // stats/norms/null-bitmap/BSBF. + Status prepare_build(io::FileWriter* posting_out); + Status finalize_build(); + // Iterates terms (from the streaming source or the materialized vector), + // splitting DICT blocks by target size and filling PODs + blocks_. + Status build_blocks(); + // Per-term driver shared by every producer. It validates the term, opens a + // block if needed, encodes it, and cuts the block at the target size. + struct BlockState; + Status process_term(StreamedTermPostings& tp, BlockState* st); + Status reserve_term_hash_for_append(); + // Region-relative byte count of the posting bytes written so far (the offset basis + // for frq_base/prx_base + frq_off_delta/prx_off_delta). During build() the only + // writes to posting_out_ are this index's posting region, so the count is the + // output offset advanced since the region began. + uint64_t posting_size() const { return posting_out_->bytes_written() - posting_off0_; } + // Serializes the current open block, streams its bytes into the dict scratch + // file, and records a compact directory entry (no block bytes retained). + Status flush_block(format::DictBlockBuilder* block, std::string first_term); + + uint64_t index_id_; + std::string index_suffix_; + format::IndexConfig index_config_; + format::IndexTier tier_; + bool has_prx_; + bool has_freq_; // tier >= T2: a freq region is encoded per window + bool has_norms_; + uint32_t doc_count_; + TrackedNullDocids null_docids_; + const std::vector& terms_; // materialized fallback (may be empty) + SpimiTermBuffer* term_source_; // streaming source (null => use terms_) + uint64_t term_count_ = 0; // distinct terms actually consumed + const std::vector& encoded_norms_; + std::optional common_grams_metadata_; + format::CommonGramsPostingPolicy common_grams_posting_policy_ = + format::CommonGramsPostingPolicy::kNone; + + uint32_t target_dict_block_bytes_; + // G16-h: zstd levels (dict whole-block / prx auto mode), from SniiIndexInput. + int dict_block_zstd_level_ = 3; + int prx_zstd_level_ = 3; + format::PrxWindowLimits prx_window_limits_ = format::kReaderPrxWindowLimits; + // The DICT region (zstd-compressed blocks) is staged here as blocks flush. It must + // land contiguously AFTER the posting region (which streams concurrently to the + // output), so it cannot stream directly; the orchestrator streams it into the + // container right after the posting region. It has NO independent local cap -- it + // spills to a temp via the writer's shared tracked-memory budget (the + // MemoryReporter from SniiIndexInput, null off-Doris). This reporter covers + // explicitly reserved structures; codec-local scratch remains governed by + // its format window limits rather than being represented as a whole-writer + // RSS hard cap. + MemoryReporter* memory_reporter_ = nullptr; + SpillableByteBuffer dict_buf_; + // The interleaved [prx][frq] posting region streams STRAIGHT into the container + // output during build() -- no temp. posting_out_ is the container writer (borrowed + // for the duration of build); posting_off0_ is its absolute offset when this index's + // region began, so posting_size() = bytes_written() - posting_off0_. + io::FileWriter* posting_out_ = nullptr; + uint64_t posting_off0_ = 0; + MemoryReporter::Reservation norms_section_reservation_; + std::vector norms_section_; + MemoryReporter::Reservation null_bitmap_section_reservation_; + std::vector null_bitmap_section_; + + std::vector blocks_; + MemoryReporter::Reservation term_hashes_reservation_; + MemoryReporter::Reservation bsbf_bytes_reservation_; + // One 8-byte XXH64 (seed 0) filter key per term, collected during the build pass + // so the whole-vocabulary string copy is never retained. + std::vector term_hashes_; + format::StatsBlock stats_; + std::vector bsbf_bytes_; // serialized block-split bloom XFilter section + bool bsbf_built_ = false; + + // Streamed-session state (T2.1). kIdle until build()/begin_streamed claims + // the writer; build() jumps straight to kFinished on completion so the two + // entry points can never interleave on one instance (single-session + // invariant). kFailed is the poison state: any failure PAST entry + // validation (process_term inside push_term, build_blocks inside build(), + // flush/finalize inside either finish path) may have left partial posting + // bytes in the sink or partial term state (term_hashes_/stats_), so the + // writer transitions to kFailed and every subsequent + // push/finish/build/begin is rejected -- a half-fed session can never + // masquerade as a sealed index (crash-safety invariant 6), and a dirty + // writer can never be re-claimed for a second session. Pure entry + // rejections (phase check, term-order guard, prepare_build validation) + // mutate nothing and therefore do NOT poison. + enum class StreamPhase : uint8_t { kIdle, kActive, kFinished, kFailed }; + StreamPhase stream_phase_ = StreamPhase::kIdle; + std::unique_ptr stream_state_; // live only while kActive + std::string last_pushed_term_; // strict term-order entry guard + bool has_pushed_term_ = false; + + friend class StreamingTermEncoder; +}; + +// TEST-ONLY observability seam (mirrors the reader-side decode-counter and the +// SPIMI vocab-materialization patterns). term_freq_scans() returns a +// process-global count of term-level fused freqs scans. Ordinary terms call +// fuse_freq_stats exactly once; complete CommonGrams entries call it zero times. +// note_term_freq_scan() bumps the counter; reset_term_freq_scans() zeroes it +// between tests; fuse_freq_stats_for_test() exposes the real fused helper so +// pure boundary tests exercise production code. Process-global; reset between +// tests. Not part of the production API. +namespace testing { +void note_term_freq_scan(); +uint64_t term_freq_scans(); +void reset_term_freq_scans(); +FreqStats fuse_freq_stats_for_test(const std::vector& freqs); +} // namespace testing + +} // namespace doris::snii::writer diff --git a/be/src/storage/index/snii/writer/memory_reporter.h b/be/src/storage/index/snii/writer/memory_reporter.h new file mode 100644 index 00000000000000..04875118f96872 --- /dev/null +++ b/be/src/storage/index/snii/writer/memory_reporter.h @@ -0,0 +1,227 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include + +#include "common/check.h" +#include "common/status.h" + +namespace doris::snii::writer { + +// Per-WRITER accurate byte counter for build-time RAM (one per SniiCompoundWriter = +// one per segment's inverted index). Legacy modules report resident-byte deltas +// after their allocation; hard-gated modules own a Reservation that atomically +// pre-charges before allocation. current_bytes() is their shared live total. +// consume_release mirrors successful changes into Doris's LOAD MemTracker; it is +// null off-Doris (bench / unit tests), where only the local atomic is updated. +class MemoryReporter { +public: + // The callback may be invoked concurrently and from Reservation destructors; + // it must be thread-safe and must not throw. Null off-Doris. + using ConsumeReleaseFn = std::function; + + enum class CapPolicy : uint8_t { + // Reservations fail before an allocation would cross the cap. Native + // compaction uses this policy so an over-budget merge can fall back to + // the raw-column rebuild path without exceeding its bounded workspace. + kHardLimit, + // The cap is a spill trigger, not an allocation limit. Ordinary index + // ingestion uses this policy because persistent vocabulary structures + // can exceed the reclaimable posting-arena threshold by design. + kSpillThreshold, + }; + + // Move-only ownership of bytes pre-charged against this reporter. Growing a + // reservation atomically charges before allocation. Hard-limit reporters + // reject an over-cap charge without changing state; spill-threshold reporters + // retain exact accounting above the threshold. Callers must release/shrink the + // physical buffer before lowering the reservation. A Reservation borrows its + // reporter, which must outlive it. + class Reservation { + public: + Reservation() = default; + Reservation(const Reservation&) = delete; + Reservation& operator=(const Reservation&) = delete; + Reservation(Reservation&& other) noexcept; + Reservation& operator=(Reservation&& other) noexcept; + ~Reservation(); + + Status set_bytes(uint64_t target_bytes); + // Pre-charges an independent allocation while this Reservation keeps + // covering the old one. After the physical replacement succeeds, move + // `replacement` back into this Reservation to release the old charge. + Status prepare_replacement(uint64_t target_bytes, Reservation* replacement) const; + void reset(); + uint64_t bytes() const { return bytes_; } + + private: + friend class MemoryReporter; + explicit Reservation(MemoryReporter* owner) : owner_(owner) {} + + MemoryReporter* owner_ = nullptr; + uint64_t bytes_ = 0; + }; + + // cap_bytes is the shared gate-2 threshold (0 = unlimited). Hard-limit + // reporters reject reservations before their covered allocations cross it. + // Spill-threshold reporters keep exact accounting above it so over_cap() can + // drive reclaim without turning irreducible vocabulary growth into an import + // failure. + explicit MemoryReporter(ConsumeReleaseFn consume_release = nullptr, uint64_t cap_bytes = 0, + CapPolicy cap_policy = CapPolicy::kHardLimit) + : consume_release_(std::move(consume_release)), + cap_bytes_(cap_bytes), + cap_policy_(cap_policy) {} + + MemoryReporter(const MemoryReporter&) = delete; + MemoryReporter& operator=(const MemoryReporter&) = delete; + + Reservation make_reservation() { return Reservation(this); } + + // Observe-only legacy path: delta > 0 grows, delta < 0 shrinks/frees. New + // hard-gated allocations must use Reservation instead. + void report(int64_t delta) { + if (delta == 0) return; + DCHECK_NE(delta, std::numeric_limits::min()); + int64_t current = current_.load(std::memory_order_relaxed); + while (true) { + DCHECK_GE(current, 0); + if (delta > 0) { + DCHECK_LE(delta, std::numeric_limits::max() - current); + } else { + DCHECK_GE(current, -delta); + } + const int64_t desired = current + delta; + if (current_.compare_exchange_weak(current, desired, std::memory_order_relaxed, + std::memory_order_relaxed)) { + if (consume_release_) consume_release_(delta); + return; + } + } + } + + int64_t current_bytes() const { return current_.load(std::memory_order_relaxed); } + + // True once all reported/reserved build RAM reaches the shared spill threshold. + bool over_cap() const { + const int64_t current = current_bytes(); + DCHECK_GE(current, 0); + return cap_bytes_ != 0 && static_cast(current) >= cap_bytes_; + } + uint64_t cap_bytes() const { return cap_bytes_; } + +private: + Status try_acquire(uint64_t bytes); + void release(uint64_t bytes); + + std::atomic current_ {0}; + ConsumeReleaseFn consume_release_; + uint64_t cap_bytes_ = 0; + CapPolicy cap_policy_ = CapPolicy::kHardLimit; +}; + +inline MemoryReporter::Reservation::Reservation(Reservation&& other) noexcept + : owner_(std::exchange(other.owner_, nullptr)), bytes_(std::exchange(other.bytes_, 0)) {} + +inline MemoryReporter::Reservation& MemoryReporter::Reservation::operator=( + Reservation&& other) noexcept { + if (this != &other) { + reset(); + owner_ = std::exchange(other.owner_, nullptr); + bytes_ = std::exchange(other.bytes_, 0); + } + return *this; +} + +inline MemoryReporter::Reservation::~Reservation() { + reset(); +} + +inline Status MemoryReporter::Reservation::set_bytes(uint64_t target_bytes) { + DORIS_CHECK(owner_ != nullptr); + if (target_bytes > bytes_) { + RETURN_IF_ERROR(owner_->try_acquire(target_bytes - bytes_)); + } else if (target_bytes < bytes_) { + owner_->release(bytes_ - target_bytes); + } + bytes_ = target_bytes; + return Status::OK(); +} + +inline Status MemoryReporter::Reservation::prepare_replacement(uint64_t target_bytes, + Reservation* replacement) const { + DORIS_CHECK(owner_ != nullptr); + DORIS_CHECK(replacement != nullptr); + DORIS_CHECK(replacement->owner_ == nullptr); + Reservation pending(owner_); + RETURN_IF_ERROR(pending.set_bytes(target_bytes)); + *replacement = std::move(pending); + return Status::OK(); +} + +inline void MemoryReporter::Reservation::reset() { + if (owner_ != nullptr && bytes_ != 0) { + owner_->release(bytes_); + bytes_ = 0; + } +} + +inline Status MemoryReporter::try_acquire(uint64_t bytes) { + if (bytes == 0) { + return Status::OK(); + } + int64_t current = current_.load(std::memory_order_relaxed); + while (true) { + DCHECK_GE(current, 0); + const uint64_t current_bytes = static_cast(current); + const bool exceeds_cap = cap_policy_ == CapPolicy::kHardLimit && cap_bytes_ != 0 && + (current_bytes > cap_bytes_ || bytes > cap_bytes_ - current_bytes); + const bool exceeds_counter = + bytes > static_cast(std::numeric_limits::max()) - current_bytes; + if (exceeds_cap || exceeds_counter) { + return Status::Error( + "SNII memory reservation exceeds limit: request={} current={} cap={}", bytes, + current_bytes, cap_bytes_); + } + const int64_t desired = current + static_cast(bytes); + if (current_.compare_exchange_weak(current, desired, std::memory_order_relaxed, + std::memory_order_relaxed)) { + if (consume_release_) { + consume_release_(static_cast(bytes)); + } + return Status::OK(); + } + } +} + +inline void MemoryReporter::release(uint64_t bytes) { + DCHECK_LE(bytes, static_cast(std::numeric_limits::max())); + const int64_t delta = static_cast(bytes); + const int64_t previous = current_.fetch_sub(delta, std::memory_order_relaxed); + DCHECK_GE(previous, delta); + if (consume_release_) { + consume_release_(-delta); + } +} + +} // namespace doris::snii::writer diff --git a/be/src/storage/index/snii/writer/posting_window_emitter.cpp b/be/src/storage/index/snii/writer/posting_window_emitter.cpp new file mode 100644 index 00000000000000..709d59b72c9ee8 --- /dev/null +++ b/be/src/storage/index/snii/writer/posting_window_emitter.cpp @@ -0,0 +1,579 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/writer/posting_window_emitter.h" + +#include +#include +#include +#include +#include +#include + +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/encoding/byte_sink.h" +#include "storage/index/snii/format/dict_entry.h" +#include "storage/index/snii/format/format_constants.h" +#include "storage/index/snii/format/frq_pod.h" +#include "storage/index/snii/format/frq_prelude.h" +#include "storage/index/snii/io/file_writer.h" +#include "storage/index/snii/writer/spillable_byte_buffer.h" + +namespace doris::snii::writer { + +namespace { + +constexpr int kRawFrqRegion = 0; +constexpr uint32_t kPreludeGroupSize = 64; + +struct PostingWindowPlan { + size_t doc_begin = 0; + size_t doc_count = 0; + uint64_t position_begin = 0; + uint64_t position_count = 0; + uint32_t max_freq = 0; +}; + +bool fits_prx_window_shape(uint64_t doc_count, uint64_t position_count, + const format::PrxWindowLimits& limits) { + return doc_count <= limits.max_docs && position_count <= limits.max_positions; +} + +// Five bytes is the maximum encoded width of every uint32 field in the raw +// payload. This gate is used only after an exact build requests a split. +bool conservatively_fits_prx_window(uint64_t doc_count, uint64_t position_count, + const format::PrxWindowLimits& limits) { + return fits_prx_window_shape(doc_count, position_count, limits) && + 1 + doc_count + position_count <= limits.max_uncomp_bytes / 5; +} + +uint8_t window_max_norm(std::span norms, std::span docs) { + if (norms.empty() || docs.empty()) { + return 0; + } +#ifdef BE_TEST + testing::note_window_norm_doc_visits(docs.size()); +#endif + uint8_t best = 0xFF; + for (uint32_t docid : docs) { + DCHECK_LT(docid, norms.size()); + best = std::min(best, norms[docid]); + } + return best == 0xFF ? 0 : best; +} + +Status build_prelude(const std::vector& windows, bool has_freq, bool has_prx, + std::vector* output) { + format::FrqPreludeColumns columns; + columns.has_freq = has_freq; + columns.has_prx = has_prx; + columns.group_size = kPreludeGroupSize; + columns.windows = windows; + ByteSink sink; + RETURN_IF_ERROR(format::build_frq_prelude(columns, &sink)); + *output = sink.take(); + return Status::OK(); +} + +Status checked_add(uint64_t increment, uint64_t* value) { + if (increment > std::numeric_limits::max() - *value) { + return Status::Error( + "window emitter: term frequency overflow"); + } + *value += increment; + return Status::OK(); +} + +} // namespace + +class WindowEmitter::Impl { +public: + explicit Impl(WindowEmitterOptions options) + : options_(options), + dd_stager_(std::numeric_limits::max(), "term_dd", options.memory_reporter), + freq_stager_(std::numeric_limits::max(), "term_freq", + options.memory_reporter) { + if (options_.posting_out != nullptr && + options_.posting_out->bytes_written() >= options_.posting_region_offset) { + prx_off_ = options_.posting_out->bytes_written() - options_.posting_region_offset; + posting_offset_valid_ = true; + } + } + + Status emit_window(const PostingRunView& run) { + if (phase_ != Phase::kActive) { + return phase_error("emit_window"); + } + Status status = emit_window_impl(run); + if (!status.ok()) { + phase_ = Phase::kFailed; + } + return status; + } + + Status finish_term(format::DictEntry* entry, TermAggregateStats* stats) { + if (phase_ != Phase::kActive) { + return phase_error("finish_term"); + } + if (entry == nullptr || stats == nullptr) { + phase_ = Phase::kFailed; + return Status::Error( + "window emitter: null finish output"); + } + if (windows_.empty()) { + phase_ = Phase::kFailed; + return Status::Error( + "window emitter: cannot finish an empty term"); + } + Status status = finish_term_impl(entry); + if (!status.ok()) { + phase_ = Phase::kFailed; + return status; + } + *stats = stats_; + phase_ = Phase::kFinished; +#ifdef BE_TEST + finished_term_counter().fetch_add(1, std::memory_order_relaxed); +#endif + return Status::OK(); + } + +private: + enum class Phase : uint8_t { kActive, kFinished, kFailed }; + + Status phase_error(std::string_view operation) const { + return Status::Error( + "window emitter: {} after {}", operation, + phase_ == Phase::kFailed ? "failure" : "finish"); + } + + Status posting_size(uint64_t* size) const { + if (options_.posting_out == nullptr) { + return Status::Error( + "window emitter: null posting sink"); + } + if (!posting_offset_valid_ || + options_.posting_out->bytes_written() < options_.posting_region_offset) { + return Status::Error( + "window emitter: invalid posting region offset"); + } + *size = options_.posting_out->bytes_written() - options_.posting_region_offset; + return Status::OK(); + } + + Status validate_run(const PostingRunView& run) const { + if (options_.posting_out == nullptr) { + return Status::Error( + "window emitter: null posting sink"); + } + if (run.docids.empty()) { + return Status::Error( + "window emitter: empty posting window"); + } + if ((!run.freqs.empty() || options_.has_freq || options_.has_prx) && + run.freqs.size() != run.docids.size()) { + return Status::Error( + "window emitter: frequency shape must match documents"); + } + if (options_.term_frequency_source == TermFrequencySource::kPositions && + !options_.has_prx) { + return Status::Error( + "window emitter: position-derived statistics require PRX offsets"); + } + if (options_.has_prx) { + if (run.position_offsets.size() != run.docids.size() + 1) { + return Status::Error( + "window emitter: position offsets must have docs plus one entries"); + } + if (run.position_offsets.front() > run.position_offsets.back() || + run.position_offsets.back() - run.position_offsets.front() != + run.positions_flat.size()) { + return Status::Error( + "window emitter: position offsets differ from the position run"); + } + } else if (!run.position_offsets.empty() || !run.positions_flat.empty()) { + return Status::Error( + "window emitter: positions require a PRX term"); + } + if (last_input_docid_.has_value() && run.docids.front() <= *last_input_docid_) { + return Status::Error( + "window emitter: posting windows must be strictly ordered"); + } + return Status::OK(); + } + + Status accumulate_constant_stats(const PostingRunView& run) { + if (run.docids.size() > std::numeric_limits::max() - stats_.df) { + return Status::Error( + "window emitter: document frequency overflow"); + } + stats_.df += static_cast(run.docids.size()); + switch (options_.term_frequency_source) { + case TermFrequencySource::kDocuments: + return checked_add(run.docids.size(), &stats_.total_freq); + case TermFrequencySource::kPositions: + return checked_add(run.position_offsets.back() - run.position_offsets.front(), + &stats_.total_freq); + case TermFrequencySource::kFrequenciesOrDocuments: + if (run.freqs.empty()) { + return checked_add(run.docids.size(), &stats_.total_freq); + } + return Status::OK(); + } + __builtin_unreachable(); + } + + uint64_t position_count(const PostingRunView& run, size_t begin, size_t count) const { + if (!options_.has_prx) { + return 0; + } + return run.position_offsets[begin + count] - run.position_offsets[begin]; + } + + Status emit_window_impl(const PostingRunView& run) { + RETURN_IF_ERROR(validate_run(run)); + RETURN_IF_ERROR(accumulate_constant_stats(run)); + + const bool accumulate_frequencies = + options_.term_frequency_source == TermFrequencySource::kFrequenciesOrDocuments && + !run.freqs.empty(); + if (!options_.has_prx && !options_.has_freq && !accumulate_frequencies) { + RETURN_IF_ERROR( + emit_planned(run, make_plan(run, 0, run.docids.size(), /*max_freq=*/0))); + last_input_docid_ = run.docids.back(); + return Status::OK(); + } + + size_t window_begin = 0; + uint32_t window_max_freq = 0; + for (size_t doc = 0; doc < run.docids.size(); ++doc) { + const uint64_t document_positions = options_.has_prx ? position_count(run, doc, 1) : 0; + if (options_.has_prx && (run.position_offsets[doc + 1] < run.position_offsets[doc] || + document_positions != run.freqs[doc])) { + return Status::Error( + "window emitter: position offsets must match frequencies"); + } + if (options_.has_prx && document_positions > options_.prx_window_limits.max_positions) { + return Status::Error( + "window emitter: one document exceeds the PRX position limit"); + } + if (accumulate_frequencies) { + RETURN_IF_ERROR(checked_add(run.freqs[doc], &stats_.total_freq)); + stats_.max_freq = std::max(stats_.max_freq, run.freqs[doc]); + } + const uint64_t candidate_docs = doc - window_begin + 1; + const uint64_t candidate_positions = position_count(run, window_begin, candidate_docs); + if (doc != window_begin && options_.has_prx && + !fits_prx_window_shape(candidate_docs, candidate_positions, + options_.prx_window_limits)) { + RETURN_IF_ERROR(emit_planned( + run, make_plan(run, window_begin, doc - window_begin, window_max_freq))); + window_begin = doc; + window_max_freq = 0; + } + if (options_.has_freq) { +#ifdef BE_TEST + testing::note_window_freq_doc_visits(); +#endif + window_max_freq = std::max(window_max_freq, run.freqs[doc]); + } + } + RETURN_IF_ERROR(emit_planned( + run, + make_plan(run, window_begin, run.docids.size() - window_begin, window_max_freq))); + last_input_docid_ = run.docids.back(); + return Status::OK(); + } + + PostingWindowPlan make_plan(const PostingRunView& run, size_t begin, size_t count, + uint32_t max_freq) const { + return { + .doc_begin = begin, + .doc_count = count, + .position_begin = options_.has_prx ? run.position_offsets[begin] - + run.position_offsets.front() + : uint64_t {0}, + .position_count = position_count(run, begin, count), + .max_freq = max_freq, + }; + } + + Status emit_planned(const PostingRunView& run, const PostingWindowPlan& plan) { + format::PrxWindowBuildOutcome outcome = format::PrxWindowBuildOutcome::kBuilt; + RETURN_IF_ERROR(emit_physical_window(run, plan, &outcome)); + if (outcome == format::PrxWindowBuildOutcome::kBuilt) { + return Status::OK(); + } + + std::vector recut; + recut_window(run, plan, &recut); + for (const PostingWindowPlan& subplan : recut) { + outcome = format::PrxWindowBuildOutcome::kBuilt; + RETURN_IF_ERROR(emit_physical_window(run, subplan, &outcome)); + if (outcome == format::PrxWindowBuildOutcome::kNeedsSplit) { + return Status::Error( + "window emitter: one document exceeds the PRX byte limit"); + } + } + return Status::OK(); + } + + void recut_window(const PostingRunView& run, const PostingWindowPlan& input, + std::vector* output) const { + size_t window_begin = input.doc_begin; + uint32_t window_max_freq = 0; + const size_t input_end = input.doc_begin + input.doc_count; + for (size_t doc = input.doc_begin; doc < input_end; ++doc) { + const uint64_t candidate_docs = doc - window_begin + 1; + const uint64_t candidate_positions = position_count(run, window_begin, candidate_docs); + if (doc != window_begin && + !conservatively_fits_prx_window(candidate_docs, candidate_positions, + options_.prx_window_limits)) { + output->push_back( + make_plan(run, window_begin, doc - window_begin, window_max_freq)); + window_begin = doc; + window_max_freq = 0; + } + if (options_.has_freq) { +#ifdef BE_TEST + testing::note_window_freq_doc_visits(); +#endif + window_max_freq = std::max(window_max_freq, run.freqs[doc]); + } + } + output->push_back(make_plan(run, window_begin, input_end - window_begin, window_max_freq)); + } + + Status emit_physical_window(const PostingRunView& run, const PostingWindowPlan& plan, + format::PrxWindowBuildOutcome* outcome) { + const auto docs = run.docids.subspan(plan.doc_begin, plan.doc_count); + const auto freqs = run.freqs.empty() ? std::span {} + : run.freqs.subspan(plan.doc_begin, plan.doc_count); + format::WindowMeta window; + window.last_docid = docs.back(); + window.win_base = window_base_; + window.doc_count = static_cast(docs.size()); + window.max_freq = options_.has_freq ? plan.max_freq : 0; + window.max_norm = options_.has_freq ? window_max_norm(options_.encoded_norms, docs) : 0; + + if (options_.has_prx) { + const auto positions = + run.positions_flat.subspan(static_cast(plan.position_begin), + static_cast(plan.position_count)); + prx_scratch_.clear(); + RETURN_IF_ERROR(format::try_build_prx_window_flat( + positions, freqs, -options_.prx_zstd_level, options_.prx_window_limits, + &prx_scratch_, outcome)); + if (*outcome == format::PrxWindowBuildOutcome::kNeedsSplit) { + return Status::OK(); + } + window.prx_off = prx_total_len_; + window.prx_len = prx_scratch_.size(); + RETURN_IF_ERROR(options_.posting_out->append(prx_scratch_.view())); + prx_total_len_ += window.prx_len; + } else { + *outcome = format::PrxWindowBuildOutcome::kBuilt; + } + + ByteSink dd_sink; + format::FrqRegionMeta dd_meta; + window.dd_off = dd_stager_.size(); + RETURN_IF_ERROR( + format::build_dd_region(docs, window_base_, kRawFrqRegion, &dd_sink, &dd_meta)); + window.dd_zstd = dd_meta.zstd; + window.dd_disk_len = dd_meta.disk_len; + window.dd_uncomp_len = dd_meta.uncomp_len; + window.crc_dd = dd_meta.crc; + RETURN_IF_ERROR(dd_stager_.append_move(dd_sink.take())); + + if (options_.has_freq) { + ByteSink freq_sink; + format::FrqRegionMeta freq_meta; + window.freq_off = freq_stager_.size(); + RETURN_IF_ERROR( + format::build_freq_region(freqs, kRawFrqRegion, &freq_sink, &freq_meta)); + window.freq_zstd = freq_meta.zstd; + window.freq_disk_len = freq_meta.disk_len; + window.freq_uncomp_len = freq_meta.uncomp_len; + window.crc_freq = freq_meta.crc; + RETURN_IF_ERROR(freq_stager_.append_move(freq_sink.take())); + } + + windows_.push_back(window); + window_base_ = window.last_docid; +#ifdef BE_TEST + physical_window_counter().fetch_add(1, std::memory_order_relaxed); +#endif + return Status::OK(); + } + + Status finish_term_impl(format::DictEntry* entry) { + std::vector prelude; + RETURN_IF_ERROR(build_prelude(windows_, options_.has_freq, options_.has_prx, &prelude)); + entry->kind = format::DictEntryKind::kPodRef; + entry->enc = format::DictEntryEnc::kWindowed; + entry->has_sb = true; + entry->prelude_len = prelude.size(); + entry->frq_docs_len = entry->prelude_len + dd_stager_.size(); + + uint64_t frq_off = 0; + RETURN_IF_ERROR(posting_size(&frq_off)); + RETURN_IF_ERROR(options_.posting_out->append(Slice(prelude))); + RETURN_IF_ERROR(dd_stager_.seal()); + RETURN_IF_ERROR(dd_stager_.stream_into_and_release(options_.posting_out)); + RETURN_IF_ERROR(freq_stager_.seal()); + RETURN_IF_ERROR(freq_stager_.stream_into_and_release(options_.posting_out)); + entry->frq_off_delta = frq_off - options_.frq_base; + uint64_t end = 0; + RETURN_IF_ERROR(posting_size(&end)); + entry->frq_len = end - frq_off; + if (options_.has_prx) { + entry->prx_off_delta = prx_off_ - options_.prx_base; + entry->prx_len = prx_total_len_; + } + return Status::OK(); + } + +#ifdef BE_TEST + static std::atomic& finished_term_counter(); + static std::atomic& physical_window_counter(); +#endif + + WindowEmitterOptions options_; + SpillableByteBuffer dd_stager_; + SpillableByteBuffer freq_stager_; + std::vector windows_; + ByteSink prx_scratch_; + TermAggregateStats stats_; + std::optional last_input_docid_; + uint64_t prx_off_ = 0; + uint64_t prx_total_len_ = 0; + uint64_t window_base_ = 0; + bool posting_offset_valid_ = false; + Phase phase_ = Phase::kActive; +}; + +#ifdef BE_TEST +namespace { +std::atomic& window_norm_doc_visit_counter() { + static std::atomic counter {0}; + return counter; +} +std::atomic& window_freq_doc_visit_counter() { + static std::atomic counter {0}; + return counter; +} +std::atomic& emitter_finished_term_counter() { + static std::atomic counter {0}; + return counter; +} +std::atomic& emitter_physical_window_counter() { + static std::atomic counter {0}; + return counter; +} +} // namespace + +std::atomic& WindowEmitter::Impl::finished_term_counter() { + return emitter_finished_term_counter(); +} + +std::atomic& WindowEmitter::Impl::physical_window_counter() { + return emitter_physical_window_counter(); +} +#endif + +WindowEmitter::WindowEmitter(WindowEmitterOptions options) + : impl_(std::make_unique(options)) {} + +WindowEmitter::~WindowEmitter() = default; + +Status WindowEmitter::emit_window(const PostingRunView& window) { + return impl_->emit_window(window); +} + +Status WindowEmitter::finish_term(format::DictEntry* entry, TermAggregateStats* stats) { + return impl_->finish_term(entry, stats); +} + +namespace testing { + +void note_window_norm_doc_visits(uint64_t count) { +#ifdef BE_TEST + window_norm_doc_visit_counter().fetch_add(count, std::memory_order_relaxed); +#endif +} + +uint64_t window_norm_doc_visits() { +#ifdef BE_TEST + return window_norm_doc_visit_counter().load(std::memory_order_relaxed); +#else + return 0; +#endif +} + +void reset_window_norm_doc_visits() { +#ifdef BE_TEST + window_norm_doc_visit_counter().store(0, std::memory_order_relaxed); +#endif +} + +void note_window_freq_doc_visits() { +#ifdef BE_TEST + window_freq_doc_visit_counter().fetch_add(1, std::memory_order_relaxed); +#endif +} + +uint64_t window_freq_doc_visits() { +#ifdef BE_TEST + return window_freq_doc_visit_counter().load(std::memory_order_relaxed); +#else + return 0; +#endif +} + +void reset_window_freq_doc_visits() { +#ifdef BE_TEST + window_freq_doc_visit_counter().store(0, std::memory_order_relaxed); +#endif +} + +uint64_t window_emitter_finished_terms() { +#ifdef BE_TEST + return emitter_finished_term_counter().load(std::memory_order_relaxed); +#else + return 0; +#endif +} + +uint64_t window_emitter_physical_windows() { +#ifdef BE_TEST + return emitter_physical_window_counter().load(std::memory_order_relaxed); +#else + return 0; +#endif +} + +void reset_window_emitter_counters() { +#ifdef BE_TEST + emitter_finished_term_counter().store(0, std::memory_order_relaxed); + emitter_physical_window_counter().store(0, std::memory_order_relaxed); +#endif +} + +} // namespace testing + +} // namespace doris::snii::writer diff --git a/be/src/storage/index/snii/writer/posting_window_emitter.h b/be/src/storage/index/snii/writer/posting_window_emitter.h new file mode 100644 index 00000000000000..e6b0e3444b31fd --- /dev/null +++ b/be/src/storage/index/snii/writer/posting_window_emitter.h @@ -0,0 +1,112 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/format/prx_pod.h" + +namespace doris::snii { +namespace format { +struct DictEntry; +} +namespace io { +class FileWriter; +} +namespace writer { + +class MemoryReporter; + +// Synchronously borrowed posting arrays for one canonical posting window. +// position_offsets has docids.size()+1 entries when positions are present. The +// offsets may start above zero when the view borrows a sub-window; positions_flat +// covers exactly position_offsets.back()-position_offsets.front() values. +struct PostingRunView { + std::span docids; + std::span freqs; + std::span position_offsets; + std::span positions_flat; +}; + +struct TermAggregateStats { + uint32_t df = 0; + uint64_t total_freq = 0; + uint32_t max_freq = 0; +}; + +// CommonGrams entries define semantic term frequency from documents or +// positions rather than the transient physical frequency array. +enum class TermFrequencySource : uint8_t { + kFrequenciesOrDocuments, + kDocuments, + kPositions, +}; + +struct WindowEmitterOptions { + io::FileWriter* posting_out = nullptr; + uint64_t posting_region_offset = 0; + uint64_t frq_base = 0; + uint64_t prx_base = 0; + std::span encoded_norms; + bool has_freq = false; + bool has_prx = false; + int prx_zstd_level = 3; + format::PrxWindowLimits prx_window_limits = format::kReaderPrxWindowLimits; + TermFrequencySource term_frequency_source = TermFrequencySource::kFrequenciesOrDocuments; + MemoryReporter* memory_reporter = nullptr; +}; + +// The single owner of windowed DD/frequency/PRX encoding and prelude metadata. +// A failed emit poisons the instance; finish_term cannot publish a partial term. +class WindowEmitter { +public: + explicit WindowEmitter(WindowEmitterOptions options); + ~WindowEmitter(); + + WindowEmitter(const WindowEmitter&) = delete; + WindowEmitter& operator=(const WindowEmitter&) = delete; + WindowEmitter(WindowEmitter&&) = delete; + WindowEmitter& operator=(WindowEmitter&&) = delete; + + Status emit_window(const PostingRunView& window); + Status finish_term(format::DictEntry* entry, TermAggregateStats* stats); + +private: + class Impl; + std::unique_ptr impl_; +}; + +// Process-global test observability for the one emitter choke point and the +// existing bounded-work counters. Reset between tests. +namespace testing { +void note_window_norm_doc_visits(uint64_t count); +uint64_t window_norm_doc_visits(); +void reset_window_norm_doc_visits(); +void note_window_freq_doc_visits(); +uint64_t window_freq_doc_visits(); +void reset_window_freq_doc_visits(); +uint64_t window_emitter_finished_terms(); +uint64_t window_emitter_physical_windows(); +void reset_window_emitter_counters(); +} // namespace testing + +} // namespace writer +} // namespace doris::snii diff --git a/be/src/storage/index/snii/writer/snii_compound_writer.cpp b/be/src/storage/index/snii/writer/snii_compound_writer.cpp new file mode 100644 index 00000000000000..472b9e1460cad4 --- /dev/null +++ b/be/src/storage/index/snii/writer/snii_compound_writer.cpp @@ -0,0 +1,398 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/writer/snii_compound_writer.h" + +#include + +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/encoding/byte_sink.h" +#include "storage/index/snii/encoding/crc32c.h" +#include "storage/index/snii/format/bootstrap_header.h" +#include "storage/index/snii/format/core_metadata.h" +#include "storage/index/snii/format/metadata_directory.h" +#include "storage/index/snii/format/tail_pointer.h" + +namespace doris::snii::writer { + +using format::BootstrapHeader; +using format::LogicalIndexMetadataRef; +using format::SectionRefs; +using format::TailPointer; + +SniiCompoundWriter::SniiCompoundWriter(io::FileWriter* out) : out_(out) {} + +Status SniiCompoundWriter::poison(Status status) { + DCHECK(!status.ok()); + if (failed_.ok()) { + failed_ = std::move(status); + } + return failed_; +} + +Status SniiCompoundWriter::append(const std::vector& bytes) { + if (bytes.empty()) return Status::OK(); + return out_->append(Slice(bytes)); +} + +// The bootstrap header occupies offset 0 and must precede the first posting region, +// which streams straight into the output during build(). Written lazily exactly once +// (on the first add, or in finish() for an empty container). +Status SniiCompoundWriter::ensure_bootstrap() { + if (!failed_.ok()) return failed_; + if (bootstrap_written_) return Status::OK(); + const Status status = write_bootstrap(); + if (!status.ok()) return poison(status); + bootstrap_written_ = true; + return Status::OK(); +} + +Status SniiCompoundWriter::add_logical_index(const SniiIndexInput& in) { + if (out_ == nullptr) + return Status::Error("compound: null file writer"); + if (finished_) + return Status::Error("compound: add after finish"); + if (!failed_.ok()) return failed_; + if (has_active_session()) + return Status::Error( + "compound: add_logical_index while a streamed index session is active (its " + "posting region streams straight into the output; interleaving would corrupt " + "both indexes)"); + RETURN_IF_ERROR(ensure_bootstrap()); + auto liw = std::make_unique(in); + Placement p; + // The posting region streams DIRECTLY into the container during build() -- no temp + // round-trip for the bulk -- followed immediately by this index's compact DICT + // trailer (produced interleaved into a temp, but laid out right after its posting + // region, preserving the per-index [posting][dict] layout). Offsets are read off + // the output writer (the single source of truth -- no separate cursor). + p.post_off = out_->bytes_written(); + Status status = liw->build(out_); + if (!status.ok()) return poison(status); + p.post_len = out_->bytes_written() - p.post_off; + p.dict_off = out_->bytes_written(); + status = liw->stream_dict_region_into(out_); + if (!status.ok()) return poison(status); + p.dict_len = out_->bytes_written() - p.dict_off; + indexes_.push_back(std::move(liw)); + placements_.push_back(p); + return Status::OK(); +} + +SniiIndexInput SniiStreamedIndexSession::attach_encoded_norms(SniiIndexInput in, + TrackedEncodedNorms* encoded_norms, + uint64_t reserved_bytes) { + DORIS_CHECK(encoded_norms != nullptr); + DORIS_CHECK(in.encoded_norms.empty()); + if (in.mem_reporter != nullptr) { + DORIS_CHECK_EQ(reserved_bytes, encoded_norms->norms_.capacity()); + } else { + DORIS_CHECK_EQ(reserved_bytes, 0); + } + in.encoded_norms = std::move(encoded_norms->norms_); + return in; +} + +SniiStreamedIndexSession::SniiStreamedIndexSession(SniiCompoundWriter* owner, SniiIndexInput in, + TrackedNullDocids null_docids, + TrackedEncodedNorms encoded_norms) + : owner_(owner), + encoded_norms_reservation_(std::move(encoded_norms.reservation_)), + input_(attach_encoded_norms(std::move(in), &encoded_norms, + encoded_norms_reservation_.bytes())), + // input_ (a member, initialized above) owns the vectors the writer + // keeps references into -- NOT the caller's already-moved-from `in`. + writer_(new LogicalIndexWriter(input_, std::move(null_docids))), + semantic_token_count_required_( + input_.common_grams_metadata.has_value() && + input_.common_grams_metadata->scoring_coverage == + segment_v2::inverted_index::ScoringCoverage::kComplete) {} + +Status SniiStreamedIndexSession::push_term(StreamedTermPostings&& tp) { + if (!owner_->failed_.ok()) return owner_->failed_; + if (finished_) { + return Status::Error( + "compound: push_term on a finished streamed index session"); + } + const Status status = writer_->push_term(std::move(tp)); + if (!status.ok()) return owner_->poison(status); + return Status::OK(); +} + +Status SniiStreamedIndexSession::set_semantic_token_count(uint64_t token_count) { + if (!owner_->failed_.ok()) return owner_->failed_; + if (finished_) { + return Status::Error( + "compound: semantic token count on a finished streamed index session"); + } + if (!semantic_token_count_required_) { + return Status::Error( + "compound: streamed index session has no complete semantic scoring metadata"); + } + if (semantic_token_count_set_) { + return Status::Error( + "compound: semantic token count was already set"); + } + DORIS_CHECK(input_.common_grams_metadata.has_value()); + DORIS_CHECK(writer_->common_grams_metadata_.has_value()); + input_.common_grams_metadata->scoring_token_count = token_count; + writer_->common_grams_metadata_->scoring_token_count = token_count; + semantic_token_count_set_ = true; + return Status::OK(); +} + +Status SniiStreamedIndexSession::finish() { + if (!owner_->failed_.ok()) return owner_->failed_; + if (finished_) { + return Status::Error( + "compound: finish on an already-finished streamed index session"); + } + if (semantic_token_count_required_ && !semantic_token_count_set_) { + return owner_->poison(Status::Error( + "compound: semantic token count must be set before streamed index finish")); + } + return owner_->finish_streamed_index(this); +} + +void SniiStreamedIndexSession::abort(const Status& cause) { + DCHECK(!cause.ok()); + static_cast(owner_->poison(cause)); +} + +Status SniiCompoundWriter::begin_streamed_index(SniiIndexInput in, + SniiStreamedIndexSession** session) { + std::vector null_docids; + null_docids.swap(in.null_docids); + MemoryReporter::Reservation null_docids_reservation = + in.mem_reporter == nullptr ? MemoryReporter::Reservation() + : in.mem_reporter->make_reservation(); + if (in.mem_reporter != nullptr) { + RETURN_IF_ERROR(null_docids_reservation.set_bytes( + static_cast(null_docids.capacity()) * sizeof(uint32_t))); + } + return begin_streamed_index( + std::move(in), + TrackedNullDocids(std::move(null_docids_reservation), std::move(null_docids)), session); +} + +Status SniiCompoundWriter::begin_streamed_index(SniiIndexInput in, TrackedNullDocids null_docids, + SniiStreamedIndexSession** session) { + std::vector encoded_norms; + encoded_norms.swap(in.encoded_norms); + MemoryReporter::Reservation encoded_norms_reservation = + in.mem_reporter == nullptr ? MemoryReporter::Reservation() + : in.mem_reporter->make_reservation(); + if (in.mem_reporter != nullptr) { + RETURN_IF_ERROR(encoded_norms_reservation.set_bytes(encoded_norms.capacity())); + } + return begin_streamed_index( + std::move(in), std::move(null_docids), + TrackedEncodedNorms(std::move(encoded_norms_reservation), std::move(encoded_norms)), + session); +} + +Status SniiCompoundWriter::begin_streamed_index(SniiIndexInput in, TrackedNullDocids null_docids, + TrackedEncodedNorms encoded_norms, + SniiStreamedIndexSession** session) { + if (session == nullptr) + return Status::Error( + "compound: null session out parameter"); + *session = nullptr; + if (out_ == nullptr) + return Status::Error("compound: null file writer"); + if (finished_) + return Status::Error("compound: begin after finish"); + if (!failed_.ok()) return failed_; + if (has_active_session()) + return Status::Error( + "compound: a streamed index session is already active (one at a time: its " + "posting region streams straight into the container output)"); + // A streamed session takes terms ONLY via push_term; a term source or a + // materialized vector would silently be ignored by the streamed writer. + if (in.term_source != nullptr || !in.terms.empty()) + return Status::Error( + "compound: a streamed index session must not carry a term source or " + "materialized terms"); + if (!in.null_docids.empty()) + return Status::Error( + "compound: tracked streamed NULL docids must not also be present in input"); + if (!in.encoded_norms.empty()) + return Status::Error( + "compound: tracked streamed norms must not also be present in input"); + RETURN_IF_ERROR(ensure_bootstrap()); + auto s = std::unique_ptr(new SniiStreamedIndexSession( + this, std::move(in), std::move(null_docids), std::move(encoded_norms))); + s->post_off_ = out_->bytes_written(); + RETURN_IF_ERROR(s->writer_->begin_streamed(out_)); + sessions_.push_back(std::move(s)); + *session = sessions_.back().get(); + return Status::OK(); +} + +Status SniiCompoundWriter::finish_streamed_index(SniiStreamedIndexSession* session) { + // Flushes the trailing DICT block and finalizes the stats / null-bitmap / + // BSBF sections (poisoning the logical writer on failure). + Status status = session->writer_->finish_streamed(); + if (!status.ok()) return poison(status); + // finalize materialized the framed norms section. Drop the source vector and + // its transferred charge before retaining that section for compound finish. + std::vector().swap(session->input_.encoded_norms); + session->encoded_norms_reservation_.reset(); + Placement p; + p.post_off = session->post_off_; + p.post_len = out_->bytes_written() - p.post_off; + p.dict_off = out_->bytes_written(); + status = session->writer_->stream_dict_region_into(out_); + if (!status.ok()) return poison(status); + p.dict_len = out_->bytes_written() - p.dict_off; + // Only now does the index join the container. Any failure above leaves + // finished_ false, so finish() keeps failing loudly instead of sealing a + // tail that silently omits an index whose posting bytes are in the file. + indexes_.push_back(std::move(session->writer_)); + placements_.push_back(p); + session->finished_ = true; + return Status::OK(); +} + +Status SniiCompoundWriter::write_bootstrap() { + BootstrapHeader bh; + bh.tail_pointer_size = static_cast(format::tail_pointer_size()); + ByteSink sink; + RETURN_IF_ERROR(format::encode_bootstrap_header(bh, &sink)); + return append(sink.buffer()); +} + +// Writes each index's norms POD then bsbf section (in add order), after all the +// per-index [posting][dict] regions. +Status SniiCompoundWriter::write_norms() { + for (size_t i = 0; i < indexes_.size(); ++i) { + const LogicalIndexWriter& w = *indexes_[i]; + if (!w.has_norms() || w.norms_bytes().empty()) continue; + Placement& p = placements_[i]; + p.norms_off = out_->bytes_written(); + RETURN_IF_ERROR(append(w.norms_bytes())); + p.norms_len = out_->bytes_written() - p.norms_off; + indexes_[i]->release_norms_bytes(); + } + for (size_t i = 0; i < indexes_.size(); ++i) { + LogicalIndexWriter& w = *indexes_[i]; + if (!w.has_null_bitmap()) continue; + Placement& p = placements_[i]; + p.null_off = out_->bytes_written(); + RETURN_IF_ERROR(append(w.null_bitmap_bytes())); + p.null_len = out_->bytes_written() - p.null_off; + w.release_null_bitmap_bytes(); + } + for (size_t i = 0; i < indexes_.size(); ++i) { + LogicalIndexWriter& w = *indexes_[i]; + if (!w.has_bsbf()) continue; + Placement& p = placements_[i]; + p.bsbf_off = out_->bytes_written(); + RETURN_IF_ERROR(append(w.bsbf_bytes())); + p.bsbf_len = out_->bytes_written() - p.bsbf_off; + w.release_bsbf_bytes(); + } + return Status::OK(); +} + +Status SniiCompoundWriter::write_tail() { + std::vector directory_entries; + directory_entries.reserve(indexes_.size()); + for (size_t i = 0; i < indexes_.size(); ++i) { + const LogicalIndexWriter& w = *indexes_[i]; + const Placement& p = placements_[i]; + + SectionRefs refs; + refs.dict_region = {.offset = p.dict_off, .length = p.dict_len}; + refs.posting_region = {.offset = p.post_off, .length = p.post_len}; + refs.norms = {.offset = p.norms_off, .length = p.norms_len}; + refs.null_bitmap = {.offset = p.null_off, .length = p.null_len}; + refs.bsbf = {.offset = p.bsbf_off, .length = p.bsbf_len}; + + SerializedMetadataGroup group; + RETURN_IF_ERROR(w.finish_metadata(refs, p.dict_off, &group)); + + LogicalIndexMetadataRef entry; + entry.index_id = w.index_id(); + entry.index_suffix = w.index_suffix(); + entry.core_metadata = {.offset = out_->bytes_written(), .length = group.core.size()}; + RETURN_IF_ERROR(append(group.core)); + DORIS_CHECK_EQ(out_->bytes_written(), + entry.core_metadata.offset + entry.core_metadata.length); + + entry.sampled_term_index = {.offset = out_->bytes_written(), + .length = group.sampled_term_index.size()}; + DORIS_CHECK_EQ(entry.sampled_term_index.offset, + entry.core_metadata.offset + entry.core_metadata.length); + RETURN_IF_ERROR(append(group.sampled_term_index)); + DORIS_CHECK_EQ(out_->bytes_written(), + entry.sampled_term_index.offset + entry.sampled_term_index.length); + + entry.dict_block_directory = {.offset = out_->bytes_written(), + .length = group.dict_block_directory.size()}; + DORIS_CHECK_EQ(entry.dict_block_directory.offset, + entry.sampled_term_index.offset + entry.sampled_term_index.length); + RETURN_IF_ERROR(append(group.dict_block_directory)); + DORIS_CHECK_EQ(out_->bytes_written(), + entry.dict_block_directory.offset + entry.dict_block_directory.length); + directory_entries.push_back(std::move(entry)); + } + + ByteSink directory_sink; + RETURN_IF_ERROR(format::encode_metadata_directory(directory_entries, &directory_sink)); + const uint64_t directory_offset = out_->bytes_written(); + RETURN_IF_ERROR(append(directory_sink.buffer())); + const uint64_t directory_length = out_->bytes_written() - directory_offset; + + TailPointer tp; + tp.directory_offset = directory_offset; + tp.directory_length = directory_length; + tp.directory_crc32c = crc32c(directory_sink.view()); + ByteSink tail_sink; + RETURN_IF_ERROR(format::encode_tail_pointer(tp, &tail_sink)); + return append(tail_sink.buffer()); +} + +Status SniiCompoundWriter::finish() { + if (out_ == nullptr) + return Status::Error("compound: null file writer"); + if (!failed_.ok()) { + return failed_; + } + if (finished_) + return Status::Error("compound: finish called twice"); + // Crash-safety invariant 6: a begun-but-unfinished streamed session already + // streamed posting bytes into the file but recorded no placement; sealing + // the container now would silently drop that index. Fail loudly -- the + // whole compaction round must fail and be retried. + if (has_active_session()) + return Status::Error( + "compound: finish with an unfinished streamed index session; the half-fed " + "index must never be sealed away silently"); + finished_ = true; + + RETURN_IF_ERROR(ensure_bootstrap()); // empty container still gets a header + Status status = write_norms(); + if (!status.ok()) return poison(status); + status = write_tail(); + if (!status.ok()) return poison(status); + status = out_->finalize(); + if (!status.ok()) return poison(status); + return Status::OK(); +} + +} // namespace doris::snii::writer diff --git a/be/src/storage/index/snii/writer/snii_compound_writer.h b/be/src/storage/index/snii/writer/snii_compound_writer.h new file mode 100644 index 00000000000000..48243581343075 --- /dev/null +++ b/be/src/storage/index/snii/writer/snii_compound_writer.h @@ -0,0 +1,206 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/io/file_writer.h" +#include "storage/index/snii/writer/logical_index_writer.h" + +// SniiCompoundWriter -- orchestrates a single-segment SNII container for one or +// more logical indexes, written front-to-back through an append-only +// io::FileWriter (no seek-back). It resolves all back-references by writing the +// metadata groups, raw directory, and fixed tail pointer LAST. +// +// CONTAINER LAYOUT PRODUCED (this is the on-disk contract the reader matches): +// [bootstrap_header] (kBootstrapHeaderSize bytes) +// for each logical index, in add order: +// [posting region] interleaved [prx][frq] per pod_ref term, term order +// (prx span empty when !has_prx) +// [DICT blocks region] concatenated DICT blocks, split by +// target_dict_block_bytes +// for each logical index, in add order: +// [norms POD] NormsPodWriter::finish (scoring only; else absent) +// [null bitmap POD] NullBitmapWriter::finish (when nulls exist) +// for each logical index, in add order: +// [Core metadata][SampledTermIndex blob][DICT block directory blob] +// [metadata directory] raw SniiMetadataDirectoryPB bytes +// [tail_pointer] encode_tail_pointer at EOF +// +// (The posting region is streamed BEFORE the DICT region per index: postings are +// the large append-only term-ordered stream; the DICT region is the compact +// compressed trailer.) +// +// OFFSET CONVENTIONS (ABSOLUTE file offsets unless stated otherwise): +// - SectionRefs in each Core metadata record ABSOLUTE file offset+length of +// that index's posting, DICT, norms, null-bitmap, and BSBF regions. Absent +// regions are (0,0); a present-but-empty posting region (all-INLINE index) +// is (off, 0). +// - DictBlockDirectory entries record each DICT block's ABSOLUTE file offset + +// length. +// - A windowed/slim pod_ref entry's absolute .frq offset = +// section_refs.posting_region.offset + frq_base + frq_off_delta +// where frq_base is the posting-region-relative running offset captured at the +// block's open (see logical_index_writer.h). prx follows the identical rule +// against the SAME region (prx_base == frq_base). +// - tail_pointer.directory_offset/length point at the raw metadata directory. +namespace doris::snii::writer { + +class SniiCompoundWriter; + +// T2.2 (compaction index merge fast path): handle for ONE streamed logical-index +// session inside a SniiCompoundWriter, obtained from begin_streamed_index(). The +// caller pushes lexicographically sorted terms (the k-way merge output) and seals +// the index with finish(), which lays the [posting][dict] regions out exactly like +// add_logical_index. The handle is owned by the compound writer and stays valid +// for the writer's lifetime; it must not outlive it. +// +// CRASH SAFETY (invariant 6): a session that was begun but never successfully +// finished keeps the container permanently unsealable -- its posting bytes are +// already in the file, so SniiCompoundWriter::finish() fails loudly instead of +// writing a tail that silently omits the half-fed index. +class SniiStreamedIndexSession { +public: + SniiStreamedIndexSession(const SniiStreamedIndexSession&) = delete; + SniiStreamedIndexSession& operator=(const SniiStreamedIndexSession&) = delete; + SniiStreamedIndexSession(SniiStreamedIndexSession&&) = delete; + SniiStreamedIndexSession& operator=(SniiStreamedIndexSession&&) = delete; + + // Terms must arrive in strictly increasing lexicographic order with + // ascending-docid postings. Unlike the standalone LogicalIndexWriter API, + // every rejection is terminal here because posting bytes may already have + // entered the compound output; all later calls return the first error. + Status push_term(StreamedTermPostings&& tp); + // Binds the semantic (plain-token) count after the merge's single postings + // pass. Complete scoring metadata requires exactly one call, including for + // an empty destination whose count is zero. + Status set_semantic_token_count(uint64_t token_count); + // Seals this index: flushes the trailing DICT block, streams the DICT region + // right after the posting region and records the placements. A failed finish + // leaves the session unfinished (and the container unsealable) -- there is + // no retry, the whole compaction round must fail and be redone. + Status finish(); + // Makes the owning compound permanently unsealable. Merge plans call this + // on every destination when any source or sibling destination fails, because + // a successfully written prefix is not a complete logical index. + void abort(const Status& cause); + bool finished() const { return finished_; } + +private: + friend class SniiCompoundWriter; + SniiStreamedIndexSession(SniiCompoundWriter* owner, SniiIndexInput in, + TrackedNullDocids null_docids, TrackedEncodedNorms encoded_norms); + static SniiIndexInput attach_encoded_norms(SniiIndexInput in, + TrackedEncodedNorms* encoded_norms, + uint64_t reserved_bytes); + + SniiCompoundWriter* owner_; + // The reservation precedes input_ so input_.encoded_norms is destroyed + // before its charge is released. + MemoryReporter::Reservation encoded_norms_reservation_; + // Owns the input: LogicalIndexWriter keeps references into it (terms / + // encoded_norms), so it must live exactly as long as the writer. + SniiIndexInput input_; + std::unique_ptr writer_; + uint64_t post_off_ = 0; + bool semantic_token_count_required_ = false; + bool semantic_token_count_set_ = false; + bool finished_ = false; +}; + +class SniiCompoundWriter { +public: + explicit SniiCompoundWriter(io::FileWriter* out); + SniiCompoundWriter(const SniiCompoundWriter&) = delete; + SniiCompoundWriter& operator=(const SniiCompoundWriter&) = delete; + SniiCompoundWriter(SniiCompoundWriter&&) = delete; + SniiCompoundWriter& operator=(SniiCompoundWriter&&) = delete; + + // Buffers one logical index: builds its section bytes and meta sub-sections. + // The actual file writing happens in finish() (single front-to-back pass). + Status add_logical_index(const SniiIndexInput& in); + + // T2.2: begins a STREAMED logical-index session (the compaction merge fast + // path) -- the caller pushes pre-merged terms through *session instead of + // handing the writer a term source, so `in` must carry NO term_source and NO + // materialized terms. Only ONE session may be active at a time (its posting + // region streams straight into the container output, so a concurrent + // add_logical_index or second session would interleave bytes); both are + // rejected while a session is unfinished, as is finish(). The returned + // handle is owned by this writer and valid for its lifetime. + Status begin_streamed_index(SniiIndexInput in, SniiStreamedIndexSession** session); + Status begin_streamed_index(SniiIndexInput in, TrackedNullDocids null_docids, + SniiStreamedIndexSession** session); + Status begin_streamed_index(SniiIndexInput in, TrackedNullDocids null_docids, + TrackedEncodedNorms encoded_norms, + SniiStreamedIndexSession** session); + + // Writes bootstrap header + all index sections + adjacent metadata groups + + // raw directory + tail pointer, then finalizes the underlying writer. + Status finish(); + +private: + // Absolute placement of one index's sections, resolved during finish(). + struct Placement { + uint64_t dict_off = 0; + uint64_t dict_len = 0; + uint64_t post_off = 0; // interleaved [prx][frq] posting region (was frq + prx) + uint64_t post_len = 0; + uint64_t norms_off = 0; + uint64_t norms_len = 0; + uint64_t null_off = 0; + uint64_t null_len = 0; + uint64_t bsbf_off = 0; + uint64_t bsbf_len = 0; + }; + + friend class SniiStreamedIndexSession; + + Status ensure_bootstrap(); + Status write_bootstrap(); + Status write_norms(); + Status write_tail(); + Status append(const std::vector& bytes); + Status poison(Status status); + // Seals one streamed session: records its placements and adopts its + // LogicalIndexWriter into indexes_ (only on FULL success -- see the + // crash-safety note on SniiStreamedIndexSession). + Status finish_streamed_index(SniiStreamedIndexSession* session); + // An unfinished streamed session (begun but not successfully sealed): + // blocks add_logical_index, another begin_streamed_index and finish(). + bool has_active_session() const { return !sessions_.empty() && !sessions_.back()->finished(); } + + io::FileWriter* out_; + std::vector> indexes_; + // Streamed sessions in begin order (at most the last one is unfinished). + // Owned here so the raw handles returned to callers stay valid for the + // writer's lifetime; a finished session is inert (its writer moved out). + std::vector> sessions_; + // Per-index placement; post_off/post_len are filled as each index's posting region + // streams in during add_logical_index, the rest during finish(). The absolute write + // offset is out_->bytes_written() (the single source of truth -- no separate cursor). + std::vector placements_; + bool bootstrap_written_ = false; + bool finished_ = false; + Status failed_ = Status::OK(); +}; + +} // namespace doris::snii::writer diff --git a/be/src/storage/index/snii/writer/spill_run_codec.cpp b/be/src/storage/index/snii/writer/spill_run_codec.cpp new file mode 100644 index 00000000000000..b85f903c26e0f4 --- /dev/null +++ b/be/src/storage/index/snii/writer/spill_run_codec.cpp @@ -0,0 +1,937 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/writer/spill_run_codec.h" + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "storage/index/snii/encoding/varint.h" +#include "storage/index/snii/format/format_constants.h" + +namespace doris::snii::writer { + +namespace { + +// Flush staging at this exact bound. A large write buffer (4 MiB) collapses the +// per-flush write() syscall count by ~64x: at 64 KiB the 5M build issued +// ~8800 write()s to ext4 (~9s of syscall overhead) for ~553 MiB of runs, versus +// a raw dd of the same bytes taking ~1.2s. Wide terms are appended and flushed +// in chunks, so the staging allocation never grows with term width. +constexpr size_t kWriteFlushBytes = 1u << 22; // 4 MiB +// RunReader reads this much per disk fill; the window slides so a single record +// never needs the whole run in RAM (only the current term's encoded span). KEEP +// this small (64 KiB): a large read chunk x many open runs would inflate the +// merge-phase peak RSS at low spill thresholds (each reader holds a window). +constexpr size_t kReadChunkBytes = 1u << 16; // 64 KiB + +enum class RunPostingShape : uint8_t { + kDocsOnlyStatless = 0, + kDocsAndFreqs = 1, + kPositioned = 2, +}; + +RunPostingShape posting_shape(const TermPostings& tp) { + if (tp.retain_positions) { + return RunPostingShape::kPositioned; + } + return tp.freqs.empty() ? RunPostingShape::kDocsOnlyStatless : RunPostingShape::kDocsAndFreqs; +} + +// Writes the full byte range [data, data+len) to fd, looping over short writes. +Status write_all(int fd, const uint8_t* data, size_t len) { + size_t off = 0; + while (off < len) { + const ssize_t n = ::write(fd, data + off, len - off); + if (n < 0) { + if (errno == EINTR) continue; + return Status::Error(std::string("run write failed: ") + + std::strerror(errno)); + } + off += static_cast(n); + } + return Status::OK(); +} + +template +Status reserve_vector_for_size(std::vector* values, size_t target, + MemoryReporter* memory_reporter, + MemoryReporter::Reservation* reservation) { + if (target <= values->capacity()) { + return Status::OK(); + } + if (target > std::numeric_limits::max() / sizeof(T)) { + return Status::Error( + "run reader: vector byte capacity overflow"); + } + if (memory_reporter == nullptr) { + values->reserve(target); + return Status::OK(); + } + MemoryReporter::Reservation replacement; + const uint64_t target_bytes = static_cast(target) * sizeof(T); + RETURN_IF_ERROR(reservation->prepare_replacement(target_bytes, &replacement)); + values->reserve(target); + DCHECK_EQ(values->capacity(), target); + *reservation = std::move(replacement); + return Status::OK(); +} + +Status reserve_write_buffer_for_append(std::vector* buffer, size_t target, + MemoryReporter* memory_reporter, + MemoryReporter::Reservation* reservation) { + if (target <= buffer->capacity()) { + return Status::OK(); + } + if (target > kWriteFlushBytes) { + return Status::Error( + "run writer: staging buffer exceeds flush bound"); + } + + size_t capacity = std::max(buffer->capacity(), 1); + while (capacity < target) { + capacity = capacity > kWriteFlushBytes / 2 ? kWriteFlushBytes : capacity * 2; + } + return reserve_vector_for_size(buffer, capacity, memory_reporter, reservation); +} + +} // namespace + +// --------------------------------------------------------------------------- +// RunWriter +// --------------------------------------------------------------------------- + +RunWriter::RunWriter(MemoryReporter* memory_reporter) + : memory_reporter_(memory_reporter), + buffer_reservation_(memory_reporter == nullptr ? MemoryReporter::Reservation() + : memory_reporter->make_reservation()) {} + +RunWriter::~RunWriter() { + if (fd_ >= 0) ::close(fd_); +} + +Status RunWriter::open(const std::string& path) { + fd_ = ::open(path.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0600); + if (fd_ < 0) { + return Status::Error("run open(" + path + + "): " + std::strerror(errno)); + } + buf_.clear(); + return Status::OK(); +} + +Status RunWriter::flush() { + if (buf_.empty()) return Status::OK(); + RETURN_IF_ERROR(write_all(fd_, buf_.data(), buf_.size())); + buf_.clear(); + return Status::OK(); +} + +Status RunWriter::append_bytes(const uint8_t* data, size_t size) { + while (size != 0) { + if (buf_.size() == kWriteFlushBytes) { + RETURN_IF_ERROR(flush()); + } + const size_t count = std::min(size, kWriteFlushBytes - buf_.size()); + const size_t target = buf_.size() + count; + RETURN_IF_ERROR(reserve_write_buffer_for_append(&buf_, target, memory_reporter_, + &buffer_reservation_)); + buf_.insert(buf_.end(), data, data + count); + data += count; + size -= count; + } + return Status::OK(); +} + +Status RunWriter::append_varint(uint64_t value) { + uint8_t bytes[10]; + const size_t size = encode_varint64(value, bytes); + return append_bytes(bytes, size); +} + +Status RunWriter::append_raw_u32(const uint32_t* values, size_t count) { + if (count > std::numeric_limits::max() / sizeof(uint32_t)) { + return Status::Error( + "run writer: raw u32 byte count overflows size_t"); + } + return append_bytes(reinterpret_cast(values), count * sizeof(uint32_t)); +} + +void RunWriter::release_buffer() { + std::vector().swap(buf_); + buffer_reservation_.reset(); +} + +Status RunWriter::write_term(uint32_t term_id, const TermPostings& tp) { + DCHECK(tp.retain_positions || tp.positions_flat.empty()); + const RunPostingShape shape = posting_shape(tp); + const size_t doc_count = tp.document_count(); + if (shape != RunPostingShape::kDocsOnlyStatless) { + DCHECK_EQ(tp.docids.size(), tp.freqs.size()); + } + RETURN_IF_ERROR(append_varint(term_id)); + RETURN_IF_ERROR(append_varint(static_cast(shape))); + RETURN_IF_ERROR(append_varint(doc_count)); + // Docids are a RAW fixed-width u32 block (bulk memcpy), NOT per-value VInt. + // Per-value varint over ~60M docids cost ~1.5s of encode CPU on the spill feed + // side; raw is a single memcpy and the decode side becomes a memcpy too. Runs + // are PRIVATE temp files written then read back from page cache, so the modestly + // larger run (no delta packing) costs ~0 extra real I/O. Absolute docids are + // stored (the merge concatenates per-term across runs and re-deltas at encode). + RETURN_IF_ERROR(append_raw_u32(tp.docids.data(), tp.docids.size())); + if (shape != RunPostingShape::kDocsOnlyStatless) { + RETURN_IF_ERROR(append_raw_u32(tp.freqs.data(), tp.freqs.size())); + } + if (shape == RunPostingShape::kPositioned) { + const uint64_t n_pos = tp.positions_flat.size(); + RETURN_IF_ERROR(append_varint(n_pos)); + RETURN_IF_ERROR(append_raw_u32(tp.positions_flat.data(), tp.positions_flat.size())); + } + return Status::OK(); +} + +Status RunWriter::close() { + if (fd_ < 0) return Status::OK(); + RETURN_IF_ERROR(flush()); + const int fd = fd_; + fd_ = -1; + if (::close(fd) != 0) { + return Status::Error(std::string("run close: ") + + std::strerror(errno)); + } + release_buffer(); + return Status::OK(); +} + +// --------------------------------------------------------------------------- +// RunReader +// --------------------------------------------------------------------------- + +RunReader::RunReader(MemoryReporter* memory_reporter) + : memory_reporter_(memory_reporter), + window_reservation_(memory_reporter == nullptr ? MemoryReporter::Reservation() + : memory_reporter->make_reservation()), + docids_reservation_(memory_reporter == nullptr ? MemoryReporter::Reservation() + : memory_reporter->make_reservation()), + freqs_reservation_(memory_reporter == nullptr ? MemoryReporter::Reservation() + : memory_reporter->make_reservation()), + positions_reservation_(memory_reporter == nullptr ? MemoryReporter::Reservation() + : memory_reporter->make_reservation()) { +} + +RunReader::~RunReader() { + if (fd_ >= 0) ::close(fd_); +} + +Status RunReader::open(const std::string& path, bool has_positions) { + fd_ = ::open(path.c_str(), O_RDONLY); + if (fd_ < 0) { + return Status::Error("run reopen(" + path + + "): " + std::strerror(errno)); + } + // Record the run's byte size so every length decoded from the stream can be + // bounded against it before allocating (no record holds more u32s than the whole + // file). Honors the header's "lengths validated against the file size" contract, + // turning a corrupt/truncated length into Status::Corruption rather than an + // uncaught std::bad_alloc from a giant resize(). + struct stat st {}; + if (::fstat(fd_, &st) != 0) { + return Status::Error(std::string("run fstat: ") + + std::strerror(errno)); + } + file_size_ = static_cast(st.st_size); + bytes_read_ = 0; + has_positions_ = has_positions; + exhausted_ = false; + eof_ = false; + pos_ = 0; + pos_count_ = 0; + pos_remaining_ = 0; + window_.clear(); + return advance(); +} + +// Slides consumed bytes out of the window, then appends one disk chunk. +Status RunReader::fill() { + if (pos_ > 0) { + window_.erase(window_.begin(), window_.begin() + pos_); + pos_ = 0; + } + if (eof_) return Status::OK(); + if (bytes_read_ > file_size_) { + return Status::Error( + "run reader: bytes read exceed file size"); + } + const uint64_t remaining = file_size_ - bytes_read_; + if (remaining == 0) { + eof_ = true; + return Status::OK(); + } + const size_t read_size = static_cast( + std::min(remaining, static_cast(kReadChunkBytes))); + const size_t base = window_.size(); + if (base > std::numeric_limits::max() - read_size) { + return Status::Error( + "run reader: decode window capacity overflow"); + } + RETURN_IF_ERROR(reserve_vector_for_size(&window_, base + read_size, memory_reporter_, + &window_reservation_)); + window_.resize(base + read_size); + ssize_t n; + do { + n = ::read(fd_, window_.data() + base, read_size); + } while (n < 0 && errno == EINTR); + if (n < 0) + return Status::Error(std::string("run read: ") + + std::strerror(errno)); + window_.resize(base + static_cast(n)); + bytes_read_ += static_cast(n); + if (n == 0 || bytes_read_ == file_size_) eof_ = true; + return Status::OK(); +} + +// Buffered bytes available to the decoder right now (from pos_ to window end). +// fill() may slide the window (erasing consumed bytes), so callers must compare +// THIS quantity -- not window_.size() -- to decide whether more data arrived. +size_t RunReader::available() const { + return window_.size() - pos_; +} + +Status RunReader::ensure(size_t n) { + while (available() < n) { + const size_t had = available(); + RETURN_IF_ERROR(fill()); + if (available() == had && eof_) { + return Status::Error( + "run truncated: needed more bytes than available"); + } + } + return Status::OK(); +} + +// Streamed varint: decode from the current window; if it straddles the buffered +// boundary, top up from disk and retry. A varint is at most 10 bytes, so this +// loops at most a couple of times. Bounds-safe: decode_varint64 never reads past +// `end`, and a partial varint at true eof is reported as corruption. +Status RunReader::read_varint(uint64_t* v) { + while (true) { + const uint8_t* p = window_.data() + pos_; + const uint8_t* end = window_.data() + window_.size(); + const uint8_t* next = nullptr; + Status s = decode_varint64(p, end, v, &next); + if (s.ok()) { + pos_ += static_cast(next - p); + return Status::OK(); + } + if (eof_) + return Status::Error( + "run truncated: incomplete varint"); + const size_t had = available(); + RETURN_IF_ERROR(fill()); + if (available() == had && eof_) { + return Status::Error( + "run truncated: incomplete varint at eof"); + } + } +} + +// Streams `count` raw little-endian u32s from the window into `dst` (caller-owned +// storage of at least count*4 bytes), topping up the window from disk as needed. +// Copies whatever is buffered each pass (the window may hold only part of a large +// block), so a high-df term's freqs/positions stream through in 64 KiB chunks +// without ever needing the whole block resident at once. +Status RunReader::pull_raw_u32(uint8_t* dst, size_t count) { + if (count == 0) return Status::OK(); + if (count > std::numeric_limits::max() / sizeof(uint32_t)) { + return Status::Error( + "run: raw u32 byte count overflows size_t"); + } + size_t need = count * sizeof(uint32_t); + size_t written = 0; + while (need > 0) { + if (available() == 0) { + const size_t had = available(); + RETURN_IF_ERROR(fill()); + if (available() == had && eof_) { + return Status::Error( + "run truncated: needed more raw bytes than available"); + } + } + const size_t take = std::min(need, available()); + std::memcpy(dst + written, window_.data() + pos_, take); + pos_ += take; + written += take; + need -= take; + } + return Status::OK(); +} + +// Bulk-decodes `count` raw u32s into `out` (resized to count). +Status RunReader::read_raw_u32(size_t count, std::vector* out, + MemoryReporter::Reservation* reservation) { + // Bound `count` against the run's byte size BEFORE resize(): a record can never + // hold more u32s than the whole file. Rejects a corrupt/truncated length varint + // (which is otherwise an unbounded resize -> uncaught std::bad_alloc). + if (count > file_size_ / sizeof(uint32_t)) { + return Status::Error( + "run: raw u32 count exceeds file size"); + } + RETURN_IF_ERROR(reserve_vector_for_size(out, count, memory_reporter_, reservation)); + out->resize(count); + if (count == 0) return Status::OK(); + return pull_raw_u32(reinterpret_cast(out->data()), count); +} + +// Materializes the current term's deferred position block into positions_flat. +// A no-op once the positions are already drained (idempotent within a term). +Status RunReader::materialize_positions() { + if (pos_remaining_ == 0) { + current_.positions_flat.clear(); + return Status::OK(); + } + if (pos_remaining_ > std::numeric_limits::max()) { + return Status::Error( + "run: position count exceeds addressable memory"); + } + const size_t n = static_cast(pos_remaining_); + RETURN_IF_ERROR(read_raw_u32(n, ¤t_.positions_flat, &positions_reservation_)); + pos_remaining_ = 0; + return Status::OK(); +} + +// Streams the next `n` positions of the current term straight from the window. +Status RunReader::stream_positions(uint32_t* dst, size_t n) { + if (n == 0) return Status::OK(); + if (n > pos_remaining_) { + return Status::Error( + "run: stream_positions past block end"); + } + RETURN_IF_ERROR(pull_raw_u32(reinterpret_cast(dst), n)); + pos_remaining_ -= n; + return Status::OK(); +} + +// Discards any positions of the current term left unread, so the window cursor +// lands at the next record boundary before advance() reads the next term. +Status RunReader::skip_remaining_positions() { + if (pos_remaining_ == 0) return Status::OK(); + std::array scratch; + while (pos_remaining_ != 0) { + const size_t count = static_cast( + std::min(pos_remaining_, static_cast(scratch.size()))); + RETURN_IF_ERROR(pull_raw_u32(reinterpret_cast(scratch.data()), count)); + pos_remaining_ -= count; + } + return Status::OK(); +} + +Status RunReader::advance() { + // Drain any positions the owner left unread for the previous term so the window + // cursor lands at the next record boundary. + RETURN_IF_ERROR(skip_remaining_positions()); + // End-of-run detection: at a record boundary, if no bytes remain we are done. + if (available() == 0) { + RETURN_IF_ERROR(fill()); + if (available() == 0 && eof_) { + exhausted_ = true; + return Status::OK(); + } + } + uint64_t term_id = 0; + RETURN_IF_ERROR(read_varint(&term_id)); + if (term_id > UINT32_MAX) + return Status::Error( + "run term_id exceeds uint32"); + current_id_ = static_cast(term_id); + current_.term.clear(); // runs store only the id; owner resolves the string + + uint64_t encoded_shape = 0; + RETURN_IF_ERROR(read_varint(&encoded_shape)); + if (encoded_shape > static_cast(RunPostingShape::kPositioned)) { + return Status::Error( + "run: unknown posting shape"); + } + const auto shape = static_cast(encoded_shape); + if (shape == RunPostingShape::kPositioned && !has_positions_) { + return Status::Error( + "run: positioned record in docs-only run"); + } + if (shape == RunPostingShape::kDocsOnlyStatless && !has_positions_) { + return Status::Error( + "run: statless record requires a positioned mixed-shape run"); + } + + uint64_t n_docs = 0; + RETURN_IF_ERROR(read_varint(&n_docs)); + if (n_docs > file_size_ / sizeof(uint32_t) || n_docs > std::numeric_limits::max()) { + return Status::Error( + "run: document count exceeds file size or addressable memory"); + } + // Docids: RAW absolute u32 block (bulk read), matching the writer's AppendRawU32. + RETURN_IF_ERROR( + read_raw_u32(static_cast(n_docs), ¤t_.docids, &docids_reservation_)); + for (size_t i = 1; i < current_.docids.size(); ++i) { + if (current_.docids[i] <= current_.docids[i - 1]) { + return Status::Error( + "run: docids must be strictly ascending within one record"); + } + } + current_.freqs.clear(); + current_.positions_flat.clear(); + pos_count_ = 0; + pos_remaining_ = 0; + if (shape == RunPostingShape::kDocsOnlyStatless) { + current_.retain_positions = false; + return Status::OK(); + } + + // Freqs: RAW u32 block (bulk read), matching the writer's AppendRawU32. + RETURN_IF_ERROR( + read_raw_u32(static_cast(n_docs), ¤t_.freqs, &freqs_reservation_)); + uint64_t total_freq = 0; + for (uint32_t freq : current_.freqs) { + if (freq == 0) { + return Status::Error( + "run: frequency must be positive"); + } + if (freq > UINT64_MAX - total_freq) { + return Status::Error( + "run: frequency sum overflows uint64"); + } + total_freq += freq; + } + if (shape == RunPostingShape::kDocsAndFreqs) { + current_.retain_positions = false; + return Status::OK(); + } + + uint64_t n_pos = 0; + RETURN_IF_ERROR(read_varint(&n_pos)); + if (n_pos != total_freq) { + return Status::Error( + "run: position count does not match frequency sum"); + } + if (n_pos > file_size_ / sizeof(uint32_t)) { + return Status::Error( + "run: position count exceeds file size"); + } + // Positions are LAZY: record the block count and leave the window cursor parked + // at the block start. The owner picks materialize_positions() for explicit + // materialization or stream_positions() for bounded writer-owned windows. + current_.retain_positions = true; + pos_count_ = n_pos; + pos_remaining_ = n_pos; + return Status::OK(); +} + +// --------------------------------------------------------------------------- +// K-way merge +// --------------------------------------------------------------------------- + +namespace { + +// Min-heap entry: orders by the run's current term-id's PRECOMPUTED integer +// string-rank (rank[term_id] == its lexicographic rank over the dense vocabulary), +// tie-broken by run index so equal terms are gathered run-order (keeping +// concatenated docids ascending). The rank is a lexicographic bijection on a dense +// vocab, so ordering by the dense 4 B rank array reproduces the exact dictionary +// order a vocab-string compare would -- with an integer compare and zero random +// vocab string access in the inner loop. +struct HeapItem { + uint32_t term_id; + size_t run; +}; +struct HeapGreater { + const std::vector* rank; + bool operator()(const HeapItem& a, const HeapItem& b) const { + const uint32_t ra = (*rank)[a.term_id]; + const uint32_t rb = (*rank)[b.term_id]; + if (ra != rb) { + return ra > rb; + } // smaller rank first (lexicographic min-heap) + return a.run > b.run; // same term across runs: run-order tie-break + } +}; + +// Appends src's postings onto dst (run order). Later runs only cover docids +// >= dst's last, so docids stay ascending. COALESCE the boundary doc: if a spill +// fell BETWEEN two tokens of the same doc, that doc ends one run and begins the +// next with the SAME docid -- merge them (sum freqs, splice positions) so the +// merged term has exactly one entry per docid (matching the in-memory build). +// +// Positions are FLAT: doc order, partitioned by freqs. Because both dst and src +// already store doc-ordered flat positions, the common (no-boundary-overlap) case +// is a single bulk append. The boundary-overlap case must INSERT src's first +// doc's positions right after dst's last doc's positions so flat order stays +// consistent with the merged (coalesced) freqs. +void concat(TermPostings* dst, const TermPostings& src, RunPostingShape shape) { + if (src.docids.empty()) return; + const bool has_positions = shape == RunPostingShape::kPositioned; + const bool statless = shape == RunPostingShape::kDocsOnlyStatless; + DCHECK(posting_shape(src) == shape); + size_t start = 0; + size_t src_pos_start = 0; // flat offset of src positions to append after splice + if (!dst->docids.empty() && dst->docids.back() == src.docids.front()) { + const uint32_t head_fc = statless ? 0 : src.freqs.front(); + if (has_positions && head_fc != 0) { + // Splice src's first-doc positions in right after dst's last-doc positions. + // dst's last doc owns dst->freqs.back() entries at the tail of positions_flat + // BEFORE we bump that freq, so insert at end() (last doc is the tail run). + auto& flat = dst->positions_flat; + flat.insert(flat.end(), src.positions_flat.begin(), + src.positions_flat.begin() + head_fc); + } + if (!statless) { + dst->freqs.back() += head_fc; + } + src_pos_start = head_fc; + start = 1; // boundary doc folded in; append the rest + } + dst->docids.insert(dst->docids.end(), src.docids.begin() + start, src.docids.end()); + if (!statless) { + dst->freqs.insert(dst->freqs.end(), src.freqs.begin() + start, src.freqs.end()); + } + if (has_positions) { + dst->positions_flat.insert(dst->positions_flat.end(), + src.positions_flat.begin() + src_pos_start, + src.positions_flat.end()); + } +} + +class RunTermPostingSource final : public TermPostingSource { +public: + RunTermPostingSource(std::vector>* readers, + const std::vector* matching, RunPostingShape shape) + : readers_(readers), matching_(matching), shape_(shape) {} + + Status fill(uint32_t target_docs, TermPostingBuffer* out, bool* exhausted) override { + if (out == nullptr || exhausted == nullptr || target_docs == 0 || !out->empty()) { + return Status::Error( + "run posting source: invalid fill arguments"); + } + + Cursor planned = cursor_; + size_t document_count = 0; + size_t position_count = 0; + while (document_count < target_docs) { + normalize(&planned); + if (planned.run == matching_->size()) { + break; + } + const uint32_t docid = current_docid(planned); + uint64_t frequency = 0; + do { + const TermPostings& postings = current_postings(planned); + if (shape_ != RunPostingShape::kDocsOnlyStatless) { + frequency += postings.freqs[planned.doc]; + if (frequency > std::numeric_limits::max()) { + return Status::Error( + "run: coalesced frequency exceeds uint32"); + } + } + advance(&planned); + normalize(&planned); + if (planned.run == matching_->size()) { + break; + } + const uint32_t next_docid = current_docid(planned); + if (next_docid < docid) { + return Status::Error( + "run: docids overlap across spill runs"); + } + if (next_docid != docid) { + break; + } + } while (true); + if (shape_ == RunPostingShape::kPositioned) { + if (frequency > std::numeric_limits::max() - position_count) { + return Status::Error( + "run posting source: position window exceeds size_t"); + } + position_count += static_cast(frequency); + } + ++document_count; + } + + MutableTermPostingSpan destination; + const bool has_freqs = shape_ != RunPostingShape::kDocsOnlyStatless; + RETURN_IF_ERROR( + out->grow_uninitialized(document_count, has_freqs, position_count, &destination)); + size_t position_offset = 0; + for (size_t output = 0; output < document_count; ++output) { + normalize(&cursor_); + DCHECK_LT(cursor_.run, matching_->size()); + const uint32_t docid = current_docid(cursor_); + uint64_t frequency = 0; + do { + const TermPostings& postings = current_postings(cursor_); + const uint32_t run_frequency = has_freqs ? postings.freqs[cursor_.doc] : 0; + if (shape_ == RunPostingShape::kPositioned) { + RunReader* reader = (*readers_)[(*matching_)[cursor_.run]].get(); + RETURN_IF_ERROR(reader->stream_positions( + destination.positions_flat.data() + position_offset, run_frequency)); + position_offset += run_frequency; + } + frequency += run_frequency; + advance(&cursor_); + normalize(&cursor_); + if (cursor_.run == matching_->size() || current_docid(cursor_) != docid) { + break; + } + } while (true); + destination.docids[output] = docid; + if (has_freqs) { + destination.freqs[output] = static_cast(frequency); + } + } + DCHECK_EQ(position_offset, destination.positions_flat.size()); + normalize(&cursor_); + *exhausted = cursor_.run == matching_->size(); + return Status::OK(); + } + + bool exhausted() { + normalize(&cursor_); + return cursor_.run == matching_->size(); + } + +private: + struct Cursor { + size_t run = 0; + size_t doc = 0; + }; + + const TermPostings& current_postings(const Cursor& cursor) const { + return (*readers_)[(*matching_)[cursor.run]]->current(); + } + + uint32_t current_docid(const Cursor& cursor) const { + return current_postings(cursor).docids[cursor.doc]; + } + + void normalize(Cursor* cursor) const { + while (cursor->run < matching_->size() && + cursor->doc == current_postings(*cursor).docids.size()) { + ++cursor->run; + cursor->doc = 0; + } + } + + static void advance(Cursor* cursor) { ++cursor->doc; } + + std::vector>* readers_; + const std::vector* matching_; + RunPostingShape shape_; + Cursor cursor_; +}; + +} // namespace + +Status merge_run_sources(const std::vector& run_paths, + const std::vector& vocab, + const std::vector& string_rank, bool has_positions, + const StreamedTermConsumer& fn, TermKeyMaterializer materialize_term_key, + MemoryReporter* memory_reporter) { + if (string_rank.size() != vocab.size()) { + return Status::Error( + "merge_run_sources: string_rank/vocab size mismatch"); + } + std::vector> readers; + readers.reserve(run_paths.size()); + std::priority_queue, HeapGreater> heap( + HeapGreater {&string_rank}); + for (size_t i = 0; i < run_paths.size(); ++i) { + auto reader = std::make_unique(memory_reporter); + RETURN_IF_ERROR(reader->open(run_paths[i], has_positions)); + if (!reader->exhausted()) { + if (reader->current_id() >= vocab.size()) { + return Status::Error( + "run term_id out of vocab range"); + } + heap.push({reader->current_id(), i}); + } + readers.push_back(std::move(reader)); + } + + std::vector matching; + while (!heap.empty()) { + const uint32_t id = heap.top().term_id; + matching.clear(); + while (!heap.empty() && heap.top().term_id == id) { + matching.push_back(heap.top().run); + heap.pop(); + } + DCHECK(!matching.empty()); + const RunPostingShape shape = posting_shape(readers[matching.front()]->current()); + for (size_t run : matching) { + if (posting_shape(readers[run]->current()) != shape) { + return Status::Error( + "run: posting shape differs across matching terms"); + } + } + + RunTermPostingSource source(&readers, &matching, shape); + StreamedTermPostings postings {.term = materialize_term_key + ? materialize_term_key(vocab[id]) + : std::string(vocab[id]), + .retain_positions = shape == RunPostingShape::kPositioned, + .source = &source}; + RETURN_IF_ERROR(fn(std::move(postings))); + if (!source.exhausted()) { + return Status::Error( + "run posting source: consumer returned before term exhaustion"); + } + + for (size_t run : matching) { + RunReader* reader = readers[run].get(); + RETURN_IF_ERROR(reader->advance()); + if (!reader->exhausted()) { + if (reader->current_id() >= vocab.size()) { + return Status::Error( + "run term_id out of vocab range"); + } + heap.push({reader->current_id(), run}); + } + } + } + return Status::OK(); +} + +Status compact_runs(const std::vector& run_paths, + const std::vector& string_rank, bool has_positions, + const std::string& out_path, MemoryReporter* memory_reporter) { + // Same heap machinery as merge_run_sources, but the output is a RUN (records keyed + // by term-id, ordered by string rank -- the exact invariant every run file + // carries), not a resolved term stream: no vocab strings are needed, and + // positions are always materialized because the run codec serializes + // positions_flat directly. + std::vector> readers; + readers.reserve(run_paths.size()); + std::priority_queue, HeapGreater> heap( + HeapGreater {&string_rank}); + for (size_t i = 0; i < run_paths.size(); ++i) { + auto r = std::make_unique(memory_reporter); + RETURN_IF_ERROR(r->open(run_paths[i], has_positions)); + if (!r->exhausted()) { + if (r->current_id() >= string_rank.size()) { + return Status::Error( + "run term_id out of rank range"); + } + heap.push({r->current_id(), i}); + } + readers.push_back(std::move(r)); + } + + RunWriter w(memory_reporter); + RETURN_IF_ERROR(w.open(out_path)); + std::vector matching; // run indices contributing the current term + while (!heap.empty()) { + const uint32_t id = heap.top().term_id; + MemoryReporter::Reservation merged_docids_reservation = + memory_reporter == nullptr ? MemoryReporter::Reservation() + : memory_reporter->make_reservation(); + MemoryReporter::Reservation merged_freqs_reservation = + memory_reporter == nullptr ? MemoryReporter::Reservation() + : memory_reporter->make_reservation(); + MemoryReporter::Reservation merged_positions_reservation = + memory_reporter == nullptr ? MemoryReporter::Reservation() + : memory_reporter->make_reservation(); + TermPostings merged; + matching.clear(); + uint64_t total_docs = 0; + uint64_t total_pos = 0; + while (!heap.empty() && heap.top().term_id == id) { + const size_t ri = heap.top().run; + heap.pop(); + const RunReader* r = readers[ri].get(); + const uint64_t run_docs = r->current().docids.size(); + const uint64_t run_positions = r->current_pos_count(); + if (run_docs > std::numeric_limits::max() - total_docs || + run_positions > std::numeric_limits::max() - total_pos) { + return Status::Error( + "run compaction: merged posting size overflows uint64"); + } + total_docs += run_docs; + total_pos += run_positions; + matching.push_back(ri); + } + DCHECK(!matching.empty()); + const RunPostingShape shape = posting_shape(readers[matching.front()]->current()); + for (size_t ri : matching) { + if (posting_shape(readers[ri]->current()) != shape) { + return Status::Error( + "run: posting shape differs across matching terms"); + } + } + const bool term_has_positions = shape == RunPostingShape::kPositioned; + const bool statless = shape == RunPostingShape::kDocsOnlyStatless; + merged.retain_positions = term_has_positions; + if (total_docs > std::numeric_limits::max() || + total_pos > std::numeric_limits::max()) { + return Status::Error( + "run compaction: merged posting exceeds addressable memory"); + } + RETURN_IF_ERROR(reserve_vector_for_size(&merged.docids, static_cast(total_docs), + memory_reporter, &merged_docids_reservation)); + if (!statless) { + RETURN_IF_ERROR(reserve_vector_for_size(&merged.freqs, static_cast(total_docs), + memory_reporter, &merged_freqs_reservation)); + } + if (term_has_positions) { + RETURN_IF_ERROR(reserve_vector_for_size(&merged.positions_flat, + static_cast(total_pos), memory_reporter, + &merged_positions_reservation)); + } + // concat (WITH boundary-doc coalescing) is deliberately the SAME + // append the final merge applies: coalescing the seam between two + // adjacent input runs here yields exactly what the final merge would + // have produced from the uncompacted pair, so compaction is invisible + // in the emitted term stream. + for (size_t ri : matching) { + RunReader* r = readers[ri].get(); + if (term_has_positions) { + RETURN_IF_ERROR(r->materialize_positions()); + } + concat(&merged, r->current(), shape); + } + RETURN_IF_ERROR(w.write_term(id, merged)); + for (size_t ri : matching) { + RunReader* r = readers[ri].get(); + RETURN_IF_ERROR(r->advance()); + if (!r->exhausted()) { + if (r->current_id() >= string_rank.size()) { + return Status::Error( + "run term_id out of rank range"); + } + heap.push({r->current_id(), ri}); + } + } + } + return w.close(); +} + +} // namespace doris::snii::writer diff --git a/be/src/storage/index/snii/writer/spill_run_codec.h b/be/src/storage/index/snii/writer/spill_run_codec.h new file mode 100644 index 00000000000000..f63bd06d6a9156 --- /dev/null +++ b/be/src/storage/index/snii/writer/spill_run_codec.h @@ -0,0 +1,236 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/writer/spimi_term_buffer.h" + +namespace doris::snii::writer { + +using TermKeyMaterializer = std::function; + +// On-disk SPIMI "run" codec for the spill / k-way-merge out-of-core build path. +// +// A RUN is a self-describing file holding a sequence of terms keyed by TERM-ID, +// each followed by its postings, in this exact wire layout. The file is produced +// and consumed by THIS module only (a private temp file -- the on-disk INDEX is +// unaffected), so the format is chosen for cheap I/O: docids, freqs and positions +// are ALL RAW fixed-width little-endian u32 BLOCKS (bulk memcpy on both ends, +// ~10x cheaper than per-value varint -- which cost ~1.5s of encode CPU over the +// 5M build's ~60M docids and compressed those streams poorly anyway). Decode +// still validates every length against the file size. +// +// run := record* (term-ids ordered by vocab string, +// strictly ascending within a run) +// record := +// VInt term_id (index into the shared vocabulary) +// VInt shape (0=docs-only-statless, 1=docs+freq, +// 2=positioned) +// VInt n_docs +// u32 docid * n_docs (RAW LE absolute ascending docids) +// shape=1: u32 freq * n_docs (RAW LE, each >= 1) +// shape=2: u32 freq * n_docs, VInt n_pos, u32 position * n_pos +// (n_pos == sum(freqs)) +// +// Shape 0 is the CommonGrams docs-only set representation. It writes neither +// synthetic all-one frequencies nor n_pos; run files are private temporaries, +// so this does not change the persisted SNII index format. +// +// Decode is fully STREAMED: a RunReader reads a small fixed buffer at a time and +// materializes only the CURRENT term's postings, never the whole run. The k-way +// merge keeps one heap slot per run (each holding only its current term-id + +// that term's postings), so peak memory is bounded by the widest single term +// summed across the runs that contain it -- not by total postings. The merge +// orders runs by a PRECOMPUTED integer string-rank (term-id -> its lexicographic +// rank over the shared dense vocabulary): an integer compare that reproduces the +// exact lexicographic order without touching a vocab string in the inner loops. + +// Writes a sorted sequence of terms (by id) to one run file. Term-ids must be +// handed to write_term in vocab-string ascending order (the spill caller sorts +// before spilling). RAII: the file is flushed and closed on close(); the partial +// file is left for the owning SpimiTermBuffer to delete on its temp-path list. +class RunWriter { +public: + explicit RunWriter(MemoryReporter* memory_reporter = nullptr); + ~RunWriter(); + + RunWriter(const RunWriter&) = delete; + RunWriter& operator=(const RunWriter&) = delete; + + // Opens `path` for writing (truncating). Returns IoError on failure. + Status open(const std::string& path); + + // Appends one term's postings under `term_id`. Empty freqs denotes the + // docs-only-statless shape; otherwise freqs parallels docids. Positioned + // postings additionally hold sum(freqs) positions in document order. + Status write_term(uint32_t term_id, const TermPostings& tp); + + // Flushes the buffer and closes the file. Safe to call once; idempotent. + Status close(); + +private: + Status flush(); + Status append_bytes(const uint8_t* data, size_t size); + Status append_varint(uint64_t value); + Status append_raw_u32(const uint32_t* values, size_t count); + void release_buffer(); + + MemoryReporter* memory_reporter_ = nullptr; + MemoryReporter::Reservation buffer_reservation_; + int fd_ = -1; + std::vector buf_; // bounded staging buffer; flushed in fixed-size chunks +}; + +// Streamed reader over one run file. After open() the first term is loaded; +// current()/current_id() expose it; advance() loads the next (or marks +// exhausted). Only the current term's postings live in memory at a time. The +// current record's `term` string is left EMPTY -- runs store only the id; the +// owner resolves the string via the shared vocabulary. +// +// LAZY POSITIONS (peak-RSS optimization for the widest merged term): advance() +// loads term_id / docids / freqs and the position-block COUNT, but does NOT read +// the position bytes -- it leaves the decode window cursor parked at the start of +// the position block. The owner then chooses, per term: +// * materialize_positions(): bulk-reads the block into current().positions_flat +// (the default; behaves exactly as the old eager reader). +// * stream_positions(dst, n): pulls the next n positions straight from the +// window in 64 KiB chunks, never materializing the whole block -- used by the +// k-way merge source to decode directly into each writer-owned window. +// advance() drains any positions left unread from the previous term before the +// next record, so a partly-streamed (or skipped) term still lands at the right +// record boundary. The yielded byte sequence is identical either way. +class RunReader { +public: + explicit RunReader(MemoryReporter* memory_reporter = nullptr); + ~RunReader(); + + RunReader(const RunReader&) = delete; + RunReader& operator=(const RunReader&) = delete; + + // Opens `path`, loading the first record (if any). has_positions declares + // whether this run may contain positioned or mixed statless records. + Status open(const std::string& path, bool has_positions); + + bool exhausted() const { return exhausted_; } + const TermPostings& current() const { return current_; } + uint32_t current_id() const { return current_id_; } + + // Number of positions in the current term's (lazily-loaded) position block. + uint64_t current_pos_count() const { return pos_count_; } + // True once the current term's positions have been materialized OR fully + // streamed (i.e. nothing remains to read before advance()). + bool positions_drained() const { return pos_remaining_ == 0; } + + // Materializes the current term's position block into current().positions_flat + // (bulk read). Idempotent within a term: a no-op once positions are drained. + Status materialize_positions(); + // Streams the next `n` positions of the current term into dst[0..n) directly + // from the decode window (64 KiB chunks topped up on demand). Caller must not + // request more than positions_remaining(); each call advances the cursor. + Status stream_positions(uint32_t* dst, size_t n); + uint64_t positions_remaining() const { return pos_remaining_; } + + // Loads the next record into current(); sets exhausted() at end of file. Any + // positions of the current term left unread are skipped first. + Status advance(); + +private: + size_t available() const; // buffered bytes from pos_ to window end + Status fill(); // tops up the decode window from disk + Status ensure(size_t n); // guarantees >= n buffered bytes (or eof) + Status read_varint(uint64_t* v); // bounds-checked streamed varint + // Bulk-reads `count` RAW little-endian u32s from the window into `out` (resized + // to count). Bounds-checked against the run's true length (Corruption on EOF). + Status read_raw_u32(size_t count, std::vector* out, + MemoryReporter::Reservation* reservation); + // Streams `count` raw u32s from the window into dst (caller-owned, sized by the + // caller); shared by read_raw_u32 (into a vector) and stream_positions. + Status pull_raw_u32(uint8_t* dst, size_t count); + // Drains (and discards) any remaining positions of the current term so the + // window cursor lands at the next record boundary. + Status skip_remaining_positions(); + + MemoryReporter* memory_reporter_ = nullptr; + // Reservations precede their vectors so allocations are destroyed first. + MemoryReporter::Reservation window_reservation_; + MemoryReporter::Reservation docids_reservation_; + MemoryReporter::Reservation freqs_reservation_; + MemoryReporter::Reservation positions_reservation_; + int fd_ = -1; + bool has_positions_ = false; + bool exhausted_ = false; + uint64_t file_size_ = 0; // total run byte size (fstat at open); bounds lengths + uint64_t bytes_read_ = 0; // bytes pulled from fd; never exceeds file_size_ + std::vector window_; // sliding decode window + size_t pos_ = 0; // consumed offset within window_ + bool eof_ = false; // no more bytes on disk + uint32_t current_id_ = 0; // current record's term-id + uint64_t pos_count_ = 0; // current term's total position count (from n_pos) + uint64_t pos_remaining_ = 0; // positions still unread in the current block + TermPostings current_; +}; + +// K-way merges the given run files into a single term stream ordered by a +// PRECOMPUTED integer string-rank (string_rank[term_id] == the term-id's +// lexicographic rank over the dense vocabulary), invoking `fn` once per distinct +// term-id with its postings concatenated across all runs that contain it (in run +// order -> docids stay ascending) and its `term` resolved from `vocab` once. +// Because a dense vocab maps each id to a distinct string, the rank is a +// lexicographic bijection: ordering by the dense 4 B rank array (an integer +// compare) reproduces the EXACT order a vocab-string compare would -- but never +// reads a vocab string in the inner heap/gather loops. Only one bounded posting +// window is materialized at a time. Returns IoError/Corruption on bad run data, or +// InternalError when string_rank.size() != vocab.size(). has_positions must match +// how the runs were written. `vocab` (term-id -> string) and `string_rank` +// (term-id -> rank) are both borrowed and MUST be sized to the vocabulary. +// +// The source callback is synchronous: matching run readers remain parked on the +// current term until the callback returns. The source fills writer-owned windows +// directly and coalesces equal docids at run boundaries. A successful callback +// must exhaust the source. +Status merge_run_sources(const std::vector& run_paths, + const std::vector& vocab, + const std::vector& string_rank, bool has_positions, + const StreamedTermConsumer& fn, + TermKeyMaterializer materialize_term_key = {}, + MemoryReporter* memory_reporter = nullptr); + +// G09 run-file cap support: k-way merges `run_paths` into ONE new run file at +// `out_path`, keyed and ordered exactly like merge_run_sources (heap on +// string_rank[term_id]; per-term postings concatenated across runs in run +// order, boundary docs coalesced -- the same concat the final merge applies, +// so compact-then-merge emits the identical term stream as merging the +// originals). Positions are fully materialized for each term because the run +// codec serializes positions_flat. +// Every record's term-id must index string_rank (else Corruption). On error +// `out_path` may hold a partial file the caller must delete; the input runs +// are never modified. Opens run_paths.size() read fds + 1 write fd for the +// call's duration -- the caller (SpimiTermBuffer::compact_runs) bounds that +// fan-in with its run-count cap. +Status compact_runs(const std::vector& run_paths, + const std::vector& string_rank, bool has_positions, + const std::string& out_path, MemoryReporter* memory_reporter = nullptr); + +} // namespace doris::snii::writer diff --git a/be/src/storage/index/snii/writer/spillable_byte_buffer.h b/be/src/storage/index/snii/writer/spillable_byte_buffer.h new file mode 100644 index 00000000000000..2760600b00f019 --- /dev/null +++ b/be/src/storage/index/snii/writer/spillable_byte_buffer.h @@ -0,0 +1,305 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "io/fs/file_reader.h" +#include "io/fs/file_reader_writer_fwd.h" +#include "io/fs/file_writer.h" +#include "io/fs/local_file_system.h" +#include "io/fs/path.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/io/file_writer.h" +#include "storage/index/snii/writer/memory_reporter.h" +#include "storage/index/snii/writer/temp_dir.h" +#include "util/slice.h" + +namespace doris::snii::writer { + +// A tiered append buffer for one build-time section. While resident it holds the +// bytes as a CHAIN OF CHUNKS (one per append) rather than a single growing vector. +// Copying appends own a right-sized allocation. Move appends preserve the caller's +// allocation, including capacity slack, and account that retained capacity as resident +// memory. Once the resident capacity crosses `cap_bytes` the buffer SPILLS to a temp +// file (resolve_temp_dir()) and routes later appends there, so a huge section stays +// RSS-bounded at ~cap_bytes while a small one is RAM-only (zero disk, spill-only build). +// Append order/bytes are identical wherever they land; stream_into() reproduces the +// logical section bytes in order. RAII-removes the temp. (cap_bytes == UINT64_MAX +// disables spilling -> always RAM.) +class SpillableByteBuffer { +public: + // `reporter` is an OPTIONAL writer-level build-RAM reporter (null off-Doris / + // unit tests). Resident chunk capacities are hard-reserved before adoption or + // allocation. If the shared cap cannot admit the next chunk, the buffer spills + // its resident prefix first and writes that chunk directly to disk. Spilled + // bytes are not resident and hold no reservation. + SpillableByteBuffer(uint64_t cap_bytes, std::string tag, MemoryReporter* reporter = nullptr) + : cap_bytes_(cap_bytes), + tag_(std::move(tag)), + reporter_(reporter), + reservation_(reporter == nullptr ? MemoryReporter::Reservation() + : reporter->make_reservation()) {} + ~SpillableByteBuffer() { release_storage(); } + SpillableByteBuffer(const SpillableByteBuffer&) = delete; + SpillableByteBuffer& operator=(const SpillableByteBuffer&) = delete; + + // Total bytes appended so far (the offset basis for callers recording sub-offsets). + uint64_t size() const { + return consumed_ ? consumed_size_ : (spilled_ ? spilled_bytes_ : ram_bytes_); + } + + // Copying append (the Slice bytes are copied into a fresh chunk). + Status append(Slice bytes) { + if (consumed_) { + return Status::Error( + "spillable buffer: append after stream"); + } + if (spilled_) { + const ::doris::Slice s(bytes.data(), bytes.size()); + RETURN_IF_ERROR(to_snii(temp_writer_->appendv(&s, 1))); + spilled_bytes_ += bytes.size(); + return Status::OK(); + } + if (!bytes.empty()) { + bool keep_resident = true; + RETURN_IF_ERROR(reserve_resident_capacity(bytes.size(), &keep_resident)); + if (!keep_resident) { + const ::doris::Slice s(bytes.data(), bytes.size()); + RETURN_IF_ERROR(to_snii(temp_writer_->appendv(&s, 1))); + spilled_bytes_ += bytes.size(); + return Status::OK(); + } + std::vector chunk; + chunk.reserve(bytes.size()); + DCHECK_EQ(chunk.capacity(), bytes.size()); + chunk.insert(chunk.end(), bytes.data(), bytes.data() + bytes.size()); + chunks_.push_back(std::move(chunk)); + ram_bytes_ += bytes.size(); + ram_capacity_bytes_ += chunks_.back().capacity(); + } + if (over_cap()) return spill_to_disk(); + return Status::OK(); + } + + // Move append: the section ADOPTS the caller's vector without copying. The common + // dict path hands off each flushed block this way. Logical bytes remain v.size(), + // while resident accounting uses the capacity retained by the adopted vector. + Status append_move(std::vector&& v) { + if (consumed_) { + return Status::Error( + "spillable buffer: append after stream"); + } + if (spilled_) { + const ::doris::Slice s(v.data(), v.size()); + RETURN_IF_ERROR(to_snii(temp_writer_->appendv(&s, 1))); + spilled_bytes_ += v.size(); + return Status::OK(); + } + if (!v.empty()) { + const size_t logical_bytes = v.size(); + const size_t retained_capacity = v.capacity(); + bool keep_resident = true; + RETURN_IF_ERROR(reserve_resident_capacity(retained_capacity, &keep_resident)); + if (!keep_resident) { + const ::doris::Slice s(v.data(), v.size()); + RETURN_IF_ERROR(to_snii(temp_writer_->appendv(&s, 1))); + spilled_bytes_ += v.size(); + return Status::OK(); + } + chunks_.push_back(std::move(v)); + ram_bytes_ += logical_bytes; + ram_capacity_bytes_ += chunks_.back().capacity(); + DCHECK_EQ(chunks_.back().capacity(), retained_capacity); + } + if (over_cap()) return spill_to_disk(); + return Status::OK(); + } + + // Must be called once after the last append, before stream_into(): flushes the temp + // (if spilled) so it can be read back. A no-op for a RAM-resident buffer. + Status seal() { + if (consumed_) { + return Status::Error( + "spillable buffer: seal after stream"); + } + if (spilled_ && !sealed_) { + RETURN_IF_ERROR(to_snii(temp_writer_->close())); + sealed_ = true; + } + return Status::OK(); + } + + // Streams the whole section (RAM chunks or sealed temp) into `out`, in append order. + Status stream_into(io::FileWriter* out) const { + if (consumed_) { + return Status::Error( + "spillable buffer: stream called twice"); + } + if (!spilled_) { + for (const auto& c : chunks_) { + if (!c.empty()) RETURN_IF_ERROR(out->append(Slice(c))); + } + return Status::OK(); + } + ::doris::io::FileReaderSPtr reader; + RETURN_IF_ERROR( + to_snii(::doris::io::global_local_filesystem()->open_file(temp_path_, &reader))); + constexpr uint64_t kMaxChunk = 1u << 20; // bounded copy window (no whole-section reload) + size_t read_capacity = static_cast(std::min(kMaxChunk, spilled_bytes_)); + MemoryReporter::Reservation read_reservation = reporter_ == nullptr + ? MemoryReporter::Reservation() + : reporter_->make_reservation(); + if (reporter_ != nullptr) { + Status reserve_status = read_reservation.set_bytes(read_capacity); + while (!reserve_status.ok() && reserve_status.is() && + read_capacity > 1) { + read_capacity = std::max(1, read_capacity / 2); + reserve_status = read_reservation.set_bytes(read_capacity); + } + RETURN_IF_ERROR(reserve_status); + } + std::vector buf; + buf.reserve(read_capacity); + DCHECK_EQ(buf.capacity(), read_capacity); + for (uint64_t off = 0; off < spilled_bytes_; off += read_capacity) { + const uint64_t n = std::min(read_capacity, spilled_bytes_ - off); + buf.resize(static_cast(n)); + size_t bytes_read = 0; + RETURN_IF_ERROR(to_snii(reader->read_at( + off, ::doris::Slice(buf.data(), static_cast(n)), &bytes_read))); + if (bytes_read != n) { + return Status::Error( + "short read from spill scratch file"); + } + RETURN_IF_ERROR(out->append(Slice(buf.data(), static_cast(n)))); + } + return Status::OK(); + } + + // The staged bytes have no readers after they are copied into the compound + // output. Releasing them here, rather than in the logical-writer destructor, + // keeps multi-destination compaction from retaining one DICT image per + // completed index until rowset close. + Status stream_into_and_release(io::FileWriter* out) { + RETURN_IF_ERROR(stream_into(out)); + release_storage(); + return Status::OK(); + } + + bool spilled() const { return spilled_; } + +private: + Status reserve_resident_capacity(size_t additional_capacity, bool* keep_resident) { + DCHECK(keep_resident != nullptr); + *keep_resident = true; + if (reporter_ == nullptr) return Status::OK(); + if (additional_capacity > std::numeric_limits::max() - ram_capacity_bytes_) { + return Status::Error( + "spillable buffer: resident capacity overflows uint64"); + } + const Status reserved = reservation_.set_bytes(ram_capacity_bytes_ + additional_capacity); + if (reserved.ok()) return Status::OK(); + if (!reserved.is()) return reserved; + RETURN_IF_ERROR(spill_to_disk()); + *keep_resident = false; + return Status::OK(); + } + + // Gate-2 spill condition (UNIFIED): spill when the writer's TOTAL build RAM crosses + // the one shared cap (reporter_->over_cap()), with the local cap_bytes_ kept only as + // a defensive per-buffer hard ceiling (e.g. when no reporter is attached). + bool over_cap() const { + return (reporter_ != nullptr && reporter_->over_cap()) || ram_capacity_bytes_ >= cap_bytes_; + } + // Bridge a Doris IO Status into SNII's Status. R01 (status migration) is not done yet, + // so this buffer still returns Status; this mirrors snii_doris_adapter's + // to_snii_status (ok -> OK, otherwise IoError carrying the Doris message). + static Status to_snii(const Status& s) { + if (s.ok()) return Status::OK(); + return Status::Error(s.to_string_no_stack()); + } + Status spill_to_disk() { + temp_path_ = resolve_temp_dir() + "/snii_" + tag_ + "_" + std::to_string(::getpid()) + "_" + + std::to_string(reinterpret_cast(this)) + ".tmp"; + RETURN_IF_ERROR(to_snii( + ::doris::io::global_local_filesystem()->create_file(temp_path_, &temp_writer_))); + for (const auto& c : chunks_) { + if (!c.empty()) { + const ::doris::Slice s(c.data(), c.size()); + RETURN_IF_ERROR(to_snii(temp_writer_->appendv(&s, 1))); + } + } + spilled_bytes_ = ram_bytes_; + std::vector>().swap(chunks_); // reclaim the RAM immediately + ram_bytes_ = 0; + ram_capacity_bytes_ = 0; + reservation_.reset(); + spilled_ = true; + return Status::OK(); + } + + void release_storage() { + if (consumed_) { + return; + } + consumed_size_ = spilled_ ? spilled_bytes_ : ram_bytes_; + std::vector>().swap(chunks_); + ram_bytes_ = 0; + ram_capacity_bytes_ = 0; + reservation_.reset(); + + // A sealed temp writer is already closed; an unsealed error-path writer + // aborts on reset. In both cases remove the scratch path best-effort. + temp_writer_.reset(); + if (!temp_path_.empty()) { + std::remove(temp_path_.c_str()); + temp_path_.clear(); + } + spilled_bytes_ = 0; + consumed_ = true; + } + + uint64_t cap_bytes_; + std::string tag_; + MemoryReporter* reporter_ = nullptr; // optional build-RAM reporter (null off-Doris) + MemoryReporter::Reservation reservation_; // resident inner-vector capacities + std::vector> chunks_; // resident tier: one chunk per append + uint64_t ram_bytes_ = 0; // logical section bytes retained in RAM + uint64_t ram_capacity_bytes_ = 0; // resident capacities of the retained chunks + bool spilled_ = false; + bool sealed_ = false; + ::doris::io::FileWriterPtr temp_writer_; // Doris local writer for the spill scratch file + std::string temp_path_; + uint64_t spilled_bytes_ = 0; + uint64_t consumed_size_ = 0; + bool consumed_ = false; +}; + +} // namespace doris::snii::writer diff --git a/be/src/storage/index/snii/writer/spimi_term_buffer.cpp b/be/src/storage/index/snii/writer/spimi_term_buffer.cpp new file mode 100644 index 00000000000000..2ee6d90bc20b81 --- /dev/null +++ b/be/src/storage/index/snii/writer/spimi_term_buffer.cpp @@ -0,0 +1,2097 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/writer/spimi_term_buffer.h" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/exception.h" +#include "storage/index/inverted/common_grams/common_grams_key_codec.h" +#include "storage/index/inverted/common_grams/common_word_set.h" +#include "storage/index/snii/encoding/varint.h" +#include "storage/index/snii/format/format_constants.h" +#include "storage/index/snii/writer/global_memory_limiter.h" +#include "storage/index/snii/writer/spill_run_codec.h" +#include "storage/index/snii/writer/temp_dir.h" + +#if defined(__GLIBC__) +#include +#endif + +namespace doris::snii::writer { + +namespace { + +constexpr size_t kCommonGramPairKeySize = 10; +constexpr char kCommonGramPairKeyTag = 'P'; + +std::array EncodeCommonGramPairKey(PlainTermId left, + PlainTermId right) { + std::array key {}; + key[0] = segment_v2::inverted_index::INTERNAL_TERM_NAMESPACE_BEGIN.front(); + key[1] = kCommonGramPairKeyTag; + for (size_t byte = 0; byte < sizeof(uint32_t); ++byte) { + const size_t shift = (sizeof(uint32_t) - byte - 1) * 8; + key[2 + byte] = static_cast((left.value >> shift) & 0xffU); + key[6 + byte] = static_cast((right.value >> shift) & 0xffU); + } + return key; +} + +bool is_common_gram_pair_key(std::string_view key) { + return key.size() == kCommonGramPairKeySize && + key.front() == segment_v2::inverted_index::INTERNAL_TERM_NAMESPACE_BEGIN.front() && + key[1] == kCommonGramPairKeyTag; +} + +struct CommonGramPairIds { + PlainTermId left; + PlainTermId right; +}; + +uint32_t decode_big_endian_uint32_unchecked(const char* bytes) { + uint32_t value = 0; + for (size_t byte = 0; byte < sizeof(uint32_t); ++byte) { + value = (value << 8) | static_cast(bytes[byte]); + } + return value; +} + +#ifdef BE_TEST +std::atomic g_common_gram_pair_unchecked_decodes {0}; +std::atomic g_common_gram_trusted_plain_decodes {0}; +std::atomic g_common_gram_pair_cache_probes {0}; +std::atomic g_common_gram_pair_cache_pair_hits {0}; +std::atomic g_common_gram_pair_cache_same_doc_hits {0}; +std::atomic g_common_gram_native_pair_probes {0}; +std::atomic g_common_gram_native_pair_hits {0}; +std::atomic g_common_gram_native_pair_inserts {0}; +std::atomic g_common_gram_logical_validations {0}; +std::atomic g_common_gram_plain_cache_probes {0}; +std::atomic g_common_gram_plain_cache_hits {0}; +std::atomic g_common_gram_plain_intern_table_probes {0}; +std::atomic g_owned_term_full_byte_comparisons {0}; +std::atomic g_fail_next_owned_term_reserve {false}; +std::atomic g_fail_next_owned_term_emplace {false}; +std::atomic g_spill_gate_checks {0}; +std::atomic g_compact_chain_varint_decodes {0}; +#endif + +CommonGramPairIds decode_common_gram_pair_key_unchecked(std::string_view key) { + DCHECK(is_common_gram_pair_key(key)); +#ifdef BE_TEST + g_common_gram_pair_unchecked_decodes.fetch_add(1, std::memory_order_relaxed); +#endif + return { + .left = PlainTermId {.value = decode_big_endian_uint32_unchecked(key.data() + 2)}, + .right = PlainTermId {.value = decode_big_endian_uint32_unchecked(key.data() + 6)}, + }; +} + +class LogicalPlainKeyView { +public: + explicit LogicalPlainKeyView(std::string_view physical) : physical_(physical) { + escaped_ = !physical.empty() && + physical.front() == segment_v2::inverted_index::PLAIN_ESCAPE_PREFIX; + if (escaped_) { + DCHECK_GE(physical.size(), 2U); + DCHECK(physical[1] == 'E' || physical[1] == 'G'); + } + } + + size_t size() const { return physical_.size() - (escaped_ ? 1 : 0); } + + uint8_t operator[](size_t index) const { + DCHECK_LT(index, size()); + if (!escaped_) { + return static_cast(physical_[index]); + } + if (index == 0) { + return static_cast( + physical_[1] == 'E' ? segment_v2::inverted_index::PLAIN_ESCAPE_PREFIX : '\x1f'); + } + return static_cast(physical_[index + 1]); + } + +private: + std::string_view physical_; + bool escaped_ = false; +}; + +std::string_view decode_logical_plain_term_trusted(std::string_view physical, + std::string* scratch) { + DCHECK(scratch != nullptr); +#ifdef BE_TEST + g_common_gram_trusted_plain_decodes.fetch_add(1, std::memory_order_relaxed); +#endif + scratch->clear(); + if (physical.empty() || physical.front() != segment_v2::inverted_index::PLAIN_ESCAPE_PREFIX) { + DCHECK(!segment_v2::inverted_index::is_internal_term_key(physical)); + return physical; + } + + DCHECK_GE(physical.size(), 2U); + DCHECK(physical[1] == 'E' || physical[1] == 'G'); + scratch->reserve(physical.size() - 1); + scratch->push_back(physical[1] == 'E' + ? segment_v2::inverted_index::PLAIN_ESCAPE_PREFIX + : segment_v2::inverted_index::INTERNAL_TERM_NAMESPACE_BEGIN.front()); + scratch->append(physical.substr(2)); + return std::string_view(*scratch); +} + +int compare_logical_plain_keys(const LogicalPlainKeyView& left, const LogicalPlainKeyView& right) { + const size_t common = std::min(left.size(), right.size()); + for (size_t i = 0; i < common; ++i) { + if (left[i] != right[i]) { + return left[i] < right[i] ? -1 : 1; + } + } + if (left.size() == right.size()) { + return 0; + } + return left.size() < right.size() ? -1 : 1; +} + +} // namespace + +struct SpimiTermBuffer::CommonGramPairCache { + struct Entry { + uint64_t pair = 0; + uint32_t term_id = std::numeric_limits::max(); + uint32_t last_docid = 0; + }; + static_assert(sizeof(Entry) == 16); + + static constexpr size_t kEntryCount = 1024; + static constexpr uint64_t kHashMultiplier = 11400714819323198485ULL; + static constexpr uint32_t kInvalidTermId = std::numeric_limits::max(); + + static size_t index(uint64_t pair) { + return static_cast((pair * kHashMultiplier) >> 54); + } + + std::array entries; + static_assert(sizeof(std::array) == 16 * 1024); +}; + +struct SpimiTermBuffer::CommonGramPlainTermCache { + struct Entry { + uint32_t fingerprint = 0; + uint32_t term_id = std::numeric_limits::max(); + }; + static_assert(sizeof(Entry) == 8); + + static constexpr size_t kSetCount = 1024; + static constexpr uint32_t kInvalidTermId = std::numeric_limits::max(); + using Set = std::array; + static_assert((kSetCount & (kSetCount - 1)) == 0); + + static size_t index(size_t term_hash) { return term_hash & (kSetCount - 1); } + + static uint32_t fingerprint(size_t term_hash) { + const auto hash = static_cast(term_hash); + return static_cast(hash ^ (hash >> 32)); + } + + static bool matches(const Entry& entry, uint32_t expected_fingerprint, std::string_view term, + const std::vector& vocab) { + if (entry.term_id == kInvalidTermId || entry.fingerprint != expected_fingerprint) { + return false; + } + DCHECK_LT(entry.term_id, vocab.size()); + return std::string_view(vocab[entry.term_id]) == term; + } + + uint32_t find(size_t term_hash, std::string_view term, const std::vector& vocab) { + Set& set = sets[index(term_hash)]; + const uint32_t expected_fingerprint = fingerprint(term_hash); + if (matches(set[0], expected_fingerprint, term, vocab)) { + return set[0].term_id; + } + if (matches(set[1], expected_fingerprint, term, vocab)) { + std::swap(set[0], set[1]); + return set[0].term_id; + } + return kInvalidTermId; + } + + void remember(size_t term_hash, uint32_t term_id) { + Set& set = sets[index(term_hash)]; + set[1] = set[0]; + set[0] = Entry {.fingerprint = fingerprint(term_hash), .term_id = term_id}; + } + + std::array sets {}; + static_assert(sizeof(std::array) == 16 * 1024); +}; + +bool SpimiTermBuffer::OwnedVocabEq::operator()(uint32_t stored, + std::string_view probe) const noexcept { +#ifdef BE_TEST + g_owned_term_full_byte_comparisons.fetch_add(1, std::memory_order_relaxed); +#endif + DCHECK_LT(stored, vocab->size()); + return std::string_view((*vocab)[stored]) == probe; +} + +bool SpimiTermBuffer::OwnedVocabEq::operator()(std::string_view probe, + uint32_t stored) const noexcept { + return (*this)(stored, probe); +} + +#ifdef BE_TEST +size_t SpimiTermBuffer::owned_term_key_size_for_test() { + return sizeof(decltype(intern_)::key_type); +} + +void SpimiTermBuffer::set_owned_term_hash_mask_for_test(size_t mask) { + DORIS_CHECK(intern_.empty()); + intern_ = decltype(intern_)(0, OwnedVocabHash {.vocab = &owned_vocab_, .hash_mask = mask}, + OwnedVocabEq {&owned_vocab_}); +} +#endif + +namespace { + +// Returns freed heap arenas to the OS (glibc only). The spill encode churns many +// small allocations whose freed chunks glibc retains in its arenas; trimming +// before the peak-RSS-defining merge phase recovers that retention. No-op (and +// harmless) on non-glibc libcs. +void trim_malloc() { +#if defined(__GLIBC__) + ::malloc_trim(0); +#endif +} + +// Process-unique temp path for a spill run under `dir` (pid + monotonic counter so +// parallel builds / multiple buffers never collide). +std::string make_run_path(const std::string& dir) { + static std::atomic counter {0}; + const uint64_t n = counter.fetch_add(1); + return dir + "/snii_spill_" + std::to_string(::getpid()) + "_" + std::to_string(n) + ".run"; +} + +// TEST-ONLY seam backing testing::vocab_string_materialization_count(). Bumped once +// per DISTINCT interned term (owned_vocab_.emplace_back), never per token. Relaxed: +// the build path is single-threaded, so only the COUNT matters, not ordering. +#ifdef BE_TEST +std::atomic g_vocab_materializations {0}; +#endif + +// G09 seam: spills that consumed a pending process-wide forced-spill request +// (the limiter flagged this buffer as one of the largest registered consumers +// while the global total exceeded the budget). Incremented under BE_TEST only +// (per-token path shared by concurrent writers). +std::atomic g_global_forced_spills {0}; + +// G09 run-file cap seam: merge-compactions of a buffer's run list (always-on: +// at most one per cap-many spills, contention-free). +std::atomic g_run_compactions {0}; + +// Test seam for complete-vocabulary rank rebuilds. The increment is compiled +// out of production because ensure_string_rank() may run on the import path. +#ifdef BE_TEST +std::atomic g_string_rank_rebuilds {0}; +std::atomic g_dense_rank_inversions {0}; +std::atomic g_rank_comparison_sorts {0}; +#endif + +// G11 bench seam: when set (BE_TEST paths only), the add-path prefetch hints +// are skipped so the locality bench can A/B them in one process. Production +// builds never read it (the hint compiles in unconditionally there). +std::atomic g_bench_disable_g11_prefetch {false}; + +// G11 add-path prefetch gate: always-on in production; toggleable under +// BE_TEST for the in-process A/B bench. The branch is perfectly predicted, so +// the bench's OFF arm measures the pre-G11 code path faithfully. +inline bool g11_prefetch_enabled() { +#ifdef BE_TEST + return !g_bench_disable_g11_prefetch.load(std::memory_order_relaxed); +#else + return true; +#endif +} + +// G08: heap payload of one owned-vocab string -- 0 while it fits the SSO buffer +// (those bytes live inside the 32 B header owned_vocab_.capacity() charges), else +// the allocated buffer (capacity + NUL). The SSO capacity is probed from the +// running stdlib so the classification is exact, not hardcoded. +uint64_t string_heap_bytes(const std::string& s) { + static const size_t kSsoCapacity = std::string().capacity(); + return s.capacity() > kSsoCapacity ? static_cast(s.capacity()) + 1 : 0; +} + +void order_ids_by_dense_rank(std::vector* ids, const std::vector& rank) { + if (ids->size() == rank.size()) { + // Touched ids are unique. Equal cardinality therefore means the run covers + // the complete vocabulary, so invert the dense rank in linear time. + for (uint32_t term_id = 0; term_id < rank.size(); ++term_id) { + (*ids)[rank[term_id]] = term_id; + } +#ifdef BE_TEST + g_dense_rank_inversions.fetch_add(1, std::memory_order_relaxed); +#endif + return; + } + + std::ranges::sort(*ids, [&](uint32_t a, uint32_t b) { return rank[a] < rank[b]; }); +#ifdef BE_TEST + g_rank_comparison_sorts.fetch_add(1, std::memory_order_relaxed); +#endif +} + +} // namespace + +namespace testing { +void set_bench_disable_g11_prefetch(bool disabled) { + g_bench_disable_g11_prefetch.store(disabled, std::memory_order_relaxed); +} +uint64_t vocab_string_materialization_count() { +#ifdef BE_TEST + return g_vocab_materializations.load(std::memory_order_relaxed); +#else + return 0; +#endif +} +void reset_vocab_string_materialization_count() { +#ifdef BE_TEST + g_vocab_materializations.store(0, std::memory_order_relaxed); +#endif +} +uint64_t global_forced_spills() { + return g_global_forced_spills.load(std::memory_order_relaxed); +} +void reset_global_forced_spills() { + g_global_forced_spills.store(0, std::memory_order_relaxed); +} +uint64_t run_compactions() { + return g_run_compactions.load(std::memory_order_relaxed); +} +void reset_run_compactions() { + g_run_compactions.store(0, std::memory_order_relaxed); +} +uint64_t string_rank_rebuilds() { +#ifdef BE_TEST + return g_string_rank_rebuilds.load(std::memory_order_relaxed); +#else + return 0; +#endif +} +void reset_string_rank_rebuilds() { +#ifdef BE_TEST + g_string_rank_rebuilds.store(0, std::memory_order_relaxed); +#endif +} +uint64_t dense_rank_inversions() { +#ifdef BE_TEST + return g_dense_rank_inversions.load(std::memory_order_relaxed); +#else + return 0; +#endif +} +uint64_t rank_comparison_sorts() { +#ifdef BE_TEST + return g_rank_comparison_sorts.load(std::memory_order_relaxed); +#else + return 0; +#endif +} +void reset_rank_ordering_counts() { +#ifdef BE_TEST + g_dense_rank_inversions.store(0, std::memory_order_relaxed); + g_rank_comparison_sorts.store(0, std::memory_order_relaxed); +#endif +} +uint64_t common_gram_pair_unchecked_decode_count() { +#ifdef BE_TEST + return g_common_gram_pair_unchecked_decodes.load(std::memory_order_relaxed); +#else + return 0; +#endif +} +uint64_t common_gram_trusted_plain_decode_count() { +#ifdef BE_TEST + return g_common_gram_trusted_plain_decodes.load(std::memory_order_relaxed); +#else + return 0; +#endif +} +void reset_common_gram_pair_fast_path_counts() { +#ifdef BE_TEST + g_common_gram_pair_unchecked_decodes.store(0, std::memory_order_relaxed); + g_common_gram_trusted_plain_decodes.store(0, std::memory_order_relaxed); +#endif +} +uint64_t common_gram_pair_cache_probes() { +#ifdef BE_TEST + return g_common_gram_pair_cache_probes.load(std::memory_order_relaxed); +#else + return 0; +#endif +} +uint64_t common_gram_pair_cache_pair_hits() { +#ifdef BE_TEST + return g_common_gram_pair_cache_pair_hits.load(std::memory_order_relaxed); +#else + return 0; +#endif +} +uint64_t common_gram_pair_cache_same_doc_hits() { +#ifdef BE_TEST + return g_common_gram_pair_cache_same_doc_hits.load(std::memory_order_relaxed); +#else + return 0; +#endif +} +void reset_common_gram_pair_cache_counts() { +#ifdef BE_TEST + g_common_gram_pair_cache_probes.store(0, std::memory_order_relaxed); + g_common_gram_pair_cache_pair_hits.store(0, std::memory_order_relaxed); + g_common_gram_pair_cache_same_doc_hits.store(0, std::memory_order_relaxed); +#endif +} +uint64_t common_gram_native_pair_probes() { +#ifdef BE_TEST + return g_common_gram_native_pair_probes.load(std::memory_order_relaxed); +#else + return 0; +#endif +} +uint64_t common_gram_native_pair_hits() { +#ifdef BE_TEST + return g_common_gram_native_pair_hits.load(std::memory_order_relaxed); +#else + return 0; +#endif +} +uint64_t common_gram_native_pair_inserts() { +#ifdef BE_TEST + return g_common_gram_native_pair_inserts.load(std::memory_order_relaxed); +#else + return 0; +#endif +} +void reset_common_gram_native_pair_intern_counts() { +#ifdef BE_TEST + g_common_gram_native_pair_probes.store(0, std::memory_order_relaxed); + g_common_gram_native_pair_hits.store(0, std::memory_order_relaxed); + g_common_gram_native_pair_inserts.store(0, std::memory_order_relaxed); +#endif +} +uint64_t common_gram_logical_validation_count() { +#ifdef BE_TEST + return g_common_gram_logical_validations.load(std::memory_order_relaxed); +#else + return 0; +#endif +} +void reset_common_gram_logical_validation_count() { +#ifdef BE_TEST + g_common_gram_logical_validations.store(0, std::memory_order_relaxed); +#endif +} +uint64_t common_gram_plain_cache_probes() { +#ifdef BE_TEST + return g_common_gram_plain_cache_probes.load(std::memory_order_relaxed); +#else + return 0; +#endif +} +uint64_t common_gram_plain_cache_hits() { +#ifdef BE_TEST + return g_common_gram_plain_cache_hits.load(std::memory_order_relaxed); +#else + return 0; +#endif +} +uint64_t common_gram_plain_intern_table_probes() { +#ifdef BE_TEST + return g_common_gram_plain_intern_table_probes.load(std::memory_order_relaxed); +#else + return 0; +#endif +} +void reset_common_gram_plain_cache_counts() { +#ifdef BE_TEST + g_common_gram_plain_cache_probes.store(0, std::memory_order_relaxed); + g_common_gram_plain_cache_hits.store(0, std::memory_order_relaxed); + g_common_gram_plain_intern_table_probes.store(0, std::memory_order_relaxed); +#endif +} +uint64_t owned_term_full_byte_comparison_count() { +#ifdef BE_TEST + return g_owned_term_full_byte_comparisons.load(std::memory_order_relaxed); +#else + return 0; +#endif +} +void reset_owned_term_full_byte_comparison_count() { +#ifdef BE_TEST + g_owned_term_full_byte_comparisons.store(0, std::memory_order_relaxed); +#endif +} +void fail_next_owned_term_reserve() { +#ifdef BE_TEST + g_fail_next_owned_term_reserve.store(true, std::memory_order_relaxed); +#endif +} +void fail_next_owned_term_emplace() { +#ifdef BE_TEST + g_fail_next_owned_term_emplace.store(true, std::memory_order_relaxed); +#endif +} +uint64_t spill_gate_check_count() { +#ifdef BE_TEST + return g_spill_gate_checks.load(std::memory_order_relaxed); +#else + return 0; +#endif +} +void reset_spill_gate_check_count() { +#ifdef BE_TEST + g_spill_gate_checks.store(0, std::memory_order_relaxed); +#endif +} +uint64_t compact_chain_varint_decode_count() { +#ifdef BE_TEST + return g_compact_chain_varint_decodes.load(std::memory_order_relaxed); +#else + return 0; +#endif +} +void reset_compact_chain_varint_decode_count() { +#ifdef BE_TEST + g_compact_chain_varint_decodes.store(0, std::memory_order_relaxed); +#endif +} +} // namespace testing + +SpimiTermBuffer::SpimiTermBuffer(const std::vector* vocab, bool has_positions, + size_t spill_threshold_bytes, MemoryReporter* reporter) + : vocab_(vocab), + // Bind the equality functor to &owned_vocab_ even in borrowed mode: + // add_token(string_view) rejects before the functor can dereference it, + // and binding unconditionally keeps both constructors symmetric. + // Initialized in the member-init list (NOT the body): the functors are + // NESTED types, whose default-constructibility is not yet established at + // the point the flat set's default ctor would be needed. The + // (bucket_count, hash, equal) constructor sidesteps that entirely. + // owned_vocab_ is constructed before intern_ (declaration order) and the + // buffer is non-movable, so &owned_vocab_ is stable for the buffer's life. + intern_(0, OwnedVocabHash {.vocab = &owned_vocab_}, OwnedVocabEq {&owned_vocab_}), + has_positions_(has_positions), + spill_threshold_bytes_(spill_threshold_bytes), + mem_reporter_(reporter) { + // Borrowed-vocab mode: only the 4 B/id slot-index array is sized to the + // vocabulary; the Term pool (slots_) grows with the LIVE touched count, so an + // all-but-empty vocabulary costs ~4 B/id instead of ~80 B/id. + slot_of_.assign(vocab_->size(), 0); + // The vocab-sized slot index is resident immediately and survives spills; report + // its initial positive delta now. + report_arena_delta(); +} + +SpimiTermBuffer::SpimiTermBuffer(bool has_positions, size_t spill_threshold_bytes, + MemoryReporter* reporter) + : vocab_(&owned_vocab_), + // Owned-vocab mode: bind both functors to the sole vocabulary so stored + // ids can rehash and string probes resolve full term equality. + // Initialized in the member-init list (NOT the body): + // the functors are NESTED types whose default-constructibility is not yet + // established where the flat set's default ctor (whose noexcept spec inspects + // the functors) would be needed for a body assignment, so the + // (bucket_count, hash, equal) constructor is used instead. owned_vocab_ is + // constructed before intern_ (declaration order) and the buffer is + // non-movable, so &owned_vocab_ is stable for the buffer's life. + intern_(0, OwnedVocabHash {.vocab = &owned_vocab_}, OwnedVocabEq {&owned_vocab_}), + has_positions_(has_positions), + spill_threshold_bytes_(spill_threshold_bytes), + mem_reporter_(reporter) { + report_arena_delta(); +} + +SpimiTermBuffer::~SpimiTermBuffer() { + // G09: leave the process-wide registry FIRST. unregister_buffer removes the + // entry (and its bytes) under the registry mutex -- the same mutex every + // flag store is made under -- so once it returns, no other thread can touch + // global_spill_requested_ while this buffer dies. + if (global_limiter_ != nullptr) { + global_limiter_->unregister_buffer(&global_spill_requested_); + global_limiter_ = nullptr; + } + // Balance the writer-level / Doris tracker on the error path: if the buffer is + // destroyed while resident bytes were reported but not yet freed-and-reported + // (e.g. a build aborts before draining), return them here so nothing leaks. + if (mem_reporter_ != nullptr && reported_resident_ != 0) { + mem_reporter_->report(-reported_resident_); + reported_resident_ = 0; + } + cleanup_runs(); +} + +void SpimiTermBuffer::attach_global_limiter(GlobalMemoryLimiter* limiter) { + // At-most-once: a re-attach would leave a stale registry entry behind (the + // dtor un-registers only the current limiter). + if (limiter == nullptr || global_limiter_ != nullptr) { + return; + } + global_limiter_ = limiter; + // Race-safe vs report: registration and every report run on the OWNER's + // thread, strictly ordered; the registry serializes them against other + // buffers' calls internally. Register with the CURRENT resident total AND + // the current spillable arena bytes (the victim-selection key) so the + // registry is exact from the first moment (a borrowed-vocab buffer + // already holds its vocab-sized slot index here). + global_limiter_->register_buffer(&global_spill_requested_, + static_cast(resident_bytes()), + static_cast(pool_.arena_bytes())); +} + +void SpimiTermBuffer::report_arena_delta() { + if (mem_reporter_ == nullptr && global_limiter_ == nullptr) { + return; + } + // Diff the REAL resident bytes (resident_bytes()) against the last reported + // total; emit the signed delta exactly once. + const auto now = static_cast(resident_bytes()); + // Per-token zero-delta debounce: skip the locked fetch_add when resident is + // unchanged (the common case -- arena_bytes() grows only ~every 32 KiB block and + // the other charged structures grow by geometric capacity steps / per new term + // only, so most tokens see delta==0). A + // delta==0 report() is a no-op (current_.fetch_add(0) plus a mirrored + // consume_release(0)) and leaves reported_resident_ == now, so current_bytes(), + // every over_cap() result, and the gate-2 spill timing stay bit-for-bit identical. + // The spill gate still evaluates the writer-level UNIFIED total whenever the + // arena is large enough to reclaim, even if this buffer's local delta is 0: + // the shared dict buffer may have crossed the cap independently. + if (now == reported_resident_) { + return; + } + if (mem_reporter_ != nullptr) { + mem_reporter_->report(now - reported_resident_); + } + // G09: forward the same debounced total -- as an ABSOLUTE, self-healing + // value -- to the process-wide registry, together with the current + // SPILLABLE arena bytes (the victim-selection key: only the arena is + // reclaimable by a forced spill; the persistent vocab/pair structures are + // not). This is the limiter's decision point: report() flags the + // largest-arena eligible buffers (possibly this one) while the global sum + // exceeds the budget. It only ever takes the registry mutex and flips + // advisory atomics; no lock is held here while spilling (any spill this + // buffer performs happens AFTER this returns, back in + // maybe_spill_after_token, on this thread). + if (global_limiter_ != nullptr) { + global_limiter_->report(&global_spill_requested_, now, + static_cast(pool_.arena_bytes())); + } + reported_resident_ = now; +} + +size_t SpimiTermBuffer::unique_terms() const { + return live_term_count_; +} + +uint64_t SpimiTermBuffer::resident_bytes() const { + // Everything live is charged by CAPACITY (the reserved tail is resident RSS + // and survives spills). All reads are O(1), since this runs once per token. + uint64_t b = pool_.arena_bytes(); // posting chains: docs + prx payload + b += static_cast(slot_of_.capacity()) * sizeof(uint32_t); // vocab-sized slot index + b += static_cast(slots_.capacity()) * sizeof(Term); // live Term pool + b += static_cast(free_slots_.capacity()) * sizeof(uint32_t); + b += static_cast(touched_ids_.capacity()) * sizeof(uint32_t); + // Owned-vocab machinery (all zero in borrowed mode): string headers by vector + // capacity, heap payloads via the incrementally-maintained counter, and the + // intern set's entries at a fixed per-entry estimate (kept at the + // pre-G10 node-set value so the gate-2 spill points are unchanged; see the + // constant's comment). + b += static_cast(owned_vocab_.capacity()) * sizeof(std::string); + b += owned_vocab_heap_bytes_; + b += static_cast(common_word_classification_.capacity()) * + sizeof(CommonWordClassification); + b += static_cast(intern_.size() + common_gram_pair_intern_.size()) * + kInternEntryEstimateBytes; + b += common_gram_pair_cache_bytes_; + b += common_gram_plain_term_cache_bytes_; + // Cached lexicographic ranks survive spills and are included by capacity. + b += static_cast(string_rank_.capacity()) * sizeof(uint32_t); + return b; +} + +// Returns the live Term for `term_id`, claiming a pool slot on first touch (1 == +// new). Reuses a freed slot from free_slots_ when available; otherwise appends a +// fresh Term to slots_. slot_of_[term_id] holds (slot index + 1); 0 means empty. +SpimiTermBuffer::Term& SpimiTermBuffer::term_slot(uint32_t term_id, bool* new_term) { + uint32_t enc = slot_of_[term_id]; + if (enc != 0) { + *new_term = false; + return slots_[enc - 1]; + } + *new_term = true; + uint32_t slot; + if (!free_slots_.empty()) { + slot = free_slots_.back(); + free_slots_.pop_back(); + } else { + slot = static_cast(slots_.size()); + slots_.emplace_back(); + } + slot_of_[term_id] = slot + 1; + return slots_[slot]; +} + +void SpimiTermBuffer::put_varint(Term* t, uint64_t v) { + if (t->head == kNoChain) { + t->head = pool_.start_chain(&t->w, &t->level); + } + if (v < 0x80U) { + pool_.append_byte(&t->w, &t->level, static_cast(v)); + return; + } + pool_.append_varint(&t->w, &t->level, v); +} + +void SpimiTermBuffer::accumulate_without_spill_gate(uint32_t term_id, uint32_t docid, uint32_t pos, + PostingChainShape shape) { + const bool retain_positions = shape == PostingChainShape::kTaggedPositioned; + const bool statless_common_gram = shape == PostingChainShape::kStatlessDocsOnly; + DCHECK(!retain_positions || has_positions_); + bool new_term = false; + Term& t = term_slot(term_id, &new_term); + if (new_term) { + t.shape = shape; + touched_ids_.push_back(term_id); + ++live_term_count_; + } else { + DCHECK(t.shape == shape); + } + // Docs-only accelerator postings are sets. Tokens for one input document are + // contiguous on the writer path, so discard repeated occurrences before they + // allocate arena bytes or enter spill/sort/posting encoding. + if (!retain_positions && t.started && t.cur_docid == docid) { + ++total_tokens_; + return; + } + // A token starts a new doc unless it continues the most-recent doc for this term. + const bool first_token = !t.started; + const bool new_doc = first_token || t.cur_docid != docid; + // A statless CommonGram singleton owns no chain. On its second distinct doc, + // backfill the first absolute docid before appending the current delta. + if (statless_common_gram && !first_token && t.head == kNoChain) { + DCHECK_EQ(t.ntok, 1U); + DCHECK_EQ(t.ndocs, 1U); + put_varint(&t, zigzag_encode(static_cast(t.cur_docid))); + } + + // Positioned and ordinary docs-only terms retain the tagged token stream used + // to reconstruct frequency. A statless CommonGram is already deduplicated per + // document and has no frequency, so its new_doc tag would be the constant 1; + // omit it and store only the document delta. + if (!statless_common_gram) { + // Widen to 64-bit so a full 32-bit position survives the shift. + const uint64_t tagged = retain_positions + ? ((static_cast(pos) << 1) | (new_doc ? 1U : 0U)) + : (new_doc ? 1U : 0U); + put_varint(&t, tagged); + } else { + DCHECK(new_doc); + } + if (new_doc) { + // Out-of-order docids are tolerated (zigzag delta is signed) and reordered at + // finalize; flag them so to_postings sorts. The delta base is the previous + // distinct doc (cur_docid), which is 0 for the very first doc (started==false). + const int64_t base = t.started ? static_cast(t.cur_docid) : 0; + if (t.started && docid < t.cur_docid) { + t.sorted = false; + } + const int64_t delta = static_cast(docid) - base; + if (!first_token || !statless_common_gram) { + put_varint(&t, zigzag_encode(delta)); + } + t.cur_docid = docid; + t.started = true; + // Exact new-doc group count; out-of-order coalescing can only shrink it. + ++t.ndocs; + } + ++t.ntok; + ++total_tokens_; +} + +void SpimiTermBuffer::accumulate(uint32_t term_id, uint32_t docid, uint32_t pos, + bool retain_positions) { + accumulate_without_spill_gate(term_id, docid, pos, + retain_positions ? PostingChainShape::kTaggedPositioned + : PostingChainShape::kTaggedDocsOnly); + maybe_spill_after_token(); +} + +// Per-input-token gate-2 tail. Ordinary adds invoke it after one posting; the +// fused CommonGrams path invokes it after its gram and right plain posting. It +// reports the token's REAL resident growth FIRST so the writer's unified total +// (reporter_->current_bytes()) reflects it before the gate check (single-source +// diff; cheap: a subtraction + relaxed atomic add), then evaluates the spill triggers: +// * Gate-2 (UNIFIED): with a reporter attached, trigger on the writer's TOTAL +// build RAM (arena + vocab structures + dict) crossing the one +// configured cap -- the same total and cap every buffer of this writer +// shares, not a per-buffer threshold. Off Doris (no reporter) fall back to +// the local spill_threshold_bytes_ against resident_bytes(). +// * G08 anti-churn floor: a gate-2 spill reclaims ONLY the posting arena +// (pool_.reset()); the vocab / slot structures resident_bytes() +// now also charges SURVIVE it. Once those persistent bytes alone exceed the +// cap, an unconditioned +// trigger would spill EVERY subsequent token -- one-block runs, k-way-merge +// and spill-fixed-cost blowup. Honor the cap only when at least a quarter of +// it is reclaimable arena: peak stays bounded at persistent + cap/4 and no +// run is smaller than cap/4, while the one-block minimum keeps small caps +// (tests, tiny configs) spilling on the first block exactly as before. +// * Hard arena safety stop, active even in unlimited mode and BYPASSING the +// floor: when the arena nears the 4 GiB uint32-offset limit, spill now -- +// without it a single >4 GiB in-memory segment wraps alloc_run and silently +// corrupts data. A forced spill + final k-way merge stays byte-identical +// regardless of when it fires. +// spill_to_run() resets the arena and reports its negative internally, so the +// unified total drops (and the trigger self-rearms) after each spill. +void SpimiTermBuffer::maybe_spill_after_token() { +#ifdef BE_TEST + g_spill_gate_checks.fetch_add(1, std::memory_order_relaxed); +#endif + constexpr uint64_t kArenaSpillCap = 0xE0000000ULL; // 3.5 GiB, < UINT32_MAX margin + const bool global_requested = global_spill_requested_.load(std::memory_order_relaxed); + const bool arena_near_limit = pool_.arena_bytes() >= kArenaSpillCap; + report_arena_delta(); + const uint64_t gate_cap = + mem_reporter_ != nullptr ? mem_reporter_->cap_bytes() : spill_threshold_bytes_; + const bool arena_worth_spilling = + pool_.arena_bytes() >= std::max(CompactPostingPool::kBlockSize, gate_cap / 4); + // G09: the process-wide limiter flagged this buffer (one of the + // largest-ARENA eligible consumers while the global total exceeded the + // budget). Honored HERE, on the owner's own thread -- never on the + // reporting thread that set the flag. The G08 anti-churn floor (cap/4) is + // deliberately BYPASSED (each victim's arena is below cap/4 by + // construction: it never reached its per-writer gate -- that is exactly + // why the global sum grew), but the FORCED-SPILL FLOOR + // (snii_forced_spill_min_arena_bytes, >= one arena block so a run is + // writable) still applies: a forced spill reclaims ONLY the arena, so + // honoring below the floor would cut a tiny run for near-zero relief. + // Below the floor the request is a NO-OP that stays PENDING -- it is NOT + // retried as a spill each token -- and is honored once the arena regrows + // past the floor (the limiter's victim selection applies the same floor, + // so a below-floor flag only arises from a floor/config race or a test + // seam). A request that finds the owner already drained is never observed + // again -- an advisory no-op (the dtor un-registers) -- and a stale + // re-request after a spill costs at most one extra floor-sized run + // (double-spill is harmless, byte-identical output). + const bool global_spill_now = + global_requested && + pool_.arena_bytes() >= std::max(CompactPostingPool::kBlockSize, + forced_spill_min_arena_bytes_); + const bool over_cap = !global_spill_now && !arena_near_limit && arena_worth_spilling && + (mem_reporter_ != nullptr ? mem_reporter_->over_cap() + : (spill_threshold_bytes_ != 0 && + resident_bytes() >= spill_threshold_bytes_)); + if ((over_cap || global_spill_now || arena_near_limit) && spill_status_.ok()) { + if (global_requested) { + // Consume the request BEFORE spilling: this spill releases exactly + // the arena a forced spill would, so it satisfies the request no + // matter which trigger won the OR above. + global_spill_requested_.store(false, std::memory_order_relaxed); +#ifdef BE_TEST + // Seam under BE_TEST only: per-token path shared by every + // concurrent writer. + g_global_forced_spills.fetch_add(1, std::memory_order_relaxed); +#endif + } + spill_status_ = spill_to_run(); + } +} + +void SpimiTermBuffer::add_token(uint32_t term_id, uint32_t docid, uint32_t pos) { + add_token(term_id, docid, pos, has_positions_); +} + +void SpimiTermBuffer::add_token(uint32_t term_id, uint32_t docid, uint32_t pos, + bool retain_positions) { + // Hot path: a pooled slot lookup + a couple of pushes. No hashing, no string + // construction per token. Reject (and latch) an out-of-range id. + if (term_id >= slot_of_.size()) { + if (spill_status_.ok()) { + spill_status_ = Status::Error( + "spimi: term_id out of vocab range"); + } + return; + } + accumulate(term_id, docid, pos, retain_positions); +} + +void SpimiTermBuffer::add_token(std::string_view term, uint32_t docid, uint32_t pos) { + add_token(term, docid, pos, has_positions_); +} + +void SpimiTermBuffer::add_token(std::string_view term, uint32_t docid, uint32_t pos, + bool retain_positions) { + // Compatibility path: intern the term into the owned vocabulary on first + // occurrence, then accumulate by its id. ONLY valid in OWNED-vocab mode. In + // BORROWED-vocab mode vocab_ points at the caller's vector, NOT &owned_vocab_: + // interning here would grow owned_vocab_ / intern_ / slot_of_ out of step with + // the active (borrowed) vocab, so the new id indexes the WRONG string and writes + // a slot_of_ entry the borrowed-vocab build never reconciles -- silent + // corruption. Reject (and latch) instead of forwarding by a bogus id. + if (vocab_ != &owned_vocab_) { + if (spill_status_.ok()) { + spill_status_ = Status::Error( + "spimi: add_token(string_view) requires owned-vocab mode"); + } + return; + } + DCHECK(!common_gram_pair_keys_); + const uint32_t term_id = find_or_intern_owned_term(term); + accumulate(term_id, docid, pos, retain_positions); +} + +PlainTermId SpimiTermBuffer::intern_plain_term(std::string_view physical_plain_term) { + DCHECK(common_gram_pair_keys_); + DCHECK(vocab_ == &owned_vocab_); + DCHECK(!physical_plain_term.empty()); + DCHECK(!segment_v2::inverted_index::is_internal_term_key(physical_plain_term)); + const size_t term_hash = intern_.hash(physical_plain_term); + uint32_t term_id = find_interned_plain_term(physical_plain_term, term_hash); + if (term_id == CommonGramPlainTermCache::kInvalidTermId) { + term_id = intern_owned_term(std::string(physical_plain_term), term_hash); + remember_plain_term(term_hash, term_id); + } + return PlainTermId {.value = term_id}; +} + +PlainTermId SpimiTermBuffer::intern_plain_term(std::string_view physical_plain_term, + std::string_view logical_plain_term) { + DCHECK(common_gram_pair_keys_); + DCHECK(vocab_ == &owned_vocab_); + DCHECK(!physical_plain_term.empty()); + DCHECK(!logical_plain_term.empty()); + DCHECK(!segment_v2::inverted_index::is_internal_term_key(physical_plain_term)); + + const size_t term_hash = intern_.hash(physical_plain_term); + uint32_t term_id = find_interned_plain_term(physical_plain_term, term_hash); + if (term_id != CommonGramPlainTermCache::kInvalidTermId) { + return PlainTermId {.value = term_id}; + } + +#ifdef BE_TEST + g_common_gram_logical_validations.fetch_add(1, std::memory_order_relaxed); +#endif + auto validation = segment_v2::inverted_index::validate_common_grams_logical_term( + logical_plain_term, "input token"); + if (!validation.ok()) { + throw Exception(validation); + } + if (physical_plain_term.size() > segment_v2::inverted_index::COMMON_GRAM_MAX_ENCODED_BYTES) { + throw Exception(Status::Error( + "CommonGrams escaped plain term would exceed the 16383-byte key limit; " + "set enable_common_grams_index_build=false and retry the import in a new " + "transaction")); + } + term_id = intern_owned_term(std::string(physical_plain_term), term_hash); + remember_plain_term(term_hash, term_id); + return PlainTermId {.value = term_id}; +} + +ClassifiedPlainTerm SpimiTermBuffer::intern_classified_plain_term( + std::string_view physical_plain_term, std::string_view logical_plain_term, + const segment_v2::inverted_index::CommonWordSet& common_words) { + DCHECK(common_gram_pair_keys_); + const PlainTermId id = intern_plain_term(physical_plain_term, logical_plain_term); + DCHECK_EQ(common_word_classification_.size(), owned_vocab_.size()); + DCHECK_LT(id.value, common_word_classification_.size()); + CommonWordClassification& classification = common_word_classification_[id.value]; + if (classification == CommonWordClassification::kUnknown) { + classification = common_words.contains(logical_plain_term) + ? CommonWordClassification::kCommon + : CommonWordClassification::kNotCommon; + } + return ClassifiedPlainTerm { + .id = id, + .is_common = classification == CommonWordClassification::kCommon, + }; +} + +void SpimiTermBuffer::add_plain_token(PlainTermId term_id, uint32_t docid, uint32_t pos) { + DCHECK(common_gram_pair_keys_); + DCHECK_LT(term_id.value, owned_vocab_.size()); + DCHECK(!segment_v2::inverted_index::is_internal_term_key(owned_vocab_[term_id.value])); + accumulate(term_id.value, docid, pos, has_positions_); +} + +void SpimiTermBuffer::add_common_gram_without_spill_gate(PlainTermId left, PlainTermId right, + uint32_t docid, uint32_t pos, + bool retain_positions) { + DCHECK(common_gram_pair_keys_); + DCHECK_LT(left.value, owned_vocab_.size()); + DCHECK_LT(right.value, owned_vocab_.size()); + DCHECK(!segment_v2::inverted_index::is_internal_term_key(owned_vocab_[left.value])); + DCHECK(!segment_v2::inverted_index::is_internal_term_key(owned_vocab_[right.value])); + DCHECK(common_gram_pair_cache_ != nullptr); + const PostingChainShape shape = retain_positions ? PostingChainShape::kTaggedPositioned + : PostingChainShape::kStatlessDocsOnly; + const uint64_t pair = (static_cast(left.value) << 32) | right.value; + CommonGramPairCache::Entry& entry = + common_gram_pair_cache_->entries[CommonGramPairCache::index(pair)]; +#ifdef BE_TEST + g_common_gram_pair_cache_probes.fetch_add(1, std::memory_order_relaxed); +#endif + if (entry.term_id != CommonGramPairCache::kInvalidTermId && entry.pair == pair) { +#ifdef BE_TEST + g_common_gram_pair_cache_pair_hits.fetch_add(1, std::memory_order_relaxed); +#endif + DCHECK_LT(entry.term_id, slot_of_.size()); + if (!retain_positions && entry.last_docid == docid) { +#ifdef BE_TEST + g_common_gram_pair_cache_same_doc_hits.fetch_add(1, std::memory_order_relaxed); +#endif + ++total_tokens_; + return; + } + entry.last_docid = docid; + accumulate_without_spill_gate(entry.term_id, docid, pos, shape); + return; + } + + const uint32_t term_id = find_or_intern_common_gram_pair(left, right, pair); + DCHECK_NE(term_id, CommonGramPairCache::kInvalidTermId); + entry = CommonGramPairCache::Entry {.pair = pair, .term_id = term_id, .last_docid = docid}; + accumulate_without_spill_gate(term_id, docid, pos, shape); +} + +void SpimiTermBuffer::add_common_gram(PlainTermId left, PlainTermId right, uint32_t docid, + uint32_t pos, bool retain_positions) { + add_common_gram_without_spill_gate(left, right, docid, pos, retain_positions); + maybe_spill_after_token(); +} + +void SpimiTermBuffer::add_common_gram_and_plain(PlainTermId left, PlainTermId right, uint32_t docid, + uint32_t gram_pos, uint32_t plain_pos, + bool retain_gram_positions) { + add_common_gram_without_spill_gate(left, right, docid, gram_pos, retain_gram_positions); + DCHECK_LT(right.value, owned_vocab_.size()); + DCHECK(!segment_v2::inverted_index::is_internal_term_key(owned_vocab_[right.value])); + accumulate_without_spill_gate(right.value, docid, plain_pos, + has_positions_ ? PostingChainShape::kTaggedPositioned + : PostingChainShape::kTaggedDocsOnly); + maybe_spill_after_token(); +} + +void SpimiTermBuffer::enable_common_gram_pair_keys() { + DORIS_CHECK(vocab_ == &owned_vocab_); + DORIS_CHECK_EQ(total_tokens_, 0); + DORIS_CHECK(owned_vocab_.empty()); + DORIS_CHECK(!common_gram_pair_keys_); + DORIS_CHECK(common_word_classification_.empty()); + auto pair_cache = std::make_unique(); + auto plain_term_cache = std::make_unique(); + common_gram_pair_keys_ = true; + common_gram_pair_cache_ = std::move(pair_cache); + common_gram_pair_cache_bytes_ = sizeof(CommonGramPairCache); + common_gram_plain_term_cache_ = std::move(plain_term_cache); + common_gram_plain_term_cache_bytes_ = sizeof(CommonGramPlainTermCache); + report_arena_delta(); +} + +uint32_t SpimiTermBuffer::find_or_intern_owned_term(std::string_view term) { + static_assert(std::is_same_v); + DCHECK_LE(term.size(), std::numeric_limits::max()); + const size_t term_hash = intern_.hash(term); + const auto found = intern_.find(term, term_hash); + if (found != intern_.end()) { + const uint32_t term_id = *found; + if (g11_prefetch_enabled()) { + __builtin_prefetch(slot_of_.data() + term_id); + } + return term_id; + } + return intern_owned_term(std::string(term), term_hash); +} + +uint32_t SpimiTermBuffer::find_or_intern_common_gram_pair(PlainTermId left, PlainTermId right, + uint64_t pair) { + DCHECK(common_gram_pair_keys_); + DCHECK_LT(left.value, owned_vocab_.size()); + DCHECK_LT(right.value, owned_vocab_.size()); + DCHECK(!segment_v2::inverted_index::is_internal_term_key(owned_vocab_[left.value])); + DCHECK(!segment_v2::inverted_index::is_internal_term_key(owned_vocab_[right.value])); +#ifdef BE_TEST + g_common_gram_native_pair_probes.fetch_add(1, std::memory_order_relaxed); +#endif + const auto found = common_gram_pair_intern_.find(pair); + if (found != common_gram_pair_intern_.end()) { +#ifdef BE_TEST + g_common_gram_native_pair_hits.fetch_add(1, std::memory_order_relaxed); +#endif + return found->second; + } + + const auto key = EncodeCommonGramPairKey(left, right); + const uint32_t term_id = append_owned_vocab_term(std::string(key.data(), key.size())); + const auto [inserted_it, inserted] = common_gram_pair_intern_.try_emplace(pair, term_id); + DCHECK(inserted); + DCHECK_EQ(inserted_it->second, term_id); +#ifdef BE_TEST + g_common_gram_native_pair_inserts.fetch_add(1, std::memory_order_relaxed); +#endif + return term_id; +} + +uint32_t SpimiTermBuffer::find_interned_plain_term(std::string_view term, size_t term_hash) { + DCHECK(common_gram_pair_keys_); + DCHECK(common_gram_plain_term_cache_ != nullptr); +#ifdef BE_TEST + g_common_gram_plain_cache_probes.fetch_add(1, std::memory_order_relaxed); +#endif + const uint32_t cached_term_id = + common_gram_plain_term_cache_->find(term_hash, term, owned_vocab_); + if (cached_term_id != CommonGramPlainTermCache::kInvalidTermId) { +#ifdef BE_TEST + g_common_gram_plain_cache_hits.fetch_add(1, std::memory_order_relaxed); +#endif + if (g11_prefetch_enabled()) { + __builtin_prefetch(slot_of_.data() + cached_term_id); + } + return cached_term_id; + } + +#ifdef BE_TEST + g_common_gram_plain_intern_table_probes.fetch_add(1, std::memory_order_relaxed); +#endif + const auto found = intern_.find(term, term_hash); + if (found == intern_.end()) { + return CommonGramPlainTermCache::kInvalidTermId; + } + const uint32_t term_id = *found; + remember_plain_term(term_hash, term_id); + if (g11_prefetch_enabled()) { + __builtin_prefetch(slot_of_.data() + term_id); + } + return term_id; +} + +void SpimiTermBuffer::remember_plain_term(size_t term_hash, uint32_t term_id) { + DCHECK(common_gram_pair_keys_); + DCHECK(common_gram_plain_term_cache_ != nullptr); + DCHECK_LT(term_id, owned_vocab_.size()); + common_gram_plain_term_cache_->remember(term_hash, term_id); +} + +bool SpimiTermBuffer::transient_term_less(uint32_t left_id, uint32_t right_id) const { + const std::vector& v = vocab(); + const std::string_view left = v[left_id]; + const std::string_view right = v[right_id]; + if (!common_gram_pair_keys_) { + return left < right; + } + + const bool left_is_pair = is_common_gram_pair_key(left); + const bool right_is_pair = is_common_gram_pair_key(right); + if (left_is_pair != right_is_pair) { + const std::string_view plain = left_is_pair ? right : left; + DCHECK(!plain.empty()); + DCHECK(!segment_v2::inverted_index::is_internal_term_key(plain)); + const bool pair_sorts_first = + static_cast( + segment_v2::inverted_index::INTERNAL_TERM_NAMESPACE_BEGIN.front()) < + static_cast(plain.front()); + return left_is_pair ? pair_sorts_first : !pair_sorts_first; + } + if (!left_is_pair) { + return left < right; + } + + const CommonGramPairIds left_ids = decode_common_gram_pair_key_unchecked(left); + const CommonGramPairIds right_ids = decode_common_gram_pair_key_unchecked(right); + DCHECK_LT(left_ids.left.value, v.size()); + DCHECK_LT(left_ids.right.value, v.size()); + DCHECK_LT(right_ids.left.value, v.size()); + DCHECK_LT(right_ids.right.value, v.size()); + + const LogicalPlainKeyView left_left_key(v[left_ids.left.value]); + const LogicalPlainKeyView left_right_key(v[left_ids.right.value]); + const LogicalPlainKeyView right_left_key(v[right_ids.left.value]); + const LogicalPlainKeyView right_right_key(v[right_ids.right.value]); + if (left_left_key.size() != right_left_key.size()) { + return left_left_key.size() < right_left_key.size(); + } + const int left_component_order = compare_logical_plain_keys(left_left_key, right_left_key); + if (left_component_order != 0) { + return left_component_order < 0; + } + return compare_logical_plain_keys(left_right_key, right_right_key) < 0; +} + +std::string SpimiTermBuffer::materialize_transient_term(std::string_view term) const { + if (!is_common_gram_pair_key(term)) { + return std::string(term); + } + + DCHECK(common_gram_pair_keys_); + const CommonGramPairIds ids = decode_common_gram_pair_key_unchecked(term); + DCHECK_LT(ids.left.value, owned_vocab_.size()); + DCHECK_LT(ids.right.value, owned_vocab_.size()); + DCHECK(!is_common_gram_pair_key(owned_vocab_[ids.left.value])); + DCHECK(!is_common_gram_pair_key(owned_vocab_[ids.right.value])); + + std::string left_scratch; + std::string right_scratch; + const std::string_view left = + decode_logical_plain_term_trusted(owned_vocab_[ids.left.value], &left_scratch); + const std::string_view right = + decode_logical_plain_term_trusted(owned_vocab_[ids.right.value], &right_scratch); + std::string output; + [[maybe_unused]] const bool encoded = + segment_v2::inverted_index::try_encode_common_gram_prevalidated(left, right, output); + DCHECK(encoded); + return output; +} + +// Prepared first-time insertion stores the string before emplace so every +// stored id remains resolvable during later growth rehashes. +uint32_t SpimiTermBuffer::intern_owned_term(std::string&& term_str, size_t term_hash) { + const size_t next_vocab_size = owned_vocab_.size() + 1; + DCHECK_LE(next_vocab_size, std::numeric_limits::max()); + + size_t target_capacity = owned_vocab_.capacity(); + if (target_capacity < next_vocab_size) { + target_capacity = target_capacity <= std::numeric_limits::max() / 2 + ? std::max(next_vocab_size, target_capacity * 2) + : next_vocab_size; + } + + // Prepare append-only vectors geometrically before publishing a vocabulary + // id. A later reserve may throw after an earlier vector already changed + // capacity, so the catch path must settle that resident delta before + // propagating the failure. + try { + owned_vocab_.reserve(target_capacity); +#ifdef BE_TEST + if (g_fail_next_owned_term_reserve.exchange(false, std::memory_order_relaxed)) { + throw std::bad_alloc(); + } +#endif + slot_of_.reserve(target_capacity); + if (common_gram_pair_keys_) { + common_word_classification_.reserve(target_capacity); + } + } catch (...) { + report_arena_delta(); + throw; + } + report_arena_delta(); + + const uint32_t term_id = append_owned_vocab_term(std::move(term_str)); + static_assert(std::is_nothrow_copy_constructible_v); + + const auto rollback_append = [&]() { +#ifdef BE_TEST + g_vocab_materializations.fetch_sub(1, std::memory_order_relaxed); +#endif + owned_vocab_heap_bytes_ -= string_heap_bytes(owned_vocab_.back()); + slot_of_.pop_back(); + if (common_gram_pair_keys_) { + common_word_classification_.pop_back(); + } + owned_vocab_.pop_back(); + report_arena_delta(); + }; + + // phmap allocates a growth table before publishing the prepared slot, and + // constructing this trivial key cannot throw. If allocation fails, the old + // table is intact and only the preceding vocabulary append needs rollback. + const auto [it, inserted] = [&]() { + try { +#ifdef BE_TEST + if (g_fail_next_owned_term_emplace.exchange(false, std::memory_order_relaxed)) { + throw std::bad_alloc(); + } +#endif + return intern_.emplace_with_hash(term_hash, term_id); + } catch (...) { + rollback_append(); + throw; + } + }(); + if (!inserted) { + rollback_append(); + } + DCHECK(inserted); + DCHECK_EQ(*it, term_id); + return term_id; +} + +uint32_t SpimiTermBuffer::append_owned_vocab_term(std::string&& term_str) { + const uint32_t term_id = static_cast(owned_vocab_.size()); + owned_vocab_.emplace_back(std::move(term_str)); + if (common_gram_pair_keys_) { + common_word_classification_.push_back(CommonWordClassification::kUnknown); + DCHECK_EQ(common_word_classification_.size(), owned_vocab_.size()); + } + slot_of_.push_back(0); // vocab grows: new id starts with no live slot + // G08: credit the stored string's heap payload (0 for SSO); the header is + // charged via owned_vocab_.capacity(). + owned_vocab_heap_bytes_ += string_heap_bytes(owned_vocab_[term_id]); +#ifdef BE_TEST + g_vocab_materializations.fetch_add(1, std::memory_order_relaxed); +#endif + return term_id; +} + +namespace { + +// Reorders a term's flat arrays into ascending-docid order, COALESCING any +// same-docid groups so the result has exactly one entry per docid -- matching the +// k-way-merge path's boundary-doc coalescing and the writer's strictly-ascending +// precondition. Only invoked for the rare term that received out-of-order docids +// (the common ascending path leaves t.sorted true and skips it). +// +// A docid may REVISIT (e.g. feed 5,1,5): the chain holds two separate doc-groups +// for doc 5. A STABLE sort keeps equal-docid groups in arrival order, then the +// coalesce pass sums their freqs and concatenates their positions in that same +// (document/arrival) order -- so the merged positions stay consistent with the +// merged freqs, exactly as the run-order merge would have produced. +template +Status reserve_tracked_vector(std::vector* values, size_t target, + MemoryReporter* memory_reporter, + MemoryReporter::Reservation* reservation) { + if (target <= values->capacity()) { + return Status::OK(); + } + if (target > std::numeric_limits::max() / sizeof(T)) { + return Status::Error( + "spimi materialization: vector byte capacity overflow"); + } + if (memory_reporter == nullptr) { + values->reserve(target); + return Status::OK(); + } + MemoryReporter::Reservation replacement; + RETURN_IF_ERROR(reservation->prepare_replacement(static_cast(target) * sizeof(T), + &replacement)); + values->reserve(target); + DCHECK_EQ(values->capacity(), target); + *reservation = std::move(replacement); + return Status::OK(); +} + +Status sort_by_docid(std::vector* docids, std::vector* freqs, + std::vector* positions_flat, bool has_positions, + MemoryReporter* memory_reporter, + MemoryReporter::Reservation* docids_reservation, + MemoryReporter::Reservation* freqs_reservation, + MemoryReporter::Reservation* positions_reservation) { + const size_t n = docids->size(); + MemoryReporter::Reservation order_reservation = memory_reporter == nullptr + ? MemoryReporter::Reservation() + : memory_reporter->make_reservation(); + MemoryReporter::Reservation pos_off_reservation = memory_reporter == nullptr + ? MemoryReporter::Reservation() + : memory_reporter->make_reservation(); + MemoryReporter::Reservation sorted_docids_reservation = + memory_reporter == nullptr ? MemoryReporter::Reservation() + : memory_reporter->make_reservation(); + MemoryReporter::Reservation sorted_freqs_reservation = + memory_reporter == nullptr ? MemoryReporter::Reservation() + : memory_reporter->make_reservation(); + MemoryReporter::Reservation sorted_positions_reservation = + memory_reporter == nullptr ? MemoryReporter::Reservation() + : memory_reporter->make_reservation(); + std::vector order; + RETURN_IF_ERROR(reserve_tracked_vector(&order, n, memory_reporter, &order_reservation)); + order.resize(n); + std::iota(order.begin(), order.end(), 0); + // The original index breaks equal-doc ties, preserving arrival order without + // stable_sort's implementation-owned allocation. + std::ranges::sort(order, [&](size_t a, size_t b) { + if ((*docids)[a] != (*docids)[b]) { + return (*docids)[a] < (*docids)[b]; + } + return a < b; + }); + + std::vector pos_off; + if (has_positions) { + RETURN_IF_ERROR(reserve_tracked_vector(&pos_off, n, memory_reporter, &pos_off_reservation)); + pos_off.resize(n); + uint32_t running = 0; + for (size_t i = 0; i < n; ++i) { + pos_off[i] = running; + running += (*freqs)[i]; + } + } + std::vector nd, nf, np; + RETURN_IF_ERROR(reserve_tracked_vector(&nd, n, memory_reporter, &sorted_docids_reservation)); + RETURN_IF_ERROR(reserve_tracked_vector(&nf, n, memory_reporter, &sorted_freqs_reservation)); + if (has_positions) { + RETURN_IF_ERROR(reserve_tracked_vector(&np, positions_flat->size(), memory_reporter, + &sorted_positions_reservation)); + } + for (size_t k : order) { + // Coalesce a revisited docid into the previous entry (it sorts adjacent now): + // sum freqs and append this group's positions right after the prior group's, + // so flat doc order stays partitioned by the merged freqs. + if (!nd.empty() && nd.back() == (*docids)[k]) { + if (has_positions) { + nf.back() += (*freqs)[k]; + } + } else { + nd.push_back((*docids)[k]); + nf.push_back((*freqs)[k]); + } + if (has_positions) { + np.insert(np.end(), positions_flat->begin() + pos_off[k], + positions_flat->begin() + pos_off[k] + (*freqs)[k]); + } + } + docids->swap(nd); + freqs->swap(nf); + std::swap(*docids_reservation, sorted_docids_reservation); + std::swap(*freqs_reservation, sorted_freqs_reservation); + if (has_positions) { + positions_flat->swap(np); + std::swap(*positions_reservation, sorted_positions_reservation); + } + return Status::OK(); +} + +} // namespace + +namespace { + +// Decodes one varint from a pool chain cursor. The chain was written by +// encode_varint*, so the same LEB128 continuation-bit loop reconstructs it. +uint64_t decode_chain_varint(CompactPostingPool::Cursor* c) { +#ifdef BE_TEST + g_compact_chain_varint_decodes.fetch_add(1, std::memory_order_relaxed); +#endif + return c->read_varint(); +} + +} // namespace + +// Decodes the compact tagged chain directly into caller-owned posting windows. +class SpimiTermBuffer::ArenaTermPostingSource final : public TermPostingSource { +public: + ArenaTermPostingSource(const CompactPostingPool* pool, const Term& term) + : shape_(term.shape), + remaining_docs_(term.ndocs), + remaining_tokens_(term.ntok), + inline_docid_(term.cur_docid) { + if (term.head != kNoChain) { + doc_cursor_.emplace(pool->cursor(term.head, term.w.cur)); + } + } + + Status fill(uint32_t target_docs, TermPostingBuffer* out, bool* exhausted) override { + if (out == nullptr || exhausted == nullptr || target_docs == 0 || !out->empty()) { + return Status::Error( + "spimi arena source: invalid fill arguments"); + } + const uint32_t count = std::min(target_docs, remaining_docs_); + if (count == 0) { + *exhausted = true; + return Status::OK(); + } + + if (shape_ == PostingChainShape::kStatlessDocsOnly) { + MutableTermPostingSpan destination; + RETURN_IF_ERROR(out->grow_uninitialized(count, /*has_freqs=*/false, + /*position_count=*/0, &destination)); + for (uint32_t i = 0; i < count; ++i) { + if (!doc_cursor_) { + DCHECK_EQ(remaining_docs_, 1U); + destination.docids[i] = inline_docid_; + } else { + absolute_docid_ += zigzag_decode(decode_chain_varint(&*doc_cursor_)); + destination.docids[i] = static_cast(absolute_docid_); + } + } + remaining_tokens_ -= count; + } else { + RETURN_IF_ERROR(fill_tagged(count, out)); + } + + remaining_docs_ -= count; + *exhausted = remaining_docs_ == 0; + if (*exhausted) { + DCHECK_EQ(remaining_tokens_, 0U); + DCHECK(!pending_new_doc_); + } + return Status::OK(); + } + + bool exhausted() const { return remaining_docs_ == 0; } + +private: + Status fill_tagged(uint32_t count, TermPostingBuffer* out) { + const bool has_positions = shape_ == PostingChainShape::kTaggedPositioned; + const bool terminal_fill = count == remaining_docs_; + const size_t position_count = has_positions && terminal_fill ? remaining_tokens_ : 0; + MutableTermPostingSpan documents; + RETURN_IF_ERROR( + out->grow_uninitialized(count, /*has_freqs=*/true, position_count, &documents)); + size_t position_index = 0; + for (uint32_t i = 0; i < count; ++i) { + uint64_t tagged = 0; + if (pending_new_doc_) { + tagged = pending_tagged_; + pending_new_doc_ = false; + } else { + DCHECK_GT(remaining_tokens_, 0U); + tagged = decode_chain_varint(&*doc_cursor_); + } + DCHECK_NE(tagged & 1U, 0U); + absolute_docid_ += zigzag_decode(decode_chain_varint(&*doc_cursor_)); + documents.docids[i] = static_cast(absolute_docid_); + uint32_t frequency = 0; + while (true) { + if (has_positions) { + if (terminal_fill) { + documents.positions_flat[position_index++] = + static_cast(tagged >> 1); + } else { + RETURN_IF_ERROR(out->append_position(static_cast(tagged >> 1))); + } + } + ++frequency; + --remaining_tokens_; + if (remaining_tokens_ == 0) { + break; + } + tagged = decode_chain_varint(&*doc_cursor_); + if ((tagged & 1U) != 0) { + pending_tagged_ = tagged; + pending_new_doc_ = true; + break; + } + } + documents.freqs[i] = frequency; + } + DCHECK_EQ(position_index, documents.positions_flat.size()); + return Status::OK(); + } + + PostingChainShape shape_; + std::optional doc_cursor_; + uint32_t remaining_docs_ = 0; + uint32_t remaining_tokens_ = 0; + uint32_t inline_docid_ = 0; + int64_t absolute_docid_ = 0; + uint64_t pending_tagged_ = 0; + bool pending_new_doc_ = false; +}; + +Status SpimiTermBuffer::to_postings(std::string term, Term&& t, + TrackedTermPostings* tracked) const { + DCHECK(tracked != nullptr); + TermPostings& postings = tracked->postings; + DCHECK(postings.docids.empty()); + DCHECK(postings.freqs.empty()); + DCHECK(postings.positions_flat.empty()); + postings.term = std::move(term); + postings.retain_positions = t.shape == PostingChainShape::kTaggedPositioned; + if (t.ntok == 0) { + return Status::OK(); + } + + RETURN_IF_ERROR(reserve_tracked_vector(&postings.docids, t.ndocs, mem_reporter_, + &tracked->docids_reservation)); + if (t.shape != PostingChainShape::kStatlessDocsOnly) { + RETURN_IF_ERROR(reserve_tracked_vector(&postings.freqs, t.ndocs, mem_reporter_, + &tracked->freqs_reservation)); + } + if (t.shape == PostingChainShape::kTaggedPositioned) { + RETURN_IF_ERROR(reserve_tracked_vector(&postings.positions_flat, t.ntok, mem_reporter_, + &tracked->positions_reservation)); + } + + ArenaTermPostingSource source(&pool_, t); + TermPostingBuffer buffer(mem_reporter_); + bool exhausted = false; + while (!exhausted) { + buffer.clear_reuse(); + RETURN_IF_ERROR(source.fill(format::kAdaptiveWindowDocs, &buffer, &exhausted)); + postings.docids.insert(postings.docids.end(), buffer.docids().begin(), + buffer.docids().end()); + postings.freqs.insert(postings.freqs.end(), buffer.freqs().begin(), buffer.freqs().end()); + postings.positions_flat.insert(postings.positions_flat.end(), + buffer.positions_flat().begin(), + buffer.positions_flat().end()); + } + if (!t.sorted && t.shape == PostingChainShape::kStatlessDocsOnly) { + std::ranges::sort(postings.docids); + postings.docids.erase(std::unique(postings.docids.begin(), postings.docids.end()), + postings.docids.end()); + } else if (!t.sorted) { + RETURN_IF_ERROR(sort_by_docid(&postings.docids, &postings.freqs, &postings.positions_flat, + postings.retain_positions, mem_reporter_, + &tracked->docids_reservation, &tracked->freqs_reservation, + &tracked->positions_reservation)); + } + return Status::OK(); +} + +void SpimiTermBuffer::ensure_string_rank() const { + const std::vector& v = vocab(); + if (string_rank_.size() == v.size()) { + return; // already built for the current append-only vocabulary + } + // Build the complete rank required by the first spill and by k-way merge + // paths. Ordinary spills with a stale rank deliberately do not call here. + if (!common_gram_pair_keys_) { + std::vector order(v.size()); + std::iota(order.begin(), order.end(), 0U); + std::ranges::sort(order, [&](uint32_t a, uint32_t b) { return transient_term_less(a, b); }); + string_rank_.assign(v.size(), 0U); + for (uint32_t rank = 0; rank < order.size(); ++rank) { + string_rank_[order[rank]] = rank; + } + } else { + size_t pair_count = 0; + for (const std::string& term : v) { + pair_count += is_common_gram_pair_key(term); + } + + std::vector plain_order; + std::vector pair_order; + plain_order.reserve(v.size() - pair_count); + pair_order.reserve(pair_count); + for (uint32_t term_id = 0; term_id < v.size(); ++term_id) { + if (is_common_gram_pair_key(v[term_id])) { + pair_order.push_back(term_id); + } else { + plain_order.push_back(term_id); + } + } + + // EscapedV1 preserves logical byte order: 0x1e maps to 0x1eE and 0x1f + // maps to 0x1eG. One physical sort therefore supplies both the final plain + // order and the logical component rank used by a gram's right term. + std::ranges::sort(plain_order, + [&](uint32_t left, uint32_t right) { return v[left] < v[right]; }); + string_rank_.assign(v.size(), 0U); + for (uint32_t rank = 0; rank < plain_order.size(); ++rank) { + string_rank_[plain_order[rank]] = rank; + } + + // Decode each transient pair exactly once. The low word remains its term id; + // the high word temporarily carries the left plain id, while the pair's + // unused rank slot carries its right component's logical rank. + for (uint64_t& decorated_pair : pair_order) { + const uint32_t pair_term_id = static_cast(decorated_pair); + const CommonGramPairIds ids = decode_common_gram_pair_key_unchecked(v[pair_term_id]); + DCHECK_LT(ids.left.value, v.size()); + DCHECK_LT(ids.right.value, v.size()); + DCHECK(!is_common_gram_pair_key(v[ids.left.value])); + DCHECK(!is_common_gram_pair_key(v[ids.right.value])); + string_rank_[pair_term_id] = string_rank_[ids.right.value]; + decorated_pair = (static_cast(ids.left.value) << 32) | pair_term_id; + } + + // The physical gram key orders its left component by fixed-width encoded + // length, then by logical bytes. Stable per-length offsets convert the + // already-logically-sorted plain ids into that compound dense rank without + // another comparison sort. + std::vector next_length_rank( + segment_v2::inverted_index::COMMON_GRAM_MAX_ENCODED_BYTES + 1, 0U); + for (uint32_t plain_id : plain_order) { + const size_t logical_size = LogicalPlainKeyView(v[plain_id]).size(); + DCHECK_LT(logical_size, next_length_rank.size()); + ++next_length_rank[logical_size]; + } + uint32_t next_rank = 0; + for (uint32_t& length_count : next_length_rank) { + const uint32_t count = length_count; + length_count = next_rank; + next_rank += count; + } + DCHECK_EQ(next_rank, plain_order.size()); + for (uint32_t plain_id : plain_order) { + const size_t logical_size = LogicalPlainKeyView(v[plain_id]).size(); + string_rank_[plain_id] = next_length_rank[logical_size]++; + } + for (uint64_t& decorated_pair : pair_order) { + const uint32_t left_plain_id = static_cast(decorated_pair >> 32); + const uint32_t pair_term_id = static_cast(decorated_pair); + decorated_pair = + (static_cast(string_rank_[left_plain_id]) << 32) | pair_term_id; + } + + std::ranges::sort(pair_order, [&](uint64_t left, uint64_t right) { + const uint32_t left_component_rank = static_cast(left >> 32); + const uint32_t right_component_rank = static_cast(right >> 32); + if (left_component_rank != right_component_rank) { + return left_component_rank < right_component_rank; + } + return string_rank_[static_cast(left)] < + string_rank_[static_cast(right)]; + }); + + // No EscapedV1 plain key enters 0x1f, so every materialized gram forms one + // contiguous namespace group between the two physical-plain ranges. + const auto pair_position = std::lower_bound( + plain_order.begin(), plain_order.end(), segment_v2::inverted_index::CG_V1_MARKER, + [&](uint32_t plain_id, std::string_view marker) { return v[plain_id] < marker; }); + uint32_t final_rank = 0; + for (auto it = plain_order.begin(); it != pair_position; ++it) { + string_rank_[*it] = final_rank++; + } + for (uint64_t decorated_pair : pair_order) { + string_rank_[static_cast(decorated_pair)] = final_rank++; + } + for (auto it = pair_position; it != plain_order.end(); ++it) { + string_rank_[*it] = final_rank++; + } + DCHECK_EQ(final_rank, v.size()); + } +#ifdef BE_TEST + g_string_rank_rebuilds.fetch_add(1, std::memory_order_relaxed); +#endif +} + +std::vector SpimiTermBuffer::sorted_ids() const { + std::vector ids = touched_ids_; + const std::vector& v = vocab(); + if (string_rank_.empty()) { + // Preserve the fixed-vocabulary fast path: the first spill pays once for + // a complete rank, then every later spill is integer-only until vocab grows. + ensure_string_rank(); + } + if (string_rank_.size() == v.size()) { + order_ids_by_dense_rank(&ids, string_rank_); + } else { + // Vocabulary grew after the last complete rank. A run needs only its touched + // terms in lexical order; defer the O(vocab log vocab) rebuild until a k-way + // merge needs rank lookups for arbitrary ids. Reserve the same persistent + // rank capacity the old rebuild allocated so resident accounting and later + // spill-trigger timing remain unchanged. + string_rank_.reserve(v.size()); + std::ranges::sort(ids, [&](uint32_t a, uint32_t b) { return transient_term_less(a, b); }); + } + return ids; +} + +void SpimiTermBuffer::release_term(uint32_t term_id) { + const uint32_t enc = slot_of_[term_id]; + DCHECK_NE(enc, 0U); + const uint32_t slot = enc - 1; + slots_[slot] = Term(); // free this term's arrays; the empty Term slot is reusable + free_slots_.push_back(slot); + slot_of_[term_id] = 0; + --live_term_count_; +} + +Status SpimiTermBuffer::drain_sorted_streamed(const StreamedTermConsumer& fn) { + const std::vector& v = vocab(); + ensure_string_rank(); + report_arena_delta(); + order_ids_by_dense_rank(&touched_ids_, string_rank_); + intern_ = decltype(intern_)(0, OwnedVocabHash {.vocab = &owned_vocab_}, + OwnedVocabEq {&owned_vocab_}); + common_gram_pair_intern_ = decltype(common_gram_pair_intern_)(); + std::vector().swap(common_word_classification_); + std::vector().swap(string_rank_); + report_arena_delta(); + + constexpr size_t kSlotIndexPrefetchDistance = 32; + constexpr size_t kTermPrefetchDistance = 16; + Status callback_status = Status::OK(); + for (size_t ordinal = 0; ordinal < touched_ids_.size(); ++ordinal) { + if (ordinal + kSlotIndexPrefetchDistance < touched_ids_.size()) { + const uint32_t future_id = touched_ids_[ordinal + kSlotIndexPrefetchDistance]; + __builtin_prefetch(slot_of_.data() + future_id); + } + if (ordinal + kTermPrefetchDistance < touched_ids_.size()) { + const uint32_t future_id = touched_ids_[ordinal + kTermPrefetchDistance]; + const uint32_t future_enc = slot_of_[future_id]; + DCHECK_NE(future_enc, 0U); + __builtin_prefetch(slots_.data() + future_enc - 1); + __builtin_prefetch(v.data() + future_id); + } + const uint32_t id = touched_ids_[ordinal]; + const uint32_t enc = slot_of_[id]; + DCHECK_NE(enc, 0U); + Term term = slots_[enc - 1]; + slots_[enc - 1] = Term(); + slot_of_[id] = 0; + --live_term_count_; + + std::string output_term = materialize_transient_term(v[id]); + if (term.sorted) { + ArenaTermPostingSource source(&pool_, term); + StreamedTermPostings postings { + .term = std::move(output_term), + .retain_positions = term.shape == PostingChainShape::kTaggedPositioned, + .source = &source}; + callback_status = fn(std::move(postings)); + if (callback_status.ok() && !source.exhausted()) { + callback_status = Status::Error( + "spimi arena source: consumer returned before term exhaustion"); + } + } else { + TrackedTermPostings materialized(mem_reporter_); + callback_status = to_postings(std::move(output_term), std::move(term), &materialized); + if (callback_status.ok()) { + SpanTermPostingSource source(materialized.postings.docids, + materialized.postings.freqs, + materialized.postings.positions_flat); + StreamedTermPostings postings { + .term = std::move(materialized.postings.term), + .retain_positions = materialized.postings.retain_positions, + .source = &source}; + callback_status = fn(std::move(postings)); + if (callback_status.ok() && !source.exhausted()) { + callback_status = Status::Error( + "spimi span source: consumer returned before term exhaustion"); + } + } + } + if (!callback_status.ok()) { + break; + } + } + + pool_.reset(); + std::vector().swap(slots_); + std::vector().swap(free_slots_); + std::vector().swap(slot_of_); + std::vector().swap(touched_ids_); + live_term_count_ = 0; + std::vector().swap(owned_vocab_); + owned_vocab_heap_bytes_ = 0; + common_gram_pair_cache_.reset(); + common_gram_pair_cache_bytes_ = 0; + common_gram_plain_term_cache_.reset(); + common_gram_plain_term_cache_bytes_ = 0; + trim_malloc(); + report_arena_delta(); + return callback_status; +} + +Status SpimiTermBuffer::drain_to_writer(RunWriter* w) { + Status st = Status::OK(); + const std::vector& v = vocab(); + // Spill writes by term-id (no string IO). Iterate touched ids in vocab-string + // order so each run is sorted; the k-way merge re-orders runs by the same key. + for (uint32_t id : sorted_ids()) { + const uint32_t enc = slot_of_[id]; + DCHECK_NE(enc, 0U); + Term term = slots_[enc - 1]; + release_term(id); + if (st.ok()) { + TrackedTermPostings materialized(mem_reporter_); + st = to_postings(v[id], std::move(term), &materialized); + if (st.ok()) { + st = w->write_term(id, materialized.postings); + } + } + } + touched_ids_.clear(); + pool_.reset(); // all chains decoded into the run; free the arena for the refill + // The spill returns the arena to 0; slot_of_ keeps its capacity (survives + // the spill). Report the arena-drop negative now so the gate-2 spill is balanced + // immediately, not deferred to the next token. + report_arena_delta(); + return st; +} + +Status SpimiTermBuffer::compact_runs() { + if (run_paths_.size() < 2) { + return Status::OK(); + } + // The compaction heap can encounter any id held by an earlier run, so it + // requires a complete rank for the current vocabulary. New append-only ids + // can shift existing lexicographic ranks, hence the explicit refresh here. + ensure_string_rank(); + const std::string out_path = make_run_path(resolve_temp_dir()); + Status s = + writer::compact_runs(run_paths_, string_rank_, has_positions_, out_path, mem_reporter_); + if (!s.ok()) { + std::remove(out_path.c_str()); // drop the partial output; inputs intact + return s; + } + // The compacted run REPLACES its inputs at the FRONT of the run order: + // it holds exactly runs [0..n) merged in run order, and any later run only + // covers strictly-later docids, so per-term run-order concatenation (the + // k-way merge invariant) is preserved. + for (const std::string& p : run_paths_) { + std::remove(p.c_str()); + } + run_paths_.clear(); + run_paths_.push_back(out_path); + g_run_compactions.fetch_add(1, std::memory_order_relaxed); + return Status::OK(); +} + +Status SpimiTermBuffer::spill_to_run() { + // G09 run-file cap: a buffer must never accumulate unbounded run files -- + // the final k-way merge (re)opens ALL of them simultaneously and holds + // the fds for its whole duration, so unbounded runs across ~100 + // concurrent writers exhausted the BE nofile rlimit ('Too many open + // files' at run reopen). At the cap, merge-compact the existing runs into + // one before cutting the new run: the merge fan-in (and its fd count) is + // bounded by cap + 1 per buffer. + if (max_run_files_ != 0 && run_paths_.size() >= max_run_files_) { + RETURN_IF_ERROR(compact_runs()); + } + const std::string dir = resolve_temp_dir(); + // Best-effort space pre-check: fail with a clear, early error rather than a + // mid-write IoError that leaves a half-written run. Best-effort only (TOCTOU; on + // tmpfs this reports RAM). The ARENA -- not full resident_bytes(), which since + // G08 also charges vocabulary structures a run never contains -- is what the + // run re-encodes, and its block slack makes it a conservative over-estimate of + // the run's on-disk size. + const uint64_t arena = pool_.arena_bytes(); + const uint64_t avail = temp_dir_available_bytes(dir); + if (avail < arena) { + return Status::Error( + "spimi: insufficient temp space in '" + dir + "' to spill ~" + + std::to_string(arena) + " B (~" + std::to_string(avail) + + " B free); set SNII_TEMP_DIR/TMPDIR to a larger disk"); + } + const std::string path = make_run_path(dir); + RunWriter w(mem_reporter_); + RETURN_IF_ERROR(w.open(path)); + run_paths_.push_back(path); // tracked for cleanup even if a later step fails + RETURN_IF_ERROR(drain_to_writer(&w)); + // The drain emptied touched_ids_ and released every live slot while retaining + // capacity for the next fill. + return w.close(); +} + +Status SpimiTermBuffer::prepare_run_merge(TermKeyMaterializer* materializer) { + if (!touched_ids_.empty()) { + Status status = spill_to_run(); + if (!status.ok() && spill_status_.ok()) { + spill_status_ = status; + } + } + if (!spill_status_.ok()) { + return spill_status_; + } + + std::vector().swap(slots_); + std::vector().swap(free_slots_); + std::vector().swap(slot_of_); + std::vector().swap(touched_ids_); + common_gram_pair_cache_.reset(); + common_gram_pair_cache_bytes_ = 0; + common_gram_plain_term_cache_.reset(); + common_gram_plain_term_cache_bytes_ = 0; + common_gram_pair_intern_ = decltype(common_gram_pair_intern_)(); + std::vector().swap(common_word_classification_); + trim_malloc(); + report_arena_delta(); + + ensure_string_rank(); + report_arena_delta(); + intern_ = decltype(intern_)(0, OwnedVocabHash {.vocab = &owned_vocab_}, + OwnedVocabEq {&owned_vocab_}); + report_arena_delta(); + if (common_gram_pair_keys_) { + *materializer = [this](std::string_view term) { return materialize_transient_term(term); }; + } + return Status::OK(); +} + +void SpimiTermBuffer::finish_run_merge() { + std::vector().swap(owned_vocab_); + owned_vocab_heap_bytes_ = 0; + std::vector().swap(string_rank_); + report_arena_delta(); + trim_malloc(); +} + +Status SpimiTermBuffer::merge_runs_streamed(const StreamedTermConsumer& fn) { + TermKeyMaterializer materializer; + RETURN_IF_ERROR(prepare_run_merge(&materializer)); + Status status = merge_run_sources(run_paths_, vocab(), string_rank_, has_positions_, fn, + std::move(materializer), mem_reporter_); + finish_run_merge(); + return status; +} + +Status SpimiTermBuffer::for_each_term_sorted(const StreamedTermConsumer& fn) { + // Single-drain contract: a second call would re-merge the (still-present) run + // files and re-emit every term, or emit nothing in the in-memory path. Return + // an error and emit NOTHING rather than produce a wrong second stream. + if (drained_) { + return Status::Error( + "spimi: already drained (single-drain contract)"); + } + drained_ = true; + if (run_paths_.empty() && spill_status_.ok()) { + return drain_sorted_streamed(fn); + } + return merge_runs_streamed(fn); +} + +std::vector SpimiTermBuffer::finalize_sorted() { + std::vector out; + out.reserve(touched_ids_.size()); + Status status = for_each_term_sorted([&out](StreamedTermPostings&& streamed) { + TermPostings materialized; + materialized.term = std::move(streamed.term); + materialized.retain_positions = streamed.retain_positions; + TermPostingBuffer buffer(nullptr); + bool exhausted = false; + while (!exhausted) { + buffer.clear_reuse(); + RETURN_IF_ERROR( + streamed.source->fill(format::kAdaptiveWindowDocs, &buffer, &exhausted)); + materialized.docids.insert(materialized.docids.end(), buffer.docids().begin(), + buffer.docids().end()); + materialized.freqs.insert(materialized.freqs.end(), buffer.freqs().begin(), + buffer.freqs().end()); + materialized.positions_flat.insert(materialized.positions_flat.end(), + buffer.positions_flat().begin(), + buffer.positions_flat().end()); + } + out.push_back(std::move(materialized)); + return Status::OK(); + }); + if (!status.ok() && spill_status_.ok()) { + spill_status_ = status; + std::vector().swap(out); + } + return out; +} + +void SpimiTermBuffer::cleanup_runs() { + for (const std::string& p : run_paths_) { + std::remove(p.c_str()); + } + run_paths_.clear(); +} + +} // namespace doris::snii::writer diff --git a/be/src/storage/index/snii/writer/spimi_term_buffer.h b/be/src/storage/index/snii/writer/spimi_term_buffer.h new file mode 100644 index 00000000000000..b5c637e161b581 --- /dev/null +++ b/be/src/storage/index/snii/writer/spimi_term_buffer.h @@ -0,0 +1,730 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/writer/compact_posting_pool.h" +#include "storage/index/snii/writer/memory_reporter.h" +#include "storage/index/snii/writer/term_posting_source.h" + +namespace doris::segment_v2::inverted_index { +class CommonWordSet; +} + +namespace doris::snii::writer { + +using StreamedTermConsumer = std::function; + +// G11: compiled-in marker for the per-token prefetch candidate (the locality +// bench keys its in-process A/B test off this). +#define SNII_G11_PREFETCH 1 + +class GlobalMemoryLimiter; // G09 process-wide build-RAM registry (see below) + +struct PlainTermId { + uint32_t value = 0; +}; + +struct ClassifiedPlainTerm { + PlainTermId id; + bool is_common = false; +}; + +// One term's posting list: docids ascending, with parallel freqs and (when +// positions are enabled) a single FLAT positions buffer. +// +// positions_flat holds every position for the term in document order, partitioned +// by freqs: doc i owns the next freqs[i] entries. This is the SAME layout the +// accumulator stores natively, so no per-doc vector-of-vectors is ever built on +// the build/merge hot path (that vector-of-vectors was the dominant peak-RSS +// driver for high-df terms). doc_positions(i) returns a non-owning span view of +// doc i's positions for consumers that want per-doc access (e.g. the prx window +// builder, tests). positions_flat is empty when positions are disabled. +struct TermPostings { + std::string term; + std::vector docids; // absolute docids + std::vector freqs; + std::vector positions_flat; // empty when positions disabled + // Per-term posting shape. A positioned logical index may mix ordinary terms + // with docs-only accelerator terms; the latter carry one posting per doc and + // deliberately omit frequencies/positions from the final index. + bool retain_positions = true; + + size_t document_count() const { return docids.size(); } + + // Byte offset of doc i's first position within positions_flat (prefix sum of + // freqs). O(i) -- callers iterating all docs should track a running offset. + size_t pos_offset(size_t doc_index) const { + size_t off = 0; + for (size_t i = 0; i < doc_index; ++i) { + off += freqs[i]; + } + return off; + } + // Non-owning view of doc i's positions (length freqs[i]) into positions_flat. + std::span doc_positions(size_t doc_index) const { + const size_t off = pos_offset(doc_index); + return {positions_flat.data() + off, freqs[doc_index]}; + } + + // Rebuilds the per-doc position lists (for callers/tests wanting per-doc access) + // from positions_flat partitioned by freqs. O(total positions); allocates. + std::vector> positions_per_doc() const { + std::vector> out(freqs.size()); + size_t off = 0; + for (size_t i = 0; i < freqs.size(); ++i) { + out[i].assign(positions_flat.begin() + off, positions_flat.begin() + off + freqs[i]); + off += freqs[i]; + } + return out; + } + + // Sets the flat positions from per-doc lists (convenience for tests / callers + // that produce per-doc positions). Does NOT touch freqs; the caller is expected + // to keep freqs[i] == per_doc[i].size() consistent (the writer validates this). + void set_positions_per_doc(const std::vector>& per_doc) { + positions_flat.clear(); + for (const auto& d : per_doc) { + positions_flat.insert(positions_flat.end(), d.begin(), d.end()); + } + } +}; + +// In-memory SPIMI (Single-Pass In-Memory Indexing) accumulator for one logical +// index. Records term occurrences and produces lexicographically sorted terms +// with ascending-docid posting lists. +// +// TERM-ID ACCUMULATION (no per-token string work): tokens are accumulated by an +// INTEGER term-id, not by hashing/constructing a std::string per token. The +// caller supplies a VOCABULARY mapping term-id -> term string; the buffer keeps +// a DENSE std::vector indexed by term-id, so the hot add_token path is a +// vector index + a couple of pushes -- no hashing, no allocation per token. The +// vocabulary is resolved to strings only once per distinct term at finalize. +// +// Two construction modes: +// * BORROWED vocab (the fast path): pass a non-null `vocab` that the caller +// owns and keeps alive; add_token(term_id, ...) indexes straight into it. +// * OWNED vocab (compatibility): pass a null `vocab`; the string-keyed +// add_token(string_view, ...) interns each new term into an internal owned +// vocabulary (assigning ids in first-seen order) and forwards to the id +// path. Existing callers that feed strings keep working unchanged. +// +// SPILL / K-WAY MERGE (out-of-core, bounds input RAM): when a non-zero +// spill_threshold_bytes is set, the REAL resident accumulator size (see +// resident_bytes(): the posting arena PLUS every live vocab / slot / rank +// structure, G08) is compared against the threshold as tokens arrive. Once it +// crosses the threshold and enough reclaimable posting arena has accumulated, +// the buffer SORTS its current terms, +// writes a self-describing sorted RUN to a temp file, and CLEARS memory. Each +// run record is keyed by the TERM-ID (varint); the k-way merge orders runs by +// the id's VOCAB STRING so the merged stream stays lexicographic. Because +// tokens arrive in globally ascending docid order, a term that reappears in a +// later run only covers strictly-later docids, so concatenating its postings in +// run order during the final merge keeps docids ascending. for_each_term_sorted +// flushes the residual buffer as a final run, then k-way merges all runs +// materializing only ONE merged term at a time -> peak memory stays bounded by +// the threshold (plus the widest single term), NOT by total postings. With the +// default threshold 0 (unlimited) the path is exactly the in-memory behavior. +// +// Internal representation is a COMPACT TAGGED VARINT byte stream per term, held in +// a shared SEGMENTED ARENA (CompactPostingPool), NOT per-term uint32 vectors. Each +// term owns ONE arena chain holding a stream of per-TOKEN entries in arrival +// order: positioned and ordinary docs-only tokens contribute +// varint((pos << 1) | new_doc_bit); when new_doc_bit is set, a +// zigzag-varint(docid - prev_docid) immediately follows. Statless CommonGrams are +// deduplicated per document and store only that document delta, omitting the +// constant new_doc tag. Frequencies are otherwise recovered as the count of +// consecutive same-doc tokens. This drops +// the entire freq stream and the second (positions) chain versus a freq/prox split, +// so the payload is ~3.4x smaller than raw uint32 docids/freqs/positions, and the +// shared arena removes per-vector doubling slack and per-term vector headers. Each +// positioned and ordinary docs-only tokens append straight into the chain. +// Stateless CommonGrams keep a singleton doc id inline and backfill it only when a +// second document arrives. The other live per-term state is the current doc id (to +// detect a doc change) and the delta base. +// The production writer drains each chain through a bounded TermPostingSource. +// to_postings() remains only for explicit materialization in run maintenance and +// test/finalize helpers. positions_flat stays empty (and pos is tagged as 0) when +// positions are disabled; freq still counts. +// +// Duplicate vocab strings: the vocab is assumed to map each id to a DISTINCT +// string (a dense vocabulary). If two ids share a string they sort adjacently +// but are emitted as two separate terms; callers must not rely on coalescing. +class SpimiTermBuffer { +public: + // BORROWED-vocab constructor: `vocab` maps term-id -> term string and is + // borrowed (NOT owned) -- the caller must keep it alive for the buffer's + // lifetime. add_token(term_id, ...) accumulates by id with no string work. + // spill_threshold_bytes is the gate-2 internal buffer cap (e.g. 512 MiB), + // sourced from config; == 0 means unlimited (pure in-memory, default). A + // positive value is a soft spill threshold for the REAL resident accumulator + // size (resident_bytes(): arena + every live vocab/slot/rank structure, G08), + // triggering a spill once enough reclaimable arena has accumulated -- NOT a + // hard cap on persistent vocabulary memory or the old per-token estimate. + // `reporter` is the OPTIONAL writer-level build-RAM reporter (null off-Doris / + // unit tests). When non-null, the accumulator reports its REAL resident-byte + // deltas -- resident_bytes() diffs -- positive on grow, negative on every + // reset/free, exactly once. NEVER reports live_bytes_ (a gated estimate that + // feeds only the spill threshold). + explicit SpimiTermBuffer(const std::vector* vocab, bool has_positions, + size_t spill_threshold_bytes = 0, MemoryReporter* reporter = nullptr); + + // OWNED-vocab (compatibility) constructor: no external vocab. The string-keyed + // add_token interns terms into an internal vocabulary on first occurrence. + explicit SpimiTermBuffer(bool has_positions, size_t spill_threshold_bytes = 0, + MemoryReporter* reporter = nullptr); + + ~SpimiTermBuffer(); + + SpimiTermBuffer(const SpimiTermBuffer&) = delete; + SpimiTermBuffer& operator=(const SpimiTermBuffer&) = delete; + + // Records one token by TERM-ID: term `term_id` occurs in `docid` at `pos`. + // `term_id` must be in [0, vocab_size). An out-of-range id latches an + // InvalidArgument into status() and is ignored. For a given term, docids are + // expected to arrive in non-decreasing order, and positions within a docid in + // ascending order; out-of-order docids (INCLUDING a REVISITED docid -- the same + // docid appearing again after a different one) are tolerated and reordered at + // finalize: sort_by_docid stably sorts by docid and COALESCES same-docid groups + // (summing freqs, concatenating positions in document order), so the emitted + // postings have exactly ONE strictly-ascending entry per docid -- matching the + // k-way merge path and the writer's strictly-ascending precondition. + void add_token(uint32_t term_id, uint32_t docid, uint32_t pos); + void add_token(uint32_t term_id, uint32_t docid, uint32_t pos, bool retain_positions); + + // Compatibility overload: records one token by TERM STRING. Valid ONLY on an + // OWNED-vocab buffer before enable_common_gram_pair_keys(); interns `term` into + // the internal vocabulary on first occurrence, then forwards by id. Pair-key + // mode must use the typed plain/gram APIs below so a physical gram and its + // transient pair key cannot become two ids for the same logical term. Called on + // a BORROWED-vocab buffer it is REJECTED (latches InvalidArgument, token ignored) + // -- interning would grow the owned vocab out of step with the borrowed one and + // corrupt the build. Interning probes a heterogeneous (string_view-keyed) set, + // so a repeat token for an already-seen term allocates NOTHING; a std::string is + // materialized only on a term's FIRST occurrence (stored once in owned_vocab_). + // The id overload remains the hot path (no hashing at all); prefer that and + // reserve this for tests / legacy string-fed callers. + void add_token(std::string_view term, uint32_t docid, uint32_t pos); + void add_token(std::string_view term, uint32_t docid, uint32_t pos, bool retain_positions); + + // SNII CommonGrams fast path. Plain terms are interned once and returned as + // stable ids; each gram occurrence hashes a fixed 10-byte pair of those ids + // instead of constructing and hashing the variable-length physical gram key. + PlainTermId intern_plain_term(std::string_view physical_plain_term); + // Production CommonGrams path. A physical-key hit proves the injectively mapped + // logical term was validated previously; a miss validates exactly once before + // materializing the vocabulary entry. + PlainTermId intern_plain_term(std::string_view physical_plain_term, + std::string_view logical_plain_term); + ClassifiedPlainTerm intern_classified_plain_term( + std::string_view physical_plain_term, std::string_view logical_plain_term, + const segment_v2::inverted_index::CommonWordSet& common_words); + void add_plain_token(PlainTermId term_id, uint32_t docid, uint32_t pos); + void add_common_gram(PlainTermId left, PlainTermId right, uint32_t docid, uint32_t pos, + bool retain_positions); + void add_common_gram_and_plain(PlainTermId left, PlainTermId right, uint32_t docid, + uint32_t gram_pos, uint32_t plain_pos, + bool retain_gram_positions); + void enable_common_gram_pair_keys(); + + // G09: joins the PROCESS-WIDE build-RAM registry. Registers this buffer's + // current resident bytes with `limiter` and forwards every subsequent + // (debounced, see report_arena_delta) resident total to it; the destructor + // un-registers. When the registered sum across ALL of the process's live + // buffers exceeds the limiter's budget, the limiter may set this buffer's + // ADVISORY spill-request flag from ANOTHER thread; the flag is observed -- + // and the forced spill run ON THIS BUFFER'S OWN THREAD -- by the next + // add_token's maybe_spill_after_token (see there for the honor rule). + // Call at most once, right after construction (extra calls are ignored); + // `limiter` must outlive this buffer. Null / never attached = the G08 + // per-writer behavior, byte-identical. + void attach_global_limiter(GlobalMemoryLimiter* limiter); + + // TEST-ONLY: G09 advisory-flag observability -- read the pending flag, and + // plant a request directly (what the limiter does cross-thread) so the + // owner-honors-at-next-token contract is testable without a registry. + bool global_spill_requested_for_test() const { + return global_spill_requested_.load(std::memory_order_relaxed); + } + void request_global_spill_for_test() { + global_spill_requested_.store(true, std::memory_order_relaxed); + } + + // G09 forced-spill floor (config snii_forced_spill_min_arena_bytes): a + // pending process-wide forced-spill request is honored only once the + // reclaimable posting arena holds at least this much (never below one + // arena block, so a run is always writable). A request planted while the + // arena is below the floor is a NO-OP that stays PENDING -- it is NOT + // retried as a spill every token -- and is honored when the arena regrows + // past the floor. Without the floor, an unreachable global budget (the + // persistent vocabulary/slot structures of all writers alone exceeding it) + // re-flagged every buffer on every report and each honored with a single + // 32 KiB arena block: thousands of tiny runs per buffer, EMFILE at the + // k-way merge reopen, failed loads (the conc=16 wikipedia field storm). + static constexpr uint64_t kDefaultForcedSpillMinArenaBytes = 64ULL << 20; // 64 MiB + void set_forced_spill_min_arena_bytes(uint64_t bytes) { forced_spill_min_arena_bytes_ = bytes; } + uint64_t forced_spill_min_arena_bytes() const { return forced_spill_min_arena_bytes_; } + + // G09 run-file cap (config snii_spill_max_run_files_per_buffer): when a + // new spill would grow the accumulated run-file count past this cap, the + // existing runs are first MERGE-COMPACTED into one (a k-way merge of the + // run files back into a single fresh run; term stream byte-identical, the + // old files deleted) so the buffer never holds more than the cap + 1 run + // files. Bounds both the final k-way merge's fan-in and -- decisively -- + // its OPEN FILE DESCRIPTORS: every run of a buffer is (re)opened + // simultaneously and held open for the whole merge, so unbounded run + // counts across ~100 concurrent writers exhausted the BE nofile rlimit + // ('Too many open files' at run reopen). 0 disables the cap. + static constexpr size_t kDefaultMaxRunFilesPerBuffer = 64; + void set_max_run_files(size_t cap) { max_run_files_ = cap; } + size_t max_run_files() const { return max_run_files_; } + + // Number of DISTINCT terms accumulated so far (touched ids still resident). + size_t unique_terms() const; + uint64_t total_tokens() const { return total_tokens_; } + bool has_positions() const { return has_positions_; } + + // OK unless an add_token validation error (out-of-range term-id, wrong vocab + // mode) was latched. for_each_term_sorted now returns its own I/O Status + // directly; callers that use add_token's latch-and-report pattern MUST check + // this after draining to surface input-side validation errors. + [[nodiscard]] Status status() const { return spill_status_; } + + // TEST-ONLY: number of spill run files currently HELD (== 0 in pure + // in-memory mode). Lets tests assert that a gate-2 spill actually fired + // once the REAL resident size crossed the configured cap. NOTE: a G09 + // run-cap merge-compaction (see set_max_run_files) collapses the list to + // ONE file, so the count is not monotonic. Not part of the production API. + size_t run_count_for_test() const { return run_paths_.size(); } + + // TEST-ONLY: the REAL resident accumulator bytes the gate-2 trigger and the + // MemoryReporter see (resident_bytes()). Lets the G08 accounting tests assert + // coverage and monotonicity without widening access to the private + // accounting. Not part of + // the production API. + uint64_t resident_bytes_for_test() const { return resident_bytes(); } + size_t string_rank_capacity_for_test() const { return string_rank_.capacity(); } +#ifdef BE_TEST + static size_t hash_term_bytes_for_test(std::string_view term) { return hash_term_bytes(term); } + static size_t owned_term_key_size_for_test(); + void set_owned_term_hash_mask_for_test(size_t mask); +#endif + + // Materializes all terms sorted lexicographically; each term's docids are + // ascending. Convenience wrapper around for_each_term_sorted that keeps the + // whole result alive at once. Prefer for_each_term_sorted for low peak memory. + // The returned vectors are caller-owned compatibility output and are not + // charged to this buffer's internal MemoryReporter after the callback returns. + // MUST be called at most once: it drains internal state. A SECOND drain (a + // repeat call, or a finalize_sorted after a for_each_term_sorted, or vice versa) + // returns EMPTY and latches an error into status() rather than re-emitting. + std::vector finalize_sorted(); + + // Streams terms to `fn` in lexicographic order. Each source is borrowed for + // the synchronous callback and fills the writer-owned transfer buffer. The + // callback must exhaust the source before returning success. + // MUST be called at most once: it drains internal state. A SECOND drain invokes + // `fn` zero times and returns an Internal error (a re-merge of the still-present + // run files would otherwise re-emit every term). Returns non-OK on spill/merge + // I/O or corruption errors, or if a prior add_token latched a validation error + // into status(). + Status for_each_term_sorted(const StreamedTermConsumer& fn); + +private: + struct CommonGramPairCache; + struct CommonGramPlainTermCache; + + enum class PostingChainShape : uint8_t { + kTaggedPositioned, + kTaggedDocsOnly, + kStatlessDocsOnly, + }; + + // Compact per-term accumulator: ONE tagged-varint arena chain plus a few cursors. + // A statless CommonGram keeps its first distinct doc inline in cur_docid and + // starts a chain only when a second doc arrives. For other posting shapes, a + // sentinel chain head marks an empty term. ntok / ndocs bound the decode loop + // and size reserves. + // Total 28 B per live term. + static constexpr uint32_t kNoChain = 0xFFFFFFFFU; + struct Term { + uint32_t head = kNoChain; // chain read entry point + CompactPostingPool::SliceWriter w; // chain cursor (8 B) + uint32_t ntok = 0; // total tokens (entries) in the chain + uint32_t cur_docid = 0; // most-recent doc id: detects doc change AND + // is the zigzag delta base for the next doc + // Exact count of new-doc groups in the chain (one per new_doc tag). It + // bounds the decode reserves and equals the distinct-doc count while the + // input remains sorted; a later out-of-order coalesce can only shrink it. + uint32_t ndocs = 0; + PostingChainShape shape = PostingChainShape::kTaggedPositioned; + uint8_t level = 0; // current slice level of w (packed here, not in w) + bool started = false; // false until the first token is accumulated + bool sorted = true; // false if a docid arrived out of ascending order + }; + static_assert(sizeof(CompactPostingPool::SliceWriter) == 8, + "SliceWriter must stay 8 bytes to keep Term compact"); + static_assert(sizeof(Term) == 28, "Term must stay compact for high-cardinality imports"); + + struct TrackedTermPostings { + explicit TrackedTermPostings(MemoryReporter* reporter) + : docids_reservation(reporter == nullptr ? MemoryReporter::Reservation() + : reporter->make_reservation()), + freqs_reservation(reporter == nullptr ? MemoryReporter::Reservation() + : reporter->make_reservation()), + positions_reservation(reporter == nullptr ? MemoryReporter::Reservation() + : reporter->make_reservation()) {} + + TrackedTermPostings(const TrackedTermPostings&) = delete; + TrackedTermPostings& operator=(const TrackedTermPostings&) = delete; + + // Reservations precede the posting vectors so physical allocations are + // destroyed before their charges are released. + MemoryReporter::Reservation docids_reservation; + MemoryReporter::Reservation freqs_reservation; + MemoryReporter::Reservation positions_reservation; + TermPostings postings; + }; + + // The active vocabulary (term-id -> string): either the borrowed pointer or, + // in owned mode, &owned_vocab_. Always non-null after construction. + const std::vector& vocab() const { return *vocab_; } + + // Accumulates one already-validated token into the per-id Term and checks the + // spill gate once for that input token. + void accumulate(uint32_t term_id, uint32_t docid, uint32_t pos, bool retain_positions); + void accumulate_without_spill_gate(uint32_t term_id, uint32_t docid, uint32_t pos, + PostingChainShape shape); + void add_common_gram_without_spill_gate(PlainTermId left, PlainTermId right, uint32_t docid, + uint32_t pos, bool retain_positions); + + // Per-token gate-2 tail of accumulate(): reports the token's resident growth, + // then spills when the unified cap / local threshold fires with a worthwhile + // reclaimable arena (the G08 anti-churn floor), when the G09 process-wide + // limiter's advisory request flag is pending (honored here, on the owner's + // own thread; bypasses the G08 floor but requires one allocated arena block + // so a run is writable), or when the arena nears its hard 4 GiB offset + // limit. Every public add path invokes this gate once; the fused CommonGrams + // path invokes it after appending both the gram and its right plain token. + void maybe_spill_after_token(); + + Status to_postings(std::string term, Term&& t, TrackedTermPostings* tracked) const; + class ArenaTermPostingSource; + + // Returns the touched term-ids sorted by their vocab string (lexicographic). + // The first spill builds the full integer string-rank. Later spills reuse it + // while the append-only vocabulary is unchanged. If the vocabulary grew, an + // ordinary spill sorts only this run's touched ids by string; run compaction + // and final merge rebuild the full rank when they actually require it. + std::vector sorted_ids() const; + // Builds string_rank_ (term-id -> lexicographic rank) for the current complete + // vocabulary. Idempotent until the append-only vocabulary grows. + void ensure_string_rank() const; + Status drain_sorted_streamed(const StreamedTermConsumer& fn); + // Spills the current buffer to a fresh sorted run file and clears memory. + Status spill_to_run(); + // G09 run-file cap enforcement (see set_max_run_files): merge-compacts the + // current run files into ONE fresh run (same term stream, ids ordered by + // the current string rank), deletes the old + // files and replaces run_paths_ with the compacted one. Called by + // spill_to_run before opening a new run once the cap is reached. + Status compact_runs(); + // Writes all current terms (sorted) to an already-open RunWriter, draining. + Status drain_to_writer(class RunWriter* w); + // REAL resident accumulator bytes -- the single source of truth for the gate-2 + // spill trigger and every MemoryReporter delta. G08: sums EVERY live input-side + // structure -- the posting arena (docs+prx payload) + // plus the vocab-sized slot index, the Term slot pool + free/touched lists, the + // owned vocabulary (headers by capacity + string heap payloads via + // owned_vocab_heap_bytes_) and its intern set, plus the cached string ranks. + // Capacity, not size, throughout: the reserved tail is resident RSS and + // survives spills. + uint64_t resident_bytes() const; + // Reports the signed change in REAL resident bytes (resident_bytes()) to + // mem_reporter_ since the previous call, then caches the new total. + // Single-source diff: every grow/reset/free emits EXACTLY ONE delta + // (self-balancing -> impossible to double-count or miss a negative). No-op when + // mem_reporter_ is null. + void report_arena_delta(); + Status merge_runs_streamed(const StreamedTermConsumer& fn); + Status prepare_run_merge(std::function* materializer); + void finish_run_merge(); + // Deletes every temp run file; called from the destructor (RAII cleanup). + void cleanup_runs(); + // Frees a drained term's accumulator (id leaves the touched set). + void release_term(uint32_t term_id); + + // Stores a first-seen owned-vocabulary term under a stable id. + uint32_t append_owned_vocab_term(std::string&& term_str); + uint32_t intern_owned_term(std::string&& term_str, size_t term_hash); + uint32_t find_or_intern_owned_term(std::string_view term); + uint32_t find_or_intern_common_gram_pair(PlainTermId left, PlainTermId right, uint64_t pair); + uint32_t find_interned_plain_term(std::string_view term, size_t term_hash); + void remember_plain_term(size_t term_hash, uint32_t term_id); + bool transient_term_less(uint32_t left_id, uint32_t right_id) const; + std::string materialize_transient_term(std::string_view term) const; + + const std::vector* vocab_; // active vocab (borrowed or &owned_) + std::vector owned_vocab_; // owned mode: interned term strings + + enum class CommonWordClassification : uint8_t { + kUnknown, + kNotCommon, + kCommon, + }; + // Stable semantic classification keyed by owned term id. Pair ids remain + // kUnknown; physical plain ids are classified once from their logical bytes. + std::vector common_word_classification_; + + // G08: running sum of the owned vocab strings' HEAP payloads (0 for SSO + // strings -- their bytes live inside the headers owned_vocab_.capacity() + // already charges; capacity+1 for heap strings). Maintained incrementally by + // intern_owned_term so resident_bytes() stays O(1); terminal drains zero it + // when owned_vocab_ is released. + uint64_t owned_vocab_heap_bytes_ = 0; + + // G08: fixed per-entry estimate for one intern-set entry. Sized for the + // pre-G10 NODE-based set (16 B next-ptr+id node, its malloc chunk rounding, + // and an amortized bucket-array share) and deliberately UNCHANGED by the G10 + // swap to the flat set: resident_bytes() feeds the gate-2 spill trigger, so + // keeping the constant keeps the resident-byte sequence -- and therefore + // every spill point and the drained output -- bit-identical to the prior + // build. It still OVER-approximates the 4-byte flat key plus control bytes + // and load-factor slack, which can only fire the gate earlier, never overshoot. + // Deterministic so the accounting tests can reason about it, and ZERO for an + // empty set so an untouched (borrowed-mode) buffer charges nothing for it. + static constexpr uint64_t kInternEntryEstimateBytes = 48; + + // The table slot stores only the stable vocabulary id. String probes hash + // their bytes once; stored ids rehash through the sole owned vocabulary. + // Equality always resolves term identity from the complete bytes, so hash + // collisions are harmless and only add comparisons. + static size_t hash_term_bytes(std::string_view s) noexcept { + return std::hash {}(s); + } + + struct OwnedVocabHash { + using is_transparent = void; + const std::vector* vocab = nullptr; + size_t hash_mask = std::numeric_limits::max(); + size_t operator()(std::string_view term) const noexcept { + return hash_term_bytes(term) & hash_mask; + } + size_t operator()(uint32_t term_id) const noexcept { + return (*this)(std::string_view((*vocab)[term_id])); + } + }; + struct OwnedVocabEq { + using is_transparent = void; + const std::vector* vocab = nullptr; + bool operator()(uint32_t left, uint32_t right) const noexcept { return left == right; } + bool operator()(uint32_t stored, std::string_view probe) const noexcept; + bool operator()(std::string_view probe, uint32_t stored) const noexcept; + }; + // One flat table is the sole ordinary-term admission path. Heterogeneous + // probes avoid temporary strings; prepared insertion materializes only a miss. + // A failed table insertion rolls back the preceding vocabulary append, and no + // iterator survives a mutation. + phmap::flat_hash_set intern_; + + // CommonGram pair ids are already a canonical, collision-free key. Keep them + // out of the string-content intern table: an L0 cache miss probes this native + // map and materializes the 10-byte transient vocabulary key only for a new + // pair. The map survives ordinary spills because the persistent vocabulary + // and term ids do; terminal drains release it before output reservations. + phmap::flat_hash_map common_gram_pair_intern_; + + bool has_positions_; + bool common_gram_pair_keys_ = false; + std::unique_ptr common_gram_pair_cache_; + uint64_t common_gram_pair_cache_bytes_ = 0; + std::unique_ptr common_gram_plain_term_cache_; + uint64_t common_gram_plain_term_cache_bytes_ = 0; + size_t spill_threshold_bytes_; // 0 => unlimited (no spilling) + uint64_t total_tokens_ = 0; + + // POOLED accumulators (replaces a dense vocab-sized std::vector, which + // cost ~80 B per vocab id even for the ~empty majority -- the single largest + // input-phase memory line). slot_of_ is the only vocab-sized array: a 4 B index + // per id (0 == no live Term; otherwise slot index + 1). slots_ holds ONE Term + // per CURRENTLY-LIVE id, so its size tracks the live touched count, not the + // vocabulary. On first touch an id claims a slot (reusing a freed one from + // free_slots_ when available, else appending). release_term frees the slot back + // to the pool and clears slot_of_[id]. touched_ids_ lists every live id so + // finalize/spill iterate touched ids without scanning the whole vocabulary. + // present_[id] is now (slot_of_[id] != 0). The hot add path is still a vector + // index + a couple of pushes: no hashing, no per-token allocation. + std::vector slot_of_; // vocab-sized: id -> slot index + 1 (0=empty) + std::vector slots_; // live Term pool (size ~ live touched count) + std::vector free_slots_; // recycled slot indices (drained terms) + std::vector touched_ids_; + size_t live_term_count_ = 0; // present (non-drained) terms; == unique_terms() + + // Shared arena backing every live term's DOC and POS varint byte chains. Holds + // the bulk of the accumulator's memory in a few large blocks (no per-term vector + // headers, no per-vector doubling slack) -- the compact-RSS win. + CompactPostingPool pool_; + + // Optional writer-level build-RAM reporter (null off-Doris / unit tests) and the + // last resident-byte total it was told about. report_arena_delta() diffs the live + // total (arena_bytes() + slot_of_.capacity()*4) against reported_resident_. + MemoryReporter* mem_reporter_ = nullptr; + int64_t reported_resident_ = 0; + + // ---- G09 process-wide limiter hookup (null / false = feature off) -------- + // The registry this buffer joined via attach_global_limiter (borrowed; must + // outlive the buffer), and the ADVISORY forced-spill request flag the + // limiter sets from other threads (only ever under the registry mutex; the + // owner reads it relaxed on its own thread each token). The flag pointer + // doubles as the buffer's registry identity. + GlobalMemoryLimiter* global_limiter_ = nullptr; + std::atomic global_spill_requested_ {false}; + // G09 forced-spill floor / run-file cap (see the public setters above). + uint64_t forced_spill_min_arena_bytes_ = kDefaultForcedSpillMinArenaBytes; + size_t max_run_files_ = kDefaultMaxRunFilesPerBuffer; + + // Returns the live Term for `term_id`, claiming a pool slot on first touch. + Term& term_slot(uint32_t term_id, bool* new_term); + + // Appends one varint to a term's chain, lazily starting the chain on first use + // (so an untouched term costs no arena bytes). + void put_varint(Term* t, uint64_t v); + + std::vector run_paths_; // spilled run temp files (deleted in dtor) + Status spill_status_; // first spill / range error, at finalize + bool drained_ = false; // set once finalize_sorted/for_each_term_sorted has run; + // a second drain would (spilled path) re-merge the run + // files and re-emit every term, or (in-memory path) emit + // nothing -- both wrong. Guard against the double-drain. + + // Lazily-built vocab-sized map: term-id -> its lexicographic rank among all + // vocab strings. `size() == vocab().size()` means the rank is current; a + // smaller non-zero size is a stale rank retained after vocabulary growth. + // Its capacity is still advanced on stale ordinary spills so resident-byte + // accounting and spill-trigger timing keep the prior full-rank charge. + mutable std::vector string_rank_; +}; + +// TEST-ONLY observability seam (mirrors the reader-side decode-counter pattern). +// Counts how many times a vocabulary string is MATERIALIZED into owned_vocab_ during +// owned-mode interning. With single-store interning this is bumped EXACTLY ONCE per +// DISTINCT term (the owned_vocab_.emplace_back) and NEVER per token -- so feeding the +// same term M times still materializes it once, and the per-token temporary probe +// string is gone entirely. Writer tests use it for deterministic allocation +// assertions (count == distinct terms). Process-global; reset between tests. Not part +// of the production API. +namespace testing { +// G11 bench seam (honored under BE_TEST only): disables the add-path +// prefetch hints so the locality bench can A/B them within ONE process. +// Production builds prefetch unconditionally. +void set_bench_disable_g11_prefetch(bool disabled); + +uint64_t vocab_string_materialization_count(); +void reset_vocab_string_materialization_count(); + +// G09 process-wide limiter seam: spills that observed -- and cleared -- a +// PENDING global forced-spill request at the moment they fired (whether or not +// the per-writer gate would also have spilled that token; the request was +// consumed either way). Incremented under BE_TEST only because the check sits +// on the per-token path of every +// concurrent writer). Deterministic on the single-threaded build path; reset +// between tests. Not part of the production API. +uint64_t global_forced_spills(); +void reset_global_forced_spills(); + +// G09 run-file cap seam: merge-compactions of a buffer's accumulated spill +// runs (each collapses the whole run list into one file). Always-on relaxed +// atomic (a compaction is rare -- at most once per cap-many spills -- so +// contention is a non-issue, unlike the per-token seams above). Deterministic +// on the single-threaded build path; reset between tests. Not part of the +// production API. +uint64_t run_compactions(); +void reset_run_compactions(); + +// Number of complete-vocabulary lexicographic rank rebuilds. Ordinary spills +// with a stale rank must not increment it; run compaction and final merge may. +uint64_t string_rank_rebuilds(); +void reset_string_rank_rebuilds(); + +// Complete touched vocabularies invert the dense rank in O(N); partial runs +// retain comparison sorting. Both counters compile out of production. +uint64_t dense_rank_inversions(); +uint64_t rank_comparison_sorts(); +void reset_rank_ordering_counts(); + +// CommonGrams pair-key terminal-ordering seam. Both counters compile out of +// production: tests use them to prove terminal sorting/materialization takes the +// trusted fixed-key path instead of re-running generic key validation. +uint64_t common_gram_pair_unchecked_decode_count(); +uint64_t common_gram_trusted_plain_decode_count(); +void reset_common_gram_pair_fast_path_counts(); + +// CommonGrams pair direct-cache seam. The counters compile out of production; +// tests use them to prove repeated pairs bypass key encoding and the intern table. +// Docs-only pairs additionally suppress same-document repeats, while positioned +// pairs reuse the cached term id and still accumulate every position. +uint64_t common_gram_pair_cache_probes(); +uint64_t common_gram_pair_cache_pair_hits(); +uint64_t common_gram_pair_cache_same_doc_hits(); +void reset_common_gram_pair_cache_counts(); + +// Native CommonGram pair-interner seam. Normal production pair ingestion must +// never route a transient pair key through the generic string-content table. +uint64_t common_gram_native_pair_probes(); +uint64_t common_gram_native_pair_hits(); +uint64_t common_gram_native_pair_inserts(); +void reset_common_gram_native_pair_intern_counts(); + +uint64_t common_gram_logical_validation_count(); +void reset_common_gram_logical_validation_count(); + +// CommonGrams plain-term hot-cache seam: total cache probes, cache hits, and +// fallbacks that reached the global intern table. +uint64_t common_gram_plain_cache_probes(); +uint64_t common_gram_plain_cache_hits(); +uint64_t common_gram_plain_intern_table_probes(); +void reset_common_gram_plain_cache_counts(); + +// Counts equality checks that must dereference owned vocabulary bytes after the +// inline length and prefix checks. Short terms must never increment this counter. +uint64_t owned_term_full_byte_comparison_count(); +void reset_owned_term_full_byte_comparison_count(); +void fail_next_owned_term_reserve(); +void fail_next_owned_term_emplace(); + +uint64_t spill_gate_check_count(); +void reset_spill_gate_check_count(); + +// Counts compact-chain varint decodes during arena source consumption. Tests use +// this to prevent positioned sources from replaying the token chain. +uint64_t compact_chain_varint_decode_count(); +void reset_compact_chain_varint_decode_count(); +} // namespace testing + +} // namespace doris::snii::writer diff --git a/be/src/storage/index/snii/writer/temp_dir.cpp b/be/src/storage/index/snii/writer/temp_dir.cpp new file mode 100644 index 00000000000000..23da77c6a2ecff --- /dev/null +++ b/be/src/storage/index/snii/writer/temp_dir.cpp @@ -0,0 +1,41 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/writer/temp_dir.h" + +#include +#include +#include + +#include "runtime/exec_env.h" +#include "storage/index/index_writer.h" // segment_v2::TmpFileDirs (full definition) + +namespace doris::snii::writer { + +std::string resolve_temp_dir() { + // Use Doris's configured spill/scratch dirs (the same source the inverted-index + // writer uses; see index_file_writer.cpp). SNII spills/section temp files live in + // a dedicated "snii" subdirectory so they do not crowd the tmp root alongside + // every other component's files. + auto dir = ExecEnv::GetInstance()->get_tmp_file_dirs()->get_tmp_file_dir(); + dir /= "snii"; + std::error_code ec; + std::filesystem::create_directories(dir, ec); + return dir.native(); +} + +} // namespace doris::snii::writer diff --git a/be/src/storage/index/snii/writer/temp_dir.h b/be/src/storage/index/snii/writer/temp_dir.h new file mode 100644 index 00000000000000..1b249087cbd213 --- /dev/null +++ b/be/src/storage/index/snii/writer/temp_dir.h @@ -0,0 +1,47 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include +#include + +namespace doris::snii::writer { + +// Scratch directory for spill runs and section temp files. Uses Doris's configured +// tmp_file_dirs (ExecEnv::get_tmp_file_dirs) in production; when those are not +// initialized (unit tests / standalone) it falls back to SNII_TEMP_DIR -> TMPDIR -> +// /tmp. Defined in temp_dir.cpp to keep exec_env.h out of this header. +// +// The fallback (SNII_TEMP_DIR / TMPDIR) should point at a REAL disk (SSD/NVMe): +// /tmp is often tmpfs (RAM-backed), where spilling does NOT reduce RSS. +std::string resolve_temp_dir(); + +// Best-effort free bytes on the filesystem backing `dir`. Returns UINT64_MAX when +// statvfs fails, so a caller's space pre-check never false-positives on an +// unstattable path. CAVEATS: this is best-effort only -- it is subject to TOCTOU +// (free space can drop before/while the write runs), and on tmpfs it reports +// RAM-backed space (use the temp-dir config to avoid tmpfs in the first place). +inline uint64_t temp_dir_available_bytes(const std::string& dir) { + struct statvfs vfs; + if (::statvfs(dir.c_str(), &vfs) != 0) return UINT64_MAX; + return static_cast(vfs.f_bavail) * static_cast(vfs.f_frsize); +} + +} // namespace doris::snii::writer diff --git a/be/src/storage/index/snii/writer/term_posting_source.h b/be/src/storage/index/snii/writer/term_posting_source.h new file mode 100644 index 00000000000000..9de6b5405fee05 --- /dev/null +++ b/be/src/storage/index/snii/writer/term_posting_source.h @@ -0,0 +1,392 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/writer/memory_reporter.h" + +namespace doris::snii::writer { + +struct MutableTermPostingSpan { + std::span docids; + std::span freqs; + std::span positions_flat; +}; + +// Reusable, reservation-backed transfer storage for one source fill. A source +// may append multiple runs during one fill, but every run must agree on whether +// transient frequency statistics are present. clear_reuse() preserves capacity +// and its memory charge. +class TermPostingBuffer { +public: + explicit TermPostingBuffer(MemoryReporter* memory_reporter) + : memory_reporter_(memory_reporter), + capacity_reservation_(memory_reporter == nullptr + ? MemoryReporter::Reservation() + : memory_reporter->make_reservation()) {} + + TermPostingBuffer(const TermPostingBuffer&) = delete; + TermPostingBuffer& operator=(const TermPostingBuffer&) = delete; + TermPostingBuffer(TermPostingBuffer&&) = delete; + TermPostingBuffer& operator=(TermPostingBuffer&&) = delete; + + size_t document_count() const { return docids_.size(); } + bool empty() const { return docids_.empty(); } + + void clear_reuse() { + docids_.clear(); + freqs_.clear(); + positions_flat_.clear(); + has_freqs_.reset(); + } + + void clear_reuse_and_release_excess(size_t max_retained_capacity) { + clear_reuse(); + bool released = false; + if (docids_.capacity() > max_retained_capacity) { + std::vector().swap(docids_); + released = true; + } + if (freqs_.capacity() > max_retained_capacity) { + std::vector().swap(freqs_); + released = true; + } + if (positions_flat_.capacity() > max_retained_capacity) { + std::vector().swap(positions_flat_); + released = true; + } + if (released && memory_reporter_ != nullptr) { + uint64_t retained_bytes = 0; + DORIS_CHECK(capacity_bytes(docids_.capacity(), freqs_.capacity(), + positions_flat_.capacity(), &retained_bytes) + .ok()); + DORIS_CHECK(capacity_reservation_.set_bytes(retained_bytes).ok()); + } + } + + Status append(std::span docids, std::span freqs, + std::span positions_flat) { + const bool has_freqs = !freqs.empty(); + if (has_freqs && freqs.size() != docids.size()) { + return Status::Error( + "term posting buffer: freqs length must equal docids"); + } + if (!has_freqs && !positions_flat.empty()) { + return Status::Error( + "term posting buffer: positions require parallel freqs"); + } + uint64_t expected_positions = 0; + for (uint32_t freq : freqs) { + if (freq > std::numeric_limits::max() - expected_positions) { + return Status::Error( + "term posting buffer: position count overflow"); + } + expected_positions += freq; + } + if (!positions_flat.empty() && expected_positions != positions_flat.size()) { + return Status::Error( + "term posting buffer: positions count must equal sum(freqs)"); + } + if (has_freqs_.has_value() && *has_freqs_ != has_freqs) { + return Status::Error( + "term posting buffer: frequency shape changed within one fill"); + } + + MutableTermPostingSpan destination; + RETURN_IF_ERROR( + grow_uninitialized(docids.size(), has_freqs, positions_flat.size(), &destination)); + std::ranges::copy(docids, destination.docids.begin()); + std::ranges::copy(freqs, destination.freqs.begin()); + std::ranges::copy(positions_flat, destination.positions_flat.begin()); + return Status::OK(); + } + + // Extends the current fill once and exposes the new tail for direct decode. + // The caller must initialize every returned element before returning from + // TermPostingSource::fill. Shape and capacity changes are committed only + // after all reservations succeed. + Status grow_uninitialized(size_t document_count, bool has_freqs, size_t position_count, + MutableTermPostingSpan* destination) { + if (destination == nullptr) { + return Status::Error( + "term posting buffer: null writable span destination"); + } + if (!has_freqs && position_count != 0) { + return Status::Error( + "term posting buffer: positions require parallel freqs"); + } + if (has_freqs_.has_value() && *has_freqs_ != has_freqs) { + return Status::Error( + "term posting buffer: frequency shape changed within one fill"); + } + + const size_t doc_begin = docids_.size(); + const size_t freq_begin = freqs_.size(); + const size_t position_begin = positions_flat_.size(); + size_t target_docids = 0; + size_t target_freqs = 0; + size_t target_positions = 0; + RETURN_IF_ERROR(checked_size(doc_begin, document_count, &target_docids)); + RETURN_IF_ERROR(checked_size(freq_begin, has_freqs ? document_count : 0, &target_freqs)); + RETURN_IF_ERROR(checked_size(position_begin, position_count, &target_positions)); + RETURN_IF_ERROR(reserve_for_append(target_docids, target_freqs, target_positions)); + + docids_.resize(target_docids); + freqs_.resize(target_freqs); + positions_flat_.resize(target_positions); + destination->docids = std::span(docids_).subspan(doc_begin, document_count); + destination->freqs = std::span(freqs_).subspan(freq_begin, has_freqs ? document_count : 0); + destination->positions_flat = + std::span(positions_flat_).subspan(position_begin, position_count); + if (document_count != 0) { + has_freqs_ = has_freqs; + } + return Status::OK(); + } + + // Appends one position while a source decodes a frequency-bearing fill. + // The common path writes into retained capacity; growth keeps replacement + // reservation accounting atomic. + Status append_position(uint32_t position) { + if (!has_freqs_.value_or(false)) { + return Status::Error( + "term posting buffer: incremental positions require parallel freqs"); + } + if (positions_flat_.size() == positions_flat_.capacity()) { + size_t target_positions = 0; + RETURN_IF_ERROR(checked_size(positions_flat_.size(), 1, &target_positions)); + RETURN_IF_ERROR(reserve_for_append(docids_.size(), freqs_.size(), target_positions)); + } + positions_flat_.push_back(position); + return Status::OK(); + } + + std::span docids() const { return docids_; } + std::span freqs() const { return freqs_; } + std::span positions_flat() const { return positions_flat_; } + +private: + static Status checked_size(size_t current, size_t additional, size_t* target) { + if (additional > std::numeric_limits::max() - current) { + return Status::Error( + "term posting buffer: capacity overflow"); + } + *target = current + additional; + if (*target > std::numeric_limits::max() / sizeof(uint32_t)) { + return Status::Error( + "term posting buffer: byte capacity overflow"); + } + return Status::OK(); + } + + static Status growth_capacity(size_t required, size_t current, size_t* target) { + *target = required; + if (current != 0 && current <= std::numeric_limits::max() / 2) { + *target = std::max(required, current * 2); + } + if (*target > std::numeric_limits::max() / sizeof(uint32_t)) { + return Status::Error( + "term posting buffer: growth capacity overflow"); + } + return Status::OK(); + } + + static Status capacity_bytes(size_t docids_capacity, size_t freqs_capacity, + size_t positions_capacity, uint64_t* bytes) { + const uint64_t docids_bytes = docids_capacity * sizeof(uint32_t); + const uint64_t freqs_bytes = freqs_capacity * sizeof(uint32_t); + const uint64_t positions_bytes = positions_capacity * sizeof(uint32_t); + if (freqs_bytes > std::numeric_limits::max() - docids_bytes || + positions_bytes > std::numeric_limits::max() - docids_bytes - freqs_bytes) { + return Status::Error( + "term posting buffer: aggregate capacity overflow"); + } + *bytes = docids_bytes + freqs_bytes + positions_bytes; + return Status::OK(); + } + + Status reserve_for_append(size_t target_docids, size_t target_freqs, size_t target_positions) { + const bool grow_docids = target_docids > docids_.capacity(); + const bool grow_freqs = target_freqs > freqs_.capacity(); + const bool grow_positions = target_positions > positions_flat_.capacity(); + if (!grow_docids && !grow_freqs && !grow_positions) { + return Status::OK(); + } + size_t docids_capacity = docids_.capacity(); + size_t freqs_capacity = freqs_.capacity(); + size_t positions_capacity = positions_flat_.capacity(); + if (grow_docids) { + RETURN_IF_ERROR(growth_capacity(target_docids, docids_.capacity(), &docids_capacity)); + } + if (grow_freqs) { + RETURN_IF_ERROR(growth_capacity(target_freqs, freqs_.capacity(), &freqs_capacity)); + } + if (grow_positions) { + RETURN_IF_ERROR(growth_capacity(target_positions, positions_flat_.capacity(), + &positions_capacity)); + } + if (memory_reporter_ == nullptr) { + if (grow_docids) docids_.reserve(docids_capacity); + if (grow_freqs) freqs_.reserve(freqs_capacity); + if (grow_positions) positions_flat_.reserve(positions_capacity); + return Status::OK(); + } + + uint64_t previous_bytes = 0; + uint64_t final_bytes = 0; + RETURN_IF_ERROR(capacity_bytes(docids_.capacity(), freqs_.capacity(), + positions_flat_.capacity(), &previous_bytes)); + RETURN_IF_ERROR( + capacity_bytes(docids_capacity, freqs_capacity, positions_capacity, &final_bytes)); + RETURN_IF_ERROR(capacity_reservation_.set_bytes(final_bytes)); + + const uint64_t overlap_bytes = + std::max({grow_docids ? docids_.capacity() * sizeof(uint32_t) : 0, + grow_freqs ? freqs_.capacity() * sizeof(uint32_t) : 0, + grow_positions ? positions_flat_.capacity() * sizeof(uint32_t) : 0}); + MemoryReporter::Reservation overlap_reservation = memory_reporter_->make_reservation(); + Status overlap_status = overlap_reservation.set_bytes(overlap_bytes); + if (!overlap_status.ok()) { + DORIS_CHECK(capacity_reservation_.set_bytes(previous_bytes).ok()); + return overlap_status; + } + + if (grow_docids) { + docids_.reserve(docids_capacity); + DCHECK_EQ(docids_.capacity(), docids_capacity); + } + if (grow_freqs) { + freqs_.reserve(freqs_capacity); + DCHECK_EQ(freqs_.capacity(), freqs_capacity); + } + if (grow_positions) { + positions_flat_.reserve(positions_capacity); + DCHECK_EQ(positions_flat_.capacity(), positions_capacity); + } + return Status::OK(); + } + + MemoryReporter* memory_reporter_ = nullptr; + // The reservation precedes vectors so their allocations are destroyed first. + MemoryReporter::Reservation capacity_reservation_; + std::vector docids_; + std::vector freqs_; + std::vector positions_flat_; + std::optional has_freqs_; +}; + +class TermPostingSource { +public: + virtual ~TermPostingSource() = default; + + // out is empty on entry. Unless this call reaches the term end, it must + // return exactly target_docs postings. exhausted means no postings remain + // after this call. The source and output are borrowed synchronously. + virtual Status fill(uint32_t target_docs, TermPostingBuffer* out, bool* exhausted) = 0; +}; + +// Synchronous non-owning adapter for callers that already hold one materialized +// posting list. It slices the arrays into the writer's requested document +// windows without copying the whole term into an intermediate object. +class SpanTermPostingSource final : public TermPostingSource { +public: + SpanTermPostingSource(std::span docids, std::span freqs, + std::span positions_flat) + : docids_(docids), freqs_(freqs), positions_flat_(positions_flat) {} + + Status fill(uint32_t target_docs, TermPostingBuffer* out, bool* exhausted) override { + if (out == nullptr || exhausted == nullptr || target_docs == 0) { + return Status::Error( + "span posting source: invalid fill arguments"); + } + if (!out->empty()) { + return Status::Error( + "span posting source: output must be empty"); + } + if (!freqs_.empty() && freqs_.size() != docids_.size()) { + return Status::Error( + "span posting source: freqs length must equal docids"); + } + if (freqs_.empty() && !positions_flat_.empty()) { + return Status::Error( + "span posting source: positions require parallel freqs"); + } + + const size_t count = + std::min(static_cast(target_docs), docids_.size() - doc_offset_); + size_t position_count = 0; + if (!positions_flat_.empty()) { + for (size_t i = 0; i < count; ++i) { + RETURN_IF_ERROR( + checked_add(position_count, freqs_[doc_offset_ + i], &position_count)); + } + if (position_count > positions_flat_.size() - position_offset_) { + return Status::Error( + "span posting source: positions shorter than sum(freqs)"); + } + } + + RETURN_IF_ERROR(out->append( + docids_.subspan(doc_offset_, count), + freqs_.empty() ? std::span {} : freqs_.subspan(doc_offset_, count), + positions_flat_.subspan(position_offset_, position_count))); + doc_offset_ += count; + position_offset_ += position_count; + *exhausted = doc_offset_ == docids_.size(); + if (*exhausted && !positions_flat_.empty() && position_offset_ != positions_flat_.size()) { + return Status::Error( + "span posting source: positions longer than sum(freqs)"); + } + return Status::OK(); + } + + bool exhausted() const { return doc_offset_ == docids_.size(); } + +private: + static Status checked_add(size_t current, uint32_t additional, size_t* result) { + if (additional > std::numeric_limits::max() - current) { + return Status::Error( + "span posting source: position count overflow"); + } + *result = current + additional; + return Status::OK(); + } + + std::span docids_; + std::span freqs_; + std::span positions_flat_; + size_t doc_offset_ = 0; + size_t position_offset_ = 0; +}; + +struct StreamedTermPostings { + std::string term; + bool retain_positions = true; + TermPostingSource* source = nullptr; +}; + +} // namespace doris::snii::writer diff --git a/be/src/storage/olap_common.h b/be/src/storage/olap_common.h index 51f23f93cd1bd1..26706d3a5da525 100644 --- a/be/src/storage/olap_common.h +++ b/be/src/storage/olap_common.h @@ -40,6 +40,7 @@ #include "common/exception.h" #include "io/io_common.h" #include "storage/index/inverted/inverted_index_stats.h" +#include "storage/index/snii/snii_query_stats.h" #include "storage/olap_define.h" #include "storage/rowset/rowset_fwd.h" #include "util/hash_util.hpp" @@ -400,6 +401,8 @@ struct OlapReaderStatistics { int64_t inverted_index_query_timer = 0; int64_t inverted_index_query_cache_hit = 0; int64_t inverted_index_query_cache_miss = 0; + int64_t inverted_index_query_cache_lookup = 0; + int64_t inverted_index_query_cache_insert = 0; int64_t inverted_index_query_null_bitmap_timer = 0; int64_t inverted_index_query_bitmap_copy_timer = 0; int64_t inverted_index_searcher_open_timer = 0; @@ -411,6 +414,8 @@ struct OlapReaderStatistics { int64_t inverted_index_downgrade_count = 0; int64_t inverted_index_analyzer_timer = 0; int64_t inverted_index_lookup_timer = 0; + // See snii_query_stats.h: one field here instead of one per SNII counter. + snii::SniiQueryStats snii_stats; InvertedIndexStatistics inverted_index_stats; int64_t ann_index_load_ns = 0; diff --git a/be/src/storage/predicate_collector.cpp b/be/src/storage/predicate_collector.cpp deleted file mode 100644 index fa8fc0117ce34f..00000000000000 --- a/be/src/storage/predicate_collector.cpp +++ /dev/null @@ -1,314 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "storage/predicate_collector.h" - -#include - -#include - -#include "exec/common/variant_util.h" -#include "exprs/vexpr.h" -#include "exprs/vexpr_context.h" -#include "exprs/vliteral.h" -#include "exprs/vsearch.h" -#include "exprs/vslot_ref.h" -#include "gen_cpp/Exprs_types.h" -#include "storage/index/index_reader_helper.h" -#include "storage/index/inverted/analyzer/analyzer.h" -#include "storage/index/inverted/util/string_helper.h" -#include "storage/tablet/tablet_schema.h" - -namespace doris { - -using namespace segment_v2; - -VSlotRef* PredicateCollector::find_slot_ref(const VExprSPtr& expr) const { - if (!expr) { - return nullptr; - } - - auto cur = VExpr::expr_without_cast(expr); - if (cur->node_type() == TExprNodeType::SLOT_REF) { - return static_cast(cur.get()); - } - - for (const auto& ch : cur->children()) { - if (auto* s = find_slot_ref(ch)) { - return s; - } - } - - return nullptr; -} - -std::string PredicateCollector::build_field_name(int32_t col_unique_id, - const std::string& suffix_path) const { - std::string field_name = std::to_string(col_unique_id); - if (!suffix_path.empty()) { - field_name += "." + suffix_path; - } - return field_name; -} - -Status MatchPredicateCollector::collect(RuntimeState* state, const TabletSchemaSPtr& tablet_schema, - const VExprSPtr& expr, CollectInfoMap* collect_infos) { - DCHECK(collect_infos != nullptr); - - auto* left_slot_ref = find_slot_ref(expr->children()[0]); - if (left_slot_ref == nullptr) { - return Status::Error( - "Index statistics collection failed: Cannot find slot reference in match predicate " - "left expression"); - } - - auto* right_literal = static_cast(expr->children()[1].get()); - DCHECK(right_literal != nullptr); - - const auto* sd = state->desc_tbl().get_slot_descriptor(left_slot_ref->slot_id()); - if (sd == nullptr) { - return Status::Error( - "Index statistics collection failed: Cannot find slot descriptor for slot_id={}", - left_slot_ref->slot_id()); - } - - int32_t col_idx = tablet_schema->field_index(left_slot_ref->column_name()); - if (col_idx == -1) { - return Status::Error( - "Index statistics collection failed: Cannot find column index for column={}", - left_slot_ref->column_name()); - } - - const auto& column = tablet_schema->column(col_idx); - auto index_metas = tablet_schema->inverted_indexs(column); - std::vector> owned_index_metas; - std::string index_suffix_path = column.suffix_path(); - - // Schema-only fallback for variant sub-columns. Collector runs at tablet - // level without segment context, so we cannot do nested-group inference - // or inherit_index runtime-type dispatch. Two paths cover what is - // resolvable from schema alone: - // 1. field_pattern templates (MATCH_NAME / MATCH_NAME_GLOB) via - // generate_sub_column_info. - // 2. Plain parent inverted index when the schema column is the dynamic - // path's VARIANT placeholder produced by _init_variant_columns. In - // that state inverted_indexs(column) misses because - // _path_set_info_map.subcolumn_indexes is only populated for typed - // paths / field_pattern outputs, not for plain parent indexes added - // by ALTER. Clone the parent's non-field-pattern indexes with the - // variant path as suffix so segment-side BM25 statistics can be - // collected. - if (index_metas.empty() && column.is_extracted_column()) { - TabletSchema::SubColumnInfo sub_column_info; - const std::string relative_path = column.path_info_ptr()->copy_pop_front().get_path(); - if (variant_util::generate_sub_column_info(*tablet_schema, column.parent_unique_id(), - relative_path, &sub_column_info) && - !sub_column_info.indexes.empty()) { - index_suffix_path = sub_column_info.column.suffix_path(); - for (auto& idx : sub_column_info.indexes) { - index_metas.push_back(idx.get()); - owned_index_metas.emplace_back(std::move(idx)); - } - } else if (column.is_variant_type()) { - const auto parent_indexes = tablet_schema->inverted_indexs(column.parent_unique_id()); - for (const auto* index : parent_indexes) { - if (!index->field_pattern().empty()) { - continue; - } - auto index_ptr = std::make_shared(*index); - index_ptr->set_escaped_escaped_index_suffix_path( - column.path_info_ptr()->get_path()); - index_metas.push_back(index_ptr.get()); - owned_index_metas.emplace_back(std::move(index_ptr)); - } - } - } - -#ifndef BE_TEST - if (index_metas.empty()) { - return Status::Error( - "Index statistics collection failed: Score query is not supported without inverted " - "index for column={}", - left_slot_ref->column_name()); - } -#endif - - for (const auto* index_meta : index_metas) { - if (!InvertedIndexAnalyzer::should_analyzer(index_meta->properties())) { - continue; - } - - if (!IndexReaderHelper::is_need_similarity_score(expr->op(), index_meta)) { - continue; - } - - auto options = DataTypeSerDe::get_default_format_options(); - options.timezone = &state->timezone_obj(); - auto term_infos = InvertedIndexAnalyzer::get_analyse_result(right_literal->value(options), - index_meta->properties()); - - std::string field_name = - build_field_name(index_meta->col_unique_ids()[0], index_suffix_path); - std::wstring ws_field_name = StringHelper::to_wstring(field_name); - - auto iter = collect_infos->find(ws_field_name); - if (iter == collect_infos->end()) { - CollectInfo collect_info; - collect_info.term_infos.insert(term_infos.begin(), term_infos.end()); - collect_info.index_meta = index_meta; - for (const auto& owned_index_meta : owned_index_metas) { - if (owned_index_meta.get() == index_meta) { - collect_info.owned_index_meta = owned_index_meta; - break; - } - } - (*collect_infos)[ws_field_name] = std::move(collect_info); - } else { - iter->second.term_infos.insert(term_infos.begin(), term_infos.end()); - } - } - - return Status::OK(); -} - -Status SearchPredicateCollector::collect(RuntimeState* state, const TabletSchemaSPtr& tablet_schema, - const VExprSPtr& expr, CollectInfoMap* collect_infos) { - DCHECK(collect_infos != nullptr); - - auto* search_expr = dynamic_cast(expr.get()); - if (search_expr == nullptr) { - return Status::InternalError("SearchPredicateCollector: expr is not VSearchExpr type"); - } - - const TSearchParam& search_param = search_expr->get_search_param(); - - RETURN_IF_ERROR(collect_from_clause(search_param.root, state, tablet_schema, collect_infos)); - - return Status::OK(); -} - -Status SearchPredicateCollector::collect_from_clause(const TSearchClause& clause, - RuntimeState* state, - const TabletSchemaSPtr& tablet_schema, - CollectInfoMap* collect_infos) { - const std::string& clause_type = clause.clause_type; - ClauseTypeCategory category = get_clause_type_category(clause_type); - - if (category == ClauseTypeCategory::COMPOUND) { - if (clause.__isset.children) { - for (const auto& child_clause : clause.children) { - RETURN_IF_ERROR( - collect_from_clause(child_clause, state, tablet_schema, collect_infos)); - } - } - return Status::OK(); - } - - return collect_from_leaf(clause, state, tablet_schema, collect_infos); -} - -Status SearchPredicateCollector::collect_from_leaf(const TSearchClause& clause, RuntimeState* state, - const TabletSchemaSPtr& tablet_schema, - CollectInfoMap* collect_infos) { - if (!clause.__isset.field_name || !clause.__isset.value) { - return Status::InvalidArgument("Search clause missing field_name or value"); - } - - const std::string& field_name = clause.field_name; - const std::string& value = clause.value; - const std::string& clause_type = clause.clause_type; - - if (!is_score_query_type(clause_type)) { - return Status::OK(); - } - - int32_t col_idx = tablet_schema->field_index(field_name); - if (col_idx == -1) { - return Status::OK(); - } - - const auto& column = tablet_schema->column(col_idx); - - auto index_metas = tablet_schema->inverted_indexs(column.unique_id(), column.suffix_path()); - if (index_metas.empty()) { - return Status::OK(); - } - - ClauseTypeCategory category = get_clause_type_category(clause_type); - for (const auto* index_meta : index_metas) { - std::set term_infos; - - if (category == ClauseTypeCategory::TOKENIZED) { - if (InvertedIndexAnalyzer::should_analyzer(index_meta->properties())) { - auto analyzed_terms = - InvertedIndexAnalyzer::get_analyse_result(value, index_meta->properties()); - term_infos.insert(analyzed_terms.begin(), analyzed_terms.end()); - } else { - term_infos.insert(TermInfo(value)); - } - } else if (category == ClauseTypeCategory::NON_TOKENIZED) { - if (clause_type == "TERM" && - InvertedIndexAnalyzer::should_analyzer(index_meta->properties())) { - auto analyzed_terms = - InvertedIndexAnalyzer::get_analyse_result(value, index_meta->properties()); - term_infos.insert(analyzed_terms.begin(), analyzed_terms.end()); - } else { - term_infos.insert(TermInfo(value)); - } - } - - std::string lucene_field_name = - build_field_name(index_meta->col_unique_ids()[0], column.suffix_path()); - std::wstring ws_field_name = StringHelper::to_wstring(lucene_field_name); - - auto iter = collect_infos->find(ws_field_name); - if (iter == collect_infos->end()) { - CollectInfo collect_info; - collect_info.term_infos = std::move(term_infos); - collect_info.index_meta = index_meta; - (*collect_infos)[ws_field_name] = std::move(collect_info); - } else { - iter->second.term_infos.insert(term_infos.begin(), term_infos.end()); - } - } - - return Status::OK(); -} - -bool SearchPredicateCollector::is_score_query_type(const std::string& clause_type) const { - return clause_type == "TERM" || clause_type == "EXACT" || clause_type == "PHRASE" || - clause_type == "MATCH" || clause_type == "ANY" || clause_type == "ALL"; -} - -SearchPredicateCollector::ClauseTypeCategory SearchPredicateCollector::get_clause_type_category( - const std::string& clause_type) const { - if (clause_type == "AND" || clause_type == "OR" || clause_type == "NOT" || - clause_type == "OCCUR_BOOLEAN") { - return ClauseTypeCategory::COMPOUND; - } else if (clause_type == "TERM" || clause_type == "EXACT") { - return ClauseTypeCategory::NON_TOKENIZED; - } else if (clause_type == "PHRASE" || clause_type == "MATCH" || clause_type == "ANY" || - clause_type == "ALL") { - return ClauseTypeCategory::TOKENIZED; - } else { - LOG(WARNING) << "Unknown clause type '" << clause_type - << "', defaulting to NON_TOKENIZED category"; - return ClauseTypeCategory::NON_TOKENIZED; - } -} - -} // namespace doris diff --git a/be/src/storage/rowset/beta_rowset.cpp b/be/src/storage/rowset/beta_rowset.cpp index 4419fc3c0d470e..08e58945c6a38d 100644 --- a/be/src/storage/rowset/beta_rowset.cpp +++ b/be/src/storage/rowset/beta_rowset.cpp @@ -831,6 +831,9 @@ Status BetaRowset::show_nested_index_file(rapidjson::Value* rowset_value, case InvertedIndexStorageFormatPB::V3: format_str = "V3"; break; + case InvertedIndexStorageFormatPB::SNII: + format_str = "SNII"; + break; default: return Status::InternalError("inverted index storage format error"); break; @@ -840,6 +843,19 @@ Status BetaRowset::show_nested_index_file(rapidjson::Value* rowset_value, rowset_value->AddMember("index_storage_format", rapidjson::Value(format_str.c_str(), allocator), allocator); rapidjson::Value segments(rapidjson::kArrayType); + auto add_file_info_to_json = [&](const std::string& path, + rapidjson::Value& json_value) -> Status { + json_value.AddMember("idx_file_path", rapidjson::Value(path.c_str(), allocator), allocator); + int64_t idx_file_size = 0; + auto st = fs->file_size(path, &idx_file_size); + if (st != Status::OK()) { + LOG(WARNING) << "show nested index file get file size error, file: " << path + << ", error: " << st.msg(); + return st; + } + json_value.AddMember("idx_file_size", rapidjson::Value(idx_file_size).Move(), allocator); + return Status::OK(); + }; for (int seg_id = 0; seg_id < num_segments(); ++seg_id) { rapidjson::Value segment(rapidjson::kObjectType); segment.AddMember("segment_id", rapidjson::Value(seg_id).Move(), allocator); @@ -850,24 +866,20 @@ Status BetaRowset::show_nested_index_file(rapidjson::Value* rowset_value, fs, std::string(index_file_path_prefix), storage_format, InvertedIndexFileInfo(), _rowset_meta->tablet_id()); RETURN_IF_ERROR(index_file_reader->init()); + if (storage_format == InvertedIndexStorageFormatPB::SNII) { + rapidjson::Value index_file(rapidjson::kObjectType); + auto index_file_path = + InvertedIndexDescriptor::get_index_file_path_v2(index_file_path_prefix); + RETURN_IF_ERROR(add_file_info_to_json(index_file_path, index_file)); + segment.AddMember("index_files", rapidjson::Value(rapidjson::kArrayType).Move(), + allocator); + auto& index_files = segment["index_files"]; + index_files.PushBack(index_file, allocator); + segments.PushBack(segment, allocator); + continue; + } auto dirs = index_file_reader->get_all_directories(); - auto add_file_info_to_json = [&](const std::string& path, - rapidjson::Value& json_value) -> Status { - json_value.AddMember("idx_file_path", rapidjson::Value(path.c_str(), allocator), - allocator); - int64_t idx_file_size = 0; - auto st = fs->file_size(path, &idx_file_size); - if (st != Status::OK()) { - LOG(WARNING) << "show nested index file get file size error, file: " << path - << ", error: " << st.msg(); - return st; - } - json_value.AddMember("idx_file_size", rapidjson::Value(idx_file_size).Move(), - allocator); - return Status::OK(); - }; - auto process_files = [&allocator, &index_file_reader](auto& index_meta, rapidjson::Value& indices, rapidjson::Value& index) -> Status { diff --git a/be/src/storage/segment/column_reader.cpp b/be/src/storage/segment/column_reader.cpp index db8f2d6819b04d..b32ae470b08e7d 100644 --- a/be/src/storage/segment/column_reader.cpp +++ b/be/src/storage/segment/column_reader.cpp @@ -56,6 +56,7 @@ #include "storage/index/index_reader.h" #include "storage/index/inverted/analyzer/analyzer.h" #include "storage/index/inverted/inverted_index_reader.h" +#include "storage/index/snii/snii_index_reader.h" #include "storage/index/zone_map/zone_map_index.h" #include "storage/iterators.h" #include "storage/olap_common.h" @@ -722,6 +723,17 @@ Status ColumnReader::_load_index(const std::shared_ptr& index_f } IndexReaderPtr index_reader; + if (index_file_reader->get_storage_format() == InvertedIndexStorageFormatPB::SNII) { + if (!is_string_type(type)) { + return Status::Error( + "SNII inverted index storage format does not support BKD index type {}", type); + } + auto reader_type = should_analyzer ? InvertedIndexReaderType::FULLTEXT + : InvertedIndexReaderType::STRING_TYPE; + index_reader = SniiIndexReader::create_shared(index_meta, index_file_reader, reader_type); + _index_readers[index_meta->index_id()] = index_reader; + return Status::OK(); + } if (is_string_type(type)) { if (should_analyzer) { diff --git a/be/src/storage/segment/column_writer.cpp b/be/src/storage/segment/column_writer.cpp index c6af6dafb8fbe4..622153dd1f8396 100644 --- a/be/src/storage/segment/column_writer.cpp +++ b/be/src/storage/segment/column_writer.cpp @@ -565,6 +565,12 @@ Status ScalarColumnWriter::init() { RETURN_IF_ERROR(IndexColumnWriter::create( get_column(), &_inverted_index_builders[i], _opts.index_file_writer, _opts.inverted_indexes[i])); + // After create() (which runs the writer's init()) and before any + // value lands: forward the write type UNCONDITIONALLY so SNII + // can select its PRX zstd level -- and the + // documented "forwarded to every created IndexColumnWriter" + // contract holds for both values. + _inverted_index_builders[i]->set_direct_load(_opts.is_direct_load); } } while (false); } @@ -1056,6 +1062,9 @@ Status ArrayColumnWriter::init() { RETURN_IF_ERROR(IndexColumnWriter::create(get_column(), &_inverted_index_writer, _opts.index_file_writer, _opts.inverted_indexes[0])); + // Same unconditional forwarding as the scalar path: after + // create()/init(), before any array value is added. + _inverted_index_writer->set_direct_load(_opts.is_direct_load); } } if (_opts.need_ann_index) { diff --git a/be/src/storage/segment/column_writer.h b/be/src/storage/segment/column_writer.h index 0b8c827367abcb..b2f7a982088a5b 100644 --- a/be/src/storage/segment/column_writer.h +++ b/be/src/storage/segment/column_writer.h @@ -86,6 +86,13 @@ struct ColumnWriterOptions { BloomFilterOptions bf_options; std::vector inverted_indexes; IndexFileWriter* index_file_writer = nullptr; + // The owning segment serves a direct load (stream/broker load, + // DataWriteType::TYPE_DIRECT) rather than compaction / schema change. Set + // once by the segment writer and propagated to variant subcolumn writers; + // forwarded to every created IndexColumnWriter via set_direct_load() so + // SNII can select its direct-load PRX zstd level without plumbing + // DataWriteType itself down here. + bool is_direct_load = false; SegmentFooterPB* footer = nullptr; io::FileWriter* file_writer = nullptr; diff --git a/be/src/storage/segment/count_on_index_fastpath.h b/be/src/storage/segment/count_on_index_fastpath.h new file mode 100644 index 00000000000000..30b0ccf7218157 --- /dev/null +++ b/be/src/storage/segment/count_on_index_fastpath.h @@ -0,0 +1,196 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +// G02 count-only fast-path caller guard (functional core, unit-testable +// without a SegmentIterator). +// +// COUNT_ON_INDEX counts the rows of THIS segment that match the pushed-down +// predicates MINUS deleted rows: SegmentIterator seeds _row_bitmap with +// [0, num_rows), intersects the index result bitmap into it, then subtracts +// the MOW delete bitmap and applies delete predicates / row ranges, and the +// scan emits |_row_bitmap| default-valued rows that the agg counts. The SNII +// fast path replaces the index result with a FABRICATED [0, df) bitmap whose +// cardinality is exact but whose row ids are not real. That is only equal in +// observable behavior when, for this segment iterator: +// 1. only the COUNT matters (COUNT_ON_INDEX agg pushdown), +// 2. the single pushed-down MATCH predicate is the ONLY filter (nothing else +// may intersect _row_bitmap before or after the index apply), +// 3. no rows are deleted (mirror of the V3 handling: _lazy_init subtracts +// the per-segment delete bitmap AFTER the index apply, and delete +// predicates filter later -- a fabricated id range cannot participate in +// either subtraction), +// 4. nothing consumes REAL row ids (rowid recording, ANN topn, BM25 +// scoring, virtual columns), and +// 5. rows are emitted as defaults without reading column data at the +// fabricated ids (the COUNT_ON_INDEX no-read-data contract must be +// active: enable_no_need_read_data_opt + DUP_KEYS or MOW). +// +// SegmentIterator fills the facts from its state right before applying the +// index and only sets IndexQueryContext::count_on_index_fastpath when this +// predicate holds. +namespace doris::segment_v2 { + +struct CountOnIndexFastpathFacts { + // (1) count-only context. + bool is_count_on_index_agg = false; + // (2) single MATCH predicate, no other conjuncts. + bool has_column_predicates = false; + size_t common_expr_count = 0; + bool single_expr_is_match_pred = false; + bool has_virtual_column_exprs = false; + // (3) deletes must be absent for this segment. + bool has_delete_predicates = false; + bool segment_delete_bitmap_empty = false; + // (2 cont.) nothing else may prune the row space around the index apply. + bool has_col_id_predicates = false; + bool has_topn_filters = false; + bool has_external_row_ranges = false; + bool row_bitmap_is_full = false; + // (4) consumers of real row ids. + bool record_rowids = false; + bool has_ann_topn = false; + bool has_score_runtime = false; + // (5) rows must be emitted as defaults (no data read at fabricated ids). + bool no_need_read_data_opt_enabled = false; + bool keys_type_supported = false; +}; + +inline bool count_on_index_fastpath_safe(const CountOnIndexFastpathFacts& f) { + return f.is_count_on_index_agg && !f.has_column_predicates && f.common_expr_count == 1 && + f.single_expr_is_match_pred && !f.has_virtual_column_exprs && !f.has_delete_predicates && + f.segment_delete_bitmap_empty && !f.has_col_id_predicates && !f.has_topn_filters && + !f.has_external_row_ranges && f.row_bitmap_is_full && !f.record_rowids && + !f.has_ann_topn && !f.has_score_runtime && f.no_need_read_data_opt_enabled && + f.keys_type_supported; +} + +// G03 count-emission shortcut guard (functional core, unit-testable without a +// SegmentIterator). +// +// After the G02 fast path answered the single MATCH predicate with a +// count-shaped bitmap, the only remaining work of the scan is to emit +// |_row_bitmap| default-valued rows batch by batch: the per-batch rowid +// iteration over the fabricated bitmap and the per-column no-read checks are +// pure overhead. The shortcut replaces them with a countdown that fills the +// block columns with defaults directly, in VStatisticsIterator-sized batches. +// +// That replacement is byte-for-byte equal to today's emission only when, at +// the end of _lazy_init: +// 1. the reader ACTUALLY answered from df (count_fastpath_hit) -- a mere +// guard pass with a row-accurate decode keeps today's path untouched, +// 2. no evaluation stage survives (vec/short-circuit/expr eval, leftover +// column predicates or common exprs, delete predicates, lazy +// materialization), +// 3. nothing consumes real row ids or per-row values (virtual columns, +// rowid recording), +// 4. batch accounting is a pure countdown (no read limit, no reverse +// key-ordered read, no condition-cache writes), and +// 5. the block is exactly the read schema and EVERY column would take a +// defaults fill in _read_columns_by_index (no real column read, no +// storage->schema cast, no version/lsn/tso rewrite) -- checked +// per-column by the iterator and summarized in one fact. +// +// Every fact is re-verified from live iterator state even though the G02 +// facts guard already implies most of them: the shortcut independently +// refuses on any drift, falling through to today's emission (which is always +// count-exact). +struct CountEmitShortcutFacts { + // (1) reader answered with a fabricated count bitmap. + bool count_fastpath_hit = false; + // (2) nothing evaluates or filters rows after the index apply. + bool needs_vec_eval = false; + bool needs_short_eval = false; + bool needs_expr_eval = false; + bool has_remaining_col_predicates = false; + bool has_remaining_common_exprs = false; + bool has_delete_predicates = false; + bool lazy_materialization_read = false; + // (3) consumers of real row ids / per-row values. + bool has_virtual_columns = false; + bool record_rowids = false; + // (4) batch accounting must be a pure countdown. + bool has_read_limit = false; + bool read_orderby_key_reverse = false; + bool has_condition_cache_digest = false; + // (5) emitted block == read schema, all columns defaults-fillable. + bool block_shape_matches_schema = false; + bool all_columns_emit_defaults = false; +}; + +inline bool count_emit_shortcut_safe(const CountEmitShortcutFacts& f) { + return f.count_fastpath_hit && !f.needs_vec_eval && !f.needs_short_eval && !f.needs_expr_eval && + !f.has_remaining_col_predicates && !f.has_remaining_common_exprs && + !f.has_delete_predicates && !f.lazy_materialization_read && !f.has_virtual_columns && + !f.record_rowids && !f.has_read_limit && !f.read_orderby_key_reverse && + !f.has_condition_cache_digest && f.block_shape_matches_schema && + f.all_columns_emit_defaults; +} + +} // namespace doris::segment_v2 + +// Deterministic seam for the G03 count-emission shortcut, mirroring the SNII +// query seam (storage/index/snii/query/internal/query_test_counters.h): +// - count_emit_shortcut_hits : engage decisions that ADMITTED the +// shortcut (incremented inside +// _should_engage_count_emit_shortcut, which +// _lazy_init calls once per iterator). Guard +// fall-throughs leave it unchanged. +// - count_emit_shortcut_batches : default-rows batches emitted by the +// shortcut (== ceil(count / 65535) per +// engaged iterator with a non-zero count). +// +// Active only under BE_TEST (library-wide define of doris_be_test); in a +// release build the macro expands to ((void)0): zero overhead, no global +// mutable state on the production path. The singleton is intentionally +// unsynchronized -- single-threaded test-only seam; reset between cases with +// `count_emit_test_counters() = {}`. +#if defined(BE_TEST) && !defined(SNII_COUNT_EMIT_TEST_COUNTERS) +#define SNII_COUNT_EMIT_TEST_COUNTERS +#endif + +#ifdef SNII_COUNT_EMIT_TEST_COUNTERS + +namespace doris::segment_v2::internal { + +struct CountEmitTestCounters { + uint64_t count_emit_shortcut_hits = 0; + uint64_t count_emit_shortcut_batches = 0; +}; + +// `inline` gives a single shared instance across all TUs that include this +// header, so counter increments made in segment_iterator.cpp are visible to +// the test that reads them. +inline CountEmitTestCounters& count_emit_test_counters() { + static CountEmitTestCounters counters; + return counters; +} + +} // namespace doris::segment_v2::internal + +#define SNII_COUNT_EMIT_COUNT(field) \ + (++::doris::segment_v2::internal::count_emit_test_counters().field) + +#else + +#define SNII_COUNT_EMIT_COUNT(field) ((void)0) + +#endif diff --git a/be/src/storage/segment/row_ranges.h b/be/src/storage/segment/row_ranges.h index 0f44f599695b8c..9d98f02e245b15 100644 --- a/be/src/storage/segment/row_ranges.h +++ b/be/src/storage/segment/row_ranges.h @@ -247,7 +247,7 @@ class RowRanges { size_t count() { return _count; } - bool is_empty() { return _count == 0; } + bool is_empty() const { return _count == 0; } bool contain(rowid_t from, rowid_t to) { // binary search diff --git a/be/src/storage/segment/segment_iterator.cpp b/be/src/storage/segment/segment_iterator.cpp index 14a5fab34c582a..a31059030b6a5d 100644 --- a/be/src/storage/segment/segment_iterator.cpp +++ b/be/src/storage/segment/segment_iterator.cpp @@ -104,6 +104,7 @@ #include "storage/segment/column_reader.h" #include "storage/segment/column_reader_cache.h" #include "storage/segment/condition_cache.h" +#include "storage/segment/count_on_index_fastpath.h" #include "storage/segment/row_ranges.h" #include "storage/segment/segment.h" #include "storage/segment/segment_prefetcher.h" @@ -614,6 +615,16 @@ Status SegmentIterator::_lazy_init(Block* block) { _init_segment_prefetchers(); + // G03: engage the count-emission shortcut. All inputs are final here (the + // index apply ran, _row_bitmap saw every subtraction/intersection above, + // _vec_init_lazy_materialization fixed the eval flags), so the post-apply + // cardinality IS the exact row count today's batch loop would emit; the + // shortcut only changes how fast those default rows are produced. + _count_emit_shortcut = _should_engage_count_emit_shortcut(block); + if (_count_emit_shortcut) { + _count_emit_rows_remaining = _row_bitmap.cardinality(); + } + return Status::OK(); } @@ -825,6 +836,18 @@ Status SegmentIterator::_get_row_ranges_by_column_conditions() { (has_index_in_iterators() || !_common_expr_ctxs_push_down.empty())) { SCOPED_RAW_TIMER(&_opts.stats->inverted_index_filter_timer); size_t input_rows = _row_bitmap.cardinality(); + // G02 count-only fast path handshake: only while the single + // pushed-down MATCH predicate of a provably filter-free + // COUNT_ON_INDEX scan is evaluated may a reader answer with a + // count-shaped bitmap (see count_on_index_fastpath.h). The reply + // direction (did the reader actually fabricate one?) is captured + // into _count_fastpath_hit and both flags are reset on every exit + // path so no later read_from_index call can observe or forge them. + if (_index_query_context != nullptr) { + _index_query_context->count_on_index_fastpath = _count_on_index_fastpath_safe(); + _index_query_context->count_on_index_fastpath_hit = false; + } + DEFER({ _capture_count_fastpath_hit(); }); // Only apply column-level inverted index if we have iterators if (has_index_in_iterators()) { RETURN_IF_ERROR(_apply_inverted_index()); @@ -1337,6 +1360,146 @@ Status SegmentIterator::_apply_index_expr() { return Status::OK(); } +bool SegmentIterator::_count_on_index_fastpath_safe() const { + CountOnIndexFastpathFacts facts; + facts.is_count_on_index_agg = _opts.push_down_agg_type_opt == TPushAggOp::COUNT_ON_INDEX; + facts.has_column_predicates = !_col_predicates.empty(); + facts.common_expr_count = _common_expr_ctxs_push_down.size(); + facts.single_expr_is_match_pred = + _common_expr_ctxs_push_down.size() == 1 && + _common_expr_ctxs_push_down.front()->root() != nullptr && + _common_expr_ctxs_push_down.front()->root()->node_type() == TExprNodeType::MATCH_PRED; + facts.has_virtual_column_exprs = !_virtual_column_exprs.empty(); + facts.has_delete_predicates = _opts.delete_condition_predicates != nullptr && + _opts.delete_condition_predicates->num_of_column_predicate() > 0; + // Mirror of the _lazy_init delete-bitmap subtraction: the fast path is only + // sound when there is nothing to subtract for THIS segment. + const auto delete_bitmap_it = _opts.delete_bitmap.find(segment_id()); + facts.segment_delete_bitmap_empty = delete_bitmap_it == _opts.delete_bitmap.end() || + delete_bitmap_it->second == nullptr || + delete_bitmap_it->second->isEmpty(); + facts.has_col_id_predicates = !_opts.col_id_to_predicates.empty(); + facts.has_topn_filters = !_opts.topn_filter_source_node_ids.empty(); + facts.has_external_row_ranges = !_opts.row_ranges.is_empty(); + // Catches every earlier pruning source in one check (condition cache, key + // ranges): the fabricated [0, df) range only counts correctly against a + // full [0, num_rows) bitmap. + facts.row_bitmap_is_full = _row_bitmap.cardinality() == uint64_t(num_rows()); + facts.record_rowids = _opts.record_rowids; + facts.has_ann_topn = _opts.ann_topn_runtime != nullptr; + facts.has_score_runtime = _score_runtime != nullptr; + // Mirror of the _need_read_data preamble: rows must be emitted as defaults, + // never materialized from the fabricated row ids. + facts.no_need_read_data_opt_enabled = + _opts.runtime_state == nullptr || + _opts.runtime_state->query_options().enable_no_need_read_data_opt; + facts.keys_type_supported = _opts.tablet_schema->keys_type() == KeysType::DUP_KEYS || + (_opts.tablet_schema->keys_type() == KeysType::UNIQUE_KEYS && + _opts.enable_unique_key_merge_on_write); + return count_on_index_fastpath_safe(facts); +} + +void SegmentIterator::_capture_count_fastpath_hit() { + if (_index_query_context == nullptr) { + return; + } + _count_fastpath_hit = _index_query_context->count_on_index_fastpath_hit; + _index_query_context->count_on_index_fastpath = false; + _index_query_context->count_on_index_fastpath_hit = false; +} + +bool SegmentIterator::_column_emits_defaults_for_count(ColumnId cid) { + // Mirror of the per-batch fill in _read_columns_by_index: a column's batch + // content is reproducible by the emission shortcut iff the column takes + // the _no_need_read_key_data defaults fill or the _prune_column defaults + // fill. Anything else -- a real column read, a storage->schema cast in + // _init_current_block/_convert_to_expected_type (whose CAST(default) need + // not equal the schema-type default), or a version/lsn/tso rewrite source + // (those columns are never no-read, so the type/read checks below already + // veto them) -- must keep today's path. + if (_is_pred_column[cid]) { + return false; + } + const auto* column_desc = _schema->column(cid); + if (column_desc == nullptr) { + return false; + } + const auto& file_column_type = _storage_name_and_type[cid].second; + DataTypePtr expected_type = Schema::get_data_type_ptr(*column_desc); + if (file_column_type == nullptr || expected_type == nullptr || + !file_column_type->equals(*expected_type)) { + return false; + } + return _no_need_read_key_data_eligible(cid) || !_need_read_data(cid); +} + +bool SegmentIterator::_should_engage_count_emit_shortcut(const Block* block) { + if (!_count_fastpath_hit) { + return false; + } + CountEmitShortcutFacts facts; + facts.count_fastpath_hit = _count_fastpath_hit; + facts.needs_vec_eval = _is_need_vec_eval; + facts.needs_short_eval = _is_need_short_eval; + facts.needs_expr_eval = _is_need_expr_eval; + facts.has_remaining_col_predicates = !_col_predicates.empty(); + facts.has_remaining_common_exprs = !_common_expr_ctxs_push_down.empty(); + facts.has_delete_predicates = _opts.delete_condition_predicates != nullptr && + _opts.delete_condition_predicates->num_of_column_predicate() > 0; + facts.lazy_materialization_read = _lazy_materialization_read; + facts.has_virtual_columns = !_virtual_column_exprs.empty(); + facts.record_rowids = _opts.record_rowids || _record_rowids; + facts.has_read_limit = _opts.read_limit > 0; + facts.read_orderby_key_reverse = _opts.read_orderby_key_reverse; + facts.has_condition_cache_digest = _opts.condition_cache_digest != 0; + facts.block_shape_matches_schema = block->columns() == _schema->num_column_ids(); + facts.all_columns_emit_defaults = true; + for (size_t i = 0; i < _schema->num_column_ids() && facts.all_columns_emit_defaults; ++i) { + facts.all_columns_emit_defaults = _column_emits_defaults_for_count(_schema->column_id(i)); + } + const bool engage = count_emit_shortcut_safe(facts); + if (engage) { + SNII_COUNT_EMIT_COUNT(count_emit_shortcut_hits); + } + return engage; +} + +Status SegmentIterator::_emit_count_shortcut_batch(Block* block) { + if (_count_emit_rows_remaining == 0) { + // EOF twin of _process_eof: shortcut batches never move the block's + // columns into _current_return_columns, so there is nothing to + // restore; deliver the empty block and release iterator memory the + // same way. + block->clear_column_data(); + _column_iterators.clear(); + _index_iterators.clear(); + return Status::EndOfFile("no more data in segment"); + } + const auto rows = static_cast( + std::min(_count_emit_rows_remaining, kCountEmitBatchRows)); + block->clear_column_data(_schema->num_column_ids()); + for (size_t i = 0; i < _schema->num_column_ids(); ++i) { + MutableColumnPtr column = std::move(*block->get_by_position(i).column).mutate(); + // Same fill as the defaults branches of _read_columns_by_index + // (_no_need_read_key_data / _prune_column): NOT-NULL defaults. + // ColumnNullable::insert_many_defaults would insert NULLs and break + // count(col) parity. + if (is_column_nullable(*column)) { + auto* nullable_col_ptr = reinterpret_cast(column.get()); + nullable_col_ptr->get_null_map_column().insert_many_defaults(rows); + nullable_col_ptr->get_nested_column_ptr()->insert_many_defaults(rows); + } else { + column->insert_many_defaults(rows); + } + block->replace_by_position(i, std::move(column)); + } + _count_emit_rows_remaining -= rows; + _opts.stats->blocks_load += 1; + _opts.stats->raw_rows_read += rows; + SNII_COUNT_EMIT_COUNT(count_emit_shortcut_batches); + return Status::OK(); +} + bool SegmentIterator::_downgrade_without_index(Status res, bool need_remaining) { bool is_fallback = _opts.runtime_state->query_options().enable_fallback_on_missing_inverted_index; @@ -2889,6 +3052,14 @@ Status SegmentIterator::_next_batch_internal(Block* block) { SCOPED_RAW_TIMER(&_opts.stats->block_load_ns); + // G03: a count-fastpath scan emits the remaining count as default rows in + // statistics-sized batches; the row-bitmap iterator and the per-rowid + // machinery below are never touched. Engaged only when read_limit == 0, + // so the limit check below stays unreachable for shortcut scans. + if (_count_emit_shortcut) { + return _emit_count_shortcut_batch(block); + } + if (_opts.read_limit > 0 && _rows_returned >= _opts.read_limit) { return _process_eof(block); } @@ -3490,8 +3661,7 @@ void SegmentIterator::_calculate_common_expr_index_exec_status() { } } -bool SegmentIterator::_no_need_read_key_data(ColumnId cid, MutableColumnPtr& column, - size_t nrows_read) { +bool SegmentIterator::_no_need_read_key_data_eligible(ColumnId cid) { if (_opts.runtime_state && !_opts.runtime_state->query_options().enable_no_need_read_data_opt) { return false; } @@ -3518,6 +3688,14 @@ bool SegmentIterator::_no_need_read_key_data(ColumnId cid, MutableColumnPtr& col return false; } + return true; +} + +bool SegmentIterator::_no_need_read_key_data(ColumnId cid, MutableColumnPtr& column, + size_t nrows_read) { + if (!_no_need_read_key_data_eligible(cid)) { + return false; + } insert_many_not_null_defaults(column, nrows_read); return true; } diff --git a/be/src/storage/segment/segment_iterator.h b/be/src/storage/segment/segment_iterator.h index b67361d53ffa7a..61ed0afccfd428 100644 --- a/be/src/storage/segment/segment_iterator.h +++ b/be/src/storage/segment/segment_iterator.h @@ -198,6 +198,32 @@ class SegmentIterator : public RowwiseIterator { bool* continue_apply); [[nodiscard]] Status _apply_ann_topn_predicate(); [[nodiscard]] Status _apply_index_expr(); + // G02: true iff answering the single pushed-down MATCH predicate by its + // match COUNT alone is indistinguishable from the row-accurate bitmap for + // this COUNT_ON_INDEX scan (no deletes, no other filters, full row bitmap, + // no row-id consumers). Gates IndexQueryContext::count_on_index_fastpath; + // the decision predicate itself lives in count_on_index_fastpath.h. + bool _count_on_index_fastpath_safe() const; + // G03: teardown of the G02 handshake. Captures whether the reader answered + // with a fabricated count bitmap into _count_fastpath_hit and clears both + // context flags so no later read_from_index call can observe or forge + // them. Runs on every exit of the index-apply scope. + void _capture_count_fastpath_hit(); + // G03: true iff the per-batch defaults fill of _read_columns_by_index + // would apply to `cid` (the _no_need_read_key_data or _prune_column + // branch) AND the block column needs no storage->schema cast, i.e. the + // emission shortcut can reproduce the column's batch content exactly. + bool _column_emits_defaults_for_count(ColumnId cid); + // G03: fills CountEmitShortcutFacts from live iterator state at the end of + // _lazy_init and returns the pure-guard verdict; the decision predicate + // itself lives in count_on_index_fastpath.h. + bool _should_engage_count_emit_shortcut(const Block* block); + // G03: one emission-shortcut batch: min(remaining, kCountEmitBatchRows) + // default rows filled straight into the block (NOT-NULL defaults for + // nullable columns, mirroring _prune_column), then EOF once the countdown + // reaches zero. Replaces the whole per-rowid _next_batch_internal body for + // engaged scans. + Status _emit_count_shortcut_batch(Block* block); bool _column_has_fulltext_index(int32_t cid); bool _column_has_ann_index(int32_t cid); @@ -314,6 +340,10 @@ class SegmentIterator : public RowwiseIterator { Status _convert_to_expected_type(const std::vector& col_ids); bool _no_need_read_key_data(ColumnId cid, MutableColumnPtr& column, size_t nrows_read); + // Side-effect-free eligibility half of _no_need_read_key_data (no column + // fill); shared by the per-batch fill and the G03 engage-time per-column + // proof so the two can never drift. + bool _no_need_read_key_data_eligible(ColumnId cid); bool _has_delete_predicate(ColumnId cid); bool _can_skip_reading_extra_column(ColumnId cid); @@ -477,6 +507,23 @@ class SegmentIterator : public RowwiseIterator { IndexQueryContextPtr _index_query_context; + // G03 count-emission shortcut state (see count_on_index_fastpath.h). + // _count_fastpath_hit: the reader answered the single MATCH predicate with + // a fabricated count bitmap (captured from the G02 handshake reply). + // _count_emit_shortcut: engaged at the end of _lazy_init when + // count_emit_shortcut_safe holds; every subsequent batch is emitted by + // _emit_count_shortcut_batch from _count_emit_rows_remaining (initialized + // to the post-apply _row_bitmap cardinality) without touching the row + // bitmap iterator. + bool _count_fastpath_hit = false; + bool _count_emit_shortcut = false; + uint64_t _count_emit_rows_remaining = 0; + // Batch size for shortcut emission: VStatisticsIterator's + // MAX_ROW_SIZE_IN_COUNT, the largest default-rows block shape already + // proven through every consumer above the segment iterator by the plain + // COUNT pushdown (rowset reader, collect iterator, block reader, scanner). + static constexpr uint64_t kCountEmitBatchRows = 65535; + // key is column uid, value is the sparse column cache std::unordered_map _variant_sparse_column_cache; diff --git a/be/src/storage/segment/segment_writer.cpp b/be/src/storage/segment/segment_writer.cpp index 0f780986f1a119..58e1dec4c79387 100644 --- a/be/src/storage/segment/segment_writer.cpp +++ b/be/src/storage/segment/segment_writer.cpp @@ -193,6 +193,8 @@ Status SegmentWriter::_create_column_writer(uint32_t cid, const TabletColumn& co if (_opts.write_type == DataWriteType::TYPE_DIRECT && schema->skip_write_index_on_load()) { skip_inverted_index = true; } + // Let SNII select the direct-load PRX zstd level. + opts.is_direct_load = _opts.write_type == DataWriteType::TYPE_DIRECT; // indexes for this column if (!skip_inverted_index) { auto inverted_indexs = schema->inverted_indexs(column); diff --git a/be/src/storage/segment/variant/variant_column_writer_impl.cpp b/be/src/storage/segment/variant/variant_column_writer_impl.cpp index 408bd9b4cd427c..f0ff9bffee27e9 100644 --- a/be/src/storage/segment/variant/variant_column_writer_impl.cpp +++ b/be/src/storage/segment/variant/variant_column_writer_impl.cpp @@ -380,6 +380,8 @@ Status prepare_materialized_subcolumn_writer( opts.rowset_ctx = base_opts.rowset_ctx; opts.file_writer = base_opts.file_writer; opts.storage_format = base_opts.storage_format; + // keep the segment writer's direct-load marking for subcolumn index writers + opts.is_direct_load = base_opts.is_direct_load; std::unique_ptr writer; variant_util::inherit_column_attributes(parent_column, tablet_column); diff --git a/be/src/storage/segment/vertical_segment_writer.cpp b/be/src/storage/segment/vertical_segment_writer.cpp index 5676e6b248bebb..cb33ace7a9e9ea 100644 --- a/be/src/storage/segment/vertical_segment_writer.cpp +++ b/be/src/storage/segment/vertical_segment_writer.cpp @@ -204,6 +204,8 @@ Status VerticalSegmentWriter::_create_column_writer(uint32_t cid, const TabletCo tablet_schema->skip_write_index_on_load()) { skip_inverted_index = true; } + // Let SNII select the direct-load PRX zstd level. + opts.is_direct_load = _opts.write_type == DataWriteType::TYPE_DIRECT; if (!skip_inverted_index) { auto inverted_indexs = tablet_schema->inverted_indexs(column); if (!inverted_indexs.empty()) { diff --git a/be/src/storage/tablet/tablet_meta.cpp b/be/src/storage/tablet/tablet_meta.cpp index 4fd34f1122ee7a..dfde6238ff0693 100644 --- a/be/src/storage/tablet/tablet_meta.cpp +++ b/be/src/storage/tablet/tablet_meta.cpp @@ -467,6 +467,9 @@ void TabletMeta::init_schema_from_thrift(const TTabletSchema& tablet_schema, case TInvertedIndexFileStorageFormat::V3: tablet_schema_pb->set_inverted_index_storage_format(InvertedIndexStorageFormatPB::V3); break; + case TInvertedIndexFileStorageFormat::SNII: + tablet_schema_pb->set_inverted_index_storage_format(InvertedIndexStorageFormatPB::SNII); + break; default: tablet_schema_pb->set_inverted_index_storage_format(InvertedIndexStorageFormatPB::V3); break; diff --git a/be/src/storage/task/index_builder.cpp b/be/src/storage/task/index_builder.cpp index f85fcee9ddf44e..bb444e96af51e3 100644 --- a/be/src/storage/task/index_builder.cpp +++ b/be/src/storage/task/index_builder.cpp @@ -83,15 +83,20 @@ Status IndexBuilder::update_inverted_index_info() { TabletSchemaSPtr output_rs_tablet_schema = std::make_shared(); const auto& input_rs_tablet_schema = input_rowset->tablet_schema(); output_rs_tablet_schema->copy_from(*input_rs_tablet_schema); + const bool is_snii_drop = + _is_drop_op && input_rs_tablet_schema->get_inverted_index_storage_format() == + InvertedIndexStorageFormatPB::SNII; int64_t total_index_size = 0; - auto* beta_rowset = reinterpret_cast(input_rowset.get()); - auto size_st = beta_rowset->get_inverted_index_size(&total_index_size); - DBUG_EXECUTE_IF("IndexBuilder::update_inverted_index_info_size_st_not_ok", { - size_st = Status::Error("debug point: get fs failed"); - }) - if (!size_st.ok() && !size_st.is() && - !size_st.is()) { - return size_st; + if (!is_snii_drop) { + auto* beta_rowset = reinterpret_cast(input_rowset.get()); + auto size_st = beta_rowset->get_inverted_index_size(&total_index_size); + DBUG_EXECUTE_IF("IndexBuilder::update_inverted_index_info_size_st_not_ok", { + size_st = Status::Error("debug point: get fs failed"); + }) + if (!size_st.ok() && !size_st.is() && + !size_st.is()) { + return size_st; + } } auto num_segments = input_rowset->num_segments(); size_t drop_index_size = 0; @@ -247,10 +252,22 @@ Status IndexBuilder::update_inverted_index_info() { without_index_uids.insert(_alter_index_ids.begin(), _alter_index_ids.end()); } + const bool preserve_snii_container = + is_snii_drop && (output_rs_tablet_schema->has_inverted_index() || + output_rs_tablet_schema->has_ann_index()); + std::set* excluded_index_ids = + without_index_uids.empty() ? &_alter_index_ids : &without_index_uids; + if (preserve_snii_container) { + // SNII logical indexes share one immutable container. Preserve that container in O(1) + // while the output schema hides the dropped logical index; compaction reclaims its + // physical bytes later. When no logical index survives, keep the exclusion set so no + // container is linked. + excluded_index_ids = nullptr; + } + // build output rowset RETURN_IF_ERROR(input_rowset->link_files_to( - _tablet->tablet_path(), output_rs_writer->rowset_id(), 0, - without_index_uids.empty() ? &_alter_index_ids : &without_index_uids)); + _tablet->tablet_path(), output_rs_writer->rowset_id(), 0, excluded_index_ids)); auto input_rowset_meta = input_rowset->rowset_meta(); RowsetMetaSharedPtr rowset_meta = std::make_shared(); @@ -272,6 +289,13 @@ Status IndexBuilder::update_inverted_index_info() { rowset_meta->set_data_disk_size(input_rowset_meta->data_disk_size()); rowset_meta->set_index_disk_size(input_rowset_meta->index_disk_size()); } + } else if (is_snii_drop) { + rowset_meta->set_total_disk_size(preserve_snii_container + ? input_rowset_meta->total_disk_size() + : input_rowset_meta->data_disk_size()); + rowset_meta->set_data_disk_size(input_rowset_meta->data_disk_size()); + rowset_meta->set_index_disk_size( + preserve_snii_container ? input_rowset_meta->index_disk_size() : 0); } else { for (int seg_id = 0; seg_id < num_segments; seg_id++) { auto seg_path = DORIS_TRY(input_rowset->segment_path(seg_id)); @@ -339,6 +363,13 @@ Status IndexBuilder::handle_single_rowset(RowsetMetaSharedPtr output_rowset_meta if (_is_drop_op) { const auto& output_rs_tablet_schema = output_rowset_meta->tablet_schema(); + if (output_rs_tablet_schema->get_inverted_index_storage_format() == + InvertedIndexStorageFormatPB::SNII) { + LOG(INFO) << "skip physical SNII inverted index rewrite for drop index. tablet_id=" + << _tablet->tablet_id() + << " rowset_id=" << output_rowset_meta->rowset_id().to_string(); + return Status::OK(); + } if (output_rs_tablet_schema->get_inverted_index_storage_format() != InvertedIndexStorageFormatPB::V1) { const auto& fs = output_rowset_meta->fs(); @@ -422,6 +453,11 @@ Status IndexBuilder::handle_single_rowset(RowsetMetaSharedPtr output_rowset_meta _olap_data_convertor->reserve(_alter_inverted_indexes.size()); std::unique_ptr index_file_writer = nullptr; + if (output_rowset_schema->get_inverted_index_storage_format() == + InvertedIndexStorageFormatPB::SNII) { + return Status::Error( + "BUILD INDEX is not supported for SNII inverted index storage format yet"); + } if (output_rowset_schema->get_inverted_index_storage_format() >= InvertedIndexStorageFormatPB::V2) { auto idx_file_reader_iter = _index_file_readers.find( @@ -517,8 +553,10 @@ Status IndexBuilder::handle_single_rowset(RowsetMetaSharedPtr output_rowset_meta if (inverted_index_builder) { auto writer_sign = std::make_pair(seg_ptr->id(), index_id); - _index_column_writers.insert( + auto [index_column_writer_it, inserted] = _index_column_writers.insert( std::make_pair(writer_sign, std::move(inverted_index_builder))); + DORIS_CHECK(inserted); + DORIS_CHECK(index_column_writer_it->second != nullptr); inverted_index_writer_signs.emplace_back(writer_sign); } } @@ -546,8 +584,10 @@ Status IndexBuilder::handle_single_rowset(RowsetMetaSharedPtr output_rowset_meta if (index_writer) { auto writer_sign = std::make_pair(seg_ptr->id(), index_id); - _index_column_writers.insert( + auto [index_column_writer_it, inserted] = _index_column_writers.insert( std::make_pair(writer_sign, std::move(index_writer))); + DORIS_CHECK(inserted); + DORIS_CHECK(index_column_writer_it->second != nullptr); inverted_index_writer_signs.emplace_back(writer_sign); } } @@ -555,7 +595,10 @@ Status IndexBuilder::handle_single_rowset(RowsetMetaSharedPtr output_rowset_meta } // DO NOT forget index_file_writer for the segment, otherwise, original inverted index will be deleted. - _index_file_writers.emplace(seg_ptr->id(), std::move(index_file_writer)); + auto [index_file_writer_it, inserted] = + _index_file_writers.emplace(seg_ptr->id(), std::move(index_file_writer)); + DORIS_CHECK(inserted); + DORIS_CHECK(index_file_writer_it->second != nullptr); if (return_columns.empty()) { // no columns to read continue; @@ -606,8 +649,7 @@ Status IndexBuilder::handle_single_rowset(RowsetMetaSharedPtr output_rowset_meta "handle_single_rowset_write_inverted_index_data_error"); }) if (!status.ok()) { - return Status::Error( - "failed to write block."); + return status; } block->clear_column_data(); } @@ -615,9 +657,10 @@ Status IndexBuilder::handle_single_rowset(RowsetMetaSharedPtr output_rowset_meta // finish write inverted index, flush data to compound file for (auto& writer_sign : inverted_index_writer_signs) { try { - if (_index_column_writers[writer_sign]) { - RETURN_IF_ERROR(_index_column_writers[writer_sign]->finish()); - } + auto index_column_writer_it = _index_column_writers.find(writer_sign); + DORIS_CHECK(index_column_writer_it != _index_column_writers.end()); + DORIS_CHECK(index_column_writer_it->second != nullptr); + RETURN_IF_ERROR(index_column_writer_it->second->finish()); DBUG_EXECUTE_IF("IndexBuilder::handle_single_rowset_index_build_finish_error", { _CLTHROWA(CL_ERR_IO, "debug point: handle_single_rowset_index_build_finish_error"); @@ -715,6 +758,11 @@ Status IndexBuilder::_add_nullable(const std::string& column_name, const std::pair& index_writer_sign, const TabletColumn* column, const uint8_t* null_map, const uint8_t** ptr, size_t num_rows) { + auto index_column_writer_it = _index_column_writers.find(index_writer_sign); + DORIS_CHECK(index_column_writer_it != _index_column_writers.end()); + DORIS_CHECK(index_column_writer_it->second != nullptr); + auto* index_column_writer = index_column_writer_it->second.get(); + // TODO: need to process null data for inverted index if (column->type() == FieldType::OLAP_FIELD_TYPE_ARRAY) { DCHECK(column->get_subtype_count() == 1); @@ -726,15 +774,14 @@ Status IndexBuilder::_add_nullable(const std::string& column_name, try { auto data = *(data_ptr + 2); auto nested_null_map = *(data_ptr + 3); - RETURN_IF_ERROR(_index_column_writers[index_writer_sign]->add_array_values( + RETURN_IF_ERROR(index_column_writer->add_array_values( field_type_size(column->get_sub_column(0).type()), reinterpret_cast(data), reinterpret_cast(nested_null_map), offsets_ptr, num_rows)); DBUG_EXECUTE_IF("IndexBuilder::_add_nullable_add_array_values_error", { _CLTHROWA(CL_ERR_IO, "debug point: _add_nullable_add_array_values_error"); }) - RETURN_IF_ERROR( - _index_column_writers[index_writer_sign]->add_array_nulls(null_map, num_rows)); + RETURN_IF_ERROR(index_column_writer->add_array_nulls(null_map, num_rows)); } catch (const std::exception& e) { return Status::Error( "CLuceneError occurred: {}", e.what()); @@ -758,11 +805,9 @@ Status IndexBuilder::_add_nullable(const std::string& column_name, do { auto step = next_run_step(); if (null_map[offset]) { - RETURN_IF_ERROR(_index_column_writers[index_writer_sign]->add_nulls( - static_cast(step))); + RETURN_IF_ERROR(index_column_writer->add_nulls(static_cast(step))); } else { - RETURN_IF_ERROR(_index_column_writers[index_writer_sign]->add_values(column_name, - *ptr, step)); + RETURN_IF_ERROR(index_column_writer->add_values(column_name, *ptr, step)); } *ptr += field_type_size(column->type()) * step; offset += step; @@ -780,6 +825,11 @@ Status IndexBuilder::_add_nullable(const std::string& column_name, Status IndexBuilder::_add_data(const std::string& column_name, const std::pair& index_writer_sign, const TabletColumn* column, const uint8_t** ptr, size_t num_rows) { + auto index_column_writer_it = _index_column_writers.find(index_writer_sign); + DORIS_CHECK(index_column_writer_it != _index_column_writers.end()); + DORIS_CHECK(index_column_writer_it->second != nullptr); + auto* index_column_writer = index_column_writer_it->second.get(); + try { if (column->type() == FieldType::OLAP_FIELD_TYPE_ARRAY) { DCHECK(column->get_subtype_count() == 1); @@ -792,14 +842,13 @@ Status IndexBuilder::_add_data(const std::string& column_name, if (element_cnt > 0) { auto data = *(data_ptr + 2); auto nested_null_map = *(data_ptr + 3); - RETURN_IF_ERROR(_index_column_writers[index_writer_sign]->add_array_values( + RETURN_IF_ERROR(index_column_writer->add_array_values( field_type_size(column->get_sub_column(0).type()), reinterpret_cast(data), reinterpret_cast(nested_null_map), offsets_ptr, num_rows)); } } else { - RETURN_IF_ERROR(_index_column_writers[index_writer_sign]->add_values(column_name, *ptr, - num_rows)); + RETURN_IF_ERROR(index_column_writer->add_values(column_name, *ptr, num_rows)); } DBUG_EXECUTE_IF("IndexBuilder::_add_data_throw_exception", { _CLTHROWA(CL_ERR_IO, "debug point: _add_data_throw_exception"); }) diff --git a/be/test/exec/scan/scanner_context_test.cpp b/be/test/exec/scan/scanner_context_test.cpp index 4eb76c27a219c9..0e2aa0a9bde9d4 100644 --- a/be/test/exec/scan/scanner_context_test.cpp +++ b/be/test/exec/scan/scanner_context_test.cpp @@ -39,6 +39,10 @@ #include "exec/scan/scanner_scheduler.h" #include "runtime/descriptors.h" #include "runtime/query_context.h" +#include "storage/options.h" +#include "storage/storage_engine.h" +#include "storage/tablet/tablet.h" +#include "storage/tablet/tablet_meta.h" #include "testutil/mock/mock_runtime_state.h" namespace doris { @@ -185,6 +189,165 @@ TEST_F(ScannerContextTest, test_init) { ASSERT_TRUE(st.ok()); } +TEST_F(ScannerContextTest, inverted_index_profile_collection_is_additive_and_idempotent) { + auto engine = std::make_unique(EngineOptions {}); + auto tablet_meta = std::make_shared(1, 2, 15673, 15674, 4, 5, TTabletSchema {}, 6, + std::unordered_map {{7, 8}}, + UniqueId(9, 10), TTabletType::TABLET_TYPE_DISK, + TCompressionType::LZ4F); + auto tablet = std::make_shared(*engine, std::move(tablet_meta), nullptr); + const int parallel_tasks = 1; + auto scan_operator = std::make_unique(obj_pool.get(), tnode, 0, *descs, + parallel_tasks, TQueryCacheParam {}); + auto local_state = OlapScanLocalState::create_unique(state.get(), scan_operator.get()); + const std::vector scan_ranges; + const std::map, + std::vector>>> + shared_state_map; + LocalStateInfo local_state_info {profile.get(), scan_ranges, nullptr, shared_state_map, 0}; + const Status init_status = local_state->init(state.get(), local_state_info); + ASSERT_TRUE(init_status.ok()) << init_status.to_string(); + + auto make_scanner = [&]() { + OlapScanner::Params params; + params.state = state.get(); + params.profile = profile.get(); + params.version = 0; + params.limit = -1; + params.aggregation = false; + return OlapScanner::create_shared(local_state.get(), std::move(params)); + }; + auto scanner1 = make_scanner(); + auto scanner2 = make_scanner(); + scanner1->_tablet_reader_params.tablet = tablet; + scanner2->_tablet_reader_params.tablet = tablet; + scanner1->_tablet_reader = std::make_unique(); + scanner2->_tablet_reader = std::make_unique(); + auto* stats1 = scanner1->_tablet_reader->mutable_stats(); + stats1->snii_stats.prx_raw_frames = 1; + stats1->snii_stats.prx_plaintext_bytes = 10; + stats1->snii_stats.prx_decode_ns = 100; + stats1->snii_stats.phrase_candidate_docs = 3; + stats1->snii_stats.common_grams_gram_plans = 1; + stats1->snii_stats.common_grams_fallback_kill_switch = 5; + stats1->snii_stats.common_grams_plain_posting_bytes = 10; + stats1->snii_stats.common_grams_gram_posting_bytes = 20; + stats1->snii_stats.common_grams_plain_estimated_candidate_df = 30; + stats1->snii_stats.common_grams_gram_estimated_candidate_df = 40; + stats1->snii_stats.common_grams_plain_estimated_cost = 50; + stats1->snii_stats.common_grams_gram_estimated_cost = 60; + stats1->snii_stats.common_grams_fallback_base_analyzer_mismatch = 61; + stats1->snii_stats.common_grams_fallback_prefix_tail_empty = 62; + stats1->snii_stats.common_grams_planning_ns = 65; + auto* stats2 = scanner2->_tablet_reader->mutable_stats(); + stats2->snii_stats.prx_raw_frames = 2; + stats2->snii_stats.prx_plaintext_bytes = 20; + stats2->snii_stats.prx_decode_ns = 200; + stats2->snii_stats.phrase_candidate_docs = 4; + stats2->snii_stats.common_grams_gram_plans = 2; + stats2->snii_stats.common_grams_fallback_kill_switch = 6; + stats2->snii_stats.common_grams_plain_posting_bytes = 1; + stats2->snii_stats.common_grams_gram_posting_bytes = 2; + stats2->snii_stats.common_grams_plain_estimated_candidate_df = 3; + stats2->snii_stats.common_grams_gram_estimated_candidate_df = 4; + stats2->snii_stats.common_grams_plain_estimated_cost = 5; + stats2->snii_stats.common_grams_gram_estimated_cost = 6; + stats2->snii_stats.common_grams_fallback_base_analyzer_mismatch = 7; + stats2->snii_stats.common_grams_fallback_prefix_tail_empty = 8; + stats2->snii_stats.common_grams_planning_ns = 11; + + RuntimeProfile* index_filter = local_state->_index_filter_profile.get(); + ASSERT_NE(index_filter, nullptr); + auto* raw_frames = index_filter->get_counter("SniiPrxRawFrames"); + auto* plaintext_bytes = index_filter->get_counter("SniiPrxPlaintextBytes"); + auto* decode_time = index_filter->get_counter("SniiPrxInclusiveDecodeTime"); + auto* phrase_candidate_docs = index_filter->get_counter("SniiPhraseCandidateDocs"); + auto* common_grams_gram_plans = index_filter->get_counter("SniiCommonGramsGramPlans"); + auto* common_grams_fallback_kill_switch = + index_filter->get_counter("SniiCommonGramsFallbackKillSwitch"); + struct ExpectedSniiCounter { + const char* name; + RuntimeProfile::Counter* counter; + int64_t scanner1_value; + int64_t combined_value; + }; + const ExpectedSniiCounter snii_counters[] = { + {"SniiCommonGramsPlainPostingBytes", + index_filter->get_counter("SniiCommonGramsPlainPostingBytes"), 10, 11}, + {"SniiCommonGramsGramPostingBytes", + index_filter->get_counter("SniiCommonGramsGramPostingBytes"), 20, 22}, + {"SniiCommonGramsPlainEstimatedCandidateDf", + index_filter->get_counter("SniiCommonGramsPlainEstimatedCandidateDf"), 30, 33}, + {"SniiCommonGramsGramEstimatedCandidateDf", + index_filter->get_counter("SniiCommonGramsGramEstimatedCandidateDf"), 40, 44}, + {"SniiCommonGramsPlainEstimatedCost", + index_filter->get_counter("SniiCommonGramsPlainEstimatedCost"), 50, 55}, + {"SniiCommonGramsGramEstimatedCost", + index_filter->get_counter("SniiCommonGramsGramEstimatedCost"), 60, 66}, + {"SniiCommonGramsFallbackBaseAnalyzerMismatch", + index_filter->get_counter("SniiCommonGramsFallbackBaseAnalyzerMismatch"), 61, 68}, + {"SniiCommonGramsFallbackPrefixTailEmpty", + index_filter->get_counter("SniiCommonGramsFallbackPrefixTailEmpty"), 62, 70}, + {"SniiCommonGramsPlanningTime", + index_filter->get_counter("SniiCommonGramsPlanningTime"), 65, 76}, + }; + + std::vector zero_nodes; + index_filter->to_thrift(&zero_nodes); + ASSERT_EQ(zero_nodes.size(), 1U); + for (const auto& expected : snii_counters) { + bool serialized = false; + for (const auto& thrift_counter : zero_nodes.front().counters) { + serialized |= thrift_counter.name == expected.name; + } + EXPECT_FALSE(serialized) << expected.name; + } + ASSERT_NE(raw_frames, nullptr); + ASSERT_NE(plaintext_bytes, nullptr); + ASSERT_NE(decode_time, nullptr); + ASSERT_NE(phrase_candidate_docs, nullptr); + ASSERT_NE(common_grams_gram_plans, nullptr); + ASSERT_NE(common_grams_fallback_kill_switch, nullptr); + for (const auto& expected : snii_counters) { + ASSERT_NE(expected.counter, nullptr) << expected.name; + EXPECT_NE(dynamic_cast(expected.counter), nullptr) + << expected.name; + } + + scanner1->_collect_profile_before_close(); + EXPECT_EQ(raw_frames->value(), 1); + EXPECT_EQ(plaintext_bytes->value(), 10); + EXPECT_EQ(decode_time->value(), 100); + EXPECT_EQ(phrase_candidate_docs->value(), 3); + EXPECT_EQ(common_grams_gram_plans->value(), 1); + EXPECT_EQ(common_grams_fallback_kill_switch->value(), 5); + for (const auto& expected : snii_counters) { + EXPECT_EQ(expected.counter->value(), expected.scanner1_value) << expected.name; + } + + scanner1->_collect_profile_before_close(); + EXPECT_EQ(raw_frames->value(), 1); + EXPECT_EQ(plaintext_bytes->value(), 10); + EXPECT_EQ(decode_time->value(), 100); + EXPECT_EQ(phrase_candidate_docs->value(), 3); + EXPECT_EQ(common_grams_gram_plans->value(), 1); + EXPECT_EQ(common_grams_fallback_kill_switch->value(), 5); + for (const auto& expected : snii_counters) { + EXPECT_EQ(expected.counter->value(), expected.scanner1_value) << expected.name; + } + + scanner2->_collect_profile_before_close(); + EXPECT_EQ(raw_frames->value(), 3); + EXPECT_EQ(plaintext_bytes->value(), 30); + EXPECT_EQ(decode_time->value(), 300); + EXPECT_EQ(phrase_candidate_docs->value(), 7); + EXPECT_EQ(common_grams_gram_plans->value(), 3); + EXPECT_EQ(common_grams_fallback_kill_switch->value(), 11); + for (const auto& expected : snii_counters) { + EXPECT_EQ(expected.counter->value(), expected.combined_value) << expected.name; + } +} + TEST_F(ScannerContextTest, test_serial_run) { const int parallel_tasks = 1; auto scan_operator = std::make_unique(obj_pool.get(), tnode, 0, *descs, diff --git a/be/test/exprs/function/function_is_null_test.cpp b/be/test/exprs/function/function_is_null_test.cpp index b10e7d76f31b05..fb5e59d391f507 100644 --- a/be/test/exprs/function/function_is_null_test.cpp +++ b/be/test/exprs/function/function_is_null_test.cpp @@ -77,11 +77,18 @@ class FunctionIsNullTest : public ::testing::Test { _inverted_index_query_cache = std::unique_ptr( InvertedIndexQueryCache::create_global_cache(inverted_index_cache_limit, 1)); + // Both caches are owned by this fixture, so the previous globals must come back in + // TearDown -- otherwise ExecEnv keeps pointing at them after the fixture is destroyed and + // the next test that reaches InvertedIndexQueryCache::instance() reads freed memory. + _previous_searcher_cache = ExecEnv::GetInstance()->get_inverted_index_searcher_cache(); + _previous_query_cache = ExecEnv::GetInstance()->get_inverted_index_query_cache(); ExecEnv::GetInstance()->set_inverted_index_searcher_cache( _inverted_index_searcher_cache.get()); - ExecEnv::GetInstance()->_inverted_index_query_cache = _inverted_index_query_cache.get(); + ExecEnv::GetInstance()->set_inverted_index_query_cache(_inverted_index_query_cache.get()); } void TearDown() override { + ExecEnv::GetInstance()->set_inverted_index_searcher_cache(_previous_searcher_cache); + ExecEnv::GetInstance()->set_inverted_index_query_cache(_previous_query_cache); EXPECT_TRUE(io::global_local_filesystem()->delete_directory(_tablet->tablet_path()).ok()); EXPECT_TRUE(io::global_local_filesystem()->delete_directory(_absolute_dir).ok()); EXPECT_TRUE(io::global_local_filesystem()->delete_directory(tmp_dir).ok()); @@ -135,6 +142,8 @@ class FunctionIsNullTest : public ::testing::Test { std::string _absolute_dir; std::string _curreent_dir; TabletSchemaPB _schema_pb; + InvertedIndexSearcherCache* _previous_searcher_cache = nullptr; + InvertedIndexQueryCache* _previous_query_cache = nullptr; std::unique_ptr _inverted_index_searcher_cache; std::unique_ptr _inverted_index_query_cache; }; diff --git a/be/test/exprs/function/function_search_test.cpp b/be/test/exprs/function/function_search_test.cpp index ac57847f32e4de..74cd72dcb79ea6 100644 --- a/be/test/exprs/function/function_search_test.cpp +++ b/be/test/exprs/function/function_search_test.cpp @@ -21,17 +21,22 @@ #include #include +#include #include #include #include +#include #include #include +#include #include "core/block/block.h" #include "core/data_type/data_type_array.h" #include "core/data_type/data_type_nullable.h" #include "core/data_type/data_type_string.h" #include "core/data_type/primitive_type.h" +#include "runtime/exec_env.h" +#include "runtime/index_policy/index_policy_mgr.h" #include "storage/index/index_file_reader.h" #include "storage/index/index_iterator.h" #include "storage/index/inverted/inverted_index_iterator.h" @@ -40,6 +45,8 @@ #include "storage/index/inverted/query_v2/phrase_query/multi_phrase_weight.h" #include "storage/index/inverted/query_v2/phrase_query/phrase_query.h" #include "storage/segment/variant/nested_group_provider.h" +#include "util/defer_op.h" +#include "util/thrift_util.h" namespace doris { @@ -115,6 +122,27 @@ class RecordingIndexIterator : public segment_v2::IndexIterator { Int32 last_int_value = 0; }; +class RecordingDirectInvertedIndexIterator final : public segment_v2::InvertedIndexIterator { +public: + Status read_from_index(const segment_v2::IndexParam& param) override { + ++read_calls; + auto* inverted_param = std::get_if(¶m); + DORIS_CHECK(inverted_param != nullptr); + DORIS_CHECK(*inverted_param != nullptr); + DORIS_CHECK((*inverted_param)->roaring != nullptr); + (*inverted_param)->roaring->add(3); + return Status::OK(); + } + + Status read_null_bitmap(segment_v2::InvertedIndexQueryCacheHandle* /*cache_handle*/) override { + return Status::OK(); + } + + Result has_null() override { return false; } + + int read_calls = 0; +}; + class DummyInvertedIndexReader final : public segment_v2::InvertedIndexReader { public: explicit DummyInvertedIndexReader(const TabletIndex* index_meta) @@ -151,6 +179,185 @@ class DummyInvertedIndexReader final : public segment_v2::InvertedIndexReader { segment_v2::InvertedIndexReaderType _reader_type = segment_v2::InvertedIndexReaderType::BKD; }; +class RejectingCluceneIndexFileReader final : public segment_v2::IndexFileReader { +public: + explicit RejectingCluceneIndexFileReader( + InvertedIndexStorageFormatPB storage_format = InvertedIndexStorageFormatPB::SNII, + const std::string& index_path = "/tmp/search_snii_native_idx") + : segment_v2::IndexFileReader(nullptr, index_path, storage_format) {} + + Status init(int32_t /*read_buffer_size*/, const io::IOContext* /*io_ctx*/) override { + ++init_calls; + return Status::OK(); + } + + Result> open( + const TabletIndex* /*index_meta*/, const io::IOContext* /*io_ctx*/) const override { + ++open_calls; + return ResultError(Status::InternalError("unexpected CLucene open for SNII search")); + } + + int init_calls = 0; + mutable int open_calls = 0; +}; + +class RecordingNativeInvertedIndexReader final : public segment_v2::InvertedIndexReader { +public: + RecordingNativeInvertedIndexReader( + const TabletIndex* index_meta, + const std::shared_ptr& index_file_reader, + segment_v2::InvertedIndexReaderType reader_type = + segment_v2::InvertedIndexReaderType::FULLTEXT) + : segment_v2::InvertedIndexReader(index_meta, index_file_reader), + _reader_type(reader_type), + _null_cache(1024 * 1024, 1), + _null_cache_key {"/tmp/search_snii_native_null", "", + segment_v2::InvertedIndexQueryType::UNKNOWN_QUERY, + std::to_string(index_meta->index_id())} { + set_has_null(false); + } + + Status new_iterator(std::unique_ptr* /*iterator*/) override { + return Status::OK(); + } + + Status query(const segment_v2::IndexQueryContextPtr& /*context*/, + const std::string& column_name, const Field& query_value, + segment_v2::InvertedIndexQueryType query_type, + std::shared_ptr& bit_map, + const InvertedIndexAnalyzerCtx* analyzer_ctx = nullptr) override { + ++query_calls; + last_column_name = column_name; + last_query_type = query_type; + last_query_value_type = query_value.get_type(); + last_analyzer_ctx = analyzer_ctx; + if (last_query_value_type == TYPE_STRING) { + last_query_value = query_value.get(); + } + + bit_map = std::make_shared(); + auto result_it = query_results.find(last_query_value); + if (result_it != query_results.end()) { + *bit_map = result_it->second; + } + return Status::OK(); + } + + Status try_query(const segment_v2::IndexQueryContextPtr& /*context*/, + const std::string& /*column_name*/, const Field& /*query_value*/, + segment_v2::InvertedIndexQueryType /*query_type*/, + size_t* /*count*/) override { + return Status::OK(); + } + + Status read_null_bitmap(const segment_v2::IndexQueryContextPtr& /*context*/, + segment_v2::InvertedIndexQueryCacheHandle* cache_handle, + lucene::store::Directory* /*dir*/ = nullptr) override { + ++null_bitmap_calls; + _null_cache.insert(_null_cache_key, std::make_shared(_null_bitmap), + cache_handle); + return Status::OK(); + } + + segment_v2::InvertedIndexReaderType type() override { return _reader_type; } + + void set_query_result(const std::string& pattern, roaring::Roaring result) { + query_results[pattern] = std::move(result); + } + + void set_null_bitmap(roaring::Roaring null_bitmap) { + _null_bitmap = std::move(null_bitmap); + set_has_null(!_null_bitmap.isEmpty()); + } + + int query_calls = 0; + int null_bitmap_calls = 0; + std::string last_column_name; + std::string last_query_value; + PrimitiveType last_query_value_type = PrimitiveType::TYPE_NULL; + segment_v2::InvertedIndexQueryType last_query_type = + segment_v2::InvertedIndexQueryType::UNKNOWN_QUERY; + const InvertedIndexAnalyzerCtx* last_analyzer_ctx = nullptr; + std::unordered_map query_results; + +private: + segment_v2::InvertedIndexReaderType _reader_type; + roaring::Roaring _null_bitmap; + segment_v2::InvertedIndexQueryCache _null_cache; + segment_v2::InvertedIndexQueryCache::CacheKey _null_cache_key; +}; + +class ScopedInvertedIndexQueryCache final { +public: + ScopedInvertedIndexQueryCache() + : _previous(ExecEnv::GetInstance()->get_inverted_index_query_cache()), + _cache(segment_v2::InvertedIndexQueryCache::create_global_cache(1024 * 1024, 1)) { + ExecEnv::GetInstance()->set_inverted_index_query_cache(_cache.get()); + } + + ~ScopedInvertedIndexQueryCache() { + ExecEnv::GetInstance()->set_inverted_index_query_cache(_previous); + } + + segment_v2::InvertedIndexQueryCache* get() const { return _cache.get(); } + +private: + segment_v2::InvertedIndexQueryCache* _previous; + std::unique_ptr _cache; +}; + +static roaring::Roaring make_bitmap(std::initializer_list docs) { + roaring::Roaring bitmap; + for (uint32_t doc : docs) { + bitmap.add(doc); + } + return bitmap; +} + +static void expect_bitmap_eq(const roaring::Roaring& actual, + std::initializer_list expected_docs) { + auto expected = make_bitmap(expected_docs); + EXPECT_EQ(expected.cardinality(), actual.cardinality()); + EXPECT_TRUE(actual == expected); +} + +static roaring::Roaring collect_docs( + const segment_v2::inverted_index::query_v2::ScorerPtr& scorer) { + roaring::Roaring docs; + for (uint32_t doc = scorer->doc(); doc != segment_v2::inverted_index::query_v2::TERMINATED; + doc = scorer->advance()) { + docs.add(doc); + } + return docs; +} + +static TSearchClause make_leaf_clause(const std::string& clause_type, const std::string& value) { + TSearchClause clause; + clause.clause_type = clause_type; + clause.field_name = "body"; + clause.value = value; + clause.__isset.field_name = true; + clause.__isset.value = true; + return clause; +} + +static Status insert_search_dsl_cache( + segment_v2::InvertedIndexQueryCache* cache, + const std::shared_ptr& index_file_reader, + const TSearchParam& search_param, roaring::Roaring bitmap) { + ThriftSerializer serializer(false, 1024); + TSearchParam copy = search_param; + std::string signature; + RETURN_IF_ERROR(serializer.serialize(©, &signature)); + + segment_v2::InvertedIndexQueryCache::CacheKey key { + index_file_reader->get_index_path_prefix(), "__search_dsl__", + segment_v2::InvertedIndexQueryType::SEARCH_DSL_QUERY, std::move(signature)}; + segment_v2::InvertedIndexQueryCacheHandle handle; + cache->insert(key, std::make_shared(std::move(bitmap)), &handle); + return Status::OK(); +} + static TabletIndex make_test_inverted_index( int64_t index_id, const std::map& properties = {}) { TabletIndex index_meta; @@ -166,6 +373,32 @@ static TabletIndex make_test_inverted_index( return index_meta; } +static Status resolve_non_variant_binding_with_mismatched_analyzer(const DataTypePtr& column_type) { + std::map index_properties; + index_properties[INVERTED_INDEX_PARSER_KEY] = INVERTED_INDEX_PARSER_STANDARD; + auto index_meta = make_test_inverted_index(13, index_properties); + auto reader = std::make_shared( + &index_meta, nullptr, segment_v2::InvertedIndexReaderType::FULLTEXT); + + segment_v2::InvertedIndexIterator iterator; + iterator.add_reader(segment_v2::InvertedIndexReaderType::FULLTEXT, reader); + + std::unordered_map data_type_with_names; + data_type_with_names.emplace("content", IndexFieldNameAndTypePair {"content", column_type}); + std::unordered_map iterators; + iterators["content"] = &iterator; + + TSearchFieldBinding field_binding; + field_binding.field_name = "content"; + field_binding.index_properties[INVERTED_INDEX_PARSER_KEY] = INVERTED_INDEX_PARSER_ENGLISH; + field_binding.__isset.index_properties = true; + + auto context = std::make_shared(); + FieldReaderResolver resolver(data_type_with_names, iterators, context, {field_binding}); + FieldReaderBinding binding; + return resolver.resolve("content", InvertedIndexQueryType::MATCH_ANY_QUERY, &binding); +} + TEST_F(FunctionSearchTest, TestGetName) { EXPECT_EQ("search", function_search->get_name()); } @@ -1716,6 +1949,7 @@ TEST_F(FunctionSearchTest, TestBuildLeafQueryPhrase) { binding.stored_field_wstr = L"content"; binding.index_properties["parser"] = "unicode"; binding.query_type = InvertedIndexQueryType::MATCH_PHRASE_QUERY; + binding.execution_mode = SearchFieldExecutionMode::CLUCENE; auto* dummy_reader = reinterpret_cast(0x1); binding.lucene_reader = std::shared_ptr( @@ -1736,6 +1970,80 @@ TEST_F(FunctionSearchTest, TestBuildLeafQueryPhrase) { EXPECT_NE(phrase_query, nullptr); } +TEST_F(FunctionSearchTest, TestBuildLeafQueryPhraseUsesPlainTerms) { + auto* exec_env = ExecEnv::GetInstance(); + auto* previous_policy_mgr = exec_env->index_policy_mgr(); + IndexPolicyMgr scoped_policy_mgr; + exec_env->_index_policy_mgr = &scoped_policy_mgr; + DEFER(exec_env->_index_policy_mgr = previous_policy_mgr); + + auto* policy_mgr = exec_env->index_policy_mgr(); + ASSERT_NE(policy_mgr, nullptr); + + TIndexPolicy tokenizer; + tokenizer.id = 910020; + tokenizer.name = "function_search_cg_tokenizer"; + tokenizer.type = TIndexPolicyType::TOKENIZER; + tokenizer.properties["type"] = "char_group"; + tokenizer.properties["tokenize_on_chars"] = "[whitespace]"; + + TIndexPolicy common_grams; + common_grams.id = 910021; + common_grams.name = "function_search_cg_filter"; + common_grams.type = TIndexPolicyType::TOKEN_FILTER; + common_grams.properties["type"] = "common_grams"; + + TIndexPolicy analyzer; + analyzer.id = 910022; + analyzer.name = "function_search_cg_analyzer"; + analyzer.type = TIndexPolicyType::ANALYZER; + analyzer.properties["tokenizer"] = tokenizer.name; + analyzer.properties["token_filter"] = "lowercase," + common_grams.name; + policy_mgr->apply_policy_changes({tokenizer, common_grams, analyzer}, {}); + + TSearchClause clause; + clause.clause_type = "PHRASE"; + clause.field_name = "content"; + clause.value = "man of the year"; + clause.__isset.field_name = true; + clause.__isset.value = true; + + auto context = std::make_shared(); + std::unordered_map data_type_with_names; + data_type_with_names.emplace("content", IndexFieldNameAndTypePair {"content", nullptr}); + std::unordered_map iterators; + FieldReaderResolver resolver(data_type_with_names, iterators, context); + + FieldReaderBinding binding; + binding.logical_field_name = "content"; + binding.stored_field_name = "content"; + binding.stored_field_wstr = L"content"; + binding.index_properties["analyzer"] = analyzer.name; + binding.query_type = InvertedIndexQueryType::MATCH_PHRASE_QUERY; + binding.execution_mode = SearchFieldExecutionMode::CLUCENE; + auto* dummy_reader = reinterpret_cast(0x1); + binding.lucene_reader = std::shared_ptr( + dummy_reader, [](lucene::index::IndexReader* /*ptr*/) {}); + binding.binding_key = + resolver.binding_key_for("content", InvertedIndexQueryType::MATCH_PHRASE_QUERY); + resolver._cache[binding.binding_key] = binding; + + inverted_index::query_v2::QueryPtr query; + std::string binding_key; + ASSERT_TRUE(function_search + ->build_leaf_query(clause, context, resolver, &query, &binding_key, "OR", 0) + .ok()); + + auto phrase = std::dynamic_pointer_cast(query); + ASSERT_NE(phrase, nullptr); + ASSERT_EQ(phrase->_term_infos.size(), 4); + EXPECT_EQ(phrase->_term_infos[0].get_single_term(), "man"); + EXPECT_EQ(phrase->_term_infos[1].get_single_term(), "of"); + EXPECT_EQ(phrase->_term_infos[2].get_single_term(), "the"); + EXPECT_EQ(phrase->_term_infos[3].get_single_term(), "year"); + policy_mgr->apply_policy_changes({}, {tokenizer.id, common_grams.id, analyzer.id}); +} + TEST_F(FunctionSearchTest, TestBuildLeafQueryVariantMissingFieldReturnsUnknown) { TSearchClause clause; clause.clause_type = "TERM"; @@ -1872,6 +2180,64 @@ TEST_F(FunctionSearchTest, EXPECT_EQ(ErrorCode::INVERTED_INDEX_FILE_NOT_FOUND, status.code()); } +TEST_F(FunctionSearchTest, + TestFieldReaderResolverNonVariantStringBindingRejectsMismatchedAnalyzer) { + auto status = resolve_non_variant_binding_with_mismatched_analyzer( + std::make_shared()); + + ASSERT_FALSE(status.ok()); + EXPECT_EQ(ErrorCode::INVERTED_INDEX_BYPASS, status.code()); +} + +TEST_F(FunctionSearchTest, + TestFieldReaderResolverNonVariantArrayStringBindingRejectsMismatchedAnalyzer) { + auto column_type = + std::make_shared(make_nullable(std::make_shared())); + auto status = resolve_non_variant_binding_with_mismatched_analyzer(column_type); + + ASSERT_FALSE(status.ok()); + EXPECT_EQ(ErrorCode::INVERTED_INDEX_BYPASS, status.code()); +} + +TEST_F(FunctionSearchTest, TestFieldReaderResolverExactIgnoresAnalyzedBindingHint) { + std::map analyzed_properties; + analyzed_properties[INVERTED_INDEX_PARSER_KEY] = INVERTED_INDEX_PARSER_STANDARD; + auto analyzed_index = make_test_inverted_index(13, analyzed_properties); + auto keyword_index = make_test_inverted_index(14); + auto index_file_reader = std::make_shared( + nullptr, "/tmp/search_exact_multi_index", InvertedIndexStorageFormatPB::SNII); + auto analyzed_reader = std::make_shared( + &analyzed_index, index_file_reader, segment_v2::InvertedIndexReaderType::FULLTEXT); + auto keyword_reader = std::make_shared( + &keyword_index, index_file_reader, segment_v2::InvertedIndexReaderType::STRING_TYPE); + + segment_v2::InvertedIndexIterator iterator; + iterator.add_reader(segment_v2::InvertedIndexReaderType::FULLTEXT, analyzed_reader); + iterator.add_reader(segment_v2::InvertedIndexReaderType::STRING_TYPE, keyword_reader); + + std::unordered_map data_type_with_names; + data_type_with_names.emplace( + "content", IndexFieldNameAndTypePair {"content", std::make_shared()}); + std::unordered_map iterators; + iterators["content"] = &iterator; + + TSearchFieldBinding field_binding; + field_binding.field_name = "content"; + field_binding.index_properties = analyzed_properties; + field_binding.__isset.index_properties = true; + + auto context = std::make_shared(); + FieldReaderResolver resolver(data_type_with_names, iterators, context, {field_binding}); + FieldReaderBinding binding; + auto status = resolver.resolve("content", InvertedIndexQueryType::EQUAL_QUERY, &binding); + + ASSERT_TRUE(status.ok()) << status; + ASSERT_NE(binding.inverted_reader, nullptr); + EXPECT_EQ(binding.inverted_reader->get_index_id(), 14); + EXPECT_EQ(binding.query_type, InvertedIndexQueryType::EQUAL_QUERY); + EXPECT_TRUE(binding.index_properties.empty()); +} + TEST_F(FunctionSearchTest, TestFieldReaderResolverVariantBkdDirectReader) { auto context = std::make_shared(); @@ -1913,6 +2279,345 @@ TEST_F(FunctionSearchTest, TestFieldReaderResolverVariantBkdDirectReader) { EXPECT_TRUE(cache.begin()->second.use_direct_index_reader()); } +TEST_F(FunctionSearchTest, TestFieldReaderResolverBindsSniiWithoutOpeningClucene) { + auto context = std::make_shared(); + auto index_meta = make_test_inverted_index( + 14, {{INVERTED_INDEX_PARSER_KEY, INVERTED_INDEX_PARSER_STANDARD}}); + auto index_file_reader = std::make_shared(); + auto reader = std::make_shared( + &index_meta, index_file_reader, segment_v2::InvertedIndexReaderType::FULLTEXT); + + segment_v2::InvertedIndexIterator iterator; + iterator.add_reader(segment_v2::InvertedIndexReaderType::FULLTEXT, reader); + + std::unordered_map data_type_with_names; + data_type_with_names.emplace( + "body", IndexFieldNameAndTypePair {"body", std::make_shared()}); + std::unordered_map iterators; + iterators["body"] = &iterator; + + TSearchFieldBinding field_binding; + field_binding.field_name = "body"; + field_binding.index_properties = index_meta.properties(); + field_binding.__isset.index_properties = true; + + FieldReaderResolver resolver(data_type_with_names, iterators, context, {field_binding}); + FieldReaderBinding binding; + auto status = resolver.resolve("body", InvertedIndexQueryType::MATCH_ANY_QUERY, &binding); + + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(0, index_file_reader->init_calls); + EXPECT_EQ(0, index_file_reader->open_calls); + EXPECT_EQ(reader, binding.inverted_reader); + EXPECT_EQ(nullptr, binding.lucene_reader); + EXPECT_TRUE(binding.use_snii_native_reader()); + EXPECT_FALSE(binding.use_direct_index_reader()); + EXPECT_EQ(SearchFieldExecutionMode::SNII_NATIVE, binding.execution_mode); +} + +TEST_F(FunctionSearchTest, TestBuildLeafQueryExecutesSelectedSniiWildcardReader) { + auto context = std::make_shared(); + std::map decoy_properties { + {INVERTED_INDEX_PARSER_KEY, INVERTED_INDEX_PARSER_ENGLISH}, + {INVERTED_INDEX_PARSER_LOWERCASE_KEY, INVERTED_INDEX_PARSER_TRUE}}; + std::map selected_properties { + {INVERTED_INDEX_PARSER_KEY, INVERTED_INDEX_PARSER_STANDARD}, + {INVERTED_INDEX_PARSER_LOWERCASE_KEY, INVERTED_INDEX_PARSER_TRUE}}; + auto decoy_meta = make_test_inverted_index(15, decoy_properties); + auto selected_meta = make_test_inverted_index(16, selected_properties); + auto decoy_file_reader = std::make_shared( + InvertedIndexStorageFormatPB::SNII, "/tmp/search_snii_decoy_idx"); + auto selected_file_reader = std::make_shared( + InvertedIndexStorageFormatPB::SNII, "/tmp/search_snii_selected_idx"); + auto decoy_reader = + std::make_shared(&decoy_meta, decoy_file_reader); + auto selected_reader = std::make_shared( + &selected_meta, selected_file_reader); + selected_reader->set_query_result("*lpha", make_bitmap({0, 2})); + selected_reader->set_null_bitmap(make_bitmap({3})); + + segment_v2::InvertedIndexIterator iterator; + iterator.add_reader(segment_v2::InvertedIndexReaderType::FULLTEXT, decoy_reader); + iterator.add_reader(segment_v2::InvertedIndexReaderType::FULLTEXT, selected_reader); + + std::unordered_map data_type_with_names; + data_type_with_names.emplace( + "body", IndexFieldNameAndTypePair {"stored_body", std::make_shared()}); + std::unordered_map iterators; + iterators["body"] = &iterator; + + TSearchFieldBinding field_binding; + field_binding.field_name = "body"; + field_binding.index_properties = selected_properties; + field_binding.__isset.index_properties = true; + + FieldReaderResolver resolver(data_type_with_names, iterators, context, {field_binding}); + auto clause = make_leaf_clause("WILDCARD", "*LPHA"); + inverted_index::query_v2::QueryPtr query; + std::string binding_key; + auto status = function_search->build_leaf_query(clause, context, resolver, &query, &binding_key, + "OR", 0, 4); + + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_NE(nullptr, query); + EXPECT_EQ(0, decoy_reader->query_calls); + EXPECT_EQ(1, selected_reader->query_calls); + EXPECT_EQ("stored_body", selected_reader->last_column_name); + EXPECT_EQ(TYPE_STRING, selected_reader->last_query_value_type); + EXPECT_EQ("*lpha", selected_reader->last_query_value); + EXPECT_EQ(InvertedIndexQueryType::WILDCARD_QUERY, selected_reader->last_query_type); + EXPECT_EQ(nullptr, selected_reader->last_analyzer_ctx); + EXPECT_EQ(0, decoy_file_reader->open_calls); + EXPECT_EQ(0, selected_file_reader->open_calls); + EXPECT_EQ(0, decoy_reader->null_bitmap_calls); + EXPECT_EQ(1, selected_reader->null_bitmap_calls); + const auto& bindings = resolver.binding_cache(); + ASSERT_EQ(1U, bindings.size()); + EXPECT_EQ(InvertedIndexQueryType::MATCH_ANY_QUERY, bindings.begin()->second.query_type); + + auto weight = query->weight(true); + ASSERT_NE(nullptr, weight); + inverted_index::query_v2::QueryExecutionContext exec_ctx; + exec_ctx.segment_num_rows = 4; + auto scorer = weight->scorer(exec_ctx, binding_key); + ASSERT_NE(nullptr, scorer); + EXPECT_EQ(0U, scorer->doc()); + EXPECT_FLOAT_EQ(1.0F, scorer->score()); + expect_bitmap_eq(collect_docs(scorer), {0, 2}); + ASSERT_TRUE(scorer->has_null_bitmap()); + const auto* null_bitmap = scorer->get_null_bitmap(); + ASSERT_NE(nullptr, null_bitmap); + expect_bitmap_eq(*null_bitmap, {3}); +} + +TEST_F(FunctionSearchTest, TestSniiWildcardPreservesThreeValuedBooleanAndFieldExists) { + auto context = std::make_shared(); + std::map properties { + {INVERTED_INDEX_PARSER_KEY, INVERTED_INDEX_PARSER_STANDARD}}; + auto index_meta = make_test_inverted_index(17, properties); + auto index_file_reader = std::make_shared(); + auto reader = + std::make_shared(&index_meta, index_file_reader); + reader->set_query_result("*lpha", make_bitmap({0})); + reader->set_query_result("beta*", make_bitmap({1})); + reader->set_null_bitmap(make_bitmap({3})); + + segment_v2::InvertedIndexIterator iterator; + iterator.add_reader(segment_v2::InvertedIndexReaderType::FULLTEXT, reader); + std::unordered_map data_type_with_names; + data_type_with_names.emplace( + "body", IndexFieldNameAndTypePair {"body", std::make_shared()}); + std::unordered_map iterators; + iterators["body"] = &iterator; + + TSearchFieldBinding field_binding; + field_binding.field_name = "body"; + field_binding.index_properties = properties; + field_binding.__isset.index_properties = true; + FieldReaderResolver resolver(data_type_with_names, iterators, context, {field_binding}); + + TSearchClause or_clause; + or_clause.clause_type = "OR"; + or_clause.children = {make_leaf_clause("WILDCARD", "*lpha"), + make_leaf_clause("WILDCARD", "beta*")}; + or_clause.__isset.children = true; + + TSearchClause not_clause; + not_clause.clause_type = "NOT"; + not_clause.children = {make_leaf_clause("WILDCARD", "*lpha")}; + not_clause.__isset.children = true; + auto exists_clause = make_leaf_clause("WILDCARD", "*"); + + auto verify_result = [&](const TSearchClause& root, + std::initializer_list expected_docs, + std::initializer_list expected_nulls) { + inverted_index::query_v2::QueryPtr query; + std::string binding_key; + auto status = function_search->build_query_recursive(root, context, resolver, &query, + &binding_key, "OR", 0, 4); + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_NE(nullptr, query); + auto weight = query->weight(false); + ASSERT_NE(nullptr, weight); + auto scorer = weight->scorer( + build_variant_search_query_execution_context(4, resolver, nullptr), binding_key); + ASSERT_NE(nullptr, scorer); + expect_bitmap_eq(collect_docs(scorer), expected_docs); + ASSERT_TRUE(scorer->has_null_bitmap()); + const auto* null_bitmap = scorer->get_null_bitmap(); + ASSERT_NE(nullptr, null_bitmap); + expect_bitmap_eq(*null_bitmap, expected_nulls); + }; + + verify_result(or_clause, {0, 1}, {3}); + verify_result(not_clause, {1, 2}, {3}); + verify_result(exists_clause, {0, 1, 2}, {3}); + EXPECT_EQ(3, reader->query_calls); +} + +TEST_F(FunctionSearchTest, TestSniiNativeRejectsNonWildcardClause) { + auto context = std::make_shared(); + auto index_meta = make_test_inverted_index( + 18, {{INVERTED_INDEX_PARSER_KEY, INVERTED_INDEX_PARSER_STANDARD}}); + auto index_file_reader = std::make_shared(); + auto reader = + std::make_shared(&index_meta, index_file_reader); + segment_v2::InvertedIndexIterator iterator; + iterator.add_reader(segment_v2::InvertedIndexReaderType::FULLTEXT, reader); + + std::unordered_map data_type_with_names; + data_type_with_names.emplace( + "body", IndexFieldNameAndTypePair {"body", std::make_shared()}); + std::unordered_map iterators; + iterators["body"] = &iterator; + TSearchFieldBinding field_binding; + field_binding.field_name = "body"; + field_binding.index_properties = index_meta.properties(); + field_binding.__isset.index_properties = true; + FieldReaderResolver resolver(data_type_with_names, iterators, context, {field_binding}); + + auto clause = make_leaf_clause("TERM", "alpha"); + inverted_index::query_v2::QueryPtr query; + std::string binding_key; + auto status = function_search->build_leaf_query(clause, context, resolver, &query, &binding_key, + "OR", 0, 4); + + ASSERT_FALSE(status.ok()); + EXPECT_EQ(ErrorCode::NOT_IMPLEMENTED_ERROR, status.code()); + EXPECT_NE(std::string::npos, status.to_string().find("TERM")); + EXPECT_NE(std::string::npos, status.to_string().find("WILDCARD")); + EXPECT_EQ(0, reader->query_calls); + EXPECT_EQ(0, index_file_reader->open_calls); +} + +TEST_F(FunctionSearchTest, TestSearchDslCacheIsDisabledForSniiNativeExecution) { + ScopedInvertedIndexQueryCache cache_guard; + auto index_meta = make_test_inverted_index( + 19, {{INVERTED_INDEX_PARSER_KEY, INVERTED_INDEX_PARSER_STANDARD}}); + auto index_file_reader = std::make_shared(); + auto reader = + std::make_shared(&index_meta, index_file_reader); + reader->set_query_result("*lpha", make_bitmap({0})); + segment_v2::InvertedIndexIterator iterator; + iterator.add_reader(segment_v2::InvertedIndexReaderType::FULLTEXT, reader); + + std::unordered_map data_type_with_names; + data_type_with_names.emplace( + "body", IndexFieldNameAndTypePair {"body", std::make_shared()}); + std::unordered_map iterators; + iterators["body"] = &iterator; + + TSearchFieldBinding field_binding; + field_binding.field_name = "body"; + field_binding.index_properties = index_meta.properties(); + field_binding.__isset.index_properties = true; + TSearchParam search_param; + search_param.original_dsl = "body:*lpha"; + search_param.root = make_leaf_clause("WILDCARD", "*lpha"); + search_param.field_bindings = {field_binding}; + ASSERT_TRUE(insert_search_dsl_cache(cache_guard.get(), index_file_reader, search_param, + make_bitmap({3})) + .ok()); + + InvertedIndexResultBitmap result; + auto status = function_search->evaluate_inverted_index_with_search_param( + search_param, data_type_with_names, iterators, 4, result, true); + + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_NE(nullptr, result.get_data_bitmap()); + expect_bitmap_eq(*result.get_data_bitmap(), {0}); + EXPECT_EQ(1, reader->query_calls); +} + +TEST_F(FunctionSearchTest, TestSearchDslCacheIsDisabledWhenScoring) { + ScopedInvertedIndexQueryCache cache_guard; + auto index_meta = make_test_inverted_index( + 20, {{INVERTED_INDEX_PARSER_KEY, INVERTED_INDEX_PARSER_STANDARD}}); + auto index_file_reader = std::make_shared( + InvertedIndexStorageFormatPB::V2, "/tmp/search_scoring_v2_idx"); + auto reader = std::make_shared( + &index_meta, index_file_reader, segment_v2::InvertedIndexReaderType::FULLTEXT); + segment_v2::InvertedIndexIterator iterator; + iterator.add_reader(segment_v2::InvertedIndexReaderType::FULLTEXT, reader); + + std::unordered_map data_type_with_names; + data_type_with_names.emplace( + "body", IndexFieldNameAndTypePair {"body", std::make_shared()}); + std::unordered_map iterators; + iterators["body"] = &iterator; + TSearchFieldBinding field_binding; + field_binding.field_name = "body"; + field_binding.index_properties = index_meta.properties(); + field_binding.__isset.index_properties = true; + TSearchParam search_param; + search_param.original_dsl = "body:alpha"; + search_param.root = make_leaf_clause("TERM", "alpha"); + search_param.field_bindings = {field_binding}; + ASSERT_TRUE(insert_search_dsl_cache(cache_guard.get(), index_file_reader, search_param, + make_bitmap({3})) + .ok()); + + auto scoring_context = std::make_shared(); + scoring_context->collection_similarity = std::make_shared(); + InvertedIndexResultBitmap result; + std::unordered_map field_name_to_column_id; + auto status = function_search->evaluate_inverted_index_with_search_param( + search_param, data_type_with_names, iterators, 4, result, true, nullptr, + field_name_to_column_id, scoring_context); + + EXPECT_FALSE(status.ok()); + EXPECT_EQ(1, index_file_reader->init_calls); + EXPECT_EQ(1, index_file_reader->open_calls); +} + +TEST_F(FunctionSearchTest, TestSearchDslCacheRemainsEnabledForUnreferencedSniiField) { + ScopedInvertedIndexQueryCache cache_guard; + auto text_index_meta = make_test_inverted_index( + 21, {{INVERTED_INDEX_PARSER_KEY, INVERTED_INDEX_PARSER_STANDARD}}); + auto number_index_meta = make_test_inverted_index(22); + auto text_file_reader = std::make_shared( + InvertedIndexStorageFormatPB::SNII, "/tmp/search_mixed_cache_idx"); + auto number_file_reader = std::make_shared( + InvertedIndexStorageFormatPB::V2, "/tmp/search_mixed_cache_idx"); + auto text_reader = std::make_shared( + &text_index_meta, text_file_reader, InvertedIndexReaderType::FULLTEXT); + auto number_reader = std::make_shared( + &number_index_meta, number_file_reader, InvertedIndexReaderType::BKD); + + InvertedIndexIterator text_iterator; + text_iterator.add_reader(InvertedIndexReaderType::FULLTEXT, text_reader); + RecordingDirectInvertedIndexIterator number_iterator; + number_iterator.add_reader(InvertedIndexReaderType::BKD, number_reader); + + std::unordered_map data_type_with_names; + data_type_with_names.emplace( + "body", IndexFieldNameAndTypePair {"body", std::make_shared()}); + data_type_with_names.emplace( + "age", IndexFieldNameAndTypePair {"age", std::make_shared()}); + std::unordered_map iterators; + iterators["body"] = &text_iterator; + iterators["age"] = &number_iterator; + + TSearchClause age_clause = make_leaf_clause("TERM", "42"); + age_clause.field_name = "age"; + TSearchParam search_param; + search_param.original_dsl = "age:42"; + search_param.root = age_clause; + ASSERT_TRUE(insert_search_dsl_cache(cache_guard.get(), number_file_reader, search_param, + make_bitmap({1})) + .ok()); + + InvertedIndexResultBitmap result; + auto status = function_search->evaluate_inverted_index_with_search_param( + search_param, data_type_with_names, iterators, 4, result, true); + + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_NE(nullptr, result.get_data_bitmap()); + expect_bitmap_eq(*result.get_data_bitmap(), {1}); + EXPECT_EQ(0, text_reader->query_calls); + EXPECT_EQ(0, number_iterator.read_calls); +} + TEST_F(FunctionSearchTest, TestBuildLeafQueryDirectUnknownClauseUsesLeafMapper) { TSearchClause clause; clause.clause_type = "PHRASE"; @@ -1942,6 +2647,7 @@ TEST_F(FunctionSearchTest, TestBuildLeafQueryDirectUnknownClauseUsesLeafMapper) binding.column_type = bool_type; binding.query_type = InvertedIndexQueryType::MATCH_PHRASE_QUERY; binding.state = SearchFieldBindingState::BOUND; + binding.execution_mode = SearchFieldExecutionMode::DIRECT_INDEX; TabletIndex index_meta; binding.inverted_reader = std::make_shared(&index_meta); @@ -2012,6 +2718,7 @@ TEST_F(FunctionSearchTest, TestBuildLeafQueryVariantBoolUsesDirectIndexReader) { binding.column_type = bool_type; binding.query_type = InvertedIndexQueryType::MATCH_ANY_QUERY; binding.state = SearchFieldBindingState::BOUND; + binding.execution_mode = SearchFieldExecutionMode::DIRECT_INDEX; TabletIndex index_meta; binding.inverted_reader = std::make_shared(&index_meta); @@ -2071,6 +2778,7 @@ TEST_F(FunctionSearchTest, TestBuildLeafQueryVariantNestedIntUsesDirectIndexRead binding.column_type = int_type; binding.query_type = InvertedIndexQueryType::MATCH_ANY_QUERY; binding.state = SearchFieldBindingState::BOUND; + binding.execution_mode = SearchFieldExecutionMode::DIRECT_INDEX; TabletIndex index_meta; binding.inverted_reader = std::make_shared(&index_meta); diff --git a/be/test/exprs/vsearch_expr_test.cpp b/be/test/exprs/vsearch_expr_test.cpp index 26316dbcbac4f5..b10fa64b6384ef 100644 --- a/be/test/exprs/vsearch_expr_test.cpp +++ b/be/test/exprs/vsearch_expr_test.cpp @@ -33,7 +33,11 @@ #include "exprs/vexpr_context.h" #include "exprs/vliteral.h" #include "exprs/vsearch.h" +#include "storage/index/index_file_reader.h" #include "storage/index/index_iterator.h" +#include "storage/index/inverted/inverted_index_iterator.h" +#include "storage/index/inverted/inverted_index_parser.h" +#include "storage/index/inverted/inverted_index_reader.h" #include "storage/segment/variant/nested_group_provider.h" #include "storage/tablet/tablet_schema.h" @@ -46,6 +50,7 @@ #if defined(__clang__) #pragma clang diagnostic pop #endif +#include "exprs/vcompound_pred.h" namespace doris { @@ -541,7 +546,7 @@ TEST_F(VSearchExprTest, TestEvaluateInvertedIndexEmptyDSL) { VExprContext context(dummy_expr); auto status = vsearch_expr->evaluate_inverted_index(&context, 100); EXPECT_FALSE(status.ok()); - EXPECT_TRUE(status.code() == ErrorCode::INVALID_ARGUMENT); + EXPECT_EQ(ErrorCode::INVALID_ARGUMENT, status.code()); EXPECT_TRUE(status.to_string().find("search DSL is empty") != std::string::npos); } @@ -1293,6 +1298,7 @@ TEST_F(VSearchExprTest, TestEvaluateInvertedIndexWithEmptyDSL) { auto status = vsearch_expr->evaluate_inverted_index(&context, 100); EXPECT_FALSE(status.ok()); // Should return error due to empty DSL EXPECT_EQ(status.code(), ErrorCode::INVALID_ARGUMENT); + EXPECT_NE(status.to_string().find("search DSL is empty"), std::string::npos); } TEST_F(VSearchExprTest, FastExecuteReturnsPrecomputedColumn) { @@ -1335,6 +1341,7 @@ TEST_F(VSearchExprTest, EvaluateInvertedIndexFailsWithoutStorageType) { auto status = expr->evaluate_inverted_index(context.get(), 128); EXPECT_FALSE(status.ok()); EXPECT_EQ(ErrorCode::INTERNAL_ERROR, status.code()); + EXPECT_NE(status.to_string().find("storage_name_type not found"), std::string::npos); } TEST_F(VSearchExprTest, EvaluateInvertedIndexWithUnsupportedChildReturnsError) { @@ -1353,6 +1360,20 @@ TEST_F(VSearchExprTest, EvaluateInvertedIndexWithUnsupportedChildReturnsError) { auto status = expr->evaluate_inverted_index(context.get(), 64); EXPECT_FALSE(status.ok()); EXPECT_EQ(ErrorCode::INVALID_ARGUMENT, status.code()); + EXPECT_NE(status.to_string().find("Unsupported child node type"), std::string::npos); + + TExprNode compound_node; + compound_node.__set_type(test_node.type); + compound_node.__set_node_type(TExprNodeType::COMPOUND_PRED); + compound_node.__set_opcode(TExprOpcode::COMPOUND_AND); + compound_node.__set_num_children(1); + compound_node.__set_is_nullable(false); + auto compound = VCompoundPred::create_shared(compound_node); + compound->add_child(expr); + status = compound->evaluate_inverted_index(context.get(), 64); + EXPECT_FALSE(status.ok()); + EXPECT_EQ(ErrorCode::INVALID_ARGUMENT, status.code()); + EXPECT_NE(status.to_string().find("Unsupported child node type"), std::string::npos); } TEST_F(VSearchExprTest, EvaluateInvertedIndexHandlesMissingIterators) { @@ -1452,7 +1473,7 @@ TEST_F(VSearchExprTest, EvaluateInvertedIndexNestedFallbackReturnsNotSupportedIn ASSERT_TRUE(provider != nullptr); if (!provider->should_enable_nested_group_read_path()) { EXPECT_FALSE(status.ok()); - EXPECT_EQ(ErrorCode::NOT_IMPLEMENTED_ERROR, status.code()); + EXPECT_EQ(ErrorCode::INVERTED_INDEX_NOT_SUPPORTED, status.code()); EXPECT_TRUE(status.to_string().find("NestedGroup support") != std::string::npos); } else { EXPECT_FALSE(status.ok()); @@ -1478,8 +1499,97 @@ TEST_F(VSearchExprTest, EvaluateInvertedIndexPropagatesFunctionFailure) { auto status = expr->evaluate_inverted_index(context.get(), 256); EXPECT_FALSE(status.ok()); - EXPECT_EQ(ErrorCode::INVERTED_INDEX_FILE_NOT_FOUND, status.code()); + EXPECT_EQ(ErrorCode::INVERTED_INDEX_NOT_SUPPORTED, status.code()); + EXPECT_FALSE(status_map[0][expr.get()]); +} + +TEST_F(VSearchExprTest, EvaluateInvertedIndexRejectsSearchFallback) { + test_node.search_param.root.clause_type = "WILDCARD"; + test_node.search_param.root.value = "hello*"; + test_node.search_param.field_bindings[0].index_properties[INVERTED_INDEX_PARSER_KEY] = + INVERTED_INDEX_PARSER_ENGLISH; + test_node.search_param.field_bindings[0].__isset.index_properties = true; + + auto expr = VSearchExpr::create_shared(test_node); + expr->add_child(create_slot_ref(0, "title")); + + TabletIndexPB index_pb; + index_pb.set_index_type(IndexType::INVERTED); + index_pb.set_index_id(1); + index_pb.set_index_name("search_fallback_index"); + index_pb.add_col_unique_id(1); + (*index_pb.mutable_properties())[INVERTED_INDEX_PARSER_KEY] = INVERTED_INDEX_PARSER_STANDARD; + TabletIndex index_meta; + index_meta.init_from_pb(index_pb); + auto index_file_reader = std::make_shared( + nullptr, "/tmp/search_fallback_idx", InvertedIndexStorageFormatPB::V2); + auto reader = std::make_shared(&index_meta, index_file_reader); + auto iterator = std::make_unique(); + iterator->add_reader(segment_v2::InvertedIndexReaderType::FULLTEXT, reader); + + std::vector col_ids = {0}; + std::vector> index_iterators; + index_iterators.emplace_back(std::move(iterator)); + std::vector storage_types; + storage_types.emplace_back("title", std::make_shared()); + std::unordered_map> status_map; + status_map[0][expr.get()] = false; + + auto inverted_ctx = make_inverted_context(col_ids, index_iterators, storage_types, status_map); + auto context = std::make_shared(expr); + context->set_index_context(inverted_ctx); + + auto status = expr->evaluate_inverted_index(context.get(), 256); + EXPECT_FALSE(status.ok()); + EXPECT_EQ(ErrorCode::INVERTED_INDEX_NOT_SUPPORTED, status.code()); + EXPECT_NE(status.to_string().find("cannot fall back"), std::string::npos); + EXPECT_NE(status.to_string().find("No inverted index found for analyzer"), std::string::npos); EXPECT_FALSE(status_map[0][expr.get()]); + + for (TExprOpcode::type opcode : {TExprOpcode::COMPOUND_AND, TExprOpcode::COMPOUND_OR}) { + TExprNode compound_node; + compound_node.__set_type(test_node.type); + compound_node.__set_node_type(TExprNodeType::COMPOUND_PRED); + compound_node.__set_opcode(opcode); + compound_node.__set_num_children(1); + compound_node.__set_is_nullable(false); + auto compound = VCompoundPred::create_shared(compound_node); + compound->add_child(expr); + + status = compound->evaluate_inverted_index(context.get(), 256); + EXPECT_FALSE(status.ok()); + EXPECT_EQ(ErrorCode::INVERTED_INDEX_NOT_SUPPORTED, status.code()); + EXPECT_NE(status.to_string().find("No inverted index found for analyzer"), + std::string::npos); + } + + test_node.search_param.root.clause_type = "TERM"; + test_node.search_param.root.value = "hello"; + test_node.search_param.field_bindings[0].index_properties[INVERTED_INDEX_PARSER_KEY] = + INVERTED_INDEX_PARSER_STANDARD; + auto snii_expr = VSearchExpr::create_shared(test_node); + snii_expr->add_child(create_slot_ref(0, "title")); + + auto snii_file_reader = std::make_shared( + nullptr, "/tmp/search_fallback_snii_idx", InvertedIndexStorageFormatPB::SNII); + auto snii_reader = + std::make_shared(&index_meta, snii_file_reader); + auto snii_iterator = std::make_unique(); + snii_iterator->add_reader(segment_v2::InvertedIndexReaderType::FULLTEXT, snii_reader); + + std::vector> snii_index_iterators; + snii_index_iterators.emplace_back(std::move(snii_iterator)); + std::unordered_map> snii_status_map; + snii_status_map[0][snii_expr.get()] = false; + auto snii_inverted_ctx = + make_inverted_context(col_ids, snii_index_iterators, storage_types, snii_status_map); + auto snii_context = std::make_shared(snii_expr); + snii_context->set_index_context(snii_inverted_ctx); + + status = snii_expr->evaluate_inverted_index(snii_context.get(), 256); + EXPECT_FALSE(status.ok()); + EXPECT_EQ(ErrorCode::INVERTED_INDEX_NOT_SUPPORTED, status.code()); + EXPECT_NE(status.to_string().find("supports only WILDCARD"), std::string::npos); } // Note: Full testing with actual IndexExecContext and real iterators diff --git a/be/test/io/cache/block_file_cache_profile_reporter_test.cpp b/be/test/io/cache/block_file_cache_profile_reporter_test.cpp index 975f1e8eddd88b..246cc539952f35 100644 --- a/be/test/io/cache/block_file_cache_profile_reporter_test.cpp +++ b/be/test/io/cache/block_file_cache_profile_reporter_test.cpp @@ -64,6 +64,10 @@ io::FileCacheStatistics make_file_cache_stats(int64_t multiplier) { stats.num_peer_race_s3_win = multiplier * 38; stats.num_peer_lazy_fetch = multiplier * 39; stats.peer_lazy_fetch_timer = multiplier * 40; + stats.inverted_index_request_bytes = multiplier * 41; + stats.inverted_index_read_bytes = multiplier * 42; + stats.inverted_index_range_read_count = multiplier * 43; + stats.inverted_index_serial_read_rounds = multiplier * 44; return stats; } @@ -114,6 +118,10 @@ void expect_file_cache_stats_eq(const io::FileCacheStatistics& actual, EXPECT_EQ(actual.num_peer_race_s3_win, expected.num_peer_race_s3_win); EXPECT_EQ(actual.num_peer_lazy_fetch, expected.num_peer_lazy_fetch); EXPECT_EQ(actual.peer_lazy_fetch_timer, expected.peer_lazy_fetch_timer); + EXPECT_EQ(actual.inverted_index_request_bytes, expected.inverted_index_request_bytes); + EXPECT_EQ(actual.inverted_index_read_bytes, expected.inverted_index_read_bytes); + EXPECT_EQ(actual.inverted_index_range_read_count, expected.inverted_index_range_read_count); + EXPECT_EQ(actual.inverted_index_serial_read_rounds, expected.inverted_index_serial_read_rounds); } } // namespace @@ -163,6 +171,14 @@ TEST(FileCacheProfileReporterTest, ReporterAggregatesDeltaReportsToExactFinalTot after_second_report.cross_cg_peer_io_timer); EXPECT_EQ(profile->get_counter("PeerLazyFetchTime")->value(), after_second_report.peer_lazy_fetch_timer); + EXPECT_EQ(profile->get_counter("InvertedIndexRequestBytes")->value(), + after_second_report.inverted_index_request_bytes); + EXPECT_EQ(profile->get_counter("InvertedIndexReadBytes")->value(), + after_second_report.inverted_index_read_bytes); + EXPECT_EQ(profile->get_counter("InvertedIndexRangeReadCount")->value(), + after_second_report.inverted_index_range_read_count); + EXPECT_EQ(profile->get_counter("InvertedIndexSerialReadRounds")->value(), + after_second_report.inverted_index_serial_read_rounds); } } // namespace doris diff --git a/be/test/runtime/fragment_mgr_cross_cluster_cancel_test.cpp b/be/test/runtime/fragment_mgr_cross_cluster_cancel_test.cpp index 24c77261183f84..c178728712d6f2 100644 --- a/be/test/runtime/fragment_mgr_cross_cluster_cancel_test.cpp +++ b/be/test/runtime/fragment_mgr_cross_cluster_cancel_test.cpp @@ -18,11 +18,15 @@ #include #include +#include "common/config.h" #include "runtime/descriptor_helper.h" #include "runtime/exec_env.h" #include "runtime/fragment_mgr.h" #include "runtime/frontend_info.h" +#include "runtime/index_policy/index_policy_mgr.h" #include "runtime/workload_group/workload_group_manager.h" +#include "storage/id_manager.h" +#include "util/defer_op.h" namespace doris { diff --git a/be/test/runtime/index_policy/index_policy_mgr_test.cpp b/be/test/runtime/index_policy/index_policy_mgr_test.cpp index 923690ef3612c8..4485217652bb11 100644 --- a/be/test/runtime/index_policy/index_policy_mgr_test.cpp +++ b/be/test/runtime/index_policy/index_policy_mgr_test.cpp @@ -19,10 +19,34 @@ #include +#include +#include +#include +#include + +#include "common/config.h" #include "runtime/exec_env.h" #include "storage/index/inverted/analysis_factory_mgr.h" +#include "storage/index/inverted/analyzer/analyzer.h" +#include "storage/index/inverted/analyzer/segment_analyzer_context.h" +#include "storage/index/inverted/common_grams/common_grams_key_codec.h" +#include "storage/index/inverted/common_grams/common_grams_segment_metadata.h" +#include "storage/index/inverted/common_grams/common_word_set.h" +#include "util/defer_op.h" namespace doris { +namespace { + +TIndexPolicy common_grams_policy(int64_t id, std::string name) { + TIndexPolicy policy; + policy.id = id; + policy.name = std::move(name); + policy.type = TIndexPolicyType::TOKEN_FILTER; + policy.properties["type"] = "common_grams"; + return policy; +} + +} // namespace class IndexPolicyMgrTest : public testing::Test { protected: @@ -176,4 +200,220 @@ TEST_F(IndexPolicyMgrTest, TestTokenFilterProcessing) { ASSERT_NE(emptyAnalyzer, nullptr); } +TEST_F(IndexPolicyMgrTest, CommonGramsProviderRetainsImmutablePurposeConfiguration) { + TIndexPolicy tokenizer; + tokenizer.id = 20; + tokenizer.name = "cg_tokenizer"; + tokenizer.type = TIndexPolicyType::TOKENIZER; + tokenizer.properties["type"] = "char_group"; + tokenizer.properties["tokenize_on_chars"] = "[whitespace]"; + + auto common_grams = common_grams_policy(21, "cg_filter"); + + TIndexPolicy analyzer_policy; + analyzer_policy.id = 22; + analyzer_policy.name = "cg_analyzer"; + analyzer_policy.type = TIndexPolicyType::ANALYZER; + analyzer_policy.properties["tokenizer"] = "cg_tokenizer"; + analyzer_policy.properties["token_filter"] = "lowercase,cg_filter"; + mgr.apply_policy_changes({tokenizer, common_grams, analyzer_policy}, {}); + + auto provider = mgr.get_analyzer_provider_by_name("cg_analyzer"); + ASSERT_NE(provider, nullptr); + auto analyze = [](const segment_v2::inverted_index::AnalyzerProviderPtr& analyzer_provider, + segment_v2::inverted_index::AnalysisPurpose purpose, std::string_view text) { + auto analyzer = analyzer_provider->get_analyzer(purpose); + auto reader = std::make_shared>(); + reader->init(text.data(), static_cast(text.size()), true); + return segment_v2::inverted_index::InvertedIndexAnalyzer::get_analyse_result( + reader, analyzer.get()); + }; + + auto index = analyze(provider, segment_v2::inverted_index::AnalysisPurpose::kIndex, + "Man of the Year"); + auto plain = analyze(provider, segment_v2::inverted_index::AnalysisPurpose::kPlainQuery, + "Man of the Year"); + auto exact = analyze(provider, segment_v2::inverted_index::AnalysisPurpose::kExactPhraseQuery, + "Man of the Year"); + auto prefix = analyze(provider, segment_v2::inverted_index::AnalysisPurpose::kPhrasePrefixQuery, + "the wo"); + EXPECT_EQ(index.size(), 7); + EXPECT_EQ(plain.size(), 4); + EXPECT_EQ(exact.size(), 3); + EXPECT_EQ(prefix.size(), 1); + auto prefix_gram = segment_v2::inverted_index::encode_common_gram("the", "wo"); + ASSERT_TRUE(prefix_gram.has_value()) << prefix_gram.error(); + EXPECT_EQ(prefix.front().get_single_term(), prefix_gram.value()); + + auto single_index = mgr.get_analyzer_by_name( + "cg_analyzer", segment_v2::inverted_index::AnalysisPurpose::kIndex); + auto single_reader = std::make_shared>(); + const std::string single_input = "Man of the Year"; + single_reader->init(single_input.data(), static_cast(single_input.size()), true); + EXPECT_EQ(segment_v2::inverted_index::InvertedIndexAnalyzer::get_analyse_result( + single_reader, single_index.get()) + .size(), + 7); +} + +TEST_F(IndexPolicyMgrTest, AnalyzerProviderPreservesPurposeInsensitiveNormalizers) { + auto builtin = mgr.get_analyzer_provider_by_name("lowercase"); + auto builtin_analyzer = + builtin->get_analyzer(segment_v2::inverted_index::AnalysisPurpose::kPlainQuery); + EXPECT_EQ(builtin->get_analyzer(segment_v2::inverted_index::AnalysisPurpose::kIndex), + builtin_analyzer); + + TIndexPolicy normalizer; + normalizer.id = 23; + normalizer.name = "test_normalizer"; + normalizer.type = TIndexPolicyType::NORMALIZER; + normalizer.properties["token_filter"] = "lowercase"; + mgr.apply_policy_changes({normalizer}, {}); + + auto configured = mgr.get_analyzer_provider_by_name("test_normalizer"); + auto configured_analyzer = + configured->get_analyzer(segment_v2::inverted_index::AnalysisPurpose::kPlainQuery); + EXPECT_EQ(configured->get_analyzer(segment_v2::inverted_index::AnalysisPurpose::kIndex), + configured_analyzer); +} + +TEST_F(IndexPolicyMgrTest, FindsFreshAnalyzerProviderBySegmentBaseFingerprint) { + using segment_v2::inverted_index::AnalysisPurpose; + using segment_v2::inverted_index::InvertedIndexAnalyzer; + + const std::map slash_to_space = { + {INVERTED_INDEX_PARSER_CHAR_FILTER_TYPE, INVERTED_INDEX_CHAR_FILTER_CHAR_REPLACE}, + {INVERTED_INDEX_PARSER_CHAR_FILTER_PATTERN, "/"}, + {INVERTED_INDEX_PARSER_CHAR_FILTER_REPLACEMENT, " "}}; + auto configured = mgr.get_analyzer_provider_by_name("analyzer1", slash_to_space); + const std::string segment_fingerprint(configured->base_analyzer_fingerprint()); + + auto first = mgr.get_analyzer_provider_by_base_fingerprint(segment_fingerprint, slash_to_space); + ASSERT_NE(first, nullptr); + EXPECT_EQ(first->base_analyzer_fingerprint(), segment_fingerprint); + EXPECT_EQ(mgr.get_analyzer_provider_by_base_fingerprint(segment_fingerprint), nullptr); + EXPECT_EQ(mgr.get_analyzer_provider_by_base_fingerprint("unknown", slash_to_space), nullptr); + + auto second = + mgr.get_analyzer_provider_by_base_fingerprint(segment_fingerprint, slash_to_space); + ASSERT_NE(second, nullptr); + EXPECT_NE(first, second); + EXPECT_NE(first->get_analyzer(AnalysisPurpose::kPlainQuery), + second->get_analyzer(AnalysisPurpose::kPlainQuery)); + + mgr.apply_policy_changes({}, {5}); + auto reader = std::make_shared>(); + const std::string input = "ASCII TERM"; + reader->init(input.data(), static_cast(input.size()), true); + const auto terms = InvertedIndexAnalyzer::get_analyse_result( + reader, first->get_analyzer(AnalysisPurpose::kPlainQuery).get()); + ASSERT_EQ(terms.size(), 2U); + EXPECT_EQ(terms[0].get_single_term(), "ascii"); + EXPECT_EQ(terms[1].get_single_term(), "term"); +} + +TEST_F(IndexPolicyMgrTest, RebuildsQueryContextForPersistedSegmentAnalyzer) { + const std::map slash_to_space = { + {INVERTED_INDEX_PARSER_CHAR_FILTER_TYPE, INVERTED_INDEX_CHAR_FILTER_CHAR_REPLACE}, + {INVERTED_INDEX_PARSER_CHAR_FILTER_PATTERN, "/"}, + {INVERTED_INDEX_PARSER_CHAR_FILTER_REPLACEMENT, " "}}; + auto segment_provider = mgr.get_analyzer_provider_by_name("analyzer1", slash_to_space); + const std::string segment_fingerprint(segment_provider->base_analyzer_fingerprint()); + + segment_v2::inverted_index::Settings tokenizer_settings; + tokenizer_settings.set("tokenize_on_chars", "[whitespace]"); + segment_v2::inverted_index::CustomAnalyzerConfig::Builder request_builder; + request_builder.with_tokenizer_config("char_group", tokenizer_settings); + auto request_provider = std::make_shared( + request_builder.build()); + ASSERT_NE(request_provider->base_analyzer_fingerprint(), segment_fingerprint); + + InvertedIndexAnalyzerCtx request_context; + request_context.analyzer_name = "request_analyzer"; + request_context.parser_type = InvertedIndexParserType::PARSER_ENGLISH; + request_context.char_filter_map = {{"stale", "filter"}}; + request_context.analyzer = request_provider->get_analyzer( + segment_v2::inverted_index::AnalysisPurpose::kPlainQuery); + request_context.analyzer_provider = request_provider; + request_context.common_grams_identity = segment_v2::inverted_index::CommonGramsQueryIdentity { + .common_grams_dictionary_identity = "stale-dictionary", + .base_analyzer_fingerprint = std::string(request_provider->base_analyzer_fingerprint()), + .common_grams_fingerprint = "stale-common-grams"}; + + const std::map physical_properties = slash_to_space; + auto rebuilt = segment_v2::inverted_index::maybe_rebuild_segment_analyzer_context( + &request_context, segment_fingerprint, physical_properties, &mgr); + ASSERT_TRUE(rebuilt.has_value()) << rebuilt.error(); + ASSERT_TRUE(rebuilt->has_value()); + const auto& effective = rebuilt->value(); + EXPECT_EQ(effective.analyzer_name, request_context.analyzer_name); + EXPECT_EQ(effective.parser_type, request_context.parser_type); + EXPECT_EQ(effective.char_filter_map, slash_to_space); + EXPECT_EQ(effective.analyzer, nullptr); + ASSERT_NE(effective.analyzer_provider, nullptr); + EXPECT_EQ(effective.analyzer_provider->base_analyzer_fingerprint(), segment_fingerprint); + EXPECT_FALSE(effective.common_grams_identity.has_value()); + EXPECT_NE(effective.analyzer_provider, segment_provider); + + InvertedIndexAnalyzerCtx matching_context = request_context; + matching_context.analyzer_provider = segment_provider; + auto unchanged = segment_v2::inverted_index::maybe_rebuild_segment_analyzer_context( + &matching_context, segment_fingerprint, physical_properties, &mgr); + ASSERT_TRUE(unchanged.has_value()) << unchanged.error(); + EXPECT_FALSE(unchanged->has_value()); + + auto unavailable = segment_v2::inverted_index::maybe_rebuild_segment_analyzer_context( + &request_context, "missing-segment-fingerprint", physical_properties, &mgr); + ASSERT_FALSE(unavailable.has_value()); + EXPECT_EQ(unavailable.error().code(), ErrorCode::INVERTED_INDEX_BYPASS); +} + +TEST_F(IndexPolicyMgrTest, SegmentAnalyzerAdmissionKeepsLegacyRequestWithoutMetadata) { + auto admitted = segment_v2::inverted_index::maybe_rebuild_segment_analyzer_context( + nullptr, std::optional {}, {}, + nullptr); + + ASSERT_TRUE(admitted.has_value()) << admitted.error(); + EXPECT_FALSE(admitted->has_value()); +} + +TEST_F(IndexPolicyMgrTest, SegmentAnalyzerAdmissionBypassesTypedMetadataWithoutBaseFingerprint) { + segment_v2::inverted_index::CommonGramsSegmentMetadata metadata; + auto admitted = segment_v2::inverted_index::maybe_rebuild_segment_analyzer_context( + nullptr, std::optional {metadata}, {}, &mgr); + + ASSERT_FALSE(admitted.has_value()); + EXPECT_EQ(admitted.error().code(), ErrorCode::INVERTED_INDEX_BYPASS); +} + +TEST_F(IndexPolicyMgrTest, SegmentAnalyzerAdmissionRebuildsTypedMetadata) { + const std::map slash_to_space = { + {INVERTED_INDEX_PARSER_CHAR_FILTER_TYPE, INVERTED_INDEX_CHAR_FILTER_CHAR_REPLACE}, + {INVERTED_INDEX_PARSER_CHAR_FILTER_PATTERN, "/"}, + {INVERTED_INDEX_PARSER_CHAR_FILTER_REPLACEMENT, " "}}; + auto segment_provider = mgr.get_analyzer_provider_by_name("analyzer1", slash_to_space); + const std::string segment_fingerprint(segment_provider->base_analyzer_fingerprint()); + + segment_v2::inverted_index::Settings tokenizer_settings; + tokenizer_settings.set("tokenize_on_chars", "[whitespace]"); + segment_v2::inverted_index::CustomAnalyzerConfig::Builder request_builder; + request_builder.with_tokenizer_config("char_group", tokenizer_settings); + auto request_provider = std::make_shared( + request_builder.build()); + + InvertedIndexAnalyzerCtx request_context; + request_context.analyzer_provider = request_provider; + request_context.analyzer = request_provider->get_analyzer( + segment_v2::inverted_index::AnalysisPurpose::kPlainQuery); + segment_v2::inverted_index::CommonGramsSegmentMetadata metadata; + metadata.base_analyzer_fingerprint = segment_fingerprint; + + auto rebuilt = segment_v2::inverted_index::maybe_rebuild_segment_analyzer_context( + &request_context, std::optional {metadata}, slash_to_space, &mgr); + + ASSERT_TRUE(rebuilt.has_value()) << rebuilt.error(); + ASSERT_TRUE(rebuilt->has_value()); + EXPECT_EQ(rebuilt->value().analyzer_provider->base_analyzer_fingerprint(), segment_fingerprint); +} + } // namespace doris \ No newline at end of file diff --git a/be/test/storage/compaction/cloud_index_change_compaction_test.cpp b/be/test/storage/compaction/cloud_index_change_compaction_test.cpp index 663dd23aebb2f7..9ff7db5b18e231 100644 --- a/be/test/storage/compaction/cloud_index_change_compaction_test.cpp +++ b/be/test/storage/compaction/cloud_index_change_compaction_test.cpp @@ -19,6 +19,8 @@ #include +#include + #include "cloud/cloud_base_compaction.h" #include "cloud/cloud_cumulative_compaction.h" #include "cpp/sync_point.h" @@ -73,6 +75,95 @@ class CloudIndexChangeCompactionTest : public testing::Test { return origin.find(sub_str) != std::string::npos; } + CloudTabletSPtr create_tablet_for_index_compaction_gate( + InvertedIndexStorageFormatPB storage_format) { + TabletSchemaPB schema_pb; + schema_pb.set_keys_type(KeysType::DUP_KEYS); + schema_pb.set_inverted_index_storage_format(storage_format); + schema_pb.set_schema_version(1); + + ColumnPB* column_pb = schema_pb.add_column(); + column_pb->set_unique_id(1); + column_pb->set_name("body"); + column_pb->set_type("STRING"); + column_pb->set_is_nullable(true); + + auto add_index = [&schema_pb](int64_t index_id, std::string_view index_name) { + TabletIndexPB* index_pb = schema_pb.add_index(); + index_pb->set_index_id(index_id); + index_pb->set_index_name(std::string(index_name)); + index_pb->set_index_type(IndexType::INVERTED); + index_pb->add_col_unique_id(1); + (*index_pb->mutable_properties())[INVERTED_INDEX_PARSER_KEY] = + INVERTED_INDEX_PARSER_UNICODE; + }; + add_index(11001, "idx_to_drop"); + add_index(11002, "idx_to_keep"); + + auto input_schema = std::make_shared(); + input_schema->init_from_pb(schema_pb); + auto input_rowset_meta = std::make_shared(); + init_rs_meta(input_rowset_meta, 2, 2); + input_rowset_meta->set_num_segments(1); + input_rowset_meta->set_tablet_schema(input_schema); + RowsetSharedPtr input_rowset = + std::make_shared(input_schema, input_rowset_meta, ""); + + TabletMetaPB tablet_meta_pb; + tablet_meta_pb.set_tablet_id(1000); + tablet_meta_pb.set_schema_hash(123456); + tablet_meta_pb.set_tablet_state(PB_RUNNING); + *tablet_meta_pb.mutable_tablet_uid() = TabletUid::gen_uid().to_proto(); + auto tablet_meta = std::make_shared(); + tablet_meta->init_from_pb(tablet_meta_pb); + CloudTabletSPtr tablet = std::make_shared(*_engine, tablet_meta); + tablet->_rs_version_map[Version(2, 2)] = input_rowset; + return tablet; + } + + void verify_index_compaction_gate(InvertedIndexStorageFormatPB storage_format, + bool initially_enabled, bool expected_enabled) { + CloudTabletSPtr tablet = create_tablet_for_index_compaction_gate(storage_format); + + TColumn body_column; + body_column.__set_column_name("body"); + body_column.__set_col_unique_id(1); + TColumnType body_type; + body_type.__set_type(TPrimitiveType::STRING); + body_column.__set_column_type(body_type); + std::vector columns {body_column}; + + TOlapTableIndex surviving_index; + surviving_index.__set_index_id(11002); + surviving_index.__set_index_name("idx_to_keep"); + surviving_index.__set_index_type(TIndexType::INVERTED); + surviving_index.__set_columns({"body"}); + surviving_index.__set_column_unique_ids({1}); + surviving_index.__set_properties( + {{INVERTED_INDEX_PARSER_KEY, INVERTED_INDEX_PARSER_UNICODE}}); + std::vector final_indexes {surviving_index}; + + CloudIndexChangeCompaction compaction(*_engine, tablet, 2, final_indexes, columns); + compaction._enable_inverted_index_compaction = initially_enabled; + auto* sync_point = SyncPoint::get_instance(); + sync_point->set_call_back("CloudMetaMgr::sync_tablet_rowsets", [](auto&& outcome) { + auto* result = try_any_cast_ret(outcome); + result->second = true; + result->first = Status::OK(); + }); + Status prepare_status = compaction.prepare_compact(); + sync_point->clear_call_back("CloudMetaMgr::sync_tablet_rowsets"); + + ASSERT_TRUE(prepare_status.ok()) << prepare_status; + ASSERT_EQ(compaction._input_rowsets.size(), 1); + EXPECT_EQ(compaction._enable_inverted_index_compaction, expected_enabled); + ASSERT_TRUE(compaction.rebuild_tablet_schema().ok()); + ASSERT_EQ(compaction._final_tablet_schema->get_inverted_index_storage_format(), + storage_format); + ASSERT_EQ(compaction._final_tablet_schema->inverted_indexes().size(), 1); + EXPECT_EQ(compaction._final_tablet_schema->inverted_indexes().front()->index_id(), 11002); + } + public: std::unique_ptr _engine; }; @@ -413,6 +504,14 @@ TEST_F(CloudIndexChangeCompactionTest, basic_compaction_test) { ASSERT_TRUE(in_predicates[0].values().size() == 2); } +TEST_F(CloudIndexChangeCompactionTest, snii_drop_index_enables_native_index_compaction) { + verify_index_compaction_gate(InvertedIndexStorageFormatPB::SNII, true, true); + verify_index_compaction_gate(InvertedIndexStorageFormatPB::SNII, false, false); + verify_index_compaction_gate(InvertedIndexStorageFormatPB::V1, true, false); + verify_index_compaction_gate(InvertedIndexStorageFormatPB::V2, true, false); + verify_index_compaction_gate(InvertedIndexStorageFormatPB::V3, true, false); +} + TEST_F(CloudIndexChangeCompactionTest, test_cloud_tablet) { TabletSchemaPB schema_pb; schema_pb.set_keys_type(KeysType::DUP_KEYS); @@ -524,4 +623,4 @@ TEST_F(CloudIndexChangeCompactionTest, test_cloud_tablet) { } } -} // namespace doris \ No newline at end of file +} // namespace doris diff --git a/be/test/storage/compaction/collection_statistics_test.cpp b/be/test/storage/compaction/collection_statistics_test.cpp index b78355b316ebdd..3f632ce7da7fc3 100644 --- a/be/test/storage/compaction/collection_statistics_test.cpp +++ b/be/test/storage/compaction/collection_statistics_test.cpp @@ -15,14 +15,19 @@ // specific language governing permissions and limitations // under the License. -#include "storage/compaction/collection_statistics.h" +#include "storage/index/inverted/similarity/collection_statistics.h" #include #include #include +#include +#include +#include #include #include +#include +#include #include "common/exception.h" #include "core/data_type/data_type_string.h" @@ -30,14 +35,27 @@ #include "exprs/vexpr.h" #include "exprs/vexpr_context.h" #include "exprs/vliteral.h" +#include "exprs/vsearch.h" #include "exprs/vslot_ref.h" #include "io/fs/local_file_system.h" -#include "storage/compaction/collection_statistics.cpp" +#include "runtime/exec_env.h" +#include "storage/index/index_file_reader.h" +#include "storage/index/index_file_writer.h" +#include "storage/index/index_writer.h" +#include "storage/index/inverted/common_grams/common_grams_key_codec.h" +#include "storage/index/inverted/inverted_index_desc.h" +#include "storage/index/inverted/similarity/collection_statistics.cpp" +#include "storage/index/snii/query/bm25_scorer.h" +#include "storage/index/snii/snii_doris_adapter.h" +#include "storage/index/snii/stats/snii_stats_provider.h" +#include "storage/index/snii/writer/snii_compound_writer.h" +#include "storage/index/snii_query_test_util.h" #include "storage/rowset/rowset.h" #include "storage/rowset/rowset_meta.h" #include "storage/rowset/rowset_reader.h" #include "storage/tablet/tablet_schema.h" #include "testutil/mock/mock_runtime_state.h" +#include "util/slice.h" namespace doris { @@ -48,6 +66,13 @@ class MockVExpr : public VExpr { MockVExpr(TExprNodeType::type node_type) : _mock_node_type(node_type) { if (node_type == TExprNodeType::MATCH_PRED) { _opcode = TExprOpcode::MATCH_PHRASE; + InvertedIndexAnalyzerConfig config; + config.parser_type = InvertedIndexParserType::PARSER_STANDARD; + config.stop_words = "none"; + _analyzer_ctx = std::make_shared(); + _analyzer_ctx->analyzer_provider = + segment_v2::inverted_index::InvertedIndexAnalyzer::create_analyzer_provider( + &config); } } @@ -80,8 +105,37 @@ class MockVExpr : public VExpr { std::string debug_string() const override { return "MockVExpr"; } + const InvertedIndexAnalyzerCtx* query_analyzer_ctx() const override { + return _analyzer_ctx.get(); + } + + void set_analyzer_ctx(InvertedIndexAnalyzerCtxSPtr analyzer_ctx) { + _analyzer_ctx = std::move(analyzer_ctx); + } + + void set_opcode(TExprOpcode::type opcode) { _opcode = opcode; } + private: TExprNodeType::type _mock_node_type; + InvertedIndexAnalyzerCtxSPtr _analyzer_ctx; +}; + +class FixedFingerprintAnalyzerProvider final : public segment_v2::inverted_index::AnalyzerProvider { +public: + FixedFingerprintAnalyzerProvider(std::shared_ptr analyzer, + std::string fingerprint) + : _analyzer(std::move(analyzer)), _fingerprint(std::move(fingerprint)) {} + + std::shared_ptr get_analyzer( + segment_v2::inverted_index::AnalysisPurpose) const override { + return _analyzer; + } + + std::string_view base_analyzer_fingerprint() const override { return _fingerprint; } + +private: + std::shared_ptr _analyzer; + std::string _fingerprint; }; class MockVSlotRef : public VSlotRef { @@ -172,6 +226,7 @@ class MockRowset : public Rowset { int64_t num_segments() const override { return _num_segments; } Result segment_path(int64_t seg_id) override { + _segment_path_requests.push_back(seg_id); if (_segment_paths.find(seg_id) != _segment_paths.end()) { return _segment_paths.at(seg_id); } @@ -184,9 +239,12 @@ class MockRowset : public Rowset { void set_num_segments(int64_t num) { _num_segments = num; } + const std::vector& segment_path_requests() const { return _segment_path_requests; } + private: int64_t _num_segments; std::map _segment_paths; + std::vector _segment_path_requests; }; class MockRowsetReader : public RowsetReader { @@ -283,11 +341,23 @@ class CollectionStatisticsTest : public ::testing::Test { return tablet_schema; } - VExprContextSPtrs create_match_expr_contexts(const std::string& search_term = "search term") { + VExprContextSPtrs create_match_expr_contexts( + const std::string& search_term = "search term", + const std::string& base_analyzer_fingerprint = "") { VExprContextSPtrs contexts; auto match_expr = std::make_shared(TExprNodeType::MATCH_PRED); + if (!base_analyzer_fingerprint.empty()) { + auto analyzer = match_expr->query_analyzer_ctx()->analyzer_provider->get_analyzer( + segment_v2::inverted_index::AnalysisPurpose::kPlainQuery); + auto provider = + std::make_shared( + std::move(analyzer), base_analyzer_fingerprint); + auto analyzer_ctx = std::make_shared(); + analyzer_ctx->analyzer_provider = std::move(provider); + match_expr->set_analyzer_ctx(std::move(analyzer_ctx)); + } auto slot_ref = std::make_shared("content", SlotId(1)); auto literal = std::make_shared(search_term); @@ -320,6 +390,389 @@ class CollectionStatisticsTest : public ::testing::Test { return splits; } + TabletSchemaSPtr create_legacy_v3_schema() { + TabletSchemaPB schema_pb; + schema_pb.set_keys_type(DUP_KEYS); + schema_pb.set_inverted_index_storage_format(InvertedIndexStorageFormatPB::V3); + auto tablet_schema = std::make_shared(); + tablet_schema->init_from_pb(schema_pb); + + TabletColumn column; + column.set_unique_id(1); + column.set_name("content"); + column.set_type(FieldType::OLAP_FIELD_TYPE_STRING); + tablet_schema->append_column(column); + + TabletIndex index; + index._index_id = 1; + index._index_type = IndexType::INVERTED; + index._col_unique_ids.push_back(1); + index._properties["parser"] = "standard"; + index._properties["support_phrase"] = "true"; + tablet_schema->append_index(std::move(index)); + return tablet_schema; + } + + TabletSchemaSPtr create_snii_schema(int64_t index_id = 1) { + TabletSchemaPB schema_pb; + schema_pb.set_keys_type(DUP_KEYS); + schema_pb.set_inverted_index_storage_format(InvertedIndexStorageFormatPB::SNII); + auto tablet_schema = std::make_shared(); + tablet_schema->init_from_pb(schema_pb); + + TabletColumn column; + column.set_unique_id(1); + column.set_name("content"); + column.set_type(FieldType::OLAP_FIELD_TYPE_STRING); + tablet_schema->append_column(column); + + TabletIndex index; + index._index_id = index_id; + index._index_type = IndexType::INVERTED; + index._col_unique_ids.push_back(1); + index._properties["parser"] = "standard"; + index._properties["support_phrase"] = "true"; + tablet_schema->append_index(std::move(index)); + return tablet_schema; + } + + Status write_legacy_v3_segment(const TabletSchemaSPtr& tablet_schema, + const std::string& segment_path) { + std::vector paths; + paths.emplace_back(test_dir_, 1024); + auto tmp_file_dirs = std::make_unique(paths); + RETURN_IF_ERROR(tmp_file_dirs->init()); + ExecEnv::GetInstance()->set_tmp_file_dir(std::move(tmp_file_dirs)); + + const std::string index_path_prefix { + segment_v2::InvertedIndexDescriptor::get_index_file_path_prefix(segment_path)}; + io::FileWriterPtr compound_file; + io::FileWriterOptions options; + auto fs = io::global_local_filesystem(); + RETURN_IF_ERROR(fs->create_file( + segment_v2::InvertedIndexDescriptor::get_index_file_path_v2(index_path_prefix), + &compound_file, &options)); + + segment_v2::IndexFileWriter file_writer(fs, index_path_prefix, "legacy_v3", 0, + InvertedIndexStorageFormatPB::V3, + std::move(compound_file)); + const auto index_metas = tablet_schema->inverted_indexs(1); + DORIS_CHECK(index_metas.size() == 1); + std::unique_ptr column_writer; + RETURN_IF_ERROR(segment_v2::IndexColumnWriter::create( + &tablet_schema->column(0), &column_writer, &file_writer, index_metas[0])); + std::vector values {Slice("alpha beta")}; + RETURN_IF_ERROR(column_writer->add_values("content", values.data(), values.size())); + RETURN_IF_ERROR(column_writer->finish()); + RETURN_IF_ERROR(file_writer.begin_close()); + return file_writer.finish_close(); + } + + Status write_snii_common_grams_segment(const std::string& segment_path, + std::string base_analyzer_fingerprint, + uint64_t scoring_token_count = 3) { + const std::string index_path_prefix { + segment_v2::InvertedIndexDescriptor::get_index_file_path_prefix(segment_path)}; + io::FileWriterPtr file_writer; + io::FileWriterOptions options; + auto fs = io::global_local_filesystem(); + RETURN_IF_ERROR(fs->create_file( + segment_v2::InvertedIndexDescriptor::get_index_file_path_v2(index_path_prefix), + &file_writer, &options)); + + segment_v2::snii_doris::DorisSniiFileWriter adapter(file_writer.get()); + snii::writer::SniiCompoundWriter writer(&adapter); + auto metadata = segment_v2::inverted_index::make_common_grams_segment_metadata( + {.common_grams_dictionary_identity = "test-stopwords-v1", + .base_analyzer_fingerprint = std::move(base_analyzer_fingerprint), + .common_grams_fingerprint = "test-common-grams-v1"}); + metadata.scoring_doc_count = 2; + metadata.scoring_token_count = scoring_token_count; + + snii::writer::TermPostings alpha; + alpha.term = "alpha"; + alpha.docids = {0, 1}; + alpha.freqs = {1, 1}; + alpha.positions_flat = {0, 0}; + + snii::writer::TermPostings beta; + beta.term = "beta"; + beta.docids = {0}; + beta.freqs = {1}; + beta.positions_flat = {1}; + + snii::writer::TermPostings gram; + gram.term = DORIS_TRY(segment_v2::inverted_index::encode_common_gram("alpha", "beta")); + gram.docids = {0}; + gram.freqs = {1}; + gram.positions_flat = {0}; + + snii::writer::SniiIndexInput input; + input.index_id = 1; + input.config = snii::format::IndexConfig::kDocsPositionsScoring; + input.doc_count = 2; + input.encoded_norms = {snii::query::encode_norm(2), snii::query::encode_norm(1)}; + input.terms = {std::move(gram), std::move(alpha), std::move(beta)}; + std::ranges::sort(input.terms, {}, &snii::writer::TermPostings::term); + input.common_grams_metadata = std::move(metadata); + + RETURN_IF_ERROR(writer.add_logical_index(input)); + RETURN_IF_ERROR(writer.finish()); + return file_writer->close(false); + } + + Status write_plain_snii_scoring_segment(const std::string& segment_path) { + const std::string index_path_prefix { + segment_v2::InvertedIndexDescriptor::get_index_file_path_prefix(segment_path)}; + io::FileWriterPtr file_writer; + io::FileWriterOptions options; + auto fs = io::global_local_filesystem(); + RETURN_IF_ERROR(fs->create_file( + segment_v2::InvertedIndexDescriptor::get_index_file_path_v2(index_path_prefix), + &file_writer, &options)); + + segment_v2::snii_doris::DorisSniiFileWriter adapter(file_writer.get()); + snii::writer::SniiCompoundWriter writer(&adapter); + + snii::writer::TermPostings alpha; + alpha.term = "alpha"; + alpha.docids = {0, 1}; + alpha.freqs = {1, 1}; + alpha.positions_flat = {0, 0}; + + snii::writer::TermPostings beta; + beta.term = "beta"; + beta.docids = {0}; + beta.freqs = {1}; + beta.positions_flat = {1}; + + snii::writer::SniiIndexInput input; + input.index_id = 1; + input.config = snii::format::IndexConfig::kDocsPositionsScoring; + input.doc_count = 2; + input.encoded_norms = {snii::query::encode_norm(2), snii::query::encode_norm(1)}; + input.terms = {std::move(alpha), std::move(beta)}; + + RETURN_IF_ERROR(writer.add_logical_index(input)); + RETURN_IF_ERROR(writer.finish()); + return file_writer->close(false); + } + + VExprContextSPtrs create_search_contexts(const std::string& clause_type, + const std::string& value) { + TSearchClause clause; + clause.clause_type = clause_type; + clause.field_name = "content"; + clause.value = value; + clause.__isset.field_name = true; + clause.__isset.value = true; + + return create_search_contexts(std::move(clause)); + } + + VExprContextSPtrs create_search_contexts(TSearchClause root, + std::vector field_bindings = {}) { + TSearchParam search_param; + search_param.root = std::move(root); + search_param.field_bindings = std::move(field_bindings); + + TExprNode node; + node.node_type = TExprNodeType::SEARCH_EXPR; + TTypeNode type_node; + type_node.type = TTypeNodeType::SCALAR; + TScalarType scalar_type; + scalar_type.__set_type(TPrimitiveType::BOOLEAN); + type_node.__set_scalar_type(scalar_type); + TTypeDesc type_desc; + type_desc.types.push_back(type_node); + node.__set_type(type_desc); + node.search_param = std::move(search_param); + node.__isset.search_param = true; + + return {std::make_shared(VSearchExpr::create_shared(node))}; + } + + TabletSchemaSPtr create_tablet_schema_with_keyword_and_fulltext_indexes() { + auto tablet_schema = std::make_shared(); + + TabletColumn column; + column.set_unique_id(1); + column.set_name("content"); + column.set_type(FieldType::OLAP_FIELD_TYPE_STRING); + tablet_schema->append_column(column); + + TabletIndex keyword_index; + keyword_index._index_id = 10; + keyword_index._index_type = IndexType::INVERTED; + keyword_index._col_unique_ids.push_back(1); + tablet_schema->append_index(std::move(keyword_index)); + + TabletIndex fulltext_index; + fulltext_index._index_id = 20; + fulltext_index._index_type = IndexType::INVERTED; + fulltext_index._col_unique_ids.push_back(1); + fulltext_index._properties["parser"] = "standard"; + fulltext_index._properties["support_phrase"] = "true"; + tablet_schema->append_index(std::move(fulltext_index)); + + return tablet_schema; + } + + TabletSchemaSPtr create_array_tablet_schema_with_keyword_and_fulltext_indexes() { + auto tablet_schema = std::make_shared(); + + TabletColumn item; + item.set_type(FieldType::OLAP_FIELD_TYPE_STRING); + TabletColumn column; + column.set_unique_id(1); + column.set_name("content"); + column.set_type(FieldType::OLAP_FIELD_TYPE_ARRAY); + column.add_sub_column(item); + tablet_schema->append_column(column); + + TabletIndex keyword_index; + keyword_index._index_id = 10; + keyword_index._index_type = IndexType::INVERTED; + keyword_index._col_unique_ids.push_back(1); + tablet_schema->append_index(std::move(keyword_index)); + + TabletIndex fulltext_index; + fulltext_index._index_id = 20; + fulltext_index._index_type = IndexType::INVERTED; + fulltext_index._col_unique_ids.push_back(1); + fulltext_index._properties["parser"] = "standard"; + fulltext_index._properties["support_phrase"] = "true"; + tablet_schema->append_index(std::move(fulltext_index)); + + return tablet_schema; + } + + TabletSchemaSPtr create_tablet_schema_with_two_fulltext_indexes() { + auto tablet_schema = std::make_shared(); + + TabletColumn column; + column.set_unique_id(1); + column.set_name("content"); + column.set_type(FieldType::OLAP_FIELD_TYPE_STRING); + tablet_schema->append_column(column); + + for (const auto& [index_id, parser] : {std::pair {10, "standard"}, + std::pair {20, "english"}}) { + TabletIndex index; + index._index_id = index_id; + index._index_type = IndexType::INVERTED; + index._col_unique_ids.push_back(1); + index._properties["parser"] = parser; + index._properties["support_phrase"] = "true"; + tablet_schema->append_index(std::move(index)); + } + + return tablet_schema; + } + + VExprContextSPtrs create_reserved_exact_search_contexts() { + return create_search_contexts( + "EXACT", std::string(segment_v2::inverted_index::CG_V1_MARKER) + "user"); + } + + void expect_no_collected_tokens(const std::wstring& field_name) { + EXPECT_THROW(stats_->get_total_term_cnt_by_col(field_name), Exception); + } + + void expect_collected_tokens(const std::wstring& field_name, uint64_t token_count) { + EXPECT_EQ(stats_->get_total_term_cnt_by_col(field_name), token_count); + } + + void expect_no_collected_term(const std::wstring& field_name, const std::wstring& term) { + EXPECT_THROW(stats_->get_term_doc_freq_by_col(field_name, term), Exception); + } + + void expect_collected_term(const std::wstring& field_name, const std::wstring& term, + uint64_t doc_frequency) { + EXPECT_EQ(stats_->get_term_doc_freq_by_col(field_name, term), doc_frequency); + } + + struct SniiScoringFieldInput { + SniiScoringFieldInput( + std::wstring field_name, + std::optional metadata, + uint64_t index_doc_count, bool has_semantic_norms, + std::optional expected_base_analyzer_fingerprint = std::nullopt) + : field_name(std::move(field_name)), + metadata(std::move(metadata)), + index_doc_count(index_doc_count), + physical_sum_total_term_freq(this->metadata ? this->metadata->scoring_token_count + : 0), + has_semantic_norms(has_semantic_norms), + expected_base_analyzer_fingerprint( + std::move(expected_base_analyzer_fingerprint)) {} + + std::wstring field_name; + std::optional metadata; + uint64_t index_doc_count = 0; + uint64_t physical_sum_total_term_freq = 0; + bool has_scoring_tier = true; + bool has_positions = true; + bool has_semantic_norms = false; + std::optional expected_base_analyzer_fingerprint; + }; + + Status stage_snii_fields_for_test( + CollectionStatistics* statistics, const std::vector& fields, + CollectionStatistics::SniiScoringSegmentAccumulator* segment_accumulator) { + for (const auto& field : fields) { + segment_v2::inverted_index::PlainTermKeyVersion key_version; + const std::string_view expected_base_analyzer_fingerprint = + field.expected_base_analyzer_fingerprint.has_value() + ? *field.expected_base_analyzer_fingerprint + : field.metadata.has_value() + ? std::string_view(field.metadata->base_analyzer_fingerprint) + : std::string_view(); + RETURN_IF_ERROR(statistics->admit_snii_scoring_segment( + field.field_name, field.metadata, expected_base_analyzer_fingerprint, + field.index_doc_count, field.physical_sum_total_term_freq, + field.has_scoring_tier, field.has_positions, field.has_semantic_norms, + &key_version, segment_accumulator)); + } + return Status::OK(); + } + + Status admit_snii_fields_for_test(CollectionStatistics* statistics, + const std::vector& fields) { + CollectionStatistics::SniiScoringSegmentAccumulator segment_accumulator; + RETURN_IF_ERROR(stage_snii_fields_for_test(statistics, fields, &segment_accumulator)); + statistics->commit_snii_scoring_segment(std::move(segment_accumulator)); + return Status::OK(); + } + + Status admit_snii_segment_for_test( + CollectionStatistics* statistics, const std::wstring& field_name, + const std::optional& metadata, + uint64_t index_doc_count, bool has_semantic_norms) { + return admit_snii_fields_for_test( + statistics, {{field_name, metadata, index_doc_count, has_semantic_norms}}); + } + + Status stage_snii_fields_then_file_not_found_for_test( + CollectionStatistics* statistics, + const std::vector& fields_before_failure) { + CollectionStatistics::SniiScoringSegmentAccumulator segment_accumulator; + RETURN_IF_ERROR(stage_snii_fields_for_test(statistics, fields_before_failure, + &segment_accumulator)); + for (const auto& field : fields_before_failure) { + add_term_doc_frequency(&segment_accumulator.term_doc_freqs, field.field_name, L"staged", + 1); + } + return Status::Error( + "simulated later field failure"); + } + + void expect_collected_stats(const std::wstring& field_name, uint64_t doc_count, + uint64_t token_count) { + EXPECT_EQ(stats_->get_doc_num(), doc_count); + expect_collected_tokens(field_name, token_count); + } + std::unique_ptr stats_; std::shared_ptr runtime_state_; std::string test_dir_; @@ -428,7 +881,8 @@ TEST_F(CollectionStatisticsTest, CollectWithMockRowsetSplits) { auto status = stats_->collect(runtime_state_.get(), splits, tablet_schema, expr_contexts, nullptr); - EXPECT_TRUE(status.ok()); + EXPECT_TRUE(status.ok()) << status; + expect_no_collected_tokens(L"1"); } TEST_F(CollectionStatisticsTest, CollectWithEmptySegments) { @@ -442,6 +896,281 @@ TEST_F(CollectionStatisticsTest, CollectWithEmptySegments) { EXPECT_TRUE(status.ok()) << status.msg(); } +TEST_F(CollectionStatisticsTest, LegacyReservedTermUsesRawV3Namespace) { + auto tablet_schema = create_legacy_v3_schema(); + const std::string segment_path = test_dir_ + "/legacy_v3_0.dat"; + ASSERT_TRUE(write_legacy_v3_segment(tablet_schema, segment_path).ok()); + + auto rowset_meta = std::make_shared(); + auto rowset = std::make_shared(tablet_schema, rowset_meta); + rowset->set_num_segments(1); + rowset->set_segment_path(0, segment_path); + auto reader = std::make_shared(rowset); + std::vector splits {RowSetSplits(reader)}; + + auto status = stats_->collect(runtime_state_.get(), splits, tablet_schema, + create_reserved_exact_search_contexts(), nullptr); + + ASSERT_TRUE(status.ok()) << status; + expect_collected_stats(L"1", 1, 2); + expect_collected_term(L"1", + segment_v2::inverted_index::StringHelper::to_wstring( + std::string(segment_v2::inverted_index::CG_V1_MARKER) + "user"), + 0); +} + +TEST_F(CollectionStatisticsTest, LegacyV3SegmentsUsePhysicalScoringStatistics) { + auto tablet_schema = create_legacy_v3_schema(); + const std::string first_segment_path = test_dir_ + "/legacy_v3_0.dat"; + const std::string second_segment_path = test_dir_ + "/legacy_v3_1.dat"; + ASSERT_TRUE(write_legacy_v3_segment(tablet_schema, first_segment_path).ok()); + ASSERT_TRUE(write_legacy_v3_segment(tablet_schema, second_segment_path).ok()); + + auto rowset_meta = std::make_shared(); + auto rowset = std::make_shared(tablet_schema, rowset_meta); + rowset->set_num_segments(2); + rowset->set_segment_path(0, first_segment_path); + rowset->set_segment_path(1, second_segment_path); + auto reader = std::make_shared(rowset); + std::vector splits {RowSetSplits(reader)}; + + auto status = stats_->collect(runtime_state_.get(), splits, tablet_schema, + create_match_expr_contexts("alpha"), nullptr); + + ASSERT_TRUE(status.ok()) << status; + expect_collected_stats(L"1", 2, 4); + expect_collected_term(L"1", L"alpha", 2); +} + +TEST_F(CollectionStatisticsTest, LegacyV3SkipsMissingSegmentAfterCollectingAvailableStatistics) { + auto tablet_schema = create_legacy_v3_schema(); + const std::string first_segment_path = test_dir_ + "/legacy_v3_0.dat"; + ASSERT_TRUE(write_legacy_v3_segment(tablet_schema, first_segment_path).ok()); + + auto rowset_meta = std::make_shared(); + auto rowset = std::make_shared(tablet_schema, rowset_meta); + rowset->set_num_segments(2); + rowset->set_segment_path(0, first_segment_path); + rowset->set_segment_path(1, test_dir_ + "/missing_v3_1.dat"); + auto reader = std::make_shared(rowset); + std::vector splits {RowSetSplits(reader)}; + + auto status = stats_->collect(runtime_state_.get(), splits, tablet_schema, + create_match_expr_contexts("alpha"), nullptr); + + ASSERT_TRUE(status.ok()) << status; + expect_collected_stats(L"1", 1, 2); + expect_collected_term(L"1", L"alpha", 1); +} + +TEST_F(CollectionStatisticsTest, LegacyV3SkipsEmptySegmentAfterCollectingAvailableStatistics) { + auto tablet_schema = create_legacy_v3_schema(); + const std::string first_segment_path = test_dir_ + "/legacy_v3_0.dat"; + const std::string empty_segment_path = test_dir_ + "/legacy_v3_empty_1.dat"; + ASSERT_TRUE(write_legacy_v3_segment(tablet_schema, first_segment_path).ok()); + + const std::string empty_index_path = + segment_v2::InvertedIndexDescriptor::get_index_file_path_v2( + segment_v2::InvertedIndexDescriptor::get_index_file_path_prefix( + empty_segment_path)); + io::FileWriterPtr empty_file; + io::FileWriterOptions options; + ASSERT_TRUE(io::global_local_filesystem() + ->create_file(empty_index_path, &empty_file, &options) + .ok()); + ASSERT_TRUE(empty_file->close(false).ok()); + + auto rowset_meta = std::make_shared(); + auto rowset = std::make_shared(tablet_schema, rowset_meta); + rowset->set_num_segments(2); + rowset->set_segment_path(0, first_segment_path); + rowset->set_segment_path(1, empty_segment_path); + auto reader = std::make_shared(rowset); + std::vector splits {RowSetSplits(reader)}; + + const Status status = stats_->collect(runtime_state_.get(), splits, tablet_schema, + create_match_expr_contexts("alpha"), nullptr); + + ASSERT_TRUE(status.ok()) << status; + expect_collected_stats(L"1", 1, 2); + expect_collected_term(L"1", L"alpha", 1); +} + +TEST_F(CollectionStatisticsTest, SniiCommonGramsUsesSemanticScoringStatistics) { + auto tablet_schema = create_snii_schema(); + auto expr_contexts = create_match_expr_contexts("alpha", "test-base-v1"); + const auto* analyzer_ctx = expr_contexts.front()->root()->query_analyzer_ctx(); + ASSERT_NE(analyzer_ctx, nullptr); + ASSERT_NE(analyzer_ctx->analyzer_provider, nullptr); + + const std::string segment_path = test_dir_ + "/snii_common_grams_0.dat"; + auto write_status = write_snii_common_grams_segment( + segment_path, + std::string(analyzer_ctx->analyzer_provider->base_analyzer_fingerprint())); + ASSERT_TRUE(write_status.ok()) << write_status; + + auto rowset_meta = std::make_shared(); + auto rowset = std::make_shared(tablet_schema, rowset_meta); + rowset->set_num_segments(1); + rowset->set_segment_path(0, segment_path); + auto reader = std::make_shared(rowset); + std::vector splits {RowSetSplits(reader)}; + + auto status = + stats_->collect(runtime_state_.get(), splits, tablet_schema, expr_contexts, nullptr); + + ASSERT_TRUE(status.ok()) << status; + expect_collected_stats(L"1", 2, 3); + expect_collected_term(L"1", L"alpha", 2); +} + +TEST_F(CollectionStatisticsTest, SniiScoringLookupUsesCallerIoContext) { + snii::snii_test::ScopedEnv force_nonresident_dict("SNII_DICT_RESIDENT_MAX", "0"); + auto tablet_schema = create_snii_schema(); + auto expr_contexts = create_match_expr_contexts("alpha", "test-base-v1"); + const auto* analyzer_ctx = expr_contexts.front()->root()->query_analyzer_ctx(); + ASSERT_NE(analyzer_ctx, nullptr); + ASSERT_NE(analyzer_ctx->analyzer_provider, nullptr); + + const std::string segment_path = test_dir_ + "/snii_io_context_0.dat"; + ASSERT_TRUE(write_snii_common_grams_segment( + segment_path, + std::string(analyzer_ctx->analyzer_provider->base_analyzer_fingerprint())) + .ok()); + + auto rowset_meta = std::make_shared(); + auto rowset = std::make_shared(tablet_schema, rowset_meta); + rowset->set_num_segments(1); + rowset->set_segment_path(0, segment_path); + auto reader = std::make_shared(rowset); + std::vector splits {RowSetSplits(reader)}; + + const std::string index_path_prefix { + segment_v2::InvertedIndexDescriptor::get_index_file_path_prefix(segment_path)}; + io::FileCacheStatistics open_stats; + io::IOContext open_io_ctx; + open_io_ctx.file_cache_stats = &open_stats; + segment_v2::IndexFileReader file_reader(io::global_local_filesystem(), index_path_prefix, + InvertedIndexStorageFormatPB::SNII); + ASSERT_TRUE(file_reader.init(config::inverted_index_read_buffer_size, &open_io_ctx).ok()); + const auto index_metas = tablet_schema->inverted_indexs(1); + ASSERT_EQ(index_metas.size(), 1); + auto logical_reader = file_reader.open_snii_index(index_metas.front(), &open_io_ctx); + ASSERT_TRUE(logical_reader.has_value()) << logical_reader.error(); + ASSERT_GT(open_stats.inverted_index_range_read_count, 0); + + io::FileCacheStatistics collect_stats; + io::IOContext collect_io_ctx; + collect_io_ctx.file_cache_stats = &collect_stats; + const auto status = stats_->collect(runtime_state_.get(), splits, tablet_schema, expr_contexts, + &collect_io_ctx); + + ASSERT_TRUE(status.ok()) << status; + EXPECT_GT(collect_stats.inverted_index_range_read_count, + open_stats.inverted_index_range_read_count); +} + +TEST_F(CollectionStatisticsTest, SniiStatsProviderUsesSemanticCommonGramsTokenCount) { + auto tablet_schema = create_snii_schema(); + auto expr_contexts = create_match_expr_contexts("alpha", "test-base-v1"); + const auto* analyzer_ctx = expr_contexts.front()->root()->query_analyzer_ctx(); + ASSERT_NE(analyzer_ctx, nullptr); + ASSERT_NE(analyzer_ctx->analyzer_provider, nullptr); + + const std::string segment_path = test_dir_ + "/snii_semantic_stats_0.dat"; + auto write_status = write_snii_common_grams_segment( + segment_path, + std::string(analyzer_ctx->analyzer_provider->base_analyzer_fingerprint())); + ASSERT_TRUE(write_status.ok()) << write_status; + + const std::string index_path_prefix { + segment_v2::InvertedIndexDescriptor::get_index_file_path_prefix(segment_path)}; + segment_v2::IndexFileReader file_reader(io::global_local_filesystem(), index_path_prefix, + InvertedIndexStorageFormatPB::SNII); + ASSERT_TRUE(file_reader.init().ok()); + const auto index_metas = tablet_schema->inverted_indexs(1); + ASSERT_EQ(index_metas.size(), 1); + auto logical_reader = file_reader.open_snii_index(index_metas.front()); + ASSERT_TRUE(logical_reader.has_value()) << logical_reader.error(); + + snii::stats::SniiStatsProvider provider; + ASSERT_TRUE(snii::stats::SniiStatsProvider::open(logical_reader->get(), &provider).ok()); + EXPECT_EQ(provider.doc_count(), 2); + EXPECT_EQ(provider.indexed_doc_count(), 2); + EXPECT_EQ(provider.sum_total_term_freq(), 3); + EXPECT_DOUBLE_EQ(provider.avgdl(), 1.5); + EXPECT_TRUE(provider.has_norms()); +} + +TEST_F(CollectionStatisticsTest, SniiWriterRejectsMissingSemanticScoringMetadata) { + const std::string segment_path = test_dir_ + "/snii_missing_scoring_metadata_0.dat"; + auto write_status = write_plain_snii_scoring_segment(segment_path); + EXPECT_EQ(write_status.code(), ErrorCode::INVERTED_INDEX_NOT_SUPPORTED); +} + +TEST_F(CollectionStatisticsTest, SniiScoringRejectsMissingSegmentForWholeCollection) { + auto tablet_schema = create_snii_schema(); + auto expr_contexts = create_match_expr_contexts("alpha", "test-base-v1"); + const auto* analyzer_ctx = expr_contexts.front()->root()->query_analyzer_ctx(); + ASSERT_NE(analyzer_ctx, nullptr); + ASSERT_NE(analyzer_ctx->analyzer_provider, nullptr); + + const std::string first_segment_path = test_dir_ + "/snii_complete_0.dat"; + auto write_status = write_snii_common_grams_segment( + first_segment_path, + std::string(analyzer_ctx->analyzer_provider->base_analyzer_fingerprint())); + ASSERT_TRUE(write_status.ok()) << write_status; + + auto rowset_meta = std::make_shared(); + auto rowset = std::make_shared(tablet_schema, rowset_meta); + rowset->set_num_segments(2); + rowset->set_segment_path(0, first_segment_path); + rowset->set_segment_path(1, test_dir_ + "/missing_snii_1.dat"); + auto reader = std::make_shared(rowset); + std::vector splits {RowSetSplits(reader)}; + + auto status = + stats_->collect(runtime_state_.get(), splits, tablet_schema, expr_contexts, nullptr); + + EXPECT_EQ(status.code(), ErrorCode::INVERTED_INDEX_NOT_SUPPORTED); +} + +TEST_F(CollectionStatisticsTest, SniiScoringRejectsMissingLogicalIndexForWholeCollection) { + auto tablet_schema = create_snii_schema(/*index_id=*/2); + auto expr_contexts = create_match_expr_contexts("alpha", "test-base-v1"); + const auto* analyzer_ctx = expr_contexts.front()->root()->query_analyzer_ctx(); + ASSERT_NE(analyzer_ctx, nullptr); + ASSERT_NE(analyzer_ctx->analyzer_provider, nullptr); + + const std::string segment_path = test_dir_ + "/snii_missing_logical_index_0.dat"; + auto write_status = write_snii_common_grams_segment( + segment_path, + std::string(analyzer_ctx->analyzer_provider->base_analyzer_fingerprint())); + ASSERT_TRUE(write_status.ok()) << write_status; + + auto rowset_meta = std::make_shared(); + auto rowset = std::make_shared(tablet_schema, rowset_meta); + rowset->set_num_segments(1); + rowset->set_segment_path(0, segment_path); + auto reader = std::make_shared(rowset); + std::vector splits {RowSetSplits(reader)}; + + auto status = + stats_->collect(runtime_state_.get(), splits, tablet_schema, expr_contexts, nullptr); + + EXPECT_EQ(status.code(), ErrorCode::INVERTED_INDEX_NOT_SUPPORTED); + expect_no_collected_tokens(L"1"); +} + +TEST_F(CollectionStatisticsTest, SniiWriterRejectsZeroSemanticTokensForNonemptyPostings) { + auto status = write_snii_common_grams_segment(test_dir_ + "/snii_zero_semantic_tokens_0.dat", + "test-base-v1", + /*scoring_token_count=*/0); + + EXPECT_EQ(status.code(), ErrorCode::INVALID_ARGUMENT); + EXPECT_THAT(status.msg(), ::testing::HasSubstr("zero semantic scoring tokens")); +} + TEST_F(CollectionStatisticsTest, CollectWithMultipleRowsetSplits) { auto tablet_schema = create_tablet_schema_with_inverted_index(); auto expr_contexts = create_match_expr_contexts(); @@ -488,6 +1217,382 @@ class CollectionStatisticsDetailedTest : public ::testing::Test { std::unique_ptr stats_; }; +segment_v2::inverted_index::CommonGramsSegmentMetadata complete_snii_scoring_metadata( + std::string base_analyzer_fingerprint = "base-v1", uint64_t doc_count = 3, + uint64_t token_count = 7) { + using namespace segment_v2::inverted_index; + segment_v2::inverted_index::CommonGramsSegmentMetadata metadata; + metadata.plain_term_key_version = PlainTermKeyVersion::kEscapedV1; + metadata.common_grams_coverage = CommonGramsCoverage::kComplete; + metadata.common_grams_semantics_version = COMMON_GRAMS_SEMANTICS_VERSION_V1; + metadata.common_grams_key_version = COMMON_GRAMS_KEY_VERSION_V1; + metadata.common_grams_dictionary_identity = "builtin-stopwords:v1"; + metadata.base_analyzer_fingerprint = std::move(base_analyzer_fingerprint); + metadata.common_grams_fingerprint = "common-grams-v1"; + metadata.scoring_coverage = ScoringCoverage::kComplete; + metadata.scoring_stats_version = COMMON_GRAMS_SCORING_STATS_VERSION_V1; + metadata.norm_semantics_version = COMMON_GRAMS_NORM_SEMANTICS_VERSION_V1; + metadata.scoring_doc_count = doc_count; + metadata.scoring_token_count = token_count; + return metadata; +} + +Result resolve_snii_scoring_segment_for_test( + std::optional metadata, + uint64_t physical_doc_count, bool has_norms) { + const uint64_t physical_sum_total_term_freq = metadata ? metadata->scoring_token_count : 0; + return resolve_snii_scoring_segment(metadata, physical_doc_count, physical_sum_total_term_freq, + /*has_scoring_tier=*/true, + /*has_positions=*/true, has_norms); +} + +TEST(CollectionStatisticsCommonGramsTest, MissingMetadataWithoutPersistedProofRejectsScoring) { + auto result = resolve_snii_scoring_segment_for_test(std::nullopt, 3, true); + + ASSERT_FALSE(result.has_value()); + EXPECT_EQ(result.error().code(), ErrorCode::INVERTED_INDEX_NOT_SUPPORTED); +} + +TEST(CollectionStatisticsCommonGramsTest, RawNoInternalMetadataWithoutScoringProofIsRejected) { + segment_v2::inverted_index::CommonGramsSegmentMetadata metadata; + metadata.plain_term_key_version = + segment_v2::inverted_index::PlainTermKeyVersion::kRawNoInternal; + metadata.common_grams_coverage = segment_v2::inverted_index::CommonGramsCoverage::kNone; + metadata.base_analyzer_fingerprint = "base-v1"; + + auto result = resolve_snii_scoring_segment_for_test(metadata, 3, true); + + ASSERT_FALSE(result.has_value()); + EXPECT_EQ(result.error().code(), ErrorCode::INVERTED_INDEX_NOT_SUPPORTED); +} + +TEST(CollectionStatisticsCommonGramsTest, ScoringCoverageNoneRejectsScoringAdmission) { + auto metadata = complete_snii_scoring_metadata(); + metadata.scoring_coverage = segment_v2::inverted_index::ScoringCoverage::kNone; + metadata.scoring_stats_version = 0; + metadata.norm_semantics_version = 0; + + auto result = resolve_snii_scoring_segment_for_test(metadata, 3, true); + + ASSERT_FALSE(result.has_value()); + EXPECT_EQ(result.error().code(), ErrorCode::INVERTED_INDEX_NOT_SUPPORTED); +} + +TEST(CollectionStatisticsCommonGramsTest, IncompatibleScoringVersionsRejectScoringAdmission) { + auto metadata = complete_snii_scoring_metadata(); + + ++metadata.scoring_stats_version; + auto scoring_version_result = resolve_snii_scoring_segment_for_test(metadata, 3, true); + ASSERT_FALSE(scoring_version_result.has_value()); + EXPECT_EQ(scoring_version_result.error().code(), ErrorCode::INVERTED_INDEX_NOT_SUPPORTED); + + --metadata.scoring_stats_version; + ++metadata.norm_semantics_version; + auto norm_version_result = resolve_snii_scoring_segment_for_test(metadata, 3, true); + ASSERT_FALSE(norm_version_result.has_value()); + EXPECT_EQ(norm_version_result.error().code(), ErrorCode::INVERTED_INDEX_NOT_SUPPORTED); +} + +TEST(CollectionStatisticsCommonGramsTest, ScoringDocCountMismatchRejectsScoringAdmission) { + auto result = resolve_snii_scoring_segment_for_test(complete_snii_scoring_metadata(), 4, true); + + ASSERT_FALSE(result.has_value()); + EXPECT_EQ(result.error().code(), ErrorCode::INVERTED_INDEX_FILE_CORRUPTED); +} + +TEST(CollectionStatisticsCommonGramsTest, MissingSemanticNormsRejectScoringAdmission) { + auto result = resolve_snii_scoring_segment_for_test(complete_snii_scoring_metadata(), 3, false); + + ASSERT_FALSE(result.has_value()); + EXPECT_EQ(result.error().code(), ErrorCode::INVERTED_INDEX_FILE_CORRUPTED); +} + +TEST(CollectionStatisticsCommonGramsTest, CompleteMetadataOnNonScoringTierIsCorruption) { + auto metadata = complete_snii_scoring_metadata(); + auto result = resolve_snii_scoring_segment(metadata, 3, 7, + /*has_scoring_tier=*/false, + /*has_positions=*/true, + /*has_semantic_norms=*/true); + + ASSERT_FALSE(result.has_value()); + EXPECT_EQ(result.error().code(), ErrorCode::INVERTED_INDEX_FILE_CORRUPTED); +} + +TEST(CollectionStatisticsCommonGramsTest, ZeroSemanticTokensWithPhysicalTermsIsCorruption) { + auto metadata = complete_snii_scoring_metadata("base-v1", 3, 0); + auto result = resolve_snii_scoring_segment(metadata, 3, 1, + /*has_scoring_tier=*/true, + /*has_positions=*/true, + /*has_semantic_norms=*/true); + + ASSERT_FALSE(result.has_value()); + EXPECT_EQ(result.error().code(), ErrorCode::INVERTED_INDEX_FILE_CORRUPTED); +} + +TEST(CollectionStatisticsCommonGramsTest, EmptyPhysicalAndSemanticTokenCountsAreValid) { + auto metadata = complete_snii_scoring_metadata("base-v1", 3, 0); + auto result = resolve_snii_scoring_segment(metadata, 3, 0, + /*has_scoring_tier=*/true, + /*has_positions=*/true, + /*has_semantic_norms=*/true); + + ASSERT_TRUE(result.has_value()) << result.error(); + EXPECT_EQ(result->token_count, 0); +} + +TEST(CollectionStatisticsCommonGramsTest, LegacyPhysicalScoringRequiresDocumentLengthNorms) { + auto result = resolve_snii_scoring_segment_for_test(std::nullopt, 3, false); + + ASSERT_FALSE(result.has_value()); + EXPECT_EQ(result.error().code(), ErrorCode::INVERTED_INDEX_NOT_SUPPORTED); +} + +TEST(CollectionStatisticsCommonGramsTest, CompleteMetadataUsesSemanticDocAndTokenCounts) { + auto result = resolve_snii_scoring_segment_for_test(complete_snii_scoring_metadata(), 3, true); + + ASSERT_TRUE(result.has_value()) << result.error(); + EXPECT_EQ(result->doc_count, 3); + EXPECT_EQ(result->token_count, 7); + EXPECT_EQ(result->plain_term_key_version, + segment_v2::inverted_index::PlainTermKeyVersion::kEscapedV1); + EXPECT_EQ(result->base_analyzer_fingerprint, "base-v1"); +} + +TEST(CollectionStatisticsCommonGramsTest, CompletePlainMetadataIsExplicitSemanticProof) { + auto metadata = complete_snii_scoring_metadata(); + metadata.plain_term_key_version = + segment_v2::inverted_index::PlainTermKeyVersion::kRawNoInternal; + metadata.common_grams_coverage = segment_v2::inverted_index::CommonGramsCoverage::kNone; + metadata.common_grams_semantics_version = 0; + metadata.common_grams_key_version = 0; + metadata.common_grams_dictionary_identity.clear(); + metadata.common_grams_fingerprint.clear(); + + auto result = resolve_snii_scoring_segment_for_test(metadata, 3, true); + + ASSERT_TRUE(result.has_value()) << result.error(); + EXPECT_EQ(result->doc_count, 3); + EXPECT_EQ(result->token_count, 7); +} + +TEST(CollectionStatisticsCommonGramsTest, PlainSemanticTokenCountMustEqualPhysicalCount) { + auto metadata = complete_snii_scoring_metadata(); + metadata.plain_term_key_version = + segment_v2::inverted_index::PlainTermKeyVersion::kRawNoInternal; + metadata.common_grams_coverage = segment_v2::inverted_index::CommonGramsCoverage::kNone; + metadata.common_grams_semantics_version = 0; + metadata.common_grams_key_version = 0; + metadata.common_grams_dictionary_identity.clear(); + metadata.common_grams_fingerprint.clear(); + + auto result = resolve_snii_scoring_segment(metadata, 3, 8, + /*has_scoring_tier=*/true, + /*has_positions=*/true, + /*has_semantic_norms=*/true); + + ASSERT_FALSE(result.has_value()); + EXPECT_EQ(result.error().code(), ErrorCode::INVERTED_INDEX_FILE_CORRUPTED); +} + +TEST(CollectionStatisticsCommonGramsTest, EmptySemanticFingerprintIsNotScoringProof) { + auto metadata = complete_snii_scoring_metadata(); + metadata.base_analyzer_fingerprint.clear(); + + auto result = resolve_snii_scoring_segment_for_test(metadata, 3, true); + + ASSERT_FALSE(result.has_value()); + EXPECT_EQ(result.error().code(), ErrorCode::INVERTED_INDEX_FILE_CORRUPTED); +} + +TEST_F(CollectionStatisticsTest, MixedBaseFingerprintRejectsAndClearsWholeCollection) { + auto first = complete_snii_scoring_metadata("base-v1", 3, 7); + ASSERT_TRUE(admit_snii_segment_for_test(stats_.get(), L"1", first, 3, true).ok()); + EXPECT_FLOAT_EQ(stats_->get_or_calculate_avg_dl(L"1"), 7.0F / 3.0F); + + auto second = complete_snii_scoring_metadata("base-v2", 3, 5); + auto status = admit_snii_segment_for_test(stats_.get(), L"1", second, 3, true); + + EXPECT_EQ(status.code(), ErrorCode::INVERTED_INDEX_NOT_SUPPORTED); + expect_no_collected_tokens(L"1"); + EXPECT_THROW(stats_->get_or_calculate_avg_dl(L"1"), Exception); +} + +TEST_F(CollectionStatisticsTest, PersistedFingerprintMustMatchRequestAnalyzer) { + auto metadata = complete_snii_scoring_metadata("persisted-base", 3, 7); + auto status = admit_snii_fields_for_test( + stats_.get(), {{L"1", metadata, 3, true, std::string("request-base")}}); + + EXPECT_EQ(status.code(), ErrorCode::INVERTED_INDEX_NOT_SUPPORTED); + expect_no_collected_tokens(L"1"); + EXPECT_THROW(stats_->get_doc_num(), Exception); +} + +TEST_F(CollectionStatisticsTest, CollectionStatisticsInstancesKeepAdmissionStateIsolated) { + CollectionStatistics first; + CollectionStatistics second; + + ASSERT_TRUE(admit_snii_segment_for_test(&first, L"1", + complete_snii_scoring_metadata("base-a", 2, 6), 2, true) + .ok()); + ASSERT_TRUE(admit_snii_segment_for_test( + &second, L"1", complete_snii_scoring_metadata("base-b", 5, 25), 5, true) + .ok()); + + EXPECT_FLOAT_EQ(first.get_or_calculate_avg_dl(L"1"), 3.0F); + EXPECT_FLOAT_EQ(second.get_or_calculate_avg_dl(L"1"), 5.0F); +} + +TEST_F(CollectionStatisticsTest, LegacyAndUnprovedRawNoInternalRejectWholeCollection) { + ASSERT_TRUE(admit_snii_segment_for_test(stats_.get(), L"1", + complete_snii_scoring_metadata("base-v1", 2, 6), 2, + true) + .ok()); + + segment_v2::inverted_index::CommonGramsSegmentMetadata plain_metadata; + plain_metadata.plain_term_key_version = + segment_v2::inverted_index::PlainTermKeyVersion::kRawNoInternal; + plain_metadata.common_grams_coverage = segment_v2::inverted_index::CommonGramsCoverage::kNone; + plain_metadata.base_analyzer_fingerprint = "base-v1"; + auto status = admit_snii_segment_for_test(stats_.get(), L"1", plain_metadata, 3, true); + + EXPECT_EQ(status.code(), ErrorCode::INVERTED_INDEX_NOT_SUPPORTED); + expect_no_collected_tokens(L"1"); + EXPECT_THROW(stats_->get_doc_num(), Exception); +} + +TEST_F(CollectionStatisticsTest, LegacyAndCommonGramsMixRejectsWholeCollection) { + ASSERT_TRUE(admit_snii_segment_for_test(stats_.get(), L"1", + complete_snii_scoring_metadata("base-v1", 2, 6), 2, + true) + .ok()); + + auto status = admit_snii_segment_for_test(stats_.get(), L"1", std::nullopt, 3, true); + + EXPECT_EQ(status.code(), ErrorCode::INVERTED_INDEX_NOT_SUPPORTED); + expect_no_collected_tokens(L"1"); + EXPECT_THROW(stats_->get_or_calculate_avg_dl(L"1"), Exception); +} + +TEST_F(CollectionStatisticsTest, ExplicitSemanticPlainAndCommonGramsSegmentsAccumulate) { + auto plain_metadata = complete_snii_scoring_metadata("base-v1", 2, 6); + plain_metadata.plain_term_key_version = + segment_v2::inverted_index::PlainTermKeyVersion::kRawNoInternal; + plain_metadata.common_grams_coverage = segment_v2::inverted_index::CommonGramsCoverage::kNone; + plain_metadata.common_grams_semantics_version = 0; + plain_metadata.common_grams_key_version = 0; + plain_metadata.common_grams_dictionary_identity.clear(); + plain_metadata.common_grams_fingerprint.clear(); + + ASSERT_TRUE(admit_snii_segment_for_test(stats_.get(), L"1", plain_metadata, 2, true).ok()); + ASSERT_TRUE(admit_snii_segment_for_test(stats_.get(), L"1", + complete_snii_scoring_metadata("base-v1", 3, 7), 3, + true) + .ok()); + + expect_collected_stats(L"1", 5, 13); + EXPECT_FLOAT_EQ(stats_->get_or_calculate_avg_dl(L"1"), 13.0F / 5.0F); +} + +TEST_F(CollectionStatisticsTest, CommonGramsSegmentsAccumulateSemanticStatistics) { + ASSERT_TRUE(admit_snii_segment_for_test(stats_.get(), L"1", + complete_snii_scoring_metadata("base-v1", 2, 6), 2, + true) + .ok()); + ASSERT_TRUE(admit_snii_segment_for_test(stats_.get(), L"1", + complete_snii_scoring_metadata("base-v1", 3, 9), 3, + true) + .ok()); + + expect_collected_stats(L"1", 5, 15); + EXPECT_FLOAT_EQ(stats_->get_or_calculate_avg_dl(L"1"), 3.0F); +} + +TEST_F(CollectionStatisticsTest, MultiFieldSegmentsCommitAndAccumulateAtomically) { + ASSERT_TRUE(admit_snii_fields_for_test( + stats_.get(), + {{L"1", complete_snii_scoring_metadata("field-1", 3, 7), 3, true}, + {L"2", complete_snii_scoring_metadata("field-2", 3, 12), 3, true}}) + .ok()); + ASSERT_TRUE(admit_snii_fields_for_test( + stats_.get(), + {{L"1", complete_snii_scoring_metadata("field-1", 2, 5), 2, true}, + {L"2", complete_snii_scoring_metadata("field-2", 2, 8), 2, true}}) + .ok()); + + expect_collected_stats(L"1", 5, 12); + expect_collected_tokens(L"2", 20); + EXPECT_FLOAT_EQ(stats_->get_or_calculate_avg_dl(L"1"), 12.0F / 5.0F); + EXPECT_FLOAT_EQ(stats_->get_or_calculate_avg_dl(L"2"), 4.0F); +} + +TEST_F(CollectionStatisticsTest, MultiFieldSegmentDocCountsMustAgree) { + auto status = admit_snii_fields_for_test( + stats_.get(), {{L"1", complete_snii_scoring_metadata("field-1", 3, 7), 3, true}, + {L"2", complete_snii_scoring_metadata("field-2", 4, 12), 4, true}}); + + EXPECT_EQ(status.code(), ErrorCode::INVERTED_INDEX_NOT_SUPPORTED); + expect_no_collected_tokens(L"1"); + expect_no_collected_tokens(L"2"); + EXPECT_THROW(stats_->get_doc_num(), Exception); +} + +TEST_F(CollectionStatisticsTest, LaterFieldFileNotFoundDoesNotPublishPartialSegment) { + ASSERT_TRUE(admit_snii_segment_for_test(stats_.get(), L"1", + complete_snii_scoring_metadata("field-1", 2, 6), 2, + true) + .ok()); + + auto status = stage_snii_fields_then_file_not_found_for_test( + stats_.get(), + {{L"2", complete_snii_scoring_metadata("staged-field-2", 3, 12), 3, true}}); + + EXPECT_EQ(status.code(), ErrorCode::INVERTED_INDEX_FILE_NOT_FOUND); + expect_collected_stats(L"1", 2, 6); + expect_no_collected_tokens(L"2"); + expect_no_collected_term(L"2", L"staged"); + + ASSERT_TRUE(admit_snii_segment_for_test( + stats_.get(), L"2", + complete_snii_scoring_metadata("different-field-2", 1, 4), 1, true) + .ok()); + expect_collected_tokens(L"2", 4); +} + +TEST(CollectionStatisticsCommonGramsTest, DocFrequencySupportsLogicalAndPhysicalTermKeys) { + std::unordered_map> + logical_frequencies; + + add_term_doc_frequency(&logical_frequencies, L"field", L"logical", 3); + EXPECT_EQ(logical_frequencies[L"field"][L"logical"], 3); + + add_term_doc_frequency(&logical_frequencies, L"field", L"same", 5); + EXPECT_EQ(logical_frequencies[L"field"][L"same"], 5); +} + +TEST(CollectionStatisticsCommonGramsTest, PhysicalAliasesCannotMergeLogicalDocFrequencies) { + using Frequencies = + std::unordered_map>; + Frequencies logical_frequencies; + + const std::wstring alias = std::wstring(1, wchar_t {0x1e}) + L"G00000001:"; + add_term_doc_frequency(&logical_frequencies, L"field", L"\x1f", 3); + add_term_doc_frequency(&logical_frequencies, L"field", alias, 5); + + EXPECT_EQ(logical_frequencies[L"field"][alias], 5); +} + +TEST(CollectionStatisticsCommonGramsTest, UnrepresentablePlainTermRegistersZeroDocFrequency) { + using Frequencies = + std::unordered_map>; + Frequencies logical_frequencies; + + add_term_doc_frequency(&logical_frequencies, L"field", L"unrepresentable", 0); + + ASSERT_TRUE(logical_frequencies.contains(L"field")); + EXPECT_TRUE(logical_frequencies.at(L"field").contains(L"unrepresentable")); + EXPECT_EQ(logical_frequencies.at(L"field").at(L"unrepresentable"), 0); +} + TEST_F(CollectionStatisticsDetailedTest, GetStatisticsWithValidData) { std::wstring field_name = L"test_field"; std::wstring term = L"test_term"; @@ -694,13 +1799,7 @@ TEST_F(CollectionStatisticsTest, ExtractCollectInfoForVariantSubcolumnIndex) { EXPECT_EQ(it->second.index_meta->index_name(), "variant_subcolumn_idx"); } -// Regression for score on a dynamic variant sub-column inherited from a plain -// parent variant inverted index (no field_pattern template). Matches the -// scan-time schema shape: _init_variant_columns materializes the accessed -// path as an extracted VARIANT placeholder, so neither inverted_indexs(column) -// nor generate_sub_column_info resolves the parent index. Collector clones -// the parent's non-field-pattern indexes with the variant path as suffix. -TEST_F(CollectionStatisticsTest, ExtractCollectInfoForVariantParentIndexWithoutTemplate) { +TEST_F(CollectionStatisticsTest, MatchScoringUsesTextSemanticsForVariantParentIndexFallback) { auto tablet_schema = std::make_shared(); constexpr int32_t kVariantUid = 9004; @@ -761,13 +1860,14 @@ TEST_F(CollectionStatisticsTest, ExtractCollectInfoForVariantParentIndexWithoutT std::unordered_map collect_infos; auto status = stats_->extract_collect_info(runtime_state_.get(), contexts, tablet_schema, &collect_infos); - ASSERT_TRUE(status.ok()) << status.msg(); - ASSERT_EQ(collect_infos.size(), 1u); + ASSERT_TRUE(status.ok()) << status; + ASSERT_EQ(collect_infos.size(), 1U); auto it = collect_infos.find(StringHelper::to_wstring(std::to_string(kVariantUid) + ".v.key")); ASSERT_NE(it, collect_infos.end()); ASSERT_NE(it->second.index_meta, nullptr); ASSERT_NE(it->second.owned_index_meta, nullptr); - EXPECT_EQ(it->second.index_meta->index_name(), "variant_parent_idx"); + EXPECT_EQ(it->second.index_meta->index_id(), 2004); + EXPECT_EQ(it->second.unique_terms, std::vector({"abc"})); } namespace { @@ -1059,7 +2159,7 @@ TEST_F(CollectionStatisticsTest, CollectDirectIndexHitFromSchema) { ASSERT_NE(it, collect_infos.end()); EXPECT_NE(it->second.index_meta, nullptr); EXPECT_EQ(it->second.owned_index_meta, nullptr); // O1: schema-direct meta is not owned - EXPECT_FALSE(it->second.term_infos.empty()); + EXPECT_FALSE(it->second.unique_terms.empty()); } // I2: Plain string column with no index and not an extracted variant @@ -1176,10 +2276,7 @@ TEST_F(CollectionStatisticsTest, CollectSkipsIndexWithoutSimilarityScore) { EXPECT_TRUE(collect_infos.empty()); } -// L4: Two MATCH predicates on the same column produce CollectInfo entries -// keyed on the same field_name; the second insertion merges term_infos -// into the first entry. -TEST_F(CollectionStatisticsTest, CollectMergesTermsForSameFieldName) { +TEST_F(CollectionStatisticsTest, CollectPreservesLogicalClauseShapesForSameFieldName) { auto tablet_schema = std::make_shared(); constexpr int32_t kColUid = 1400; @@ -1215,16 +2312,539 @@ TEST_F(CollectionStatisticsTest, CollectMergesTermsForSameFieldName) { MatchPredicateCollector collector; std::unordered_map collect_infos; - auto first = collector.collect(runtime_state_.get(), tablet_schema, build_match("alpha"), + auto first = collector.collect(runtime_state_.get(), tablet_schema, build_match("alpha beta"), &collect_infos); ASSERT_TRUE(first.ok()) << first.msg(); - auto second = collector.collect(runtime_state_.get(), tablet_schema, build_match("beta"), - &collect_infos); + auto second = collector.collect(runtime_state_.get(), tablet_schema, + build_match("alpha alpha beta"), &collect_infos); ASSERT_TRUE(second.ok()) << second.msg(); ASSERT_EQ(collect_infos.size(), 1u); auto it = collect_infos.find(StringHelper::to_wstring(std::to_string(kColUid))); ASSERT_NE(it, collect_infos.end()); - EXPECT_GE(it->second.term_infos.size(), 2u); // both "alpha" and "beta" present + ASSERT_EQ(it->second.unique_terms, std::vector({"alpha", "beta"})); + ASSERT_EQ(it->second.unique_term_slots.size(), 2u); + EXPECT_EQ(it->second.unique_term_slots.at("alpha"), 0u); + EXPECT_EQ(it->second.unique_term_slots.at("beta"), 1u); + ASSERT_EQ(it->second.logical_scoring_leaves.size(), 2u); + ASSERT_EQ(it->second.logical_scoring_leaves[0].clauses.size(), 2u); + EXPECT_EQ(it->second.logical_scoring_leaves[0].clauses[0].df_slot, 0u); + EXPECT_EQ(it->second.logical_scoring_leaves[0].clauses[0].position, 1); + EXPECT_EQ(it->second.logical_scoring_leaves[0].clauses[1].df_slot, 1u); + EXPECT_EQ(it->second.logical_scoring_leaves[0].clauses[1].position, 2); + ASSERT_EQ(it->second.logical_scoring_leaves[1].clauses.size(), 3u); + EXPECT_EQ(it->second.logical_scoring_leaves[1].clauses[0].df_slot, 0u); + EXPECT_EQ(it->second.logical_scoring_leaves[1].clauses[0].position, 1); + EXPECT_EQ(it->second.logical_scoring_leaves[1].clauses[1].df_slot, 0u); + EXPECT_EQ(it->second.logical_scoring_leaves[1].clauses[1].position, 2); + EXPECT_EQ(it->second.logical_scoring_leaves[1].clauses[2].df_slot, 1u); + EXPECT_EQ(it->second.logical_scoring_leaves[1].clauses[2].position, 3); +} + +TEST_F(CollectionStatisticsTest, CollectUsesMatchRequestAnalyzerProviderAndFingerprint) { + auto tablet_schema = create_tablet_schema_with_inverted_index(); + + auto analyzer = segment_v2::inverted_index::InvertedIndexAnalyzer::create_builtin_analyzer( + InvertedIndexParserType::PARSER_ENGLISH, "", INVERTED_INDEX_PARSER_FALSE, "none"); + auto provider = std::make_shared( + std::move(analyzer), "request-base-v1"); + auto analyzer_ctx = std::make_shared(); + analyzer_ctx->analyzer_provider = provider; + + auto match_expr = std::make_shared(TExprNodeType::MATCH_PRED); + match_expr->set_analyzer_ctx(std::move(analyzer_ctx)); + auto slot_ref = std::make_shared("content", SlotId(1)); + auto literal = std::make_shared("Alpha ALPHA"); + match_expr->_children.push_back(slot_ref); + match_expr->_children.push_back(literal); + + MatchPredicateCollector collector; + CollectInfoMap collect_infos; + auto status = + collector.collect(runtime_state_.get(), tablet_schema, match_expr, &collect_infos); + + ASSERT_TRUE(status.ok()) << status.msg(); + ASSERT_EQ(collect_infos.size(), 1u); + const auto& collect_info = collect_infos.begin()->second; + EXPECT_EQ(collect_info.expected_base_analyzer_fingerprint, "request-base-v1"); + ASSERT_EQ(collect_info.unique_terms, std::vector({"Alpha", "ALPHA"})); + ASSERT_EQ(collect_info.logical_scoring_leaves.size(), 1u); + ASSERT_EQ(collect_info.logical_scoring_leaves[0].clauses.size(), 2u); + EXPECT_EQ(collect_info.logical_scoring_leaves[0].clauses[0].df_slot, 0u); + EXPECT_EQ(collect_info.logical_scoring_leaves[0].clauses[1].df_slot, 1u); +} + +TEST_F(CollectionStatisticsTest, CollectPhrasePrefixExcludesScoringTail) { + auto tablet_schema = create_tablet_schema_with_inverted_index(); + auto match_expr = std::make_shared(TExprNodeType::MATCH_PRED); + match_expr->set_opcode(TExprOpcode::MATCH_PHRASE_PREFIX); + match_expr->_children.push_back( + std::make_shared("content", SlotId(1))); + match_expr->_children.push_back( + std::make_shared("alpha beta gamma")); + + MatchPredicateCollector collector; + CollectInfoMap collect_infos; + auto status = + collector.collect(runtime_state_.get(), tablet_schema, match_expr, &collect_infos); + + ASSERT_TRUE(status.ok()) << status.msg(); + ASSERT_EQ(collect_infos.size(), 1u); + const auto& collect_info = collect_infos.begin()->second; + EXPECT_EQ(collect_info.unique_terms, std::vector({"alpha", "beta"})); + ASSERT_EQ(collect_info.logical_scoring_leaves.size(), 1u); + ASSERT_EQ(collect_info.logical_scoring_leaves[0].clauses.size(), 2u); + EXPECT_EQ(collect_info.logical_scoring_leaves[0].clauses[0].position, 1); + EXPECT_EQ(collect_info.logical_scoring_leaves[0].clauses[1].position, 2); +} + +TEST_F(CollectionStatisticsTest, CollectSingleTermPhrasePrefixHasEmptyScoringLeaf) { + auto tablet_schema = create_tablet_schema_with_inverted_index(); + auto match_expr = std::make_shared(TExprNodeType::MATCH_PRED); + match_expr->set_opcode(TExprOpcode::MATCH_PHRASE_PREFIX); + match_expr->_children.push_back( + std::make_shared("content", SlotId(1))); + match_expr->_children.push_back(std::make_shared("alpha")); + + MatchPredicateCollector collector; + CollectInfoMap collect_infos; + auto status = + collector.collect(runtime_state_.get(), tablet_schema, match_expr, &collect_infos); + + ASSERT_TRUE(status.ok()) << status.msg(); + ASSERT_EQ(collect_infos.size(), 1u); + const auto& collect_info = collect_infos.begin()->second; + EXPECT_TRUE(collect_info.unique_terms.empty()); + ASSERT_EQ(collect_info.logical_scoring_leaves.size(), 1u); + EXPECT_TRUE(collect_info.logical_scoring_leaves[0].clauses.empty()); +} + +TEST_F(CollectionStatisticsTest, SearchMatchCollectsRawExecutionTerm) { + auto tablet_schema = create_tablet_schema_with_inverted_index(); + auto contexts = create_search_contexts("MATCH", "alpha beta"); + CollectInfoMap collect_infos; + + auto status = stats_->extract_collect_info(runtime_state_.get(), contexts, tablet_schema, + &collect_infos); + + ASSERT_TRUE(status.ok()) << status.msg(); + ASSERT_EQ(collect_infos.size(), 1u); + const auto& collect_info = collect_infos.begin()->second; + EXPECT_EQ(collect_info.unique_terms, std::vector({"alpha beta"})); + ASSERT_EQ(collect_info.logical_scoring_leaves.size(), 1u); + ASSERT_EQ(collect_info.logical_scoring_leaves[0].clauses.size(), 1u); + EXPECT_EQ(collect_info.logical_scoring_leaves[0].clauses[0].df_slot, 0u); +} + +TEST_F(CollectionStatisticsTest, MatchSelectsOnlyTheRuntimeAnalyzerIndex) { + auto tablet_schema = create_tablet_schema_with_two_fulltext_indexes(); + auto match_expr = std::make_shared(TExprNodeType::MATCH_PRED); + match_expr->_children.push_back( + std::make_shared("content", SlotId(1))); + match_expr->_children.push_back( + std::make_shared("running quickly")); + + InvertedIndexAnalyzerConfig config; + config.analyzer_name = "english"; + config.parser_type = InvertedIndexParserType::PARSER_ENGLISH; + config.stop_words = "none"; + auto analyzer_ctx = std::make_shared(); + analyzer_ctx->analyzer_name = "english"; + analyzer_ctx->analyzer_provider = + segment_v2::inverted_index::InvertedIndexAnalyzer::create_analyzer_provider(&config); + match_expr->set_analyzer_ctx(std::move(analyzer_ctx)); + + MatchPredicateCollector collector; + CollectInfoMap collect_infos; + auto status = + collector.collect(runtime_state_.get(), tablet_schema, match_expr, &collect_infos); + + ASSERT_TRUE(status.ok()) << status.msg(); + ASSERT_EQ(collect_infos.size(), 1u); + const auto& collect_info = collect_infos.begin()->second; + ASSERT_NE(collect_info.index_meta, nullptr); + EXPECT_EQ(collect_info.index_meta->index_id(), 20); + EXPECT_EQ(collect_info.logical_scoring_leaves.size(), 1u); +} + +TEST_F(CollectionStatisticsTest, MatchArrayStringSelectsFulltextLeafIndex) { + auto tablet_schema = create_array_tablet_schema_with_keyword_and_fulltext_indexes(); + auto match_expr = std::make_shared(TExprNodeType::MATCH_PRED); + match_expr->_children.push_back( + std::make_shared("content", SlotId(1))); + match_expr->_children.push_back( + std::make_shared("alpha beta")); + + MatchPredicateCollector collector; + CollectInfoMap collect_infos; + auto status = + collector.collect(runtime_state_.get(), tablet_schema, match_expr, &collect_infos); + + ASSERT_TRUE(status.ok()) << status.msg(); + ASSERT_EQ(collect_infos.size(), 1u); + ASSERT_NE(collect_infos.begin()->second.index_meta, nullptr); + EXPECT_EQ(collect_infos.begin()->second.index_meta->index_id(), 20); +} + +TEST_F(CollectionStatisticsTest, SearchTermSelectsOnlyTheRuntimeFullTextIndex) { + auto tablet_schema = create_tablet_schema_with_keyword_and_fulltext_indexes(); + auto contexts = create_search_contexts("TERM", "alpha beta"); + CollectInfoMap collect_infos; + + auto status = stats_->extract_collect_info(runtime_state_.get(), contexts, tablet_schema, + &collect_infos); + + ASSERT_TRUE(status.ok()) << status.msg(); + ASSERT_EQ(collect_infos.size(), 1u); + const auto& collect_info = collect_infos.begin()->second; + ASSERT_NE(collect_info.index_meta, nullptr); + EXPECT_EQ(collect_info.index_meta->index_id(), 20); + EXPECT_EQ(collect_info.logical_scoring_leaves.size(), 1u); +} + +TEST_F(CollectionStatisticsTest, SearchArrayStringSelectsFulltextLeafIndex) { + auto tablet_schema = create_array_tablet_schema_with_keyword_and_fulltext_indexes(); + auto contexts = create_search_contexts("TERM", "alpha beta"); + CollectInfoMap collect_infos; + + auto status = stats_->extract_collect_info(runtime_state_.get(), contexts, tablet_schema, + &collect_infos); + + ASSERT_TRUE(status.ok()) << status.msg(); + ASSERT_EQ(collect_infos.size(), 1u); + ASSERT_NE(collect_infos.begin()->second.index_meta, nullptr); + EXPECT_EQ(collect_infos.begin()->second.index_meta->index_id(), 20); +} + +TEST_F(CollectionStatisticsTest, SearchExactIgnoresAnalyzedBindingHint) { + auto tablet_schema = create_tablet_schema_with_keyword_and_fulltext_indexes(); + TSearchClause clause; + clause.clause_type = "EXACT"; + clause.field_name = "content"; + clause.value = "running quickly"; + clause.__isset.field_name = true; + clause.__isset.value = true; + TSearchFieldBinding binding; + binding.field_name = "content"; + binding.slot_index = 0; + binding.index_properties["parser"] = "english"; + binding.index_properties["support_phrase"] = "true"; + binding.__isset.index_properties = true; + CollectInfoMap collect_infos; + + auto status = stats_->extract_collect_info( + runtime_state_.get(), create_search_contexts(std::move(clause), {std::move(binding)}), + tablet_schema, &collect_infos); + + ASSERT_TRUE(status.ok()) << status.msg(); + ASSERT_EQ(collect_infos.size(), 1u); + ASSERT_NE(collect_infos.begin()->second.index_meta, nullptr); + EXPECT_EQ(collect_infos.begin()->second.index_meta->index_id(), 10); + EXPECT_EQ(collect_infos.begin()->second.unique_terms, + std::vector({"running quickly"})); +} + +TEST_F(CollectionStatisticsTest, SearchScoringRejectsMissingField) { + auto tablet_schema = create_tablet_schema_with_inverted_index(); + TSearchClause clause; + clause.clause_type = "TERM"; + clause.field_name = "missing"; + clause.value = "alpha"; + clause.__isset.field_name = true; + clause.__isset.value = true; + CollectInfoMap collect_infos; + + auto status = stats_->extract_collect_info(runtime_state_.get(), + create_search_contexts(std::move(clause)), + tablet_schema, &collect_infos); + + EXPECT_EQ(status.code(), ErrorCode::INVERTED_INDEX_NOT_SUPPORTED); + EXPECT_TRUE(collect_infos.empty()); +} + +TEST_F(CollectionStatisticsTest, SearchScoringRejectsMissingIndex) { + auto tablet_schema = std::make_shared(); + TabletColumn column; + column.set_unique_id(1); + column.set_name("content"); + column.set_type(FieldType::OLAP_FIELD_TYPE_STRING); + tablet_schema->append_column(column); + CollectInfoMap collect_infos; + + auto status = stats_->extract_collect_info(runtime_state_.get(), + create_search_contexts("TERM", "alpha"), + tablet_schema, &collect_infos); + + EXPECT_EQ(status.code(), ErrorCode::INVERTED_INDEX_NOT_SUPPORTED); + EXPECT_TRUE(collect_infos.empty()); +} + +TEST_F(CollectionStatisticsTest, SearchTypedVariantBindingSelectsItsAnalyzerIndex) { + auto tablet_schema = std::make_shared(); + constexpr int32_t kVariantUid = 9010; + + TabletColumn variant_column; + variant_column.set_unique_id(kVariantUid); + variant_column.set_name("v"); + variant_column.set_type(FieldType::OLAP_FIELD_TYPE_VARIANT); + tablet_schema->append_column(variant_column); + + TabletColumn subcolumn; + subcolumn.set_unique_id(-1); + subcolumn.set_name("v.host"); + subcolumn.set_type(FieldType::OLAP_FIELD_TYPE_STRING); + subcolumn.set_parent_unique_id(kVariantUid); + subcolumn.set_path_info(PathInData("v.host", true)); + tablet_schema->append_column(subcolumn); + + TabletSchema::PathsSetInfo path_set_info; + TabletSchema::SubColumnInfo typed_path_info; + typed_path_info.column = subcolumn; + for (const auto& [index_id, parser] : {std::pair {3010, "standard"}, + std::pair {3020, "english"}}) { + auto index = std::make_shared(); + TabletIndexPB index_pb; + index_pb.set_index_id(index_id); + index_pb.set_index_name(parser + "_variant_idx"); + index_pb.set_index_type(IndexType::INVERTED); + index_pb.add_col_unique_id(kVariantUid); + (*index_pb.mutable_properties())["parser"] = parser; + (*index_pb.mutable_properties())["support_phrase"] = "true"; + index->init_from_pb(index_pb); + typed_path_info.indexes.push_back(std::move(index)); + } + path_set_info.typed_path_set.emplace("host", std::move(typed_path_info)); + std::unordered_map path_set_info_map; + path_set_info_map.emplace(kVariantUid, std::move(path_set_info)); + tablet_schema->set_path_set_info(std::move(path_set_info_map)); + + TSearchClause clause; + clause.clause_type = "TERM"; + clause.field_name = "v.host"; + clause.value = "running"; + clause.__isset.field_name = true; + clause.__isset.value = true; + TSearchFieldBinding binding; + binding.field_name = "v.host"; + binding.slot_index = 0; + binding.is_variant_subcolumn = true; + binding.__isset.is_variant_subcolumn = true; + binding.parent_field_name = "v"; + binding.__isset.parent_field_name = true; + binding.subcolumn_path = "host"; + binding.__isset.subcolumn_path = true; + binding.index_properties["parser"] = "english"; + binding.index_properties["support_phrase"] = "true"; + binding.__isset.index_properties = true; + CollectInfoMap collect_infos; + + auto status = stats_->extract_collect_info( + runtime_state_.get(), create_search_contexts(std::move(clause), {std::move(binding)}), + tablet_schema, &collect_infos); + + ASSERT_TRUE(status.ok()) << status.msg(); + ASSERT_EQ(collect_infos.size(), 1u); + const auto& collect_info = collect_infos.begin()->second; + ASSERT_NE(collect_info.index_meta, nullptr); + EXPECT_EQ(collect_info.index_meta->index_id(), 3020); + EXPECT_EQ(collect_info.unique_terms, std::vector({"running"})); +} + +TEST_F(CollectionStatisticsTest, SearchScoringUsesTextSemanticsForVariantParentIndexFallback) { + auto tablet_schema = std::make_shared(); + constexpr int32_t kVariantUid = 9015; + + TabletColumn variant_column; + variant_column.set_unique_id(kVariantUid); + variant_column.set_name("v"); + variant_column.set_type(FieldType::OLAP_FIELD_TYPE_VARIANT); + tablet_schema->append_column(variant_column); + + TabletIndex parent_index; + parent_index._index_id = 3025; + parent_index._index_type = IndexType::INVERTED; + parent_index._col_unique_ids.push_back(kVariantUid); + parent_index._properties["parser"] = "standard"; + parent_index._properties["support_phrase"] = "true"; + tablet_schema->append_index(std::move(parent_index)); + + TSearchClause clause; + clause.clause_type = "PHRASE"; + clause.field_name = "v.dynamic"; + clause.value = "alpha beta"; + clause.__isset.field_name = true; + clause.__isset.value = true; + TSearchFieldBinding binding; + binding.field_name = "v.dynamic"; + binding.slot_index = 0; + binding.is_variant_subcolumn = true; + binding.__isset.is_variant_subcolumn = true; + binding.parent_field_name = "v"; + binding.__isset.parent_field_name = true; + binding.subcolumn_path = "dynamic"; + binding.__isset.subcolumn_path = true; + binding.index_properties["parser"] = "standard"; + binding.index_properties["support_phrase"] = "true"; + binding.__isset.index_properties = true; + CollectInfoMap collect_infos; + + auto status = stats_->extract_collect_info( + runtime_state_.get(), create_search_contexts(std::move(clause), {std::move(binding)}), + tablet_schema, &collect_infos); + + ASSERT_TRUE(status.ok()) << status; + ASSERT_EQ(collect_infos.size(), 1U); + auto it = collect_infos.find( + StringHelper::to_wstring(std::to_string(kVariantUid) + ".v.dynamic")); + ASSERT_NE(it, collect_infos.end()); + ASSERT_NE(it->second.index_meta, nullptr); + ASSERT_NE(it->second.owned_index_meta, nullptr); + EXPECT_EQ(it->second.index_meta->index_id(), 3025); + EXPECT_EQ(it->second.unique_terms, std::vector({"alpha", "beta"})); +} + +TEST_F(CollectionStatisticsTest, SearchVariantFieldPatternKeepsSelectedMetadataAlive) { + auto tablet_schema = std::make_shared(); + constexpr int32_t kVariantUid = 9020; + + TabletColumn variant_column; + variant_column.set_unique_id(kVariantUid); + variant_column.set_name("meta"); + variant_column.set_type(FieldType::OLAP_FIELD_TYPE_VARIANT); + auto subcolumn_template = make_subcolumn_template("user.*", PatternTypePB::MATCH_NAME_GLOB); + variant_column.add_sub_column(subcolumn_template); + tablet_schema->append_column(variant_column); + + TabletIndexPB index_pb; + index_pb.set_index_id(3030); + index_pb.set_index_name("variant_search_field_pattern_idx"); + index_pb.set_index_type(IndexType::INVERTED); + index_pb.add_col_unique_id(kVariantUid); + (*index_pb.mutable_properties())["parser"] = "standard"; + (*index_pb.mutable_properties())["support_phrase"] = "true"; + (*index_pb.mutable_properties())["field_pattern"] = "user.*"; + TabletIndex index; + index.init_from_pb(index_pb); + tablet_schema->append_index(std::move(index)); + + TSearchClause clause; + clause.clause_type = "PHRASE"; + clause.field_name = "meta.user.name"; + clause.value = "alice smith"; + clause.__isset.field_name = true; + clause.__isset.value = true; + TSearchFieldBinding binding; + binding.field_name = "meta.user.name"; + binding.slot_index = 0; + binding.is_variant_subcolumn = true; + binding.__isset.is_variant_subcolumn = true; + binding.parent_field_name = "meta"; + binding.__isset.parent_field_name = true; + binding.subcolumn_path = "user.name"; + binding.__isset.subcolumn_path = true; + binding.index_properties["parser"] = "standard"; + binding.index_properties["support_phrase"] = "true"; + binding.__isset.index_properties = true; + CollectInfoMap collect_infos; + + auto status = stats_->extract_collect_info( + runtime_state_.get(), create_search_contexts(std::move(clause), {std::move(binding)}), + tablet_schema, &collect_infos); + + ASSERT_TRUE(status.ok()) << status.msg(); + ASSERT_EQ(collect_infos.size(), 1u); + auto iter = collect_infos.find( + StringHelper::to_wstring(std::to_string(kVariantUid) + ".meta.user.name")); + ASSERT_NE(iter, collect_infos.end()); + ASSERT_NE(iter->second.index_meta, nullptr); + ASSERT_NE(iter->second.owned_index_meta, nullptr); + EXPECT_EQ(iter->second.index_meta->index_id(), 3030); +} + +TEST_F(CollectionStatisticsTest, SearchScoringRejectsNumericBkdLeaf) { + auto tablet_schema = std::make_shared(); + TabletColumn column; + column.set_unique_id(2); + column.set_name("number"); + column.set_type(FieldType::OLAP_FIELD_TYPE_INT); + tablet_schema->append_column(column); + TabletIndex index; + index._index_id = 3040; + index._index_type = IndexType::INVERTED; + index._col_unique_ids.push_back(2); + tablet_schema->append_index(std::move(index)); + TSearchClause clause; + clause.clause_type = "TERM"; + clause.field_name = "number"; + clause.value = "42"; + clause.__isset.field_name = true; + clause.__isset.value = true; + CollectInfoMap collect_infos; + + auto status = stats_->extract_collect_info(runtime_state_.get(), + create_search_contexts(std::move(clause)), + tablet_schema, &collect_infos); + + EXPECT_EQ(status.code(), ErrorCode::INVERTED_INDEX_NOT_SUPPORTED); + EXPECT_TRUE(collect_infos.empty()); +} + +TEST_F(CollectionStatisticsTest, NestedSearchScoringIsRejected) { + TSearchClause phrase; + phrase.clause_type = "PHRASE"; + phrase.field_name = "content"; + phrase.value = "alpha beta"; + phrase.__isset.field_name = true; + phrase.__isset.value = true; + + TSearchClause nested; + nested.clause_type = "NESTED"; + nested.children.push_back(std::move(phrase)); + nested.__isset.children = true; + + CollectInfoMap collect_infos; + auto status = stats_->extract_collect_info( + runtime_state_.get(), create_search_contexts(std::move(nested)), + create_tablet_schema_with_inverted_index(), &collect_infos); + + EXPECT_EQ(status.code(), ErrorCode::INVERTED_INDEX_NOT_SUPPORTED); + EXPECT_TRUE(collect_infos.empty()); +} + +TEST_F(CollectionStatisticsTest, OneScoringFieldCannotSelectDifferentPhysicalIndexes) { + auto tablet_schema = create_tablet_schema_with_two_fulltext_indexes(); + auto build_match = [](const std::string& analyzer_name, InvertedIndexParserType parser_type) { + auto match_expr = + std::make_shared(TExprNodeType::MATCH_PRED); + match_expr->_children.push_back( + std::make_shared("content", SlotId(1))); + match_expr->_children.push_back( + std::make_shared("alpha beta")); + + InvertedIndexAnalyzerConfig config; + config.analyzer_name = analyzer_name; + config.parser_type = parser_type; + config.stop_words = "none"; + auto analyzer_ctx = std::make_shared(); + analyzer_ctx->analyzer_name = analyzer_name; + analyzer_ctx->analyzer_provider = + segment_v2::inverted_index::InvertedIndexAnalyzer::create_analyzer_provider( + &config); + match_expr->set_analyzer_ctx(std::move(analyzer_ctx)); + return match_expr; + }; + + MatchPredicateCollector collector; + CollectInfoMap collect_infos; + auto first = collector.collect( + runtime_state_.get(), tablet_schema, + build_match("standard", InvertedIndexParserType::PARSER_STANDARD), &collect_infos); + ASSERT_TRUE(first.ok()) << first.msg(); + + auto second = collector.collect(runtime_state_.get(), tablet_schema, + build_match("english", InvertedIndexParserType::PARSER_ENGLISH), + &collect_infos); + + EXPECT_EQ(second.code(), ErrorCode::INVERTED_INDEX_NOT_SUPPORTED); } // Test-only subclass that exposes the protected helpers of PredicateCollector. @@ -1270,41 +2890,4 @@ TEST_F(CollectionStatisticsTest, BuildFieldNameWithoutSuffix) { EXPECT_EQ(collector.build_field_name(42, ""), "42"); } -TEST(TermInfoComparerTest, OrdersByTermAndDedups) { - using doris::TermInfoComparer; - using doris::segment_v2::TermInfo; - - std::set terms; - - TermInfo t1; - t1.term = std::string("banana"); - t1.position = 2; - - TermInfo t2; - t2.term = std::string("apple"); - t2.position = 10; - - TermInfo t3; - t3.term = std::string("cherry"); - t3.position = 1; - - TermInfo dup; - dup.term = std::string("banana"); - dup.position = 100; - - terms.insert(t1); - terms.insert(t2); - terms.insert(t3); - terms.insert(dup); - - std::vector ordered; - ordered.reserve(terms.size()); - for (const auto& t : terms) { - ordered.push_back(t.get_single_term()); - } - - EXPECT_EQ(terms.size(), 3u); - EXPECT_THAT(ordered, ::testing::ElementsAre("apple", "banana", "cherry")); -} - } // namespace doris diff --git a/be/test/storage/compaction/segcompaction_test.cpp b/be/test/storage/compaction/segcompaction_test.cpp index d3b843c050da2e..2c6697bc592c59 100644 --- a/be/test/storage/compaction/segcompaction_test.cpp +++ b/be/test/storage/compaction/segcompaction_test.cpp @@ -19,12 +19,15 @@ #include #include +#include +#include #include #include #include #include #include "common/config.h" +#include "cpp/sync_point.h" #include "io/fs/local_file_system.h" #include "runtime/exec_env.h" #include "runtime/memory/mem_tracker.h" @@ -114,6 +117,10 @@ class SegCompactionTest : public testing::Test { } void TearDown() { + auto* sync_point = SyncPoint::get_instance(); + sync_point->disable_processing(); + sync_point->clear_all_call_backs(); + sync_point->clear_trace(); config::enable_segcompaction = false; ExecEnv* exec_env = doris::ExecEnv::GetInstance(); l_engine = nullptr; @@ -288,6 +295,55 @@ class SegCompactionTest : public testing::Test { std::unique_ptr _inverted_index_searcher_cache; }; +TEST_F(SegCompactionTest, FatalStatusPreservesOriginalError) { + config::segcompaction_candidate_max_rows = 10; + config::segcompaction_batch_size = 2; + + auto tablet_schema = std::make_shared(); + create_tablet_schema(tablet_schema, DUP_KEYS); + + RowsetWriterContext writer_context; + create_rowset_writer_context(10046, tablet_schema, &writer_context); + auto writer_result = RowsetFactory::create_rowset_writer(*l_engine, writer_context, false); + ASSERT_TRUE(writer_result.has_value()) << writer_result.error(); + auto rowset_writer = std::move(writer_result).value(); + + const auto expected_status = Status::Error( + "CommonGrams analysis failed; set enable_common_grams_index_build=false and retry " + "the load as a new transaction"); + std::promise worker_started; + auto worker_started_future = worker_started.get_future(); + auto* sync_point = SyncPoint::get_instance(); + SyncPoint::CallbackGuard callback_guard; + sync_point->set_call_back( + "SegcompactionWorker::_do_compact_segments", + [&worker_started, expected_status](auto&& args) { + auto* result = try_any_cast_ret(args); + result->first = expected_status; + result->second = true; + worker_started.set_value(); + }, + &callback_guard); + sync_point->enable_processing(); + + for (int segment_id = 0; segment_id < 3; ++segment_id) { + Block block = tablet_schema->create_block(); + auto columns = std::move(block).mutate_columns(); + for (uint32_t column_id = 0; column_id < columns.size(); ++column_id) { + const uint32_t value = segment_id + column_id; + columns[column_id]->insert_data(reinterpret_cast(&value), sizeof(value)); + } + ASSERT_TRUE(add_block_with_columns(rowset_writer.get(), &block, &columns).ok()); + ASSERT_TRUE(rowset_writer->flush().ok()); + } + ASSERT_EQ(worker_started_future.wait_for(std::chrono::seconds(10)), std::future_status::ready); + + RowsetSharedPtr rowset; + const auto status = rowset_writer->build(rowset); + EXPECT_EQ(status.code(), INVERTED_INDEX_ANALYZER_ERROR); + EXPECT_EQ(status.to_string(), expected_status.to_string()); +} + TEST_F(SegCompactionTest, SegCompactionThenRead) { config::enable_segcompaction = true; Status s; diff --git a/be/test/storage/index/index_builder_test.cpp b/be/test/storage/index/index_builder_test.cpp index 4d5d890c887bb5..23141021e831be 100644 --- a/be/test/storage/index/index_builder_test.cpp +++ b/be/test/storage/index/index_builder_test.cpp @@ -20,7 +20,12 @@ #include #include +#include + +#include "common/config.h" +#include "storage/index/index_file_reader.h" #include "storage/index/index_writer.h" +#include "storage/index/snii/query/term_query.h" #include "storage/olap_common.h" #include "storage/rowset/beta_rowset.h" #include "storage/rowset/rowset_factory.h" @@ -28,10 +33,29 @@ #include "storage/storage_engine.h" #include "storage/tablet/tablet_fwd.h" #include "storage/tablet/tablet_schema.h" +#include "util/debug_points.h" namespace doris { using namespace testing; +class ScopedIndexBuilderDebugPoints { +public: + ScopedIndexBuilderDebugPoints() : _debug_points_enabled(config::enable_debug_points) { + config::enable_debug_points = true; + DebugPoints::instance()->clear(); + } + + ~ScopedIndexBuilderDebugPoints() { + DebugPoints::instance()->clear(); + config::enable_debug_points = _debug_points_enabled; + } + + void enable(const std::string& name) { DebugPoints::instance()->add(name); } + +private: + bool _debug_points_enabled; +}; + class IndexBuilderTest : public ::testing::Test { protected: void SetUp() override { @@ -170,6 +194,230 @@ class IndexBuilderTest : public ::testing::Test { rs_meta->set_tablet_schema(tablet_schema); } + void prepare_single_index_build(int64_t rowset_id) { + auto tablet_path = _absolute_dir + "/" + std::to_string(rowset_id); + _tablet->_tablet_path = tablet_path; + ASSERT_TRUE(io::global_local_filesystem()->delete_directory(tablet_path).ok()); + ASSERT_TRUE(io::global_local_filesystem()->create_directory(tablet_path).ok()); + + RowsetWriterContext writer_context; + writer_context.rowset_id.init(rowset_id); + writer_context.tablet_id = _tablet->tablet_id(); + writer_context.tablet_schema_hash = _tablet_meta->schema_hash(); + writer_context.partition_id = 10; + writer_context.rowset_type = BETA_ROWSET; + writer_context.tablet_path = tablet_path; + writer_context.rowset_state = VISIBLE; + writer_context.tablet_schema = _tablet_schema; + writer_context.version = Version(10, 10); + + auto result = RowsetFactory::create_rowset_writer(*_engine_ref, writer_context, false); + ASSERT_TRUE(result.has_value()) << result.error(); + auto rowset_writer = std::move(result).value(); + + Block block = _tablet_schema->create_block(); + auto columns = std::move(block).mutate_columns(); + for (int i = 0; i < 8; ++i) { + int32_t k1 = i * 10; + int32_t k2 = i; + columns[0]->insert_data(reinterpret_cast(&k1), sizeof(k1)); + columns[1]->insert_data(reinterpret_cast(&k2), sizeof(k2)); + } + block.set_columns(std::move(columns)); + ASSERT_TRUE(rowset_writer->add_block(&block).ok()); + ASSERT_TRUE(rowset_writer->flush().ok()); + + RowsetSharedPtr rowset; + ASSERT_TRUE(rowset_writer->build(rowset).ok()); + ASSERT_TRUE(_tablet->add_rowset(rowset).ok()); + + TOlapTableIndex index; + index.index_id = 101; + index.index_name = "k1_index"; + index.columns.emplace_back("k1"); + index.column_unique_ids.push_back(1); + index.index_type = TIndexType::INVERTED; + _alter_indexes.push_back(std::move(index)); + } + + Status build_single_index() { + IndexBuilder builder(*_engine_ref, _tablet, _columns, _alter_indexes, false); + RETURN_IF_ERROR(builder.init()); + return builder.do_build_inverted_index(); + } + + static TabletSchemaSPtr create_snii_drop_schema() { + TabletSchemaPB schema_pb; + schema_pb.set_keys_type(DUP_KEYS); + schema_pb.set_inverted_index_storage_format(InvertedIndexStorageFormatPB::SNII); + auto tablet_schema = std::make_shared(); + tablet_schema->init_from_pb(schema_pb); + + TabletColumn key_column; + key_column.set_unique_id(1); + key_column.set_name("k1"); + key_column.set_type(FieldType::OLAP_FIELD_TYPE_INT); + key_column.set_length(4); + key_column.set_index_length(4); + key_column.set_is_key(true); + key_column.set_is_nullable(false); + tablet_schema->append_column(key_column); + + for (const auto& [unique_id, name] : {std::pair {2, "body_a"}, + std::pair {3, "body_b"}}) { + TabletColumn column; + column.set_unique_id(unique_id); + column.set_name(std::string(name)); + column.set_type(FieldType::OLAP_FIELD_TYPE_VARCHAR); + column.set_length(65535); + column.set_is_nullable(false); + tablet_schema->append_column(column); + } + + for (const auto& [index_id, index_name, column_unique_id] : + {std::tuple {1, "idx_a", 2}, + std::tuple {2, "idx_b", 3}}) { + TabletIndex index; + index._index_id = index_id; + index._index_name = index_name; + index._index_type = IndexType::INVERTED; + index._col_unique_ids.push_back(column_unique_id); + index._properties["parser"] = "english"; + index._properties["lower_case"] = "true"; + index._properties["support_phrase"] = "true"; + tablet_schema->append_index(std::move(index)); + } + return tablet_schema; + } + + Status create_snii_drop_tablet(const TabletSchemaSPtr& tablet_schema, + const std::string& tablet_path, TabletSharedPtr* tablet) { + RETURN_IF_ERROR(io::global_local_filesystem()->delete_directory(tablet_path)); + RETURN_IF_ERROR(io::global_local_filesystem()->create_directory(tablet_path)); + auto tablet_meta = create_tablet_meta(); + tablet_meta->_schema = tablet_schema; + *tablet = std::make_shared(*_engine_ref, tablet_meta, _data_dir.get()); + (*tablet)->_tablet_path = tablet_path; + return (*tablet)->init(); + } + + Status create_snii_source_rowset(const TabletSharedPtr& tablet, + const TabletSchemaSPtr& tablet_schema, + const std::string& tablet_path, + RowsetSharedPtr* rowset) const { + RowsetWriterContext writer_context; + writer_context.rowset_id.init(15691); + writer_context.tablet_id = tablet->tablet_id(); + writer_context.tablet_schema_hash = tablet->schema_hash(); + writer_context.partition_id = 10; + writer_context.rowset_type = BETA_ROWSET; + writer_context.tablet_path = tablet_path; + writer_context.rowset_state = VISIBLE; + writer_context.tablet_schema = tablet_schema; + writer_context.version = Version(10, 10); + + auto rowset_writer = + DORIS_TRY(RowsetFactory::create_rowset_writer(*_engine_ref, writer_context, false)); + Block block = tablet_schema->create_block(); + auto columns = std::move(block).mutate_columns(); + const std::vector dropped_values = {"drop alpha", "drop beta"}; + const std::vector surviving_values = {"keep alpha", "keep beta"}; + for (int32_t i = 0; i < 2; ++i) { + columns[0]->insert_data(reinterpret_cast(&i), sizeof(i)); + columns[1]->insert_data(dropped_values[i].data(), dropped_values[i].size()); + columns[2]->insert_data(surviving_values[i].data(), surviving_values[i].size()); + } + block = tablet_schema->create_block(); + block.set_columns(std::move(columns)); + RETURN_IF_ERROR(rowset_writer->add_block(&block)); + RETURN_IF_ERROR(rowset_writer->flush()); + RETURN_IF_ERROR(rowset_writer->build(*rowset)); + return tablet->add_rowset(*rowset); + } + + static TOlapTableIndex create_drop_index(int64_t index_id, std::string index_name, + std::string column_name, int32_t column_unique_id) { + TOlapTableIndex index; + index.index_id = index_id; + index.index_name = std::move(index_name); + index.index_type = TIndexType::INVERTED; + index.columns.emplace_back(std::move(column_name)); + index.column_unique_ids.push_back(column_unique_id); + return index; + } + + Status drop_snii_index(const TabletSharedPtr& tablet, TOlapTableIndex index, + RowsetSharedPtr* output_rowset) const { + std::vector drop_indexes {std::move(index)}; + IndexBuilder builder(*_engine_ref, tablet, _columns, drop_indexes, true); + RETURN_IF_ERROR(builder.init()); + RETURN_IF_ERROR(builder.do_build_inverted_index()); + DORIS_CHECK_EQ(builder._output_rowsets.size(), 1); + *output_rowset = builder._output_rowsets.front(); + return Status::OK(); + } + + static void assert_snii_surviving_index(const RowsetSharedPtr& source_rowset, + const RowsetSharedPtr& output_rowset) { + const auto& output_schema = output_rowset->tablet_schema(); + EXPECT_FALSE(output_schema->has_inverted_index_with_index_id(1)); + ASSERT_TRUE(output_schema->has_inverted_index_with_index_id(2)); + EXPECT_EQ(output_rowset->index_disk_size(), source_rowset->index_disk_size()); + EXPECT_EQ(output_rowset->data_disk_size(), source_rowset->data_disk_size()); + EXPECT_EQ(output_rowset->total_disk_size(), source_rowset->total_disk_size()); + + auto source_segment_path = source_rowset->segment_path(0); + ASSERT_TRUE(source_segment_path.has_value()) << source_segment_path.error(); + auto output_segment_path = output_rowset->segment_path(0); + ASSERT_TRUE(output_segment_path.has_value()) << output_segment_path.error(); + const auto source_index_path = segment_v2::InvertedIndexDescriptor::get_index_file_path_v2( + segment_v2::InvertedIndexDescriptor::get_index_file_path_prefix( + source_segment_path.value())); + const auto output_index_path = segment_v2::InvertedIndexDescriptor::get_index_file_path_v2( + segment_v2::InvertedIndexDescriptor::get_index_file_path_prefix( + output_segment_path.value())); + std::error_code equivalent_error; + EXPECT_TRUE( + std::filesystem::equivalent(source_index_path, output_index_path, equivalent_error)) + << equivalent_error.message(); + } + + static void assert_snii_term_query(const RowsetSharedPtr& rowset, int64_t tablet_id) { + auto segment_path = rowset->segment_path(0); + ASSERT_TRUE(segment_path.has_value()) << segment_path.error(); + const std::string index_path_prefix { + segment_v2::InvertedIndexDescriptor::get_index_file_path_prefix( + segment_path.value())}; + segment_v2::IndexFileReader index_file_reader( + io::global_local_filesystem(), index_path_prefix, + InvertedIndexStorageFormatPB::SNII, InvertedIndexFileInfo(), tablet_id); + ASSERT_TRUE(index_file_reader.init().ok()); + + const auto& surviving_indexes = rowset->tablet_schema()->inverted_indexs(3); + ASSERT_EQ(surviving_indexes.size(), 1); + auto logical_index = index_file_reader.open_snii_index(surviving_indexes.front()); + ASSERT_TRUE(logical_index.has_value()) << logical_index.error(); + std::vector docids; + ASSERT_TRUE(snii::query::term_query(*logical_index.value(), "keep", &docids).ok()); + EXPECT_EQ(docids, (std::vector {0, 1})); + } + + static void assert_last_snii_index_dropped(const RowsetSharedPtr& source_rowset, + const RowsetSharedPtr& rowset) { + EXPECT_FALSE(rowset->tablet_schema()->has_inverted_index()); + EXPECT_EQ(rowset->index_disk_size(), 0); + EXPECT_EQ(rowset->data_disk_size(), source_rowset->data_disk_size()); + EXPECT_EQ(rowset->total_disk_size(), source_rowset->data_disk_size()); + auto segment_path = rowset->segment_path(0); + ASSERT_TRUE(segment_path.has_value()) << segment_path.error(); + const auto index_path = segment_v2::InvertedIndexDescriptor::get_index_file_path_v2( + segment_v2::InvertedIndexDescriptor::get_index_file_path_prefix( + segment_path.value())); + bool index_exists = true; + ASSERT_TRUE(io::global_local_filesystem()->exists(index_path, &index_exists).ok()); + EXPECT_FALSE(index_exists); + } + StorageEngine* _engine_ref = nullptr; TabletSharedPtr _tablet; TabletMetaSharedPtr _tablet_meta; @@ -203,6 +451,17 @@ TEST_F(IndexBuilderTest, BasicBuildTest) { EXPECT_EQ(builder._alter_index_ids.size(), 1); } +TEST_F(IndexBuilderTest, HandleSingleRowsetPreservesOrdinaryAppendFailure) { + prepare_single_index_build(16604); + ScopedIndexBuilderDebugPoints debug_points; + debug_points.enable("IndexBuilder::handle_single_rowset_write_inverted_index_data_error"); + + auto status = build_single_index(); + + EXPECT_TRUE(status.is()) << status; + EXPECT_EQ(status.msg(), "debug point: handle_single_rowset_write_inverted_index_data_error"); +} + TEST_F(IndexBuilderTest, DropInvertedIndexTest) { // 0. prepare tablet path auto tablet_path = _absolute_dir + "/" + std::to_string(15676); @@ -3132,4 +3391,27 @@ TEST_F(IndexBuilderTest, DropOneIndexNotAffectOtherIndexesOnSameColumnTest) { << "Should have exactly 1 inverted index remaining after drop"; } +TEST_F(IndexBuilderTest, DropOneSniiIndexPreservesSurvivingPhysicalIndex) { + const auto tablet_path = _absolute_dir + "/15691"; + auto tablet_schema = create_snii_drop_schema(); + TabletSharedPtr tablet; + ASSERT_TRUE(create_snii_drop_tablet(tablet_schema, tablet_path, &tablet).ok()); + RowsetSharedPtr source_rowset; + ASSERT_TRUE(create_snii_source_rowset(tablet, tablet_schema, tablet_path, &source_rowset).ok()); + ASSERT_GT(source_rowset->index_disk_size(), 0); + + RowsetSharedPtr output_rowset; + ASSERT_TRUE(drop_snii_index(tablet, create_drop_index(1, "idx_a", "body_a", 2), &output_rowset) + .ok()); + assert_snii_surviving_index(source_rowset, output_rowset); + assert_snii_term_query(output_rowset, tablet->tablet_id()); + + ScopedIndexBuilderDebugPoints debug_points; + debug_points.enable("IndexBuilder::update_inverted_index_info_index_file_reader_init_not_ok"); + RowsetSharedPtr final_rowset; + ASSERT_TRUE(drop_snii_index(tablet, create_drop_index(2, "idx_b", "body_b", 3), &final_rowset) + .ok()); + assert_last_snii_index_dropped(source_rowset, final_rowset); +} + } // namespace doris diff --git a/be/test/storage/index/inverted/ananlyzer/analyzer_test.cpp b/be/test/storage/index/inverted/ananlyzer/analyzer_test.cpp index 4a51609a68806d..6d7d6e02221f6d 100644 --- a/be/test/storage/index/inverted/ananlyzer/analyzer_test.cpp +++ b/be/test/storage/index/inverted/ananlyzer/analyzer_test.cpp @@ -345,4 +345,23 @@ TEST_F(AnalyzerTest, TestAnalyzerFunctionality) { } } +TEST_F(AnalyzerTest, CommonGramsQueryPurposeSelection) { + EXPECT_EQ(select_analysis_purpose(InvertedIndexQueryType::MATCH_ANY_QUERY, 0, false), + AnalysisPurpose::kPlainQuery); + EXPECT_EQ(select_analysis_purpose(InvertedIndexQueryType::MATCH_ALL_QUERY, 0, false), + AnalysisPurpose::kPlainQuery); + EXPECT_EQ(select_analysis_purpose(InvertedIndexQueryType::MATCH_PHRASE_QUERY, 0, false), + AnalysisPurpose::kExactPhraseQuery); + EXPECT_EQ(select_analysis_purpose(InvertedIndexQueryType::MATCH_PHRASE_QUERY, 2, false), + AnalysisPurpose::kPlainQuery); + EXPECT_EQ(select_analysis_purpose(InvertedIndexQueryType::MATCH_PHRASE_QUERY, 0, true), + AnalysisPurpose::kPlainQuery); + EXPECT_EQ(select_analysis_purpose(InvertedIndexQueryType::MATCH_PHRASE_PREFIX_QUERY, 0, false), + AnalysisPurpose::kPhrasePrefixQuery); + EXPECT_EQ(select_analysis_purpose(InvertedIndexQueryType::MATCH_PHRASE_PREFIX_QUERY, 0, true), + AnalysisPurpose::kPlainQuery); + EXPECT_EQ(select_analysis_purpose(InvertedIndexQueryType::MATCH_REGEXP_QUERY, 0, false), + AnalysisPurpose::kPlainQuery); +} + } // namespace doris::segment_v2::inverted_index \ No newline at end of file diff --git a/be/test/storage/index/inverted/ananlyzer/common_grams_filter_test.cpp b/be/test/storage/index/inverted/ananlyzer/common_grams_filter_test.cpp new file mode 100644 index 00000000000000..9d4feb731bb150 --- /dev/null +++ b/be/test/storage/index/inverted/ananlyzer/common_grams_filter_test.cpp @@ -0,0 +1,644 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/inverted/token_filter/common_grams_filter.h" + +#include + +#include +#include +#include +#include +#include + +#include "common/exception.h" +#include "storage/index/inverted/common_grams/common_grams_key_codec.h" +#include "storage/index/inverted/common_grams/common_word_set.h" + +namespace doris::segment_v2::inverted_index { +namespace { + +struct ScriptedToken { + std::string term; + int32_t position_increment = 1; + int32_t start_offset = 0; + int32_t end_offset = 0; + const TCHAR* type = Token::getDefaultType(); +}; + +class ScriptedTokenStream final : public TokenStream { +public: + explicit ScriptedTokenStream(std::vector tokens) : _tokens(std::move(tokens)) {} + + Token* next(Token* token) override { + if (_next == _tokens.size()) { + return nullptr; + } + const auto& scripted = _tokens[_next++]; + _scratch = scripted.term; + token->clear(); + token->setTextNoCopy(_scratch.data(), static_cast(_scratch.size())); + token->positionIncrement = scripted.position_increment; + token->setStartOffset(scripted.start_offset); + token->setEndOffset(scripted.end_offset); + token->setType(scripted.type); + return token; + } + + void close() override {} + void reset() override { _next = 0; } + + void set_tokens(std::vector tokens) { + _tokens = std::move(tokens); + _next = 0; + } + + size_t consumed() const { return _next; } + +private: + std::vector _tokens; + size_t _next = 0; + std::string _scratch; +}; + +struct ActualToken { + std::string term; + int32_t position_increment; + int32_t start_offset; + int32_t end_offset; + std::wstring type; + + bool operator==(const ActualToken&) const = default; +}; + +struct TokenSemantics { + std::string term; + int32_t position; + std::wstring type; + + bool operator==(const TokenSemantics&) const = default; +}; + +template +concept HasAnalyzerCommonGramClassification = requires(Event event) { + event.has_preceding_gram; + event.preceding_gram_both_common; +}; + +static_assert(!HasAnalyzerCommonGramClassification); + +std::string gram(std::string_view left, std::string_view right) { + auto encoded = encode_common_gram(left, right); + EXPECT_TRUE(encoded.has_value()) << encoded.error(); + return encoded.value(); +} + +std::vector collect(const TokenStreamPtr& stream) { + std::vector result; + Token token; + while (stream->next(&token) != nullptr) { + result.push_back({std::string(token.termBuffer(), token.termLength()), + token.getPositionIncrement(), token.startOffset(), token.endOffset(), + token.type()}); + } + return result; +} + +std::vector collect_semantics(const TokenStreamPtr& stream) { + std::vector result; + Token token; + int32_t position = 0; + while (stream->next(&token) != nullptr) { + position += token.getPositionIncrement(); + result.push_back({std::string(token.termBuffer(), token.termLength()), position, + token.type()}); + } + return result; +} + +std::vector expand_snii_index_events(CommonGramsFilter* stream, size_t* event_count, + size_t* both_common_gram_count) { + std::vector result; + std::optional previous_logical_term; + bool previous_is_common = false; + int32_t position = 0; + SniiCommonGramsIndexEvent event; + while (stream->next_snii_index_event(&event)) { + ++*event_count; + const std::string physical_plain_term(event.plain_term); + const std::string logical_term = + decode_plain_term(physical_plain_term, PlainTermKeyVersion::kEscapedV1).value(); + EXPECT_EQ(event.logical_term, logical_term); + const bool current_is_common = + CommonWordSet::builtin_english_stop_words_v1().contains(logical_term); + const bool has_preceding_gram = previous_logical_term.has_value() && + (previous_is_common || current_is_common) && + common_gram_component_sizes_encodable( + previous_logical_term->size(), logical_term.size()); + if (has_preceding_gram) { + EXPECT_TRUE(previous_logical_term.has_value()); + *both_common_gram_count += previous_is_common && current_is_common; + result.push_back({gram(previous_logical_term.value(), logical_term), position, + COMMON_GRAM_TOKEN_TYPE}); + } + ++position; + result.push_back({physical_plain_term, position, Token::getDefaultType()}); + previous_logical_term = logical_term; + previous_is_common = current_is_common; + } + return result; +} + +std::vector words(std::initializer_list terms) { + std::vector result; + int32_t offset = 0; + for (auto term : terms) { + result.push_back( + {std::string(term), 1, offset, offset + static_cast(term.size())}); + offset += static_cast(term.size()) + 1; + } + return result; +} + +std::vector terms(const std::vector& tokens) { + std::vector result; + for (const auto& token : tokens) { + EXPECT_EQ(token.position_increment, 1); + result.push_back(token.term); + } + return result; +} + +std::shared_ptr builtin_common_words() { + static const auto common_words = std::shared_ptr( + &CommonWordSet::builtin_english_stop_words_v1(), [](const CommonWordSet*) {}); + return common_words; +} + +TokenStreamPtr index_stream(const std::shared_ptr& input) { + return std::make_shared(input, builtin_common_words()); +} + +TokenStreamPtr escaped_index_stream(const std::shared_ptr& input) { + return std::make_shared(input, builtin_common_words(), + CommonGramsOutputMode::kEscapedV1Index); +} + +TokenStreamPtr spimi_index_stream(const std::shared_ptr& input) { + return std::make_shared(input, builtin_common_words(), + CommonGramsOutputMode::kEscapedV1SpimiIndex); +} + +TokenStreamPtr plain_stream(const std::shared_ptr& input) { + return std::make_shared(input); +} + +TokenStreamPtr exact_stream(const std::shared_ptr& input) { + return std::make_shared(index_stream(input), builtin_common_words()); +} + +TokenStreamPtr prefix_stream(const std::shared_ptr& input) { + return std::make_shared(index_stream(input), + builtin_common_words()); +} + +TEST(CommonGramsFilterTest, IndexPreservesUnigramsAndAddsEligibleOwnedGrams) { + auto input = std::make_shared(words({"man", "of", "the", "year"})); + auto stream = index_stream(input); + + EXPECT_EQ(collect(stream), (std::vector { + {"man", 1, 0, 3, Token::getDefaultType()}, + {gram("man", "of"), 0, 0, 6, COMMON_GRAM_TOKEN_TYPE}, + {"of", 1, 4, 6, Token::getDefaultType()}, + {gram("of", "the"), 0, 4, 10, COMMON_GRAM_TOKEN_TYPE}, + {"the", 1, 7, 10, Token::getDefaultType()}, + {gram("the", "year"), 0, 7, 15, COMMON_GRAM_TOKEN_TYPE}, + {"year", 1, 11, 15, Token::getDefaultType()}, + })); +} + +TEST(CommonGramsFilterTest, QueryModesNormalizeBothCommonGramType) { + auto input = std::make_shared(words({"of", "the"})); + EXPECT_EQ(collect(exact_stream(input)), + (std::vector { + {gram("of", "the"), 1, 0, 6, COMMON_GRAM_TOKEN_TYPE}, + })); + + input = std::make_shared(words({"of", "the"})); + EXPECT_EQ(collect(prefix_stream(input)), + (std::vector { + {gram("of", "the"), 1, 0, 6, COMMON_GRAM_TOKEN_TYPE}, + })); +} + +TEST(CommonGramsFilterTest, QueryGramEligibilityMatchesPurposeSpecificFilters) { + const std::vector> cases { + {}, + {"alpha"}, + {"alpha", "beta"}, + {"alpha", "the"}, + {"the", "alpha"}, + {"alpha", "beta", "the"}, + {"alpha", "the", "beta"}, + {"alpha", "beta", "gamma", "delta"}, + }; + for (const auto& query_terms : cases) { + SCOPED_TRACE(testing::PrintToString(query_terms)); + std::vector scripted; + scripted.reserve(query_terms.size()); + for (const auto& term : query_terms) { + scripted.push_back({.term = term}); + } + + for (const auto mode : + {CommonGramsQueryMode::kExact, CommonGramsQueryMode::kPhrasePrefix}) { + auto input = std::make_shared(scripted); + const auto output = + collect(mode == CommonGramsQueryMode::kExact ? exact_stream(input) + : prefix_stream(input)); + bool filter_used_gram = false; + for (const auto& token : output) { + filter_used_gram = + filter_used_gram || token.type == std::wstring(COMMON_GRAM_TOKEN_TYPE); + } + EXPECT_EQ(common_grams_query_may_use_gram(query_terms, mode, *builtin_common_words()), + filter_used_gram); + } + } +} + +TEST(CommonGramsFilterTest, PhysicalIndexModeEscapesPlainTermsButGramsUseLogicalBytes) { + const std::string internal_plain = std::string(1, '\x1f') + "literal"; + auto input = std::make_shared(words({internal_plain, "of"})); + auto stream = escaped_index_stream(input); + + EXPECT_EQ(collect(stream), + (std::vector { + {std::string(1, PLAIN_ESCAPE_PREFIX) + "Gliteral", 1, 0, + static_cast(internal_plain.size()), Token::getDefaultType()}, + {gram(internal_plain, "of"), 0, 0, + static_cast(internal_plain.size() + 3), COMMON_GRAM_TOKEN_TYPE}, + {"of", 1, static_cast(internal_plain.size() + 1), + static_cast(internal_plain.size() + 3), Token::getDefaultType()}, + })); + + const std::string escape_plain = std::string(1, PLAIN_ESCAPE_PREFIX) + "literal"; + input->set_tokens(words({escape_plain})); + stream->reset(); + EXPECT_EQ(terms(collect(stream)), + (std::vector {std::string(1, PLAIN_ESCAPE_PREFIX) + "Eliteral"})); +} + +TEST(CommonGramsFilterTest, SpimiIndexModeEmitsPhysicalGramsAndEscapedPlainTerms) { + const std::string internal_plain = std::string(1, '\x1f') + "literal"; + auto input = std::make_shared(words({internal_plain, "of"})); + + const std::string physical = gram(internal_plain, "of"); + EXPECT_EQ(collect(spimi_index_stream(input)), + (std::vector { + {std::string(1, PLAIN_ESCAPE_PREFIX) + "Gliteral", 1, 0, + static_cast(internal_plain.size()), Token::getDefaultType()}, + {physical, 0, 0, static_cast(internal_plain.size() + 3), + COMMON_GRAM_TOKEN_TYPE}, + {"of", 1, static_cast(internal_plain.size() + 1), + static_cast(internal_plain.size() + 3), Token::getDefaultType()}, + })); +} + +TEST(CommonGramsFilterTest, SniiIndexEventsExpandToExistingSpimiTokenSemantics) { + const std::string internal_plain = std::string(1, '\x1f') + "literal"; + const std::string escaped_plain = std::string(1, PLAIN_ESCAPE_PREFIX) + "literal"; + const std::vector input_tokens = + words({internal_plain, "of", "the", escaped_plain, "中文词"}); + + auto legacy_input = std::make_shared(input_tokens); + const std::vector expected = + collect_semantics(spimi_index_stream(legacy_input)); + + auto event_input = std::make_shared(input_tokens); + CommonGramsFilter event_stream(event_input, builtin_common_words(), + CommonGramsOutputMode::kEscapedV1SpimiIndex); + size_t event_count = 0; + size_t both_common_gram_count = 0; + const std::vector actual = + expand_snii_index_events(&event_stream, &event_count, &both_common_gram_count); + + EXPECT_EQ(event_count, input_tokens.size()); + EXPECT_EQ(both_common_gram_count, 1U); + EXPECT_EQ(actual, expected); +} + +TEST(CommonGramsFilterTest, SniiIndexEventsDeferCommonWordClassificationToWriter) { + auto input = std::make_shared( + words({"the", "database", "of", "the", "world", "and", "search"})); + CommonGramsFilter stream(input, builtin_common_words(), + CommonGramsOutputMode::kEscapedV1SpimiIndex); + + common_grams_testing::reset_common_word_membership_lookup_count(); + SniiCommonGramsIndexEvent event; + size_t event_count = 0; + while (stream.next_snii_index_event(&event)) { + EXPECT_FALSE(event.logical_term.empty()); + EXPECT_FALSE(event.plain_term.empty()); + ++event_count; + } + + EXPECT_EQ(event_count, 7U); + EXPECT_EQ(common_grams_testing::common_word_membership_lookup_count(), 0U); +} + +TEST(CommonGramsFilterTest, IndexOmitsNonCommonPair) { + auto input = std::make_shared(words({"man", "year"})); + EXPECT_EQ(terms(collect(index_stream(input))), (std::vector {"man", "year"})); +} + +TEST(CommonGramsFilterTest, CachesCommonWordMembershipWithoutDefeatingShortCircuit) { + auto input = std::make_shared(words({"single"})); + common_grams_testing::reset_common_word_membership_lookup_count(); + collect(index_stream(input)); + EXPECT_EQ(common_grams_testing::common_word_membership_lookup_count(), 0); + + input = std::make_shared(words({"of", "dog"})); + common_grams_testing::reset_common_word_membership_lookup_count(); + collect(index_stream(input)); + EXPECT_EQ(common_grams_testing::common_word_membership_lookup_count(), 1); + + input = std::make_shared(words({"of", "the", "and", "to"})); + common_grams_testing::reset_common_word_membership_lookup_count(); + collect(exact_stream(input)); + EXPECT_EQ(common_grams_testing::common_word_membership_lookup_count(), 6); + + input = std::make_shared(words({"man", "dog"})); + common_grams_testing::reset_common_word_membership_lookup_count(); + collect(prefix_stream(input)); + EXPECT_EQ(common_grams_testing::common_word_membership_lookup_count(), 3); + + input = std::make_shared(words({"man", "dog", "year", "thing"})); + common_grams_testing::reset_common_word_membership_lookup_count(); + EXPECT_EQ(terms(collect(index_stream(input))), + (std::vector {"man", "dog", "year", "thing"})); + EXPECT_EQ(common_grams_testing::common_word_membership_lookup_count(), 4); + + input = std::make_shared(words({"man", "dog", "year", "thing"})); + common_grams_testing::reset_common_word_membership_lookup_count(); + EXPECT_EQ(terms(collect(exact_stream(input))), + (std::vector {"man", "dog", "year", "thing"})); + EXPECT_EQ(common_grams_testing::common_word_membership_lookup_count(), 8); +} + +TEST(CommonGramsFilterTest, ExactRewriteTruthTable) { + struct Case { + std::vector input; + std::vector expected; + }; + const std::vector cases = { + {words({"man", "dog", "year"}), {"man", "dog", "year"}}, + {words({"man", "dog", "the"}), {"man", gram("dog", "the")}}, + {words({"man", "of", "year"}), {gram("man", "of"), gram("of", "year")}}, + {words({"man", "of", "the"}), {gram("man", "of"), gram("of", "the")}}, + {words({"of", "dog", "year"}), {gram("of", "dog"), "dog", "year"}}, + {words({"of", "dog", "the"}), {gram("of", "dog"), gram("dog", "the")}}, + {words({"of", "the", "year"}), {gram("of", "the"), gram("the", "year")}}, + {words({"of", "the", "and"}), {gram("of", "the"), gram("the", "and")}}, + {words({"the", "the", "the"}), {gram("the", "the"), gram("the", "the")}}, + }; + + for (const auto& test_case : cases) { + auto input = std::make_shared(test_case.input); + EXPECT_EQ(terms(collect(exact_stream(input))), test_case.expected); + } +} + +TEST(CommonGramsFilterTest, PhrasePrefixRewritesOnlyCommonLeftBoundary) { + struct Case { + std::vector input; + std::vector expected; + }; + const std::vector cases = { + {words({"the", "wo"}), {gram("the", "wo")}}, + {words({"foo", "the"}), {"foo", "the"}}, + {words({"foo", "of", "th"}), {gram("foo", "of"), gram("of", "th")}}, + {words({"the", "bar", "ba"}), {gram("the", "bar"), "bar", "ba"}}, + {words({"the"}), {"the"}}, + }; + + for (const auto& test_case : cases) { + auto input = std::make_shared(test_case.input); + EXPECT_EQ(terms(collect(prefix_stream(input))), test_case.expected); + } +} + +TEST(CommonGramsFilterTest, ResetAndReuseRestoresUnigramMetadata) { + auto input = std::make_shared(words({"man", "of"})); + auto stream = index_stream(input); + ASSERT_EQ(collect(stream).size(), 3); + + input->set_tokens({{"plain", 1, 17, 22, Token::getDefaultType()}}); + stream->reset(); + EXPECT_EQ(collect(stream), (std::vector { + {"plain", 1, 17, 22, Token::getDefaultType()}, + })); +} + +TEST(CommonGramsFilterTest, PreservesUpstreamUnigramTypes) { + auto input = std::make_shared(std::vector { + {"man", 1, 3, 6, L"left_type"}, {"of", 1, 7, 9, L"right_type"}}); + EXPECT_EQ(collect(index_stream(input)), + (std::vector { + {"man", 1, 3, 6, L"left_type"}, + {gram("man", "of"), 0, 3, 9, COMMON_GRAM_TOKEN_TYPE}, + {"of", 1, 7, 9, L"right_type"}, + })); + + input->set_tokens({{"man", 1, 3, 6, L"left_type"}, {"year", 1, 7, 11, L"right_type"}}); + auto exact = exact_stream(input); + EXPECT_EQ(collect(exact), (std::vector { + {"man", 1, 3, 6, L"left_type"}, + {"year", 1, 7, 11, L"right_type"}, + })); +} + +TEST(CommonGramsFilterTest, ExactAndPrefixResetReuseIndependentInput) { + auto exact_input = std::make_shared(words({"man", "of"})); + auto exact = exact_stream(exact_input); + EXPECT_EQ(terms(collect(exact)), (std::vector {gram("man", "of")})); + exact_input->set_tokens(words({"plain", "terms"})); + exact->reset(); + EXPECT_EQ(terms(collect(exact)), (std::vector {"plain", "terms"})); + + auto prefix_input = std::make_shared(words({"the", "term"})); + auto prefix = prefix_stream(prefix_input); + EXPECT_EQ(terms(collect(prefix)), (std::vector {gram("the", "term")})); + prefix_input->set_tokens(words({"plain", "terms"})); + prefix->reset(); + EXPECT_EQ(terms(collect(prefix)), (std::vector {"plain", "terms"})); +} + +TEST(CommonGramsFilterTest, EmptyAndSingleTokenStreamsAreStable) { + auto input = std::make_shared(std::vector {}); + auto stream = index_stream(input); + EXPECT_TRUE(collect(stream).empty()); + + input->set_tokens(words({"of"})); + stream->reset(); + EXPECT_EQ(terms(collect(stream)), (std::vector {"of"})); +} + +TEST(CommonGramsFilterTest, RejectsNonUnitInputForEveryStreamPurpose) { + using Factory = TokenStreamPtr (*)(const std::shared_ptr&); + for (Factory factory : {index_stream, plain_stream, exact_stream, prefix_stream}) { + for (int32_t bad_increment : {-1, 0, 2}) { + auto input = std::make_shared( + std::vector {{"of", bad_increment, 0, 2}}); + try { + collect(factory(input)); + FAIL() << "expected analyzer error for increment " << bad_increment; + } catch (const Exception& error) { + EXPECT_EQ(error.code(), ErrorCode::INVERTED_INDEX_ANALYZER_ERROR); + } + } + + for (int32_t bad_increment : {-1, 0, 2}) { + auto empty_input = std::make_shared( + std::vector {{"", bad_increment, 0, 0}}); + try { + collect(factory(empty_input)); + FAIL() << "expected analyzer error for empty token increment " << bad_increment; + } catch (const Exception& error) { + EXPECT_EQ(error.code(), ErrorCode::INVERTED_INDEX_ANALYZER_ERROR); + } + } + + auto input = std::make_shared( + std::vector {{"man", 1, 0, 3}, {"of", 2, 4, 6}}); + auto stream = factory(input); + try { + collect(stream); + FAIL() << "expected analyzer error after a valid token"; + } catch (const Exception& error) { + EXPECT_EQ(error.code(), ErrorCode::INVERTED_INDEX_ANALYZER_ERROR); + EXPECT_EQ(input->consumed(), 2); + } + } +} + +TEST(CommonGramsFilterTest, RejectsEmptyInputForEveryStreamPurpose) { + using Factory = TokenStreamPtr (*)(const std::shared_ptr&); + for (Factory factory : {index_stream, plain_stream, exact_stream, prefix_stream}) { + auto input = + std::make_shared(std::vector {{"", 1, 0, 0}}); + try { + collect(factory(input)); + FAIL() << "expected analyzer error for an empty token"; + } catch (const Exception& error) { + EXPECT_EQ(error.code(), ErrorCode::INVERTED_INDEX_ANALYZER_ERROR); + } + } +} + +TEST(CommonGramsFilterTest, QueryPreparationLatchesFailureUntilReset) { + for (auto factory : {exact_stream, prefix_stream}) { + auto input = std::make_shared( + std::vector {{"the", 1, 0, 3}, {"term", 2, 4, 8}}); + auto stream = factory(input); + for (int attempt = 0; attempt < 2; ++attempt) { + try { + collect(stream); + FAIL() << "expected latched analyzer error on attempt " << attempt; + } catch (const Exception& error) { + EXPECT_EQ(error.code(), ErrorCode::INVERTED_INDEX_ANALYZER_ERROR); + } + } + + input->set_tokens(words({"the", "term"})); + stream->reset(); + EXPECT_EQ(terms(collect(stream)), (std::vector {gram("the", "term")})); + } +} + +TEST(CommonGramsFilterTest, UnencodableRequiredGramFallsBackToWholePlainSequence) { + const std::string huge(COMMON_GRAM_MAX_ENCODED_BYTES, 'x'); + + for (auto factory : {exact_stream, prefix_stream}) { + auto input = std::make_shared(words({"the", huge})); + EXPECT_EQ(terms(collect(factory(input))), (std::vector {"the", huge})); + + input = std::make_shared(words({"the", "of", huge})); + EXPECT_EQ(terms(collect(factory(input))), (std::vector {"the", "of", huge})); + } + + auto input = std::make_shared(words({"the", huge})); + EXPECT_EQ(terms(collect(index_stream(input))), (std::vector {"the", huge})); +} + +TEST(CommonGramsFilterTest, MaximumMarkerLeadingUnigramFailsWithBuildSwitchRecovery) { + for (const char marker : {PLAIN_ESCAPE_PREFIX, '\x1f'}) { + std::string term(COMMON_GRAM_MAX_ENCODED_BYTES, 'x'); + term.front() = marker; + auto input = std::make_shared( + std::vector {{term, 1, 0, static_cast(term.size())}}); + try { + static_cast(collect(escaped_index_stream(input))); + FAIL() << "expected CommonGrams escaped-term overflow"; + } catch (const Exception& e) { + EXPECT_EQ(e.code(), ErrorCode::INVERTED_INDEX_ANALYZER_ERROR); + EXPECT_NE(std::string(e.what()).find("enable_common_grams_index_build=false"), + std::string::npos); + EXPECT_NE(std::string(e.what()).find("new transaction"), std::string::npos); + } + } +} + +TEST(CommonGramsFilterTest, LargestEscapableMarkerLeadingUnigramUsesPhysicalKeyLimit) { + for (const char marker : {PLAIN_ESCAPE_PREFIX, '\x1f'}) { + std::string term(COMMON_GRAM_MAX_ENCODED_BYTES - 1, 'x'); + term.front() = marker; + auto input = std::make_shared( + std::vector {{term, 1, 0, static_cast(term.size())}}); + const auto tokens = collect(escaped_index_stream(input)); + ASSERT_EQ(tokens.size(), 1); + EXPECT_EQ(tokens[0].term.size(), COMMON_GRAM_MAX_ENCODED_BYTES); + } +} + +TEST(CommonGramsFilterTest, InvalidLogicalTermsRemainHardAnalyzerErrors) { + for (const std::string& term : {std::string("bad\0term", 8), std::string("\xc3", 1)}) { + auto input = std::make_shared( + std::vector {{term, 1, 0, static_cast(term.size())}}); + try { + collect(escaped_index_stream(input)); + FAIL() << "expected analyzer error"; + } catch (const Exception& error) { + EXPECT_EQ(error.code(), ErrorCode::INVERTED_INDEX_ANALYZER_ERROR); + } + } +} + +TEST(CommonGramsFilterTest, RejectsOverlongLogicalToken) { + auto input = std::make_shared( + std::vector {{std::string(COMMON_GRAM_MAX_ENCODED_BYTES + 1, 'x'), 1, 0, + static_cast(COMMON_GRAM_MAX_ENCODED_BYTES + 1)}}); + try { + collect(index_stream(input)); + FAIL() << "expected analyzer error"; + } catch (const Exception& error) { + EXPECT_EQ(error.code(), ErrorCode::INVERTED_INDEX_ANALYZER_ERROR); + } +} + +} // namespace +} // namespace doris::segment_v2::inverted_index diff --git a/be/test/storage/index/inverted/ananlyzer/common_grams_key_codec_test.cpp b/be/test/storage/index/inverted/ananlyzer/common_grams_key_codec_test.cpp new file mode 100644 index 00000000000000..05ef2bb4e56453 --- /dev/null +++ b/be/test/storage/index/inverted/ananlyzer/common_grams_key_codec_test.cpp @@ -0,0 +1,553 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/inverted/common_grams/common_grams_key_codec.h" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "util/utf8_check.h" + +namespace doris::segment_v2::inverted_index { +namespace { + +TEST(CommonGramsKeyCodecTest, GramGoldenBytesAndMarkerRange) { + const std::string expected = + "\x1f" + "DORIS_COMMON_GRAM_V1" + "\x1f" + "00000001:ab"; + auto encoded = encode_common_gram("a", "b"); + ASSERT_TRUE(encoded.has_value()) << encoded.error(); + EXPECT_EQ(encoded.value(), expected); + + EXPECT_EQ(CG_V1_MARKER, std::string_view("\x1f" + "DORIS_COMMON_GRAM_V1" + "\x1f", + 22)); + EXPECT_EQ(CG_V1_MARKER_END, std::string_view("\x1f" + "DORIS_COMMON_GRAM_V1" + "\x20", + 22)); + EXPECT_GE(encoded.value(), CG_V1_MARKER); + EXPECT_LT(encoded.value(), CG_V1_MARKER_END); + EXPECT_EQ(encoded->find('\0'), std::string::npos); + EXPECT_TRUE(validate_utf8(encoded->data(), encoded->size())); + + auto hexadecimal_length = encode_common_gram("0123456789", "x"); + ASSERT_TRUE(hexadecimal_length.has_value()) << hexadecimal_length.error(); + EXPECT_EQ(hexadecimal_length.value(), + "\x1f" + "DORIS_COMMON_GRAM_V1" + "\x1f" + "0000000a:0123456789x"); + + auto different_boundary = encode_common_gram("ab", "c"); + ASSERT_TRUE(different_boundary.has_value()) << different_boundary.error(); + auto same_text = encode_common_gram("a", "bc"); + ASSERT_TRUE(same_text.has_value()) << same_text.error(); + EXPECT_NE(different_boundary.value(), same_text.value()); +} + +TEST(CommonGramsKeyCodecTest, ValidatesLogicalTermsWithoutEncoding) { + EXPECT_TRUE(validate_common_grams_logical_term("valid", "test term").ok()); + EXPECT_TRUE(validate_common_grams_logical_term("", "test term").ok()); + + const std::string nul_term("a\0b", 3); + const std::string invalid_utf8("\xc3\x28", 2); + const std::string overlong(COMMON_GRAM_MAX_ENCODED_BYTES + 1, 'x'); + for (const auto& term : {nul_term, invalid_utf8, overlong}) { + auto status = validate_common_grams_logical_term(term, "test term"); + EXPECT_EQ(status.code(), ErrorCode::INVERTED_INDEX_ANALYZER_ERROR); + } +} + +TEST(CommonGramsKeyCodecTest, TryEncodeDistinguishesTooLongAndReusesOutputCapacity) { + std::string output; + output.reserve(COMMON_GRAM_MAX_ENCODED_BYTES); + const char* reserved_data = output.data(); + + auto encoded = try_encode_common_gram("a", "b", &output); + ASSERT_TRUE(encoded.has_value()) << encoded.error(); + EXPECT_TRUE(encoded.value()); + EXPECT_EQ(output, encode_common_gram("a", "b").value()); + EXPECT_EQ(output.data(), reserved_data); + + const std::string huge(COMMON_GRAM_MAX_ENCODED_BYTES, 'x'); + auto too_long = try_encode_common_gram("the", huge, &output); + ASSERT_TRUE(too_long.has_value()) << too_long.error(); + EXPECT_FALSE(too_long.value()); + EXPECT_TRUE(output.empty()); + EXPECT_EQ(output.data(), reserved_data); + + const std::string invalid_utf8("\xc3\x28", 2); + auto invalid = try_encode_common_gram(invalid_utf8, "valid", &output); + EXPECT_FALSE(invalid.has_value()); + EXPECT_EQ(invalid.error().code(), ErrorCode::INVERTED_INDEX_ANALYZER_ERROR); + EXPECT_TRUE(output.empty()); +} + +TEST(CommonGramsKeyCodecTest, PrevalidatedEncoderMatchesCheckedEncoder) { + for (const auto& [left, right] : std::vector> { + {"a", "b"}, {"the", "term"}, {"", "right"}}) { + std::string checked; + auto checked_result = try_encode_common_gram(left, right, &checked); + ASSERT_TRUE(checked_result.has_value()) << checked_result.error(); + + std::string prevalidated; + EXPECT_EQ(try_encode_common_gram_prevalidated(left, right, prevalidated), + checked_result.value()); + EXPECT_EQ(prevalidated, checked); + } + + std::string too_long(COMMON_GRAM_MAX_ENCODED_BYTES, 'x'); + std::string output = "stale"; + EXPECT_FALSE(try_encode_common_gram_prevalidated("left", too_long, output)); + EXPECT_TRUE(output.empty()); +} + +TEST(CommonGramsKeyCodecTest, PrevalidatedPlainEncoderMatchesCheckedEncoderAndBoundaries) { + std::string output; + output.reserve(COMMON_GRAM_MAX_ENCODED_BYTES); + const char* reserved_data = output.data(); + + for (const char marker : {PLAIN_ESCAPE_PREFIX, '\x1f'}) { + const std::string logical = std::string(1, marker) + "plain"; + std::string checked; + auto checked_result = + try_encode_plain_term(logical, PlainTermKeyVersion::kEscapedV1, &checked); + ASSERT_TRUE(checked_result.has_value()) << checked_result.error(); + ASSERT_TRUE(checked_result.value()); + + EXPECT_TRUE(try_encode_escaped_plain_term_prevalidated(logical, output)); + EXPECT_EQ(output, checked); + EXPECT_EQ(output.data(), reserved_data); + } + + const std::string maximum_encodable = std::string(1, PLAIN_ESCAPE_PREFIX) + + std::string(COMMON_GRAM_MAX_ENCODED_BYTES - 2, 'x'); + EXPECT_TRUE(try_encode_escaped_plain_term_prevalidated(maximum_encodable, output)); + EXPECT_EQ(output.size(), COMMON_GRAM_MAX_ENCODED_BYTES); + EXPECT_EQ(output.data(), reserved_data); + + const std::string too_long = std::string(1, PLAIN_ESCAPE_PREFIX) + + std::string(COMMON_GRAM_MAX_ENCODED_BYTES - 1, 'x'); + EXPECT_FALSE(try_encode_escaped_plain_term_prevalidated(too_long, output)); + EXPECT_TRUE(output.empty()); + EXPECT_EQ(output.data(), reserved_data); +} + +TEST(CommonGramsKeyCodecTest, PlainTermViewBorrowsOrdinaryKeysAndUsesScratchOnlyForEscapes) { + const std::string ordinary = "ordinary"; + std::string scratch = "stale"; + auto borrowed = try_encode_plain_term_view(ordinary, PlainTermKeyVersion::kEscapedV1, &scratch); + ASSERT_TRUE(borrowed.has_value()) << borrowed.error(); + ASSERT_TRUE(borrowed->has_value()); + EXPECT_EQ(**borrowed, ordinary); + EXPECT_EQ(borrowed->value().data(), ordinary.data()); + EXPECT_TRUE(scratch.empty()); + + const std::string escaped = std::string(1, '\x1f') + "literal"; + auto encoded = try_encode_plain_term_view(escaped, PlainTermKeyVersion::kEscapedV1, &scratch); + ASSERT_TRUE(encoded.has_value()) << encoded.error(); + ASSERT_TRUE(encoded->has_value()); + EXPECT_EQ(**encoded, *encode_plain_term(escaped, PlainTermKeyVersion::kEscapedV1)); + EXPECT_EQ(encoded->value().data(), scratch.data()); + + std::string unrepresentable(COMMON_GRAM_MAX_ENCODED_BYTES, 'x'); + unrepresentable.front() = PLAIN_ESCAPE_PREFIX; + auto absent = + try_encode_plain_term_view(unrepresentable, PlainTermKeyVersion::kEscapedV1, &scratch); + ASSERT_TRUE(absent.has_value()) << absent.error(); + EXPECT_FALSE(absent->has_value()); + EXPECT_TRUE(scratch.empty()); +} + +TEST(CommonGramsKeyCodecTest, GramComponentsUseUnescapedLogicalBytes) { + const std::array cases { + std::tuple {std::string(1, '\x1e'), std::string(1, '\x1f'), + std::string("\x1f" + "DORIS_COMMON_GRAM_V1" + "\x1f" + "00000001:\x1e\x1f")}, + std::tuple {std::string(1, '\x1f'), std::string(1, '\x1e'), + std::string("\x1f" + "DORIS_COMMON_GRAM_V1" + "\x1f" + "00000001:\x1f\x1e")}, + std::tuple {std::string("\x1e" + "L"), + std::string("\x1f" + "R"), + std::string("\x1f" + "DORIS_COMMON_GRAM_V1" + "\x1f" + "00000002:\x1e" + "L\x1f" + "R")}, + std::tuple {std::string("\x1f" + "L"), + std::string("\x1e" + "R"), + std::string("\x1f" + "DORIS_COMMON_GRAM_V1" + "\x1f" + "00000002:\x1f" + "L\x1e" + "R")}, + }; + for (const auto& [left, right, expected] : cases) { + auto encoded = encode_common_gram(left, right); + ASSERT_TRUE(encoded.has_value()) << encoded.error(); + EXPECT_EQ(encoded.value(), expected); + } +} + +TEST(CommonGramsKeyCodecTest, PlainKeyVersionsHaveReversibleGoldenBytes) { + const std::array raw_versions {PlainTermKeyVersion::kLegacyRaw, + PlainTermKeyVersion::kRawNoInternal}; + const std::string escape_leading = + "\x1e" + "alpha"; + const std::string gram_leading = + "\x1f" + "alpha"; + + for (PlainTermKeyVersion version : raw_versions) { + auto escape_encoded = encode_plain_term(escape_leading, version); + ASSERT_TRUE(escape_encoded.has_value()) << escape_encoded.error(); + EXPECT_EQ(escape_encoded.value(), escape_leading); + auto escape_decoded = decode_plain_term(escape_encoded.value(), version); + ASSERT_TRUE(escape_decoded.has_value()) << escape_decoded.error(); + EXPECT_EQ(escape_decoded.value(), escape_leading); + + auto gram_encoded = encode_plain_term(gram_leading, version); + ASSERT_TRUE(gram_encoded.has_value()) << gram_encoded.error(); + EXPECT_EQ(gram_encoded.value(), gram_leading); + auto gram_decoded = decode_plain_term(gram_encoded.value(), version); + ASSERT_TRUE(gram_decoded.has_value()) << gram_decoded.error(); + EXPECT_EQ(gram_decoded.value(), gram_leading); + } + + auto escape_encoded = encode_plain_term(escape_leading, PlainTermKeyVersion::kEscapedV1); + ASSERT_TRUE(escape_encoded.has_value()) << escape_encoded.error(); + EXPECT_EQ(escape_encoded.value(), + "\x1e" + "Ealpha"); + auto escape_decoded = + decode_plain_term(escape_encoded.value(), PlainTermKeyVersion::kEscapedV1); + ASSERT_TRUE(escape_decoded.has_value()) << escape_decoded.error(); + EXPECT_EQ(escape_decoded.value(), escape_leading); + + auto gram_encoded = encode_plain_term(gram_leading, PlainTermKeyVersion::kEscapedV1); + ASSERT_TRUE(gram_encoded.has_value()) << gram_encoded.error(); + EXPECT_EQ(gram_encoded.value(), + "\x1e" + "Galpha"); + auto gram_decoded = decode_plain_term(gram_encoded.value(), PlainTermKeyVersion::kEscapedV1); + ASSERT_TRUE(gram_decoded.has_value()) << gram_decoded.error(); + EXPECT_EQ(gram_decoded.value(), gram_leading); + + EXPECT_FALSE(decode_plain_term("\x1e", PlainTermKeyVersion::kEscapedV1).has_value()); + EXPECT_FALSE(decode_plain_term("\x1e" + "Xalpha", + PlainTermKeyVersion::kEscapedV1) + .has_value()); +} + +TEST(CommonGramsKeyCodecTest, DecodePlainTermViewBorrowsOrdinaryInput) { + const std::array cases { + std::pair {PlainTermKeyVersion::kLegacyRaw, std::string("legacy")}, + std::pair {PlainTermKeyVersion::kEscapedV1, std::string("ordinary")}, + }; + for (const auto& [version, physical_term] : cases) { + std::string scratch; + auto decoded = decode_plain_term_view(physical_term, version, &scratch); + ASSERT_TRUE(decoded.has_value()) << decoded.error(); + EXPECT_EQ(decoded.value(), physical_term); + EXPECT_EQ(decoded->data(), physical_term.data()); + EXPECT_TRUE(scratch.empty()); + } +} + +TEST(CommonGramsKeyCodecTest, DecodePlainTermViewUsesScratchForEscapes) { + const std::array cases { + std::pair {std::string("\x1e" + "Ealpha"), + std::string("\x1e" + "alpha")}, + std::pair {std::string("\x1e" + "Galpha"), + std::string("\x1f" + "alpha")}, + }; + for (const auto& [physical_term, logical_term] : cases) { + std::string scratch = "stale"; + auto decoded = + decode_plain_term_view(physical_term, PlainTermKeyVersion::kEscapedV1, &scratch); + ASSERT_TRUE(decoded.has_value()) << decoded.error(); + EXPECT_EQ(decoded.value(), logical_term); + EXPECT_EQ(scratch, logical_term); + EXPECT_EQ(decoded->data(), scratch.data()); + } +} + +TEST(CommonGramsKeyCodecTest, EscapedPlainKeysCannotEnterGramMarkerRange) { + const std::array logical_terms {std::string("plain"), + std::string("\x1e" + "plain"), + std::string("\x1f" + "plain")}; + for (const auto& term : logical_terms) { + auto encoded = encode_plain_term(term, PlainTermKeyVersion::kEscapedV1); + ASSERT_TRUE(encoded.has_value()) << encoded.error(); + EXPECT_FALSE(encoded.value() >= CG_V1_MARKER && encoded.value() < CG_V1_MARKER_END); + auto decoded = decode_plain_term(encoded.value(), PlainTermKeyVersion::kEscapedV1); + ASSERT_TRUE(decoded.has_value()) << decoded.error(); + EXPECT_EQ(decoded.value(), term); + } +} + +TEST(CommonGramsKeyCodecTest, InternalNamespaceAndLegacyBypassAreSeparated) { + const std::string legacy_marker = + "\x1f" + "SNII_PHRASE_BIGRAM" + "\x1f"; + EXPECT_EQ(INTERNAL_TERM_NAMESPACE_BEGIN, std::string_view("\x1f", 1)); + EXPECT_EQ(INTERNAL_TERM_NAMESPACE_END, std::string_view("\x20", 1)); + + EXPECT_TRUE(is_internal_term_key(encode_common_gram("a", "b").value())); + EXPECT_TRUE(is_internal_term_key(legacy_marker + "payload")); + EXPECT_TRUE( + is_internal_term_key("\x1f" + "FUTURE_INTERNAL_TERM")); + EXPECT_FALSE( + is_internal_term_key("\x1e" + "Gescaped")); + EXPECT_FALSE(is_internal_term_key("plain")); + + EXPECT_TRUE(legacy_raw_exact_requires_bypass(CG_V1_MARKER)); + EXPECT_TRUE(legacy_raw_exact_requires_bypass(legacy_marker + "payload")); + EXPECT_FALSE(legacy_raw_exact_requires_bypass(std::string(1, '\x1f'))); + EXPECT_FALSE( + legacy_raw_exact_requires_bypass("\x1f" + "literal")); + EXPECT_FALSE(legacy_raw_exact_requires_bypass("plain")); + + EXPECT_TRUE(legacy_raw_prefix_requires_bypass("")); + EXPECT_TRUE(legacy_raw_prefix_requires_bypass(std::string(1, '\x1f'))); + EXPECT_TRUE( + legacy_raw_prefix_requires_bypass("\x1f" + "DORIS_COMMON")); + EXPECT_TRUE(legacy_raw_prefix_requires_bypass(CG_V1_MARKER)); + EXPECT_TRUE(legacy_raw_prefix_requires_bypass(legacy_marker + "payload")); + EXPECT_FALSE( + legacy_raw_prefix_requires_bypass("\x1f" + "literal")); + EXPECT_FALSE(legacy_raw_prefix_requires_bypass("plain")); +} + +TEST(CommonGramsKeyCodecTest, PlainEncodingPreservesLogicalPrefixRanges) { + const std::array logical_prefixes { + std::string("plain"), + std::string("\x1e" + "escape"), + std::string("\x1f" + "marker"), + }; + const std::array logical_terms { + logical_prefixes[0] + "-suffix", + logical_prefixes[1] + "-suffix", + logical_prefixes[2] + "-suffix", + }; + for (const std::string& logical_prefix : logical_prefixes) { + auto physical_prefix = encode_plain_term(logical_prefix, PlainTermKeyVersion::kEscapedV1); + ASSERT_TRUE(physical_prefix.has_value()) << physical_prefix.error(); + for (const std::string& logical_term : logical_terms) { + auto physical_term = encode_plain_term(logical_term, PlainTermKeyVersion::kEscapedV1); + ASSERT_TRUE(physical_term.has_value()) << physical_term.error(); + EXPECT_EQ(logical_term.starts_with(logical_prefix), + physical_term->starts_with(physical_prefix.value())); + } + } +} + +TEST(CommonGramsKeyCodecTest, EncodedLengthBoundariesAreShared) { + EXPECT_EQ(COMMON_GRAM_MAX_ENCODED_BYTES, 16383); + + const std::array all_versions {PlainTermKeyVersion::kLegacyRaw, PlainTermKeyVersion::kEscapedV1, + PlainTermKeyVersion::kRawNoInternal}; + for (PlainTermKeyVersion version : all_versions) { + for (size_t size : {size_t {16382}, size_t {16383}}) { + std::string term(size, 'p'); + auto encoded = encode_plain_term(term, version); + ASSERT_TRUE(encoded.has_value()) << encoded.error(); + EXPECT_EQ(encoded->size(), size); + auto decoded = decode_plain_term(encoded.value(), version); + ASSERT_TRUE(decoded.has_value()) << decoded.error(); + EXPECT_EQ(decoded.value(), term); + } + const std::string overlong(16384, 'p'); + EXPECT_FALSE(encode_plain_term(overlong, version).has_value()); + EXPECT_FALSE(decode_plain_term(overlong, version).has_value()); + } + + for (const auto& [logical_size, encoded_size] : + std::array {std::pair {size_t {16381}, size_t {16382}}, + std::pair {size_t {16382}, size_t {16383}}}) { + std::string term(logical_size, 'p'); + term.front() = '\x1e'; + auto encoded = encode_plain_term(term, PlainTermKeyVersion::kEscapedV1); + ASSERT_TRUE(encoded.has_value()) << encoded.error(); + EXPECT_EQ(encoded->size(), encoded_size); + } + + std::string escape_overflow(16383, 'p'); + escape_overflow.front() = '\x1f'; + EXPECT_FALSE(encode_plain_term(escape_overflow, PlainTermKeyVersion::kEscapedV1).has_value()); + std::string try_output = "stale"; + auto try_overflow = + try_encode_plain_term(escape_overflow, PlainTermKeyVersion::kEscapedV1, &try_output); + ASSERT_TRUE(try_overflow.has_value()) << try_overflow.error(); + EXPECT_FALSE(try_overflow.value()); + EXPECT_TRUE(try_output.empty()); + + const std::string invalid_utf8("\xc3\x28", 2); + auto try_invalid = + try_encode_plain_term(invalid_utf8, PlainTermKeyVersion::kEscapedV1, &try_output); + EXPECT_FALSE(try_invalid.has_value()); + EXPECT_EQ(try_invalid.error().code(), ErrorCode::INVERTED_INDEX_ANALYZER_ERROR); + EXPECT_TRUE(try_output.empty()); + + auto raw_fallback = encode_plain_term(escape_overflow, PlainTermKeyVersion::kRawNoInternal); + ASSERT_TRUE(raw_fallback.has_value()) << raw_fallback.error(); + EXPECT_EQ(raw_fallback->size(), 16383); + EXPECT_EQ(raw_fallback.value(), escape_overflow); + + for (const auto& [encoded_size, right_size, expected_encodable] : + std::array {std::tuple {size_t {16382}, size_t {16350}, true}, + std::tuple {size_t {16383}, size_t {16351}, true}, + std::tuple {size_t {16384}, size_t {16352}, false}}) { + std::string right(right_size, 'r'); + EXPECT_EQ(is_common_gram_encodable("a", right), expected_encodable); + auto encoded = encode_common_gram("a", right); + EXPECT_EQ(encoded.has_value(), expected_encodable); + if (encoded.has_value()) { + EXPECT_EQ(encoded->size(), encoded_size); + } + } +} + +TEST(CommonGramsKeyCodecTest, GramLengthBoundariesCanBeConcentratedInLeft) { + for (const auto& [encoded_size, left_size, expected_encodable] : + std::array {std::tuple {size_t {16382}, size_t {16350}, true}, + std::tuple {size_t {16383}, size_t {16351}, true}, + std::tuple {size_t {16384}, size_t {16352}, false}}) { + std::string left(left_size, 'l'); + EXPECT_EQ(is_common_gram_encodable(left, "r"), expected_encodable); + auto encoded = encode_common_gram(left, "r"); + EXPECT_EQ(encoded.has_value(), expected_encodable); + if (encoded.has_value()) { + EXPECT_EQ(encoded->size(), encoded_size); + } + } + + const std::string e_acute = "\xc3\xa9"; + std::string multibyte_left; + multibyte_left.reserve(16352); + for (size_t i = 0; i < 8175; ++i) { + multibyte_left.append(e_acute); + } + auto at_16382 = encode_common_gram(multibyte_left, "r"); + ASSERT_TRUE(at_16382.has_value()) << at_16382.error(); + EXPECT_EQ(at_16382->size(), 16382); + + multibyte_left.push_back('x'); + auto at_16383 = encode_common_gram(multibyte_left, "r"); + ASSERT_TRUE(at_16383.has_value()) << at_16383.error(); + EXPECT_EQ(at_16383->size(), 16383); + + multibyte_left.push_back('x'); + EXPECT_FALSE(is_common_gram_encodable(multibyte_left, "r")); + EXPECT_FALSE(encode_common_gram(multibyte_left, "r").has_value()); +} + +TEST(CommonGramsKeyCodecTest, LengthUsesUtf8BytesRatherThanCodePoints) { + const std::string chinese = "\xe4\xbd\xa0"; + const std::string e_acute = "\xc3\xa9"; + std::string right; + right.reserve(16350); + for (size_t i = 0; i < 8174; ++i) { + right.append(e_acute); + } + + auto at_16382 = encode_common_gram(chinese, right); + ASSERT_TRUE(at_16382.has_value()) << at_16382.error(); + EXPECT_EQ(at_16382->size(), 16382); + EXPECT_NE(at_16382->find("00000003:"), std::string::npos); + + right.push_back('x'); + auto at_16383 = encode_common_gram(chinese, right); + ASSERT_TRUE(at_16383.has_value()) << at_16383.error(); + EXPECT_EQ(at_16383->size(), 16383); + + right.push_back('x'); + EXPECT_FALSE(is_common_gram_encodable(chinese, right)); + EXPECT_FALSE(encode_common_gram(chinese, right).has_value()); +} + +TEST(CommonGramsKeyCodecTest, RejectsNulAndInvalidUtf8) { + const std::string nul_term("a\0b", 3); + const std::string invalid_utf8("\xc3\x28", 2); + for (const auto& term : {nul_term, invalid_utf8}) { + EXPECT_FALSE(encode_plain_term(term, PlainTermKeyVersion::kLegacyRaw).has_value()); + EXPECT_FALSE(encode_plain_term(term, PlainTermKeyVersion::kEscapedV1).has_value()); + EXPECT_FALSE(encode_plain_term(term, PlainTermKeyVersion::kRawNoInternal).has_value()); + EXPECT_FALSE(decode_plain_term(term, PlainTermKeyVersion::kLegacyRaw).has_value()); + EXPECT_FALSE(is_common_gram_encodable(term, "valid")); + EXPECT_FALSE(is_common_gram_encodable("valid", term)); + EXPECT_FALSE(encode_common_gram(term, "valid").has_value()); + EXPECT_FALSE(encode_common_gram("valid", term).has_value()); + } + + const std::string escaped_nul( + "\x1e" + "Ea\0b", + 5); + const std::string escaped_invalid( + "\x1e" + "G\xc3\x28", + 4); + EXPECT_FALSE(decode_plain_term(escaped_nul, PlainTermKeyVersion::kEscapedV1).has_value()); + EXPECT_FALSE(decode_plain_term(escaped_invalid, PlainTermKeyVersion::kEscapedV1).has_value()); + for (PlainTermKeyVersion version : + {PlainTermKeyVersion::kLegacyRaw, PlainTermKeyVersion::kRawNoInternal}) { + EXPECT_FALSE(decode_plain_term(nul_term, version).has_value()); + EXPECT_FALSE(decode_plain_term(invalid_utf8, version).has_value()); + } +} + +} // namespace +} // namespace doris::segment_v2::inverted_index diff --git a/be/test/storage/index/inverted/ananlyzer/common_word_set_test.cpp b/be/test/storage/index/inverted/ananlyzer/common_word_set_test.cpp new file mode 100644 index 00000000000000..2f1ce9fca4fe00 --- /dev/null +++ b/be/test/storage/index/inverted/ananlyzer/common_word_set_test.cpp @@ -0,0 +1,186 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/inverted/common_grams/common_word_set.h" + +#include + +#include +#include +#include + +#include "common/config.h" + +namespace doris::segment_v2::inverted_index { + +namespace common_grams_testing { +uint64_t common_word_hash_lookup_count(); +void reset_common_word_hash_lookup_count(); +} // namespace common_grams_testing + +namespace { + +TEST(CommonWordSetTest, DefaultWordSetLivesUnderTheSharedDictionaryRoot) { + // The word list follows the same layout as the icu, ik and pinyin dictionaries, so relocating + // inverted_index_dict_path moves all of them together instead of leaving this one behind. + const std::string saved = config::inverted_index_dict_path; + config::inverted_index_dict_path = "/somewhere/else"; + EXPECT_EQ(CommonWordSet::default_word_set_path(), + "/somewhere/else/common_grams/default_words.txt"); + config::inverted_index_dict_path = saved; +} + +TEST(CommonWordSetTest, BuiltinEnglishStopV1IsTheExactFrozen33WordSet) { + EXPECT_EQ(BUILTIN_COMMON_WORDS_RESOURCE, "builtin:lucene_english_stop:v1"); + const auto& words = CommonWordSet::builtin_english_stop_words_v1(); + constexpr std::array expected = { + "a", "an", "and", "are", "as", "at", "be", "but", "by", "for", "if", + "in", "into", "is", "it", "no", "not", "of", "on", "or", "such", "that", + "the", "their", "then", "there", "these", "they", "this", "to", "was", "will", "with"}; + + EXPECT_EQ(words.size(), expected.size()); + for (std::string_view word : expected) { + EXPECT_TRUE(words.contains(word)) << word; + } + EXPECT_FALSE(words.contains("english")); + EXPECT_FALSE(words.contains("The")); +} + +TEST(CommonWordSetTest, RejectsImpossibleShapesBeforeHashLookup) { + const auto& words = CommonWordSet::builtin_english_stop_words_v1(); + common_grams_testing::reset_common_word_hash_lookup_count(); + + EXPECT_FALSE(words.contains("encyclopedia")); // longer than every builtin word + EXPECT_FALSE(words.contains("zoo")); // impossible first byte + EXPECT_FALSE(words.contains("tea")); // possible shape, absent from the set + EXPECT_TRUE(words.contains("the")); + + EXPECT_EQ(common_grams_testing::common_word_hash_lookup_count(), 2); +} + +TEST(CommonWordSetTest, WordsetV1IgnoresEmptyAndCommentLinesAndDeduplicates) { + EXPECT_EQ(WORDSET_FORMAT_V1, "wordset:v1"); + const std::string content = + "# comment\n" + "alpha\n" + "\n" + "beta\n" + "alpha\n" + "# trailing comment\n" + "\xe4\xbd\xa0\xe5\xa5\xbd\n"; + auto parsed = CommonWordSet::parse_words(content); + ASSERT_TRUE(parsed.has_value()) << parsed.error(); + EXPECT_EQ(parsed->size(), 3); + EXPECT_TRUE(parsed->contains("alpha")); + EXPECT_TRUE(parsed->contains("beta")); + EXPECT_TRUE(parsed->contains("\xe4\xbd\xa0\xe5\xa5\xbd")); + EXPECT_FALSE(parsed->contains("# comment")); + EXPECT_FALSE(parsed->contains("")); +} + +TEST(CommonWordSetTest, WordsetV1PreservesTermBytesAndAcceptsCrLfAndFinalLine) { + const std::string content = + "alpha\r\n" + "beta\r\n" + "inline#hash\n" + " leading\n" + "trailing \n" + " #not-comment\n" + "final"; + auto parsed = CommonWordSet::parse_words(content); + ASSERT_TRUE(parsed.has_value()) << parsed.error(); + EXPECT_EQ(parsed->size(), 7); + for (std::string_view term : + {"alpha", "beta", "inline#hash", " leading", "trailing ", " #not-comment", "final"}) { + EXPECT_TRUE(parsed->contains(term)) << term; + } + EXPECT_FALSE(parsed->contains("alpha\r")); + EXPECT_FALSE(parsed->contains("beta\r")); +} + +TEST(CommonWordSetTest, UnterminatedFinalCarriageReturnIsTermData) { + auto final_with_carriage_return = CommonWordSet::parse_words("final\r"); + ASSERT_TRUE(final_with_carriage_return.has_value()) << final_with_carriage_return.error(); + EXPECT_EQ(final_with_carriage_return->size(), 1); + EXPECT_TRUE(final_with_carriage_return->contains("final\r")); + EXPECT_FALSE(final_with_carriage_return->contains("final")); + + auto lone_carriage_return = CommonWordSet::parse_words("\r"); + ASSERT_TRUE(lone_carriage_return.has_value()) << lone_carriage_return.error(); + EXPECT_EQ(lone_carriage_return->size(), 1); + EXPECT_TRUE(lone_carriage_return->contains("\r")); + EXPECT_FALSE(lone_carriage_return->contains("")); +} + +TEST(CommonWordSetTest, RejectsNulAndInvalidUtf8Terms) { + const std::string nul_content("alpha\na\0b\n", 10); + EXPECT_FALSE(CommonWordSet::parse_words(nul_content).has_value()); + + const std::string invalid_utf8("alpha\n\xc3\x28\n", 9); + EXPECT_FALSE(CommonWordSet::parse_words(invalid_utf8).has_value()); + + const std::string nul_comment("# bad\0comment\nalpha\n", 20); + EXPECT_FALSE(CommonWordSet::parse_words(nul_comment).has_value()); + const std::string invalid_comment("# bad\xc3\x28\nalpha\n", 14); + EXPECT_FALSE(CommonWordSet::parse_words(invalid_comment).has_value()); +} + +TEST(CommonWordSetTest, WordsetV1IsCaseSensitive) { + auto parsed = CommonWordSet::parse_words("Alpha\nalpha\n"); + ASSERT_TRUE(parsed.has_value()) << parsed.error(); + EXPECT_EQ(parsed->size(), 2); + EXPECT_TRUE(parsed->contains("Alpha")); + EXPECT_TRUE(parsed->contains("alpha")); +} + +TEST(CommonWordSetTest, BuiltinIdentityIsTheFrozenResourceName) { + EXPECT_EQ(CommonWordSet::builtin_english_stop_words_v1().identity(), + BUILTIN_COMMON_WORDS_RESOURCE); +} + +// A segment stamps this identity and the querying analyzer compares against it, so two BEs +// reading different word lists must not agree. Deriving it from the file bytes is what makes +// the BE-local word list safe; a fixed constant would let mismatched grams pass unnoticed. +TEST(CommonWordSetTest, ParsedIdentityIsDerivedFromContent) { + auto first = CommonWordSet::parse_words("alpha\nbeta\n"); + auto same = CommonWordSet::parse_words("alpha\nbeta\n"); + auto different = CommonWordSet::parse_words("alpha\ngamma\n"); + ASSERT_TRUE(first.has_value()) << first.error(); + ASSERT_TRUE(same.has_value()) << same.error(); + ASSERT_TRUE(different.has_value()) << different.error(); + + EXPECT_EQ(first->identity(), same->identity()); + EXPECT_NE(first->identity(), different->identity()); + EXPECT_TRUE(first->identity().starts_with("wordset:md5:")); + EXPECT_NE(first->identity(), BUILTIN_COMMON_WORDS_RESOURCE); +} + +// Comments and blank lines do not change the parsed word set, but they do change the identity. +// That direction is the safe one: a superfluous re-plan costs a cost comparison, whereas reusing +// grams across an edit that DID change the list would be wrong. +TEST(CommonWordSetTest, IdentityTracksRawBytesNotJustTheParsedWords) { + auto plain = CommonWordSet::parse_words("alpha\nbeta\n"); + auto commented = CommonWordSet::parse_words("# a note\nalpha\nbeta\n"); + ASSERT_TRUE(plain.has_value()) << plain.error(); + ASSERT_TRUE(commented.has_value()) << commented.error(); + + EXPECT_EQ(plain->size(), commented->size()); + EXPECT_NE(plain->identity(), commented->identity()); +} + +} // namespace +} // namespace doris::segment_v2::inverted_index diff --git a/be/test/storage/index/inverted/ananlyzer/custom_analyzer_test.cpp b/be/test/storage/index/inverted/ananlyzer/custom_analyzer_test.cpp index a96edc1f38b346..e6d0d73f5cf2a9 100644 --- a/be/test/storage/index/inverted/ananlyzer/custom_analyzer_test.cpp +++ b/be/test/storage/index/inverted/ananlyzer/custom_analyzer_test.cpp @@ -21,12 +21,16 @@ #include #include +#include #include "CLucene/store/Directory.h" #include "CLucene/store/FSDirectory.h" #include "roaring/roaring.hh" #include "runtime/exec_env.h" #include "storage/index/inverted/analysis_factory_mgr.h" +#include "storage/index/inverted/analyzer/analyzer.h" +#include "storage/index/inverted/common_grams/common_grams_key_codec.h" +#include "storage/index/inverted/common_grams/common_word_set.h" #include "storage/index/inverted/query/phrase_prefix_query.h" #include "storage/index/inverted/query/phrase_query.h" #include "storage/index/inverted/setting.h" @@ -255,6 +259,437 @@ TEST_F(CustomAnalyzerTest, TokenStreamWithReaderPtr) { delete token_stream; } +CustomAnalyzerConfigPtr common_grams_config( + const std::string& tokenizer, const Settings& tokenizer_settings, + const std::vector>& filters = {}, + const Settings& common_grams_settings = {}) { + CustomAnalyzerConfig::Builder builder; + builder.with_tokenizer_config(tokenizer, tokenizer_settings); + for (const auto& [name, settings] : filters) { + builder.add_token_filter_config(name, settings); + } + builder.add_token_filter_config("common_grams", common_grams_settings); + return builder.build(); +} + +Settings whitespace_tokenizer_settings() { + Settings settings; + settings.set("tokenize_on_chars", "[whitespace]"); + return settings; +} + +std::string expected_gram(std::string_view left, std::string_view right) { + auto result = encode_common_gram(left, right); + EXPECT_TRUE(result.has_value()) << result.error(); + return result.value(); +} + +void expect_common_grams_analyzer_error(const CustomAnalyzerConfigPtr& config, + AnalysisPurpose purpose) { + try { + CustomAnalyzer::build_custom_analyzer(config, purpose); + FAIL() << "expected CommonGrams analyzer error"; + } catch (const Exception& error) { + EXPECT_EQ(error.code(), ErrorCode::INVERTED_INDEX_ANALYZER_ERROR); + } +} + +TEST_F(CustomAnalyzerTest, CommonGramsBuildsIndependentPurposeStreams) { + auto config = + common_grams_config("char_group", whitespace_tokenizer_settings(), {{"lowercase", {}}}); + auto index = CustomAnalyzer::build_custom_analyzer(config, AnalysisPurpose::kIndex); + auto snii_index = + CustomAnalyzer::build_custom_analyzer(config, AnalysisPurpose::kSniiTransientIndex); + auto plain = CustomAnalyzer::build_custom_analyzer(config, AnalysisPurpose::kPlainQuery); + auto exact = CustomAnalyzer::build_custom_analyzer(config, AnalysisPurpose::kExactPhraseQuery); + auto prefix = + CustomAnalyzer::build_custom_analyzer(config, AnalysisPurpose::kPhrasePrefixQuery); + + EXPECT_EQ(tokenize1(index, "Man of the Year"), + (std::vector {{"man", 1}, + {expected_gram("man", "of"), 0}, + {"of", 1}, + {expected_gram("of", "the"), 0}, + {"the", 1}, + {expected_gram("the", "year"), 0}, + {"year", 1}})); + EXPECT_EQ(tokenize1(snii_index, "Man of the Year"), + (std::vector {{"man", 1}, + {expected_gram("man", "of"), 0}, + {"of", 1}, + {expected_gram("of", "the"), 0}, + {"the", 1}, + {expected_gram("the", "year"), 0}, + {"year", 1}})); + EXPECT_EQ(tokenize1(plain, "Man of the Year"), + (std::vector {{"man", 1}, {"of", 1}, {"the", 1}, {"year", 1}})); + EXPECT_EQ(tokenize1(exact, "Man of the Year"), + (std::vector {{expected_gram("man", "of"), 1}, + {expected_gram("of", "the"), 1}, + {expected_gram("the", "year"), 1}})); + EXPECT_EQ(tokenize1(prefix, "the wo"), + (std::vector {{expected_gram("the", "wo"), 1}})); + + EXPECT_EQ(tokenize1(exact, "plain terms"), + (std::vector {{"plain", 1}, {"terms", 1}})); + EXPECT_EQ(tokenize1(prefix, "of term"), + (std::vector {{expected_gram("of", "term"), 1}})); +} + +TEST_F(CustomAnalyzerTest, CommonGramsReusableIndexStreamDoesNotBridgeRows) { + auto analyzer = CustomAnalyzer::build_custom_analyzer( + common_grams_config("char_group", whitespace_tokenizer_settings(), {{"lowercase", {}}}), + AnalysisPurpose::kIndex); + auto reader = std::make_shared>(); + + auto tokenize_row = [&](std::string_view row) { + reader->init(row.data(), static_cast(row.size()), false); + auto* stream = analyzer->reusableTokenStream(L"", reader); + stream->reset(); + std::vector tokens; + Token token; + while (stream->next(&token)) { + tokens.emplace_back(std::string(token.termBuffer(), token.termLength()), + token.getPositionIncrement()); + } + return std::pair {stream, std::move(tokens)}; + }; + + auto [first_stream, first] = tokenize_row("foo of"); + auto [second_stream, second] = tokenize_row("the bar"); + + EXPECT_EQ(first_stream, second_stream); + EXPECT_EQ(first, (std::vector { + {"foo", 1}, {expected_gram("foo", "of"), 0}, {"of", 1}})); + EXPECT_EQ(second, (std::vector { + {"the", 1}, {expected_gram("the", "bar"), 0}, {"bar", 1}})); +} + +TEST_F(CustomAnalyzerTest, CommonGramsEscapesPlainKeysOnlyForIndexPurpose) { + Settings tokenizer_settings; + tokenizer_settings.set("tokenize_on_chars", "[\\u0020]"); + auto config = common_grams_config("char_group", tokenizer_settings); + auto index = CustomAnalyzer::build_custom_analyzer(config, AnalysisPurpose::kIndex); + auto plain = CustomAnalyzer::build_custom_analyzer(config, AnalysisPurpose::kPlainQuery); + auto exact = CustomAnalyzer::build_custom_analyzer(config, AnalysisPurpose::kExactPhraseQuery); + auto prefix = + CustomAnalyzer::build_custom_analyzer(config, AnalysisPurpose::kPhrasePrefixQuery); + + const std::string logical = std::string(1, '\x1f') + "literal"; + const std::string input = "the " + logical; + const std::string physical = std::string(1, PLAIN_ESCAPE_PREFIX) + "Gliteral"; + const std::string common_gram = expected_gram("the", logical); + + EXPECT_EQ(tokenize1(index, input), + (std::vector {{"the", 1}, {common_gram, 0}, {physical, 1}})); + EXPECT_EQ(tokenize1(plain, input), (std::vector {{"the", 1}, {logical, 1}})); + EXPECT_EQ(tokenize1(exact, input), (std::vector {{common_gram, 1}})); + EXPECT_EQ(tokenize1(prefix, input), (std::vector {{common_gram, 1}})); +} + +TEST_F(CustomAnalyzerTest, AnalyseResultPreservesCommonGramTermKind) { + auto config = + common_grams_config("char_group", whitespace_tokenizer_settings(), {{"lowercase", {}}}); + auto exact = CustomAnalyzer::build_custom_analyzer(config, AnalysisPurpose::kExactPhraseQuery); + auto reader = InvertedIndexAnalyzer::create_reader({}); + const std::string input = "Man of year"; + reader->init(input.data(), static_cast(input.size()), true); + + const auto terms = InvertedIndexAnalyzer::get_analyse_result(reader, exact.get()); + ASSERT_EQ(terms.size(), 2U); + EXPECT_EQ(terms[0].get_single_term(), expected_gram("man", "of")); + EXPECT_EQ(terms[0].key_kind, TermKeyKind::kCommonGram); + EXPECT_EQ(terms[1].get_single_term(), expected_gram("of", "year")); + EXPECT_EQ(terms[1].key_kind, TermKeyKind::kCommonGram); + + auto plain = CustomAnalyzer::build_custom_analyzer(config, AnalysisPurpose::kPlainQuery); + reader = InvertedIndexAnalyzer::create_reader({}); + reader->init(input.data(), static_cast(input.size()), true); + const auto plain_terms = InvertedIndexAnalyzer::get_analyse_result(reader, plain.get()); + ASSERT_EQ(plain_terms.size(), 3U); + for (const auto& term : plain_terms) { + EXPECT_EQ(term.key_kind, TermKeyKind::kPlain); + } +} + +TEST_F(CustomAnalyzerTest, AnalyseResultPreservesBothCommonGramTermKind) { + auto config = + common_grams_config("char_group", whitespace_tokenizer_settings(), {{"lowercase", {}}}); + auto index = CustomAnalyzer::build_custom_analyzer(config, AnalysisPurpose::kIndex); + auto reader = InvertedIndexAnalyzer::create_reader({}); + const std::string input = "of the"; + reader->init(input.data(), static_cast(input.size()), true); + + const auto terms = InvertedIndexAnalyzer::get_analyse_result(reader, index.get()); + ASSERT_EQ(terms.size(), 3U); + EXPECT_EQ(terms[1].get_single_term(), expected_gram("of", "the")); + EXPECT_EQ(terms[1].key_kind, TermKeyKind::kCommonGram); +} + +TEST_F(CustomAnalyzerTest, CommonGramsAllowsAbsentOrOneTerminalFilter) { + CustomAnalyzerConfig::Builder missing; + missing.with_tokenizer_config("char_group", whitespace_tokenizer_settings()); + auto missing_config = missing.build(); + for (AnalysisPurpose purpose : + {AnalysisPurpose::kIndex, AnalysisPurpose::kPlainQuery, AnalysisPurpose::kExactPhraseQuery, + AnalysisPurpose::kPhrasePrefixQuery}) { + EXPECT_NO_THROW(CustomAnalyzer::build_custom_analyzer(missing_config, purpose)); + } + + CustomAnalyzerConfig::Builder duplicate; + duplicate.with_tokenizer_config("char_group", whitespace_tokenizer_settings()); + duplicate.add_token_filter_config("common_grams", {}); + duplicate.add_token_filter_config("common_grams", {}); + expect_common_grams_analyzer_error(duplicate.build(), AnalysisPurpose::kIndex); + + CustomAnalyzerConfig::Builder not_last; + not_last.with_tokenizer_config("char_group", whitespace_tokenizer_settings()); + not_last.add_token_filter_config("common_grams", {}); + not_last.add_token_filter_config("lowercase", {}); + expect_common_grams_analyzer_error(not_last.build(), AnalysisPurpose::kIndex); +} + +TEST_F(CustomAnalyzerTest, CommonGramsValidatesPlainConfigurationButOmitsGrams) { + Settings unknown; + unknown.set("unknown_setting", "true"); + auto invalid = common_grams_config("char_group", whitespace_tokenizer_settings(), {}, unknown); + expect_common_grams_analyzer_error(invalid, AnalysisPurpose::kPlainQuery); + + auto valid = common_grams_config("char_group", whitespace_tokenizer_settings()); + auto plain = CustomAnalyzer::build_custom_analyzer(valid, AnalysisPurpose::kPlainQuery); + EXPECT_EQ(tokenize1(plain, "the term"), (std::vector {{"the", 1}, {"term", 1}})); +} + +TEST_F(CustomAnalyzerTest, CommonGramsIndexChainRejectsInvalidUtf8AfterAValidToken) { + auto analyzer = CustomAnalyzer::build_custom_analyzer( + common_grams_config("char_group", whitespace_tokenizer_settings(), {{"lowercase", {}}}), + AnalysisPurpose::kIndex); + const std::string input = std::string("valid b") + static_cast(0xFF) + std::string("ad"); + auto reader = std::make_shared>(); + reader->init(input.data(), static_cast(input.size()), false); + auto* stream = analyzer->reusableTokenStream(L"", reader); + stream->reset(); + + Token token; + ASSERT_NE(stream->next(&token), nullptr); + EXPECT_EQ(std::string(token.termBuffer(), token.termLength()), "valid"); + try { + stream->next(&token); + FAIL() << "expected malformed UTF-8 to fail the analyzer chain"; + } catch (const Exception& error) { + EXPECT_EQ(error.code(), ErrorCode::INVERTED_INDEX_ANALYZER_ERROR); + } +} + +TEST_F(CustomAnalyzerTest, CommonGramsDeniesUnsafePositionFactories) { + for (const std::string& tokenizer : {"standard", "pinyin", "basic", "icu", "keyword"}) { + expect_common_grams_analyzer_error(common_grams_config(tokenizer, {}), + AnalysisPurpose::kIndex); + } + + Settings preserve_original; + preserve_original.set("preserve_original", "true"); + expect_common_grams_analyzer_error( + common_grams_config("char_group", whitespace_tokenizer_settings(), + {{"asciifolding", preserve_original}}), + AnalysisPurpose::kIndex); + expect_common_grams_analyzer_error( + common_grams_config("char_group", whitespace_tokenizer_settings(), + {{"word_delimiter", {}}}), + AnalysisPurpose::kIndex); + expect_common_grams_analyzer_error( + common_grams_config("char_group", whitespace_tokenizer_settings(), {{"pinyin", {}}}), + AnalysisPurpose::kIndex); +} + +TEST_F(CustomAnalyzerTest, CommonGramsAcceptsReviewedUnitPositionFactories) { + for (const std::string& tokenizer : {"empty", "char_group", "ngram", "edge_ngram"}) { + EXPECT_NO_THROW(CustomAnalyzer::build_custom_analyzer( + common_grams_config(tokenizer, tokenizer == "char_group" + ? whitespace_tokenizer_settings() + : Settings {}), + AnalysisPurpose::kIndex)); + } + + for (const std::string& filter : {"empty", "lowercase", "icu_normalizer", "asciifolding"}) { + EXPECT_NO_THROW(CustomAnalyzer::build_custom_analyzer( + common_grams_config("char_group", whitespace_tokenizer_settings(), {{filter, {}}}), + AnalysisPurpose::kIndex)); + } +} + +TEST_F(CustomAnalyzerTest, CommonGramsRejectsTokensNormalizedToEmpty) { + const auto config = common_grams_config("char_group", whitespace_tokenizer_settings(), + {{"icu_normalizer", {}}}); + const std::string input = std::string("\xC2\xAD") + " the"; + for (AnalysisPurpose purpose : + {AnalysisPurpose::kIndex, AnalysisPurpose::kPlainQuery, AnalysisPurpose::kExactPhraseQuery, + AnalysisPurpose::kPhrasePrefixQuery}) { + auto analyzer = CustomAnalyzer::build_custom_analyzer(config, purpose); + try { + tokenize1(analyzer, input); + ADD_FAILURE() << "expected analyzer error for purpose " << static_cast(purpose); + } catch (const Exception& error) { + EXPECT_EQ(error.code(), ErrorCode::INVERTED_INDEX_ANALYZER_ERROR); + } + } +} + +TEST_F(CustomAnalyzerTest, CommonGramsInternalQueryFiltersAreNotRegistered) { + for (const std::string& internal : {"common_grams_query", "common_grams_phrase_prefix"}) { + CustomAnalyzerConfig::Builder builder; + builder.with_tokenizer_config("char_group", whitespace_tokenizer_settings()); + builder.add_token_filter_config(internal, {}); + builder.add_token_filter_config("common_grams", {}); + EXPECT_THROW( + CustomAnalyzer::build_custom_analyzer(builder.build(), AnalysisPurpose::kIndex), + Exception); + } +} + +// "the" is a member of the built-in stop-word list, which is what default_word_set() resolves to +// when no wordset file is installed -- the provider can no longer be handed a word list of its own. +TEST_F(CustomAnalyzerTest, CommonGramsProviderCachesPurposeAnalyzersWithOneWordSetSnapshot) { + auto provider = std::make_shared(common_grams_config( + "char_group", whitespace_tokenizer_settings(), {{"lowercase", {}}})); + + auto index = provider->get_analyzer(AnalysisPurpose::kIndex); + auto snii_index = provider->get_analyzer(AnalysisPurpose::kSniiTransientIndex); + auto plain = provider->get_analyzer(AnalysisPurpose::kPlainQuery); + auto exact = provider->get_analyzer(AnalysisPurpose::kExactPhraseQuery); + auto prefix = provider->get_analyzer(AnalysisPurpose::kPhrasePrefixQuery); + + EXPECT_EQ(tokenize1(std::dynamic_pointer_cast(index), "The year"), + (std::vector { + {"the", 1}, {expected_gram("the", "year"), 0}, {"year", 1}})); + EXPECT_EQ(tokenize1(std::dynamic_pointer_cast(snii_index), "The year"), + (std::vector { + {"the", 1}, {expected_gram("the", "year"), 0}, {"year", 1}})); + EXPECT_EQ(tokenize1(std::dynamic_pointer_cast(plain), "The year"), + (std::vector {{"the", 1}, {"year", 1}})); + EXPECT_EQ(tokenize1(std::dynamic_pointer_cast(exact), "The year"), + (std::vector {{expected_gram("the", "year"), 1}})); + EXPECT_EQ(tokenize1(std::dynamic_pointer_cast(prefix), "The ye"), + (std::vector {{expected_gram("the", "ye"), 1}})); + + // Every purpose analyzer shares the one process-wide word list. + EXPECT_EQ(provider->common_words(), CommonWordSet::default_word_set()); + EXPECT_EQ(provider->get_analyzer(AnalysisPurpose::kPlainQuery), plain); + EXPECT_NE(index, plain); + EXPECT_NE(index, snii_index); + EXPECT_NE(plain, exact); + EXPECT_NE(exact, prefix); +} + +TEST_F(CustomAnalyzerTest, CommonGramsProviderOwnsDeterministicBuiltinIdentity) { + Settings first_settings; + first_settings.set("max_token_length", "16383"); + first_settings.set("tokenize_on_chars", "[whitespace]"); + Settings second_settings; + second_settings.set("tokenize_on_chars", "[whitespace]"); + second_settings.set("max_token_length", "16383"); + + CustomAnalyzerProvider first( + common_grams_config("char_group", first_settings, {{"lowercase", {}}})); + CustomAnalyzerProvider second( + common_grams_config("char_group", second_settings, {{"lowercase", {}}})); + + ASSERT_NE(first.common_grams_identity(), nullptr); + ASSERT_NE(second.common_grams_identity(), nullptr); + EXPECT_EQ(first.common_grams_identity()->common_grams_dictionary_identity, + BUILTIN_COMMON_WORDS_RESOURCE); + EXPECT_EQ(*first.common_grams_identity(), *second.common_grams_identity()); + EXPECT_EQ(first.common_grams_identity()->base_analyzer_fingerprint.size(), 64U); + EXPECT_EQ(first.common_grams_identity()->common_grams_fingerprint.size(), 64U); +} + +TEST_F(CustomAnalyzerTest, CommonGramsProviderSeparatesBaseAndDictionaryIdentity) { + Settings first_settings = whitespace_tokenizer_settings(); + first_settings.set("max_token_length", "100"); + Settings second_settings = whitespace_tokenizer_settings(); + second_settings.set("max_token_length", "101"); + CustomAnalyzerProvider first(common_grams_config("char_group", first_settings)); + CustomAnalyzerProvider second(common_grams_config("char_group", second_settings)); + + ASSERT_NE(first.common_grams_identity(), nullptr); + ASSERT_NE(second.common_grams_identity(), nullptr); + EXPECT_NE(first.common_grams_identity()->base_analyzer_fingerprint, + second.common_grams_identity()->base_analyzer_fingerprint); + EXPECT_EQ(first.common_grams_identity()->common_grams_fingerprint, + second.common_grams_identity()->common_grams_fingerprint); + + // The dictionary half of the identity is the word list's own content identity. Neither the + // provider nor an index policy can supply one, so every provider in this process agrees on it. + EXPECT_EQ(first.common_grams_identity()->common_grams_dictionary_identity, + CommonWordSet::default_word_set()->identity()); + EXPECT_EQ(second.common_grams_identity()->common_grams_dictionary_identity, + first.common_grams_identity()->common_grams_dictionary_identity); +} + +TEST_F(CustomAnalyzerTest, CommonGramsIdentityIncludesOuterCharFilter) { + CustomAnalyzerProvider no_outer_filter( + common_grams_config("char_group", whitespace_tokenizer_settings())); + ASSERT_NE(no_outer_filter.common_grams_identity(), nullptr); + + const std::map slash_to_space = { + {INVERTED_INDEX_PARSER_CHAR_FILTER_TYPE, INVERTED_INDEX_CHAR_FILTER_CHAR_REPLACE}, + {INVERTED_INDEX_PARSER_CHAR_FILTER_PATTERN, "/"}, + {INVERTED_INDEX_PARSER_CHAR_FILTER_REPLACEMENT, " "}}; + const std::map dash_to_space = { + {INVERTED_INDEX_PARSER_CHAR_FILTER_TYPE, INVERTED_INDEX_CHAR_FILTER_CHAR_REPLACE}, + {INVERTED_INDEX_PARSER_CHAR_FILTER_PATTERN, "-"}, + {INVERTED_INDEX_PARSER_CHAR_FILTER_REPLACEMENT, " "}}; + CustomAnalyzerProvider slash_filter( + common_grams_config("char_group", whitespace_tokenizer_settings()), slash_to_space); + CustomAnalyzerProvider repeated_slash_filter( + common_grams_config("char_group", whitespace_tokenizer_settings()), slash_to_space); + CustomAnalyzerProvider dash_filter( + common_grams_config("char_group", whitespace_tokenizer_settings()), dash_to_space); + ASSERT_NE(slash_filter.common_grams_identity(), nullptr); + ASSERT_NE(repeated_slash_filter.common_grams_identity(), nullptr); + ASSERT_NE(dash_filter.common_grams_identity(), nullptr); + + EXPECT_EQ(*slash_filter.common_grams_identity(), + *repeated_slash_filter.common_grams_identity()); + EXPECT_EQ(slash_filter.common_grams_identity()->common_grams_dictionary_identity, + no_outer_filter.common_grams_identity()->common_grams_dictionary_identity); + EXPECT_EQ(slash_filter.common_grams_identity()->common_grams_fingerprint, + no_outer_filter.common_grams_identity()->common_grams_fingerprint); + EXPECT_NE(slash_filter.common_grams_identity()->base_analyzer_fingerprint, + no_outer_filter.common_grams_identity()->base_analyzer_fingerprint); + EXPECT_NE(slash_filter.common_grams_identity()->base_analyzer_fingerprint, + dash_filter.common_grams_identity()->base_analyzer_fingerprint); +} + +TEST_F(CustomAnalyzerTest, ProviderSharesOneLegacyAnalyzerWhenCommonGramsIsAbsent) { + CustomAnalyzerConfig::Builder builder; + builder.with_tokenizer_config("char_group", whitespace_tokenizer_settings()); + builder.add_token_filter_config("lowercase", {}); + auto provider = std::make_shared(builder.build()); + + auto analyzer = provider->get_analyzer(AnalysisPurpose::kIndex); + EXPECT_EQ(provider->get_analyzer(AnalysisPurpose::kPlainQuery), analyzer); + EXPECT_EQ(provider->get_analyzer(AnalysisPurpose::kExactPhraseQuery), analyzer); + EXPECT_EQ(provider->get_analyzer(AnalysisPurpose::kPhrasePrefixQuery), analyzer); + EXPECT_EQ(provider->common_grams_identity(), nullptr); +} + +TEST_F(CustomAnalyzerTest, ProviderExposesBaseFingerprintWithoutCommonGrams) { + CustomAnalyzerConfig::Builder plain_builder; + plain_builder.with_tokenizer_config("char_group", whitespace_tokenizer_settings()); + plain_builder.add_token_filter_config("lowercase", {}); + CustomAnalyzerProvider plain(plain_builder.build()); + + CustomAnalyzerProvider common_grams(common_grams_config( + "char_group", whitespace_tokenizer_settings(), {{"lowercase", {}}})); + + EXPECT_EQ(plain.base_analyzer_fingerprint().size(), 64U); + EXPECT_EQ(plain.base_analyzer_fingerprint(), common_grams.base_analyzer_fingerprint()); + ASSERT_NE(common_grams.common_grams_identity(), nullptr); + EXPECT_EQ(common_grams.base_analyzer_fingerprint(), + common_grams.common_grams_identity()->base_analyzer_fingerprint); +} + // TEST_F(CustomAnalyzerTest, test) { // std::string name = "name"; // std::string path = "/mnt/disk3/yangsiyu/clucene"; diff --git a/be/test/storage/index/inverted/common/single_flight_test.cpp b/be/test/storage/index/inverted/common/single_flight_test.cpp new file mode 100644 index 00000000000000..5f6527230552ae --- /dev/null +++ b/be/test/storage/index/inverted/common/single_flight_test.cpp @@ -0,0 +1,220 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/inverted/common/single_flight.h" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/check.h" + +namespace doris::segment_v2::inverted_index { + +struct BlockingMoveState { + std::mutex mutex; + std::condition_variable condition; + bool move_started = false; + bool release_move = false; +}; + +struct BlockingMoveResult { + explicit BlockingMoveResult(std::shared_ptr state_) + : state(std::move(state_)) {} + + BlockingMoveResult(const BlockingMoveResult&) = default; + BlockingMoveResult& operator=(const BlockingMoveResult&) = default; + + BlockingMoveResult(BlockingMoveResult&& other) noexcept : state(std::move(other.state)) { + std::unique_lock lock(state->mutex); + state->move_started = true; + state->condition.notify_all(); + state->condition.wait(lock, [&] { return state->release_move; }); + } + + BlockingMoveResult& operator=(BlockingMoveResult&&) = delete; + + std::shared_ptr state; +}; + +// A lone caller leads, a second concurrent caller follows, and the follower reuses the +// leader's published result. The in-flight entry is cleared on publish. +TEST(SingleFlight, LeadFollowReuse) { + SingleFlight sf; + + auto leader = sf.join_or_lead("k"); + EXPECT_FALSE(leader.has_value()); // first caller leads + EXPECT_EQ(sf.inflight_size(), 1U); + + auto follower = sf.join_or_lead("k"); + ASSERT_TRUE(follower.has_value()); // second caller follows the in-flight leader + EXPECT_EQ(sf.inflight_size(), 1U); // still one in-flight key + + sf.publish("k", 7); + EXPECT_EQ(follower->get(), 7); // follower reuses the leader's result + EXPECT_EQ(sf.inflight_size(), 0U); +} + +// Different keys execute independently -- each first caller leads. +TEST(SingleFlight, DistinctKeysLeadIndependently) { + SingleFlight sf; + EXPECT_FALSE(sf.join_or_lead("a").has_value()); + EXPECT_FALSE(sf.join_or_lead("b").has_value()); + EXPECT_EQ(sf.inflight_size(), 2U); + sf.publish("a", 1); + sf.publish("b", 2); + EXPECT_EQ(sf.inflight_size(), 0U); +} + +// After a key is published, the next caller of the same key leads a fresh execution. +TEST(SingleFlight, ReLeadAfterPublish) { + SingleFlight sf; + EXPECT_FALSE(sf.join_or_lead("k").has_value()); + sf.publish("k", 1); + EXPECT_EQ(sf.inflight_size(), 0U); + EXPECT_FALSE(sf.join_or_lead("k").has_value()); // leads again + EXPECT_EQ(sf.inflight_size(), 1U); + sf.publish("k", 2); +} + +TEST(SingleFlight, JoinDuringPublicationStillFollowsPublishedFlight) { + SingleFlight sf; + ASSERT_FALSE(sf.join_or_lead("k").has_value()); + + auto state = std::make_shared(); + BlockingMoveResult result(state); + std::thread publisher([&] { sf.publish("k", result); }); + { + std::unique_lock lock(state->mutex); + state->condition.wait(lock, [&] { return state->move_started; }); + } + + auto follower = sf.join_or_lead("k"); + EXPECT_TRUE(follower.has_value()); + { + std::lock_guard lock(state->mutex); + state->release_move = true; + } + state->condition.notify_all(); + publisher.join(); + + ASSERT_TRUE(follower.has_value()); + EXPECT_EQ(follower->get().state, state); + EXPECT_EQ(sf.inflight_size(), 0U); +} + +// The motivating scenario: many threads issue the same key concurrently. Exactly one leads; +// all followers receive the single shared result (one shared_ptr payload, shared read-only). +TEST(SingleFlight, ConcurrentCollapsesToOneLeader) { + constexpr int kThreads = 16; + SingleFlight> sf; + + std::atomic leader_count {0}; + std::latch joined(kThreads); + std::latch release_leader(1); + std::vector> results(kThreads); + auto payload = std::make_shared(42); + + std::vector threads; + threads.reserve(kThreads); + for (int i = 0; i < kThreads; ++i) { + threads.emplace_back([&, i] { + auto follow = sf.join_or_lead("hot-segment"); + const bool is_leader = !follow.has_value(); + joined.count_down(); + if (is_leader) { + leader_count.fetch_add(1, std::memory_order_relaxed); + joined.wait(); // ensure every thread has joined before we publish + release_leader.wait(); + sf.publish("hot-segment", payload); + } else { + results[i] = follow->get(); + } + }); + } + joined.wait(); + EXPECT_EQ(sf.inflight_size(), 1U); + release_leader.count_down(); + for (auto& t : threads) { + t.join(); + } + + EXPECT_EQ(leader_count.load(), 1); // only one open/execute happened + EXPECT_EQ(sf.inflight_size(), 0U); + for (int i = 0; i < kThreads; ++i) { + if (results[i] != nullptr) { + EXPECT_EQ(results[i], payload); // same shared object, not a recomputed copy + EXPECT_EQ(*results[i], 42); + } + } +} + +TEST(SingleFlight, ConcurrentDifferentRawQueriesNeverJoin) { + constexpr int kThreads = 16; + SingleFlight sf; + std::atomic leader_count {0}; + std::latch all_registered(kThreads); + std::latch release_publishers(1); + std::vector threads; + threads.reserve(kThreads); + for (int i = 0; i < kThreads; ++i) { + threads.emplace_back([&, i] { + const std::string key = "raw-query-" + std::to_string(i); + auto follower = sf.join_or_lead(key); + if (!follower.has_value()) { + leader_count.fetch_add(1, std::memory_order_relaxed); + } + all_registered.count_down(); + release_publishers.wait(); + DORIS_CHECK(!follower.has_value()); + sf.publish(key, i); + }); + } + all_registered.wait(); + EXPECT_EQ(sf.inflight_size(), kThreads); + release_publishers.count_down(); + for (auto& thread : threads) { + thread.join(); + } + + EXPECT_EQ(leader_count.load(std::memory_order_relaxed), kThreads); + EXPECT_EQ(sf.inflight_size(), 0U); +} + +// A leader that fails publishes its failure; followers observe it (and would then fall back +// to computing independently in the production path). +TEST(SingleFlight, ErrorResultPropagates) { + SingleFlight> sf; + auto leader = sf.join_or_lead("k"); + ASSERT_FALSE(leader.has_value()); + auto follower = sf.join_or_lead("k"); + ASSERT_TRUE(follower.has_value()); + + sf.publish("k", std::make_pair(false, 0)); // leader failed + auto [ok, value] = follower->get(); + EXPECT_FALSE(ok); + EXPECT_EQ(value, 0); +} + +} // namespace doris::segment_v2::inverted_index diff --git a/be/test/storage/index/inverted/compaction/index_compaction_test.cpp b/be/test/storage/index/inverted/compaction/index_compaction_test.cpp index 9f8eb7070dc81a..017cd2d48d9a7b 100644 --- a/be/test/storage/index/inverted/compaction/index_compaction_test.cpp +++ b/be/test/storage/index/inverted/compaction/index_compaction_test.cpp @@ -17,9 +17,13 @@ #include +#include + #include "storage/index/index_writer.h" #include "storage/index/inverted/compaction/util/index_compaction_utils.cpp" #include "storage/utils.h" +#include "util/debug_points.h" +#include "util/defer_op.h" namespace doris { @@ -732,6 +736,62 @@ class IndexCompactionTest : public ::testing::Test { } } + void _build_snii_multi_index_tablet(bool second_supports_phrase = true) { + TabletSchemaPB schema_pb; + schema_pb.set_keys_type(KeysType::DUP_KEYS); + schema_pb.set_inverted_index_storage_format(InvertedIndexStorageFormatPB::SNII); + + IndexCompactionUtils::construct_column(schema_pb.add_column(), 0, "INT", "key"); + IndexCompactionUtils::construct_column(schema_pb.add_column(), 1, "STRING", "v1"); + IndexCompactionUtils::construct_column(schema_pb.add_column(), 2, "STRING", "v2"); + IndexCompactionUtils::construct_column(schema_pb.add_column(), 3, "INT", "v3"); + + auto add_index = [&schema_pb](int64_t index_id, std::string_view name, + bool supports_phrase) { + TabletIndexPB* index = schema_pb.add_index(); + index->set_index_id(index_id); + index->set_index_name(std::string(name)); + index->set_index_type(IndexType::INVERTED); + index->add_col_unique_id(1); + auto* properties = index->mutable_properties(); + (*properties)[INVERTED_INDEX_PARSER_KEY] = INVERTED_INDEX_PARSER_UNICODE; + (*properties)[INVERTED_INDEX_PARSER_PHRASE_SUPPORT_KEY] = + supports_phrase ? INVERTED_INDEX_PARSER_PHRASE_SUPPORT_YES + : INVERTED_INDEX_PARSER_PHRASE_SUPPORT_NO; + (*properties)[INVERTED_INDEX_PARSER_LOWERCASE_KEY] = INVERTED_INDEX_PARSER_TRUE; + }; + add_index(11001, "v1_phrase_a", true); + add_index(11002, "v1_phrase_b", second_supports_phrase); + + _tablet_schema = std::make_shared(); + _tablet_schema->init_from_pb(schema_pb); + TabletMetaSharedPtr tablet_meta(new TabletMeta(_tablet_schema)); + _tablet = std::make_shared(*_engine_ref, tablet_meta, _data_dir.get()); + EXPECT_TRUE(_tablet->init().ok()); + } + + std::vector _build_snii_source_rowsets() { + EXPECT_TRUE(io::global_local_filesystem()->delete_directory(_tablet->tablet_path()).ok()); + EXPECT_TRUE(io::global_local_filesystem()->create_directory(_tablet->tablet_path()).ok()); + const std::vector data_files = { + _current_dir + "/be/test/storage/index/inverted/data/data1.csv", + _current_dir + "/be/test/storage/index/inverted/data/data2.csv"}; + std::vector rowsets(data_files.size()); + auto check_indexes = [](const int32_t& size) { EXPECT_EQ(size, 2); }; + IndexCompactionUtils::build_rowsets( + _data_dir, _tablet_schema, _tablet, _engine_ref, rowsets, data_files, _inc_id, + check_indexes, false, 1000); + return rowsets; + } + + static std::string _read_index_file_bytes(const RowsetSharedPtr& rowset, uint32_t segment_id) { + const std::string path = fmt::format("{}/{}_{}.idx", rowset->tablet_path(), + rowset->rowset_id().to_string(), segment_id); + std::ifstream input(path, std::ios::binary); + EXPECT_TRUE(input.is_open()) << path; + return {std::istreambuf_iterator(input), std::istreambuf_iterator()}; + } + private: TabletSchemaSPtr _tablet_schema = nullptr; StorageEngine* _engine_ref = nullptr; @@ -1688,4 +1748,208 @@ TEST_F(IndexCompactionTest, test_inverted_index_ram_dir_disable_with_debug_point config::inverted_index_ram_dir_enable = original_ram_dir_enable; config::enable_debug_points = original_enable_debug_points; } + +TEST_F(IndexCompactionTest, snii_native_merge_validates_rowids_once_and_matches_raw_rebuild) { + const bool old_common_grams = config::enable_common_grams_index_build; + const bool old_debug_points = config::enable_debug_points; + const bool old_write_freq = config::snii_positions_index_write_freq; + config::enable_common_grams_index_build = false; + config::enable_debug_points = true; + config::snii_positions_index_write_freq = false; + constexpr std::string_view kValidationPoint = + "Compaction::snii_validated_rowid_conversion_created"; + constexpr std::string_view kReaderInitPoint = "Compaction::snii_eligibility_reader_initialized"; + DEFER({ + DebugPoints::instance()->remove(std::string(kValidationPoint)); + DebugPoints::instance()->remove(std::string(kReaderInitPoint)); + config::enable_common_grams_index_build = old_common_grams; + config::enable_debug_points = old_debug_points; + config::snii_positions_index_write_freq = old_write_freq; + }); + + _build_snii_multi_index_tablet(); + const std::vector rowsets = _build_snii_source_rowsets(); + size_t validation_count = 0; + size_t reader_init_count = 0; + std::function count_validation = [&validation_count]() { ++validation_count; }; + std::function count_reader_init = [&reader_init_count]() { ++reader_init_count; }; + DebugPoints::instance()->add_with_callback(std::string(kValidationPoint), count_validation); + DebugPoints::instance()->add_with_callback(std::string(kReaderInitPoint), count_reader_init); + + auto check_native_merge = [](const BaseCompaction& compaction, const RowsetWriterContext& ctx) { + EXPECT_EQ(ctx.columns_to_do_index_compaction, (std::set {1})); + EXPECT_EQ(compaction._output_rowset->num_segments(), 2); + }; + RowsetSharedPtr native_merge; + Status status = IndexCompactionUtils::do_compaction(rowsets, _engine_ref, _tablet, true, + native_merge, check_native_merge, 1000); + ASSERT_TRUE(status.ok()) << status; + ASSERT_NE(native_merge, nullptr); + EXPECT_EQ(validation_count, 1); + EXPECT_EQ(reader_init_count, rowsets.size()); + DebugPoints::instance()->remove(std::string(kValidationPoint)); + DebugPoints::instance()->remove(std::string(kReaderInitPoint)); + + auto check_raw_rebuild = [](const BaseCompaction& compaction, const RowsetWriterContext& ctx) { + EXPECT_TRUE(ctx.columns_to_do_index_compaction.empty()); + EXPECT_EQ(compaction._output_rowset->num_segments(), 2); + }; + RowsetSharedPtr raw_rebuild; + status = IndexCompactionUtils::do_compaction(rowsets, _engine_ref, _tablet, false, raw_rebuild, + check_raw_rebuild, 1000); + ASSERT_TRUE(status.ok()) << status; + ASSERT_NE(raw_rebuild, nullptr); + ASSERT_EQ(native_merge->num_segments(), raw_rebuild->num_segments()); + for (uint32_t segment_id = 0; segment_id < native_merge->num_segments(); ++segment_id) { + EXPECT_EQ(_read_index_file_bytes(native_merge, segment_id), + _read_index_file_bytes(raw_rebuild, segment_id)); + } +} + +TEST_F(IndexCompactionTest, snii_native_merge_falls_back_for_entire_multi_index_column) { + const bool old_common_grams = config::enable_common_grams_index_build; + const bool old_write_freq = config::snii_positions_index_write_freq; + config::enable_common_grams_index_build = false; + config::snii_positions_index_write_freq = false; + DEFER({ + config::enable_common_grams_index_build = old_common_grams; + config::snii_positions_index_write_freq = old_write_freq; + }); + + _build_snii_multi_index_tablet(/*second_supports_phrase=*/false); + const std::vector rowsets = _build_snii_source_rowsets(); + auto check_fallback = [](const BaseCompaction& compaction, const RowsetWriterContext& ctx) { + EXPECT_TRUE(ctx.columns_to_do_index_compaction.empty()); + EXPECT_EQ(compaction._output_rowset->num_segments(), 2); + }; + RowsetSharedPtr output; + const Status status = IndexCompactionUtils::do_compaction(rowsets, _engine_ref, _tablet, true, + output, check_fallback, 1000); + ASSERT_TRUE(status.ok()) << status; + ASSERT_NE(output, nullptr); + for (uint32_t segment_id = 0; segment_id < output->num_segments(); ++segment_id) { + const auto segment_path = output->segment_path(segment_id); + ASSERT_TRUE(segment_path.has_value()) << segment_path.error(); + auto file_reader = IndexCompactionUtils::init_index_file_reader( + output, segment_path.value(), InvertedIndexStorageFormatPB::SNII); + for (const TabletIndex* index : _tablet_schema->inverted_indexes()) { + const auto logical_index = file_reader->open_snii_index(index); + EXPECT_TRUE(logical_index.has_value()) << logical_index.error(); + } + } +} + +TEST_F(IndexCompactionTest, snii_native_merge_aborts_after_partial_destination_creation) { + const bool old_common_grams = config::enable_common_grams_index_build; + const bool old_debug_points = config::enable_debug_points; + const bool old_write_freq = config::snii_positions_index_write_freq; + config::enable_common_grams_index_build = false; + config::enable_debug_points = true; + config::snii_positions_index_write_freq = false; + constexpr std::string_view kFailurePoint = "Compaction::before_add_snii_destination_session"; + constexpr std::string_view kAbortPoint = "Compaction::snii_destination_session_aborted"; + DEFER({ + DebugPoints::instance()->remove(std::string(kFailurePoint)); + DebugPoints::instance()->remove(std::string(kAbortPoint)); + config::enable_common_grams_index_build = old_common_grams; + config::enable_debug_points = old_debug_points; + config::snii_positions_index_write_freq = old_write_freq; + }); + + _build_snii_multi_index_tablet(); + const std::vector rowsets = _build_snii_source_rowsets(); + std::function fail_second_destination = [](size_t destination_ordinal, + Status* status) { + if (destination_ordinal == 1) { + *status = Status::Error( + "injected SNII destination session failure"); + } + }; + DebugPoints::instance()->add_with_callback(std::string(kFailurePoint), fail_second_destination); + size_t abort_count = 0; + std::function count_abort = [&abort_count](size_t) { ++abort_count; }; + DebugPoints::instance()->add_with_callback(std::string(kAbortPoint), count_abort); + + RowsetSharedPtr output; + const Status status = IndexCompactionUtils::do_compaction(rowsets, _engine_ref, _tablet, true, + output, nullptr, 1000); + EXPECT_FALSE(status.ok()); + EXPECT_TRUE(status.is()); + EXPECT_THAT(status.to_string(), + testing::HasSubstr("injected SNII destination session failure")); + EXPECT_EQ(output, nullptr); + EXPECT_EQ(abort_count, 1); + for (const RowsetSharedPtr& rowset : rowsets) { + EXPECT_FALSE(rowset->is_skip_index_compaction(1)); + } +} + +TEST_F(IndexCompactionTest, snii_native_merge_mem_limit_arms_raw_rebuild_without_wrapping) { + const bool old_common_grams = config::enable_common_grams_index_build; + const bool old_debug_points = config::enable_debug_points; + const bool old_write_freq = config::snii_positions_index_write_freq; + config::enable_common_grams_index_build = false; + config::enable_debug_points = true; + config::snii_positions_index_write_freq = false; + constexpr std::string_view kFailurePoint = "Compaction::before_execute_snii_merge"; + DEFER({ + DebugPoints::instance()->remove(std::string(kFailurePoint)); + config::enable_common_grams_index_build = old_common_grams; + config::enable_debug_points = old_debug_points; + config::snii_positions_index_write_freq = old_write_freq; + }); + + _build_snii_multi_index_tablet(); + const std::vector rowsets = _build_snii_source_rowsets(); + std::function inject_mem_limit = [](Status* status) { + *status = Status::Error("injected SNII merge memory limit"); + }; + DebugPoints::instance()->add_with_callback(std::string(kFailurePoint), inject_mem_limit); + + RowsetSharedPtr output; + const Status status = IndexCompactionUtils::do_compaction(rowsets, _engine_ref, _tablet, true, + output, nullptr, 1000); + EXPECT_FALSE(status.ok()); + EXPECT_TRUE(status.is()); + EXPECT_THAT(status.to_string(), testing::HasSubstr("injected SNII merge memory limit")); + EXPECT_EQ(output, nullptr); + for (const RowsetSharedPtr& rowset : rowsets) { + EXPECT_TRUE(rowset->is_skip_index_compaction(1)); + } +} + +TEST_F(IndexCompactionTest, snii_native_merge_corruption_arms_raw_rebuild_fallback) { + const bool old_common_grams = config::enable_common_grams_index_build; + const bool old_debug_points = config::enable_debug_points; + const bool old_write_freq = config::snii_positions_index_write_freq; + config::enable_common_grams_index_build = false; + config::enable_debug_points = true; + config::snii_positions_index_write_freq = false; + constexpr std::string_view kFailurePoint = "Compaction::before_execute_snii_merge"; + DEFER({ + DebugPoints::instance()->remove(std::string(kFailurePoint)); + config::enable_common_grams_index_build = old_common_grams; + config::enable_debug_points = old_debug_points; + config::snii_positions_index_write_freq = old_write_freq; + }); + + _build_snii_multi_index_tablet(); + const std::vector rowsets = _build_snii_source_rowsets(); + std::function inject_corruption = [](Status* status) { + *status = Status::Error( + "injected SNII merge corruption"); + }; + DebugPoints::instance()->add_with_callback(std::string(kFailurePoint), inject_corruption); + + RowsetSharedPtr output; + const Status status = IndexCompactionUtils::do_compaction(rowsets, _engine_ref, _tablet, true, + output, nullptr, 1000); + EXPECT_FALSE(status.ok()); + EXPECT_TRUE(status.is()) << status; + EXPECT_THAT(status.to_string(), testing::HasSubstr("injected SNII merge corruption")); + EXPECT_EQ(output, nullptr); + for (const RowsetSharedPtr& rowset : rowsets) { + EXPECT_TRUE(rowset->is_skip_index_compaction(1)); + } +} } // namespace doris diff --git a/be/test/storage/index/inverted/compaction/index_compaction_write_index_test.cpp b/be/test/storage/index/inverted/compaction/index_compaction_write_index_test.cpp index a530be96da3775..48803988d876f6 100644 --- a/be/test/storage/index/inverted/compaction/index_compaction_write_index_test.cpp +++ b/be/test/storage/index/inverted/compaction/index_compaction_write_index_test.cpp @@ -28,7 +28,6 @@ #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wshadow-field" #include // IWYU pragma: keep -#include #include #include "CLucene/analysis/Analyzers.h" diff --git a/be/test/storage/index/inverted/compaction/util/index_compaction_utils.cpp b/be/test/storage/index/inverted/compaction/util/index_compaction_utils.cpp index 6ca726770cfb5d..7feb99bf16d2ec 100644 --- a/be/test/storage/index/inverted/compaction/util/index_compaction_utils.cpp +++ b/be/test/storage/index/inverted/compaction/util/index_compaction_utils.cpp @@ -427,7 +427,8 @@ class IndexCompactionUtils { const TabletSharedPtr& tablet, bool is_index_compaction, RowsetSharedPtr& rowset_ptr, const std::function custom_check = nullptr, - int64_t max_rows_per_segment = 100000) { + int64_t max_rows_per_segment = 100000, + const std::optional& output_storage_resource = std::nullopt) { config::inverted_index_compaction_enable = is_index_compaction; // control max rows in one block config::compaction_batch_size = max_rows_per_segment; @@ -441,6 +442,8 @@ class IndexCompactionUtils { RowsetWriterContext ctx; ctx.max_rows_per_segment = max_rows_per_segment; + // Empty for a local output rowset, which is what is_local_rowset() keys off. + ctx.storage_resource = output_storage_resource; RETURN_IF_ERROR(compaction.construct_output_rowset_writer(ctx)); compaction._stats.rowid_conversion = compaction._rowid_conversion.get(); @@ -585,7 +588,8 @@ class IndexCompactionUtils { InvertedIndexDescriptor::get_index_file_path_prefix(seg_path.value()); auto index_file_reader = std::make_shared( fs, std::string(index_file_path_prefix), - tablet_schema->get_inverted_index_storage_format(), index_info); + tablet_schema->get_inverted_index_storage_format(), index_info, + output_rowset->rowset_meta()->tablet_id()); EXPECT_TRUE(index_file_reader->init().ok()); const auto& dirs = index_file_reader->get_all_directories(); EXPECT_TRUE(dirs.has_value()); @@ -616,11 +620,11 @@ class IndexCompactionUtils { } } - static RowsetWriterContext rowset_writer_context(const std::unique_ptr& data_dir, - const TabletSchemaSPtr& schema, - const std::string& tablet_path, - int64_t& inc_id, - int64_t max_rows_per_segment = 200) { + static RowsetWriterContext rowset_writer_context( + const std::unique_ptr& data_dir, const TabletSchemaSPtr& schema, + const std::string& tablet_path, int64_t& inc_id, int64_t max_rows_per_segment = 200, + const std::optional& storage_resource = std::nullopt, + int64_t tablet_id = 0, bool write_file_cache = false) { RowsetWriterContext context; RowsetId rowset_id; rowset_id.init(inc_id); @@ -630,8 +634,16 @@ class IndexCompactionUtils { context.rowset_state = VISIBLE; context.tablet_schema = schema; context.tablet_path = tablet_path; + // Remote rowsets need this: CachedRemoteFileReader asserts tablet_id > 0 for Doris tables. + context.tablet_id = tablet_id; context.version = Version(inc_id, inc_id); context.max_rows_per_segment = max_rows_per_segment; + // Set only by callers that want the rowset to live on remote storage; a local rowset + // leaves it empty, which is what is_local_rowset() keys off. + context.storage_resource = storage_resource; + // Cloud sets this from the load request (cloud_rowset_builder.cpp), which is what leaves + // the block cache warm for compaction. Off by default, matching non-cloud writes. + context.write_file_cache = write_file_cache; inc_id++; return context; } @@ -643,7 +655,9 @@ class IndexCompactionUtils { const std::vector& data_files, int64_t& inc_id, const std::function custom_check = nullptr, const bool& is_performance = false, - int64_t max_rows_per_segment = 200) { + int64_t max_rows_per_segment = 200, + const std::optional& storage_resource = std::nullopt, + bool write_file_cache = false) { std::vector> data; for (const auto& file : data_files) { data.emplace_back(read_data(file)); @@ -652,7 +666,8 @@ class IndexCompactionUtils { const auto& res = RowsetFactory::create_rowset_writer( *engine_ref, rowset_writer_context(data_dir, schema, tablet->tablet_path(), inc_id, - max_rows_per_segment), + max_rows_per_segment, storage_resource, + tablet->tablet_id(), write_file_cache), false); EXPECT_TRUE(res.has_value()) << res.error(); const auto& rowset_writer = res.value(); @@ -715,8 +730,10 @@ class IndexCompactionUtils { (rowsets[i]->num_rows() / max_rows_per_segment)) << rowsets[i]->num_segments(); - // check rowset meta and file - for (int seg_id = 0; seg_id < rowsets[i]->num_segments(); seg_id++) { + // check rowset meta and file -- local only: the paths below are local, so a remote + // rowset would always report size 0 here. + for (int seg_id = 0; rowsets[i]->is_local() && seg_id < rowsets[i]->num_segments(); + seg_id++) { const auto& index_info = rowsets[i]->_rowset_meta->inverted_index_file_info(seg_id); EXPECT_TRUE(index_info.has_index_size()); const auto& fs = rowsets[i]->_rowset_meta->fs(); @@ -733,13 +750,26 @@ class IndexCompactionUtils { InvertedIndexDescriptor::get_index_file_path_prefix(seg_path.value()); auto index_file_reader = std::make_shared( fs, std::string(index_file_path_prefix), - schema->get_inverted_index_storage_format(), index_info); + schema->get_inverted_index_storage_format(), index_info, + rowsets[i]->rowset_meta()->tablet_id()); st = index_file_reader->init(); EXPECT_TRUE(st.ok()) << st.to_string(); - const auto& dirs = index_file_reader->get_all_directories(); - EXPECT_TRUE(dirs.has_value()); - if (custom_check) { - custom_check(dirs.value().size()); + if (schema->get_inverted_index_storage_format() == + InvertedIndexStorageFormatPB::SNII) { + const auto& indexes = schema->inverted_indexes(); + for (const TabletIndex* index : indexes) { + const auto logical_index = index_file_reader->open_snii_index(index); + EXPECT_TRUE(logical_index.has_value()) << logical_index.error(); + } + if (custom_check) { + custom_check(indexes.size()); + } + } else { + const auto& dirs = index_file_reader->get_all_directories(); + EXPECT_TRUE(dirs.has_value()); + if (custom_check) { + custom_check(dirs.value().size()); + } } } } diff --git a/be/test/storage/index/inverted/empty_index_file_test.cpp b/be/test/storage/index/inverted/empty_index_file_test.cpp index af1e7f0630f7eb..7acf6b13ef14a7 100644 --- a/be/test/storage/index/inverted/empty_index_file_test.cpp +++ b/be/test/storage/index/inverted/empty_index_file_test.cpp @@ -31,35 +31,56 @@ constexpr int64_t LOAD_ID_LO = 1; constexpr int64_t LOAD_ID_HI = 2; constexpr int64_t NUM_STREAM = 3; constexpr static std::string_view tmp_dir = "./ut_dir/tmp"; -class EmptyIndexFileTest : public testing::Test { +class EmptyIndexFileTest : public testing::TestWithParam { + struct WriteState { + size_t bytes_appended = 0; + int data_calls = 0; + int eos_calls = 0; + }; + class MockStreamStub : public LoadStreamStub { public: - MockStreamStub(PUniqueId load_id, int64_t src_id) + MockStreamStub(PUniqueId load_id, int64_t src_id, std::shared_ptr state) : LoadStreamStub(load_id, src_id, std::make_shared(), - std::make_shared()) {}; + std::make_shared()), + _state(std::move(state)) {}; - virtual ~MockStreamStub() = default; + ~MockStreamStub() override = default; // APPEND_DATA - virtual Status append_data(int64_t partition_id, int64_t index_id, int64_t tablet_id, - int32_t segment_id, uint64_t offset, std::span data, - bool segment_eos = false, - FileType file_type = FileType::SEGMENT_FILE) override { - EXPECT_TRUE(segment_eos); + Status append_data(int64_t partition_id, int64_t index_id, int64_t tablet_id, + int32_t segment_id, uint64_t offset, std::span data, + bool segment_eos = false, + FileType file_type = FileType::SEGMENT_FILE) override { + EXPECT_EQ(offset, _state->bytes_appended); + if (segment_eos) { + ++_state->eos_calls; + EXPECT_TRUE(data.empty()); + return Status::OK(); + } + ++_state->data_calls; + for (const auto& slice : data) { + _state->bytes_appended += slice.size; + } return Status::OK(); } + + private: + std::shared_ptr _state; }; public: EmptyIndexFileTest() = default; - ~EmptyIndexFileTest() = default; + ~EmptyIndexFileTest() override = default; protected: - virtual void SetUp() { + void SetUp() override { _load_id.set_hi(LOAD_ID_HI); _load_id.set_lo(LOAD_ID_LO); for (int src_id = 0; src_id < NUM_STREAM; src_id++) { - _streams.emplace_back(new MockStreamStub(_load_id, src_id)); + auto state = std::make_shared(); + _write_states.push_back(state); + _streams.emplace_back(new MockStreamStub(_load_id, src_id, std::move(state))); } EXPECT_TRUE(io::global_local_filesystem()->delete_directory(tmp_dir).ok()); EXPECT_TRUE(io::global_local_filesystem()->create_directory(tmp_dir).ok()); @@ -70,25 +91,34 @@ class EmptyIndexFileTest : public testing::Test { ExecEnv::GetInstance()->set_tmp_file_dir(std::move(tmp_file_dirs)); } - virtual void TearDown() { + void TearDown() override { EXPECT_TRUE(io::global_local_filesystem()->delete_directory(tmp_dir).ok()); } PUniqueId _load_id; std::vector> _streams; + std::vector> _write_states; }; -TEST_F(EmptyIndexFileTest, test_empty_index_file) { +TEST_P(EmptyIndexFileTest, PreservesZeroByteFileWhenNoLogicalIndexes) { io::FileWriterPtr file_writer = std::make_unique(_streams); auto fs = io::global_local_filesystem(); std::string index_path = "/tmp/empty_index_file_test"; std::string rowset_id = "1234567890"; int64_t seg_id = 1234567890; auto index_file_writer = std::make_unique( - fs, index_path, rowset_id, seg_id, InvertedIndexStorageFormatPB::V2, - std::move(file_writer), false); + fs, index_path, rowset_id, seg_id, GetParam(), std::move(file_writer), false); EXPECT_TRUE(index_file_writer->begin_close().ok()); EXPECT_TRUE(index_file_writer->finish_close().ok()); + for (const auto& state : _write_states) { + EXPECT_EQ(state->bytes_appended, 0); + EXPECT_EQ(state->data_calls, 0); + EXPECT_EQ(state->eos_calls, 1); + } } +INSTANTIATE_TEST_SUITE_P(LegacyCompoundFormats, EmptyIndexFileTest, + testing::Values(InvertedIndexStorageFormatPB::V2, + InvertedIndexStorageFormatPB::V3)); + } // namespace doris diff --git a/be/test/storage/index/inverted/inverted_index_reader_analysis_purpose_test.cpp b/be/test/storage/index/inverted/inverted_index_reader_analysis_purpose_test.cpp new file mode 100644 index 00000000000000..f3f8703c55f789 --- /dev/null +++ b/be/test/storage/index/inverted/inverted_index_reader_analysis_purpose_test.cpp @@ -0,0 +1,697 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "common/exception.h" +#include "io/fs/local_file_system.h" +#include "runtime/exec_env.h" +#include "runtime/runtime_state.h" +#include "storage/compaction/collection_similarity.h" +#include "storage/index/index_file_reader.h" +#include "storage/index/inverted/analyzer/analyzer.h" +#include "storage/index/inverted/analyzer/analyzer_provider.h" +#include "storage/index/inverted/analyzer/custom_analyzer.h" +#include "storage/index/inverted/common_grams/common_grams_key_codec.h" +#include "storage/index/inverted/inverted_index_cache.h" +#include "storage/index/inverted/inverted_index_reader.h" +#include "storage/index/inverted/token_filter/common_grams_filter.h" +#include "storage/index/snii/snii_index_reader.h" +#include "storage/tablet/tablet_schema.h" +#include "util/defer_op.h" +#include "util/time.h" + +namespace doris::segment_v2 { +namespace { + +using inverted_index::AnalysisPurpose; +using inverted_index::AnalyzerProvider; + +class RecordingFailingAnalyzerProvider final : public AnalyzerProvider { +public: + std::shared_ptr get_analyzer( + AnalysisPurpose purpose) const override { + purposes.push_back(purpose); + throw Exception(ErrorCode::INVERTED_INDEX_ANALYZER_ERROR, + "forced analyzer provider failure"); + } + + mutable std::vector purposes; +}; + +class IdentityFailingAnalyzerProvider final : public AnalyzerProvider { +public: + explicit IdentityFailingAnalyzerProvider(inverted_index::CommonGramsQueryIdentity identity) + : _identity(std::move(identity)) {} + + std::shared_ptr get_analyzer( + AnalysisPurpose purpose) const override { + purposes.push_back(purpose); + throw Exception(ErrorCode::INVERTED_INDEX_ANALYZER_ERROR, + "forced analyzer provider failure"); + } + + bool uses_common_grams() const override { return true; } + + const inverted_index::CommonGramsQueryIdentity* common_grams_identity() const override { + return &_identity; + } + + mutable std::vector purposes; + +private: + inverted_index::CommonGramsQueryIdentity _identity; +}; + +class PartialFailureTokenStream final : public lucene::analysis::TokenStream { +public: + explicit PartialFailureTokenStream(std::shared_ptr> emitted_tokens) + : _emitted_tokens(std::move(emitted_tokens)) {} + + lucene::analysis::Token* next(lucene::analysis::Token* token) override { + if (!_emitted) { + _emitted = true; + _term = "partial"; + token->clear(); + token->setTextNoCopy(_term.data(), static_cast(_term.size())); + token->positionIncrement = 1; + _emitted_tokens->fetch_add(1, std::memory_order_relaxed); + return token; + } + throw Exception(ErrorCode::INVERTED_INDEX_ANALYZER_ERROR, + "forced failure after first token"); + } + + void close() override {} + void reset() override { _emitted = false; } + +private: + std::shared_ptr> _emitted_tokens; + bool _emitted = false; + std::string _term; +}; + +class PartialFailureAnalyzer final : public lucene::analysis::Analyzer { +public: + explicit PartialFailureAnalyzer(std::shared_ptr> emitted_tokens) + : _emitted_tokens(std::move(emitted_tokens)) {} + + bool isSDocOpt() override { return true; } + + lucene::analysis::TokenStream* tokenStream(const TCHAR*, lucene::util::Reader*) override { + return new PartialFailureTokenStream(_emitted_tokens); + } + + lucene::analysis::TokenStream* reusableTokenStream(const TCHAR*, + lucene::util::Reader*) override { + _reusable = std::make_unique(_emitted_tokens); + return _reusable.get(); + } + + lucene::analysis::TokenStream* tokenStream(const TCHAR*, + const inverted_index::ReaderPtr&) override { + return new PartialFailureTokenStream(_emitted_tokens); + } + + lucene::analysis::TokenStream* reusableTokenStream(const TCHAR*, + const inverted_index::ReaderPtr&) override { + _reusable = std::make_unique(_emitted_tokens); + return _reusable.get(); + } + +private: + std::shared_ptr> _emitted_tokens; + std::unique_ptr _reusable; +}; + +class RecordingPartialFailureAnalyzerProvider final : public AnalyzerProvider { +public: + std::shared_ptr get_analyzer( + AnalysisPurpose purpose) const override { + purposes.push_back(purpose); + return std::make_shared(emitted_tokens); + } + + mutable std::vector purposes; + std::shared_ptr> emitted_tokens = + std::make_shared>(0); +}; + +class GeneratedTokenStream final : public lucene::analysis::TokenStream { +public: + GeneratedTokenStream(std::string term, bool mark_common_gram) + : _term(std::move(term)), _mark_common_gram(mark_common_gram) {} + + lucene::analysis::Token* next(lucene::analysis::Token* token) override { + if (_emitted) { + return nullptr; + } + _emitted = true; + token->clear(); + token->setTextNoCopy(_term.data(), static_cast(_term.size())); + token->setPositionIncrement(1); + if (_mark_common_gram) { + token->setType(inverted_index::COMMON_GRAM_TOKEN_TYPE); + } + return token; + } + + void close() override {} + void reset() override { _emitted = false; } + +private: + std::string _term; + bool _mark_common_gram = false; + bool _emitted = false; +}; + +class GeneratedTokenAnalyzer final : public lucene::analysis::Analyzer { +public: + GeneratedTokenAnalyzer(std::string term, bool mark_common_gram) + : _term(std::move(term)), _mark_common_gram(mark_common_gram) {} + + bool isSDocOpt() override { return true; } + + lucene::analysis::TokenStream* tokenStream(const TCHAR*, lucene::util::Reader*) override { + return new GeneratedTokenStream(_term, _mark_common_gram); + } + + lucene::analysis::TokenStream* reusableTokenStream(const TCHAR*, + lucene::util::Reader*) override { + _reusable = std::make_unique(_term, _mark_common_gram); + return _reusable.get(); + } + + lucene::analysis::TokenStream* tokenStream(const TCHAR*, + const inverted_index::ReaderPtr&) override { + return new GeneratedTokenStream(_term, _mark_common_gram); + } + + lucene::analysis::TokenStream* reusableTokenStream(const TCHAR*, + const inverted_index::ReaderPtr&) override { + _reusable = std::make_unique(_term, _mark_common_gram); + return _reusable.get(); + } + +private: + std::string _term; + bool _mark_common_gram = false; + std::unique_ptr _reusable; +}; + +class GeneratedGramAnalyzerProvider final : public AnalyzerProvider { +public: + GeneratedGramAnalyzerProvider() + : _gram(*inverted_index::encode_common_gram("the", "history")), + _identity {.common_grams_dictionary_identity = "builtin-stopwords:v1", + .base_analyzer_fingerprint = "base:v1", + .common_grams_fingerprint = "grams:v1"} {} + + std::shared_ptr get_analyzer( + AnalysisPurpose purpose) const override { + purposes.push_back(purpose); + return std::make_shared(_gram, true); + } + + bool uses_common_grams() const override { return true; } + const inverted_index::CommonGramsQueryIdentity* common_grams_identity() const override { + return &_identity; + } + + mutable std::vector purposes; + +private: + std::string _gram; + inverted_index::CommonGramsQueryIdentity _identity; +}; + +struct QueryExecutionContext { + explicit QueryExecutionContext(bool scoring) { + TQueryOptions query_options; + query_options.enable_inverted_index_query_cache = true; + query_options.enable_inverted_index_searcher_cache = true; + runtime_state.set_query_options(query_options); + context->io_ctx = &io_ctx; + context->stats = &stats; + context->runtime_state = &runtime_state; + if (scoring) { + context->collection_similarity = std::make_shared(); + } + } + + OlapReaderStatistics stats; + io::IOContext io_ctx; + RuntimeState runtime_state; + IndexQueryContextPtr context = std::make_shared(); +}; + +class InvertedIndexReaderAnalysisPurposeTest : public testing::Test { +protected: + void SetUp() override { + _previous_searcher_cache = ExecEnv::GetInstance()->get_inverted_index_searcher_cache(); + _previous_query_cache = ExecEnv::GetInstance()->get_inverted_index_query_cache(); + _searcher_cache.reset(InvertedIndexSearcherCache::create_global_instance(1024 * 1024, 1)); + _query_cache.reset(InvertedIndexQueryCache::create_global_cache(1024 * 1024, 1)); + ExecEnv::GetInstance()->set_inverted_index_searcher_cache(_searcher_cache.get()); + ExecEnv::GetInstance()->set_inverted_index_query_cache(_query_cache.get()); + + TabletIndexPB pb; + pb.set_index_type(IndexType::INVERTED); + pb.set_index_id(73); + pb.set_index_name("analysis_purpose_idx"); + pb.add_col_unique_id(0); + pb.mutable_properties()->insert({"parser", "english"}); + pb.mutable_properties()->insert({"lower_case", "true"}); + pb.mutable_properties()->insert({"support_phrase", "true"}); + _meta.init_from_pb(pb); + + _snii_file_reader = std::make_shared( + io::global_local_filesystem(), "./ut_dir/missing_snii_analysis_purpose", + InvertedIndexStorageFormatPB::SNII); + _snii_reader = SniiIndexReader::create_shared(&_meta, _snii_file_reader, + InvertedIndexReaderType::FULLTEXT); + } + + void TearDown() override { + _snii_reader.reset(); + _snii_file_reader.reset(); + ExecEnv::GetInstance()->set_inverted_index_searcher_cache(_previous_searcher_cache); + ExecEnv::GetInstance()->set_inverted_index_query_cache(_previous_query_cache); + _searcher_cache.reset(); + _query_cache.reset(); + } + + // This entry is admission-only: tests must exit during analysis before dereferencing the + // SNII postings reader. + void preload_legacy_searcher_cache_entries() { + const InvertedIndexSearcherCache::CacheKey snii_key( + _snii_file_reader->get_index_file_cache_key(&_meta)); + _searcher_cache->insert( + snii_key, new InvertedIndexSearcherCache::CacheValue( + std::make_unique(), 1, + UnixMillis(), _snii_file_reader)); + } + + template + void expect_analysis_failure_after_segment_admission(const std::shared_ptr& reader, + InvertedIndexQueryType query_type, + std::string query, bool scoring, + AnalysisPurpose expected_purpose, + const std::shared_ptr& provider, + int64_t expected_query_cache_lookups = 0, + int64_t expected_searcher_cache_hits = 1) { + QueryExecutionContext execution(scoring); + InvertedIndexAnalyzerCtx analyzer_ctx; + analyzer_ctx.parser_type = InvertedIndexParserType::PARSER_ENGLISH; + analyzer_ctx.analyzer_provider = provider; + + auto original_bitmap = std::make_shared(); + original_bitmap->add(999); + std::shared_ptr bitmap = original_bitmap; + const Field query_value = Field::create_field(std::move(query)); + + Status status; + EXPECT_NO_THROW(status = reader->query(execution.context, "content", query_value, + query_type, bitmap, &analyzer_ctx)); + EXPECT_EQ(status.code(), ErrorCode::INVERTED_INDEX_ANALYZER_ERROR) << status; + EXPECT_EQ(provider->purposes, (std::vector {expected_purpose})); + EXPECT_EQ(bitmap, original_bitmap); + EXPECT_EQ(bitmap->cardinality(), 1); + EXPECT_TRUE(bitmap->contains(999)); + EXPECT_EQ(execution.stats.inverted_index_query_cache_hit, 0); + EXPECT_EQ(execution.stats.inverted_index_query_cache_miss, expected_query_cache_lookups); + EXPECT_EQ(execution.stats.inverted_index_query_cache_lookup, expected_query_cache_lookups); + EXPECT_EQ(execution.stats.inverted_index_query_cache_insert, 0); + EXPECT_EQ(execution.stats.inverted_index_searcher_cache_hit, expected_searcher_cache_hits); + EXPECT_EQ(execution.stats.inverted_index_searcher_cache_miss, 0); + } + + template + void expect_provider_failure_after_segment_admission(const std::shared_ptr& reader, + InvertedIndexQueryType query_type, + std::string query, bool scoring, + AnalysisPurpose expected_purpose, + int64_t expected_query_cache_lookups = 0, + int64_t expected_searcher_cache_hits = 1) { + expect_analysis_failure_after_segment_admission( + reader, query_type, std::move(query), scoring, expected_purpose, + std::make_shared(), expected_query_cache_lookups, + expected_searcher_cache_hits); + } + + template + void expect_generated_gram_bypass_after_segment_admission( + const std::shared_ptr& reader) { + QueryExecutionContext execution(/*scoring=*/false); + auto provider = std::make_shared(); + InvertedIndexAnalyzerCtx analyzer_ctx; + analyzer_ctx.parser_type = InvertedIndexParserType::PARSER_ENGLISH; + analyzer_ctx.analyzer_provider = provider; + auto original_bitmap = std::make_shared(); + original_bitmap->add(999); + std::shared_ptr bitmap = original_bitmap; + const Field query_value = Field::create_field("the history"); + + const Status status = + reader->query(execution.context, "content", query_value, + InvertedIndexQueryType::MATCH_PHRASE_QUERY, bitmap, &analyzer_ctx); + EXPECT_EQ(status.code(), ErrorCode::INVERTED_INDEX_BYPASS) << status; + EXPECT_EQ(provider->purposes, + (std::vector {AnalysisPurpose::kPlainQuery})); + EXPECT_EQ(bitmap, original_bitmap); + EXPECT_EQ(execution.stats.inverted_index_query_cache_hit, 0); + EXPECT_EQ(execution.stats.inverted_index_query_cache_miss, 1); + EXPECT_EQ(execution.stats.inverted_index_query_cache_lookup, 1); + EXPECT_EQ(execution.stats.inverted_index_query_cache_insert, 0); + EXPECT_EQ(execution.stats.inverted_index_searcher_cache_hit, 1); + EXPECT_EQ(execution.stats.inverted_index_searcher_cache_miss, 0); + } + + template + void expect_raw_query_bypasses_analyzer(const std::shared_ptr& reader, + InvertedIndexQueryType query_type, std::string query) { + QueryExecutionContext execution(/*scoring=*/false); + auto provider = std::make_shared(); + InvertedIndexAnalyzerCtx analyzer_ctx; + analyzer_ctx.parser_type = InvertedIndexParserType::PARSER_ENGLISH; + analyzer_ctx.analyzer_provider = provider; + std::shared_ptr bitmap; + const Field query_value = Field::create_field(std::move(query)); + + const Status status = reader->query(execution.context, "content", query_value, query_type, + bitmap, &analyzer_ctx); + EXPECT_NE(status.code(), ErrorCode::INVERTED_INDEX_ANALYZER_ERROR) << status; + EXPECT_TRUE(provider->purposes.empty()); + } + + template + void expect_raw_cache_hit_before_analysis(const std::shared_ptr& reader, + const std::shared_ptr& file_reader, + const std::shared_ptr& provider, + uint64_t common_grams_cache_generation) { + QueryExecutionContext execution(/*scoring=*/false); + InvertedIndexAnalyzerCtx analyzer_ctx; + analyzer_ctx.parser_type = InvertedIndexParserType::PARSER_ENGLISH; + analyzer_ctx.analyzer_provider = provider; + + const std::string raw_query = "the history"; + const InvertedIndexRawQuerySemantic semantic { + .raw_query_bytes = raw_query, + .query_type = InvertedIndexQueryType::MATCH_PHRASE_QUERY, + .slop = 0, + .ordered = false, + .max_expansions = + execution.runtime_state.query_options().inverted_index_max_expansions, + .common_grams_cache_generation = common_grams_cache_generation}; + const InvertedIndexQueryCache::CacheKey key { + file_reader->get_index_file_cache_key(&_meta), "content", + InvertedIndexQueryType::MATCH_PHRASE_QUERY, semantic.encode()}; + auto cached = std::make_shared(); + cached->add(7); + InvertedIndexQueryCacheHandle insert_handle; + _query_cache->insert(key, cached, &insert_handle); + + std::shared_ptr bitmap; + const Field query_value = Field::create_field(raw_query); + const Status status = + reader->query(execution.context, "content", query_value, + InvertedIndexQueryType::MATCH_PHRASE_QUERY, bitmap, &analyzer_ctx); + EXPECT_TRUE(status.ok()) << status; + EXPECT_TRUE(provider->purposes.empty()); + ASSERT_NE(bitmap, nullptr); + EXPECT_EQ(bitmap->cardinality(), 1); + EXPECT_TRUE(bitmap->contains(7)); + EXPECT_EQ(execution.stats.inverted_index_query_cache_hit, 1); + EXPECT_EQ(execution.stats.inverted_index_query_cache_miss, 0); + EXPECT_EQ(execution.stats.inverted_index_query_cache_lookup, 1); + EXPECT_EQ(execution.stats.inverted_index_query_cache_insert, 0); + EXPECT_EQ(execution.stats.inverted_index_searcher_cache_hit, 0); + EXPECT_EQ(execution.stats.inverted_index_searcher_cache_miss, 0); + } + + InvertedIndexSearcherCache* _previous_searcher_cache = nullptr; + InvertedIndexQueryCache* _previous_query_cache = nullptr; + std::unique_ptr _searcher_cache; + std::unique_ptr _query_cache; + TabletIndex _meta; + std::shared_ptr _snii_file_reader; + std::shared_ptr _snii_reader; +}; + +TEST(InvertedIndexRawQuerySemanticTest, EncodesOnlyRawSemanticDimensionsWithoutDelimiters) { + const std::string raw_query("a/b\0c", 5); + InvertedIndexRawQuerySemantic base {.raw_query_bytes = raw_query, + .query_type = InvertedIndexQueryType::MATCH_PHRASE_QUERY, + .slop = 2, + .ordered = true, + .max_expansions = 50, + .cache_semantics_version = 3, + .common_grams_cache_generation = 7}; + const std::string encoded = base.encode(); + constexpr size_t kFixedEncodedBytes = sizeof(uint32_t) + sizeof(uint64_t) + sizeof(uint32_t) + + sizeof(uint32_t) + sizeof(uint8_t) + sizeof(uint32_t) + + sizeof(uint64_t); + EXPECT_EQ(encoded.size(), kFixedEncodedBytes + raw_query.size()); + + auto changed = base; + changed.raw_query_bytes = std::string_view(raw_query).substr(0, 3); + EXPECT_NE(changed.encode(), encoded); + changed = base; + changed.query_type = InvertedIndexQueryType::MATCH_PHRASE_PREFIX_QUERY; + EXPECT_NE(changed.encode(), encoded); + changed = base; + changed.slop = 3; + EXPECT_NE(changed.encode(), encoded); + changed = base; + changed.ordered = false; + EXPECT_NE(changed.encode(), encoded); + changed = base; + changed.max_expansions = 51; + EXPECT_NE(changed.encode(), encoded); + changed = base; + changed.cache_semantics_version = 4; + EXPECT_NE(changed.encode(), encoded); + changed = base; + changed.common_grams_cache_generation = 8; + EXPECT_NE(changed.encode(), encoded); +} + +TEST(InvertedIndexRawQuerySemanticTest, CacheEnvelopeSeparatesSlashAndNulBoundaries) { + const InvertedIndexQueryCache::CacheKey slash_left { + io::Path("a/b"), "c", InvertedIndexQueryType::MATCH_PHRASE_QUERY, "d"}; + const InvertedIndexQueryCache::CacheKey slash_right { + io::Path("a"), "b/c", InvertedIndexQueryType::MATCH_PHRASE_QUERY, "d"}; + EXPECT_NE(slash_left.encode(), slash_right.encode()); + + const InvertedIndexQueryCache::CacheKey nul_left { + io::Path("a"), std::string("b\0c", 3), InvertedIndexQueryType::MATCH_PHRASE_QUERY, "d"}; + const InvertedIndexQueryCache::CacheKey nul_right { + io::Path(std::string("a\0b", 3)), "c", InvertedIndexQueryType::MATCH_PHRASE_QUERY, "d"}; + EXPECT_NE(nul_left.encode(), nul_right.encode()); +} + +TEST(InvertedIndexRawQuerySemanticTest, KillSwitchTransitionsAdvanceCacheGeneration) { + const bool original = config::enable_common_grams_query_plan; + const auto before = config::common_grams_query_plan_config_snapshot(); + + ASSERT_TRUE(config::set_config("enable_common_grams_query_plan", original ? "false" : "true", + /*need_persist=*/false) + .ok()); + const auto disabled_or_enabled = config::common_grams_query_plan_config_snapshot(); + ASSERT_TRUE(config::set_config("enable_common_grams_query_plan", original ? "true" : "false", + /*need_persist=*/false) + .ok()); + const auto restored = config::common_grams_query_plan_config_snapshot(); + + EXPECT_EQ(before.enabled, original); + EXPECT_EQ(disabled_or_enabled.enabled, !original); + EXPECT_GT(disabled_or_enabled.cache_generation, before.cache_generation); + EXPECT_EQ(restored.enabled, original); + EXPECT_GT(restored.cache_generation, disabled_or_enabled.cache_generation); +} + +TEST(InvertedIndexRawQuerySemanticTest, CommonGramsQueryPlanIsDisabledByDefault) { + EXPECT_FALSE(config::enable_common_grams_query_plan); + EXPECT_FALSE(config::common_grams_query_plan_config_snapshot().enabled); +} + +TEST(InvertedIndexRawQuerySemanticTest, CostModelSnapshotTracksDynamicConfigAsOneVersion) { + const int32_t original_ratio = config::common_grams_plan_cost_ratio_percent; + const int32_t original_factor = config::common_grams_position_verify_factor; + const int32_t changed_ratio = original_ratio == 84 ? 85 : 84; + const int32_t changed_factor = original_factor == 7 ? 8 : 7; + + ASSERT_TRUE(config::set_config("common_grams_plan_cost_ratio_percent", + std::to_string(changed_ratio)) + .ok()); + ASSERT_TRUE(config::set_config("common_grams_position_verify_factor", + std::to_string(changed_factor)) + .ok()); + const auto changed = config::common_grams_query_plan_config_snapshot(); + EXPECT_EQ(changed.plan_cost_ratio_percent, changed_ratio); + EXPECT_EQ(changed.position_verify_factor, changed_factor); + EXPECT_EQ(changed.cost_model_generation, + (static_cast(static_cast(changed_factor)) << 32) | + static_cast(changed_ratio)); + + ASSERT_TRUE(config::set_config("common_grams_plan_cost_ratio_percent", + std::to_string(original_ratio)) + .ok()); + ASSERT_TRUE(config::set_config("common_grams_position_verify_factor", + std::to_string(original_factor)) + .ok()); +} + +TEST(InvertedIndexRawQuerySemanticTest, InvalidCostModelConfigDoesNotMutatePublishedState) { + const auto before = config::common_grams_query_plan_config_snapshot(); + + for (const auto& [field, value] : std::vector> { + {"common_grams_plan_cost_ratio_percent", "-1"}, + {"common_grams_plan_cost_ratio_percent", "101"}, + {"common_grams_position_verify_factor", "-1"}}) { + EXPECT_FALSE(config::set_config(field, value).ok()); + const auto after = config::common_grams_query_plan_config_snapshot(); + EXPECT_EQ(after.plan_cost_ratio_percent, before.plan_cost_ratio_percent); + EXPECT_EQ(after.position_verify_factor, before.position_verify_factor); + EXPECT_EQ(after.cost_model_generation, before.cost_model_generation); + } +} + +TEST(InvertedIndexAnalyzerCtxTest, UsesProviderCommonGramsIdentityWithoutCopyingIt) { + auto provider = std::make_shared(); + InvertedIndexAnalyzerCtx analyzer_ctx; + analyzer_ctx.analyzer_provider = provider; + EXPECT_EQ(analyzer_ctx.get_common_grams_identity(), provider->common_grams_identity()); +} + +TEST_F(InvertedIndexReaderAnalysisPurposeTest, + SniiRawCacheLookupIsIndependentOfRequestAnalyzerIdentity) { + const bool original = config::enable_common_grams_query_plan; + Defer restore([original] { + EXPECT_TRUE(config::set_config("enable_common_grams_query_plan", + original ? "true" : "false", /*need_persist=*/false) + .ok()); + }); + ASSERT_TRUE(config::set_config("enable_common_grams_query_plan", "true", + /*need_persist=*/false) + .ok()); + const auto safety = config::common_grams_query_plan_config_snapshot(); + const inverted_index::CommonGramsQueryIdentity complete_identity { + .common_grams_dictionary_identity = "dictionary:complete", + .base_analyzer_fingerprint = "base:complete", + .common_grams_fingerprint = "grams:complete"}; + const inverted_index::CommonGramsQueryIdentity empty_identity; + for (const auto& identity : {complete_identity, empty_identity}) { + expect_raw_cache_hit_before_analysis( + _snii_reader, _snii_file_reader, + std::make_shared(identity), + safety.cache_generation); + } + expect_raw_cache_hit_before_analysis(_snii_reader, _snii_file_reader, + std::make_shared(), + safety.cache_generation); +} + +TEST_F(InvertedIndexReaderAnalysisPurposeTest, DisabledResultCacheDoesNotLookupCountOrInsert) { + QueryExecutionContext execution(/*scoring=*/false); + TQueryOptions disabled_options; + disabled_options.enable_inverted_index_query_cache = false; + disabled_options.enable_inverted_index_searcher_cache = true; + execution.runtime_state.set_query_options(disabled_options); + + const InvertedIndexQueryCache::CacheKey key {io::Path("disabled-cache"), "content", + InvertedIndexQueryType::MATCH_PHRASE_QUERY, + "raw-semantic"}; + auto bitmap = std::make_shared(); + bitmap->add(3); + InvertedIndexQueryCacheHandle handle; + _snii_reader->insert_query_cache(execution.context, _query_cache.get(), key, bitmap, &handle); + std::shared_ptr lookup_bitmap; + EXPECT_FALSE(_snii_reader->handle_query_cache(execution.context, _query_cache.get(), key, + &handle, lookup_bitmap)); + EXPECT_EQ(execution.stats.inverted_index_query_cache_hit, 0); + EXPECT_EQ(execution.stats.inverted_index_query_cache_miss, 0); + EXPECT_EQ(execution.stats.inverted_index_query_cache_lookup, 0); + EXPECT_EQ(execution.stats.inverted_index_query_cache_insert, 0); + + TQueryOptions enabled_options; + enabled_options.enable_inverted_index_query_cache = true; + enabled_options.enable_inverted_index_searcher_cache = true; + execution.runtime_state.set_query_options(enabled_options); + EXPECT_FALSE(_snii_reader->handle_query_cache(execution.context, _query_cache.get(), key, + &handle, lookup_bitmap)); + EXPECT_EQ(execution.stats.inverted_index_query_cache_miss, 1); + EXPECT_EQ(execution.stats.inverted_index_query_cache_lookup, 1); +} + +TEST_F(InvertedIndexReaderAnalysisPurposeTest, SniiSelectsPurposeAfterSegmentAdmission) { + preload_legacy_searcher_cache_entries(); + expect_provider_failure_after_segment_admission( + _snii_reader, InvertedIndexQueryType::MATCH_PHRASE_QUERY, "the history", false, + AnalysisPurpose::kPlainQuery, /*expected_query_cache_lookups=*/1); + expect_provider_failure_after_segment_admission( + _snii_reader, InvertedIndexQueryType::MATCH_PHRASE_QUERY, "the history ~2", false, + AnalysisPurpose::kPlainQuery, /*expected_query_cache_lookups=*/1); + expect_provider_failure_after_segment_admission( + _snii_reader, InvertedIndexQueryType::MATCH_PHRASE_PREFIX_QUERY, "the hist", false, + AnalysisPurpose::kPlainQuery, /*expected_query_cache_lookups=*/1); + expect_provider_failure_after_segment_admission( + _snii_reader, InvertedIndexQueryType::MATCH_PHRASE_QUERY, "the history", true, + AnalysisPurpose::kPlainQuery); +} + +TEST_F(InvertedIndexReaderAnalysisPurposeTest, PartialAnalysisFailureDoesNotPublishState) { + preload_legacy_searcher_cache_entries(); + auto snii_provider = std::make_shared(); + expect_analysis_failure_after_segment_admission( + _snii_reader, InvertedIndexQueryType::MATCH_PHRASE_QUERY, "the history", false, + AnalysisPurpose::kPlainQuery, snii_provider, /*expected_query_cache_lookups=*/1); + EXPECT_EQ(snii_provider->emitted_tokens->load(std::memory_order_relaxed), 1); +} + +TEST_F(InvertedIndexReaderAnalysisPurposeTest, + SniiGeneratedGramBypassesAfterSegmentAdmissionBeforeSearch) { + preload_legacy_searcher_cache_entries(); + const bool original = config::enable_common_grams_query_plan; + Defer restore([original] { + EXPECT_TRUE(config::set_config("enable_common_grams_query_plan", + original ? "true" : "false", /*need_persist=*/false) + .ok()); + }); + ASSERT_TRUE(config::set_config("enable_common_grams_query_plan", "true", + /*need_persist=*/false) + .ok()); + expect_generated_gram_bypass_after_segment_admission(_snii_reader); +} + +TEST_F(InvertedIndexReaderAnalysisPurposeTest, RegexpAndWildcardBypassAnalyzer) { + for (const auto query_type : + {InvertedIndexQueryType::MATCH_REGEXP_QUERY, InvertedIndexQueryType::WILDCARD_QUERY}) { + expect_raw_query_bypasses_analyzer(_snii_reader, query_type, "hist.*"); + } +} + +} // namespace +} // namespace doris::segment_v2 diff --git a/be/test/storage/index/inverted/query/phrase_edge_query_test.cpp b/be/test/storage/index/inverted/query/phrase_edge_query_test.cpp index 2da72758470a27..975ae80051678f 100644 --- a/be/test/storage/index/inverted/query/phrase_edge_query_test.cpp +++ b/be/test/storage/index/inverted/query/phrase_edge_query_test.cpp @@ -62,12 +62,19 @@ class PhraseEdgeQueryTest : public testing::Test { _inverted_index_query_cache = std::unique_ptr( InvertedIndexQueryCache::create_global_cache(inverted_index_cache_limit, 1)); + // Both caches are owned by this fixture, so the previous globals must come back in + // TearDown -- otherwise ExecEnv keeps pointing at them after the fixture is destroyed and + // the next test that reaches InvertedIndexQueryCache::instance() reads freed memory. + _previous_searcher_cache = ExecEnv::GetInstance()->get_inverted_index_searcher_cache(); + _previous_query_cache = ExecEnv::GetInstance()->get_inverted_index_query_cache(); ExecEnv::GetInstance()->set_inverted_index_searcher_cache( _inverted_index_searcher_cache.get()); - ExecEnv::GetInstance()->_inverted_index_query_cache = _inverted_index_query_cache.get(); + ExecEnv::GetInstance()->set_inverted_index_query_cache(_inverted_index_query_cache.get()); } void TearDown() override { + ExecEnv::GetInstance()->set_inverted_index_searcher_cache(_previous_searcher_cache); + ExecEnv::GetInstance()->set_inverted_index_query_cache(_previous_query_cache); EXPECT_TRUE(io::global_local_filesystem()->delete_directory(kTestDir).ok()); } @@ -180,6 +187,8 @@ class PhraseEdgeQueryTest : public testing::Test { ~PhraseEdgeQueryTest() override = default; private: + InvertedIndexSearcherCache* _previous_searcher_cache = nullptr; + InvertedIndexQueryCache* _previous_query_cache = nullptr; std::unique_ptr _inverted_index_searcher_cache; std::unique_ptr _inverted_index_query_cache; }; diff --git a/be/test/storage/index/inverted/similarity/bm25_similarity_test.cpp b/be/test/storage/index/inverted/similarity/bm25_similarity_test.cpp index 24fdfbd2e62709..9ceba92421e87f 100644 --- a/be/test/storage/index/inverted/similarity/bm25_similarity_test.cpp +++ b/be/test/storage/index/inverted/similarity/bm25_similarity_test.cpp @@ -22,8 +22,8 @@ #include #include "common/be_mock_util.h" -#include "storage/compaction/collection_statistics.h" #include "storage/index/index_query_context.h" +#include "storage/index/inverted/similarity/collection_statistics.h" using namespace doris; using namespace doris::segment_v2; diff --git a/be/test/storage/index/inverted/token_filter/lower_case_filter_factory_test.cpp b/be/test/storage/index/inverted/token_filter/lower_case_filter_factory_test.cpp index cd8c72755e16ce..6c19d437c9da59 100644 --- a/be/test/storage/index/inverted/token_filter/lower_case_filter_factory_test.cpp +++ b/be/test/storage/index/inverted/token_filter/lower_case_filter_factory_test.cpp @@ -20,10 +20,17 @@ #include +#include + #include "storage/index/inverted/tokenizer/keyword/keyword_tokenizer_factory.h" namespace doris::segment_v2::inverted_index { +namespace lower_case_testing { +uint64_t unicode_path_count(); +void reset_unicode_path_count(); +} // namespace lower_case_testing + TokenStreamPtr create_lowercase_filter(const std::string& text, Settings settings = Settings()) { ReaderPtr reader = std::make_shared>(); reader->init(text.data(), text.size(), false); @@ -45,6 +52,32 @@ struct ExpectedToken { int pos_inc; }; +class ScriptedLowercaseInput final : public TokenStream { +public: + explicit ScriptedLowercaseInput(std::vector terms) : _terms(std::move(terms)) {} + + Token* next(Token* token) override { + if (_next == _terms.size()) { + return nullptr; + } + const auto& term = _terms[_next++]; + token->clear(); + token->setTextNoCopy(term.data(), static_cast(term.size())); + token->setPositionIncrement(3); + token->setStartOffset(7); + token->setEndOffset(19); + token->setType(_T("scripted")); + return token; + } + + void close() override {} + void reset() override { _next = 0; } + +private: + std::vector _terms; + size_t _next = 0; +}; + class LowerCaseFilterTest : public ::testing::Test { protected: void assert_filter_output(const std::string& text, const std::vector& expected) { @@ -75,10 +108,73 @@ TEST_F(LowerCaseFilterTest, HandlesMixedCase) { assert_filter_output("HeLLo WoRLd", {{"hello world", 1}}); } +TEST_F(LowerCaseFilterTest, ASCIIBypassesUnicodeConversion) { + lower_case_testing::reset_unicode_path_count(); + assert_filter_output("already lowercase", {{"already lowercase", 1}}); + assert_filter_output("ASCII UPPER", {{"ascii upper", 1}}); + EXPECT_EQ(lower_case_testing::unicode_path_count(), 0); + + assert_filter_output( + "\xC3\x9C" + "BER", + {{"\xC3\xBC" + "ber", + 1}}); + EXPECT_EQ(lower_case_testing::unicode_path_count(), 1); +} + +TEST_F(LowerCaseFilterTest, ASCIIPreservesMetadataAndEmbeddedNul) { + auto input = std::make_shared( + std::vector {std::string("A\0B", 3), "already lower"}); + LowerCaseFilterFactory factory; + factory.initialize({}); + auto filter = factory.create(input); + filter->reset(); + + Token token; + ASSERT_NE(filter->next(&token), nullptr); + EXPECT_EQ(std::string(token.termBuffer(), token.termLength()), + std::string("a\0b", 3)); + EXPECT_EQ(token.getPositionIncrement(), 3); + EXPECT_EQ(token.startOffset(), 7); + EXPECT_EQ(token.endOffset(), 19); + EXPECT_EQ(std::wstring(token.type()), L"scripted"); + + ASSERT_NE(filter->next(&token), nullptr); + EXPECT_EQ(std::string(token.termBuffer(), token.termLength()), "already lower"); + EXPECT_EQ(token.getPositionIncrement(), 3); + EXPECT_EQ(token.startOffset(), 7); + EXPECT_EQ(token.endOffset(), 19); + EXPECT_EQ(std::wstring(token.type()), L"scripted"); +} + TEST_F(LowerCaseFilterTest, ConvertsUnicodeCharacters) { assert_filter_output("ÜBER ΜΈΓΑ", {{"über μέγα", 1}}); } +TEST_F(LowerCaseFilterTest, RetriesUnicodeExpansionWithRequiredBufferSize) { + assert_filter_output("\xC4\xB0", {{"i\xCC\x87", 1}}); +} + +TEST_F(LowerCaseFilterTest, RejectsInvalidUtf8WithAnalyzerError) { + auto input = std::make_shared( + std::vector {"VALID", std::string(1, static_cast(0xFF))}); + LowerCaseFilterFactory factory; + factory.initialize({}); + auto filter = factory.create(input); + filter->reset(); + + Token token; + ASSERT_NE(filter->next(&token), nullptr); + EXPECT_EQ(std::string(token.termBuffer(), token.termLength()), "valid"); + try { + filter->next(&token); + FAIL() << "expected malformed UTF-8 to fail analysis"; + } catch (const Exception& error) { + EXPECT_EQ(error.code(), ErrorCode::INVERTED_INDEX_ANALYZER_ERROR); + } +} + TEST_F(LowerCaseFilterTest, HandlesNumbersAndSymbols) { assert_filter_output("123!@# ABC", {{"123!@# abc", 1}}); } diff --git a/be/test/storage/index/inverted/tokenizer/char_group_tokenizer_factory_test.cpp b/be/test/storage/index/inverted/tokenizer/char_group_tokenizer_factory_test.cpp index 986074b052ce7e..a124c1793eb2ba 100644 --- a/be/test/storage/index/inverted/tokenizer/char_group_tokenizer_factory_test.cpp +++ b/be/test/storage/index/inverted/tokenizer/char_group_tokenizer_factory_test.cpp @@ -23,6 +23,11 @@ namespace doris::segment_v2::inverted_index { +namespace char_tokenizer_testing { +uint64_t non_ascii_decode_count(); +void reset_non_ascii_decode_count(); +} // namespace char_tokenizer_testing + class CharGroupTokenizerTest : public ::testing::Test { protected: std::vector tokenize(CharGroupTokenizerFactory& factory, const std::string& text) { @@ -65,6 +70,22 @@ TEST_F(CharGroupTokenizerTest, TokenizeOnSpace) { ASSERT_EQ(tokens, expected); } +TEST_F(CharGroupTokenizerTest, ASCIIUsesPrecomputedClassification) { + CharGroupTokenizerFactory factory; + Settings settings; + settings.set("tokenize_on_chars", "[whitespace], [punctuation]"); + factory.initialize(settings); + + char_tokenizer_testing::reset_non_ascii_decode_count(); + EXPECT_EQ(tokenize(factory, "Hello, ASCII world!"), + (std::vector {"Hello", "ASCII", "world"})); + EXPECT_EQ(char_tokenizer_testing::non_ascii_decode_count(), 0); + + EXPECT_EQ(tokenize(factory, "Hello \xE4\xB8\x96\xE7\x95\x8C"), + (std::vector {"Hello", "\xE4\xB8\x96\xE7\x95\x8C"})); + EXPECT_EQ(char_tokenizer_testing::non_ascii_decode_count(), 2); +} + TEST_F(CharGroupTokenizerTest, TokenizeOnLetter) { CharGroupTokenizerFactory factory; Settings settings; diff --git a/be/test/storage/index/inverted_index_parser_test.cpp b/be/test/storage/index/inverted_index_parser_test.cpp index 719f8fb170f21e..00174207fec417 100644 --- a/be/test/storage/index/inverted_index_parser_test.cpp +++ b/be/test/storage/index/inverted_index_parser_test.cpp @@ -418,6 +418,23 @@ TEST_F(InvertedIndexParserTest, BuildAnalyzerKeyFromProperties_CustomOverridesPa EXPECT_EQ(build_analyzer_key_from_properties(properties), "my_custom"); } +TEST_F(InvertedIndexParserTest, BuildAnalyzerKeyFromProperties_ParserOverridesNormalizer) { + std::map properties; + properties[INVERTED_INDEX_NORMALIZER_NAME_KEY] = "MY_NORMALIZER"; + properties[INVERTED_INDEX_PARSER_KEY] = "chinese"; + + EXPECT_EQ(build_analyzer_key_from_properties(properties), "chinese"); +} + +TEST_F(InvertedIndexParserTest, BuildAnalyzerKeyFromProperties_AnalyzerOverridesNormalizer) { + std::map properties; + properties[INVERTED_INDEX_ANALYZER_NAME_KEY] = "MY_ANALYZER"; + properties[INVERTED_INDEX_NORMALIZER_NAME_KEY] = "MY_NORMALIZER"; + properties[INVERTED_INDEX_PARSER_KEY] = "chinese"; + + EXPECT_EQ(build_analyzer_key_from_properties(properties), "my_analyzer"); +} + // ============================================================================ // AnalyzerConfigParser Tests // ============================================================================ diff --git a/be/test/storage/index/snii/bench/snii_vs_v3_benchmark_test.cpp b/be/test/storage/index/snii/bench/snii_vs_v3_benchmark_test.cpp new file mode 100644 index 00000000000000..586e39e0e85d09 --- /dev/null +++ b/be/test/storage/index/snii/bench/snii_vs_v3_benchmark_test.cpp @@ -0,0 +1,1996 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// +// SNII vs V3 baseline over the same wikipedia corpus, covering the three phases that matter for the +// format: building the index while loading, compacting it, and querying it across MATCH_ANY, +// MATCH_ALL, MATCH_PHRASE and MATCH_PHRASE_PREFIX. +// +// Why this lives in a unit test rather than a cluster benchmark: a UT iterates in minutes instead +// of a deploy cycle, and it measures one process we control end to end. +// +// Why CPU time is the headline number: this machine is shared, so wall clock moves with whatever +// else is running. Process CPU time barely does. Wall time is still reported -- a large wall/CPU +// gap means the run was IO bound or descheduled and the comparison should be rerun -- but the +// SNII/V3 verdict is taken from CPU. +// +// Corpus is not committed (~41 MB). Point SNII_BENCH_CORPUS_DIR at a directory of wikipedia_*.json +// with {"title","content"} per line: +// +// SNII_BENCH_CORPUS_DIR=/path/to/corpus \ +// SNII_BENCH_QUERY_ITERATIONS=30 \ +// ./run-be-ut.sh --run --filter='*SniiVsV3Benchmark*' -j +// +// SNII_BENCH_QUERY_ITERATIONS defaults to 30. The benchmark reports nearest-rank +// p50/p99 query CPU and wall time from the sorted per-iteration samples. +// +// SNII_BENCH_QUERY_INPUT_ROWSETS=1 points the query phase at the input rowsets instead of the +// compacted output. With one corpus file per rowset that is the shape of a tablet still ingesting: +// N indexes open at once and N rounds of remote reads per cold query, rather than one merged index. +// It also skips compaction, whose only purpose here is to produce the rowset the query reads -- +// compaction is benchmarked by the default compacted-output mode, where it is on the measured path. +// +// SNII_BENCH_ONLY_CASE=