From eb76e965db955c921eeae63dc34c0278633060ff Mon Sep 17 00:00:00 2001 From: Pascal Seitz Date: Mon, 31 Aug 2026 17:29:59 +0800 Subject: [PATCH 1/4] Lower default field list size limit to 10k --- quickwit/quickwit-search/src/list_fields/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/quickwit/quickwit-search/src/list_fields/mod.rs b/quickwit/quickwit-search/src/list_fields/mod.rs index b62ab5a850a..0398e5ea3f8 100644 --- a/quickwit/quickwit-search/src/list_fields/mod.rs +++ b/quickwit/quickwit-search/src/list_fields/mod.rs @@ -36,7 +36,7 @@ pub use crate::list_fields::root::root_list_fields; /// a JSON type with random field names. Retaining the most common fields bounds /// response memory while pruning the long tail of rare fields. fn field_list_size_limit() -> usize { - quickwit_common::get_from_env_cached!(usize, "QW_FIELD_LIST_SIZE_LIMIT", 100_000, false) + quickwit_common::get_from_env_cached!(usize, "QW_FIELD_LIST_SIZE_LIMIT", 10_000, false) } // Sorts and deduplicates the list of fields. From 0b9de91587d8a9918db612a2798bd5e6e774484c Mon Sep 17 00:00:00 2001 From: Pascal Seitz Date: Mon, 31 Aug 2026 17:35:45 +0800 Subject: [PATCH 2/4] Document field list limit rationale --- quickwit/quickwit-search/src/list_fields/mod.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/quickwit/quickwit-search/src/list_fields/mod.rs b/quickwit/quickwit-search/src/list_fields/mod.rs index 0398e5ea3f8..46ca31e324b 100644 --- a/quickwit/quickwit-search/src/list_fields/mod.rs +++ b/quickwit/quickwit-search/src/list_fields/mod.rs @@ -34,7 +34,8 @@ pub use crate::list_fields::root::root_list_fields; /// /// Having many fields can happen when a user is creating fields dynamically in /// a JSON type with random field names. Retaining the most common fields bounds -/// response memory while pruning the long tail of rare fields. +/// response memory while pruning the long tail of rare fields. The default is +/// 10,000 because responses with 100,000 fields may exceed gRPC message size limits. fn field_list_size_limit() -> usize { quickwit_common::get_from_env_cached!(usize, "QW_FIELD_LIST_SIZE_LIMIT", 10_000, false) } From b7db385dd163c3d3e6dd14b3f83d0876970666a7 Mon Sep 17 00:00:00 2001 From: Pascal Seitz Date: Mon, 31 Aug 2026 18:41:40 +0800 Subject: [PATCH 3/4] Add per-request field list limit Allow field capabilities and mappings requests to override the default field limit, and propagate the limit through root and leaf merges. --- docs/reference/es_compatible_api.md | 3 +++ .../protos/quickwit/search.proto | 8 +++++- .../src/codegen/quickwit/quickwit.search.rs | 6 +++++ .../quickwit-search/src/list_fields/leaf.rs | 10 ++++--- .../quickwit-search/src/list_fields/mod.rs | 27 ++++++++++++++----- .../quickwit-search/src/list_fields/root.rs | 8 +++--- quickwit/quickwit-search/src/service.rs | 1 + .../model/field_capability.rs | 6 +++++ .../model/index_mapping_query_params.rs | 10 +++++++ .../src/elasticsearch_api/rest_handler.rs | 1 + 10 files changed, 67 insertions(+), 13 deletions(-) diff --git a/docs/reference/es_compatible_api.md b/docs/reference/es_compatible_api.md index b1a0ba3702b..041368154a1 100644 --- a/docs/reference/es_compatible_api.md +++ b/docs/reference/es_compatible_api.md @@ -391,9 +391,12 @@ The [field capabilities API](https://www.elastic.co/guide/en/elasticsearch/refer | `allow_no_indices` | `Boolean` | If `true`, missing or closed indices are not an error. | (Optional) | | `expand_wildcards` | `String` | Controls what kind of indices that wildcard patterns can match. | (Optional) | | `ignore_unavailable` | `Boolean` | If `true`, unavailable indices are ignored. | (Optional) | +| `limit` | `Integer` | *(Quickwit-specific)* Maximum number of fields to return. Overrides `QW_FIELD_LIST_SIZE_LIMIT` for this request. | (Optional) | | `start_timestamp` | `Integer` | *(Quickwit-specific)* If set, restricts splits to documents with a timestamp range start >= `start_timestamp` (seconds since epoch). | (Optional) | | `end_timestamp` | `Integer` | *(Quickwit-specific)* If set, restricts splits to documents with a timestamp range end < `end_timestamp` (seconds since epoch). | (Optional) | +The Quickwit-specific `limit` parameter is also supported by the `_mapping` and `_mappings` APIs. + #### Supported Request Body parameters | Variable | Type | Description | Default value | diff --git a/quickwit/quickwit-proto/protos/quickwit/search.proto b/quickwit/quickwit-proto/protos/quickwit/search.proto index d266f889508..a8a9e83ed80 100644 --- a/quickwit/quickwit-proto/protos/quickwit/search.proto +++ b/quickwit/quickwit-proto/protos/quickwit/search.proto @@ -141,8 +141,11 @@ message ListFieldsRequest { // When provided, only fields from documents matching this query are returned. optional string query_ast = 5; + // Maximum number of fields to return. Overrides QW_FIELD_LIST_SIZE_LIMIT. + optional uint32 limit = 6; + // Control if the request will fail if split_ids contains a split that does not exist. - // optional bool fail_on_missing_index = 6; + // optional bool fail_on_missing_index = 7; } message LeafListFieldsRequest { @@ -157,6 +160,9 @@ message LeafListFieldsRequest { // Optional limit query to a list of fields // Wildcard expressions are supported. repeated string field_patterns = 4; + + // Maximum number of fields to return. Overrides QW_FIELD_LIST_SIZE_LIMIT. + optional uint32 limit = 5; } /// Message returned by leaf and root list fields requests. diff --git a/quickwit/quickwit-proto/src/codegen/quickwit/quickwit.search.rs b/quickwit/quickwit-proto/src/codegen/quickwit/quickwit.search.rs index 347892fb124..ad0c1099cf7 100644 --- a/quickwit/quickwit-proto/src/codegen/quickwit/quickwit.search.rs +++ b/quickwit/quickwit-proto/src/codegen/quickwit/quickwit.search.rs @@ -86,6 +86,9 @@ pub struct ListFieldsRequest { /// When provided, only fields from documents matching this query are returned. #[prost(string, optional, tag = "5")] pub query_ast: ::core::option::Option<::prost::alloc::string::String>, + /// Maximum number of fields to return. Overrides QW_FIELD_LIST_SIZE_LIMIT. + #[prost(uint32, optional, tag = "6")] + pub limit: ::core::option::Option, } #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -104,6 +107,9 @@ pub struct LeafListFieldsRequest { /// Wildcard expressions are supported. #[prost(string, repeated, tag = "4")] pub field_patterns: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, + /// Maximum number of fields to return. Overrides QW_FIELD_LIST_SIZE_LIMIT. + #[prost(uint32, optional, tag = "5")] + pub limit: ::core::option::Option, } /// / Message returned by leaf and root list fields requests. #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] diff --git a/quickwit/quickwit-search/src/list_fields/leaf.rs b/quickwit/quickwit-search/src/list_fields/leaf.rs index cd416037b14..646b86966c5 100644 --- a/quickwit/quickwit-search/src/list_fields/leaf.rs +++ b/quickwit/quickwit-search/src/list_fields/leaf.rs @@ -27,7 +27,7 @@ use tracing::{Span, instrument}; use crate::leaf::open_split_bundle; use crate::list_fields::patterns::FieldPatterns; -use crate::list_fields::{merge_entries, sort_and_dedup}; +use crate::list_fields::{merge_entries_with_limit_override, sort_and_dedup}; use crate::search_thread_pool; use crate::service::SearcherContext; @@ -45,6 +45,7 @@ pub async fn leaf_list_fields( index_id: IndexId, field_patterns_strs: &[String], split_footers: Vec, + limit: Option, searcher_ctx: Arc, storage: Arc, ) -> crate::Result { @@ -62,7 +63,7 @@ pub async fn leaf_list_fields( ) .await?; - let merged_entries: Vec = merge_fields_metadata(all_entries).await?; + let merged_entries: Vec = merge_fields_metadata(all_entries, limit).await?; let response = ListFieldsResponse { entries: merged_entries, @@ -229,10 +230,13 @@ fn filter_fields_metadata( #[instrument(skip_all, fields(num_splits = all_entries.len()))] async fn merge_fields_metadata( all_entries: Vec>, + limit: Option, ) -> crate::Result> { let parent_span = Span::current(); search_thread_pool() - .run_cpu_intensive(move || parent_span.in_scope(|| merge_entries(all_entries))) + .run_cpu_intensive(move || { + parent_span.in_scope(|| merge_entries_with_limit_override(all_entries, limit)) + }) .await .context("failed to merge single split list fields")? } diff --git a/quickwit/quickwit-search/src/list_fields/mod.rs b/quickwit/quickwit-search/src/list_fields/mod.rs index 46ca31e324b..eb8b5d4bb8a 100644 --- a/quickwit/quickwit-search/src/list_fields/mod.rs +++ b/quickwit/quickwit-search/src/list_fields/mod.rs @@ -28,16 +28,18 @@ use tracing::instrument; pub use crate::list_fields::leaf::leaf_list_fields; pub use crate::list_fields::root::root_list_fields; -/// QW_FIELD_LIST_SIZE_LIMIT defines a hard limit on the number of fields that -/// can be returned. When the limit is exceeded, the fields present in the most -/// splits are retained. +/// QW_FIELD_LIST_SIZE_LIMIT defines the default limit on the number of fields +/// that can be returned. A request-specific limit takes precedence. When the +/// limit is exceeded, the fields present in the most splits are retained. /// /// Having many fields can happen when a user is creating fields dynamically in /// a JSON type with random field names. Retaining the most common fields bounds /// response memory while pruning the long tail of rare fields. The default is /// 10,000 because responses with 100,000 fields may exceed gRPC message size limits. -fn field_list_size_limit() -> usize { - quickwit_common::get_from_env_cached!(usize, "QW_FIELD_LIST_SIZE_LIMIT", 10_000, false) +fn field_list_size_limit(limit: Option) -> usize { + limit.map(|limit| limit as usize).unwrap_or_else(|| { + quickwit_common::get_from_env_cached!(usize, "QW_FIELD_LIST_SIZE_LIMIT", 10_000, false) + }) } // Sorts and deduplicates the list of fields. @@ -64,10 +66,18 @@ fn sort_and_dedup(entries: &mut Vec) { }); } +#[cfg(test)] fn merge_entries(entry_groups: Vec>) -> crate::Result> { + merge_entries_with_limit_override(entry_groups, None) +} + +fn merge_entries_with_limit_override( + entry_groups: Vec>, + limit: Option, +) -> crate::Result> { Ok(merge_entries_with_limit( entry_groups, - field_list_size_limit(), + field_list_size_limit(limit), )) } @@ -205,6 +215,11 @@ mod tests { use super::*; + #[test] + fn request_limit_overrides_configured_limit() { + assert_eq!(field_list_size_limit(Some(123)), 123); + } + #[test] fn merge_leaf_list_fields_identical_test() { let entry1 = ListFieldsEntry { diff --git a/quickwit/quickwit-search/src/list_fields/root.rs b/quickwit/quickwit-search/src/list_fields/root.rs index 6e0382f9874..f2bb73ffaee 100644 --- a/quickwit/quickwit-search/src/list_fields/root.rs +++ b/quickwit/quickwit-search/src/list_fields/root.rs @@ -30,7 +30,7 @@ use quickwit_proto::types::{IndexId, IndexUid}; use quickwit_query::query_ast::QueryAst; use tracing::{Span, instrument}; -use crate::list_fields::{merge_entries, sort_and_dedup}; +use crate::list_fields::{merge_entries_with_limit_override, sort_and_dedup}; use crate::search_job_placer::group_jobs_by_index_id; use crate::{ ClusterClient, SearchError, SearchJob, list_relevant_splits, resolve_index_patterns, @@ -141,7 +141,7 @@ pub async fn root_list_fields( .into_iter() .map(|response| response.entries) .collect(); - let merged_entries = merge_fields_metadata(leaf_entries).await?; + let merged_entries = merge_fields_metadata(leaf_entries, list_fields_req.limit).await?; let response = ListFieldsResponse { entries: merged_entries, }; @@ -170,6 +170,7 @@ fn jobs_to_leaf_requests( index_uri: index_meta.index_uri.to_string(), field_patterns: search_request_for_leaf.field_patterns.clone(), split_offsets: job_group.into_iter().map(|job| job.offsets).collect(), + limit: search_request_for_leaf.limit, }; leaf_search_requests.push(leaf_search_request); Ok(()) @@ -181,6 +182,7 @@ fn jobs_to_leaf_requests( #[instrument(skip_all, fields(num_leaves = entry_groups.len()))] async fn merge_fields_metadata( mut entry_groups: Vec>, + limit: Option, ) -> crate::Result> { let parent_span = Span::current(); search_thread_pool() @@ -200,7 +202,7 @@ async fn merge_fields_metadata( sort_and_dedup(entry_group); } } - merge_entries(entry_groups) + merge_entries_with_limit_override(entry_groups, limit) }) }) .await diff --git a/quickwit/quickwit-search/src/service.rs b/quickwit/quickwit-search/src/service.rs index 52bfc846696..7281eed8176 100644 --- a/quickwit/quickwit-search/src/service.rs +++ b/quickwit/quickwit-search/src/service.rs @@ -301,6 +301,7 @@ impl SearchService for SearchServiceImpl { index_id, &list_fields_req.field_patterns, split_ids, + list_fields_req.limit, self.searcher_context.clone(), storage, ) diff --git a/quickwit/quickwit-serve/src/elasticsearch_api/model/field_capability.rs b/quickwit/quickwit-serve/src/elasticsearch_api/model/field_capability.rs index 292f7202644..87a93aaeecc 100644 --- a/quickwit/quickwit-serve/src/elasticsearch_api/model/field_capability.rs +++ b/quickwit/quickwit-serve/src/elasticsearch_api/model/field_capability.rs @@ -42,6 +42,9 @@ pub struct FieldCapabilityQueryParams { pub fields: Option>, #[serde(default)] pub ignore_unavailable: Option, + /// Non-ES parameter. Overrides `QW_FIELD_LIST_SIZE_LIMIT` for this request. + #[serde(default)] + pub limit: Option, /// Non-ES Parameter. If set, restricts splits to documents with a `time_range.start >= /// start_timestamp`. pub start_timestamp: Option, @@ -229,6 +232,7 @@ pub fn build_list_field_request_for_es_api( start_timestamp: search_params.start_timestamp, end_timestamp: search_params.end_timestamp, query_ast: query_ast_json, + limit: search_params.limit, }) } @@ -349,6 +353,7 @@ mod tests { fields: Some(vec!["field1".to_string(), "field2".to_string()]), start_timestamp: Some(1000), end_timestamp: Some(2000), + limit: Some(123), ..Default::default() }; @@ -370,6 +375,7 @@ mod tests { ); assert_eq!(result.start_timestamp, Some(1000)); assert_eq!(result.end_timestamp, Some(2000)); + assert_eq!(result.limit, Some(123)); assert!(result.query_ast.is_some()); } diff --git a/quickwit/quickwit-serve/src/elasticsearch_api/model/index_mapping_query_params.rs b/quickwit/quickwit-serve/src/elasticsearch_api/model/index_mapping_query_params.rs index 030eb81613c..9c47e02029b 100644 --- a/quickwit/quickwit-serve/src/elasticsearch_api/model/index_mapping_query_params.rs +++ b/quickwit/quickwit-serve/src/elasticsearch_api/model/index_mapping_query_params.rs @@ -28,6 +28,9 @@ pub struct IndexMappingQueryParams { pub start_timestamp: Option, #[serde(default)] pub end_timestamp: Option, + /// Overrides `QW_FIELD_LIST_SIZE_LIMIT` for this request. + #[serde(default)] + pub limit: Option, /// Accepts both `field_patterns` (Quickwit) and `fields` (ES-compatible). #[serde(default, alias = "fields", deserialize_with = "empty_string_as_none")] pub field_patterns: Option, @@ -70,9 +73,16 @@ mod tests { let params: IndexMappingQueryParams = serde_qs::from_str("").unwrap(); assert!(params.start_timestamp.is_none()); assert!(params.end_timestamp.is_none()); + assert!(params.limit.is_none()); assert!(params.field_patterns.is_none()); } + #[test] + fn limit_param_present() { + let params: IndexMappingQueryParams = serde_qs::from_str("limit=123").unwrap(); + assert_eq!(params.limit, Some(123)); + } + #[test] fn both_params_present_yield_some() { let qs = "start_timestamp=1712160204&end_timestamp=1712764984"; diff --git a/quickwit/quickwit-serve/src/elasticsearch_api/rest_handler.rs b/quickwit/quickwit-serve/src/elasticsearch_api/rest_handler.rs index cec582d934d..6dadc76c710 100644 --- a/quickwit/quickwit-serve/src/elasticsearch_api/rest_handler.rs +++ b/quickwit/quickwit-serve/src/elasticsearch_api/rest_handler.rs @@ -224,6 +224,7 @@ pub(crate) async fn es_compat_index_mapping( start_timestamp: params.start_timestamp, end_timestamp: params.end_timestamp, query_ast: None, + limit: params.limit, }; let list_fields_response = match search_service.root_list_fields(list_fields_request).await { Ok(response) => Some(response), From 91a5b52c1551a97f3365e14cb74d612d01b28c60 Mon Sep 17 00:00:00 2001 From: Pascal Seitz Date: Tue, 1 Sep 2026 12:37:02 +0800 Subject: [PATCH 4/4] Restrict field limit override to protobuf requests --- docs/reference/es_compatible_api.md | 3 --- quickwit/quickwit-proto/protos/quickwit/search.proto | 3 +-- .../src/elasticsearch_api/model/field_capability.rs | 7 +------ .../model/index_mapping_query_params.rs | 10 ---------- .../src/elasticsearch_api/rest_handler.rs | 2 +- 5 files changed, 3 insertions(+), 22 deletions(-) diff --git a/docs/reference/es_compatible_api.md b/docs/reference/es_compatible_api.md index 041368154a1..b1a0ba3702b 100644 --- a/docs/reference/es_compatible_api.md +++ b/docs/reference/es_compatible_api.md @@ -391,12 +391,9 @@ The [field capabilities API](https://www.elastic.co/guide/en/elasticsearch/refer | `allow_no_indices` | `Boolean` | If `true`, missing or closed indices are not an error. | (Optional) | | `expand_wildcards` | `String` | Controls what kind of indices that wildcard patterns can match. | (Optional) | | `ignore_unavailable` | `Boolean` | If `true`, unavailable indices are ignored. | (Optional) | -| `limit` | `Integer` | *(Quickwit-specific)* Maximum number of fields to return. Overrides `QW_FIELD_LIST_SIZE_LIMIT` for this request. | (Optional) | | `start_timestamp` | `Integer` | *(Quickwit-specific)* If set, restricts splits to documents with a timestamp range start >= `start_timestamp` (seconds since epoch). | (Optional) | | `end_timestamp` | `Integer` | *(Quickwit-specific)* If set, restricts splits to documents with a timestamp range end < `end_timestamp` (seconds since epoch). | (Optional) | -The Quickwit-specific `limit` parameter is also supported by the `_mapping` and `_mappings` APIs. - #### Supported Request Body parameters | Variable | Type | Description | Default value | diff --git a/quickwit/quickwit-proto/protos/quickwit/search.proto b/quickwit/quickwit-proto/protos/quickwit/search.proto index a8a9e83ed80..a1a10efbe09 100644 --- a/quickwit/quickwit-proto/protos/quickwit/search.proto +++ b/quickwit/quickwit-proto/protos/quickwit/search.proto @@ -144,8 +144,7 @@ message ListFieldsRequest { // Maximum number of fields to return. Overrides QW_FIELD_LIST_SIZE_LIMIT. optional uint32 limit = 6; - // Control if the request will fail if split_ids contains a split that does not exist. - // optional bool fail_on_missing_index = 7; + reserved 7; } message LeafListFieldsRequest { diff --git a/quickwit/quickwit-serve/src/elasticsearch_api/model/field_capability.rs b/quickwit/quickwit-serve/src/elasticsearch_api/model/field_capability.rs index 87a93aaeecc..128fd5983b5 100644 --- a/quickwit/quickwit-serve/src/elasticsearch_api/model/field_capability.rs +++ b/quickwit/quickwit-serve/src/elasticsearch_api/model/field_capability.rs @@ -42,9 +42,6 @@ pub struct FieldCapabilityQueryParams { pub fields: Option>, #[serde(default)] pub ignore_unavailable: Option, - /// Non-ES parameter. Overrides `QW_FIELD_LIST_SIZE_LIMIT` for this request. - #[serde(default)] - pub limit: Option, /// Non-ES Parameter. If set, restricts splits to documents with a `time_range.start >= /// start_timestamp`. pub start_timestamp: Option, @@ -232,7 +229,7 @@ pub fn build_list_field_request_for_es_api( start_timestamp: search_params.start_timestamp, end_timestamp: search_params.end_timestamp, query_ast: query_ast_json, - limit: search_params.limit, + limit: None, }) } @@ -353,7 +350,6 @@ mod tests { fields: Some(vec!["field1".to_string(), "field2".to_string()]), start_timestamp: Some(1000), end_timestamp: Some(2000), - limit: Some(123), ..Default::default() }; @@ -375,7 +371,6 @@ mod tests { ); assert_eq!(result.start_timestamp, Some(1000)); assert_eq!(result.end_timestamp, Some(2000)); - assert_eq!(result.limit, Some(123)); assert!(result.query_ast.is_some()); } diff --git a/quickwit/quickwit-serve/src/elasticsearch_api/model/index_mapping_query_params.rs b/quickwit/quickwit-serve/src/elasticsearch_api/model/index_mapping_query_params.rs index 9c47e02029b..030eb81613c 100644 --- a/quickwit/quickwit-serve/src/elasticsearch_api/model/index_mapping_query_params.rs +++ b/quickwit/quickwit-serve/src/elasticsearch_api/model/index_mapping_query_params.rs @@ -28,9 +28,6 @@ pub struct IndexMappingQueryParams { pub start_timestamp: Option, #[serde(default)] pub end_timestamp: Option, - /// Overrides `QW_FIELD_LIST_SIZE_LIMIT` for this request. - #[serde(default)] - pub limit: Option, /// Accepts both `field_patterns` (Quickwit) and `fields` (ES-compatible). #[serde(default, alias = "fields", deserialize_with = "empty_string_as_none")] pub field_patterns: Option, @@ -73,16 +70,9 @@ mod tests { let params: IndexMappingQueryParams = serde_qs::from_str("").unwrap(); assert!(params.start_timestamp.is_none()); assert!(params.end_timestamp.is_none()); - assert!(params.limit.is_none()); assert!(params.field_patterns.is_none()); } - #[test] - fn limit_param_present() { - let params: IndexMappingQueryParams = serde_qs::from_str("limit=123").unwrap(); - assert_eq!(params.limit, Some(123)); - } - #[test] fn both_params_present_yield_some() { let qs = "start_timestamp=1712160204&end_timestamp=1712764984"; diff --git a/quickwit/quickwit-serve/src/elasticsearch_api/rest_handler.rs b/quickwit/quickwit-serve/src/elasticsearch_api/rest_handler.rs index 6dadc76c710..657668b57b7 100644 --- a/quickwit/quickwit-serve/src/elasticsearch_api/rest_handler.rs +++ b/quickwit/quickwit-serve/src/elasticsearch_api/rest_handler.rs @@ -224,7 +224,7 @@ pub(crate) async fn es_compat_index_mapping( start_timestamp: params.start_timestamp, end_timestamp: params.end_timestamp, query_ast: None, - limit: params.limit, + limit: None, }; let list_fields_response = match search_service.root_list_fields(list_fields_request).await { Ok(response) => Some(response),