feat(search): use stable node IDs for split affinity to keep disk cache warm when searcher pods restart

docs/configuration/node-config.md

@@ -21,7 +21,7 @@ A commented example is available here: [quickwit.yaml](https://github.com/quickw
 | --- | --- | --- | --- |
 | `version` | Config file version. `0.7` is the only available value with a retro compatibility on `0.5` and `0.4`. | | |
 | `cluster_id` | Unique identifier of the cluster the node will be joining. Clusters sharing the same network should use distinct cluster IDs.| `QW_CLUSTER_ID` | `quickwit-default-cluster` |
-| `node_id` | Unique identifier of the node. It must be distinct from the node IDs of its cluster peers. Defaults to the instance's short hostname if not set. | `QW_NODE_ID` | short hostname |
+| `node_id` | Unique identifier of the node. It must be distinct from the node IDs of its cluster peers. Searchers hash this ID for split affinity, so keep it stable across restarts (for example a StatefulSet pod name) if the same splits should keep landing on the same node. Defaults to the instance's short hostname if not set. | `QW_NODE_ID` | short hostname |
 | `enabled_services` | Enabled services (control_plane, indexer, janitor, metastore, metastore_read_replica, searcher) | `QW_ENABLED_SERVICES` | all services except metastore_read_replica |
 | `listen_address` | The IP address or hostname that Quickwit service binds to for starting REST and GRPC server and connecting this node to other nodes. By default, Quickwit binds itself to 127.0.0.1 (localhost). This default is not valid when trying to form a cluster. | `QW_LISTEN_ADDRESS` | `127.0.0.1` |
 | `advertise_address` | IP address advertised by the node, i.e. the IP address that peer nodes should use to connect to the node for RPCs. | `QW_ADVERTISE_ADDRESS` | `listen_address` |

quickwit/quickwit-common/src/lib.rs

@@ -52,24 +52,20 @@ pub mod type_map;
 pub mod uri;
 
 mod metrics_specific;
-pub use env::{
-    get_bool_from_env, get_bool_from_env_opt, get_duration_from_env, get_from_env,
-    get_from_env_opt, parse_bool_lenient,
-};
-pub use metrics_specific::*;
-
-mod socket_addr_legacy_hash;
-
 use std::fmt::Display;
 use std::future::Future;
 use std::ops::{Range, RangeInclusive};
 
 pub use coolid::new_coolid;
 pub use cpus::num_cpus;
+pub use env::{
+    get_bool_from_env, get_bool_from_env_opt, get_duration_from_env, get_from_env,
+    get_from_env_opt, parse_bool_lenient,
+};
 pub use kill_switch::KillSwitch;
+pub use metrics_specific::*;
 pub use path_hasher::PathHasher;
 pub use progress::{Progress, ProtectedZoneGuard};
-pub use socket_addr_legacy_hash::SocketAddrLegacyHash;
 pub use stream_utils::{BoxStream, ServiceStream};
 
 /// Returns true at compile time. This function is mostly used with serde to initialize boolean

quickwit/quickwit-common/src/rendezvous_hasher.rs

@@ -35,54 +35,29 @@ pub fn sort_by_rendez_vous_hash<T: Hash, U: Hash>(nodes: &mut [T], key: U) {
 
 #[cfg(test)]
 mod tests {
-    use std::net::SocketAddr;
-
     use super::*;
-    use crate::SocketAddrLegacyHash;
-
-    fn test_socket_addr(last_byte: u8) -> SocketAddr {
-        ([127, 0, 0, last_byte], 10_000u16).into()
-    }
 
     #[test]
     fn test_utils_sort_by_rendez_vous_hash() {
-        let socket1 = test_socket_addr(1);
-        let socket2 = test_socket_addr(2);
-        let socket3 = test_socket_addr(3);
-        let socket4 = test_socket_addr(4);
-
-        let legacy_socket1 = SocketAddrLegacyHash(&socket1);
-        let legacy_socket2 = SocketAddrLegacyHash(&socket2);
-        let legacy_socket3 = SocketAddrLegacyHash(&socket3);
-        let legacy_socket4 = SocketAddrLegacyHash(&socket4);
-
-        let mut socket_set1 = vec![
-            legacy_socket4,
-            legacy_socket3,
-            legacy_socket1,
-            legacy_socket2,
-        ];
-        sort_by_rendez_vous_hash(&mut socket_set1, "key");
-
-        let mut socket_set2 = vec![legacy_socket1, legacy_socket2, legacy_socket4];
-        sort_by_rendez_vous_hash(&mut socket_set2, "key");
-
-        let mut socket_set3 = vec![legacy_socket1, legacy_socket4];
-        sort_by_rendez_vous_hash(&mut socket_set3, "key");
-
-        assert_eq!(
-            socket_set1,
-            &[
-                legacy_socket1,
-                legacy_socket2,
-                legacy_socket3,
-                legacy_socket4
-            ]
-        );
-        assert_eq!(
-            socket_set2,
-            &[legacy_socket1, legacy_socket2, legacy_socket4]
-        );
-        assert_eq!(socket_set3, &[legacy_socket1, legacy_socket4]);
+        let mut nodes_four = vec!["node-4", "node-3", "node-1", "node-2"];
+        sort_by_rendez_vous_hash(&mut nodes_four, "key");
+
+        let mut nodes_three = vec!["node-1", "node-2", "node-4"];
+        sort_by_rendez_vous_hash(&mut nodes_three, "key");
+        let expected_three: Vec<&str> = nodes_four
+            .iter()
+            .copied()
+            .filter(|node| *node != "node-3")
+            .collect();
+        assert_eq!(nodes_three, expected_three);
+
+        let mut nodes_two = vec!["node-1", "node-4"];
+        sort_by_rendez_vous_hash(&mut nodes_two, "key");
+        let expected_two: Vec<&str> = nodes_four
+            .iter()
+            .copied()
+            .filter(|node| *node == "node-1" || *node == "node-4")
+            .collect();
+        assert_eq!(nodes_two, expected_two);
     }
 }

quickwit/quickwit-datafusion/tests/distributed.rs

@@ -34,7 +34,8 @@ use quickwit_datafusion::test_utils::make_batch;
 use quickwit_datafusion::{
     DataFusionSessionBuilder, QuickwitObjectStoreRegistry, QuickwitWorkerResolver,
 };
-use quickwit_search::{SearcherPool, create_search_client_from_grpc_addr};
+use quickwit_proto::types::NodeId;
+use quickwit_search::{SearcherNode, SearcherPool, create_search_client_from_grpc_addr};
 
 mod common;
 mod metrics_splits;
@@ -104,12 +105,15 @@ async fn test_distributed_tasks_not_shuffles() {
     let worker_b = spawn_df_worker(make_builder()).await;
 
     // Populate the pool with the real worker gRPC addresses.
-    // Pool value is a SearchServiceClient — only the key (addr) matters for
+    // Pool value is a SearcherNode — only the key (addr) matters for
     // QuickwitWorkerResolver, which calls pool.keys() to get URLs.
     for addr in [worker_a.addr, worker_b.addr] {
         pool.insert(
             addr,
-            create_search_client_from_grpc_addr(addr, bytesize::ByteSize::mib(20)),
+            SearcherNode {
+                node_id: NodeId::from_str(&format!("worker-{}", addr.port())),
+                client: create_search_client_from_grpc_addr(addr, bytesize::ByteSize::mib(20)),
+            },
         );
     }