From c15718ff18aa5827154387457428235a94271a9c Mon Sep 17 00:00:00 2001 From: Vitalii Parfonov Date: Thu, 13 Aug 2026 15:54:15 +0300 Subject: [PATCH] feat(startup): add --raise-fd-limit CLI flag to raise file descriptor soft limit Backport of ViaQ/vector#277 from v0.54.0-rh. Add an opt-in --raise-fd-limit flag (and VECTOR_RAISE_FD_LIMIT env var) that raises the RLIMIT_NOFILE soft limit to the hard limit at startup, preventing "Too many open files" errors when monitoring many log files. On macOS, falls back to kern.maxfilesperproc if the hard limit is too high. Also fix clippy error int the aws_cloudwatch_logs healthcheck integration test Signed-off-by: Vitalii Parfonov --- Cargo.toml | 3 +- changelog.d/raise_fd_limit_cli_flag.fix.md | 7 + src/app.rs | 5 + src/cli.rs | 179 ++++++++++++++++++ .../aws_cloudwatch_logs/integration_tests.rs | 2 +- 5 files changed, 194 insertions(+), 2 deletions(-) create mode 100644 changelog.d/raise_fd_limit_cli_flag.fix.md diff --git a/Cargo.toml b/Cargo.toml index ec45b1c6d2666..ba5c68ed4bb36 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -411,7 +411,8 @@ byteorder = "1.5.0" windows-service = "0.7.0" [target.'cfg(unix)'.dependencies] -nix = { version = "0.26.2", default-features = false, features = ["socket", "signal"] } +libc = "0.2.172" +nix = { version = "0.26.2", default-features = false, features = ["socket", "signal", "resource"] } [target.'cfg(target_os = "linux")'.dependencies] netlink-packet-utils = "0.5.2" diff --git a/changelog.d/raise_fd_limit_cli_flag.fix.md b/changelog.d/raise_fd_limit_cli_flag.fix.md new file mode 100644 index 0000000000000..10b8c2c306856 --- /dev/null +++ b/changelog.d/raise_fd_limit_cli_flag.fix.md @@ -0,0 +1,7 @@ +A new `--raise-fd-limit` CLI flag (or `VECTOR_RAISE_FD_LIMIT` environment variable) +raises the file descriptor soft limit to the hard limit at startup. This prevents +"Too many open files" errors when Vector monitors large numbers of log files. On +macOS, Vector falls back to the kernel-enforced per-process file limit if the hard +limit is too high. + +authors: vparfonov diff --git a/src/app.rs b/src/app.rs index 04023ebba4bab..e64dc6854df8e 100644 --- a/src/app.rs +++ b/src/app.rs @@ -204,6 +204,11 @@ impl Application { opts.root.internal_log_rate_limit, ); + #[cfg(unix)] + if opts.root.raise_fd_limit { + crate::cli::raise_file_descriptor_limit(); + } + // Can only log this after initializing the logging subsystem if opts.root.openssl_no_probe { debug!(message = "Disabled probing and configuration of root certificate locations on the system for OpenSSL."); diff --git a/src/cli.rs b/src/cli.rs index 279560154585c..931b8262d9d1c 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -236,6 +236,15 @@ pub struct RootOpts { /// `--watch-config`. #[arg(long, env = "VECTOR_ALLOW_EMPTY_CONFIG", default_value = "false")] pub allow_empty_config: bool, + + /// Raise the file descriptor soft limit (RLIMIT_NOFILE) to the hard limit at startup. + /// + /// Many systems default the soft limit to 1024 (Linux) or 256 (macOS), which is too low + /// when Vector monitors large numbers of log files. This flag raises the soft limit to + /// prevent "Too many open files" errors without requiring manual sysadmin intervention. + #[cfg(unix)] + #[arg(long, env = "VECTOR_RAISE_FD_LIMIT", default_value = "false")] + pub raise_fd_limit: bool, } impl RootOpts { @@ -267,6 +276,88 @@ impl RootOpts { } } +/// Raise the soft file descriptor limit (RLIMIT_NOFILE) as high as the OS allows. +/// +/// Many systems default the soft limit to 1024 (Linux) or 256 (macOS), which is too low +/// for Vector when it monitors large numbers of log files. Raising it prevents +/// "Too many open files (os error 24)" errors without requiring manual sysadmin intervention. +/// +/// On Linux, the soft limit is raised to the hard limit (typically 65536+). +/// On macOS, the hard limit can be RLIM_INFINITY, so we first try the hard limit, +/// then fall back to the kernel-enforced `kern.maxfilesperproc` (typically 10240). +#[cfg(unix)] +pub(crate) fn raise_file_descriptor_limit() { + use nix::sys::resource::{getrlimit, setrlimit, Resource}; + use tracing::{info, warn}; + + let (soft, hard) = match getrlimit(Resource::RLIMIT_NOFILE) { + Ok(limits) => limits, + Err(err) => { + warn!(message = "Failed to get file descriptor limit.", %err); + return; + } + }; + + if soft >= hard { + return; // Already at maximum + } + + // Try setting soft limit to hard limit (works on Linux, may fail on macOS) + if setrlimit(Resource::RLIMIT_NOFILE, hard, hard).is_ok() { + info!( + message = "Raised file descriptor limit.", + from = soft, + to = hard, + ); + return; + } + + // On macOS, the hard limit can be RLIM_INFINITY which setrlimit rejects. + // Fall back to the kernel-enforced kern.maxfilesperproc. + #[cfg(target_os = "macos")] + { + if let Some(maxfiles) = macos_maxfilesperproc() { + if maxfiles > soft && setrlimit(Resource::RLIMIT_NOFILE, maxfiles, hard).is_ok() { + info!( + message = "Raised file descriptor limit.", + from = soft, + to = maxfiles, + ); + return; + } + } + } + + warn!( + message = "Failed to raise file descriptor limit.", + current = soft, + attempted = hard, + ); +} + +/// Query the macOS kernel limit on per-process open files. +#[cfg(target_os = "macos")] +fn macos_maxfilesperproc() -> Option { + let mut maxfiles: libc::c_int = 0; + let mut len = std::mem::size_of::() as libc::size_t; + // Safety: sysctlbyname with a valid null-terminated name and correctly sized output buffer. + // No safe wrapper exists for this macOS-specific call. + let ret = unsafe { + libc::sysctlbyname( + c"kern.maxfilesperproc".as_ptr(), + &mut maxfiles as *mut libc::c_int as *mut libc::c_void, + &mut len, + std::ptr::null_mut(), + 0, + ) + }; + if ret == 0 && maxfiles > 0 { + Some(maxfiles as libc::rlim_t) + } else { + None + } +} + #[derive(Parser, Debug)] #[command(rename_all = "kebab-case")] pub enum SubCommand { @@ -399,3 +490,91 @@ pub fn handle_config_errors(errors: Vec) -> exitcode::ExitCode { exitcode::CONFIG } + +#[cfg(test)] +mod tests { + #[cfg(unix)] + fn run_in_subprocess(test_name: &str) { + let exe = std::env::current_exe().unwrap(); + let output = std::process::Command::new(exe) + .env("__VECTOR_SUBPROCESS_TEST", "1") + .args(["--exact", test_name, "--nocapture"]) + .output() + .unwrap(); + assert!( + output.status.success(), + "subprocess test failed:\nstdout: {}\nstderr: {}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + ); + } + + #[test] + #[cfg(unix)] + fn test_raise_file_descriptor_limit() { + if std::env::var("__VECTOR_SUBPROCESS_TEST").is_err() { + run_in_subprocess("cli::tests::test_raise_file_descriptor_limit"); + return; + } + + use nix::sys::resource::{getrlimit, setrlimit, Resource}; + + let (original_soft, hard) = getrlimit(Resource::RLIMIT_NOFILE).unwrap(); + let lowered = std::cmp::min(original_soft, 256); + if lowered < hard { + setrlimit(Resource::RLIMIT_NOFILE, lowered, hard).unwrap(); + + let (soft_before, _) = getrlimit(Resource::RLIMIT_NOFILE).unwrap(); + assert_eq!(soft_before, lowered); + + super::raise_file_descriptor_limit(); + + let (soft_after, _) = getrlimit(Resource::RLIMIT_NOFILE).unwrap(); + assert!( + soft_after > lowered, + "Expected soft limit to be raised above {lowered}, got {soft_after}" + ); + } + } + + #[test] + #[cfg(unix)] + fn test_raise_file_descriptor_limit_already_at_max() { + if std::env::var("__VECTOR_SUBPROCESS_TEST").is_err() { + run_in_subprocess("cli::tests::test_raise_file_descriptor_limit_already_at_max"); + return; + } + + use nix::sys::resource::{getrlimit, setrlimit, Resource}; + + let (_, hard) = getrlimit(Resource::RLIMIT_NOFILE).unwrap(); + + if setrlimit(Resource::RLIMIT_NOFILE, hard, hard).is_err() { + #[cfg(target_os = "macos")] + if let Some(maxfiles) = super::macos_maxfilesperproc() { + let _ = setrlimit(Resource::RLIMIT_NOFILE, maxfiles, hard); + } + } + + let (soft_before, _) = getrlimit(Resource::RLIMIT_NOFILE).unwrap(); + + super::raise_file_descriptor_limit(); + + let (soft_after, _) = getrlimit(Resource::RLIMIT_NOFILE).unwrap(); + assert_eq!(soft_before, soft_after); + } + + #[test] + #[cfg(target_os = "macos")] + fn test_macos_maxfilesperproc_returns_positive() { + let result = super::macos_maxfilesperproc(); + assert!( + result.is_some(), + "macos_maxfilesperproc() should return Some on macOS" + ); + assert!( + result.unwrap() > 0, + "kern.maxfilesperproc should be positive" + ); + } +} diff --git a/src/sinks/aws_cloudwatch_logs/integration_tests.rs b/src/sinks/aws_cloudwatch_logs/integration_tests.rs index 889d663f0911a..c59ffcd931326 100644 --- a/src/sinks/aws_cloudwatch_logs/integration_tests.rs +++ b/src/sinks/aws_cloudwatch_logs/integration_tests.rs @@ -573,7 +573,7 @@ async fn cloudwatch_healthcheck() { tags: None, }; - let client = config.create_client(&ProxyConfig::default()).await.unwrap(); + let (client, _region) = config.create_client(&ProxyConfig::default()).await.unwrap(); healthcheck(config, client).await.unwrap(); }