From 770d5a824e0e0bb3a5107e8f421de326e0105609 Mon Sep 17 00:00:00 2001 From: zackees Date: Sun, 9 Aug 2026 15:59:02 -0700 Subject: [PATCH] fix(avr): download registry lib_deps before compilation Registry and remote lib_deps entries (e.g., fastled/FastLED@^3.10.3) were classified by 'fbuild sync' but never downloaded by the AVR build orchestrator, so the compile step couldn't find their headers. Add a shared ensure_lib_deps() helper to the build pipeline library module and call it from the AVR orchestrator before the compiler is created, so downloaded library include directories are available during compilation and their archives are linked into the firmware. The ESP32 orchestrator already called ensure_libraries() directly; this brings the AVR path up to parity. Other non-ESP32 orchestrators using run_sequential_build_with_libs() (Teensy, STM32, RP2040, SAM, CH32V, ESP8266, NRF52, Renesas, Silabs, Apollo3, NXP LPC) can adopt the same ensure_lib_deps() helper in follow-up PRs. Fixes #1276. Co-Authored-By: Claude --- .../src/pipeline/library.rs | 82 ++++++++++++++++++- .../fbuild-build-engine/src/pipeline/mod.rs | 2 +- .../fbuild-build-mcu/src/avr/orchestrator.rs | 69 +++++++++++++++- 3 files changed, 147 insertions(+), 6 deletions(-) diff --git a/crates/fbuild-build-engine/src/pipeline/library.rs b/crates/fbuild-build-engine/src/pipeline/library.rs index f1fdf9b73..c13e75e7d 100644 --- a/crates/fbuild-build-engine/src/pipeline/library.rs +++ b/crates/fbuild-build-engine/src/pipeline/library.rs @@ -1,5 +1,7 @@ //! Library compilation helpers: `LibraryBuildEnv`, archiver selection, -//! extra library roots, and the "project-as-library" compile path. +//! extra library roots, the "project-as-library" compile path, and +//! `ensure_lib_deps` for downloading registry/remote `lib_deps` entries +//! before the compiler is created. use std::path::{Path, PathBuf}; @@ -303,6 +305,84 @@ pub async fn compile_project_as_library( } } +/// Download and compile `lib_deps` entries from `platformio.ini`. +/// +/// Called by build orchestrators **before** the compiler is created, so the +/// returned include directories can be folded into the compiler's search path. +/// The caller is responsible for: +/// +/// 1. Adding the returned `include_dirs` to the compiler's include list. +/// 2. Passing the returned `archives` to the linker (via +/// `run_sequential_build_with_libs`' `extra_link_inputs` parameter). +/// +/// `base_includes` is the set of include directories available before any +/// `lib_deps` are downloaded (core, variant, sketch, toolchain sysroot). +/// `libs_dir` is where downloaded libraries will be staged — callers typically +/// use `/libs`. +/// +/// When `lib_deps` is empty this returns `(vec![], vec![])` immediately with +/// no I/O or network access. +/// +/// FastLED/fbuild#1276: registry dependencies (`fastled/FastLED@^3.10.3`) +/// were classified by `fbuild sync` but never downloaded by the build +/// orchestrator, so the compile step couldn't find their headers. +#[allow(clippy::too_many_arguments)] +pub async fn ensure_lib_deps( + lib_deps: &[String], + lib_ignore: &[String], + gcc_path: &Path, + gxx_path: &Path, + ar_path: &Path, + c_flags: &[String], + cpp_flags: &[String], + base_includes: &[PathBuf], + project_dir: &Path, + libs_dir: &Path, + verbose: bool, + jobs: usize, + compiler_cache: Option<&Path>, +) -> Result<(Vec, Vec)> { + if lib_deps.is_empty() { + return Ok((vec![], vec![])); + } + + tracing::info!( + "downloading {} lib_deps to {}", + lib_deps.len(), + libs_dir.display() + ); + + let lib_result = fbuild_packages::library::library_manager::ensure_libraries( + lib_deps, + lib_ignore, + gcc_path, + gxx_path, + ar_path, + c_flags, + cpp_flags, + base_includes, + project_dir, + libs_dir, + verbose, + jobs, + compiler_cache, + ) + .await?; + + // FastLED/fbuild#966: sort include dirs so `-I` flags are deterministic + // across checkouts (read_dir order varies by filesystem). + let mut lib_include_dirs = lib_result.include_dirs; + lib_include_dirs.sort(); + + tracing::info!( + "lib_deps: {} include dirs, {} archives", + lib_include_dirs.len(), + lib_result.archives.len() + ); + + Ok((lib_include_dirs, lib_result.archives)) +} + #[cfg(test)] mod pick_archiver_tests { use super::*; diff --git a/crates/fbuild-build-engine/src/pipeline/mod.rs b/crates/fbuild-build-engine/src/pipeline/mod.rs index ff0e63ae0..cc27f2bd8 100644 --- a/crates/fbuild-build-engine/src/pipeline/mod.rs +++ b/crates/fbuild-build-engine/src/pipeline/mod.rs @@ -22,7 +22,7 @@ pub use compile::{ pub use context::BuildContext; pub use library::{ LibraryBuildEnv, add_extra_library_include_dirs, compile_extra_libraries, - compile_project_as_library, discover_extra_library_roots, pick_archiver, + compile_project_as_library, discover_extra_library_roots, ensure_lib_deps, pick_archiver, }; pub use link::{assemble_build_result, handle_link_result}; pub use project_discovery::{discover_project_includes, is_platform_project, is_project_a_library}; diff --git a/crates/fbuild-build-mcu/src/avr/orchestrator.rs b/crates/fbuild-build-mcu/src/avr/orchestrator.rs index b89b373fb..419467a47 100644 --- a/crates/fbuild-build-mcu/src/avr/orchestrator.rs +++ b/crates/fbuild-build-mcu/src/avr/orchestrator.rs @@ -231,9 +231,9 @@ impl BuildOrchestrator for AvrOrchestrator { sources.variant_sources.len(), ); - // 6. Build include dirs + compiler + // 6. Build include dirs (initial set — lib_deps dirs are added below). let defines = ctx.board.get_defines(); - // Use the resolved core_dir/variant_dir directly — board.get_include_paths() + // Use the resolved core_dir/variant_dir directly — board.get_include_paths() // uses the raw board core name which may differ from the actual directory // (e.g. MiniCore's core dir is "MCUdude_corefiles", not "MiniCore"). let mut include_dirs = vec![core_dir.clone(), variant_dir.clone()]; @@ -244,6 +244,66 @@ impl BuildOrchestrator for AvrOrchestrator { let mcu_config = super::mcu_config::get_avr_config()?; + // 6a. Download `lib_deps` from the registry / remote URLs before + // creating the compiler, so the downloaded library include directories + // are available during compilation (FastLED/fbuild#1276). + let lib_deps = ctx.config.get_lib_deps(¶ms.env_name)?; + let lib_ignore = ctx + .config + .get_lib_ignore(¶ms.env_name) + .unwrap_or_default(); + let lib_archives: Vec; + if !lib_deps.is_empty() { + // Build a temp compiler solely to get the c/cxx flags for library + // compilation. The temp compiler is discarded — the *real* compiler + // is created afterwards with the full include-dir set. + let temp_compiler = AvrCompiler::new( + toolchain.get_gcc_path(), + toolchain.get_gxx_path(), + &ctx.board.mcu, + &ctx.board.f_cpu, + defines.clone(), + include_dirs.clone(), + mcu_config.clone(), + params.profile, + params.verbose, + ); + let c_flags_temp = temp_compiler.c_flags(); + let cpp_flags_temp = temp_compiler.cpp_flags(); + let dep_ar_path = toolchain.get_ar_path(); + let dep_gcc_ar_path = toolchain.get_gcc_ar_path(); + let dep_lib_ar_path = pipeline::pick_archiver( + &dep_ar_path, + &dep_gcc_ar_path, + &c_flags_temp, + &cpp_flags_temp, + ); + let libs_dir = build_dir.join("libs"); + let (lib_include_dirs, archives) = pipeline::ensure_lib_deps( + &lib_deps, + &lib_ignore, + &toolchain.get_gcc_path(), + &toolchain.get_gxx_path(), + dep_lib_ar_path, + &c_flags_temp, + &cpp_flags_temp, + &include_dirs, + ¶ms.project_dir, + &libs_dir, + params.verbose, + crate::parallel::effective_jobs(params.jobs), + compiler_cache.as_deref(), + ) + .await?; + include_dirs.extend(lib_include_dirs); + lib_archives = archives; + } else { + lib_archives = Vec::new(); + } + // temp_compiler is dropped here — its only purpose was c_flags/cpp_flags. + + // 6b. Create compiler with the full include-dir set (core, variant, + // toolchain sysroot, *and* downloaded library paths). let compiler = AvrCompiler::new( toolchain.get_gcc_path(), toolchain.get_gxx_path(), @@ -296,14 +356,15 @@ impl BuildOrchestrator for AvrOrchestrator { compiler_cache: None, }; - // 9. Run shared sequential build pipeline + // 9. Run shared sequential build pipeline — pass downloaded library + // archives as extra link inputs (FastLED/fbuild#1276). let build_result = pipeline::run_sequential_build_with_libs( &compiler, &linker, ctx, params, &sources, - &[], + &lib_archives, Some(&lib_env), TargetArchitecture::Avr, "AVR",