Uh oh!
There was an error while loading. Please reload this page.
- Notifications
You must be signed in to change notification settings - Fork 367
llext: install modules appropriately for testing#9013
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Uh oh!
There was an error while loading. Please reload this page.
Changes from all commits
b04e779279387cf69cabe71f9603f79b23bff87687173e144a71dc09File filter
Filter by extension
Conversations
Uh oh!
There was an error while loading. Please reload this page.
Jump to
Uh oh!
There was an error while loading. Please reload this page.
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change | ||||
|---|---|---|---|---|---|---|
| @@ -63,6 +63,8 @@ | ||||||
| sof_fw_version = None | ||||||
| signing_key = None | ||||||
| if py_platform.system() == "Windows": | ||||||
| xtensa_tools_version_postfix = "-win32" | ||||||
| elif py_platform.system() == "Linux": | ||||||
| @@ -601,7 +603,7 @@ def clean_staging(platform): | ||||||
| def rimage_west_configuration(platform_dict, dest_dir): | ||||||
| """Configure rimage in a new file `dest_dir/westconfig.ini`, starting | ||||||
| from the workspace .west/config. | ||||||
| Returns the pathlib.Path to the new file. | ||||||
| Returns a tuple (west ConfigFile, pathlib.Path to that new file). | ||||||
| """ | ||||||
| saved_local_var = os.environ.get('WEST_CONFIG_LOCAL') | ||||||
| @@ -641,7 +643,7 @@ def rimage_west_configuration(platform_dict, dest_dir): | ||||||
| platform_wconfig.set("rimage.extra-args", shlex.join(extra_args)) | ||||||
| return platform_west_config_path | ||||||
| return platform_wconfig, platform_west_config_path | ||||||
| def build_rimage(): | ||||||
| @@ -671,12 +673,12 @@ def rimage_options(platform_dict): | ||||||
| example: [ (-f, 2.5.0), (-b, 1), (-k, key.pem),... ] | ||||||
| """ | ||||||
| global signing_key | ||||||
| opts = [] | ||||||
| if args.verbose > 0: | ||||||
| opts.append(("-v",) * args.verbose) | ||||||
| signing_key = None | ||||||
| if args.key: | ||||||
| key_path = pathlib.Path(args.key) | ||||||
| assert key_path.exists(), f"{key_path} not found" | ||||||
| @@ -831,10 +833,11 @@ def build_platforms(): | ||||||
| see https://docs.zephyrproject.org/latest/guides/west/build-flash-debug.html#one-time-cmake-arguments | ||||||
| Try "west config build.cmake-args -- ..." instead.""") | ||||||
| platf_build_environ['WEST_CONFIG_LOCAL'] = str(rimage_west_configuration( | ||||||
| platform_wcfg, wcfg_path = rimage_west_configuration( | ||||||
| platform_dict, | ||||||
| STAGING_DIR / "sof-info" / platform | ||||||
| )) | ||||||
| ) | ||||||
| platf_build_environ['WEST_CONFIG_LOCAL'] = str(wcfg_path) | ||||||
| # Make sure the build logs don't leave anything hidden | ||||||
| execute_command(['west', 'config', '-l'], cwd=west_top, | ||||||
| @@ -869,7 +872,7 @@ def build_platforms(): | ||||||
| if platform not in RI_INFO_UNSUPPORTED: | ||||||
| reproducible_checksum(platform, west_top / platform_build_dir_name / "zephyr" / "zephyr.ri") | ||||||
| install_platform(platform, sof_platform_output_dir, platf_build_environ) | ||||||
| install_platform(platform, sof_platform_output_dir, platf_build_environ, platform_wcfg) | ||||||
| src_dest_list = [] | ||||||
| tools_output_dir = pathlib.Path(STAGING_DIR, "tools") | ||||||
| @@ -902,7 +905,69 @@ def build_platforms(): | ||||||
| symlinks=True, ignore_dangling_symlinks=True, dirs_exist_ok=True) | ||||||
| def install_platform(platform, sof_output_dir, platf_build_environ): | ||||||
| def install_lib(sof_lib_dir, abs_build_dir, platform_wconfig): | ||||||
| """[summary] Sign loadable llext modules, if any, copy them to the | ||||||
| deployment tree and create UUID links for the kernel to find and load | ||||||
| them.""" | ||||||
| global signing_key | ||||||
| with os.scandir(str(abs_build_dir)) as iter: | ||||||
marc-hb marked this conversation as resolved.
Uh oh!There was an error while loading. Please reload this page. | ||||||
| if args.key_type_subdir != "none": | ||||||
| sof_lib_dir = sof_lib_dir / args.key_type_subdir | ||||||
| sof_lib_dir.mkdir(parents=True, exist_ok=True) | ||||||
| for entry in iter: | ||||||
| if (not entry.is_dir or | ||||||
| not entry.name.endswith('_llext')): | ||||||
| continue | ||||||
| entry_path = pathlib.Path(entry.path) | ||||||
| uuids = entry_path / 'llext.uuid' | ||||||
| if not os.path.exists(uuids): | ||||||
marc-hb marked this conversation as resolved.
Uh oh!There was an error while loading. Please reload this page. | ||||||
| print(f"Directory {entry.name} has no llext.uuid file. Skipping.") | ||||||
| continue | ||||||
| # replace '_llext' with '.llext', e.g. | ||||||
| # eq_iir_llext/eq_iir.llext | ||||||
| llext_base = entry.name[:-6] | ||||||
| llext_file = llext_base + '.llext' | ||||||
marc-hb marked this conversation as resolved.
Uh oh!There was an error while loading. Please reload this page. | ||||||
| dst = sof_lib_dir / llext_file | ||||||
| rimage_cfg = entry_path / 'rimage_config.toml' | ||||||
| llext_input = entry_path / (llext_base + '.so') | ||||||
| llext_output = entry_path / llext_file | ||||||
| sign_cmd = [str(platform_wconfig.get("rimage.path")), "-o", str(llext_output), | ||||||
Collaborator There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Suggested change
CollaboratorAuthor There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. addressed in de87535
| ||||||
| "-e", "-c", str(rimage_cfg), | ||||||
| "-k", str(signing_key), "-l", "-r", | ||||||
| str(llext_input)] | ||||||
Collaborator There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. You can't hardcode rimage.path and arguments like that, the user has the ability to override those, see Collaborator There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Similar to CMake, I still can't see what is temporary versus not here. Please add the relevant TODOs. Even more important considering the high-level signing design sounds like "do whatever you want". CollaboratorAuthor There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. @marc-hb everything is temporary... I'm far from being a Python expert, so I expect improvements throughout this code - both during this review and after this is (hopefully soon) merged
| ||||||
| execute_command(sign_cmd, cwd=west_top) | ||||||
| # An intuitive way to make this multiline would be | ||||||
| # with (open(dst, 'wb') as fdst, open(llext_output, 'rb') as fllext, | ||||||
| # open(llext_output.with_suffix('.llext.xman'), 'rb') as fman): | ||||||
| # but a Python version, used on Windows errored out on this. | ||||||
| # Thus we're left with a choice between a 150-character | ||||||
| # long line and an illogical split like this | ||||||
| with open(dst, 'wb') as fdst, open(llext_output, 'rb') as fllext, open( | ||||||
| llext_output.with_suffix('.llext.xman'), 'rb') as fman: | ||||||
| # Concatenate the manifest and the llext | ||||||
| shutil.copyfileobj(fman, fdst) | ||||||
| shutil.copyfileobj(fllext, fdst) | ||||||
| # Create symbolic links for all UUIDs | ||||||
| with open(uuids, 'r') as uuids_f: | ||||||
| for uuid in uuids_f: | ||||||
| linkname = uuid.strip() + '.bin' | ||||||
| symlink_or_copy(sof_lib_dir, llext_file, | ||||||
| sof_lib_dir, linkname) | ||||||
| def install_platform(platform, sof_output_dir, platf_build_environ, platform_wconfig): | ||||||
| # Keep in sync with caller | ||||||
| platform_build_dir_name = f"build-{platform}" | ||||||
| @@ -948,6 +1013,10 @@ def install_platform(platform, sof_output_dir, platf_build_environ): | ||||||
| for p_alias in platform_configs[platform].aliases: | ||||||
| symlink_or_copy(install_key_dir, output_fwname, install_key_dir, f"sof-{p_alias}.ri") | ||||||
| if args.deployable_build and platform_configs[platform].ipc4: | ||||||
| install_lib(sof_output_dir / '..' / 'sof-ipc4-lib' / platform, abs_build_dir, | ||||||
| platform_wconfig) | ||||||
| # sof-info/ directory | ||||||
| @@ -1033,7 +1102,6 @@ class InstFile: | ||||||
| gzip_res.result() # throws exception if gzip unexpectedly failed | ||||||
| gzip_threads.shutdown() | ||||||
| # Zephyr's CONFIG_KERNEL_BIN_NAME default value | ||||||
| BIN_NAME = 'zephyr' | ||||||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,6 +1,10 @@ | ||
| # Copyright (c) 2023 Intel Corporation. | ||
| # SPDX-License-Identifier: Apache-2.0 | ||
| # FIXME: This *WILL* be converted to add_llext_target() as long as that's | ||
| # reasonably possible, and if it isn't, a *significant* effort will be made to | ||
| # make that possible | ||
| cmake_minimum_required(VERSION 3.20.0) | ||
| find_package(Zephyr REQUIRED HINTS $ENV{ZEPHYR_BASE}) | ||
| project(smart_amp_test) | ||
| @@ -10,6 +14,16 @@ SET_PROPERTY(GLOBAL PROPERTY TARGET_SUPPORTS_SHARED_LIBS TRUE) | ||
| set(MODULE "smart_amp_test") | ||
| cmake_path(SET SOF_BASE NORMALIZE ${PROJECT_SOURCE_DIR}/../../../..) | ||
| file(STRINGS ${CMAKE_CURRENT_LIST_DIR}/../${MODULE}.toml uuids REGEX "^[ \t]*uuid *=") | ||
| file(WRITE ${PROJECT_BINARY_DIR}/llext.uuid "") | ||
| foreach(line IN LISTS uuids) | ||
| # extract UUID value - drop the 'uuid = ' part of the assignment line | ||
| string(REGEX REPLACE "^[ \t]*uuid *= \"([0-9A-F\-]*)\"" "\\1" uuid ${line}) | ||
| file(APPEND ${PROJECT_BINARY_DIR}/llext.uuid "${uuid}\n") | ||
| endforeach() | ||
Member There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. why is this a smart_amp specific operation? Are we going to replicate this for each and every module? CollaboratorAuthor There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. @plbossart so far this CMakeLists.txt would be largely repeated for each new module, yes. But we plan to switch to using recently added Zephyr cmake support for LLEXT code ASAP Collaborator There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Corresponding FIXME now added in later commit, thanks! (I reviewed commits one by one and initially missed it) | ||
| add_library(${MODULE} SHARED) | ||
| target_sources(${MODULE} PRIVATE | ||
| @@ -55,12 +69,12 @@ target_compile_options(${MODULE} PRIVATE | ||
| if("${ZEPHYR_TOOLCHAIN_VARIANT}" STREQUAL "zephyr") | ||
| set(MODULE_LINKER_PARAMS -nostdlib -nodefaultlibs) | ||
| set(EXTRA_LINKED_PARAMS -shared) | ||
| set(COPY_CMD ${CMAKE_STRIP} -R .xt.* -o ${MODULE}_out.so ${MODULE}_llext.so) | ||
| set(EXTRA_LINKER_PARAMS -shared) | ||
| set(COPY_CMD ${CMAKE_STRIP} -R .xt.* -o ${MODULE}.so ${MODULE}_pre.so) | ||
| else() | ||
| set(MODULE_LINKER_PARAMS -nostdlib -nodefaultlibs -r) | ||
| set(EXTRA_LINKED_PARAMS) | ||
| set(COPY_CMD ${CMAKE_OBJCOPY} -R .xt.* ${MODULE}_llext.so ${MODULE}_out.so) | ||
| set(EXTRA_LINKER_PARAMS) | ||
| set(COPY_CMD ${CMAKE_OBJCOPY} -R .xt.* ${MODULE}_pre.so ${MODULE}.so) | ||
| endif() | ||
Member There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. you should split code, script changes. Not sure why they belong in the same patch? Also there's no reason why smart_amp should be included here, this is a separate addition IMHO. CollaboratorAuthor There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. @plbossart smart-amp is currently the only component in "main," that can be built as a module. So while adding generic relocatable format support to SOF, we also add it to smart-amp, but this can be split, yes | ||
| target_link_options(${MODULE} PRIVATE | ||
| @@ -69,10 +83,14 @@ target_link_options(${MODULE} PRIVATE | ||
| add_custom_command(OUTPUT lib${MODULE}_out.so | ||
| DEPENDS ${MODULE} | ||
| COMMAND ${CMAKE_C_COMPILER} -E ${CMAKE_CURRENT_LIST_DIR}/llext.toml.h -P -DREM= | ||
| -I${SOF_BASE} -I${SOF_BASE}src | ||
| -imacros ../include/generated/autoconf.h | ||
| -o rimage_config.toml | ||
marc-hb marked this conversation as resolved.
Uh oh!There was an error while loading. Please reload this page. | ||
| COMMAND ${SOF_BASE}scripts/llext_link_helper.py | ||
| -f lib${MODULE}.so -t "0xa06ca000" ${CMAKE_C_COMPILER} -- | ||
| ${MODULE_LINKER_PARAMS} ${EXTRA_LINKED_PARAMS} -fPIC | ||
| -o ${MODULE}_llext.so $<TARGET_OBJECTS:${MODULE}> | ||
| ${MODULE_LINKER_PARAMS} ${EXTRA_LINKER_PARAMS} -fPIC | ||
| -o ${MODULE}_pre.so $<TARGET_OBJECTS:${MODULE}> | ||
Member There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. problem statement/commit message: a) why do we need to sign modules? CollaboratorAuthor There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. @plbossart here the same key is used as the one for the base firmware. In principle any key, accepted by the DSP can be used. Why - for security, for the same reason why we sign the base firmware Member There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. @lyakh we do need the ability to specify that alternate key for modules that are not part of the base firmware. Actually maybe it's more complicated even, e.g. if Intel releases modules then presumably they are signed with the same key than the base firmware, but a 3rd party would be required to use a separate key, so the key would need to be specified on a module-by-module basis. It's not 9am and that already gives me a migraine.
| ||
| COMMAND ${COPY_CMD} | ||
| COMMAND_EXPAND_LISTS | ||
| ) | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,4 +1,5 @@ | ||
| #include <tools/rimage/config/platform.toml> | ||
| #define LOAD_TYPE "2" | ||
| #include "../smart_amp_test.toml" | ||
| [module] | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -794,8 +794,8 @@ if(CONFIG_IPC_MAJOR_3) | ||
| ) | ||
| elseif(CONFIG_IPC_MAJOR_4) | ||
| if(CONFIG_SAMPLE_SMART_AMP STREQUAL "m") | ||
| add_subdirectory(${SOF_SAMPLES_PATH}/audio/smart_amp_llext | ||
| ${PROJECT_BINARY_DIR}/smart_amp_llext) | ||
| add_subdirectory(${SOF_SAMPLES_PATH}/audio/smart_amp_test_llext | ||
| ${PROJECT_BINARY_DIR}/smart_amp_test_llext) | ||
Member There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. why the rename? was this a mistake that should be corrected separated, something done in a previous patch? CollaboratorAuthor There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. @plbossart previously this name wasn't a problem, since each module had its own supporting processing code, so names could be arbitrary. Now we need automation, so names must be unified as | ||
| add_dependencies(app smart_amp_test_llext) | ||
| elseif(CONFIG_SAMPLE_SMART_AMP) | ||
| zephyr_library_sources( | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
should this be a module_signing_key? Is this different from the base firmware key? why is this global?