diff --git a/app/boards/intel_adsp_ace30_ptl.conf b/app/boards/intel_adsp_ace30_ptl.conf index b6ac41938398..9a63751b1bde 100644 --- a/app/boards/intel_adsp_ace30_ptl.conf +++ b/app/boards/intel_adsp_ace30_ptl.conf @@ -21,9 +21,9 @@ CONFIG_KCPS_DYNAMIC_CLOCK_CONTROL=n CONFIG_PROBE=y CONFIG_PROBE_DMA_MAX=2 CONFIG_SOF_TELEMETRY=y -CONFIG_SOF_TELEMETRY_IO_PERFORMANCE_MEASUREMENTS=y -CONFIG_SOF_TELEMETRY_PERFORMANCE_MEASUREMENTS=y -CONFIG_COLD_STORE_EXECUTE_DRAM=y +CONFIG_SOF_TELEMETRY_IO_PERFORMANCE_MEASUREMENTS=n +CONFIG_SOF_TELEMETRY_PERFORMANCE_MEASUREMENTS=n +CONFIG_COLD_STORE_EXECUTE_DRAM=n # SOF / loadable modules CONFIG_INTEL_MODULES=y @@ -40,10 +40,10 @@ CONFIG_SOF_LOG_LEVEL_INF=y CONFIG_COUNTER=y CONFIG_HEAP_MEM_POOL_SIZE=8192 CONFIG_LLEXT=y -CONFIG_LLEXT_STORAGE_WRITABLE=y -CONFIG_LLEXT_EXPERIMENTAL=y +CONFIG_LLEXT_STORAGE_WRITABLE=n +CONFIG_LLEXT_EXPERIMENTAL=n CONFIG_LLEXT_EDK=n -CONFIG_MODULES=y +CONFIG_MODULES=n # Zephyr / device drivers CONFIG_DAI_INIT_PRIORITY=70 @@ -78,3 +78,19 @@ CONFIG_SOF_USERSPACE_PROXY=y CONFIG_MAX_THREAD_BYTES=3 CONFIG_MAX_DOMAIN_PARTITIONS=32 + +# Userspace LL (was app/overlays/ptl/ll_userspace_overlay.conf) +# Run Low-Latency audio pipelines in user-space threads by default. +CONFIG_SOF_USERSPACE_LL=y +CONFIG_SOF_USERSPACE_INTERFACE_DMA=y +CONFIG_DAI_USERSPACE=y + +# Settings currently required to enable user-space LL. The cold-store, +# telemetry, loadable-module and misc feature disables above/here are not +# yet user-space compatible (see the former overlay for rationale). +CONFIG_COLD_STORE_EXECUTE_DEBUG=n +CONFIG_SOF_BOOT_TEST_ALLOWED=n +CONFIG_CROSS_CORE_STREAM=n +CONFIG_INTEL_ADSP_MIC_PRIVACY=n +CONFIG_XRUN_NOTIFICATIONS_ENABLE=n +CONFIG_ZEPHYR_DP_SCHEDULER=n diff --git a/app/boards/intel_adsp_ace30_wcl.conf b/app/boards/intel_adsp_ace30_wcl.conf index 2196af333e65..d825a2a37c95 100644 --- a/app/boards/intel_adsp_ace30_wcl.conf +++ b/app/boards/intel_adsp_ace30_wcl.conf @@ -21,9 +21,9 @@ CONFIG_KCPS_DYNAMIC_CLOCK_CONTROL=n CONFIG_PROBE=y CONFIG_PROBE_DMA_MAX=2 CONFIG_SOF_TELEMETRY=y -CONFIG_SOF_TELEMETRY_IO_PERFORMANCE_MEASUREMENTS=y -CONFIG_SOF_TELEMETRY_PERFORMANCE_MEASUREMENTS=y -CONFIG_COLD_STORE_EXECUTE_DRAM=y +CONFIG_SOF_TELEMETRY_IO_PERFORMANCE_MEASUREMENTS=n +CONFIG_SOF_TELEMETRY_PERFORMANCE_MEASUREMENTS=n +CONFIG_COLD_STORE_EXECUTE_DRAM=n # SOF / loadable modules CONFIG_INTEL_MODULES=y @@ -39,10 +39,10 @@ CONFIG_SOF_LOG_LEVEL_INF=y # Zephyr / OS features CONFIG_HEAP_MEM_POOL_SIZE=8192 CONFIG_LLEXT=y -CONFIG_LLEXT_STORAGE_WRITABLE=y -CONFIG_LLEXT_EXPERIMENTAL=y +CONFIG_LLEXT_STORAGE_WRITABLE=n +CONFIG_LLEXT_EXPERIMENTAL=n CONFIG_LLEXT_EDK=n -CONFIG_MODULES=y +CONFIG_MODULES=n # Zephyr / device drivers CONFIG_DAI_INIT_PRIORITY=70 @@ -64,3 +64,32 @@ CONFIG_PM_DEVICE_RUNTIME_ASYNC=n CONFIG_LOG_BACKEND_ADSP=n CONFIG_LOG_FLUSH_SLEEP_US=5000 CONFIG_WINSTREAM_CONSOLE=n + +# Userspace base (mirrored from intel_adsp_ace30_ptl.conf) +# Required so that user-space LL (below) can actually be enabled, since +# CONFIG_SOF_USERSPACE_LL depends on CONFIG_USERSPACE. +CONFIG_USERSPACE=y +CONFIG_DYNAMIC_THREAD=y +CONFIG_DYNAMIC_THREAD_ALLOC=y +CONFIG_DYNAMIC_THREAD_PREFER_ALLOC=y +CONFIG_SOF_STACK_SIZE=8192 +CONFIG_SOF_USERSPACE_PROXY=y +CONFIG_MAX_THREAD_BYTES=3 +CONFIG_MAX_DOMAIN_PARTITIONS=32 +CONFIG_XTENSA_MMU_NUM_L2_TABLES=128 + +# Userspace LL (was app/overlays/ptl/ll_userspace_overlay.conf) +# Run Low-Latency audio pipelines in user-space threads by default. +CONFIG_SOF_USERSPACE_LL=y +CONFIG_SOF_USERSPACE_INTERFACE_DMA=y +CONFIG_DAI_USERSPACE=y + +# Settings currently required to enable user-space LL. The cold-store, +# telemetry, loadable-module and misc feature disables above/here are not +# yet user-space compatible (see the former overlay for rationale). +CONFIG_COLD_STORE_EXECUTE_DEBUG=n +CONFIG_SOF_BOOT_TEST_ALLOWED=n +CONFIG_CROSS_CORE_STREAM=n +CONFIG_INTEL_ADSP_MIC_PRIVACY=n +CONFIG_XRUN_NOTIFICATIONS_ENABLE=n +CONFIG_ZEPHYR_DP_SCHEDULER=n diff --git a/app/overlays/ptl/ll_userspace_overlay.conf b/app/overlays/ptl/ll_userspace_overlay.conf index 8888a66f4c24..53b881320e6f 100644 --- a/app/overlays/ptl/ll_userspace_overlay.conf +++ b/app/overlays/ptl/ll_userspace_overlay.conf @@ -36,3 +36,4 @@ CONFIG_SOF_BOOT_TEST_ALLOWED=n CONFIG_CROSS_CORE_STREAM=n CONFIG_INTEL_ADSP_MIC_PRIVACY=n CONFIG_XRUN_NOTIFICATIONS_ENABLE=n +CONFIG_ZEPHYR_DP_SCHEDULER=n diff --git a/posix/include/rtos/mutex.h b/posix/include/rtos/mutex.h index 19b360bdaea5..f82b4169d73c 100644 --- a/posix/include/rtos/mutex.h +++ b/posix/include/rtos/mutex.h @@ -16,6 +16,7 @@ #include #define K_FOREVER ((k_timeout_t) { .ticks = 0xffffffff }) +#define K_MUTEX_DEFINE(name) struct k_mutex name struct k_mutex { struct k_spinlock lock; diff --git a/src/audio/chain_dma.c b/src/audio/chain_dma.c index e7207e636ee8..2752318c4d06 100644 --- a/src/audio/chain_dma.c +++ b/src/audio/chain_dma.c @@ -15,7 +15,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -61,13 +63,13 @@ struct chain_dma_data { /* local host DMA config */ struct sof_dma *dma_host; - struct dma_chan_data *chan_host; + int chan_host_index; struct dma_config z_config_host; struct dma_block_config dma_block_cfg_host; /* local link DMA config */ struct sof_dma *dma_link; - struct dma_chan_data *chan_link; + int chan_link_index; struct dma_config z_config_link; struct dma_block_config dma_block_cfg_link; @@ -79,18 +81,18 @@ static int chain_host_start(struct comp_dev *dev) struct chain_dma_data *cd = comp_get_drvdata(dev); int err; - if (!cd->chan_host || !cd->chan_host->dma) { + if (cd->chan_host_index < 0 || !cd->dma_host) { comp_err(dev, "incomplete initialization detected, aborting host %p", - cd->chan_host); + cd->dma_host); return -ENODEV; } - err = dma_start(cd->chan_host->dma->z_dev, cd->chan_host->index); + err = sof_dma_start(cd->dma_host, cd->chan_host_index); if (err < 0) return err; comp_info(dev, "dma_start() host chan_index = %u", - cd->chan_host->index); + cd->chan_host_index); return 0; } @@ -99,12 +101,12 @@ static int chain_link_start(struct comp_dev *dev) struct chain_dma_data *cd = comp_get_drvdata(dev); int err; - err = dma_start(cd->chan_link->dma->z_dev, cd->chan_link->index); + err = sof_dma_start(cd->dma_link, cd->chan_link_index); if (err < 0) return err; comp_info(dev, "dma_start() link chan_index = %u", - cd->chan_link->index); + cd->chan_link_index); return 0; } @@ -113,12 +115,12 @@ static int chain_link_stop(struct comp_dev *dev) struct chain_dma_data *cd = comp_get_drvdata(dev); int err; - err = dma_stop(cd->chan_link->dma->z_dev, cd->chan_link->index); + err = sof_dma_stop(cd->dma_link, cd->chan_link_index); if (err < 0) return err; comp_info(dev, "dma_stop() link chan_index = %u", - cd->chan_link->index); + cd->chan_link_index); return 0; } @@ -128,12 +130,12 @@ static int chain_host_stop(struct comp_dev *dev) struct chain_dma_data *cd = comp_get_drvdata(dev); int err; - err = dma_stop(cd->chan_host->dma->z_dev, cd->chan_host->index); + err = sof_dma_stop(cd->dma_host, cd->chan_host_index); if (err < 0) return err; comp_info(dev, "dma_stop() host chan_index = %u", - cd->chan_host->index); + cd->chan_host_index); return 0; } @@ -171,7 +173,7 @@ static enum task_state chain_task_run(void *data) /* Link DMA can return -EPIPE and current status if xrun occurs, then it is not critical * and flow shall continue. Other error values will be treated as critical. */ - ret = dma_get_status(cd->chan_link->dma->z_dev, cd->chan_link->index, &stat); + ret = sof_dma_get_status(cd->dma_link, cd->chan_link_index, &stat); switch (ret) { case 0: #if CONFIG_XRUN_NOTIFICATIONS_ENABLE @@ -195,7 +197,7 @@ static enum task_state chain_task_run(void *data) link_read_pos = stat.read_position; /* Host DMA does not report xruns. All error values will be treated as critical. */ - ret = dma_get_status(cd->chan_host->dma->z_dev, cd->chan_host->index, &stat); + ret = sof_dma_get_status(cd->dma_host, cd->chan_host_index, &stat); if (ret < 0) { tr_err(&chain_dma_tr, "dma_get_status() error, ret = %d", ret); return SOF_TASK_STATE_COMPLETED; @@ -213,14 +215,14 @@ static enum task_state chain_task_run(void *data) */ const size_t increment = MIN(host_free_bytes, link_avail_bytes); - ret = dma_reload(cd->chan_host->dma->z_dev, cd->chan_host->index, 0, 0, increment); + ret = sof_dma_reload(cd->dma_host, cd->chan_host_index, increment); if (ret < 0) { tr_err(&chain_dma_tr, "dma_reload() host error, ret = %d", ret); return SOF_TASK_STATE_COMPLETED; } - ret = dma_reload(cd->chan_link->dma->z_dev, cd->chan_link->index, 0, 0, increment); + ret = sof_dma_reload(cd->dma_link, cd->chan_link_index, increment); if (ret < 0) { tr_err(&chain_dma_tr, "dma_reload() link error, ret = %d", ret); @@ -236,9 +238,8 @@ static enum task_state chain_task_run(void *data) const size_t half_buff_size = buff_size / 2; if (!cd->first_data_received && host_avail_bytes > half_buff_size) { - ret = dma_reload(cd->chan_link->dma->z_dev, - cd->chan_link->index, 0, 0, - MIN(host_avail_bytes, link_free_bytes)); + ret = sof_dma_reload(cd->dma_link, cd->chan_link_index, + MIN(host_avail_bytes, link_free_bytes)); if (ret < 0) { tr_err(&chain_dma_tr, "dma_reload() link error, ret = %d", ret); @@ -252,8 +253,8 @@ static enum task_state chain_task_run(void *data) host_read_pos, buff_size); - ret = dma_reload(cd->chan_host->dma->z_dev, cd->chan_host->index, - 0, 0, transferred); + ret = sof_dma_reload(cd->dma_host, cd->chan_host_index, + transferred); if (ret < 0) { tr_err(&chain_dma_tr, "dma_reload() host error, ret = %d", ret); @@ -262,8 +263,8 @@ static enum task_state chain_task_run(void *data) if (host_avail_bytes >= half_buff_size && link_free_bytes >= half_buff_size) { - ret = dma_reload(cd->chan_link->dma->z_dev, cd->chan_link->index, - 0, 0, half_buff_size); + ret = sof_dma_reload(cd->dma_link, cd->chan_link_index, + half_buff_size); if (ret < 0) { tr_err(&chain_dma_tr, "dma_reload() link error, ret = %d", ret); @@ -373,9 +374,9 @@ __cold static void chain_release(struct comp_dev *dev) assert_can_be_cold(); - dma_release_channel(cd->chan_host->dma->z_dev, cd->chan_host->index); + sof_dma_release_channel(cd->dma_host, cd->chan_host_index); sof_dma_put(cd->dma_host); - dma_release_channel(cd->chan_link->dma->z_dev, cd->chan_link->index); + sof_dma_release_channel(cd->dma_link, cd->chan_link_index); sof_dma_put(cd->dma_link); if (cd->dma_buffer) { @@ -463,16 +464,16 @@ __cold static int chain_init(struct comp_dev *dev, void *addr, size_t length) /* get host DMA channel */ channel = cd->host_connector_node_id.f.v_index; - channel = dma_request_channel(cd->dma_host->z_dev, &channel); + channel = sof_dma_request_channel(cd->dma_host, channel); if (channel < 0) { comp_err(dev, "host dma_request_channel() failed for %u", cd->host_connector_node_id.f.v_index); return channel; } - cd->chan_host = &cd->dma_host->chan[channel]; + cd->chan_host_index = channel; - err = dma_config(cd->dma_host->z_dev, cd->chan_host->index, dma_cfg_host); + err = sof_dma_config(cd->dma_host, cd->chan_host_index, dma_cfg_host); if (err < 0) { comp_err(dev, "host dma_config() failed for %d", channel); goto error_host; @@ -480,7 +481,7 @@ __cold static int chain_init(struct comp_dev *dev, void *addr, size_t length) /* get link DMA channel */ channel = cd->link_connector_node_id.f.v_index; - channel = dma_request_channel(cd->dma_link->z_dev, &channel); + channel = sof_dma_request_channel(cd->dma_link, channel); if (channel < 0) { comp_err(dev, "link dma_request_channel() failed for %u", cd->link_connector_node_id.f.v_index); @@ -488,9 +489,9 @@ __cold static int chain_init(struct comp_dev *dev, void *addr, size_t length) goto error_host; } - cd->chan_link = &cd->dma_link->chan[channel]; + cd->chan_link_index = channel; - err = dma_config(cd->dma_link->z_dev, cd->chan_link->index, dma_cfg_link); + err = sof_dma_config(cd->dma_link, cd->chan_link_index, dma_cfg_link); if (err < 0) { comp_err(dev, "link dma_config() failed for %d", channel); goto error_link; @@ -498,11 +499,9 @@ __cold static int chain_init(struct comp_dev *dev, void *addr, size_t length) return 0; error_link: - dma_release_channel(cd->dma_link->z_dev, cd->chan_link->index); - cd->chan_link = NULL; + sof_dma_release_channel(cd->dma_link, cd->chan_link_index); error_host: - dma_release_channel(cd->dma_host->z_dev, cd->chan_host->index); - cd->chan_host = NULL; + sof_dma_release_channel(cd->dma_host, cd->chan_host_index); return err; } @@ -510,6 +509,7 @@ __cold static int chain_task_init(struct comp_dev *dev, uint8_t host_dma_id, uin uint32_t fifo_size) { struct chain_dma_data *cd = comp_get_drvdata(dev); + struct mod_alloc_ctx *alloc_ctx = NULL; uint32_t addr_align; size_t buff_size; void *buff_addr; @@ -559,8 +559,8 @@ __cold static int chain_task_init(struct comp_dev *dev, uint8_t host_dma_id, uin } /* retrieve DMA buffer address alignment */ - ret = dma_get_attribute(cd->dma_host->z_dev, DMA_ATTR_BUFFER_ADDRESS_ALIGNMENT, - &addr_align); + ret = sof_dma_get_attribute(cd->dma_host, DMA_ATTR_BUFFER_ADDRESS_ALIGNMENT, + &addr_align); if (ret < 0) { comp_err(dev, "could not get dma buffer address alignment, err = %d", ret); @@ -588,8 +588,14 @@ __cold static int chain_task_init(struct comp_dev *dev, uint8_t host_dma_id, uin } fifo_size = ALIGN_UP_INTERNAL(fifo_size, addr_align); + +#ifdef CONFIG_SOF_USERSPACE_LL + alloc_ctx = ipc_get()->ll_alloc; +#endif + /* allocate not shared buffer */ - cd->dma_buffer = buffer_alloc(NULL, fifo_size, SOF_MEM_FLAG_USER | SOF_MEM_FLAG_DMA, + cd->dma_buffer = buffer_alloc(alloc_ctx, fifo_size, + SOF_MEM_FLAG_USER | SOF_MEM_FLAG_DMA, addr_align, BUFFER_USAGE_NOT_SHARED); if (!cd->dma_buffer) { @@ -631,6 +637,70 @@ static int chain_task_trigger(struct comp_dev *dev, int cmd) } } +/* + * comp_dev and private data allocation helpers. For user-space LL both + * objects must live on the user heap so the (unprivileged) user LL thread + * can access them; otherwise the normal component/rmalloc paths are used. + */ +#ifdef CONFIG_SOF_USERSPACE_LL +__cold static struct comp_dev *chain_dev_alloc(const struct comp_driver *drv) +{ + struct comp_dev *dev; + + dev = sof_heap_alloc(sof_sys_user_heap_get(), + SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, + sizeof(*dev), 0); + if (!dev) + return NULL; + + memset(dev, 0, sizeof(*dev)); + comp_init(drv, dev, sizeof(*dev)); + + return dev; +} + +__cold static struct chain_dma_data *chain_cd_alloc(void) +{ + struct chain_dma_data *cd; + + cd = sof_heap_alloc(sof_sys_user_heap_get(), SOF_MEM_FLAG_USER, sizeof(*cd), 0); + if (cd) + memset(cd, 0, sizeof(*cd)); + + return cd; +} + +__cold static void chain_dev_free(struct comp_dev *dev) +{ + sof_heap_free(sof_sys_user_heap_get(), dev); +} + +__cold static void chain_cd_free(struct chain_dma_data *cd) +{ + sof_heap_free(sof_sys_user_heap_get(), cd); +} +#else +__cold static struct comp_dev *chain_dev_alloc(const struct comp_driver *drv) +{ + return comp_alloc(drv, sizeof(struct comp_dev)); +} + +__cold static struct chain_dma_data *chain_cd_alloc(void) +{ + return rzalloc(SOF_MEM_FLAG_USER, sizeof(struct chain_dma_data)); +} + +__cold static void chain_dev_free(struct comp_dev *dev) +{ + comp_free_device(dev); +} + +__cold static void chain_cd_free(struct chain_dma_data *cd) +{ + rfree(cd); +} +#endif + __cold static struct comp_dev *chain_task_create(const struct comp_driver *drv, const struct comp_ipc_config *ipc_config, const void *ipc_specific_config) @@ -649,17 +719,19 @@ __cold static struct comp_dev *chain_task_create(const struct comp_driver *drv, if (host_dma_id >= max_chain_number) return NULL; - dev = comp_alloc(drv, sizeof(*dev)); + dev = chain_dev_alloc(drv); if (!dev) return NULL; - cd = rzalloc(SOF_MEM_FLAG_USER, sizeof(*cd)); + cd = chain_cd_alloc(); if (!cd) goto error; cd->first_data_received = false; cd->cs = scs ? 2 : 4; cd->chain_task.state = SOF_TASK_STATE_INIT; + cd->chan_host_index = -EINVAL; + cd->chan_link_index = -EINVAL; comp_set_drvdata(dev, cd); @@ -667,9 +739,9 @@ __cold static struct comp_dev *chain_task_create(const struct comp_driver *drv, if (!ret) return dev; - rfree(cd); + chain_cd_free(cd); error: - comp_free_device(dev); + chain_dev_free(dev); return NULL; } @@ -680,8 +752,8 @@ __cold static void chain_task_free(struct comp_dev *dev) assert_can_be_cold(); chain_release(dev); - rfree(cd); - comp_free_device(dev); + chain_cd_free(cd); + chain_dev_free(dev); } static const struct comp_driver comp_chain_dma = { diff --git a/src/audio/module_adapter/module/generic.c b/src/audio/module_adapter/module/generic.c index ee1b2df92829..37071a2448e3 100644 --- a/src/audio/module_adapter/module/generic.c +++ b/src/audio/module_adapter/module/generic.c @@ -28,6 +28,15 @@ LOG_MODULE_DECLARE(module_adapter, CONFIG_SOF_LOG_LEVEL); +/* + * A single lock shared by all module resource pools. The resource API is only + * ever entered from supervisor context (the z_impl_* syscall bodies), so the + * lock does not need to live in the per-module, user-writable struct - a static + * kernel object serialises all pools. Contention is negligible because resource + * bookkeeping only happens at module setup/teardown, not on the processing path. + */ +static K_MUTEX_DEFINE(mod_res_lock); + int module_load_config(struct comp_dev *dev, const void *cfg, size_t size) { int ret; @@ -79,7 +88,6 @@ void mod_resource_init(struct processing_module *mod) struct module_resources *res = &mod->priv.resources; /* Init memory list */ - k_mutex_init(&res->lock); list_init(&res->objpool.list); res->objpool.heap = res->alloc->heap; res->objpool.vreg = res->alloc->vreg; @@ -179,18 +187,18 @@ void *z_impl_mod_balloc_align(struct processing_module *mod, size_t size, size_t struct module_resources *res = &mod->priv.resources; struct module_resource *container; - k_mutex_lock(&res->lock, K_FOREVER); + k_mutex_lock(&mod_res_lock, K_FOREVER); container = container_get(mod); if (!container) { - k_mutex_unlock(&res->lock); + k_mutex_unlock(&mod_res_lock); return NULL; } if (!size) { comp_err(mod->dev, "requested allocation of 0 bytes."); container_put(mod, container); - k_mutex_unlock(&res->lock); + k_mutex_unlock(&mod_res_lock); return NULL; } @@ -202,7 +210,7 @@ void *z_impl_mod_balloc_align(struct processing_module *mod, size_t size, size_t comp_err(mod->dev, "Failed to alloc %zu bytes %zu alignment for comp %#x.", size, alignment, dev_comp_id(mod->dev)); container_put(mod, container); - k_mutex_unlock(&res->lock); + k_mutex_unlock(&mod_res_lock); return NULL; } /* Store reference to allocated memory */ @@ -214,7 +222,7 @@ void *z_impl_mod_balloc_align(struct processing_module *mod, size_t size, size_t if (res->heap_usage > res->heap_high_water_mark) res->heap_high_water_mark = res->heap_usage; - k_mutex_unlock(&res->lock); + k_mutex_unlock(&mod_res_lock); return ptr; } EXPORT_SYMBOL(z_impl_mod_balloc_align); @@ -235,18 +243,18 @@ void *z_impl_mod_alloc_ext(struct processing_module *mod, uint32_t flags, size_t struct module_resources *res = &mod->priv.resources; struct module_resource *container; - k_mutex_lock(&res->lock, K_FOREVER); + k_mutex_lock(&mod_res_lock, K_FOREVER); container = container_get(mod); if (!container) { - k_mutex_unlock(&res->lock); + k_mutex_unlock(&mod_res_lock); return NULL; } if (!size) { comp_err(mod->dev, "requested allocation of 0 bytes."); container_put(mod, container); - k_mutex_unlock(&res->lock); + k_mutex_unlock(&mod_res_lock); return NULL; } @@ -257,7 +265,7 @@ void *z_impl_mod_alloc_ext(struct processing_module *mod, uint32_t flags, size_t comp_err(mod->dev, "Failed to alloc %zu bytes %zu alignment for comp %#x.", size, alignment, dev_comp_id(mod->dev)); container_put(mod, container); - k_mutex_unlock(&res->lock); + k_mutex_unlock(&mod_res_lock); return NULL; } /* Store reference to allocated memory */ @@ -269,7 +277,7 @@ void *z_impl_mod_alloc_ext(struct processing_module *mod, uint32_t flags, size_t if (res->heap_usage > res->heap_high_water_mark) res->heap_high_water_mark = res->heap_usage; - k_mutex_unlock(&res->lock); + k_mutex_unlock(&mod_res_lock); return ptr; } EXPORT_SYMBOL(z_impl_mod_alloc_ext); @@ -284,22 +292,21 @@ EXPORT_SYMBOL(z_impl_mod_alloc_ext); #if CONFIG_COMP_BLOB struct comp_data_blob_handler *z_impl_mod_data_blob_handler_new(struct processing_module *mod) { - struct module_resources *res = &mod->priv.resources; struct comp_data_blob_handler *bhp; struct module_resource *container; - k_mutex_lock(&res->lock, K_FOREVER); + k_mutex_lock(&mod_res_lock, K_FOREVER); container = container_get(mod); if (!container) { - k_mutex_unlock(&res->lock); + k_mutex_unlock(&mod_res_lock); return NULL; } bhp = comp_data_blob_handler_new_ext(mod->dev, false, NULL, NULL); if (!bhp) { container_put(mod, container); - k_mutex_unlock(&res->lock); + k_mutex_unlock(&mod_res_lock); return NULL; } @@ -307,7 +314,7 @@ struct comp_data_blob_handler *z_impl_mod_data_blob_handler_new(struct processin container->size = 0; container->type = MOD_RES_BLOB_HANDLER; - k_mutex_unlock(&res->lock); + k_mutex_unlock(&mod_res_lock); return bhp; } EXPORT_SYMBOL(z_impl_mod_data_blob_handler_new); @@ -328,18 +335,18 @@ const void *z_impl_mod_fast_get(struct processing_module *mod, const void * cons struct module_resource *container; const void *ptr; - k_mutex_lock(&res->lock, K_FOREVER); + k_mutex_lock(&mod_res_lock, K_FOREVER); container = container_get(mod); if (!container) { - k_mutex_unlock(&res->lock); + k_mutex_unlock(&mod_res_lock); return NULL; } ptr = fast_get(res->alloc, dram_ptr, size); if (!ptr) { container_put(mod, container); - k_mutex_unlock(&res->lock); + k_mutex_unlock(&mod_res_lock); return NULL; } @@ -347,7 +354,7 @@ const void *z_impl_mod_fast_get(struct processing_module *mod, const void * cons container->size = 0; container->type = MOD_RES_FAST_GET; - k_mutex_unlock(&res->lock); + k_mutex_unlock(&mod_res_lock); return ptr; } EXPORT_SYMBOL(z_impl_mod_fast_get); @@ -425,10 +432,10 @@ int z_impl_mod_free(struct processing_module *mod, const void *ptr) /* Find which container holds this memory */ struct mod_res_cb_arg cb_arg = {mod, ptr}; - k_mutex_lock(&res->lock, K_FOREVER); + k_mutex_lock(&mod_res_lock, K_FOREVER); int ret = objpool_iterate(&res->objpool, mod_res_free, &cb_arg); - k_mutex_unlock(&res->lock); + k_mutex_unlock(&mod_res_lock); if (ret < 0) comp_err(mod->dev, "error: could not find memory pointed by %p", ptr); @@ -501,6 +508,20 @@ int z_vrfy_mod_free(struct processing_module *mod, const void *ptr) } #include +void z_vrfy_mod_free_all(struct processing_module *mod) +{ + size_t h_size = 0; + uintptr_t h_start; + + K_OOPS(K_SYSCALL_MEMORY_WRITE(mod, sizeof(*mod))); + mod_heap_info(mod, &h_size, &h_start); + if (h_size) + K_OOPS(K_SYSCALL_MEMORY_WRITE(h_start, h_size)); + + z_impl_mod_free_all(mod); +} +#include + #if CONFIG_COMP_BLOB struct comp_data_blob_handler *z_vrfy_mod_data_blob_handler_new(struct processing_module *mod) { @@ -734,21 +755,22 @@ int module_reset(struct processing_module *mod) * * This function is called automatically when the module is unloaded. */ -void mod_free_all(struct processing_module *mod) +void z_impl_mod_free_all(struct processing_module *mod) { struct module_resources *res = &mod->priv.resources; /* Free all contents found in used containers */ struct mod_res_cb_arg cb_arg = {mod, NULL}; - k_mutex_lock(&res->lock, K_FOREVER); + k_mutex_lock(&mod_res_lock, K_FOREVER); objpool_iterate(&res->objpool, mod_res_free, &cb_arg); objpool_prune(&res->objpool); - k_mutex_unlock(&res->lock); + k_mutex_unlock(&mod_res_lock); /* Make sure resource lists and accounting are reset */ mod_resource_init(mod); } +EXPORT_SYMBOL(z_impl_mod_free_all); int module_free(struct processing_module *mod) { diff --git a/src/audio/pipeline/pipeline-graph.c b/src/audio/pipeline/pipeline-graph.c index 6e154dfbfba6..cb8b3860cecd 100644 --- a/src/audio/pipeline/pipeline-graph.c +++ b/src/audio/pipeline/pipeline-graph.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -44,10 +45,20 @@ DECLARE_TR_CTX(pipe_tr, SOF_UUID(pipe_uuid), LOG_LEVEL_INFO); /* lookup table to determine busy/free pipeline metadata objects */ struct pipeline_posn { bool posn_offset[PPL_POSN_OFFSETS]; /**< available offsets */ +#ifndef CONFIG_SOF_USERSPACE_LL struct k_spinlock lock; /**< lock mechanism */ +#endif }; /* the pipeline position lookup table */ -static SHARED_DATA struct pipeline_posn pipeline_posn_shared; +static APP_SYSUSER_BSS SHARED_DATA struct pipeline_posn pipeline_posn_shared; + +#ifdef CONFIG_SOF_USERSPACE_LL +/* Mutex pointer in user-accessible partition so user-space threads + * can read the pointer for syscalls. Kept outside the SHARED_DATA + * struct to avoid kernel object tracking issues. + */ +static APP_SYSUSER_BSS struct k_mutex *pipeline_posn_mutex; +#endif /** * \brief Retrieves pipeline position structure. @@ -55,9 +66,48 @@ static SHARED_DATA struct pipeline_posn pipeline_posn_shared; */ static inline struct pipeline_posn *pipeline_posn_get(void) { +#ifdef CONFIG_SOF_USERSPACE_LL + return &pipeline_posn_shared; +#else return sof_get()->pipeline_posn; +#endif +} + +/* + * Position table locking. User-space LL cannot use a spinlock (disabling + * interrupts is privileged), so it uses a mutex; the config split is kept + * here so the callers below stay identical for both configurations. + */ +#ifdef CONFIG_SOF_USERSPACE_LL +typedef int pipeline_posn_key_t; + +static inline pipeline_posn_key_t pipeline_posn_lock(struct pipeline_posn *posn) +{ + (void)posn; + k_mutex_lock(pipeline_posn_mutex, K_FOREVER); + return 0; } +static inline void pipeline_posn_unlock(struct pipeline_posn *posn, pipeline_posn_key_t key) +{ + (void)posn; + (void)key; + k_mutex_unlock(pipeline_posn_mutex); +} +#else +typedef k_spinlock_key_t pipeline_posn_key_t; + +static inline pipeline_posn_key_t pipeline_posn_lock(struct pipeline_posn *posn) +{ + return k_spin_lock(&posn->lock); +} + +static inline void pipeline_posn_unlock(struct pipeline_posn *posn, pipeline_posn_key_t key) +{ + k_spin_unlock(&posn->lock, key); +} +#endif + /** * \brief Retrieves first free pipeline position offset. * \param[in,out] posn_offset Pipeline position offset to be set. @@ -68,9 +118,7 @@ static inline int pipeline_posn_offset_get(uint32_t *posn_offset) struct pipeline_posn *pipeline_posn = pipeline_posn_get(); int ret = -EINVAL; uint32_t i; - k_spinlock_key_t key; - - key = k_spin_lock(&pipeline_posn->lock); + pipeline_posn_key_t key = pipeline_posn_lock(pipeline_posn); for (i = 0; i < PPL_POSN_OFFSETS; ++i) { if (!pipeline_posn->posn_offset[i]) { @@ -81,8 +129,7 @@ static inline int pipeline_posn_offset_get(uint32_t *posn_offset) } } - - k_spin_unlock(&pipeline_posn->lock, key); + pipeline_posn_unlock(pipeline_posn, key); return ret; } @@ -95,20 +142,34 @@ static inline void pipeline_posn_offset_put(uint32_t posn_offset) { struct pipeline_posn *pipeline_posn = pipeline_posn_get(); int i = posn_offset / sizeof(struct sof_ipc_stream_posn); - k_spinlock_key_t key; - - key = k_spin_lock(&pipeline_posn->lock); + pipeline_posn_key_t key = pipeline_posn_lock(pipeline_posn); pipeline_posn->posn_offset[i] = false; - k_spin_unlock(&pipeline_posn->lock, key); + pipeline_posn_unlock(pipeline_posn, key); } void pipeline_posn_init(struct sof *sof) { sof->pipeline_posn = &pipeline_posn_shared; +#ifdef CONFIG_SOF_USERSPACE_LL + pipeline_posn_mutex = k_object_alloc(K_OBJ_MUTEX); + if (!pipeline_posn_mutex) { + pipe_cl_err("pipeline posn mutex alloc failed"); + k_panic(); + } + k_mutex_init(pipeline_posn_mutex); +#else k_spinlock_init(&sof->pipeline_posn->lock); +#endif +} + +#ifdef CONFIG_SOF_USERSPACE_LL +void pipeline_posn_grant_access(struct k_thread *thread) +{ + k_thread_access_grant(thread, pipeline_posn_mutex); } +#endif /* create new pipeline - returns pipeline id or negative error */ struct pipeline *pipeline_new(struct k_heap *heap, uint32_t pipeline_id, uint32_t priority, @@ -140,12 +201,21 @@ struct pipeline *pipeline_new(struct k_heap *heap, uint32_t pipeline_id, uint32_ p->pipeline_id = pipeline_id; p->status = COMP_STATE_INIT; p->trigger.cmd = COMP_TRIGGER_NO_ACTION; + +#ifndef CONFIG_SOF_USERSPACE_LL + /* + * pipe_tr lives in the .trace_ctx section, which is not mapped into + * the sysuser partition, so it cannot be read from a user-mode thread. + * The copy is also unnecessary in that configuration: with Zephyr + * logging the pipe_*() macros use the global pipe_tr, not p->tctx. + */ ret = memcpy_s(&p->tctx, sizeof(struct tr_ctx), &pipe_tr, sizeof(struct tr_ctx)); if (ret < 0) { pipe_err(p, "failed to copy trace settings"); goto free; } +#endif ret = pipeline_posn_offset_get(&p->posn_offset); if (ret < 0) { diff --git a/src/include/sof/audio/module_adapter/module/generic.h b/src/include/sof/audio/module_adapter/module/generic.h index 6740593a7cf1..70fb602d1aca 100644 --- a/src/include/sof/audio/module_adapter/module/generic.h +++ b/src/include/sof/audio/module_adapter/module/generic.h @@ -125,7 +125,6 @@ struct module_param { * when the module unloads. */ struct module_resources { - struct k_mutex lock; struct objpool_head objpool; size_t heap_usage; size_t heap_high_water_mark; @@ -197,12 +196,15 @@ void mod_heap_info(struct processing_module *mod, size_t *size, uintptr_t *start __syscall void *mod_alloc_ext(struct processing_module *mod, uint32_t flags, size_t size, size_t alignment); __syscall int mod_free(struct processing_module *mod, const void *ptr); +__syscall void mod_free_all(struct processing_module *mod); #else void *z_impl_mod_alloc_ext(struct processing_module *mod, uint32_t flags, size_t size, size_t alignment); int z_impl_mod_free(struct processing_module *mod, const void *ptr); +void z_impl_mod_free_all(struct processing_module *mod); #define mod_alloc_ext z_impl_mod_alloc_ext #define mod_free z_impl_mod_free +#define mod_free_all z_impl_mod_free_all #endif /** @@ -259,7 +261,6 @@ const void *z_impl_mod_fast_get(struct processing_module *mod, const void * cons #endif void mod_fast_put(struct processing_module *mod, const void *sram_ptr); #endif -void mod_free_all(struct processing_module *mod); int module_prepare(struct processing_module *mod, struct sof_source **sources, int num_of_sources, struct sof_sink **sinks, int num_of_sinks); diff --git a/src/include/sof/audio/pipeline-trace.h b/src/include/sof/audio/pipeline-trace.h index d1f89583ad45..dbd5ae7f7c84 100644 --- a/src/include/sof/audio/pipeline-trace.h +++ b/src/include/sof/audio/pipeline-trace.h @@ -59,6 +59,10 @@ extern struct tr_ctx pipe_tr; #else +#if defined(__ZEPHYR__) && defined(CONFIG_SOF_USERSPACE_LL) +#error "Invalid build config: User-space cannot access trace context." +#endif + #define pipe_err(pipe_p, __e, ...) \ trace_dev_err(trace_pipe_get_tr_ctx, trace_pipe_get_id, \ trace_pipe_get_subid, pipe_p, __e, ##__VA_ARGS__) diff --git a/src/include/sof/audio/pipeline.h b/src/include/sof/audio/pipeline.h index 913a569c208c..ff456fbceb7d 100644 --- a/src/include/sof/audio/pipeline.h +++ b/src/include/sof/audio/pipeline.h @@ -206,6 +206,14 @@ int pipeline_complete(struct pipeline *p, struct comp_dev *source, */ void pipeline_posn_init(struct sof *sof); +#ifdef CONFIG_SOF_USERSPACE_LL +/** + * \brief Grants user-space thread access to pipeline position mutex. + * \param[in] thread Thread to grant access to. + */ +void pipeline_posn_grant_access(struct k_thread *thread); +#endif + /** * \brief Resets the pipeline and free runtime resources. * \param[in] p pipeline. diff --git a/src/include/sof/schedule/ll_schedule_domain.h b/src/include/sof/schedule/ll_schedule_domain.h index b4b58e6923e2..ffa954bf1e59 100644 --- a/src/include/sof/schedule/ll_schedule_domain.h +++ b/src/include/sof/schedule/ll_schedule_domain.h @@ -328,6 +328,7 @@ struct ll_schedule_domain *zephyr_domain_init(int clk); #define timer_domain_init(timer, clk) zephyr_domain_init(clk) #ifdef CONFIG_SOF_USERSPACE_LL struct k_thread *zephyr_domain_thread_tid(struct ll_schedule_domain *domain); +struct k_thread *zephyr_domain_thread_tid_for_core(struct ll_schedule_domain *domain, int core); struct k_mem_domain *zephyr_ll_mem_domain(void); #endif /* CONFIG_SOF_USERSPACE_LL */ #ifdef CONFIG_SOF_FULL_ZEPHYR_APPLICATION diff --git a/src/init/init.c b/src/init/init.c index 7976e2eb673e..5990cfebc2dc 100644 --- a/src/init/init.c +++ b/src/init/init.c @@ -32,6 +32,7 @@ #include #include #include +#include #include #if CONFIG_IPC_MAJOR_4 #include @@ -232,6 +233,11 @@ __cold static int primary_core_init(int argc, char *argv[], struct sof *sof) zephyr_ll_user_resources_init(); #endif + /* init pipeline position offsets - must be before platform_init() + * which calls ipc_init() -> ipc_user_init() that needs the posn mutex. + */ + pipeline_posn_init(sof); + /* init the platform */ if (platform_init(sof) < 0) sof_panic(SOF_IPC_PANIC_PLATFORM); diff --git a/src/ipc/ipc-common.c b/src/ipc/ipc-common.c index afc8fe45de05..95b4bf28485d 100644 --- a/src/ipc/ipc-common.c +++ b/src/ipc/ipc-common.c @@ -466,6 +466,7 @@ __cold static void ipc_user_init(void) sof_panic(SOF_IPC_PANIC_IPC); } user_ll_grant_access(&ipc_user_thread, PLATFORM_PRIMARY_CORE_ID); + pipeline_posn_grant_access(&ipc_user_thread); k_mem_domain_add_thread(zephyr_ll_mem_domain(), &ipc_user_thread); k_thread_cpu_pin(&ipc_user_thread, PLATFORM_PRIMARY_CORE_ID); diff --git a/src/ipc/ipc4/dai.c b/src/ipc/ipc4/dai.c index 1540d95cf0f0..ee5821ad31c0 100644 --- a/src/ipc/ipc4/dai.c +++ b/src/ipc/ipc4/dai.c @@ -425,7 +425,7 @@ int dai_common_position(struct dai_data *dd, struct comp_dev *dev, platform_dai_wallclock(dev, &dd->wallclock); posn->wallclock = dd->wallclock; - ret = dma_get_status(dd->dma->z_dev, dd->chan_index, &status); + ret = sof_dma_get_status(dd->dma, dd->chan_index, &status); if (ret < 0) return ret; diff --git a/src/schedule/zephyr_domain.c b/src/schedule/zephyr_domain.c index 681b0c872f58..f6ddcf28165d 100644 --- a/src/schedule/zephyr_domain.c +++ b/src/schedule/zephyr_domain.c @@ -498,6 +498,23 @@ struct k_thread *zephyr_domain_thread_tid(struct ll_schedule_domain *domain) return dt->ll_thread; } +/* + * Return the LL scheduling thread for an explicitly given core. + * + * Unlike zephyr_domain_thread_tid(), this does not rely on cpu_get_id() and + * is therefore safe to call from a syscall context that may run on a core + * different from the task's target core. + */ +struct k_thread *zephyr_domain_thread_tid_for_core(struct ll_schedule_domain *domain, int core) +{ + struct zephyr_domain *zephyr_domain = ll_sch_domain_get_pdata(domain); + + if (core < 0 || core >= CONFIG_CORE_COUNT) + return NULL; + + return zephyr_domain->domain_thread[core].ll_thread; +} + #endif /* CONFIG_SOF_USERSPACE_LL */ #if CONFIG_CROSS_CORE_STREAM diff --git a/src/schedule/zephyr_ll.c b/src/schedule/zephyr_ll.c index fd70a4fe31df..ae612c1975e4 100644 --- a/src/schedule/zephyr_ll.c +++ b/src/schedule/zephyr_ll.c @@ -474,6 +474,26 @@ int z_impl_zephyr_ll_task_sem_alloc(struct task *task) k_sem_init(ts->sem, 0, 1); +#if CONFIG_SOF_USERSPACE_LL + /* + * The per-task semaphore is signalled from zephyr_ll_task_done(), + * which runs in the (unprivileged) LL scheduler thread when a task is + * freed while it is still running. k_object_alloc() only grants access + * to the calling thread (the IPC handler that creates the task), so the + * LL thread must be granted access explicitly, otherwise its + * k_sem_give() traps with a userspace permission fault. + */ + struct zephyr_ll *sch = task->sch ? task->sch->data : NULL; + + if (sch && sch->ll_domain) { + struct k_thread *ll_tid = + zephyr_domain_thread_tid_for_core(sch->ll_domain, task->core); + + if (ll_tid) + k_thread_access_grant(ll_tid, ts->sem); + } +#endif + ts->task = task; pdata->sem_p = ts->sem; /* List is protected by IPC serialization */ diff --git a/zephyr/CMakeLists.txt b/zephyr/CMakeLists.txt index 0ed4a13ee742..e0e7e8bfb302 100644 --- a/zephyr/CMakeLists.txt +++ b/zephyr/CMakeLists.txt @@ -620,6 +620,8 @@ zephyr_library_sources_ifdef(CONFIG_SHELL sof_shell.c ) +zephyr_syscall_header(include/sof/lib/cpu.h) +zephyr_library_sources_ifdef(CONFIG_USERSPACE syscall/cpu.c) zephyr_syscall_header(${SOF_SRC_PATH}/include/sof/audio/module_adapter/module/generic.h) zephyr_syscall_header(${SOF_SRC_PATH}/include/sof/lib/fast-get.h) zephyr_syscall_header(${SOF_SRC_PATH}/include/sof/ipc/ipc_reply.h) diff --git a/zephyr/include/sof/lib/cpu.h b/zephyr/include/sof/lib/cpu.h index c23405e85121..a47b93898973 100644 --- a/zephyr/include/sof/lib/cpu.h +++ b/zephyr/include/sof/lib/cpu.h @@ -40,13 +40,32 @@ void cpu_notify_state_exit(enum pm_state state); #endif /* CONFIG_PM */ +/* + * cpu_get_id() is exposed as a Zephyr system call so that user-mode + * threads (e.g. user-space LL pipelines) can query the current core + * id. The underlying arch_proc_id() reads a privileged special + * register (e.g. Xtensa PRID) which would fault if executed directly + * from user mode. In supervisor context the generated wrapper inlines + * the z_impl_cpu_get_id() body, so there is no overhead there. + */ +#if defined(CONFIG_SOF_FULL_ZEPHYR_APPLICATION) +__syscall int cpu_get_id(void); +#endif + /* let the compiler optimise when in single core mode */ #if CONFIG_MULTICORE && CONFIG_SMP +#if defined(CONFIG_SOF_FULL_ZEPHYR_APPLICATION) +static inline int z_impl_cpu_get_id(void) +{ + return arch_proc_id(); +} +#else static inline int cpu_get_id(void) { return arch_proc_id(); } +#endif static inline bool cpu_is_primary(int id) { @@ -73,7 +92,11 @@ int cpu_restore_secondary_cores(void); int cpu_secondary_cores_prepare_d0ix(void); #else +#if defined(CONFIG_SOF_FULL_ZEPHYR_APPLICATION) +static inline int z_impl_cpu_get_id(void) { return 0; }; +#else static inline int cpu_get_id(void) { return 0; }; +#endif static inline bool cpu_is_primary(int id) { return 1; }; @@ -93,6 +116,10 @@ static inline int cpu_secondary_cores_prepare_d0ix(void) { return 0; }; #endif /* CONFIG_MULTICORE && CONFIG_SMP */ +#if defined(CONFIG_SOF_FULL_ZEPHYR_APPLICATION) +#include +#endif + #endif #endif /* __SOF_LIB_CPU_H__ */ diff --git a/zephyr/syscall/cpu.c b/zephyr/syscall/cpu.c new file mode 100644 index 000000000000..ee1990db8fd4 --- /dev/null +++ b/zephyr/syscall/cpu.c @@ -0,0 +1,22 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Copyright(c) 2026 Intel Corporation. + +#include +#include +#include + +/** + * \brief Userspace verification wrapper for cpu_get_id(). + * + * The system call takes no arguments and passes no pointers, so no + * access validation is required; the call is simply forwarded to the + * implementation running in supervisor context. + * + * @return Id of the DSP core executing the call. + */ +static inline int z_vrfy_cpu_get_id(void) +{ + return z_impl_cpu_get_id(); +} +#include diff --git a/zephyr/wrapper.c b/zephyr/wrapper.c index 9bbb43f8a798..afdc5b54a2e9 100644 --- a/zephyr/wrapper.c +++ b/zephyr/wrapper.c @@ -177,9 +177,6 @@ int task_main_start(struct sof *sof) /* init default audio components */ sys_comp_init(sof); - /* init pipeline position offsets */ - pipeline_posn_init(sof); - return 0; }