From bb92e9b1bf82ccada186a5269ec3b5eeeab17bab Mon Sep 17 00:00:00 2001 From: Kai Vehmanen Date: Mon, 23 Mar 2026 20:38:09 +0200 Subject: [PATCH 001/108] (---section submitted PRs START) From 4983cddeac9307f4a87fbef269022c248beeaa10 Mon Sep 17 00:00:00 2001 From: Jyri Sarha Date: Wed, 24 Jun 2026 16:34:34 +0300 Subject: [PATCH 002/108] schedule: use k_thread_cpu_pin() for CPU affinity Replace the k_thread_cpu_mask_clear() + k_thread_cpu_mask_enable() sequence with a single k_thread_cpu_pin() call. The clear-then-enable pattern momentarily leaves the thread with a zero CPU mask, which is invalid under CONFIG_SCHED_CPU_MASK_PIN_ONLY and triggers an assertion after Zephyr commit 7798570a031d ("kernel: sched: enforce non-zero CPU mask invariant in PIN_ONLY mode"). k_thread_cpu_pin() atomically sets exactly one bit in the mask, avoiding the transient zero-mask state. Signed-off-by: Jyri Sarha --- src/schedule/zephyr_dma_domain.c | 3 +-- src/schedule/zephyr_domain.c | 6 ++---- zephyr/edf_schedule.c | 3 +-- 3 files changed, 4 insertions(+), 8 deletions(-) diff --git a/src/schedule/zephyr_dma_domain.c b/src/schedule/zephyr_dma_domain.c index 9bb76660ab29..50901ef7b2af 100644 --- a/src/schedule/zephyr_dma_domain.c +++ b/src/schedule/zephyr_dma_domain.c @@ -462,8 +462,7 @@ static int zephyr_dma_domain_register(struct ll_schedule_domain *domain, K_FOREVER); #ifdef CONFIG_SCHED_CPU_MASK - k_thread_cpu_mask_clear(thread); - k_thread_cpu_mask_enable(thread, core); + k_thread_cpu_pin(thread, core); #endif k_thread_name_set(thread, thread_name); diff --git a/src/schedule/zephyr_domain.c b/src/schedule/zephyr_domain.c index 3837f0b135c6..c878e1972c49 100644 --- a/src/schedule/zephyr_domain.c +++ b/src/schedule/zephyr_domain.c @@ -218,8 +218,7 @@ static int zephyr_domain_register(struct ll_schedule_domain *domain, NULL, CONFIG_LL_THREAD_PRIORITY, 0, K_FOREVER); #ifdef CONFIG_SCHED_CPU_MASK - k_thread_cpu_mask_clear(thread); - k_thread_cpu_mask_enable(thread, core); + k_thread_cpu_pin(thread, core); #endif k_thread_name_set(thread, thread_name); @@ -345,8 +344,7 @@ static int zephyr_domain_thread_init(struct ll_schedule_domain *domain, K_USER, K_FOREVER); #ifdef CONFIG_SCHED_CPU_MASK - k_thread_cpu_mask_clear(thread); - k_thread_cpu_mask_enable(thread, core); + k_thread_cpu_pin(thread, core); #endif k_thread_name_set(thread, thread_name); diff --git a/zephyr/edf_schedule.c b/zephyr/edf_schedule.c index a14ed31d060d..c15f36b7be58 100644 --- a/zephyr/edf_schedule.c +++ b/zephyr/edf_schedule.c @@ -117,8 +117,7 @@ __cold int scheduler_init_edf(void) k_thread_heap_assign(thread, sof_sys_heap_get()); #ifdef CONFIG_SCHED_CPU_MASK - k_thread_cpu_mask_clear(thread); - k_thread_cpu_mask_enable(thread, PLATFORM_PRIMARY_CORE_ID); + k_thread_cpu_pin(thread, PLATFORM_PRIMARY_CORE_ID); #endif k_thread_name_set(thread, "edf_workq"); From fcd70109f8aa816e9c7b6baadf3b04b8608e6b58 Mon Sep 17 00:00:00 2001 From: Kai Vehmanen Date: Mon, 23 Mar 2026 20:38:09 +0200 Subject: [PATCH 003/108] (---section submitted PRs STOP) From a90f9f5569512ef844bd527ec4c71f219c5c370b Mon Sep 17 00:00:00 2001 From: Kai Vehmanen Date: Thu, 19 Feb 2026 14:51:22 +0200 Subject: [PATCH 004/108] (---section dai-zephyr START) From bb29ab29bc489c3309959516465f21291bc56136 Mon Sep 17 00:00:00 2001 From: Kai Vehmanen Date: Wed, 10 Jun 2026 12:26:54 +0300 Subject: [PATCH 005/108] WIP: audio: dai-zephyr: temporarily disable spinlock for dai properties The real fix is to remove the locking around dai_get_properties() altogether, but this depends on fixes in Zephyr DAI drivers. To unblock user-space work, remove the calls to spinlocks for now. This opens up possibility to hit issues with concurrent playback and capture cases on multiple cores, so this commit remains a WIP until fixes in Zephyr drivers land. Signed-off-by: Kai Vehmanen --- src/audio/dai-zephyr.c | 29 ++++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/src/audio/dai-zephyr.c b/src/audio/dai-zephyr.c index e0295526d054..73e99aca4f72 100644 --- a/src/audio/dai-zephyr.c +++ b/src/audio/dai-zephyr.c @@ -238,12 +238,18 @@ __cold int dai_set_config(struct dai *dai, struct ipc_config_dai *common_config, /* called from ipc/ipc3/dai.c */ int dai_get_handshake(struct dai *dai, int direction, int stream_id) { + /* Remove for all builds once Zephyr drivers fixed to make + * get_properties() safe. */ +#ifndef CONFIG_SOF_USERSPACE_LL k_spinlock_key_t key = k_spin_lock(&dai->lock); +#endif const struct dai_properties *props = dai_get_properties(dai->dev, direction, stream_id); int hs_id = props->dma_hs_id; +#ifndef CONFIG_SOF_USERSPACE_LL k_spin_unlock(&dai->lock, key); +#endif return hs_id; } @@ -252,40 +258,52 @@ int dai_get_handshake(struct dai *dai, int direction, int stream_id) int dai_get_fifo_depth(struct dai *dai, int direction) { const struct dai_properties *props; +#ifndef CONFIG_SOF_USERSPACE_LL k_spinlock_key_t key; +#endif int fifo_depth; if (!dai) return 0; +#ifndef CONFIG_SOF_USERSPACE_LL key = k_spin_lock(&dai->lock); +#endif props = dai_get_properties(dai->dev, direction, 0); fifo_depth = props->fifo_depth; +#ifndef CONFIG_SOF_USERSPACE_LL k_spin_unlock(&dai->lock, key); - +#endif return fifo_depth; } int dai_get_stream_id(struct dai *dai, int direction) { +#ifndef CONFIG_SOF_USERSPACE_LL k_spinlock_key_t key = k_spin_lock(&dai->lock); +#endif const struct dai_properties *props = dai_get_properties(dai->dev, direction, 0); int stream_id = props->stream_id; +#ifndef CONFIG_SOF_USERSPACE_LL k_spin_unlock(&dai->lock, key); +#endif return stream_id; } static int dai_get_fifo(struct dai *dai, int direction, int stream_id) { +#ifndef CONFIG_SOF_USERSPACE_LL k_spinlock_key_t key = k_spin_lock(&dai->lock); +#endif const struct dai_properties *props = dai_get_properties(dai->dev, direction, stream_id); int fifo_address = props->fifo_address; +#ifndef CONFIG_SOF_USERSPACE_LL k_spin_unlock(&dai->lock, key); - +#endif return fifo_address; } @@ -1983,17 +2001,22 @@ static int dai_ts_stop_op(struct comp_dev *dev) uint32_t dai_get_init_delay_ms(struct dai *dai) { const struct dai_properties *props; +#ifndef CONFIG_SOF_USERSPACE_LL k_spinlock_key_t key; +#endif uint32_t init_delay; if (!dai) return 0; +#ifndef CONFIG_SOF_USERSPACE_LL key = k_spin_lock(&dai->lock); +#endif props = dai_get_properties(dai->dev, 0, 0); init_delay = props->reg_init_delay; +#ifndef CONFIG_SOF_USERSPACE_LL k_spin_unlock(&dai->lock, key); - +#endif return init_delay; } From bb1a7edfbb6a8a0be026ec550a2d6065e0abf7b7 Mon Sep 17 00:00:00 2001 From: Kai Vehmanen Date: Wed, 18 Feb 2026 15:49:22 +0200 Subject: [PATCH 006/108] audio: dai-zephyr: migrate to use dai_get_properties_copy() Modify code to allocate DAI properties object on stack and use dai_get_properties_copy(). This is required when DAI code is run in user-space and a syscall is needed to talk to the DAI driver. It's not possible to return a pointer to kernel memory, so instead data needs to be copied to caller stack. Signed-off-by: Kai Vehmanen --- src/audio/dai-zephyr.c | 69 ++++++++++++++++++++++++------------------ 1 file changed, 40 insertions(+), 29 deletions(-) diff --git a/src/audio/dai-zephyr.c b/src/audio/dai-zephyr.c index 73e99aca4f72..7e83d8e9d769 100644 --- a/src/audio/dai-zephyr.c +++ b/src/audio/dai-zephyr.c @@ -238,73 +238,83 @@ __cold int dai_set_config(struct dai *dai, struct ipc_config_dai *common_config, /* called from ipc/ipc3/dai.c */ int dai_get_handshake(struct dai *dai, int direction, int stream_id) { + struct dai_properties props; + int ret; + /* Remove for all builds once Zephyr drivers fixed to make * get_properties() safe. */ #ifndef CONFIG_SOF_USERSPACE_LL k_spinlock_key_t key = k_spin_lock(&dai->lock); #endif - const struct dai_properties *props = dai_get_properties(dai->dev, direction, - stream_id); - int hs_id = props->dma_hs_id; - + ret = dai_get_properties_copy(dai->dev, direction, stream_id, &props); #ifndef CONFIG_SOF_USERSPACE_LL k_spin_unlock(&dai->lock, key); #endif + if (ret < 0) + return ret; - return hs_id; + return props.dma_hs_id; } /* called from ipc/ipc3/dai.c and ipc/ipc4/dai.c */ int dai_get_fifo_depth(struct dai *dai, int direction) { - const struct dai_properties *props; -#ifndef CONFIG_SOF_USERSPACE_LL - k_spinlock_key_t key; -#endif - int fifo_depth; + struct dai_properties props; + int ret; if (!dai) return 0; #ifndef CONFIG_SOF_USERSPACE_LL - key = k_spin_lock(&dai->lock); + k_spinlock_key_t key = k_spin_lock(&dai->lock); #endif - props = dai_get_properties(dai->dev, direction, 0); - fifo_depth = props->fifo_depth; + + ret = dai_get_properties_copy(dai->dev, direction, 0, &props); #ifndef CONFIG_SOF_USERSPACE_LL k_spin_unlock(&dai->lock, key); #endif - return fifo_depth; + if (ret < 0) + return 0; + + return props.fifo_depth; } int dai_get_stream_id(struct dai *dai, int direction) { + struct dai_properties props; + int ret; + #ifndef CONFIG_SOF_USERSPACE_LL k_spinlock_key_t key = k_spin_lock(&dai->lock); #endif - const struct dai_properties *props = dai_get_properties(dai->dev, direction, 0); - int stream_id = props->stream_id; + ret = dai_get_properties_copy(dai->dev, direction, 0, &props); #ifndef CONFIG_SOF_USERSPACE_LL k_spin_unlock(&dai->lock, key); #endif + if (ret < 0) + return ret; - return stream_id; + return props.stream_id; } static int dai_get_fifo(struct dai *dai, int direction, int stream_id) { + struct dai_properties props; + int ret; + #ifndef CONFIG_SOF_USERSPACE_LL k_spinlock_key_t key = k_spin_lock(&dai->lock); #endif - const struct dai_properties *props = dai_get_properties(dai->dev, direction, - stream_id); - int fifo_address = props->fifo_address; + ret = dai_get_properties_copy(dai->dev, direction, stream_id, &props); #ifndef CONFIG_SOF_USERSPACE_LL k_spin_unlock(&dai->lock, key); #endif - return fifo_address; + if (ret < 0) + return ret; + + return props.fifo_address; } /* this is called by DMA driver every time descriptor has completed */ @@ -2000,23 +2010,24 @@ static int dai_ts_stop_op(struct comp_dev *dev) uint32_t dai_get_init_delay_ms(struct dai *dai) { - const struct dai_properties *props; -#ifndef CONFIG_SOF_USERSPACE_LL - k_spinlock_key_t key; -#endif - uint32_t init_delay; + struct dai_properties props; + uint32_t init_delay = 0; + int ret; if (!dai) return 0; #ifndef CONFIG_SOF_USERSPACE_LL - key = k_spin_lock(&dai->lock); + k_spinlock_key_t key = k_spin_lock(&dai->lock); #endif - props = dai_get_properties(dai->dev, 0, 0); - init_delay = props->reg_init_delay; + + ret = dai_get_properties_copy(dai->dev, 0, 0, &props); + if (!ret) + init_delay = props.reg_init_delay; #ifndef CONFIG_SOF_USERSPACE_LL k_spin_unlock(&dai->lock, key); #endif + return init_delay; } From 249dd70a174d3578d9b1d111df70eba3dcca7100 Mon Sep 17 00:00:00 2001 From: Kai Vehmanen Date: Thu, 19 Feb 2026 15:05:22 +0200 Subject: [PATCH 007/108] (---section dai-zephyr STOP) From 46568f7e7c39431df244cb9963cb55f68dce3e11 Mon Sep 17 00:00:00 2001 From: Kai Vehmanen Date: Tue, 3 Mar 2026 15:54:37 +0200 Subject: [PATCH 008/108] (---section schduler changes START) From 4d36a44a87ce500228d851e76bc80069bcaa0f03 Mon Sep 17 00:00:00 2001 From: Kai Vehmanen Date: Fri, 13 Feb 2026 18:48:43 +0200 Subject: [PATCH 009/108] schedule: zephyr_ll: convert pdata->sem into a dynamic object Turn the pdata->sem into a dynamic object in userspace LL builds, implemented with Zephyr k_sem. Add POSIX no-op stubs for sys_sem to maintain testbench build compatibility. Keep statically allocated semaphore for kernel LL builds. Signed-off-by: Kai Vehmanen --- posix/include/rtos/mutex.h | 21 +++++++++++++++++++++ src/schedule/zephyr_ll.c | 9 +++++---- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/posix/include/rtos/mutex.h b/posix/include/rtos/mutex.h index 19b360bdaea5..3bd01342ced5 100644 --- a/posix/include/rtos/mutex.h +++ b/posix/include/rtos/mutex.h @@ -62,4 +62,25 @@ static inline int sys_mutex_unlock(struct sys_mutex *mutex) return 0; } +/* provide a no-op implementation for zephyr/sys/sem.h */ + +struct sys_sem { +}; + +static inline int sys_sem_init(struct sys_sem *sem, unsigned int initial_count, + unsigned int limit) +{ + return 0; +} + +static inline int sys_sem_give(struct sys_sem *sem) +{ + return 0; +} + +static inline int sys_sem_take(struct sys_sem *sem, k_timeout_t timeout) +{ + return 0; +} + #endif diff --git a/src/schedule/zephyr_ll.c b/src/schedule/zephyr_ll.c index 33d2d76014bd..2c18226e6dd7 100644 --- a/src/schedule/zephyr_ll.c +++ b/src/schedule/zephyr_ll.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include @@ -41,7 +42,7 @@ struct zephyr_ll { struct zephyr_ll_pdata { bool run; bool freeing; - struct k_sem sem; + struct sys_sem sem; }; #if CONFIG_SOF_USERSPACE_LL @@ -136,7 +137,7 @@ static void zephyr_ll_task_done(struct zephyr_ll *sch, * zephyr_ll_task_free() is trying to free this task. Complete * it and signal the semaphore to let the function proceed */ - k_sem_give(&pdata->sem); + sys_sem_give(&pdata->sem); tr_info(&ll_tr, "task complete %p %pU", task, task->uid); tr_info(&ll_tr, "num_tasks %d total_num_tasks %ld", @@ -505,7 +506,7 @@ static int zephyr_ll_task_free(void *data, struct task *task) if (must_wait) /* Wait for up to 100 periods */ - k_sem_take(&pdata->sem, K_USEC(LL_TIMER_PERIOD_US * 100)); + sys_sem_take(&pdata->sem, K_USEC(LL_TIMER_PERIOD_US * 100)); /* Protect against racing with schedule_task() */ zephyr_ll_lock(sch, &flags); @@ -691,7 +692,7 @@ int zephyr_ll_task_init(struct task *task, memset(pdata, 0, sizeof(*pdata)); - k_sem_init(&pdata->sem, 0, 1); + sys_sem_init(&pdata->sem, 0, 1); task->priv_data = pdata; From 8edbd21367712a1894d03aad7706af5b000e54b4 Mon Sep 17 00:00:00 2001 From: Kai Vehmanen Date: Wed, 10 Jun 2026 18:18:54 +0300 Subject: [PATCH 010/108] schedule: add scheduler_get_data_for_core() Add function scheduler_get_data_for_core() to look up scheduler data for a particular type of scheduler. This variant allows to pass the core number as an argument, so it can be called from unprivileged code. Signed-off-by: Kai Vehmanen --- src/include/sof/schedule/schedule.h | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/src/include/sof/schedule/schedule.h b/src/include/sof/schedule/schedule.h index 03ce567dc9ea..940df24fc310 100644 --- a/src/include/sof/schedule/schedule.h +++ b/src/include/sof/schedule/schedule.h @@ -206,6 +206,31 @@ static inline void *scheduler_get_data(uint16_t type) return NULL; } +#if CONFIG_SOF_USERSPACE_LL +/** + * Retrieves scheduler's data for a specific core. + * @param type SOF_SCHEDULE_ type. + * @param core Core ID to get scheduler data for. + * @return Pointer to scheduler's data. + * + * Safe to call from user-space context — does not use cpu_get_id(). + */ +static inline void *scheduler_get_data_for_core(uint16_t type, int core) +{ + struct schedulers *schedulers = *arch_user_schedulers_get_for_core(core); + struct schedule_data *sch; + struct list_item *slist; + + list_for_item(slist, &schedulers->list) { + sch = container_of(slist, struct schedule_data, list); + if (type == sch->type) + return sch->data; + } + + return NULL; +} +#endif + /** See scheduler_ops::schedule_task_running */ static inline int schedule_task_running(struct task *task) { From 05821fc1ad2684692e9af453adc3d21b681a9601 Mon Sep 17 00:00:00 2001 From: Kai Vehmanen Date: Fri, 26 Jun 2026 18:41:43 +0300 Subject: [PATCH 011/108] schedule: zephyr_ll: add user_ll_grant_access() Add function user_ll_grant_access() to allow other threads to access the scheduler mutex. This is needed if work is submitted from other threads to the scheduler. Signed-off-by: Kai Vehmanen --- src/include/sof/schedule/ll_schedule_domain.h | 1 + src/schedule/zephyr_ll.c | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/src/include/sof/schedule/ll_schedule_domain.h b/src/include/sof/schedule/ll_schedule_domain.h index 9b0fd46b371e..f356566ca0aa 100644 --- a/src/include/sof/schedule/ll_schedule_domain.h +++ b/src/include/sof/schedule/ll_schedule_domain.h @@ -120,6 +120,7 @@ struct task *zephyr_ll_task_alloc(void); struct k_heap *zephyr_ll_user_heap(void); bool zephyr_ll_user_heap_verify(struct k_heap *heap); void zephyr_ll_user_resources_init(void); +void user_ll_grant_access(struct k_thread *thread, int core); void user_ll_lock_sched(int core); void user_ll_unlock_sched(int core); #ifdef CONFIG_ASSERT diff --git a/src/schedule/zephyr_ll.c b/src/schedule/zephyr_ll.c index 2c18226e6dd7..1a85667f5430 100644 --- a/src/schedule/zephyr_ll.c +++ b/src/schedule/zephyr_ll.c @@ -626,6 +626,13 @@ struct task *zephyr_ll_task_alloc(void) return task; } +void user_ll_grant_access(struct k_thread *thread, int core) +{ + assert(core < CONFIG_CORE_COUNT && zephyr_ll_locks[core] != NULL); + + k_thread_access_grant(thread, zephyr_ll_locks[core]); +} + /** * Lock the LL scheduler to prevent it from processing tasks. * From f5dce6169178250ecc6f9fddb1674be4f58f8406 Mon Sep 17 00:00:00 2001 From: Kai Vehmanen Date: Tue, 24 Mar 2026 18:19:26 +0200 Subject: [PATCH 012/108] schedule: zephyr_domain: use a different thread name for user LL When LL scheduler is run in user-space, use a different Zephyr thread name. Signed-off-by: Kai Vehmanen --- src/schedule/zephyr_domain.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/schedule/zephyr_domain.c b/src/schedule/zephyr_domain.c index c878e1972c49..89defd4f49f9 100644 --- a/src/schedule/zephyr_domain.c +++ b/src/schedule/zephyr_domain.c @@ -300,7 +300,7 @@ static int zephyr_domain_thread_init(struct ll_schedule_domain *domain, { struct zephyr_domain *zephyr_domain = ll_sch_domain_get_pdata(domain); struct zephyr_domain_thread *dt; - char thread_name[] = "ll_thread0"; + char thread_name[] = "userll_thread0"; k_tid_t thread; int core = task->core; From f8dc5a9cdcbb12da720814dd024d137f73e2df50 Mon Sep 17 00:00:00 2001 From: Kai Vehmanen Date: Fri, 20 Mar 2026 21:40:57 +0200 Subject: [PATCH 013/108] coherent: disable core debug checks for user-space builds The COHERENT_CHECK_NONSHARED_CORES debug macros call cpu_get_id() which invokes arch_proc_id() - a privileged hardware register read that faults in user-space context. Disable the entire debug block at compile time when CONFIG_SOF_USERSPACE_LL is enabled. This also fixes the same latent issue in CORE_CHECK_STRUCT and CORE_CHECK_STRUCT_INIT. Signed-off-by: Kai Vehmanen --- src/include/sof/coherent.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/include/sof/coherent.h b/src/include/sof/coherent.h index 172e45b4ed92..ba6b8d8c7e52 100644 --- a/src/include/sof/coherent.h +++ b/src/include/sof/coherent.h @@ -86,8 +86,8 @@ STATIC_ASSERT(sizeof(struct coherent) <= DCACHE_LINE_SIZE, DCACHE_LINE_SIZE_too #define ADDR_IS_COHERENT(_c) #endif -/* debug sharing amongst cores */ -#ifdef COHERENT_CHECK_NONSHARED_CORES +/* debug sharing amongst cores - not available in user-space builds */ +#if defined(COHERENT_CHECK_NONSHARED_CORES) && !defined(CONFIG_SOF_USERSPACE_LL) #define CORE_CHECK_STRUCT_FIELD uint32_t __core; bool __is_shared #define CORE_CHECK_STRUCT_INIT(_c, is_shared) { (_c)->__core = cpu_get_id(); \ From 9dd3c9831a7563cd27e30154e4cc17a6fb3e84ee Mon Sep 17 00:00:00 2001 From: Kai Vehmanen Date: Wed, 10 Jun 2026 18:26:34 +0300 Subject: [PATCH 014/108] schedule: zephyr_ll: make zephyr_ll_assert_core() user-space safe Modify the checks in zephyr_ll_assert_core() to make them safe to call from user-space LL threads. Signed-off-by: Kai Vehmanen --- src/schedule/zephyr_ll.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/schedule/zephyr_ll.c b/src/schedule/zephyr_ll.c index 1a85667f5430..7e914a21c322 100644 --- a/src/schedule/zephyr_ll.c +++ b/src/schedule/zephyr_ll.c @@ -113,8 +113,15 @@ static void zephyr_ll_unlock(struct zephyr_ll *sch, uint32_t *flags) static void zephyr_ll_assert_core(const struct zephyr_ll *sch) { - assert(CONFIG_CORE_COUNT == 1 || IS_ENABLED(CONFIG_SOF_USERSPACE_LL) || - sch->core == cpu_get_id()); +#if CONFIG_SOF_USERSPACE_LL + /* In user-space mode, cpu_get_id() is not available. + * Core correctness is ensured by task->core routing in + * schedule.h and verified at task schedule time. + */ + (void)sch; +#else + assert(CONFIG_CORE_COUNT == 1 || sch->core == cpu_get_id()); +#endif } /* Locking: caller should hold the domain lock */ From 6fa83f89b5a6e4618ad0e377901113aed5972971 Mon Sep 17 00:00:00 2001 From: Kai Vehmanen Date: Tue, 9 Jun 2026 22:01:17 +0300 Subject: [PATCH 015/108] schedule: zephyr_ll: use correct method to get scheduler data Needs more review, but makes the tests pass again. Signed-off-by: Kai Vehmanen --- src/schedule/zephyr_ll.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/schedule/zephyr_ll.c b/src/schedule/zephyr_ll.c index 7e914a21c322..37bcbce60326 100644 --- a/src/schedule/zephyr_ll.c +++ b/src/schedule/zephyr_ll.c @@ -772,7 +772,7 @@ void scheduler_get_task_info_ll(struct scheduler_props *scheduler_props, uint32_t flags; scheduler_props->processing_domain = COMP_PROCESSING_DOMAIN_LL; - struct zephyr_ll *ll_sch = (struct zephyr_ll *)scheduler_get_data(SOF_SCHEDULE_LL_TIMER); + struct zephyr_ll *ll_sch = (struct zephyr_ll *)scheduler_get_data_for_core(SOF_SCHEDULE_LL_TIMER, 0); zephyr_ll_lock(ll_sch, &flags); scheduler_get_task_info(scheduler_props, data_off_size, &ll_sch->tasks); @@ -782,7 +782,7 @@ void scheduler_get_task_info_ll(struct scheduler_props *scheduler_props, /* Return a pointer to the LL scheduler timer domain */ struct ll_schedule_domain *zephyr_ll_domain(void) { - struct zephyr_ll *ll_sch = scheduler_get_data(SOF_SCHEDULE_LL_TIMER); + struct zephyr_ll *ll_sch = scheduler_get_data_for_core(SOF_SCHEDULE_LL_TIMER, 0); return ll_sch->ll_domain; } From 109529caa54645ffaf8bc6c78cf572ec85212ffe Mon Sep 17 00:00:00 2001 From: Kai Vehmanen Date: Tue, 16 Jun 2026 20:45:41 +0300 Subject: [PATCH 016/108] schedule: zephyr_dp_sched_app: fix build with no thread names Build fails when building with CONFIG_THREAD_NAME disabled. Fix the issue by conditional compilation of code using CONFIG_THREAD_MAX_NAME_LEN. Signed-off-by: Kai Vehmanen --- src/schedule/zephyr_dp_schedule_application.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/schedule/zephyr_dp_schedule_application.c b/src/schedule/zephyr_dp_schedule_application.c index 791e16ab190d..e00972bd467b 100644 --- a/src/schedule/zephyr_dp_schedule_application.c +++ b/src/schedule/zephyr_dp_schedule_application.c @@ -414,11 +414,13 @@ void scheduler_dp_internal_free(struct task *task) #ifdef CONFIG_THREAD_NAME static void scheduler_dp_thread_name_set(k_tid_t thread_id, struct processing_module *mod) { +#ifdef CONFIG_THREAD_NAME char name[CONFIG_THREAD_MAX_NAME_LEN]; snprintf(name, sizeof(name), "DP:%#x", mod->dev->ipc_config.id); k_thread_name_set(thread_id, name); +#endif } #else /* k_thread_name_set() is a no-op so skip constructing a thread name */ From cc42a761d7a8c37ca4012b2e22cf129a2390ee8f Mon Sep 17 00:00:00 2001 From: Kai Vehmanen Date: Tue, 3 Mar 2026 15:54:51 +0200 Subject: [PATCH 017/108] (---section schduler changes END) From 4b3c1e7cdceaeb6988e6c4d44d7055286777b920 Mon Sep 17 00:00:00 2001 From: Kai Vehmanen Date: Wed, 25 Mar 2026 19:31:48 +0200 Subject: [PATCH 018/108] (---section audio-user PRs START) From 830afaf5e8f656de2d538ccb0ef3cf5568cae3d9 Mon Sep 17 00:00:00 2001 From: Kai Vehmanen Date: Wed, 10 Jun 2026 18:11:45 +0300 Subject: [PATCH 019/108] audio: pipeline-schedule: make pipeline_schedule_triggered() user safe Add support to run pipeline_schedule_triggered() in user-space. Use the user_ll_lock/unlock_sched() interface if building with CONFIG_SOF_USERSPACE_LL. Signed-off-by: Kai Vehmanen --- src/audio/pipeline/pipeline-schedule.c | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/src/audio/pipeline/pipeline-schedule.c b/src/audio/pipeline/pipeline-schedule.c index cb4ec8fd3c62..3612b63df08a 100644 --- a/src/audio/pipeline/pipeline-schedule.c +++ b/src/audio/pipeline/pipeline-schedule.c @@ -282,6 +282,18 @@ void pipeline_schedule_triggered(struct pipeline_walk_context *ctx, struct pipeline_data *ppl_data = ctx->comp_data; struct list_item *tlist; struct pipeline *p; + +#ifdef CONFIG_SOF_USERSPACE_LL + /* + * In user-space irq_local_disable() is not available. Use the LL + * scheduler mutex to prevent the scheduler from processing tasks + * while pipeline state is being updated. The k_mutex is re-entrant + * so schedule_task() calls inside the critical section are safe. + */ + int sched_core = ppl_data->start->ipc_config.core; + + user_ll_lock_sched(sched_core); +#else uint32_t flags; #ifdef CONFIG_IPC_MAJOR_4 @@ -300,6 +312,7 @@ void pipeline_schedule_triggered(struct pipeline_walk_context *ctx, * immediately before all pipelines achieved a consistent state. */ irq_local_disable(flags); +#endif switch (cmd) { case COMP_TRIGGER_PAUSE: @@ -355,8 +368,11 @@ void pipeline_schedule_triggered(struct pipeline_walk_context *ctx, p->xrun_bytes = 1; } } - +#ifdef CONFIG_SOF_USERSPACE_LL + user_ll_unlock_sched(sched_core); +#else irq_local_enable(flags); +#endif } int pipeline_comp_ll_task_init(struct pipeline *p) From c19f6411806145109a48869ade3e837b967b6d93 Mon Sep 17 00:00:00 2001 From: Kai Vehmanen Date: Wed, 10 Jun 2026 18:19:30 +0300 Subject: [PATCH 020/108] init: secondary_core_init: set up user-space LL scheduler context In user-space LL builds the low-latency scheduler runs its work in a dedicated privileged domain thread, created together with its timer and access grants by scheduler_init_context() (zephyr_ll_init_context() -> domain_thread_init()). This context is per-core and must exist on every core that runs LL tasks. So far it was only established for the primary core, so LL tasks could not be scheduled on secondary cores when CONFIG_SOF_USERSPACE_LL is enabled. Allocate an LL task in secondary_core_init() and run scheduler_init_context() on it, giving each secondary core its own LL domain thread. A dedicated sec_core_init UUID is registered for the task. The whole block is compiled in only for CONFIG_SOF_USERSPACE_LL. Signed-off-by: Kai Vehmanen --- src/init/init.c | 16 ++++++++++++++++ uuid-registry.txt | 1 + 2 files changed, 17 insertions(+) diff --git a/src/init/init.c b/src/init/init.c index c76b4842d88e..55e7c7e31ba9 100644 --- a/src/init/init.c +++ b/src/init/init.c @@ -47,6 +47,10 @@ LOG_MODULE_REGISTER(init, CONFIG_SOF_LOG_LEVEL); +#if CONFIG_SOF_USERSPACE_LL +SOF_DEFINE_REG_UUID(sec_core_init); +#endif + /* main firmware context */ static struct sof sof; @@ -139,6 +143,18 @@ __cold int secondary_core_init(struct sof *sof) return err; #endif /* CONFIG_ZEPHYR_DP_SCHEDULER */ +#if CONFIG_SOF_USERSPACE_LL + /* Create domain thread for this secondary core's LL scheduler */ + { + struct task *task = zephyr_ll_task_alloc(); + + schedule_task_init_ll(task, SOF_UUID(sec_core_init_uuid), + SOF_SCHEDULE_LL_TIMER, + 0, NULL, NULL, cpu_get_id(), 0); + scheduler_init_context(task); + } +#endif + /* initialize IDC mechanism */ trace_point(TRACE_BOOT_PLATFORM_IDC); err = idc_init(); diff --git a/uuid-registry.txt b/uuid-registry.txt index e9e8f8e77876..b4d3f437e9b1 100644 --- a/uuid-registry.txt +++ b/uuid-registry.txt @@ -147,6 +147,7 @@ d7f6712d-131c-45a7-82ed6aa9dc2291ea pm_runtime 9302adf5-88be-4234-a0a7dca538ef81f4 sai 3dee06de-f25a-4e10-ae1fabc9573873ea schedule 70d223ef-2b91-4aac-b444d89a0db2793a sdma +bdcb1461-34f5-4047-b9cc70fdf8dfb234 sec_core_init 55a88ed5-3d18-46ca-88f10ee6eae9930f selector 32fe92c1-1e17-4fc2-9758c7f3542e980a selector4 cf90d851-68a2-4987-a2de85aed0c8531c sgen_mt8186 From 1398b35dac798d01453b64c84fe53945b365acbc Mon Sep 17 00:00:00 2001 From: Kai Vehmanen Date: Wed, 15 Apr 2026 13:52:05 +0300 Subject: [PATCH 021/108] audio: make comp_drivers_get() accessible from user-space Make comp_drivers_get() usable from user-space when SOF is built with CONFIG_SOF_USERSPACE_LL. Signed-off-by: Kai Vehmanen --- src/audio/component.c | 8 ++++++++ src/include/sof/audio/component_ext.h | 4 ++++ 2 files changed, 12 insertions(+) diff --git a/src/audio/component.c b/src/audio/component.c index 8a6d35776971..62cbdac689f7 100644 --- a/src/audio/component.c +++ b/src/audio/component.c @@ -38,6 +38,14 @@ LOG_MODULE_REGISTER(component, CONFIG_SOF_LOG_LEVEL); static APP_SYSUSER_BSS SHARED_DATA struct comp_driver_list cd; +#ifdef CONFIG_SOF_USERSPACE_LL +struct comp_driver_list *comp_drivers_get(void) +{ + return &cd; +} +EXPORT_SYMBOL(comp_drivers_get); +#endif + SOF_DEFINE_REG_UUID(component); DECLARE_TR_CTX(comp_tr, SOF_UUID(component_uuid), LOG_LEVEL_INFO); diff --git a/src/include/sof/audio/component_ext.h b/src/include/sof/audio/component_ext.h index 70324175fea8..8532664488df 100644 --- a/src/include/sof/audio/component_ext.h +++ b/src/include/sof/audio/component_ext.h @@ -424,10 +424,14 @@ static inline void comp_make_shared(struct comp_dev *dev) dev->is_shared = true; } +#ifdef CONFIG_SOF_USERSPACE_LL +struct comp_driver_list *comp_drivers_get(void); +#else static inline struct comp_driver_list *comp_drivers_get(void) { return sof_get()->comp_drivers; } +#endif #if CONFIG_IPC_MAJOR_4 static inline int comp_ipc4_bind_remote(struct comp_dev *dev, struct bind_info *bind_data) From 15b3fb8387e8ed90a4c8fca1e0d7bb1f43efa0f0 Mon Sep 17 00:00:00 2001 From: Kai Vehmanen Date: Tue, 21 Apr 2026 16:09:15 +0300 Subject: [PATCH 022/108] audio: chain_dma: add user-space memory and scheduling support Add CONFIG_SOF_USERSPACE_LL support to chain DMA component: - Allocate comp_dev, private data, and DMA buffer from user heap - Use k_work_delayable for periodic task instead of LL timer scheduler to keep DMA operations in kernel context - Guard all changes with #ifdef CONFIG_SOF_USERSPACE_LL Signed-off-by: Kai Vehmanen --- src/audio/chain_dma.c | 91 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 91 insertions(+) diff --git a/src/audio/chain_dma.c b/src/audio/chain_dma.c index e7207e636ee8..91d38828aeb1 100644 --- a/src/audio/chain_dma.c +++ b/src/audio/chain_dma.c @@ -15,7 +15,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -59,6 +61,15 @@ struct chain_dma_data { bool xrun_notification_sent; #endif +#ifdef CONFIG_SOF_USERSPACE_LL + /** Kernel workqueue scheduling for chain DMA in userspace builds. + * Chain DMA needs kernel context for DMA operations, so it cannot + * run on the user-space LL timer thread. + */ + struct k_work_delayable dma_work; + bool stopped; +#endif + /* local host DMA config */ struct sof_dma *dma_host; struct dma_chan_data *chan_host; @@ -275,6 +286,25 @@ static enum task_state chain_task_run(void *data) return SOF_TASK_STATE_RESCHEDULE; } +#ifdef CONFIG_SOF_USERSPACE_LL +/** Kernel workqueue handler for chain DMA periodic task. + * Runs chain_task_run() in kernel context and reschedules if needed. + */ +static void chain_dma_work_handler(struct k_work *work) +{ + struct k_work_delayable *dwork = k_work_delayable_from_work(work); + struct chain_dma_data *cd = CONTAINER_OF(dwork, struct chain_dma_data, dma_work); + enum task_state state; + + if (cd->stopped) + return; + + state = chain_task_run(cd); + if (state == SOF_TASK_STATE_RESCHEDULE && !cd->stopped) + k_work_reschedule(dwork, K_USEC(LL_TIMER_PERIOD_US)); +} +#endif + static int chain_task_start(struct comp_dev *dev) { struct chain_dma_data *cd = comp_get_drvdata(dev); @@ -316,6 +346,12 @@ static int chain_task_start(struct comp_dev *dev) } } +#ifdef CONFIG_SOF_USERSPACE_LL + cd->stopped = false; + k_work_init_delayable(&cd->dma_work, chain_dma_work_handler); + k_work_reschedule(&cd->dma_work, K_NO_WAIT); + cd->chain_task.state = SOF_TASK_STATE_QUEUED; +#else ret = schedule_task_init_ll(&cd->chain_task, SOF_UUID(chain_dma_uuid), SOF_SCHEDULE_LL_TIMER, SOF_TASK_PRI_HIGH, chain_task_run, cd, 0, 0); @@ -330,16 +366,19 @@ static int chain_task_start(struct comp_dev *dev) schedule_task_free(&cd->chain_task); goto error_task; } +#endif pm_policy_state_lock_get(PM_STATE_RUNTIME_IDLE, PM_ALL_SUBSTATES); return 0; +#ifndef CONFIG_SOF_USERSPACE_LL error_task: chain_host_stop(dev); chain_link_stop(dev); return ret; +#endif } static int chain_task_pause(struct comp_dev *dev) @@ -347,10 +386,18 @@ static int chain_task_pause(struct comp_dev *dev) struct chain_dma_data *cd = comp_get_drvdata(dev); int ret, ret2; +#ifdef CONFIG_SOF_USERSPACE_LL + if (cd->chain_task.state == SOF_TASK_STATE_FREE) + return 0; + + cd->stopped = true; + cd->first_data_received = false; +#else if (cd->chain_task.state == SOF_TASK_STATE_FREE) return 0; cd->first_data_received = false; +#endif if (cd->stream_direction == SOF_IPC_STREAM_PLAYBACK) { ret = chain_host_stop(dev); ret2 = chain_link_stop(dev); @@ -361,7 +408,12 @@ static int chain_task_pause(struct comp_dev *dev) if (!ret) ret = ret2; +#ifdef CONFIG_SOF_USERSPACE_LL + k_work_cancel_delayable_sync(&cd->dma_work, &(struct k_work_sync){}); + cd->chain_task.state = SOF_TASK_STATE_FREE; +#else schedule_task_free(&cd->chain_task); +#endif pm_policy_state_lock_put(PM_STATE_RUNTIME_IDLE, PM_ALL_SUBSTATES); return ret; @@ -589,8 +641,14 @@ __cold static int chain_task_init(struct comp_dev *dev, uint8_t host_dma_id, uin fifo_size = ALIGN_UP_INTERNAL(fifo_size, addr_align); /* allocate not shared buffer */ +#ifdef CONFIG_SOF_USERSPACE_LL + cd->dma_buffer = buffer_alloc(sof_sys_user_heap_get(), fifo_size, + SOF_MEM_FLAG_USER | SOF_MEM_FLAG_DMA, + addr_align, BUFFER_USAGE_NOT_SHARED); +#else cd->dma_buffer = buffer_alloc(NULL, fifo_size, SOF_MEM_FLAG_USER | SOF_MEM_FLAG_DMA, addr_align, BUFFER_USAGE_NOT_SHARED); +#endif if (!cd->dma_buffer) { comp_err(dev, "failed to alloc dma buffer"); @@ -649,14 +707,31 @@ __cold static struct comp_dev *chain_task_create(const struct comp_driver *drv, if (host_dma_id >= max_chain_number) return NULL; +#ifdef CONFIG_SOF_USERSPACE_LL + dev = sof_heap_alloc(sof_sys_user_heap_get(), + SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, + sizeof(*dev), 0); + if (!dev) + return NULL; + + memset(dev, 0, sizeof(*dev)); + comp_init(drv, dev, sizeof(*dev)); +#else dev = comp_alloc(drv, sizeof(*dev)); if (!dev) return NULL; +#endif +#ifdef CONFIG_SOF_USERSPACE_LL + cd = sof_heap_alloc(sof_sys_user_heap_get(), SOF_MEM_FLAG_USER, sizeof(*cd), 0); +#else cd = rzalloc(SOF_MEM_FLAG_USER, sizeof(*cd)); +#endif if (!cd) goto error; + memset(cd, 0, sizeof(*cd)); + cd->first_data_received = false; cd->cs = scs ? 2 : 4; cd->chain_task.state = SOF_TASK_STATE_INIT; @@ -667,9 +742,17 @@ __cold static struct comp_dev *chain_task_create(const struct comp_driver *drv, if (!ret) return dev; +#ifdef CONFIG_SOF_USERSPACE_LL + sof_heap_free(sof_sys_user_heap_get(), cd); +#else rfree(cd); +#endif error: +#ifdef CONFIG_SOF_USERSPACE_LL + sof_heap_free(sof_sys_user_heap_get(), dev); +#else comp_free_device(dev); +#endif return NULL; } @@ -680,8 +763,16 @@ __cold static void chain_task_free(struct comp_dev *dev) assert_can_be_cold(); chain_release(dev); +#ifdef CONFIG_SOF_USERSPACE_LL + sof_heap_free(sof_sys_user_heap_get(), cd); +#else rfree(cd); +#endif +#ifdef CONFIG_SOF_USERSPACE_LL + sof_heap_free(sof_sys_user_heap_get(), dev); +#else comp_free_device(dev); +#endif } static const struct comp_driver comp_chain_dma = { From b98e676bdf17a9463167834299903ddc6f2312de Mon Sep 17 00:00:00 2001 From: Kai Vehmanen Date: Fri, 8 May 2026 14:38:52 +0300 Subject: [PATCH 023/108] audio: chain-dma: use module context for allocations Use a module context instead of heap to allocate objects in chain-dma module. This is required to support vregion use. Signed-off-by: Kai Vehmanen --- src/audio/chain_dma.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/audio/chain_dma.c b/src/audio/chain_dma.c index 91d38828aeb1..b3eee06eb3fb 100644 --- a/src/audio/chain_dma.c +++ b/src/audio/chain_dma.c @@ -562,6 +562,7 @@ __cold static int chain_task_init(struct comp_dev *dev, uint8_t host_dma_id, uin uint32_t fifo_size) { struct chain_dma_data *cd = comp_get_drvdata(dev); + struct mod_alloc_ctx *alloc_ctx = NULL; uint32_t addr_align; size_t buff_size; void *buff_addr; @@ -640,15 +641,15 @@ __cold static int chain_task_init(struct comp_dev *dev, uint8_t host_dma_id, uin } fifo_size = ALIGN_UP_INTERNAL(fifo_size, addr_align); - /* allocate not shared buffer */ + #ifdef CONFIG_SOF_USERSPACE_LL - cd->dma_buffer = buffer_alloc(sof_sys_user_heap_get(), fifo_size, + alloc_ctx = ipc_get()->ll_alloc; +#endif + + /* allocate not shared buffer */ + cd->dma_buffer = buffer_alloc(alloc_ctx, fifo_size, SOF_MEM_FLAG_USER | SOF_MEM_FLAG_DMA, addr_align, BUFFER_USAGE_NOT_SHARED); -#else - cd->dma_buffer = buffer_alloc(NULL, fifo_size, SOF_MEM_FLAG_USER | SOF_MEM_FLAG_DMA, - addr_align, BUFFER_USAGE_NOT_SHARED); -#endif if (!cd->dma_buffer) { comp_err(dev, "failed to alloc dma buffer"); From 944164363745a77ffcd6a447f648f82b852e56df Mon Sep 17 00:00:00 2001 From: Kai Vehmanen Date: Mon, 13 Jul 2026 17:42:47 +0300 Subject: [PATCH 024/108] audio: chain_dma: run chain DMA in the user LL thread Commit f515f3b kept chain DMA in kernel context for CONFIG_SOF_USERSPACE_LL by driving chain_task_run() from a k_work_delayable on the system workqueue, because chain DMA used the raw Zephyr DMA API (dma_get_status/dma_reload/dma_start/dma_stop) which is not callable from an unprivileged user thread. Running a ~1ms (500us on AMD) audio DMA cadence on the low-priority system workqueue, rescheduled completion-relative via k_work_reschedule(), is prone to preemption and drift and risks xruns. Port chain DMA to SOF's sof_dma_* syscall wrappers (as host and dai already do) so it can run unprivileged, and schedule it as a normal SOF_SCHEDULE_LL_TIMER task on the user LL thread. This removes the kernel-workqueue path entirely; both userspace and non-userspace builds now share the LL scheduler path with scheduler-managed task lifecycle. Channel handles are stored as integer indices instead of kernel-only struct dma_chan_data pointers. The user-heap allocations for the component, private data and DMA buffer are retained so the user thread can access them. Signed-off-by: Kai Vehmanen --- src/audio/chain_dma.c | 121 ++++++++++++------------------------------ 1 file changed, 35 insertions(+), 86 deletions(-) diff --git a/src/audio/chain_dma.c b/src/audio/chain_dma.c index b3eee06eb3fb..e9be5e142e2e 100644 --- a/src/audio/chain_dma.c +++ b/src/audio/chain_dma.c @@ -61,24 +61,15 @@ struct chain_dma_data { bool xrun_notification_sent; #endif -#ifdef CONFIG_SOF_USERSPACE_LL - /** Kernel workqueue scheduling for chain DMA in userspace builds. - * Chain DMA needs kernel context for DMA operations, so it cannot - * run on the user-space LL timer thread. - */ - struct k_work_delayable dma_work; - bool stopped; -#endif - /* local host DMA config */ struct sof_dma *dma_host; - struct dma_chan_data *chan_host; + int chan_host_index; struct dma_config z_config_host; struct dma_block_config dma_block_cfg_host; /* local link DMA config */ struct sof_dma *dma_link; - struct dma_chan_data *chan_link; + int chan_link_index; struct dma_config z_config_link; struct dma_block_config dma_block_cfg_link; @@ -90,18 +81,18 @@ static int chain_host_start(struct comp_dev *dev) struct chain_dma_data *cd = comp_get_drvdata(dev); int err; - if (!cd->chan_host || !cd->chan_host->dma) { + if (cd->chan_host_index < 0 || !cd->dma_host) { comp_err(dev, "incomplete initialization detected, aborting host %p", - cd->chan_host); + cd->dma_host); return -ENODEV; } - err = dma_start(cd->chan_host->dma->z_dev, cd->chan_host->index); + err = sof_dma_start(cd->dma_host, cd->chan_host_index); if (err < 0) return err; comp_info(dev, "dma_start() host chan_index = %u", - cd->chan_host->index); + cd->chan_host_index); return 0; } @@ -110,12 +101,12 @@ static int chain_link_start(struct comp_dev *dev) struct chain_dma_data *cd = comp_get_drvdata(dev); int err; - err = dma_start(cd->chan_link->dma->z_dev, cd->chan_link->index); + err = sof_dma_start(cd->dma_link, cd->chan_link_index); if (err < 0) return err; comp_info(dev, "dma_start() link chan_index = %u", - cd->chan_link->index); + cd->chan_link_index); return 0; } @@ -124,12 +115,12 @@ static int chain_link_stop(struct comp_dev *dev) struct chain_dma_data *cd = comp_get_drvdata(dev); int err; - err = dma_stop(cd->chan_link->dma->z_dev, cd->chan_link->index); + err = sof_dma_stop(cd->dma_link, cd->chan_link_index); if (err < 0) return err; comp_info(dev, "dma_stop() link chan_index = %u", - cd->chan_link->index); + cd->chan_link_index); return 0; } @@ -139,12 +130,12 @@ static int chain_host_stop(struct comp_dev *dev) struct chain_dma_data *cd = comp_get_drvdata(dev); int err; - err = dma_stop(cd->chan_host->dma->z_dev, cd->chan_host->index); + err = sof_dma_stop(cd->dma_host, cd->chan_host_index); if (err < 0) return err; comp_info(dev, "dma_stop() host chan_index = %u", - cd->chan_host->index); + cd->chan_host_index); return 0; } @@ -182,7 +173,7 @@ static enum task_state chain_task_run(void *data) /* Link DMA can return -EPIPE and current status if xrun occurs, then it is not critical * and flow shall continue. Other error values will be treated as critical. */ - ret = dma_get_status(cd->chan_link->dma->z_dev, cd->chan_link->index, &stat); + ret = sof_dma_get_status(cd->dma_link, cd->chan_link_index, &stat); switch (ret) { case 0: #if CONFIG_XRUN_NOTIFICATIONS_ENABLE @@ -206,7 +197,7 @@ static enum task_state chain_task_run(void *data) link_read_pos = stat.read_position; /* Host DMA does not report xruns. All error values will be treated as critical. */ - ret = dma_get_status(cd->chan_host->dma->z_dev, cd->chan_host->index, &stat); + ret = sof_dma_get_status(cd->dma_host, cd->chan_host_index, &stat); if (ret < 0) { tr_err(&chain_dma_tr, "dma_get_status() error, ret = %d", ret); return SOF_TASK_STATE_COMPLETED; @@ -224,14 +215,14 @@ static enum task_state chain_task_run(void *data) */ const size_t increment = MIN(host_free_bytes, link_avail_bytes); - ret = dma_reload(cd->chan_host->dma->z_dev, cd->chan_host->index, 0, 0, increment); + ret = sof_dma_reload(cd->dma_host, cd->chan_host_index, increment); if (ret < 0) { tr_err(&chain_dma_tr, "dma_reload() host error, ret = %d", ret); return SOF_TASK_STATE_COMPLETED; } - ret = dma_reload(cd->chan_link->dma->z_dev, cd->chan_link->index, 0, 0, increment); + ret = sof_dma_reload(cd->dma_link, cd->chan_link_index, increment); if (ret < 0) { tr_err(&chain_dma_tr, "dma_reload() link error, ret = %d", ret); @@ -247,9 +238,8 @@ static enum task_state chain_task_run(void *data) const size_t half_buff_size = buff_size / 2; if (!cd->first_data_received && host_avail_bytes > half_buff_size) { - ret = dma_reload(cd->chan_link->dma->z_dev, - cd->chan_link->index, 0, 0, - MIN(host_avail_bytes, link_free_bytes)); + ret = sof_dma_reload(cd->dma_link, cd->chan_link_index, + MIN(host_avail_bytes, link_free_bytes)); if (ret < 0) { tr_err(&chain_dma_tr, "dma_reload() link error, ret = %d", ret); @@ -263,8 +253,8 @@ static enum task_state chain_task_run(void *data) host_read_pos, buff_size); - ret = dma_reload(cd->chan_host->dma->z_dev, cd->chan_host->index, - 0, 0, transferred); + ret = sof_dma_reload(cd->dma_host, cd->chan_host_index, + transferred); if (ret < 0) { tr_err(&chain_dma_tr, "dma_reload() host error, ret = %d", ret); @@ -273,8 +263,8 @@ static enum task_state chain_task_run(void *data) if (host_avail_bytes >= half_buff_size && link_free_bytes >= half_buff_size) { - ret = dma_reload(cd->chan_link->dma->z_dev, cd->chan_link->index, - 0, 0, half_buff_size); + ret = sof_dma_reload(cd->dma_link, cd->chan_link_index, + half_buff_size); if (ret < 0) { tr_err(&chain_dma_tr, "dma_reload() link error, ret = %d", ret); @@ -286,25 +276,6 @@ static enum task_state chain_task_run(void *data) return SOF_TASK_STATE_RESCHEDULE; } -#ifdef CONFIG_SOF_USERSPACE_LL -/** Kernel workqueue handler for chain DMA periodic task. - * Runs chain_task_run() in kernel context and reschedules if needed. - */ -static void chain_dma_work_handler(struct k_work *work) -{ - struct k_work_delayable *dwork = k_work_delayable_from_work(work); - struct chain_dma_data *cd = CONTAINER_OF(dwork, struct chain_dma_data, dma_work); - enum task_state state; - - if (cd->stopped) - return; - - state = chain_task_run(cd); - if (state == SOF_TASK_STATE_RESCHEDULE && !cd->stopped) - k_work_reschedule(dwork, K_USEC(LL_TIMER_PERIOD_US)); -} -#endif - static int chain_task_start(struct comp_dev *dev) { struct chain_dma_data *cd = comp_get_drvdata(dev); @@ -346,12 +317,6 @@ static int chain_task_start(struct comp_dev *dev) } } -#ifdef CONFIG_SOF_USERSPACE_LL - cd->stopped = false; - k_work_init_delayable(&cd->dma_work, chain_dma_work_handler); - k_work_reschedule(&cd->dma_work, K_NO_WAIT); - cd->chain_task.state = SOF_TASK_STATE_QUEUED; -#else ret = schedule_task_init_ll(&cd->chain_task, SOF_UUID(chain_dma_uuid), SOF_SCHEDULE_LL_TIMER, SOF_TASK_PRI_HIGH, chain_task_run, cd, 0, 0); @@ -366,19 +331,16 @@ static int chain_task_start(struct comp_dev *dev) schedule_task_free(&cd->chain_task); goto error_task; } -#endif pm_policy_state_lock_get(PM_STATE_RUNTIME_IDLE, PM_ALL_SUBSTATES); return 0; -#ifndef CONFIG_SOF_USERSPACE_LL error_task: chain_host_stop(dev); chain_link_stop(dev); return ret; -#endif } static int chain_task_pause(struct comp_dev *dev) @@ -386,18 +348,11 @@ static int chain_task_pause(struct comp_dev *dev) struct chain_dma_data *cd = comp_get_drvdata(dev); int ret, ret2; -#ifdef CONFIG_SOF_USERSPACE_LL if (cd->chain_task.state == SOF_TASK_STATE_FREE) return 0; - cd->stopped = true; cd->first_data_received = false; -#else - if (cd->chain_task.state == SOF_TASK_STATE_FREE) - return 0; - cd->first_data_received = false; -#endif if (cd->stream_direction == SOF_IPC_STREAM_PLAYBACK) { ret = chain_host_stop(dev); ret2 = chain_link_stop(dev); @@ -408,12 +363,7 @@ static int chain_task_pause(struct comp_dev *dev) if (!ret) ret = ret2; -#ifdef CONFIG_SOF_USERSPACE_LL - k_work_cancel_delayable_sync(&cd->dma_work, &(struct k_work_sync){}); - cd->chain_task.state = SOF_TASK_STATE_FREE; -#else schedule_task_free(&cd->chain_task); -#endif pm_policy_state_lock_put(PM_STATE_RUNTIME_IDLE, PM_ALL_SUBSTATES); return ret; @@ -425,9 +375,9 @@ __cold static void chain_release(struct comp_dev *dev) assert_can_be_cold(); - dma_release_channel(cd->chan_host->dma->z_dev, cd->chan_host->index); + sof_dma_release_channel(cd->dma_host, cd->chan_host_index); sof_dma_put(cd->dma_host); - dma_release_channel(cd->chan_link->dma->z_dev, cd->chan_link->index); + sof_dma_release_channel(cd->dma_link, cd->chan_link_index); sof_dma_put(cd->dma_link); if (cd->dma_buffer) { @@ -515,16 +465,16 @@ __cold static int chain_init(struct comp_dev *dev, void *addr, size_t length) /* get host DMA channel */ channel = cd->host_connector_node_id.f.v_index; - channel = dma_request_channel(cd->dma_host->z_dev, &channel); + channel = sof_dma_request_channel(cd->dma_host, channel); if (channel < 0) { comp_err(dev, "host dma_request_channel() failed for %u", cd->host_connector_node_id.f.v_index); return channel; } - cd->chan_host = &cd->dma_host->chan[channel]; + cd->chan_host_index = channel; - err = dma_config(cd->dma_host->z_dev, cd->chan_host->index, dma_cfg_host); + err = sof_dma_config(cd->dma_host, cd->chan_host_index, dma_cfg_host); if (err < 0) { comp_err(dev, "host dma_config() failed for %d", channel); goto error_host; @@ -532,7 +482,7 @@ __cold static int chain_init(struct comp_dev *dev, void *addr, size_t length) /* get link DMA channel */ channel = cd->link_connector_node_id.f.v_index; - channel = dma_request_channel(cd->dma_link->z_dev, &channel); + channel = sof_dma_request_channel(cd->dma_link, channel); if (channel < 0) { comp_err(dev, "link dma_request_channel() failed for %u", cd->link_connector_node_id.f.v_index); @@ -540,9 +490,9 @@ __cold static int chain_init(struct comp_dev *dev, void *addr, size_t length) goto error_host; } - cd->chan_link = &cd->dma_link->chan[channel]; + cd->chan_link_index = channel; - err = dma_config(cd->dma_link->z_dev, cd->chan_link->index, dma_cfg_link); + err = sof_dma_config(cd->dma_link, cd->chan_link_index, dma_cfg_link); if (err < 0) { comp_err(dev, "link dma_config() failed for %d", channel); goto error_link; @@ -550,11 +500,9 @@ __cold static int chain_init(struct comp_dev *dev, void *addr, size_t length) return 0; error_link: - dma_release_channel(cd->dma_link->z_dev, cd->chan_link->index); - cd->chan_link = NULL; + sof_dma_release_channel(cd->dma_link, cd->chan_link_index); error_host: - dma_release_channel(cd->dma_host->z_dev, cd->chan_host->index); - cd->chan_host = NULL; + sof_dma_release_channel(cd->dma_host, cd->chan_host_index); return err; } @@ -612,8 +560,8 @@ __cold static int chain_task_init(struct comp_dev *dev, uint8_t host_dma_id, uin } /* retrieve DMA buffer address alignment */ - ret = dma_get_attribute(cd->dma_host->z_dev, DMA_ATTR_BUFFER_ADDRESS_ALIGNMENT, - &addr_align); + ret = sof_dma_get_attribute(cd->dma_host, DMA_ATTR_BUFFER_ADDRESS_ALIGNMENT, + &addr_align); if (ret < 0) { comp_err(dev, "could not get dma buffer address alignment, err = %d", ret); @@ -736,6 +684,7 @@ __cold static struct comp_dev *chain_task_create(const struct comp_driver *drv, cd->first_data_received = false; cd->cs = scs ? 2 : 4; cd->chain_task.state = SOF_TASK_STATE_INIT; + cd->chan_host_index = -EINVAL; comp_set_drvdata(dev, cd); From 153e46ffaeeb0ffcb6cdb878bf75feeef8491048 Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Tue, 23 Jun 2026 15:37:48 +0200 Subject: [PATCH 025/108] userspace: fix debug-overlay builds Currently building with userspace LL and the debug overlay generates non-functional images. This is due to two incompatibilities: (1) cpu_get_id() called in userspace in zephyr_ll_task_init() and (2) the heap_alloc boot-test currently causing an exception. Fix the former and disable the latter to re-enabld debug builds with the userspace enabled. Signed-off-by: Guennadi Liakhovetski --- src/schedule/zephyr_ll.c | 2 +- zephyr/test/CMakeLists.txt | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/schedule/zephyr_ll.c b/src/schedule/zephyr_ll.c index 37bcbce60326..83d8bb99d1a6 100644 --- a/src/schedule/zephyr_ll.c +++ b/src/schedule/zephyr_ll.c @@ -687,7 +687,7 @@ int zephyr_ll_task_init(struct task *task, * schedule_task_init() must be run on target core, see * sof/zephyr/schedule.c:arch_schedulers_get() */ - assert(cpu_get_id() == core); + assert(IS_ENABLED(CONFIG_SOF_USERSPACE_LL) || cpu_get_id() == core); ret = schedule_task_init(task, uid, type, priority, run, data, core, flags); diff --git a/zephyr/test/CMakeLists.txt b/zephyr/test/CMakeLists.txt index 397e9e31b938..29acdb24264c 100644 --- a/zephyr/test/CMakeLists.txt +++ b/zephyr/test/CMakeLists.txt @@ -11,7 +11,8 @@ if(CONFIG_SOF_BOOT_TEST) zephyr_library_sources_ifdef(CONFIG_USERSPACE userspace/ksem.c ) - if(CONFIG_USERSPACE AND CONFIG_SOF_USERSPACE_INTERFACE_ALLOC) + + if(CONFIG_USERSPACE AND CONFIG_SOF_USERSPACE_INTERFACE_ALLOC AND NOT CONFIG_SOF_USERSPACE_LL) zephyr_library_sources(userspace/test_heap_alloc.c) endif() endif() From 7726e135a2fab74e046cf80c28499ec72fad41f7 Mon Sep 17 00:00:00 2001 From: Kai Vehmanen Date: Wed, 25 Mar 2026 19:31:49 +0200 Subject: [PATCH 026/108] (---section audio user PRs STOP) From a19ec28c98da76a16aa1d87abb1b85690b6681d3 Mon Sep 17 00:00:00 2001 From: Kai Vehmanen Date: Tue, 17 Mar 2026 20:21:06 +0200 Subject: [PATCH 027/108] (---section: IPC user support START) From 6a7af4cf6ba3efcdf285b6e17034f77a1e52330f Mon Sep 17 00:00:00 2001 From: Kai Vehmanen Date: Tue, 10 Feb 2026 15:17:16 +0200 Subject: [PATCH 028/108] ipc: move standalone-test check later in ipc_init() If CONFIG_SOF_BOOT_TEST_STANDALONE is set, ipc_init() is terminated early. This ensures SOF will not start to generate or respond to IPC messages that could potentially interfere with standalone test cases (some of which send and receive IPCs). The current implementation leaves the component list uninitialized and this can cause trouble to standalone tests that want to utilzie common IPC code to build messages. Fix this problem by executing more of ipc_init() also in the standalone mode. Signed-off-by: Kai Vehmanen --- src/ipc/ipc-common.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/ipc/ipc-common.c b/src/ipc/ipc-common.c index d0d248c9ec77..cde586f83c55 100644 --- a/src/ipc/ipc-common.c +++ b/src/ipc/ipc-common.c @@ -294,11 +294,6 @@ __cold int ipc_init(struct sof *sof) tr_dbg(&ipc_tr, "entry"); -#if CONFIG_SOF_BOOT_TEST_STANDALONE - LOG_INF("SOF_BOOT_TEST_STANDALONE, disabling IPC."); - return 0; -#endif - /* init ipc data */ sof->ipc = rzalloc(SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, sizeof(*sof->ipc)); if (!sof->ipc) { @@ -329,6 +324,11 @@ __cold int ipc_init(struct sof *sof) io_perf_monitor_init_data(&sof->ipc->io_perf_out_msg_count, &init_data); #endif +#if CONFIG_SOF_BOOT_TEST_STANDALONE + LOG_INF("SOF_BOOT_TEST_STANDALONE, disabling IPC."); + return 0; +#endif + #ifdef __ZEPHYR__ struct k_thread *thread = &sof->ipc->ipc_send_wq.thread; From 747d7b8aa4f9dd300a8125ae17fdfeb027949774 Mon Sep 17 00:00:00 2001 From: Kai Vehmanen Date: Thu, 18 Jun 2026 13:56:03 +0300 Subject: [PATCH 029/108] ipc: add global IPC context and ll_alloc for user-space LL When CONFIG_SOF_USERSPACE_LL is enabled the IPC context must live in user-accessible application memory rather than being allocated from the kernel heap and reached via sof->ipc. Place struct ipc in a dedicated K_APP_BMEM partition and provide an out-of-line ipc_get() returning it. Add the user-space bookkeeping fields to struct ipc (ipc_user_pdata and ll_alloc) and the struct ipc_user descriptor later used to forward commands to a user thread. Rework ipc_init() to set up ll_alloc from the LL user heap and allocate the IPC objects accordingly. A no-op ipc_user_init() stub is added here and implemented later. This also makes the tree build with CONFIG_SOF_USERSPACE_LL again, as chain_dma.c already references ipc_get()->ll_alloc. Signed-off-by: Kai Vehmanen --- src/include/sof/ipc/common.h | 45 ++++++++++++++++++ src/ipc/ipc-common.c | 92 ++++++++++++++++++++++++++++-------- src/ipc/ipc4/helper.c | 10 +++- zephyr/include/rtos/sof.h | 2 + 4 files changed, 129 insertions(+), 20 deletions(-) diff --git a/src/include/sof/ipc/common.h b/src/include/sof/ipc/common.h index e46fc10b9521..74b75d7ac232 100644 --- a/src/include/sof/ipc/common.h +++ b/src/include/sof/ipc/common.h @@ -22,6 +22,7 @@ #include #include +struct comp_driver; struct dma_sg_elem_array; struct ipc_msg; @@ -53,6 +54,37 @@ extern struct tr_ctx ipc_tr; #define IPC_TASK_SECONDARY_CORE BIT(2) #define IPC_TASK_POWERDOWN BIT(3) +struct ipc_user { + struct k_thread *thread; + struct k_sem *sem; + struct k_event *event; + /** @brief Copy of IPC4 message primary word forwarded to user thread */ + uint32_t ipc_msg_pri; + /** @brief Copy of IPC4 message extension word forwarded to user thread */ + uint32_t ipc_msg_ext; + /** @brief Result code from user thread processing */ + int result; + /** @brief Reply extension word from user thread (e.g. CONFIG_GET result) */ + uint32_t reply_ext; + /** @brief Reply TX data size from user thread (e.g. LARGE_CONFIG_GET result) */ + uint32_t reply_tx_size; + /** @brief Reply TX data pointer from user thread (e.g. LARGE_CONFIG_GET result) */ + void *reply_tx_data; + struct ipc *ipc; + struct k_thread *audio_thread; + /** @brief Original kernel driver pointer for restoring dev->drv after create */ + const struct comp_driver *init_drv; + /** + * @brief User-accessible copy of comp_driver + tr_ctx for create(). + * + * The comp_driver and tr_ctx structs reside in kernel memory + * (.rodata/.data) which is not user-readable. The kernel handler + * copies them here before forwarding to the user thread. + * Size verified by BUILD_ASSERT in handler-user.c. + */ + uint8_t init_drv_data[160] __aligned(4); +}; + struct ipc { struct k_spinlock lock; /* locking mechanism */ void *comp_data; @@ -74,6 +106,11 @@ struct ipc { struct task ipc_task; #endif +#ifdef CONFIG_SOF_USERSPACE_LL + struct ipc_user *ipc_user_pdata; + struct mod_alloc_ctx *ll_alloc; +#endif + #ifdef CONFIG_SOF_TELEMETRY_IO_PERFORMANCE_MEASUREMENTS /* io performance measurement */ struct io_perf_data_item *io_perf_in_msg_count; @@ -95,6 +132,12 @@ struct ipc { extern struct task_ops ipc_task_ops; +#ifdef CONFIG_SOF_USERSPACE_LL + +struct ipc *ipc_get(void); + +#else + /** * \brief Get the IPC global context. * @return The global IPC context. @@ -104,6 +147,8 @@ static inline struct ipc *ipc_get(void) return sof_get()->ipc; } +#endif /* CONFIG_SOF_USERSPACE_LL */ + /** * \brief Initialise global IPC context. * @param[in,out] sof Global SOF context. diff --git a/src/ipc/ipc-common.c b/src/ipc/ipc-common.c index cde586f83c55..3177201af493 100644 --- a/src/ipc/ipc-common.c +++ b/src/ipc/ipc-common.c @@ -24,6 +24,8 @@ #include #include #include +#include +#include #include #include #include @@ -35,6 +37,15 @@ #include #include +#ifdef __ZEPHYR__ +#include +#endif + +#ifdef CONFIG_SOF_USERSPACE_LL +#include +#include +#endif + #include LOG_MODULE_REGISTER(ipc, CONFIG_SOF_LOG_LEVEL); @@ -43,6 +54,18 @@ SOF_DEFINE_REG_UUID(ipc); DECLARE_TR_CTX(ipc_tr, SOF_UUID(ipc_uuid), LOG_LEVEL_INFO); +#ifdef CONFIG_SOF_USERSPACE_LL +K_APPMEM_PARTITION_DEFINE(ipc_context_part); + +K_APP_BMEM(ipc_context_part) static struct ipc ipc_context; + +struct ipc *ipc_get(void) +{ + return &ipc_context; +} +EXPORT_SYMBOL(ipc_get); +#endif + int ipc_process_on_core(uint32_t core, bool blocking) { struct ipc *ipc = ipc_get(); @@ -288,29 +311,56 @@ void ipc_schedule_process(struct ipc *ipc) #endif } +static int ipc_user_init(void) +{ + return 0; +} + __cold int ipc_init(struct sof *sof) { + struct k_heap *heap; + struct ipc *ipc; + assert_can_be_cold(); tr_dbg(&ipc_tr, "entry"); +#ifdef CONFIG_SOF_USERSPACE_LL + heap = zephyr_ll_user_heap(); + + ipc = ipc_get(); + memset(ipc, 0, sizeof(*ipc)); + ipc->ll_alloc = sof_heap_alloc(heap, SOF_MEM_FLAG_USER, sizeof(*ipc->ll_alloc), 0); + if (!ipc->ll_alloc) { + tr_err(&ipc_tr, "Unable to allocate IPC ll_alloc"); + return -ENOMEM; + } + ipc->ll_alloc->heap = heap; + ipc->ll_alloc->vreg = NULL; +#else + heap = NULL; + /* init ipc data */ - sof->ipc = rzalloc(SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, sizeof(*sof->ipc)); - if (!sof->ipc) { + ipc = rzalloc(SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, sizeof(*ipc)); + if (!ipc) { tr_err(&ipc_tr, "Unable to allocate IPC data"); return -ENOMEM; } - sof->ipc->comp_data = rzalloc(SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, - SOF_IPC_MSG_MAX_SIZE); - if (!sof->ipc->comp_data) { + sof->ipc = ipc; +#endif + + ipc->comp_data = sof_heap_alloc(heap, SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, + SOF_IPC_MSG_MAX_SIZE, 0); + if (!ipc->comp_data) { tr_err(&ipc_tr, "Unable to allocate IPC component data"); - rfree(sof->ipc); + sof_heap_free(heap, ipc); return -ENOMEM; } + memset(ipc->comp_data, 0, SOF_IPC_MSG_MAX_SIZE); - k_spinlock_init(&sof->ipc->lock); - list_init(&sof->ipc->msg_list); - list_init(&sof->ipc->comp_list); + k_spinlock_init(&ipc->lock); + list_init(&ipc->msg_list); + list_init(&ipc->comp_list); #ifdef CONFIG_SOF_TELEMETRY_IO_PERFORMANCE_MEASUREMENTS struct io_perf_data_item init_data = {IO_PERF_IPC_ID, @@ -319,20 +369,17 @@ __cold int ipc_init(struct sof *sof) IO_PERF_POWERED_UP_ENABLED, IO_PERF_D0IX_POWER_MODE, 0, 0, 0 }; - io_perf_monitor_init_data(&sof->ipc->io_perf_in_msg_count, &init_data); + io_perf_monitor_init_data(&ipc->io_perf_in_msg_count, &init_data); init_data.direction = IO_PERF_OUTPUT_DIRECTION; - io_perf_monitor_init_data(&sof->ipc->io_perf_out_msg_count, &init_data); + io_perf_monitor_init_data(&ipc->io_perf_out_msg_count, &init_data); #endif -#if CONFIG_SOF_BOOT_TEST_STANDALONE - LOG_INF("SOF_BOOT_TEST_STANDALONE, disabling IPC."); - return 0; -#endif #ifdef __ZEPHYR__ - struct k_thread *thread = &sof->ipc->ipc_send_wq.thread; + struct k_thread *thread = &ipc->ipc_send_wq.thread; - k_work_queue_start(&sof->ipc->ipc_send_wq, ipc_send_wq_stack, + k_work_queue_init(&ipc->ipc_send_wq); + k_work_queue_start(&ipc->ipc_send_wq, ipc_send_wq_stack, K_THREAD_STACK_SIZEOF(ipc_send_wq_stack), 1, NULL); k_thread_suspend(thread); @@ -344,10 +391,17 @@ __cold int ipc_init(struct sof *sof) k_thread_resume(thread); - k_work_init_delayable(&sof->ipc->z_delayed_work, ipc_work_handler); + k_work_init_delayable(&ipc->z_delayed_work, ipc_work_handler); +#endif + + ipc_user_init(); + +#if CONFIG_SOF_BOOT_TEST_STANDALONE + LOG_INF("SOF_BOOT_TEST_STANDALONE, skipping platform IPC init."); + return 0; #endif - return platform_ipc_init(sof->ipc); + return platform_ipc_init(ipc); } /* Locking: call with ipc->lock held and interrupts disabled */ diff --git a/src/ipc/ipc4/helper.c b/src/ipc/ipc4/helper.c index 5881fbbfb0b5..d09755c240e6 100644 --- a/src/ipc/ipc4/helper.c +++ b/src/ipc/ipc4/helper.c @@ -558,7 +558,10 @@ __cold static struct comp_buffer *ipc4_create_buffer(struct comp_dev *src, bool ipc_buf.size = buf_size; ipc_buf.comp.id = IPC4_COMP_ID(src_queue, dst_queue); ipc_buf.comp.pipeline_id = src->ipc_config.pipeline_id; - ipc_buf.comp.core = cpu_get_id(); + + assert(IS_ENABLED(CONFIG_SOF_USERSPACE_LL) || src->ipc_config.core == cpu_get_id()); + ipc_buf.comp.core = src->ipc_config.core; + return buffer_new(alloc, &ipc_buf, is_shared); } @@ -701,6 +704,11 @@ __cold int ipc_comp_connect(struct ipc *ipc, ipc_pipe_comp_connect *_connect) #else alloc = NULL; #endif /* CONFIG_ZEPHYR_DP_SCHEDULER */ + +#ifdef CONFIG_SOF_USERSPACE_LL + if (!alloc) + alloc = ipc->ll_alloc; +#endif bool cross_core_bind = source->ipc_config.core != sink->ipc_config.core; /* If both components are on same core -- process IPC on that core, diff --git a/zephyr/include/rtos/sof.h b/zephyr/include/rtos/sof.h index 573e6ac63d77..efe0c1e6acab 100644 --- a/zephyr/include/rtos/sof.h +++ b/zephyr/include/rtos/sof.h @@ -46,8 +46,10 @@ struct sof { int argc; char **argv; +#ifndef CONFIG_SOF_USERSPACE_LL /* ipc */ struct ipc *ipc; +#endif /* system agent */ struct sa *sa; From c4d8211c8c484a741e53b914e5eaf77096571c84 Mon Sep 17 00:00:00 2001 From: Kai Vehmanen Date: Thu, 18 Jun 2026 14:03:44 +0300 Subject: [PATCH 030/108] ipc4: handler: make config and pipeline-state processing context-agnostic Extract the per-message processing logic for module CONFIG_GET/SET and LARGE_CONFIG_GET/SET into standalone functions that return their results via output parameters instead of writing the global msg_reply directly, and give SET_PIPELINE_STATE external linkage. This is a pure refactor with no functional change. The new ipc4_process_module_config(), ipc4_process_large_config_get(), ipc4_process_large_config_set() and ipc4_set_pipeline_state() can be called from any execution context, which is needed to later dispatch these messages from a separate IPC user-space thread. Signed-off-by: Kai Vehmanen --- src/include/ipc4/handler.h | 37 +++++++ src/ipc/ipc4/handler-user.c | 213 +++++++++++++++++++++++++++++++++++- 2 files changed, 246 insertions(+), 4 deletions(-) diff --git a/src/include/ipc4/handler.h b/src/include/ipc4/handler.h index b25cb98e9427..340a3a2908ea 100644 --- a/src/include/ipc4/handler.h +++ b/src/include/ipc4/handler.h @@ -16,6 +16,36 @@ struct ipc4_message_request; */ int ipc4_user_process_module_message(struct ipc4_message_request *ipc4, struct ipc_msg *reply); +/** + * @brief Process MOD_CONFIG_GET or MOD_CONFIG_SET in any execution context. + * @param[in] ipc4 IPC4 message request. + * @param[in] set true for CONFIG_SET, false for CONFIG_GET. + * @param[out] reply_ext Receives extension value for CONFIG_GET (may be NULL). + * @return IPC4 status code (0 on success). + */ +int ipc4_process_module_config(struct ipc4_message_request *ipc4, + bool set, uint32_t *reply_ext); + +/** + * @brief Process MOD_LARGE_CONFIG_GET in any execution context. + * @param[in] ipc4 IPC4 message request. + * @param[out] reply_ext Receives extension value for reply. + * @param[out] reply_tx_size Receives TX data size for reply. + * @param[out] reply_tx_data Receives TX data pointer for reply. + * @return IPC4 status code (0 on success). + */ +int ipc4_process_large_config_get(struct ipc4_message_request *ipc4, + uint32_t *reply_ext, + uint32_t *reply_tx_size, + void **reply_tx_data); + +/** + * @brief Process MOD_LARGE_CONFIG_SET in any execution context. + * @param[in] ipc4 IPC4 message request. + * @return IPC4 status code (0 on success). + */ +int ipc4_process_large_config_set(struct ipc4_message_request *ipc4); + /** * \brief Processes IPC4 userspace global message. * @param[in] ipc4 IPC4 message request. @@ -24,6 +54,13 @@ int ipc4_user_process_module_message(struct ipc4_message_request *ipc4, struct i */ int ipc4_user_process_glb_message(struct ipc4_message_request *ipc4, struct ipc_msg *reply); +/** + * \brief Process SET_PIPELINE_STATE IPC4 message (prepare + trigger phases). + * @param[in] ipc4 IPC4 message request. + * @return 0 on success, IPC4 error code otherwise. + */ +int ipc4_set_pipeline_state(struct ipc4_message_request *ipc4); + /** * \brief Increment the IPC compound message pre-start counter. * @param[in] msg_id IPC message ID. diff --git a/src/ipc/ipc4/handler-user.c b/src/ipc/ipc4/handler-user.c index 48131f2f6aae..f877094e5ad7 100644 --- a/src/ipc/ipc4/handler-user.c +++ b/src/ipc/ipc4/handler-user.c @@ -424,8 +424,12 @@ __cold const struct ipc4_pipeline_set_state_data *ipc4_get_pipeline_data_wrapper return ipc4_get_pipeline_data(); } -/* Entry point for ipc4_pipeline_trigger(), therefore cannot be cold */ -static int ipc4_set_pipeline_state(struct ipc4_message_request *ipc4) +/** + * \brief Process SET_PIPELINE_STATE IPC4 message (prepare + trigger phases). + * @param[in] ipc4 IPC4 message request. + * @return 0 on success, IPC4 error code otherwise. + */ +int ipc4_set_pipeline_state(struct ipc4_message_request *ipc4) { const struct ipc4_pipeline_set_state_data *ppl_data; struct ipc4_pipeline_set_state state; @@ -809,7 +813,22 @@ __cold static int ipc4_unbind_module_instance(struct ipc4_message_request *ipc4) return ipc_comp_disconnect(ipc, (ipc_pipe_comp_connect *)&bu); } -static int ipc4_set_get_config_module_instance(struct ipc4_message_request *ipc4, bool set) +/** + * @brief Process MOD_CONFIG_GET or MOD_CONFIG_SET in any execution context. + * + * Looks up the target component by module_id:instance_id, verifies the + * driver supports get_attribute/set_attribute, and dispatches the + * operation. For GET, the retrieved value is written to @p reply_ext. + * + * Callable from both the IPC kernel task and the IPC user thread. + * + * @param ipc4 Pointer to the IPC4 message request (primary + extension) + * @param set true for CONFIG_SET, false for CONFIG_GET + * @param reply_ext Output: receives the extension value for CONFIG_GET (may be NULL for SET) + * @return IPC4 status code (0 on success) + */ +__cold int ipc4_process_module_config(struct ipc4_message_request *ipc4, + bool set, uint32_t *reply_ext) { struct ipc4_module_config *config = (struct ipc4_module_config *)ipc4; int (*function)(struct comp_dev *dev, uint32_t type, void *value); @@ -859,8 +878,21 @@ static int ipc4_set_get_config_module_instance(struct ipc4_message_request *ipc4 ret = IPC4_INVALID_CONFIG_PARAM_ID; } + if (!set && reply_ext) + *reply_ext = config->extension.dat; + + return ret; +} + +static int ipc4_set_get_config_module_instance(struct ipc4_message_request *ipc4, bool set) +{ + uint32_t reply_ext; + int ret; + + ret = ipc4_process_module_config(ipc4, set, &reply_ext); + if (!set) - msg_reply->extension = config->extension.dat; + msg_reply->extension = reply_ext; return ret; } @@ -990,6 +1022,108 @@ __cold static int ipc4_get_vendor_config_module_instance(struct comp_dev *dev, return IPC4_SUCCESS; } +__cold int ipc4_process_large_config_get(struct ipc4_message_request *ipc4, + uint32_t *reply_ext, + uint32_t *reply_tx_size, + void **reply_tx_data) +{ + struct ipc4_module_large_config_reply reply; + struct ipc4_module_large_config config; + char *data = ipc_get()->comp_data; + const struct comp_driver *drv; + struct comp_dev *dev = NULL; + uint32_t data_offset; + + assert_can_be_cold(); + + int ret = memcpy_s(&config, sizeof(config), ipc4, sizeof(*ipc4)); + + if (ret < 0) + return IPC4_FAILURE; + + tr_dbg(&ipc_tr, "%x : %x", + (uint32_t)config.primary.r.module_id, (uint32_t)config.primary.r.instance_id); + + /* get component dev for non-basefw since there is no + * component dev for basefw + */ + if (config.primary.r.module_id) { + uint32_t comp_id; + + comp_id = IPC4_COMP_ID(config.primary.r.module_id, + config.primary.r.instance_id); + dev = ipc4_get_comp_dev(comp_id); + if (!dev) + return IPC4_MOD_INVALID_ID; + + drv = dev->drv; + + /* Multicore disabled for userspace forwarding; + * non-userspace path retains ipc4_process_on_core() + */ + } else { + drv = ipc4_get_comp_drv(config.primary.r.module_id); + } + + if (!drv) + return IPC4_MOD_INVALID_ID; + + if (!drv->ops.get_large_config) + return IPC4_INVALID_REQUEST; + + data_offset = config.extension.r.data_off_size; + + /* check for vendor param first */ + if (config.extension.r.large_param_id == VENDOR_CONFIG_PARAM) { + /* For now only vendor_config case uses payload from hostbox */ + dcache_invalidate_region((__sparse_force void __sparse_cache *)MAILBOX_HOSTBOX_BASE, + config.extension.r.data_off_size); + ret = ipc4_get_vendor_config_module_instance(dev, drv, + config.extension.r.init_block, + config.extension.r.final_block, + &data_offset, + data, + (const char *)MAILBOX_HOSTBOX_BASE); + } else { +#if CONFIG_LIBRARY + data += sizeof(reply); +#endif + ipc4_prepare_for_kcontrol_get(dev, config.extension.r.large_param_id, + data, data_offset); + + ret = drv->ops.get_large_config(dev, config.extension.r.large_param_id, + config.extension.r.init_block, + config.extension.r.final_block, + &data_offset, data); + } + + /* set up ipc4 error code for reply data */ + if (ret < 0) + ret = IPC4_MOD_INVALID_ID; + + /* Copy host config and overwrite */ + reply.extension.dat = config.extension.dat; + reply.extension.r.data_off_size = data_offset; + + /* The last block, no more data */ + if (!config.extension.r.final_block && data_offset < SOF_IPC_MSG_MAX_SIZE) + reply.extension.r.final_block = 1; + + /* Indicate last block if error occurs */ + if (ret) + reply.extension.r.final_block = 1; + + /* no need to allocate memory for reply msg */ + if (ret) + return ret; + + /* Output via parameters instead of msg_reply */ + *reply_ext = reply.extension.dat; + *reply_tx_size = data_offset; + *reply_tx_data = data; + return ret; +} + __cold static int ipc4_get_large_config_module_instance(struct ipc4_message_request *ipc4) { struct ipc4_module_large_config_reply reply; @@ -1182,6 +1316,77 @@ __cold static int ipc4_set_vendor_config_module_instance(struct comp_dev *dev, data_off_size, data); } +__cold int ipc4_process_large_config_set(struct ipc4_message_request *ipc4) +{ + struct ipc4_module_large_config config; + struct comp_dev *dev = NULL; + const struct comp_driver *drv; + + assert_can_be_cold(); + + int ret = memcpy_s(&config, sizeof(config), ipc4, sizeof(*ipc4)); + + if (ret < 0) + return IPC4_FAILURE; + + dcache_invalidate_region((__sparse_force void __sparse_cache *)MAILBOX_HOSTBOX_BASE, + config.extension.r.data_off_size); + tr_dbg(&ipc_tr, "%x : %x", + (uint32_t)config.primary.r.module_id, (uint32_t)config.primary.r.instance_id); + + if (config.primary.r.module_id) { + uint32_t comp_id; + + comp_id = IPC4_COMP_ID(config.primary.r.module_id, config.primary.r.instance_id); + dev = ipc4_get_comp_dev(comp_id); + if (!dev) + return IPC4_MOD_INVALID_ID; + + drv = dev->drv; + + /* Multicore disabled for userspace forwarding; + * non-userspace path retains ipc4_process_on_core() + */ + } else { + drv = ipc4_get_comp_drv(config.primary.r.module_id); + } + + if (!drv) + return IPC4_MOD_INVALID_ID; + + if (!drv->ops.set_large_config) + return IPC4_INVALID_REQUEST; + + /* check for vendor param first */ + if (config.extension.r.large_param_id == VENDOR_CONFIG_PARAM) { + ret = ipc4_set_vendor_config_module_instance(dev, drv, + (uint32_t)config.primary.r.module_id, + (uint32_t)config.primary.r.instance_id, + config.extension.r.init_block, + config.extension.r.final_block, + config.extension.r.data_off_size, + (const char *)MAILBOX_HOSTBOX_BASE); + } else { +#if CONFIG_LIBRARY + struct ipc *ipc = ipc_get(); + const char *data = (const char *)ipc->comp_data + sizeof(config); +#else + const char *data = (const char *)MAILBOX_HOSTBOX_BASE; +#endif + ret = drv->ops.set_large_config(dev, config.extension.r.large_param_id, + config.extension.r.init_block, config.extension.r.final_block, + config.extension.r.data_off_size, data); + if (ret < 0) { + ipc_cmd_err(&ipc_tr, "failed to set large_config_module_instance %x : %x", + (uint32_t)config.primary.r.module_id, + (uint32_t)config.primary.r.instance_id); + ret = IPC4_INVALID_RESOURCE_ID; + } + } + + return ret; +} + __cold static int ipc4_set_large_config_module_instance(struct ipc4_message_request *ipc4) { struct ipc4_module_large_config config; From 10e5065b36b898a0b43d27472ddd300adbe8e8e8 Mon Sep 17 00:00:00 2001 From: Kai Vehmanen Date: Thu, 18 Jun 2026 14:07:24 +0300 Subject: [PATCH 031/108] ipc: make ipc_msg_reply and compound-message handlers syscalls The IPC reply and compound-message completion entry points need to be callable from the IPC user-space thread. Declare ipc_msg_reply(), ipc_compound_pre_start(), ipc_compound_post_start() and ipc_wait_for_compound_msg() as Zephyr syscalls when CONFIG_SOF_USERSPACE_LL is enabled, renaming the implementations to z_impl_* and adding z_vrfy_* verification wrappers. A new ipc_reply.h carries the ipc_msg_reply declaration so it can be pulled in as a syscall header. Register the new syscall headers in CMakeLists.txt. No functional change for non-userspace builds, where the functions keep direct external linkage via z_impl_* defines. Signed-off-by: Kai Vehmanen --- src/include/ipc4/handler.h | 33 ++++++++++----- src/include/sof/ipc/common.h | 53 ++++++++++++++++++++++- src/include/sof/ipc/ipc_reply.h | 28 +++++++++++++ src/ipc/ipc3/helper.c | 2 +- src/ipc/ipc4/handler-kernel.c | 74 ++++++++++++++++++++++++++++++--- zephyr/CMakeLists.txt | 2 + 6 files changed, 175 insertions(+), 17 deletions(-) create mode 100644 src/include/sof/ipc/ipc_reply.h diff --git a/src/include/ipc4/handler.h b/src/include/ipc4/handler.h index 340a3a2908ea..6073e4dc83c0 100644 --- a/src/include/ipc4/handler.h +++ b/src/include/ipc4/handler.h @@ -62,30 +62,43 @@ int ipc4_user_process_glb_message(struct ipc4_message_request *ipc4, struct ipc_ int ipc4_set_pipeline_state(struct ipc4_message_request *ipc4); /** - * \brief Increment the IPC compound message pre-start counter. + * \brief Complete the IPC compound message. * @param[in] msg_id IPC message ID. + * @param[in] error Error code of the IPC command. */ -void ipc_compound_pre_start(int msg_id); +void ipc_compound_msg_done(uint32_t msg_id, int error); +#if defined(__ZEPHYR__) && defined(CONFIG_SOF_USERSPACE_LL) /** - * \brief Decrement the IPC compound message pre-start counter on return value status. + * \brief Increment the IPC compound message pre-start counter. * @param[in] msg_id IPC message ID. - * @param[in] ret Return value of the IPC command. - * @param[in] delayed True if the reply is delayed. */ -void ipc_compound_post_start(uint32_t msg_id, int ret, bool delayed); +__syscall void ipc_compound_pre_start(int msg_id); /** - * \brief Complete the IPC compound message. + * \brief Decrement the IPC compound message pre-start counter on return value status. * @param[in] msg_id IPC message ID. - * @param[in] error Error code of the IPC command. + * @param[in] ret Return value of the IPC command. + * @param[in] delayed True if the reply is delayed. */ -void ipc_compound_msg_done(uint32_t msg_id, int error); +__syscall void ipc_compound_post_start(uint32_t msg_id, int ret, bool delayed); /** * \brief Wait for the IPC compound message to complete. * @return 0 on success, error code otherwise on timeout. */ -int ipc_wait_for_compound_msg(void); +__syscall int ipc_wait_for_compound_msg(void); +#else +void z_impl_ipc_compound_pre_start(int msg_id); +#define ipc_compound_pre_start z_impl_ipc_compound_pre_start +void z_impl_ipc_compound_post_start(uint32_t msg_id, int ret, bool delayed); +#define ipc_compound_post_start z_impl_ipc_compound_post_start +int z_impl_ipc_wait_for_compound_msg(void); +#define ipc_wait_for_compound_msg z_impl_ipc_wait_for_compound_msg +#endif + +#if defined(__ZEPHYR__) && defined(CONFIG_SOF_USERSPACE_LL) +#include +#endif #endif /* __SOF_IPC4_HANDLER_H__ */ diff --git a/src/include/sof/ipc/common.h b/src/include/sof/ipc/common.h index 74b75d7ac232..8726f53b8fdf 100644 --- a/src/include/sof/ipc/common.h +++ b/src/include/sof/ipc/common.h @@ -25,6 +25,7 @@ struct comp_driver; struct dma_sg_elem_array; struct ipc_msg; +struct ipc4_message_request; /* validates internal non tail structures within IPC command structure */ #define IPC_IS_SIZE_INVALID(object) \ @@ -211,6 +212,56 @@ struct dai_data; */ int ipc_dai_data_config(struct dai_data *dd, struct comp_dev *dev); +/** + * \brief Processes IPC4 userspace module message. + * @param[in] ipc4 IPC4 message request. + * @param[in] reply IPC message reply structure. + * @return IPC4_SUCCESS on success, error code otherwise. + */ +int ipc4_user_process_module_message(struct ipc4_message_request *ipc4, struct ipc_msg *reply); + +/** + * \brief Processes IPC4 userspace global message. + * @param[in] ipc4 IPC4 message request. + * @param[in] reply IPC message reply structure. + * @return IPC4_SUCCESS on success, error code otherwise. + */ +int ipc4_user_process_glb_message(struct ipc4_message_request *ipc4, struct ipc_msg *reply); + +/* + * When CONFIG_SOF_USERSPACE_LL is enabled, compound message functions are + * declared as syscalls in ipc4/handler.h — do not re-declare here with + * external linkage as that conflicts with the static inline syscall wrappers. + */ +#if !(defined(__ZEPHYR__) && defined(CONFIG_SOF_USERSPACE_LL)) +/** + * \brief Increment the IPC compound message pre-start counter. + * @param[in] msg_id IPC message ID. + */ +void ipc_compound_pre_start(int msg_id); + +/** + * \brief Decrement the IPC compound message pre-start counter on return value status. + * @param[in] msg_id IPC message ID. + * @param[in] ret Return value of the IPC command. + * @param[in] delayed True if the reply is delayed. + */ +void ipc_compound_post_start(uint32_t msg_id, int ret, bool delayed); + +/** + * \brief Wait for the IPC compound message to complete. + * @return 0 on success, error code otherwise on timeout. + */ +int ipc_wait_for_compound_msg(void); +#endif /* !CONFIG_SOF_USERSPACE_LL */ + +/** + * \brief Complete the IPC compound message. + * @param[in] msg_id IPC message ID. + * @param[in] error Error code of the IPC command. + */ +void ipc_compound_msg_done(uint32_t msg_id, int error); + /** * \brief create a IPC boot complete message. * @param[in] header header. @@ -285,7 +336,7 @@ int ipc_process_on_core(uint32_t core, bool blocking); * \brief reply to an IPC message. * @param[in] reply pointer to the reply structure. */ -void ipc_msg_reply(struct sof_ipc_reply *reply); +#include /** * \brief Call platform-specific IPC completion function. diff --git a/src/include/sof/ipc/ipc_reply.h b/src/include/sof/ipc/ipc_reply.h new file mode 100644 index 000000000000..756fd535ca30 --- /dev/null +++ b/src/include/sof/ipc/ipc_reply.h @@ -0,0 +1,28 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * + * Copyright(c) 2026 Intel Corporation. All rights reserved. + */ + +#ifndef __SOF_IPC_IPC_REPLY_H__ +#define __SOF_IPC_IPC_REPLY_H__ + +#include + +struct sof_ipc_reply; + +/** + * \brief reply to an IPC message. + * @param[in] reply pointer to the reply structure. + */ +#if defined(__ZEPHYR__) && defined(CONFIG_SOF_USERSPACE_LL) +__syscall void ipc_msg_reply(struct sof_ipc_reply *reply); +#else +void z_impl_ipc_msg_reply(struct sof_ipc_reply *reply); +#define ipc_msg_reply z_impl_ipc_msg_reply +#endif + +#if defined(__ZEPHYR__) && defined(CONFIG_SOF_USERSPACE_LL) +#include +#endif + +#endif /* __SOF_IPC_IPC_REPLY_H__ */ diff --git a/src/ipc/ipc3/helper.c b/src/ipc/ipc3/helper.c index e962a3670a86..7fa66228942f 100644 --- a/src/ipc/ipc3/helper.c +++ b/src/ipc/ipc3/helper.c @@ -727,7 +727,7 @@ int ipc_comp_new(struct ipc *ipc, ipc_comp *_comp) return 0; } -void ipc_msg_reply(struct sof_ipc_reply *reply) +void z_impl_ipc_msg_reply(struct sof_ipc_reply *reply) { struct ipc *ipc = ipc_get(); k_spinlock_key_t key; diff --git a/src/ipc/ipc4/handler-kernel.c b/src/ipc/ipc4/handler-kernel.c index d7d9f417806a..f58bf3787229 100644 --- a/src/ipc/ipc4/handler-kernel.c +++ b/src/ipc/ipc4/handler-kernel.c @@ -40,6 +40,11 @@ #include #include +#ifdef __ZEPHYR__ +#include +#include +#endif + #include #include #include @@ -128,7 +133,7 @@ __cold static bool is_any_ppl_active(void) return false; } -void ipc_compound_pre_start(int msg_id) +void z_impl_ipc_compound_pre_start(int msg_id) { /* ipc thread will wait for all scheduled tasks to be complete * Use a reference count to check status of these tasks. @@ -136,7 +141,23 @@ void ipc_compound_pre_start(int msg_id) atomic_add(&msg_data.delayed_reply, 1); } -void ipc_compound_post_start(uint32_t msg_id, int ret, bool delayed) +#ifdef CONFIG_USERSPACE +/** + * \brief Userspace verification wrapper for ipc_compound_pre_start(). + * + * Forwards the call to z_impl_ipc_compound_pre_start(). No pointer + * validation is needed as only primitive types are passed. + * + * @param[in] msg_id IPC message ID. + */ +void z_vrfy_ipc_compound_pre_start(int msg_id) +{ + z_impl_ipc_compound_pre_start(msg_id); +} +#include +#endif + +void z_impl_ipc_compound_post_start(uint32_t msg_id, int ret, bool delayed) { if (ret) { ipc_cmd_err(&ipc_tr, "failed to process msg %d status %d", msg_id, ret); @@ -149,6 +170,24 @@ void ipc_compound_post_start(uint32_t msg_id, int ret, bool delayed) atomic_sub(&msg_data.delayed_reply, 1); } +#ifdef CONFIG_USERSPACE +/** + * \brief Userspace verification wrapper for ipc_compound_post_start(). + * + * Forwards the call to z_impl_ipc_compound_post_start(). No pointer + * validation is needed as only primitive types are passed. + * + * @param[in] msg_id IPC message ID. + * @param[in] ret Return value of the IPC command. + * @param[in] delayed True if the reply is delayed. + */ +void z_vrfy_ipc_compound_post_start(uint32_t msg_id, int ret, bool delayed) +{ + z_impl_ipc_compound_post_start(msg_id, ret, delayed); +} +#include +#endif + void ipc_compound_msg_done(uint32_t msg_id, int error) { if (!atomic_read(&msg_data.delayed_reply)) { @@ -170,13 +209,13 @@ void ipc_compound_msg_done(uint32_t msg_id, int error) * be always IPC4_FAILURE. Therefore the compound messages handling is simplified. The pipeline * triggers will require an explicit scheduler call to get the components to desired state. */ -int ipc_wait_for_compound_msg(void) +int z_impl_ipc_wait_for_compound_msg(void) { atomic_set(&msg_data.delayed_reply, 0); return IPC4_SUCCESS; } #else -int ipc_wait_for_compound_msg(void) +int z_impl_ipc_wait_for_compound_msg(void) { int try_count = 30; @@ -192,6 +231,22 @@ int ipc_wait_for_compound_msg(void) return IPC4_SUCCESS; } + +#ifdef CONFIG_USERSPACE +/** + * \brief Userspace verification wrapper for ipc_wait_for_compound_msg(). + * + * Forwards the call to z_impl_ipc_wait_for_compound_msg(). No pointer + * validation is needed as no pointers are passed. + * + * @return IPC4_SUCCESS on success, IPC4_FAILURE on timeout. + */ +int z_vrfy_ipc_wait_for_compound_msg(void) +{ + return z_impl_ipc_wait_for_compound_msg(); +} +#include +#endif #endif #if CONFIG_LIBRARY_MANAGER @@ -509,7 +564,7 @@ void ipc_send_buffer_status_notify(void) } #endif -void ipc_msg_reply(struct sof_ipc_reply *reply) +void z_impl_ipc_msg_reply(struct sof_ipc_reply *reply) { struct ipc4_message_request in; @@ -517,6 +572,15 @@ void ipc_msg_reply(struct sof_ipc_reply *reply) ipc_compound_msg_done(in.primary.r.type, reply->error); } +#ifdef CONFIG_USERSPACE +void z_vrfy_ipc_msg_reply(struct sof_ipc_reply *reply) +{ + K_OOPS(K_SYSCALL_MEMORY_READ(reply, sizeof(*reply))); + z_impl_ipc_msg_reply(reply); +} +#include +#endif + void ipc_cmd(struct ipc_cmd_hdr *_hdr) { struct ipc4_message_request *in = ipc4_get_message_request(); diff --git a/zephyr/CMakeLists.txt b/zephyr/CMakeLists.txt index 3b3386fc8a77..4b61a9517d46 100644 --- a/zephyr/CMakeLists.txt +++ b/zephyr/CMakeLists.txt @@ -622,6 +622,8 @@ zephyr_library_sources_ifdef(CONFIG_SHELL zephyr_syscall_header(${SOF_SRC_PATH}/include/sof/audio/module_adapter/module/generic.h) zephyr_syscall_header(${SOF_SRC_PATH}/include/sof/lib/fast-get.h) +zephyr_syscall_header(${SOF_SRC_PATH}/include/sof/ipc/ipc_reply.h) +zephyr_syscall_header(${SOF_SRC_PATH}/include/ipc4/handler.h) zephyr_syscall_header(include/rtos/alloc.h) zephyr_library_sources_ifdef(CONFIG_SOF_USERSPACE_INTERFACE_ALLOC syscall/alloc.c) zephyr_syscall_header(${SOF_SRC_PATH}/include/sof/lib/dai-zephyr.h) From 8cb55a275b4e1c9ccc8daf81701fec6508b2a84a Mon Sep 17 00:00:00 2001 From: Kai Vehmanen Date: Thu, 18 Jun 2026 14:13:25 +0300 Subject: [PATCH 032/108] ipc: allocate IPC objects from the user-accessible system heap The pipeline, component and chain-DMA bookkeeping objects must be reachable from the user-space IPC thread, so allocate them from the system user heap via sof_heap_alloc()/sof_heap_free() instead of the generic rzalloc()/rfree(). As sof_heap_alloc() does not zero the allocation, add an explicit memset() to preserve the previous behaviour. Give ipc4_add_comp_dev() external linkage (declared in topology.h) so the user thread can register a created component, and skip the buffer tr_ctx copy for user-space LL builds where the kernel tr_ctx is not mapped. No functional change for non-userspace builds. Signed-off-by: Kai Vehmanen --- src/include/sof/ipc/topology.h | 1 + src/ipc/ipc-helper.c | 4 +++- src/ipc/ipc4/helper.c | 25 ++++++++++++++++--------- 3 files changed, 20 insertions(+), 10 deletions(-) diff --git a/src/include/sof/ipc/topology.h b/src/include/sof/ipc/topology.h index 84df84abb069..6bfad1cdb49a 100644 --- a/src/include/sof/ipc/topology.h +++ b/src/include/sof/ipc/topology.h @@ -49,6 +49,7 @@ typedef uint32_t ipc_comp; struct ipc_comp_dev; const struct comp_driver *ipc4_get_comp_drv(uint32_t module_id); struct comp_dev *ipc4_get_comp_dev(uint32_t comp_id); +int ipc4_add_comp_dev(struct comp_dev *dev); int ipc4_chain_manager_create(struct ipc4_chain_dma *cdma); int ipc4_chain_dma_state(struct comp_dev *dev, struct ipc4_chain_dma *cdma); int ipc4_create_chain_dma(struct ipc *ipc, struct ipc4_chain_dma *cdma); diff --git a/src/ipc/ipc-helper.c b/src/ipc/ipc-helper.c index 1a2fcef4194e..cbd17d00da4c 100644 --- a/src/ipc/ipc-helper.c +++ b/src/ipc/ipc-helper.c @@ -89,8 +89,10 @@ __cold struct comp_buffer *buffer_new(struct mod_alloc_ctx *alloc, buffer->stream.runtime_stream_params.pipeline_id = desc->comp.pipeline_id; buffer->core = desc->comp.core; +#if !defined(CONFIG_SOF_USERSPACE_LL) memcpy_s(&buffer->tctx, sizeof(struct tr_ctx), &buffer_tr, sizeof(struct tr_ctx)); +#endif } return buffer; @@ -388,7 +390,7 @@ __cold int ipc_comp_free(struct ipc *ipc, uint32_t comp_id) icd->cd = NULL; list_item_del(&icd->list); - rfree(icd); + sof_heap_free(sof_sys_user_heap_get(), icd); return 0; } diff --git a/src/ipc/ipc4/helper.c b/src/ipc/ipc4/helper.c index d09755c240e6..2cca34b948e4 100644 --- a/src/ipc/ipc4/helper.c +++ b/src/ipc/ipc4/helper.c @@ -64,7 +64,6 @@ LOG_MODULE_DECLARE(ipc, CONFIG_SOF_LOG_LEVEL); extern struct tr_ctx comp_tr; static const struct comp_driver *ipc4_get_drv(const void *uuid); -static int ipc4_add_comp_dev(struct comp_dev *dev); void ipc_build_stream_posn(struct sof_ipc_stream_posn *posn, uint32_t type, uint32_t id) @@ -341,9 +340,12 @@ __cold static int ipc4_create_pipeline(struct ipc4_pipeline_create *pipe_desc, struct ipc_comp_dev *ipc_pipe; struct pipeline *pipe; struct ipc *ipc = ipc_get(); + struct k_heap *heap = sof_sys_user_heap_get(); assert_can_be_cold(); + LOG_INF("pipe_desc %x, instance %u", pipe_desc, pipe_desc->primary.r.instance_id); + /* check whether pipeline id is already taken or in use */ ipc_pipe = ipc_get_pipeline_by_id(ipc, pipe_desc->primary.r.instance_id); if (ipc_pipe) { @@ -353,8 +355,9 @@ __cold static int ipc4_create_pipeline(struct ipc4_pipeline_create *pipe_desc, } /* create the pipeline */ - pipe = pipeline_new(NULL, pipe_desc->primary.r.instance_id, + pipe = pipeline_new(heap, pipe_desc->primary.r.instance_id, pipe_desc->primary.r.ppl_priority, 0, pparams); + LOG_INF("pipeline_new() -> %p", pipe); if (!pipe) { tr_err(&ipc_tr, "ipc: pipeline_new() failed"); return IPC4_OUT_OF_MEMORY; @@ -369,12 +372,13 @@ __cold static int ipc4_create_pipeline(struct ipc4_pipeline_create *pipe_desc, pipe->core = pipe_desc->extension.r.core_id; /* allocate the IPC pipeline container */ - ipc_pipe = rzalloc(SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, - sizeof(struct ipc_comp_dev)); + ipc_pipe = sof_heap_alloc(heap, SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, + sizeof(struct ipc_comp_dev), 0); if (!ipc_pipe) { pipeline_free(pipe); return IPC4_OUT_OF_MEMORY; } + memset(ipc_pipe, 0, sizeof(*ipc_pipe)); ipc_pipe->pipeline = pipe; ipc_pipe->type = COMP_TYPE_PIPELINE; @@ -385,6 +389,8 @@ __cold static int ipc4_create_pipeline(struct ipc4_pipeline_create *pipe_desc, /* add new pipeline to the list */ list_item_append(&ipc_pipe->list, &ipc->comp_list); + LOG_INF("success"); + return IPC4_SUCCESS; } @@ -541,7 +547,7 @@ __cold int ipc_pipeline_free(struct ipc *ipc, uint32_t comp_id) ipc_pipe->pipeline = NULL; list_item_del(&ipc_pipe->list); - rfree(ipc_pipe); + sof_heap_free(sof_sys_user_heap_get(), ipc_pipe); return IPC4_SUCCESS; } @@ -1089,7 +1095,7 @@ __cold int ipc4_chain_dma_state(struct comp_dev *dev, struct ipc4_chain_dma *cdm if (icd->cd != dev) continue; list_item_del(&icd->list); - rfree(icd); + sof_heap_free(sof_sys_user_heap_get(), icd); break; } comp_free(dev); @@ -1305,7 +1311,7 @@ struct comp_dev *ipc4_get_comp_dev(uint32_t comp_id) } EXPORT_SYMBOL(ipc4_get_comp_dev); -__cold static int ipc4_add_comp_dev(struct comp_dev *dev) +__cold int ipc4_add_comp_dev(struct comp_dev *dev) { struct ipc *ipc = ipc_get(); struct ipc_comp_dev *icd; @@ -1320,12 +1326,13 @@ __cold static int ipc4_add_comp_dev(struct comp_dev *dev) } /* allocate the IPC component container */ - icd = rzalloc(SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, - sizeof(struct ipc_comp_dev)); + icd = sof_heap_alloc(sof_sys_user_heap_get(), SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, + sizeof(struct ipc_comp_dev), 0); if (!icd) { tr_err(&ipc_tr, "alloc failed"); return IPC4_OUT_OF_MEMORY; } + memset(icd, 0, sizeof(*icd)); icd->cd = dev; icd->type = COMP_TYPE_COMPONENT; From cef6a8723af78e050f9bd449c689d6d79fbacc6b Mon Sep 17 00:00:00 2001 From: Kai Vehmanen Date: Thu, 18 Jun 2026 14:35:27 +0300 Subject: [PATCH 033/108] ipc: implement user-space IPC handling thread Add a dedicated user-space thread for handling IPC commands that operate on audio pipelines when CONFIG_SOF_USERSPACE_LL is enabled. The kernel IPC handler forwards IPC4 messages to the user thread via k_event signaling and collects the result through a k_sem handshake. The IPC task_mask IPC_TASK_IN_THREAD bit prevents host completion until the user thread finishes. Component creation (drv->ops.create) runs in the user thread so untrusted module code does not execute with kernel privileges; the kernel resolves the driver and copies it into a user-readable buffer first. HOSTBOX partitions are mapped into the user thread's memory domain for module init parameter reads. Current implementation only covers IPC4, but the generic infra can be extended to cover other major IPC protocol variants. Signed-off-by: Kai Vehmanen --- src/include/sof/ipc/common.h | 10 ++ src/include/sof/ipc/topology.h | 6 + src/ipc/ipc-common.c | 313 +++++++++++++++++++++++++++++++++ src/ipc/ipc4/handler-user.c | 140 +++++++++++++++ src/ipc/ipc4/helper.c | 99 +++++++++++ zephyr/Kconfig | 8 + zephyr/lib/userspace_helper.c | 37 ++++ 7 files changed, 613 insertions(+) diff --git a/src/include/sof/ipc/common.h b/src/include/sof/ipc/common.h index 8726f53b8fdf..fc749f0b33a9 100644 --- a/src/include/sof/ipc/common.h +++ b/src/include/sof/ipc/common.h @@ -346,4 +346,14 @@ void ipc_complete_cmd(struct ipc *ipc); /* GDB stub: should enter GDB after completing the IPC processing */ extern bool ipc_enter_gdb; +#ifdef CONFIG_SOF_USERSPACE_LL +struct ipc4_message_request; +/** + * @brief Forward an IPC4 command to the user-space thread. + * @param ipc4 Pointer to the IPC4 message request + * @return Result from user thread processing + */ +int ipc_user_forward_cmd(struct ipc4_message_request *ipc4); +#endif + #endif /* __SOF_DRIVERS_IPC_H__ */ diff --git a/src/include/sof/ipc/topology.h b/src/include/sof/ipc/topology.h index 6bfad1cdb49a..a76c299aa60e 100644 --- a/src/include/sof/ipc/topology.h +++ b/src/include/sof/ipc/topology.h @@ -50,6 +50,12 @@ struct ipc_comp_dev; const struct comp_driver *ipc4_get_comp_drv(uint32_t module_id); struct comp_dev *ipc4_get_comp_dev(uint32_t comp_id); int ipc4_add_comp_dev(struct comp_dev *dev); +#ifdef CONFIG_SOF_USERSPACE_LL +struct ipc4_message_request; +struct comp_driver; +struct comp_dev *comp_new_ipc4_user(struct ipc4_message_request *ipc4, + const struct comp_driver *drv); +#endif int ipc4_chain_manager_create(struct ipc4_chain_dma *cdma); int ipc4_chain_dma_state(struct comp_dev *dev, struct ipc4_chain_dma *cdma); int ipc4_create_chain_dma(struct ipc *ipc, struct ipc4_chain_dma *cdma); diff --git a/src/ipc/ipc-common.c b/src/ipc/ipc-common.c index 3177201af493..6e073628623d 100644 --- a/src/ipc/ipc-common.c +++ b/src/ipc/ipc-common.c @@ -44,6 +44,9 @@ #ifdef CONFIG_SOF_USERSPACE_LL #include #include +#include +#include +#include #endif #include @@ -279,7 +282,11 @@ void ipc_msg_send(struct ipc_msg *msg, void *data, bool high_priority) list_item_append(&msg->list, &ipc->msg_list); } +#if 0 /*def CONFIG_SOF_USERSPACE_LL */ + LOG_WRN("Skipping IPC worker schedule. TODO to fix\n"); +#else schedule_ipc_worker(); +#endif k_spin_unlock(&ipc->lock, key); } @@ -311,10 +318,316 @@ void ipc_schedule_process(struct ipc *ipc) #endif } +#ifdef CONFIG_SOF_USERSPACE_LL +/* User-space thread for pipeline_two_components test */ + +#define IPC_USER_EVENT_CMD BIT(0) +#define IPC_USER_EVENT_STOP BIT(1) + +static struct k_thread ipc_user_thread; +static K_THREAD_STACK_DEFINE(ipc_user_stack, CONFIG_SOF_IPC_USER_THREAD_STACK_SIZE); + +/** + * @brief Forward an IPC4 command to the user-space thread. + * + * Called from kernel context (IPC EDF task) to forward the IPC4 + * message to the user-space thread for processing. Sets + * IPC_TASK_IN_THREAD in task_mask so the host is not signaled + * until the user thread completes. Blocks until the user thread + * finishes processing and returns the result. + * + * @param ipc4 Pointer to the IPC4 message request + * @return Result from user thread processing + */ +int ipc_user_forward_cmd(struct ipc4_message_request *ipc4) +{ + struct ipc *ipc = ipc_get(); + struct ipc_user *pdata = ipc->ipc_user_pdata; + k_spinlock_key_t key; + int ret; + + LOG_DBG("IPC: forward cmd %08x", ipc4->primary.dat); + + /* Copy message words — original buffer may be reused */ + pdata->ipc_msg_pri = ipc4->primary.dat; + pdata->ipc_msg_ext = ipc4->extension.dat; + pdata->ipc = ipc; + + /* Prevent host completion until user thread finishes */ + key = k_spin_lock(&ipc->lock); + ipc->task_mask |= IPC_TASK_IN_THREAD; + k_spin_unlock(&ipc->lock, key); + + /* Wake the user thread */ + k_event_set(pdata->event, IPC_USER_EVENT_CMD); + + /* Wait for user thread to complete */ + ret = k_sem_take(pdata->sem, K_MSEC(10)); + if (ret) { + LOG_ERR("IPC user: sem error %d\n", ret); + return ret; + } + + /* Clear the task mask bit and check for completion */ + key = k_spin_lock(&ipc->lock); + ipc->task_mask &= ~IPC_TASK_IN_THREAD; + ipc_complete_cmd(ipc); + k_spin_unlock(&ipc->lock, key); + + return pdata->result; +} + +/** + * User-space thread entry point for pipeline_two_components test. + * p1 points to the ppl_test_ctx shared with the kernel launcher. + */ +static void ipc_user_thread_fn(void *p1, void *p2, void *p3) +{ + struct ipc_user *ipc_user = p1; + + ARG_UNUSED(p2); + ARG_UNUSED(p3); + + __ASSERT(k_is_user_context(), "expected user context"); + + /* Signal startup complete — unblocks init waiting on semaphore */ + k_sem_give(ipc_user->sem); + LOG_INF("IPC user-space thread started"); + + for (;;) { + uint32_t mask = k_event_wait_safe(ipc_user->event, + IPC_USER_EVENT_CMD | IPC_USER_EVENT_STOP, + false, K_MSEC(5000)); + + LOG_DBG("IPC user wake, mask %u", mask); + + if (mask & IPC_USER_EVENT_CMD) { + struct ipc4_message_request msg; + + /* Reconstruct the IPC4 message from copied words */ + msg.primary.dat = ipc_user->ipc_msg_pri; + msg.extension.dat = ipc_user->ipc_msg_ext; + + ipc_user->reply_ext = 0; + + if (msg.primary.r.msg_tgt == SOF_IPC4_MESSAGE_TARGET_MODULE_MSG) { + /* Module message dispatch */ + switch (msg.primary.r.type) { + case SOF_IPC4_MOD_CONFIG_GET: + ipc_user->result = + ipc4_process_module_config( + &msg, false, + &ipc_user->reply_ext); + break; + case SOF_IPC4_MOD_CONFIG_SET: + ipc_user->result = + ipc4_process_module_config( + &msg, true, NULL); + break; + case SOF_IPC4_MOD_BIND: { + struct ipc4_module_bind_unbind bu; + + memcpy_s(&bu, sizeof(bu), &msg, sizeof(msg)); + ipc_user->result = ipc_comp_connect( + ipc_user->ipc, + (ipc_pipe_comp_connect *)&bu); + break; + } + case SOF_IPC4_MOD_UNBIND: { + struct ipc4_module_bind_unbind bu; + + memcpy_s(&bu, sizeof(bu), &msg, sizeof(msg)); + ipc_user->result = ipc_comp_disconnect( + ipc_user->ipc, + (ipc_pipe_comp_connect *)&bu); + break; + } + case SOF_IPC4_MOD_INIT_INSTANCE: { + /* User thread creates the component — + * drv->ops.create() runs in user-space so + * untrusted module code does not execute + * with kernel privileges. + * + * init_drv = original kernel pointer + * init_drv_data = user-accessible copy + */ + const struct comp_driver *orig_drv = + ipc_user->init_drv; + const struct comp_driver *drv_copy = + (const struct comp_driver *) + ipc_user->init_drv_data; + + ipc_user->init_drv = NULL; + if (!orig_drv) { + ipc_user->result = + IPC4_MOD_NOT_INITIALIZED; + break; + } + + struct comp_dev *dev = + comp_new_ipc4_user(&msg, drv_copy); + + if (!dev) { + ipc_user->result = + IPC4_MOD_NOT_INITIALIZED; + break; + } + + /* Restore original kernel driver pointer. + * comp_init() set dev->drv to the copy; + * runtime code expects the canonical + * kernel address. + */ + dev->drv = orig_drv; + + ipc_user->result = + ipc4_add_comp_dev(dev); + if (ipc_user->result != IPC4_SUCCESS) + break; + + comp_update_ibs_obs_cpc(dev); + ipc_user->result = 0; + break; + } + case SOF_IPC4_MOD_DELETE_INSTANCE: { + struct ipc4_module_delete_instance module; + + memcpy_s(&module, sizeof(module), &msg, sizeof(msg)); + uint32_t comp_id = IPC4_COMP_ID( + module.primary.r.module_id, + module.primary.r.instance_id); + ipc_user->result = ipc_comp_free( + ipc_user->ipc, comp_id); + if (ipc_user->result < 0) + ipc_user->result = + IPC4_INVALID_RESOURCE_ID; + break; + } + case SOF_IPC4_MOD_LARGE_CONFIG_GET: + ipc_user->result = + ipc4_process_large_config_get( + &msg, + &ipc_user->reply_ext, + &ipc_user->reply_tx_size, + &ipc_user->reply_tx_data); + break; + case SOF_IPC4_MOD_LARGE_CONFIG_SET: + ipc_user->result = + ipc4_process_large_config_set( + &msg); + break; + default: + LOG_ERR("IPC user: unsupported module cmd type %d", + msg.primary.r.type); + ipc_user->result = -EINVAL; + break; + } + } else { + /* Global message dispatch */ + switch (msg.primary.r.type) { + case SOF_IPC4_GLB_CREATE_PIPELINE: + ipc_user->result = + ipc_pipeline_new(ipc_user->ipc, + (ipc_pipe_new *)&msg); + break; + case SOF_IPC4_GLB_DELETE_PIPELINE: { + struct ipc4_pipeline_delete *pipe = + (struct ipc4_pipeline_delete *)&msg; + ipc_user->result = + ipc_pipeline_free( + ipc_user->ipc, + pipe->primary.r.instance_id); + break; + } + case SOF_IPC4_GLB_SET_PIPELINE_STATE: + ipc_user->result = + ipc4_set_pipeline_state(&msg); + break; + default: + LOG_ERR("IPC user: unsupported glb cmd type %d", + msg.primary.r.type); + ipc_user->result = -EINVAL; + break; + } + } + + /* Signal completion — kernel side will finish IPC */ + k_sem_give(ipc_user->sem); + } + + if (mask & IPC_USER_EVENT_STOP) + break; + } +} + +__cold int ipc_user_init(void) +{ + struct ipc *ipc = ipc_get(); + struct ipc_user *ipc_user = sof_heap_alloc(sof_sys_user_heap_get(), SOF_MEM_FLAG_USER, + sizeof(*ipc_user), 0); + int ret; + + ipc_user->sem = k_object_alloc(K_OBJ_SEM); + if (!ipc_user->sem) { + LOG_ERR("user IPC sem alloc failed"); + k_panic(); + } + + ret = k_mem_domain_add_partition(zephyr_ll_mem_domain(), &ipc_context_part); + + k_sem_init(ipc_user->sem, 0, 1); + + /* Allocate kernel objects for the user-space thread */ + ipc_user->event = k_object_alloc(K_OBJ_EVENT); + if (!ipc_user->event) { + LOG_ERR("user IPC event alloc failed"); + k_panic(); + } + k_event_init(ipc_user->event); + + k_thread_create(&ipc_user_thread, ipc_user_stack, + CONFIG_SOF_IPC_USER_THREAD_STACK_SIZE, + ipc_user_thread_fn, ipc_user, NULL, NULL, + -1, K_USER, K_FOREVER); + + ipc_user->thread = &ipc_user_thread; + k_thread_access_grant(&ipc_user_thread, ipc_user->sem, ipc_user->event); + user_grant_dai_access_all(&ipc_user_thread); + user_grant_dma_access_all(&ipc_user_thread); + user_access_to_mailbox(zephyr_ll_mem_domain(), &ipc_user_thread); + user_ll_grant_access(&ipc_user_thread, PLATFORM_PRIMARY_CORE_ID); + k_mem_domain_add_thread(zephyr_ll_mem_domain(), &ipc_user_thread); + + k_thread_cpu_pin(&ipc_user_thread, PLATFORM_PRIMARY_CORE_ID); + k_thread_name_set(&ipc_user_thread, "ipc_user"); + + /* Store references in ipc struct so kernel handler can forward commands */ + ipc->ipc_user_pdata = ipc_user; + + k_thread_start(&ipc_user_thread); + + struct task *task = zephyr_ll_task_alloc(); + schedule_task_init_ll(task, SOF_UUID(ipc_uuid), SOF_SCHEDULE_LL_TIMER, + 0, NULL, NULL, cpu_get_id(), 0); + ipc_user->audio_thread = scheduler_init_context(task); + + /* Grant ipc_user thread permission on the audio thread object. + * Needed so user-space dai_common_new() can call + * k_thread_access_grant(audio_thread, dai_mutex) from user context. + */ + k_thread_access_grant(&ipc_user_thread, ipc_user->audio_thread); + + /* Wait for user thread startup — consumes the initial k_sem_give from thread */ + k_sem_take(ipc->ipc_user_pdata->sem, K_FOREVER); + + return 0; +} +#else static int ipc_user_init(void) { return 0; } +#endif /* CONFIG_SOF_USERSPACE_LL */ __cold int ipc_init(struct sof *sof) { diff --git a/src/ipc/ipc4/handler-user.c b/src/ipc/ipc4/handler-user.c index f877094e5ad7..8a390cea621f 100644 --- a/src/ipc/ipc4/handler-user.c +++ b/src/ipc/ipc4/handler-user.c @@ -90,6 +90,7 @@ static inline const struct ipc4_pipeline_set_state_data *ipc4_get_pipeline_data( /* * Global IPC Operations. */ +#ifndef CONFIG_SOF_USERSPACE_LL __cold static int ipc4_new_pipeline(struct ipc4_message_request *ipc4) { struct ipc *ipc = ipc_get(); @@ -98,7 +99,9 @@ __cold static int ipc4_new_pipeline(struct ipc4_message_request *ipc4) return ipc_pipeline_new(ipc, (ipc_pipe_new *)ipc4); } +#endif +#ifndef CONFIG_SOF_USERSPACE_LL __cold static int ipc4_delete_pipeline(struct ipc4_message_request *ipc4) { struct ipc4_pipeline_delete *pipe; @@ -111,6 +114,7 @@ __cold static int ipc4_delete_pipeline(struct ipc4_message_request *ipc4) return ipc_pipeline_free(ipc, pipe->primary.r.instance_id); } +#endif static int ipc4_pcm_params(struct ipc_comp_dev *pcm_dev) { @@ -689,13 +693,26 @@ int ipc4_user_process_glb_message(struct ipc4_message_request *ipc4, /* pipeline settings */ case SOF_IPC4_GLB_CREATE_PIPELINE: + /* Implementation in progress: forward only CREATE_PIPELINE for now */ +#ifdef CONFIG_SOF_USERSPACE_LL + ret = ipc_user_forward_cmd(ipc4); +#else ret = ipc4_new_pipeline(ipc4); +#endif break; case SOF_IPC4_GLB_DELETE_PIPELINE: +#ifdef CONFIG_SOF_USERSPACE_LL + ret = ipc_user_forward_cmd(ipc4); +#else ret = ipc4_delete_pipeline(ipc4); +#endif break; case SOF_IPC4_GLB_SET_PIPELINE_STATE: +#ifdef CONFIG_SOF_USERSPACE_LL + ret = ipc_user_forward_cmd(ipc4); +#else ret = ipc4_set_pipeline_state(ipc4); +#endif break; case SOF_IPC4_GLB_GET_PIPELINE_STATE: @@ -1503,28 +1520,151 @@ __cold int ipc4_user_process_module_message(struct ipc4_message_request *ipc4, switch (type) { case SOF_IPC4_MOD_INIT_INSTANCE: +#ifdef CONFIG_SOF_USERSPACE_LL + { + /* User-space init: kernel does driver lookup only (requires + * access to IMR manifest and driver list in kernel memory). + * Component creation (drv->ops.create) runs in user thread + * so untrusted module code does not execute in kernel context. + * Cross-core creation stays fully in kernel. + */ + struct ipc4_module_init_instance mi; + + BUILD_ASSERT(sizeof(struct comp_driver) + sizeof(struct tr_ctx) <= + sizeof(((struct ipc_user *)0)->init_drv_data), + "ipc_user.init_drv_data too small for driver copy"); + + memcpy_s(&mi, sizeof(mi), ipc4, sizeof(*ipc4)); + if (!cpu_is_me(mi.extension.r.core_id)) { + ret = ipc4_init_module_instance(ipc4); + } else { + struct ipc *ipc = ipc_get(); + uint32_t comp_id = IPC4_COMP_ID(mi.primary.r.module_id, + mi.primary.r.instance_id); + const struct comp_driver *drv = ipc4_get_comp_drv( + IPC4_MOD_ID(comp_id)); + + if (!drv) { + ret = IPC4_MOD_NOT_INITIALIZED; + } else { + struct ipc_user *pdata = ipc->ipc_user_pdata; + + /* Copy comp_driver and tr_ctx into + * user-accessible ipc_user buffer — + * originals are in kernel .rodata/.data + * and not readable from user mode. + */ + struct comp_driver *drv_copy = + (struct comp_driver *)pdata->init_drv_data; + struct tr_ctx *tctx_copy = + (struct tr_ctx *)(pdata->init_drv_data + + sizeof(struct comp_driver)); + + memcpy_s(drv_copy, sizeof(*drv_copy), + drv, sizeof(*drv)); + if (drv->tctx) { + memcpy_s(tctx_copy, sizeof(*tctx_copy), + drv->tctx, sizeof(*drv->tctx)); + drv_copy->tctx = tctx_copy; + } + + pdata->init_drv = drv; + ret = ipc_user_forward_cmd(ipc4); + } + } + } +#else ret = ipc4_init_module_instance(ipc4); +#endif break; case SOF_IPC4_MOD_CONFIG_GET: +#ifdef CONFIG_SOF_USERSPACE_LL + /* Forward to user thread for privilege-separated execution */ + ret = ipc_user_forward_cmd(ipc4); + if (!ret) { + struct ipc *ipc = ipc_get(); + struct ipc_user *pdata = ipc->ipc_user_pdata; + + msg_reply->extension = pdata->reply_ext; + } +#else ret = ipc4_set_get_config_module_instance(ipc4, false); +#endif break; case SOF_IPC4_MOD_CONFIG_SET: +#ifdef CONFIG_SOF_USERSPACE_LL + /* Forward to user thread for privilege-separated execution */ + ret = ipc_user_forward_cmd(ipc4); +#else ret = ipc4_set_get_config_module_instance(ipc4, true); +#endif break; case SOF_IPC4_MOD_LARGE_CONFIG_GET: +#ifdef CONFIG_SOF_USERSPACE_LL + { + struct ipc4_module_large_config config; + + memcpy_s(&config, sizeof(config), ipc4, sizeof(*ipc4)); + if (config.primary.r.module_id) { + /* Module case: forward to user thread */ + ret = ipc_user_forward_cmd(ipc4); + if (!ret) { + struct ipc *ipc = ipc_get(); + struct ipc_user *pdata = ipc->ipc_user_pdata; + + msg_reply->extension = pdata->reply_ext; + msg_reply->tx_size = pdata->reply_tx_size; + msg_reply->tx_data = pdata->reply_tx_data; + } + } else { + /* Base firmware (module_id==0): keep in kernel — + * ipc4_get_comp_drv() accesses IMR manifest which + * has no user-space partition. + */ + ret = ipc4_get_large_config_module_instance(ipc4); + } + } +#else ret = ipc4_get_large_config_module_instance(ipc4); +#endif break; case SOF_IPC4_MOD_LARGE_CONFIG_SET: +#ifdef CONFIG_SOF_USERSPACE_LL + { + struct ipc4_module_large_config config; + + memcpy_s(&config, sizeof(config), ipc4, sizeof(*ipc4)); + if (config.primary.r.module_id) { + ret = ipc_user_forward_cmd(ipc4); + } else { + /* Base firmware: keep in kernel (IMR access) */ + ret = ipc4_set_large_config_module_instance(ipc4); + } + } +#else ret = ipc4_set_large_config_module_instance(ipc4); +#endif break; case SOF_IPC4_MOD_BIND: +#ifdef CONFIG_SOF_USERSPACE_LL + ret = ipc_user_forward_cmd(ipc4); +#else ret = ipc4_bind_module_instance(ipc4); +#endif break; case SOF_IPC4_MOD_UNBIND: +#ifdef CONFIG_SOF_USERSPACE_LL + ret = ipc_user_forward_cmd(ipc4); +#else ret = ipc4_unbind_module_instance(ipc4); +#endif break; case SOF_IPC4_MOD_DELETE_INSTANCE: +#ifdef CONFIG_SOF_USERSPACE_LL + ret = ipc_user_forward_cmd(ipc4); +#else ret = ipc4_delete_module_instance(ipc4); +#endif break; case SOF_IPC4_MOD_ENTER_MODULE_RESTORE: case SOF_IPC4_MOD_EXIT_MODULE_RESTORE: diff --git a/src/ipc/ipc4/helper.c b/src/ipc/ipc4/helper.c index 2cca34b948e4..2b9312e03dba 100644 --- a/src/ipc/ipc4/helper.c +++ b/src/ipc/ipc4/helper.c @@ -208,6 +208,105 @@ __cold struct comp_dev *comp_new_ipc4(struct ipc4_module_init_instance *module_i return dev; } +#ifdef CONFIG_SOF_USERSPACE_LL +/** + * comp_new_ipc4_user - Create component in user-space IPC thread context. + * + * Called from the user-space IPC thread. Receives a pre-resolved driver + * pointer from the kernel handler. Performs IPC4 message parsing, HOSTBOX + * data read, and calls drv->ops.create() in user-space context. + * + * @param ipc4 IPC4 message request (reconstructed from ipc_user pri/ext words) + * @param drv Component driver resolved by kernel via ipc4_get_comp_drv() + * @return Created component device, or NULL on failure + */ +__cold struct comp_dev *comp_new_ipc4_user(struct ipc4_message_request *ipc4, + const struct comp_driver *drv) +{ + struct ipc4_module_init_instance module_init; + struct comp_ipc_config ipc_config; + struct comp_dev *dev; + uint32_t comp_id; + char *data; + int ret; + + assert_can_be_cold(); + + ret = memcpy_s(&module_init, sizeof(module_init), ipc4, sizeof(*ipc4)); + if (ret < 0) + return NULL; + + comp_id = IPC4_COMP_ID(module_init.primary.r.module_id, + module_init.primary.r.instance_id); + + if (ipc4_get_comp_dev(comp_id)) { + tr_err(&ipc_tr, "comp 0x%x exists", comp_id); + return NULL; + } + + if (module_init.extension.r.core_id >= CONFIG_CORE_COUNT) { + tr_err(&ipc_tr, "ipc: comp->core = %u", + (uint32_t)module_init.extension.r.core_id); + return NULL; + } + + memset(&ipc_config, 0, sizeof(ipc_config)); + ipc_config.id = comp_id; + ipc_config.pipeline_id = module_init.extension.r.ppl_instance_id; + ipc_config.core = module_init.extension.r.core_id; + ipc_config.ipc_config_size = + module_init.extension.r.param_block_size * sizeof(uint32_t); + ipc_config.ipc_extended_init = module_init.extension.r.extended_init; + if (ipc_config.ipc_config_size > MAILBOX_HOSTBOX_SIZE) { + tr_err(&ipc_tr, + "IPC payload size %u too big for the message window", + ipc_config.ipc_config_size); + return NULL; + } +#ifdef CONFIG_DCACHE_LINE_SIZE + if (!IS_ENABLED(CONFIG_LIBRARY)) + sys_cache_data_invd_range( + (__sparse_force void __sparse_cache *)MAILBOX_HOSTBOX_BASE, + ALIGN_UP(ipc_config.ipc_config_size, + CONFIG_DCACHE_LINE_SIZE)); +#endif + data = ipc4_get_comp_new_data(); + +#if CONFIG_ZEPHYR_DP_SCHEDULER + if (module_init.extension.r.proc_domain) + ipc_config.proc_domain = COMP_PROCESSING_DOMAIN_DP; + else + ipc_config.proc_domain = COMP_PROCESSING_DOMAIN_LL; +#else + if (module_init.extension.r.proc_domain) { + tr_err(&ipc_tr, + "ipc: DP scheduling is disabled, cannot create comp 0x%x", + comp_id); + return NULL; + } + ipc_config.proc_domain = COMP_PROCESSING_DOMAIN_LL; +#endif + + if (drv->type == SOF_COMP_MODULE_ADAPTER) { + const struct ipc_config_process spec = { + .data = (const unsigned char *)data, + .size = ipc_config.ipc_config_size, + }; + + dev = drv->ops.create(drv, &ipc_config, (const void *)&spec); + } else { + dev = drv->ops.create(drv, &ipc_config, (const void *)data); + } + if (!dev) + return NULL; + + list_init(&dev->bsource_list); + list_init(&dev->bsink_list); + + return dev; +} +#endif /* CONFIG_SOF_USERSPACE_LL */ + /* Called from ipc4_set_pipeline_state(), so cannot be cold */ struct ipc_comp_dev *ipc_get_comp_by_ppl_id(struct ipc *ipc, uint16_t type, uint32_t ppl_id, diff --git a/zephyr/Kconfig b/zephyr/Kconfig index 9ff220efdc55..ab88efcb9a6a 100644 --- a/zephyr/Kconfig +++ b/zephyr/Kconfig @@ -327,3 +327,11 @@ config STACK_SIZE_IPC_TX default 2048 help IPC sender work-queue thread stack size. Keep a power of 2. + +config SOF_IPC_USER_THREAD_STACK_SIZE + int "IPC user thread stack size" + default 8192 + depends on SOF_USERSPACE_LL + help + Stack size for the userspace IPC thread. + Keep a power of 2. diff --git a/zephyr/lib/userspace_helper.c b/zephyr/lib/userspace_helper.c index ce4368adc077..2abc2fa0e2f2 100644 --- a/zephyr/lib/userspace_helper.c +++ b/zephyr/lib/userspace_helper.c @@ -110,6 +110,43 @@ int user_access_to_mailbox(struct k_mem_domain *domain, k_tid_t thread_id) if (ret < 0) return ret; +#if defined(CONFIG_SOF_USERSPACE_LL) && defined(CONFIG_IPC_MAJOR_4) + /* HOSTBOX partitions for IPC4 module init parameter block reads. + * comp_new_ipc4() accesses MAILBOX_HOSTBOX_BASE directly to get + * the module configuration data sent by the host. + */ + { + struct k_mem_partition hostbox_partition; + + /* Uncached HOSTBOX partition */ + hostbox_partition.start = + (uintptr_t)sys_cache_uncached_ptr_get( + (void __sparse_cache *)MAILBOX_HOSTBOX_BASE); + hostbox_partition.size = ALIGN_UP(MAILBOX_HOSTBOX_SIZE, + CONFIG_MMU_PAGE_SIZE); + hostbox_partition.attr = K_MEM_PARTITION_P_RO_U_RO; + + ret = k_mem_domain_add_partition(domain, &hostbox_partition); + if (ret < 0) + return ret; + + /* Cached HOSTBOX partition for cache invalidation path. + * sys_cache_data_invd_range() syscall verification requires + * write access to the region, so use RW instead of RO. + */ + hostbox_partition.start = + (uintptr_t)sys_cache_cached_ptr_get( + (void *)MAILBOX_HOSTBOX_BASE); + hostbox_partition.size = ALIGN_UP(MAILBOX_HOSTBOX_SIZE, + CONFIG_MMU_PAGE_SIZE); + hostbox_partition.attr = K_MEM_PARTITION_P_RW_U_RW; + + ret = k_mem_domain_add_partition(domain, &hostbox_partition); + if (ret < 0) + return ret; + } +#endif /* CONFIG_IPC_MAJOR_4 */ + #ifndef CONFIG_IPC_MAJOR_4 /* * Next mailbox_stream (not available in IPC4). Stream access is cached, From 38950e6950fb96109ac9b8506df37153abb076f5 Mon Sep 17 00:00:00 2001 From: Kai Vehmanen Date: Thu, 25 Jun 2026 14:12:12 +0300 Subject: [PATCH 034/108] zephyr: lib: make vregion_alloc/free system calls Make vregion_alloc(), vregion_alloc_coherent(), vregion_alloc_align(), vregion_alloc_coherent_align(), and vregion_free() available as Zephyr system calls for user-space threads. Add K_SYSCALL_MEMORY_WRITE verification to all syscall handlers to validate the calling thread has access to the vregion's managed memory area. Add CONFIG_SOF_USERSPACE_INTERFACE_VREGION Kconfig option to control the feature. It is auto-selected by SOF_USERSPACE_LL when SOF_VREGIONS is enabled. Signed-off-by: Kai Vehmanen --- src/include/sof/lib/vregion.h | 26 ++++++++++--- zephyr/CMakeLists.txt | 2 + zephyr/Kconfig | 9 +++++ zephyr/lib/vregion.c | 25 ++++++------ zephyr/syscall/vregion.c | 73 +++++++++++++++++++++++++++++++++++ 5 files changed, 119 insertions(+), 16 deletions(-) create mode 100644 zephyr/syscall/vregion.c diff --git a/src/include/sof/lib/vregion.h b/src/include/sof/lib/vregion.h index 5c066c90dbc8..93233bd2caf6 100644 --- a/src/include/sof/lib/vregion.h +++ b/src/include/sof/lib/vregion.h @@ -6,6 +6,8 @@ #define __SOF_LIB_VREGION_H__ #include +#include +#include #ifdef __cplusplus extern "C" { @@ -80,12 +82,16 @@ struct vregion *vregion_put(struct vregion *vr); * @param[in] size Size of memory to allocate in bytes. * @return void* Pointer to the allocated memory, or NULL on failure. */ -void *vregion_alloc(struct vregion *vr, size_t size); +__syscall void *vregion_alloc(struct vregion *vr, size_t size); + +void *z_impl_vregion_alloc(struct vregion *vr, size_t size); /** * @brief like vregion_alloc() but allocates coherent memory */ -void *vregion_alloc_coherent(struct vregion *vr, size_t size); +__syscall void *vregion_alloc_coherent(struct vregion *vr, size_t size); + +void *z_impl_vregion_alloc_coherent(struct vregion *vr, size_t size); /** * @brief Allocate aligned memory from the specified virtual region. @@ -98,12 +104,16 @@ void *vregion_alloc_coherent(struct vregion *vr, size_t size); * @param[in] alignment Alignment of memory to allocate in bytes. * @return void* Pointer to the allocated memory, or NULL on failure. */ -void *vregion_alloc_align(struct vregion *vr, size_t size, size_t alignment); +__syscall void *vregion_alloc_align(struct vregion *vr, size_t size, size_t alignment); + +void *z_impl_vregion_alloc_align(struct vregion *vr, size_t size, size_t alignment); /** * @brief like vregion_alloc_align() but allocates coherent memory */ -void *vregion_alloc_coherent_align(struct vregion *vr, size_t size, size_t alignment); +__syscall void *vregion_alloc_coherent_align(struct vregion *vr, size_t size, size_t alignment); + +void *z_impl_vregion_alloc_coherent_align(struct vregion *vr, size_t size, size_t alignment); /** * @brief Free memory allocated from the specified virtual region. @@ -113,7 +123,9 @@ void *vregion_alloc_coherent_align(struct vregion *vr, size_t size, size_t align * @param[in] vr Pointer to the virtual region instance. * @param[in] ptr Pointer to the memory to free. */ -void vregion_free(struct vregion *vr, void *ptr); +__syscall void vregion_free(struct vregion *vr, void *ptr); + +void z_impl_vregion_free(struct vregion *vr, void *ptr); /** * @brief Log virtual region memory usage. @@ -181,4 +193,8 @@ static inline void vregion_mem_info(struct vregion *vr, size_t *size, uintptr_t } #endif +#if CONFIG_SOF_VREGIONS +#include +#endif + #endif /* __SOF_LIB_VREGION_H__ */ diff --git a/zephyr/CMakeLists.txt b/zephyr/CMakeLists.txt index 4b61a9517d46..d715a24a1da8 100644 --- a/zephyr/CMakeLists.txt +++ b/zephyr/CMakeLists.txt @@ -626,6 +626,8 @@ zephyr_syscall_header(${SOF_SRC_PATH}/include/sof/ipc/ipc_reply.h) zephyr_syscall_header(${SOF_SRC_PATH}/include/ipc4/handler.h) zephyr_syscall_header(include/rtos/alloc.h) zephyr_library_sources_ifdef(CONFIG_SOF_USERSPACE_INTERFACE_ALLOC syscall/alloc.c) +zephyr_syscall_header(${SOF_SRC_PATH}/include/sof/lib/vregion.h) +zephyr_library_sources(syscall/vregion.c) zephyr_syscall_header(${SOF_SRC_PATH}/include/sof/lib/dai-zephyr.h) zephyr_library_sources_ifdef(CONFIG_USERSPACE syscall/dai.c) diff --git a/zephyr/Kconfig b/zephyr/Kconfig index ab88efcb9a6a..2d03a6d6f7ae 100644 --- a/zephyr/Kconfig +++ b/zephyr/Kconfig @@ -36,11 +36,20 @@ config SOF_USERSPACE_INTERFACE_ALLOC Allow user-space threads to use sof_heap_alloc/sof_heap_free as Zephyr system calls. +config SOF_USERSPACE_INTERFACE_VREGION + bool "Enable SOF vregion interface to userspace threads" + depends on USERSPACE + depends on SOF_VREGIONS + help + Allow user-space threads to use vregion_alloc/vregion_free + and their variants as Zephyr system calls. + config SOF_USERSPACE_LL bool "Run Low-Latency pipelines in userspace threads" depends on USERSPACE select SOF_USERSPACE_INTERFACE_ALLOC select SOF_USERSPACE_INTERFACE_DMA + select SOF_USERSPACE_INTERFACE_VREGION if SOF_VREGIONS help Run Low-Latency (LL) pipelines in userspace threads. This adds memory protection between operating system resources and diff --git a/zephyr/lib/vregion.c b/zephyr/lib/vregion.c index 9c8c94c23f97..ff7d3ef0f98d 100644 --- a/zephyr/lib/vregion.c +++ b/zephyr/lib/vregion.c @@ -365,7 +365,7 @@ static void lifetime_free(struct vlinear_heap *heap, void *ptr) * @param vr Pointer to the virtual region instance. * @param ptr Pointer to the memory to free. */ -void vregion_free(struct vregion *vr, void *ptr) +void z_impl_vregion_free(struct vregion *vr, void *ptr) { if (!vr || !ptr) return; @@ -390,7 +390,7 @@ void vregion_free(struct vregion *vr, void *ptr) k_mutex_unlock(&vr->lock); } -EXPORT_SYMBOL(vregion_free); +EXPORT_SYMBOL(z_impl_vregion_free); /** * @brief Allocate memory from the virtual region. @@ -401,7 +401,8 @@ EXPORT_SYMBOL(vregion_free); * * @return void* Pointer to the allocated memory, or NULL on failure. */ -void *vregion_alloc_align(struct vregion *vr, size_t size, size_t alignment) +void *z_impl_vregion_alloc_align(struct vregion *vr, + size_t size, size_t alignment) { void *p; @@ -429,7 +430,7 @@ void *vregion_alloc_align(struct vregion *vr, size_t size, size_t alignment) return p; } -EXPORT_SYMBOL(vregion_alloc_align); +EXPORT_SYMBOL(z_impl_vregion_alloc_align); /** * @brief Allocate memory from the virtual region. @@ -437,17 +438,17 @@ EXPORT_SYMBOL(vregion_alloc_align); * @param[in] size Size of the allocation. * @return void* Pointer to the allocated memory, or NULL on failure. */ -void *vregion_alloc(struct vregion *vr, size_t size) +void *z_impl_vregion_alloc(struct vregion *vr, size_t size) { - return vregion_alloc_align(vr, size, 0); + return z_impl_vregion_alloc_align(vr, size, 0); } -EXPORT_SYMBOL(vregion_alloc); +EXPORT_SYMBOL(z_impl_vregion_alloc); -void *vregion_alloc_coherent(struct vregion *vr, size_t size) +void *z_impl_vregion_alloc_coherent(struct vregion *vr, size_t size) { size = ALIGN_UP(size, CONFIG_DCACHE_LINE_SIZE); - void *p = vregion_alloc_align(vr, size, CONFIG_DCACHE_LINE_SIZE); + void *p = z_impl_vregion_alloc_align(vr, size, CONFIG_DCACHE_LINE_SIZE); if (!p) return NULL; @@ -456,14 +457,15 @@ void *vregion_alloc_coherent(struct vregion *vr, size_t size) return sys_cache_uncached_ptr_get(p); } +EXPORT_SYMBOL(z_impl_vregion_alloc_coherent); -void *vregion_alloc_coherent_align(struct vregion *vr, size_t size, size_t alignment) +void *z_impl_vregion_alloc_coherent_align(struct vregion *vr, size_t size, size_t alignment) { if (alignment < CONFIG_DCACHE_LINE_SIZE) alignment = CONFIG_DCACHE_LINE_SIZE; size = ALIGN_UP(size, CONFIG_DCACHE_LINE_SIZE); - void *p = vregion_alloc_align(vr, size, alignment); + void *p = z_impl_vregion_alloc_align(vr, size, alignment); if (!p) return NULL; @@ -472,6 +474,7 @@ void *vregion_alloc_coherent_align(struct vregion *vr, size_t size, size_t align return sys_cache_uncached_ptr_get(p); } +EXPORT_SYMBOL(z_impl_vregion_alloc_coherent_align); /** * @brief Log virtual region memory usage. diff --git a/zephyr/syscall/vregion.c b/zephyr/syscall/vregion.c new file mode 100644 index 000000000000..70fb038eba05 --- /dev/null +++ b/zephyr/syscall/vregion.c @@ -0,0 +1,73 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Copyright(c) 2026 Intel Corporation. + +#include +#include +#include + +static inline void *z_vrfy_vregion_alloc(struct vregion *vr, size_t size) +{ + size_t vr_size = 0; + uintptr_t vr_start; + + vregion_mem_info(vr, &vr_size, &vr_start); + if (vr_size) + K_OOPS(K_SYSCALL_MEMORY_WRITE((void *)vr_start, vr_size)); + + return z_impl_vregion_alloc(vr, size); +} +#include + +static inline void *z_vrfy_vregion_alloc_coherent(struct vregion *vr, size_t size) +{ + size_t vr_size = 0; + uintptr_t vr_start; + + vregion_mem_info(vr, &vr_size, &vr_start); + if (vr_size) + K_OOPS(K_SYSCALL_MEMORY_WRITE((void *)vr_start, vr_size)); + + return z_impl_vregion_alloc_coherent(vr, size); +} +#include + +static inline void *z_vrfy_vregion_alloc_align(struct vregion *vr, size_t size, size_t alignment) +{ + size_t vr_size = 0; + uintptr_t vr_start; + + vregion_mem_info(vr, &vr_size, &vr_start); + if (vr_size) + K_OOPS(K_SYSCALL_MEMORY_WRITE((void *)vr_start, vr_size)); + + return z_impl_vregion_alloc_align(vr, size, alignment); +} +#include + +static inline void *z_vrfy_vregion_alloc_coherent_align(struct vregion *vr, + size_t size, size_t alignment) +{ + size_t vr_size = 0; + uintptr_t vr_start; + + vregion_mem_info(vr, &vr_size, &vr_start); + if (vr_size) + K_OOPS(K_SYSCALL_MEMORY_WRITE((void *)vr_start, vr_size)); + + return z_impl_vregion_alloc_coherent_align(vr, size, alignment); +} +#include + +static inline void z_vrfy_vregion_free(struct vregion *vr, void *ptr) +{ + size_t vr_size = 0; + uintptr_t vr_start; + + vregion_mem_info(vr, &vr_size, &vr_start); + if (vr_size) + K_OOPS(K_SYSCALL_MEMORY_WRITE((void *)vr_start, vr_size)); + + z_impl_vregion_free(vr, ptr); +} +#include From ad41e9345a260ab733eb21c0104eed8d21965409 Mon Sep 17 00:00:00 2001 From: Kai Vehmanen Date: Fri, 27 Mar 2026 12:45:25 +0200 Subject: [PATCH 035/108] (---section: IPC user support STOP) From a75e3f588fdb668dbc0c190d46c68874a7a2822a Mon Sep 17 00:00:00 2001 From: Kai Vehmanen Date: Fri, 8 May 2026 18:58:45 +0300 Subject: [PATCH 036/108] (---section: IPC notifications START) From 924b194395ffee2f125a5a2da5c169ba9c8d3347 Mon Sep 17 00:00:00 2001 From: Jyri Sarha Date: Tue, 21 Apr 2026 00:27:48 +0300 Subject: [PATCH 037/108] ipc: turn ipc_msg_send() into a system call if SOF_USERSPACE_LL=y Make ipc_msg_send() a Zephyr system call so audio processing modules running in user-space LL threads can queue IPC messages (e.g. position updates, notifications) back to the host. The change takes effect only if CONFIG_SOF_USERSPACE_LL=y. Follows the same pattern used for ipc_msg_reply(): a dedicated header with __syscall declaration, z_impl/z_vrfy split, and syscall header registration in CMakeLists.txt. The verifier validates that the msg struct is writable (the implementation touches the list linkage) and that the data buffer, when provided, is readable up to msg->tx_size bytes. Signed-off-by: Jyri Sarha (cherry picked from commit b25c15cd33674444bfc4e808592ed92e9828e809) --- src/include/sof/ipc/ipc_msg_send.h | 32 ++++++++++++++++++++++++++++++ src/include/sof/ipc/msg.h | 8 +------- src/ipc/ipc-common.c | 26 ++++++++++++++++++++++++ zephyr/CMakeLists.txt | 1 + 4 files changed, 60 insertions(+), 7 deletions(-) create mode 100644 src/include/sof/ipc/ipc_msg_send.h diff --git a/src/include/sof/ipc/ipc_msg_send.h b/src/include/sof/ipc/ipc_msg_send.h new file mode 100644 index 000000000000..c5b9b4a4448a --- /dev/null +++ b/src/include/sof/ipc/ipc_msg_send.h @@ -0,0 +1,32 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * + * Copyright(c) 2026 Intel Corporation. All rights reserved. + */ + +#ifndef __SOF_IPC_IPC_MSG_SEND_H__ +#define __SOF_IPC_IPC_MSG_SEND_H__ + +#include + +struct ipc_msg; + +/** + * \brief Queues an IPC message for transmission. + * @param msg The IPC message. + * @param data The message data. + * @param high_priority True if a high priority message. + */ +#if defined(__ZEPHYR__) && defined(CONFIG_SOF_USERSPACE_LL) +__syscall void ipc_msg_send(struct ipc_msg *msg, void *data, + bool high_priority); +#else +void z_impl_ipc_msg_send(struct ipc_msg *msg, void *data, + bool high_priority); +#define ipc_msg_send z_impl_ipc_msg_send +#endif + +#if defined(__ZEPHYR__) && defined(CONFIG_SOF_USERSPACE_LL) +#include +#endif + +#endif /* __SOF_IPC_IPC_MSG_SEND_H__ */ diff --git a/src/include/sof/ipc/msg.h b/src/include/sof/ipc/msg.h index 160a9be9ec7c..74c2b66fb916 100644 --- a/src/include/sof/ipc/msg.h +++ b/src/include/sof/ipc/msg.h @@ -108,13 +108,7 @@ static inline void ipc_msg_free(struct ipc_msg *msg) */ void ipc_send_queued_msg(void); -/** - * \brief Queues an IPC message for transmission. - * @param msg The IPC message to be freed. - * @param data The message data. - * @param high_priority True if a high priortity message. - */ -void ipc_msg_send(struct ipc_msg *msg, void *data, bool high_priority); +#include /** * \brief Send an IPC message directly for emergency. diff --git a/src/ipc/ipc-common.c b/src/ipc/ipc-common.c index 6e073628623d..46d28e5403dc 100644 --- a/src/ipc/ipc-common.c +++ b/src/ipc/ipc-common.c @@ -39,6 +39,7 @@ #ifdef __ZEPHYR__ #include +#include #endif #ifdef CONFIG_SOF_USERSPACE_LL @@ -242,7 +243,11 @@ __cold void ipc_msg_send_direct(struct ipc_msg *msg, void *data) k_spin_unlock(&ipc->lock, key); } +#ifdef CONFIG_SOF_USERSPACE_LL +void z_impl_ipc_msg_send(struct ipc_msg *msg, void *data, bool high_priority) +#else void ipc_msg_send(struct ipc_msg *msg, void *data, bool high_priority) +#endif { struct ipc *ipc = ipc_get(); k_spinlock_key_t key; @@ -292,6 +297,27 @@ void ipc_msg_send(struct ipc_msg *msg, void *data, bool high_priority) } EXPORT_SYMBOL(ipc_msg_send); +#ifdef CONFIG_SOF_USERSPACE_LL +static inline bool z_vrfy_ipc_msg_send_check_data(struct ipc_msg *msg, void *data) +{ + /* If data != NULL and tx_size > 0, verify the data buffer */ + if (data && msg->tx_size > 0) + K_OOPS(K_SYSCALL_MEMORY_READ(data, msg->tx_size)); + + return true; +} + +void z_vrfy_ipc_msg_send(struct ipc_msg *msg, void *data, bool high_priority) +{ + K_OOPS(K_SYSCALL_MEMORY_WRITE(msg, sizeof(*msg))); + + z_vrfy_ipc_msg_send_check_data(msg, data); + + z_impl_ipc_msg_send(msg, data, high_priority); +} +#include +#endif + #ifdef __ZEPHYR__ static void ipc_work_handler(struct k_work *work) { diff --git a/zephyr/CMakeLists.txt b/zephyr/CMakeLists.txt index d715a24a1da8..87078267c454 100644 --- a/zephyr/CMakeLists.txt +++ b/zephyr/CMakeLists.txt @@ -624,6 +624,7 @@ zephyr_syscall_header(${SOF_SRC_PATH}/include/sof/audio/module_adapter/module/ge zephyr_syscall_header(${SOF_SRC_PATH}/include/sof/lib/fast-get.h) zephyr_syscall_header(${SOF_SRC_PATH}/include/sof/ipc/ipc_reply.h) zephyr_syscall_header(${SOF_SRC_PATH}/include/ipc4/handler.h) +zephyr_syscall_header(${SOF_SRC_PATH}/include/sof/ipc/ipc_msg_send.h) zephyr_syscall_header(include/rtos/alloc.h) zephyr_library_sources_ifdef(CONFIG_SOF_USERSPACE_INTERFACE_ALLOC syscall/alloc.c) zephyr_syscall_header(${SOF_SRC_PATH}/include/sof/lib/vregion.h) From e2f10c3c242f1bc1afda536f8ec0c795bf8123c1 Mon Sep 17 00:00:00 2001 From: Kai Vehmanen Date: Thu, 25 Jun 2026 14:19:02 +0300 Subject: [PATCH 038/108] ipc: make IPC message allocation userspace-safe Add an optional heap parameter to ipc_msg_w_ext_init() and ipc_msg_init() so callers can direct allocations to a specific heap. When the heap argument is NULL the existing rzalloc() path is used; when non-NULL, sof_heap_alloc()/sof_heap_free() are used instead. This allows IPC messages to be allocated from userspace-accessible heaps. For audio module contexts, introduce mod_ipc_msg_w_ext_init() and mod_ipc_msg_init() in generic.h. These use mod_zalloc()/ mod_free() for allocations that are automatically tracked and freed with the module lifecycle. ipc_msg_w_ext_init() is moved from a static inline in msg.h to a non-inline function in ipc-common.c due to the additional sof_heap_alloc dependency. Update all existing callers: - Module context callers (cadence, tdfb, sound_dose) use the new mod_ipc_msg_*() variants. - host-zephyr.c uses hd->heap, pipeline-graph.c uses the heap parameter from pipeline_new(). - Remaining kernel-context callers pass NULL for the default heap. Signed-off-by: Jyri Sarha (cherry picked from commit 11a17d542f135524d4c2d348a5ea60756d1859ff) --- src/audio/google/google_hotword_detect.c | 2 +- src/audio/host-legacy.c | 2 +- src/audio/host-zephyr.c | 2 +- src/audio/mfcc/mfcc_ipc4.c | 8 ++-- .../module_adapter/module/cadence_ipc4.c | 2 +- src/audio/pipeline/pipeline-graph.c | 2 +- src/audio/sound_dose/sound_dose-ipc4.c | 6 +-- src/audio/tdfb/tdfb_ipc3.c | 2 +- src/audio/tdfb/tdfb_ipc4.c | 4 +- .../sof/audio/module_adapter/module/generic.h | 42 ++++++++++++++++++ src/include/sof/ipc/msg.h | 38 +++++----------- src/ipc/ipc-common.c | 44 +++++++++++++++++++ src/library_manager/lib_notification.c | 2 +- src/samples/audio/detect_test.c | 4 +- src/trace/dma-trace.c | 2 +- 15 files changed, 115 insertions(+), 47 deletions(-) diff --git a/src/audio/google/google_hotword_detect.c b/src/audio/google/google_hotword_detect.c index 1055f7914e1e..1280368fbd4a 100644 --- a/src/audio/google/google_hotword_detect.c +++ b/src/audio/google/google_hotword_detect.c @@ -112,7 +112,7 @@ static struct comp_dev *ghd_create(const struct comp_driver *drv, cd->event.event_type = SOF_CTRL_EVENT_KD; cd->event.num_elems = 0; - cd->msg = ipc_msg_init(cd->event.rhdr.hdr.cmd, cd->event.rhdr.hdr.size); + cd->msg = ipc_msg_init(NULL, cd->event.rhdr.hdr.cmd, cd->event.rhdr.hdr.size); if (!cd->msg) { comp_err(dev, "ipc_msg_init failed"); goto cd_fail; diff --git a/src/audio/host-legacy.c b/src/audio/host-legacy.c index 4560959bb044..900fdd03bb73 100644 --- a/src/audio/host-legacy.c +++ b/src/audio/host-legacy.c @@ -560,7 +560,7 @@ int host_common_new(struct host_data *hd, struct comp_dev *dev, ipc_build_stream_posn(&hd->posn, SOF_IPC_STREAM_POSITION, config_id); #if CONFIG_HOST_DMA_IPC_POSITION_UPDATES - hd->msg = ipc_msg_init(hd->posn.rhdr.hdr.cmd, hd->posn.rhdr.hdr.size); + hd->msg = ipc_msg_init(NULL, hd->posn.rhdr.hdr.cmd, hd->posn.rhdr.hdr.size); if (!hd->msg) { comp_err(dev, "ipc_msg_init failed"); dma_put(hd->dma); diff --git a/src/audio/host-zephyr.c b/src/audio/host-zephyr.c index 95f8f35a3736..379bbb690834 100644 --- a/src/audio/host-zephyr.c +++ b/src/audio/host-zephyr.c @@ -728,7 +728,7 @@ __cold int host_common_new(struct host_data *hd, struct comp_dev *dev, ipc_build_stream_posn(&hd->posn, SOF_IPC_STREAM_POSITION, config_id); #if CONFIG_HOST_DMA_IPC_POSITION_UPDATES - hd->msg = ipc_msg_init(hd->posn.rhdr.hdr.cmd, sizeof(hd->posn)); + hd->msg = ipc_msg_init(hd->heap, hd->posn.rhdr.hdr.cmd, sizeof(hd->posn)); if (!hd->msg) { comp_err(dev, "ipc_msg_init failed"); sof_dma_put(hd->dma); diff --git a/src/audio/mfcc/mfcc_ipc4.c b/src/audio/mfcc/mfcc_ipc4.c index bb20d85e413b..21850be8a5d6 100644 --- a/src/audio/mfcc/mfcc_ipc4.c +++ b/src/audio/mfcc/mfcc_ipc4.c @@ -49,10 +49,10 @@ int mfcc_ipc_notification_init(struct processing_module *mod) primary->r.type = SOF_IPC4_GLB_NOTIFICATION; primary->r.rsp = SOF_IPC4_MESSAGE_DIR_MSG_REQUEST; primary->r.msg_tgt = SOF_IPC4_MESSAGE_TARGET_FW_GEN_MSG; - cd->msg = ipc_msg_w_ext_init(msg_proto.header, msg_proto.extension, - sizeof(struct sof_ipc4_notify_module_data) + - sizeof(struct sof_ipc4_control_msg_payload) + - sizeof(struct sof_ipc4_ctrl_value_chan)); + cd->msg = mod_ipc_msg_w_ext_init(mod, msg_proto.header, msg_proto.extension, + sizeof(struct sof_ipc4_notify_module_data) + + sizeof(struct sof_ipc4_control_msg_payload) + + sizeof(struct sof_ipc4_ctrl_value_chan)); if (!cd->msg) { comp_err(dev, "Failed to initialize VAD notification"); return -ENOMEM; diff --git a/src/audio/module_adapter/module/cadence_ipc4.c b/src/audio/module_adapter/module/cadence_ipc4.c index 763272964d61..4133ac7ebd6c 100644 --- a/src/audio/module_adapter/module/cadence_ipc4.c +++ b/src/audio/module_adapter/module/cadence_ipc4.c @@ -233,7 +233,7 @@ static struct ipc_msg *cadence_codec_notification_init(struct processing_module primary.r.type = SOF_IPC4_GLB_NOTIFICATION; primary.r.rsp = SOF_IPC4_MESSAGE_DIR_MSG_REQUEST; primary.r.msg_tgt = SOF_IPC4_MESSAGE_TARGET_FW_GEN_MSG; - msg = ipc_msg_w_ext_init(primary.dat, 0, sizeof(*msg_module_data)); + msg = mod_ipc_msg_w_ext_init(mod, primary.dat, 0, sizeof(*msg_module_data)); if (!msg) return NULL; diff --git a/src/audio/pipeline/pipeline-graph.c b/src/audio/pipeline/pipeline-graph.c index adcb00a80719..0395a5791c42 100644 --- a/src/audio/pipeline/pipeline-graph.c +++ b/src/audio/pipeline/pipeline-graph.c @@ -158,7 +158,7 @@ struct pipeline *pipeline_new(struct k_heap *heap, uint32_t pipeline_id, uint32_ ipc_build_stream_posn(&posn, SOF_IPC_STREAM_TRIG_XRUN, p->comp_id); if (posn.rhdr.hdr.size) { - p->msg = ipc_msg_init(posn.rhdr.hdr.cmd, posn.rhdr.hdr.size); + p->msg = ipc_msg_init(heap, posn.rhdr.hdr.cmd, posn.rhdr.hdr.size); if (!p->msg) { pipe_err(p, "ipc_msg_init failed"); goto free; diff --git a/src/audio/sound_dose/sound_dose-ipc4.c b/src/audio/sound_dose/sound_dose-ipc4.c index fd3a8151e984..3c3f537dc338 100644 --- a/src/audio/sound_dose/sound_dose-ipc4.c +++ b/src/audio/sound_dose/sound_dose-ipc4.c @@ -32,9 +32,9 @@ static struct ipc_msg *sound_dose_notification_init(struct processing_module *mo primary->r.type = SOF_IPC4_GLB_NOTIFICATION; primary->r.rsp = SOF_IPC4_MESSAGE_DIR_MSG_REQUEST; primary->r.msg_tgt = SOF_IPC4_MESSAGE_TARGET_FW_GEN_MSG; - msg = ipc_msg_w_ext_init(msg_proto.header, msg_proto.extension, - sizeof(struct sof_ipc4_notify_module_data) + - sizeof(struct sof_ipc4_control_msg_payload)); + msg = mod_ipc_msg_w_ext_init(mod, msg_proto.header, msg_proto.extension, + sizeof(struct sof_ipc4_notify_module_data) + + sizeof(struct sof_ipc4_control_msg_payload)); if (!msg) return NULL; diff --git a/src/audio/tdfb/tdfb_ipc3.c b/src/audio/tdfb/tdfb_ipc3.c index 1ebdb9641ccf..f7b655065674 100644 --- a/src/audio/tdfb/tdfb_ipc3.c +++ b/src/audio/tdfb/tdfb_ipc3.c @@ -36,7 +36,7 @@ static int init_get_ctl_ipc(struct processing_module *mod) cd->ctrl_data->rhdr.hdr.cmd = SOF_IPC_GLB_COMP_MSG | SOF_IPC_COMP_GET_VALUE | comp_id; cd->ctrl_data->rhdr.hdr.size = TDFB_GET_CTRL_DATA_SIZE; - cd->msg = ipc_msg_init(cd->ctrl_data->rhdr.hdr.cmd, cd->ctrl_data->rhdr.hdr.size); + cd->msg = mod_ipc_msg_init(mod, cd->ctrl_data->rhdr.hdr.cmd, cd->ctrl_data->rhdr.hdr.size); cd->ctrl_data->comp_id = comp_id; cd->ctrl_data->type = SOF_CTRL_TYPE_VALUE_CHAN_GET; diff --git a/src/audio/tdfb/tdfb_ipc4.c b/src/audio/tdfb/tdfb_ipc4.c index 2f7e7e865214..a7b4e34977c7 100644 --- a/src/audio/tdfb/tdfb_ipc4.c +++ b/src/audio/tdfb/tdfb_ipc4.c @@ -48,8 +48,8 @@ static struct ipc_msg *tdfb_notification_init(struct processing_module *mod, primary->r.type = SOF_IPC4_GLB_NOTIFICATION; primary->r.rsp = SOF_IPC4_MESSAGE_DIR_MSG_REQUEST; primary->r.msg_tgt = SOF_IPC4_MESSAGE_TARGET_FW_GEN_MSG; - msg = ipc_msg_w_ext_init(msg_proto.header, msg_proto.extension, - sizeof(struct sof_ipc4_notify_module_data) + + msg = mod_ipc_msg_w_ext_init(mod, msg_proto.header, msg_proto.extension, + sizeof(struct sof_ipc4_notify_module_data) + sizeof(struct sof_ipc4_control_msg_payload) + sizeof(struct sof_ipc4_ctrl_value_chan)); if (!msg) diff --git a/src/include/sof/audio/module_adapter/module/generic.h b/src/include/sof/audio/module_adapter/module/generic.h index 6740593a7cf1..bfc4b19969c5 100644 --- a/src/include/sof/audio/module_adapter/module/generic.h +++ b/src/include/sof/audio/module_adapter/module/generic.h @@ -19,6 +19,7 @@ #include #include #include +#include #include "module_interface.h" #include @@ -239,6 +240,47 @@ static inline void *mod_zalloc(struct processing_module *mod, size_t size) return ret; } +/** + * \brief Initialize a new IPC message using the module allocator. + * @param mod Module to allocate from + * @param header Message header metadata + * @param extension Message header extension metadata + * @param size Message data size in bytes. + * @return New IPC message. + */ +static inline struct ipc_msg *mod_ipc_msg_w_ext_init(struct processing_module *mod, + uint32_t header, + uint32_t extension, + uint32_t size) +{ + struct ipc_msg *msg; + + msg = mod_zalloc(mod, sizeof(*msg)); + if (!msg) + return NULL; + + if (size) { + msg->tx_data = mod_zalloc(mod, size); + if (!msg->tx_data) { + mod_free(mod, msg); + return NULL; + } + } + + msg->header = header; + msg->extension = extension; + msg->tx_size = size; + list_init(&msg->list); + + return msg; +} + +static inline struct ipc_msg *mod_ipc_msg_init(struct processing_module *mod, + uint32_t header, uint32_t size) +{ + return mod_ipc_msg_w_ext_init(mod, header, 0, size); +} + #if CONFIG_COMP_BLOB #if defined(__ZEPHYR__) && defined(CONFIG_SOF_FULL_ZEPHYR_APPLICATION) __syscall struct comp_data_blob_handler *mod_data_blob_handler_new(struct processing_module *mod); diff --git a/src/include/sof/ipc/msg.h b/src/include/sof/ipc/msg.h index 74c2b66fb916..2471b8c2004b 100644 --- a/src/include/sof/ipc/msg.h +++ b/src/include/sof/ipc/msg.h @@ -29,6 +29,7 @@ struct dai_config; struct dma; struct dma_sg_elem_array; +struct k_heap; struct ipc_msg { uint32_t header; /* specific to platform */ @@ -40,46 +41,27 @@ struct ipc_msg { }; /** - * \brief Initialize a new IPC message. + * \brief Initialize a new IPC message using a specific heap. + * @param heap Heap to allocate from (NULL = default kernel heap) * @param header Message header metadata * @param extension Message header extension metadata * @param size Message data size in bytes. * @return New IPC message. */ -static inline struct ipc_msg *ipc_msg_w_ext_init(uint32_t header, uint32_t extension, - uint32_t size) -{ - struct ipc_msg *msg; - - msg = rzalloc(SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, sizeof(*msg)); - if (!msg) - return NULL; - - if (size) { - msg->tx_data = rzalloc(SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, size); - if (!msg->tx_data) { - rfree(msg); - return NULL; - } - } - - msg->header = header; - msg->extension = extension; - msg->tx_size = size; - list_init(&msg->list); - - return msg; -} +struct ipc_msg *ipc_msg_w_ext_init(struct k_heap *heap, uint32_t header, + uint32_t extension, uint32_t size); /** - * \brief Initialise a new IPC message. + * \brief Initialize a new IPC message using a specific heap. + * @param heap Heap to allocate from (NULL = default kernel heap) * @param header Message header metadata * @param size Message data size in bytes. * @return New IPC message. */ -static inline struct ipc_msg *ipc_msg_init(uint32_t header, uint32_t size) +static inline struct ipc_msg *ipc_msg_init(struct k_heap *heap, + uint32_t header, uint32_t size) { - return ipc_msg_w_ext_init(header, 0, size); + return ipc_msg_w_ext_init(heap, header, 0, size); } /** diff --git a/src/ipc/ipc-common.c b/src/ipc/ipc-common.c index 46d28e5403dc..51e4477b2191 100644 --- a/src/ipc/ipc-common.c +++ b/src/ipc/ipc-common.c @@ -70,6 +70,50 @@ struct ipc *ipc_get(void) EXPORT_SYMBOL(ipc_get); #endif +struct ipc_msg *ipc_msg_w_ext_init(struct k_heap *heap, uint32_t header, + uint32_t extension, uint32_t size) +{ + struct ipc_msg *msg; + + if (heap) { + msg = sof_heap_alloc(heap, SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, + sizeof(*msg), 0); + if (msg) + memset(msg, 0, sizeof(*msg)); + } else { + msg = rzalloc(SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, sizeof(*msg)); + } + if (!msg) + return NULL; + + if (size) { + if (heap) { + msg->tx_data = sof_heap_alloc(heap, + SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, + size, 0); + if (msg->tx_data) + memset(msg->tx_data, 0, size); + } else { + msg->tx_data = rzalloc(SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, + size); + } + if (!msg->tx_data) { + if (heap) + sof_heap_free(heap, msg); + else + rfree(msg); + return NULL; + } + } + + msg->header = header; + msg->extension = extension; + msg->tx_size = size; + list_init(&msg->list); + + return msg; +} + int ipc_process_on_core(uint32_t core, bool blocking) { struct ipc *ipc = ipc_get(); diff --git a/src/library_manager/lib_notification.c b/src/library_manager/lib_notification.c index 36aa85835e70..0feab9660c3e 100644 --- a/src/library_manager/lib_notification.c +++ b/src/library_manager/lib_notification.c @@ -53,7 +53,7 @@ struct ipc_msg *lib_notif_msg_init(uint32_t header, uint32_t size) sizeof(*msg_pool_elem)); if (!msg_pool_elem) return NULL; - msg = ipc_msg_init(header, SRAM_OUTBOX_SIZE); + msg = ipc_msg_init(NULL, header, SRAM_OUTBOX_SIZE); if (!msg) { rfree(msg_pool_elem); return NULL; diff --git a/src/samples/audio/detect_test.c b/src/samples/audio/detect_test.c index 7979d7db3d6c..27068bc52838 100644 --- a/src/samples/audio/detect_test.c +++ b/src/samples/audio/detect_test.c @@ -472,7 +472,7 @@ static struct ipc_msg *ipc4_kd_notification_init(uint32_t word_id, notif.extension.r.sv_score = score; - msg = ipc_msg_w_ext_init(notif.primary.dat, + msg = ipc_msg_w_ext_init(NULL, notif.primary.dat, notif.extension.dat, 0); if (!msg) @@ -731,7 +731,7 @@ static struct comp_dev *test_keyword_new(const struct comp_driver *drv, cd->msg = ipc4_kd_notification_init(NOTIFICATION_DEFAULT_WORD_ID, NOTIFICATION_DEFAULT_SCORE); #else - cd->msg = ipc_msg_init(cd->event.rhdr.hdr.cmd, sizeof(cd->event)); + cd->msg = ipc_msg_init(NULL, cd->event.rhdr.hdr.cmd, sizeof(cd->event)); #endif /* CONFIG_IPC_MAJOR_4 */ if (!cd->msg) { diff --git a/src/trace/dma-trace.c b/src/trace/dma-trace.c index 554204ac5c70..fc7e7b2e953c 100644 --- a/src/trace/dma-trace.c +++ b/src/trace/dma-trace.c @@ -160,7 +160,7 @@ int dma_trace_init_early(struct sof *sof) k_spinlock_init(&sof->dmat->lock); ipc_build_trace_posn(&sof->dmat->posn); - sof->dmat->msg = ipc_msg_init(sof->dmat->posn.rhdr.hdr.cmd, + sof->dmat->msg = ipc_msg_init(NULL, sof->dmat->posn.rhdr.hdr.cmd, sof->dmat->posn.rhdr.hdr.size); if (!sof->dmat->msg) { ret = -ENOMEM; From 284392e0971c31fb49659a25d70da597e41c359b Mon Sep 17 00:00:00 2001 From: Jyri Sarha Date: Fri, 24 Apr 2026 00:14:17 +0300 Subject: [PATCH 039/108] ipc4: notification: make send_resource_notif() a syscall Move user-facing notification functions (send_copier_gateway_xrun_notif_msg, send_gateway_xrun_notif_msg, send_mixer_underrun_notif_msg, send_process_data_error_notif_msg) to a new notification-user.c file so they can run in userspace. The send_resource_notif() function, which depends on the kernel-side notification pool and IPC message infrastructure, is converted to a Zephyr syscall. The implementation is renamed to z_impl_send_resource_notif() and remains in notification.c alongside is_notif_filtered_out() and ipc4_update_notification_mask(). The send_resource_notif() is converted to a system call only if CONFIG_SOF_USERSPACE_LL=y, without it the behaviour is same as befofe. A z_vrfy_send_resource_notif() handler is added to validate the user-provided data buffer and other parameters before forwarding to the kernel implementation. Signed-off-by: Jyri Sarha (cherry picked from commit e385f101a6cdd4965bbe7f88722f9e16206d4723) --- src/include/ipc4/notification.h | 17 +++++++++ src/ipc/ipc4/CMakeLists.txt | 1 + src/ipc/ipc4/notification-user.c | 54 +++++++++++++++++++++++++++ src/ipc/ipc4/notification.c | 64 ++++++++++++++------------------ zephyr/CMakeLists.txt | 2 + 5 files changed, 102 insertions(+), 36 deletions(-) create mode 100644 src/ipc/ipc4/notification-user.c diff --git a/src/include/ipc4/notification.h b/src/include/ipc4/notification.h index 614a926d16ad..bad08a39082c 100644 --- a/src/include/ipc4/notification.h +++ b/src/include/ipc4/notification.h @@ -297,4 +297,21 @@ void send_mixer_underrun_notif_msg(uint32_t resource_id, uint32_t eos_flag, uint uint32_t expected_data_mixed); void ipc4_update_notification_mask(uint32_t ntfy_mask, uint32_t enabled_mask); +#ifdef CONFIG_SOF_USERSPACE_LL + +__syscall bool send_resource_notif(uint32_t resource_id, uint32_t event_type, + uint32_t resource_type, void *data, uint32_t data_size); + +bool z_impl_send_resource_notif(uint32_t resource_id, uint32_t event_type, + uint32_t resource_type, void *data, uint32_t data_size); + +#include + +#else + +bool send_resource_notif(uint32_t resource_id, uint32_t event_type, + uint32_t resource_type, void *data, uint32_t data_size); + +#endif /* CONFIG_SOF_USERSPACE_LL */ + #endif /* __IPC4_NOTIFICATION_H__ */ diff --git a/src/ipc/ipc4/CMakeLists.txt b/src/ipc/ipc4/CMakeLists.txt index 360a4e988650..c9f9fcb5710d 100644 --- a/src/ipc/ipc4/CMakeLists.txt +++ b/src/ipc/ipc4/CMakeLists.txt @@ -7,6 +7,7 @@ add_local_sources(sof helper.c logging.c notification.c + notification-user.c ) # The DAI interface is not implemented in library builds and diff --git a/src/ipc/ipc4/notification-user.c b/src/ipc/ipc4/notification-user.c new file mode 100644 index 000000000000..37a881e32829 --- /dev/null +++ b/src/ipc/ipc4/notification-user.c @@ -0,0 +1,54 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright(c) 2023 Intel Corporation. All rights reserved. + * + * Author: Piotr Makaruk + * Adrian Warecki + */ + +#include +#include +#include + +#include + +static enum sof_ipc4_resource_event_type dir_to_xrun_event(enum sof_ipc_stream_direction dir) +{ + return (dir == SOF_IPC_STREAM_PLAYBACK) ? SOF_IPC4_GATEWAY_UNDERRUN_DETECTED : + SOF_IPC4_GATEWAY_OVERRUN_DETECTED; +} + +bool send_copier_gateway_xrun_notif_msg(uint32_t pipeline_id, enum sof_ipc_stream_direction dir) +{ + return send_resource_notif(pipeline_id, dir_to_xrun_event(dir), SOF_IPC4_PIPELINE, NULL, + 0); +} + +bool send_gateway_xrun_notif_msg(uint32_t resource_id, enum sof_ipc_stream_direction dir) +{ + return send_resource_notif(resource_id, dir_to_xrun_event(dir), SOF_IPC4_GATEWAY, NULL, 0); +} + +void send_mixer_underrun_notif_msg(uint32_t resource_id, uint32_t eos_flag, uint32_t data_mixed, + uint32_t expected_data_mixed) +{ + struct ipc4_mixer_underrun_event_data mixer_underrun_data; + + mixer_underrun_data.eos_flag = eos_flag; + mixer_underrun_data.data_mixed = data_mixed; + mixer_underrun_data.expected_data_mixed = expected_data_mixed; + + send_resource_notif(resource_id, SOF_IPC4_MIXER_UNDERRUN_DETECTED, SOF_IPC4_PIPELINE, + &mixer_underrun_data, sizeof(mixer_underrun_data)); +} +EXPORT_SYMBOL(send_mixer_underrun_notif_msg); + +void send_process_data_error_notif_msg(uint32_t resource_id, uint32_t error_code) +{ + struct ipc4_process_data_error_event_data error_data; + + error_data.error_code = error_code; + + send_resource_notif(resource_id, SOF_IPC4_PROCESS_DATA_ERROR, SOF_IPC4_MODULE_INSTANCE, + &error_data, sizeof(error_data)); +} diff --git a/src/ipc/ipc4/notification.c b/src/ipc/ipc4/notification.c index 9fd566ee1f93..485b875f8468 100644 --- a/src/ipc/ipc4/notification.c +++ b/src/ipc/ipc4/notification.c @@ -12,7 +12,9 @@ #include #include -#include +#ifdef CONFIG_SOF_USERSPACE_LL +#include +#endif static uint32_t notification_mask = 0xFFFFFFFF; @@ -37,8 +39,13 @@ static bool is_notif_filtered_out(uint32_t event_type) return (notification_mask & BIT(notif_idx)) == 0; } -static bool send_resource_notif(uint32_t resource_id, uint32_t event_type, uint32_t resource_type, - void *data, uint32_t data_size) +#ifdef CONFIG_SOF_USERSPACE_LL +bool z_impl_send_resource_notif(uint32_t resource_id, uint32_t event_type, + uint32_t resource_type, void *data, uint32_t data_size) +#else +bool send_resource_notif(uint32_t resource_id, uint32_t event_type, + uint32_t resource_type, void *data, uint32_t data_size) +#endif { struct ipc_msg *msg; @@ -80,49 +87,34 @@ static bool send_resource_notif(uint32_t resource_id, uint32_t event_type, uint3 return true; } -static enum sof_ipc4_resource_event_type dir_to_xrun_event(enum sof_ipc_stream_direction dir) -{ - return (dir == SOF_IPC_STREAM_PLAYBACK) ? SOF_IPC4_GATEWAY_UNDERRUN_DETECTED : - SOF_IPC4_GATEWAY_OVERRUN_DETECTED; -} - void ipc4_update_notification_mask(uint32_t ntfy_mask, uint32_t enabled_mask) { notification_mask &= enabled_mask | (~ntfy_mask); notification_mask |= enabled_mask & ntfy_mask; } -bool send_copier_gateway_xrun_notif_msg(uint32_t pipeline_id, enum sof_ipc_stream_direction dir) +#ifdef CONFIG_SOF_USERSPACE_LL +static inline bool z_vrfy_send_resource_notif(uint32_t resource_id, uint32_t event_type, + uint32_t resource_type, void *data, + uint32_t data_size) { - return send_resource_notif(pipeline_id, dir_to_xrun_event(dir), SOF_IPC4_PIPELINE, NULL, - 0); -} + /* Validate event_type is a known resource event */ + K_OOPS(K_SYSCALL_VERIFY(event_type < SOF_IPC4_INVALID_RESOURCE_EVENT_TYPE)); -bool send_gateway_xrun_notif_msg(uint32_t resource_id, enum sof_ipc_stream_direction dir) -{ - return send_resource_notif(resource_id, dir_to_xrun_event(dir), SOF_IPC4_GATEWAY, NULL, 0); -} + /* Validate resource_type is a known resource type */ + K_OOPS(K_SYSCALL_VERIFY(resource_type < SOF_IPC4_INVALID_RESOURCE_TYPE)); -void send_mixer_underrun_notif_msg(uint32_t resource_id, uint32_t eos_flag, uint32_t data_mixed, - uint32_t expected_data_mixed) -{ - struct ipc4_mixer_underrun_event_data mixer_underrun_data; - - mixer_underrun_data.eos_flag = eos_flag; - mixer_underrun_data.data_mixed = data_mixed; - mixer_underrun_data.expected_data_mixed = expected_data_mixed; + /* data and data_size must be consistent */ + K_OOPS(K_SYSCALL_VERIFY((!data && !data_size) || (data && data_size))); - send_resource_notif(resource_id, SOF_IPC4_MIXER_UNDERRUN_DETECTED, SOF_IPC4_PIPELINE, - &mixer_underrun_data, sizeof(mixer_underrun_data)); -} -EXPORT_SYMBOL(send_mixer_underrun_notif_msg); - -void send_process_data_error_notif_msg(uint32_t resource_id, uint32_t error_code) -{ - struct ipc4_process_data_error_event_data error_data; + /* Payload must fit in the event_data union and the IPC message */ + K_OOPS(K_SYSCALL_VERIFY(data_size <= sizeof(union ipc4_resource_event_data))); + K_OOPS(K_SYSCALL_VERIFY(data_size <= SOF_IPC_MSG_MAX_SIZE)); - error_data.error_code = error_code; + if (data && data_size) + K_OOPS(K_SYSCALL_MEMORY_READ(data, data_size)); - send_resource_notif(resource_id, SOF_IPC4_PROCESS_DATA_ERROR, SOF_IPC4_MODULE_INSTANCE, - &error_data, sizeof(error_data)); + return z_impl_send_resource_notif(resource_id, event_type, resource_type, data, data_size); } +#include +#endif diff --git a/zephyr/CMakeLists.txt b/zephyr/CMakeLists.txt index 87078267c454..4042d56ae546 100644 --- a/zephyr/CMakeLists.txt +++ b/zephyr/CMakeLists.txt @@ -634,6 +634,8 @@ zephyr_library_sources_ifdef(CONFIG_USERSPACE syscall/dai.c) zephyr_syscall_header(${SOF_SRC_PATH}/include/user/debug_stream_slot.h) +zephyr_syscall_header(${SOF_SRC_PATH}/include/ipc4/notification.h) + zephyr_library_link_libraries(SOF) target_link_libraries(SOF INTERFACE zephyr_interface) From 54fbca1658413ded5f87e913a7193d2027c34c70 Mon Sep 17 00:00:00 2001 From: Kai Vehmanen Date: Fri, 8 May 2026 18:58:45 +0300 Subject: [PATCH 040/108] (---section: IPC notifications STOP) From 79650de37ac622d5020757a27d377c5d478833a1 Mon Sep 17 00:00:00 2001 From: Kai Vehmanen Date: Thu, 19 Feb 2026 16:24:36 +0200 Subject: [PATCH 041/108] (---section test-case START) From 2cfc9cf7b3b0d05780bbfef762b34772b411ec1c Mon Sep 17 00:00:00 2001 From: Kai Vehmanen Date: Wed, 22 Apr 2026 15:17:31 +0300 Subject: [PATCH 042/108] zephyr: test: userspace: add pipeline_two_components test Add a new test to userspace_ll set that tests more of the functionality needed to run full audio pipelines in user-space. The test creates a pipeline with two components (IPC4 host and DAI copiers), does pipeline prepare, one copy cycle in prepared state and tears down the pipeline. One user-space thread is created to manage the pipelines. This would be equivalent to user IPC handler thread. Another user-space thread is created for the LL scheduler. Signed-off-by: Kai Vehmanen --- zephyr/test/userspace/test_ll_task.c | 397 ++++++++++++++++++++++++++- 1 file changed, 395 insertions(+), 2 deletions(-) diff --git a/zephyr/test/userspace/test_ll_task.c b/zephyr/test/userspace/test_ll_task.c index 234423defc60..2ada2ce1c0c4 100644 --- a/zephyr/test/userspace/test_ll_task.c +++ b/zephyr/test/userspace/test_ll_task.c @@ -14,9 +14,20 @@ #include #include #include +#include +#include +#include +#include #include #include #include +#include +#include +#include +#include +#include +#include +#include #include #include @@ -24,6 +35,7 @@ #include #include /* offsetof() */ +#include LOG_MODULE_DECLARE(sof_boot_test, LOG_LEVEL_DBG); @@ -36,10 +48,20 @@ K_APPMEM_PARTITION_DEFINE(userspace_ll_part); /* Global variable for test runs counter, accessible from user-space */ K_APP_BMEM(userspace_ll_part) static int test_runs; +/* User-space thread for pipeline_two_components test */ +#define PPL_USER_STACKSIZE 4096 + +static struct k_thread ppl_user_thread; +static K_THREAD_STACK_DEFINE(ppl_user_stack, PPL_USER_STACKSIZE); + +static void test_ll_task_free(struct task *task) +{ + sof_heap_free(zephyr_ll_user_heap(), task); +} + static enum task_state task_callback(void *arg) { LOG_INF("entry"); - if (++test_runs > 3) return SOF_TASK_STATE_COMPLETED; @@ -77,7 +99,7 @@ static void ll_task_test(void) LOG_INF("task scheduled and running"); /* Let the task run for a bit */ - k_sleep(K_MSEC(10)); + k_sleep(K_MSEC(100)); /* Cancel the task to stop any scheduled execution */ ret = schedule_task_cancel(task); @@ -87,6 +109,9 @@ static void ll_task_test(void) ret = schedule_task_free(task); zassert_equal(ret, 0); + k_mem_domain_remove_partition(zephyr_ll_mem_domain(), &userspace_ll_part); + test_ll_task_free(task); + LOG_INF("test complete"); } @@ -129,6 +154,374 @@ ZTEST(userspace_ll, pipeline_check) pipeline_check(); } +/* Copier UUID: 9ba00c83-ca12-4a83-943c-1fa2e82f9dda */ +static const uint8_t copier_uuid[16] = { + 0x83, 0x0c, 0xa0, 0x9b, 0x12, 0xca, 0x83, 0x4a, + 0x94, 0x3c, 0x1f, 0xa2, 0xe8, 0x2f, 0x9d, 0xda +}; + +/** + * Find the module_id (manifest entry index) for the copier module + * by iterating the firmware manifest and matching the copier UUID. + */ +static int find_copier_module_id(void) +{ + const struct sof_man_fw_desc *desc = basefw_vendor_get_manifest(); + const struct sof_man_module *mod; + uint32_t i; + + if (!desc) + return -1; + + for (i = 0; i < desc->header.num_module_entries; i++) { + mod = (const struct sof_man_module *)((const char *)desc + + SOF_MAN_MODULE_OFFSET(i)); + if (!memcmp(&mod->uuid, copier_uuid, sizeof(copier_uuid))) + return (int)i; + } + + return -1; +} + +/** + * IPC4 copier module config - used as payload for comp_new_ipc4(). + * Placed at MAILBOX_HOSTBOX_BASE before calling comp_new_ipc4(). + * Layout matches struct ipc4_copier_module_cfg from copier.h. + */ +struct copier_init_data { + struct ipc4_base_module_cfg base; + struct ipc4_audio_format out_fmt; + uint32_t copier_feature_mask; + /* Gateway config (matches struct ipc4_copier_gateway_cfg) */ + union ipc4_connector_node_id node_id; + uint32_t dma_buffer_size; + uint32_t config_length; +} __packed __aligned(4); + +static void fill_audio_format(struct ipc4_audio_format *fmt) +{ + memset(fmt, 0, sizeof(*fmt)); + fmt->sampling_frequency = IPC4_FS_48000HZ; + fmt->depth = IPC4_DEPTH_32BIT; + fmt->ch_cfg = IPC4_CHANNEL_CONFIG_STEREO; + fmt->channels_count = 2; + fmt->valid_bit_depth = 32; + fmt->s_type = IPC4_TYPE_MSB_INTEGER; + fmt->interleaving_style = IPC4_CHANNELS_INTERLEAVED; +} + +/** + * Create a copier component via IPC4. + * + * @param module_id Copier module_id from manifest + * @param instance_id Instance ID for this component + * @param pipeline_id Parent pipeline ID + * @param node_id Gateway node ID (type + virtual DMA index) + */ +static struct comp_dev *create_copier(int module_id, int instance_id, + int pipeline_id, + union ipc4_connector_node_id node_id) +{ + struct ipc4_module_init_instance module_init; + struct copier_init_data cfg; + struct comp_dev *dev; + + /* Prepare copier config payload */ + memset(&cfg, 0, sizeof(cfg)); + fill_audio_format(&cfg.base.audio_fmt); + /* 2 channels * 4 bytes * 48 frames = 384 bytes */ + cfg.base.ibs = 384; + cfg.base.obs = 384; + cfg.base.is_pages = 0; + cfg.base.cpc = 0; + cfg.out_fmt = cfg.base.audio_fmt; + cfg.copier_feature_mask = 0; + cfg.node_id = node_id; + cfg.dma_buffer_size = 768; + cfg.config_length = 0; + + /* Write config data to mailbox hostbox (where comp_new_ipc4 reads it). + * Flush cache so that data is visible in SRAM before comp_new_ipc4() + * invalidates the cache line (in normal IPC flow, host writes via DMA + * directly to SRAM, so the invalidation reads fresh data; here the DSP + * core itself writes, so an explicit flush is needed). + */ + memcpy((void *)MAILBOX_HOSTBOX_BASE, &cfg, sizeof(cfg)); + sys_cache_data_flush_range((void *)MAILBOX_HOSTBOX_BASE, sizeof(cfg)); + + /* Prepare IPC4 module init header */ + memset(&module_init, 0, sizeof(module_init)); + module_init.primary.r.module_id = module_id; + module_init.primary.r.instance_id = instance_id; + module_init.primary.r.type = SOF_IPC4_MOD_INIT_INSTANCE; + module_init.primary.r.msg_tgt = SOF_IPC4_MESSAGE_TARGET_MODULE_MSG; + module_init.primary.r.rsp = SOF_IPC4_MESSAGE_DIR_MSG_REQUEST; + + module_init.extension.r.param_block_size = sizeof(cfg) / sizeof(uint32_t); + module_init.extension.r.ppl_instance_id = pipeline_id; + module_init.extension.r.core_id = 0; + module_init.extension.r.proc_domain = 0; /* LL */ + + dev = comp_new_ipc4(&module_init); + + return dev; +} + +/** + * Context shared between kernel setup and the user-space pipeline thread. + */ +struct ppl_test_ctx { + struct pipeline *p; + struct k_heap *heap; + struct mod_alloc_ctx *alloc; + struct comp_dev *host_comp; + struct comp_dev *dai_comp; + struct comp_buffer *buf; + struct ipc *ipc; + struct ipc_comp_dev *ipc_pipe; +}; + +/** + * Pipeline operations: connect, complete, prepare, copy, verify, and clean up. + * This function is called either directly (kernel mode) or from a user-space + * thread, exercising pipeline_*() calls from the requested context. + */ +static void pipeline_ops(struct ppl_test_ctx *ctx) +{ + struct pipeline *p = ctx->p; + struct comp_dev *host_comp = ctx->host_comp; + struct comp_dev *dai_comp = ctx->dai_comp; + struct comp_buffer *buf = ctx->buf; + int ret; + + LOG_INF("pipeline_ops: user_context=%d", k_is_user_context()); + + /* Step: Connect host -> buffer -> DAI */ + ret = pipeline_connect(host_comp, buf, PPL_CONN_DIR_COMP_TO_BUFFER); + zassert_equal(ret, 0, "connect host to buffer failed"); + + ret = pipeline_connect(dai_comp, buf, PPL_CONN_DIR_BUFFER_TO_COMP); + zassert_equal(ret, 0, "connect buffer to DAI failed"); + + LOG_INF("host -> buffer -> DAI connected"); + + /* Step: Complete the pipeline */ + ret = pipeline_complete(p, host_comp, dai_comp); + zassert_equal(ret, 0, "pipeline complete failed"); + + /* Step: Prepare the pipeline */ + p->sched_comp = host_comp; + + ret = pipeline_prepare(p, host_comp); + zassert_equal(ret, 0, "pipeline prepare failed"); + + ret = pipeline_trigger(p, host_comp, COMP_TRIGGER_PRE_START); + //zassert_equal(ret, 0, "pipeline TRIGGER_START failed"); + + LOG_INF("pipeline complete, status = %d", p->status); + + /* Step: Run copies */ + pipeline_schedule_copy(p, 1000); + + /* Step: let run for 3 msec */ + k_sleep(K_MSEC(3)); + + /* Verify pipeline source and sink assignments */ + zassert_equal(p->source_comp, host_comp, "source comp mismatch"); + zassert_equal(p->sink_comp, dai_comp, "sink comp mismatch"); + + LOG_INF("pipeline_ops done"); +} + +/** + * User-space thread entry point for pipeline_two_components test. + * p1 points to the ppl_test_ctx shared with the kernel launcher. + */ +static void pipeline_user_thread(void *p1, void *p2, void *p3) +{ + struct ppl_test_ctx *ctx = (struct ppl_test_ctx *)p1; + + zassert_true(k_is_user_context(), "expected user context"); + pipeline_ops(ctx); +} + +/** + * Test creating a pipeline with a host copier and a DAI (link) copier, + * connected through a shared buffer. + * + * When run_in_user is true, all pipeline_*() calls are made from a + * separate user-space thread. + */ +static void pipeline_two_components(void) +{ + struct ppl_test_ctx *ctx; + struct k_heap *heap = NULL; + uint32_t pipeline_id = 2; + uint32_t priority = 0; + struct task *task; + uint32_t comp_id; + int copier_module_id; + int host_instance_id = 0; + int dai_instance_id = 1; + int core = 0; + int ret; + + /* Step: Find the copier module_id from the firmware manifest */ + copier_module_id = find_copier_module_id(); + zassert_true(copier_module_id >= 0, "copier module not found in manifest"); + LOG_INF("copier module_id = %d", copier_module_id); + + /* Step: Create pipeline */ + LOG_INF("running test with user memory domain"); + heap = zephyr_ll_user_heap(); + zassert_not_null(heap, "user heap not found"); + + task = zephyr_ll_task_alloc(); + zassert_not_null(task, "task allocation failed"); + + ctx = sof_heap_alloc(heap, SOF_MEM_FLAG_USER, sizeof(*ctx), 0); + ctx->alloc = sof_heap_alloc(heap, SOF_MEM_FLAG_USER, sizeof(struct mod_alloc_ctx), 0); + ctx->alloc->heap = heap; + ctx->alloc->vreg = NULL; + ctx->heap = heap; + ctx->ipc = ipc_get(); + + comp_id = IPC4_COMP_ID(copier_module_id, host_instance_id); + ctx->p = pipeline_new(ctx->heap, pipeline_id, priority, comp_id, NULL); + zassert_not_null(ctx->p, "pipeline creation failed"); + + /* create the LL scheduler thread by initializing one task */ + k_mem_domain_add_partition(zephyr_ll_mem_domain(), &userspace_ll_part); + + test_runs = 0; + ret = schedule_task_init_ll(task, SOF_UUID(test_task_uuid), SOF_SCHEDULE_LL_TIMER, + priority, task_callback, + (void *)&test_runs, core, 0); + zassert_equal(ret, 0); + + LOG_INF("task init done"); + + /* Set pipeline period so components get correct dev->period and dev->frames. + * This mirrors what ipc4_create_pipeline() does in normal IPC flow. + */ + ctx->p->time_domain = SOF_TIME_DOMAIN_TIMER; + ctx->p->period = LL_TIMER_PERIOD_US; + + /* Register pipeline in IPC component list so comp_new_ipc4() can + * find it via ipc_get_comp_by_ppl_id() and set dev->period. + */ + ctx->ipc_pipe = rzalloc(SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, + sizeof(struct ipc_comp_dev)); + zassert_not_null(ctx->ipc_pipe, "ipc_comp_dev alloc failed"); + ctx->ipc_pipe->pipeline = ctx->p; + ctx->ipc_pipe->type = COMP_TYPE_PIPELINE; + ctx->ipc_pipe->id = pipeline_id; + ctx->ipc_pipe->core = 0; + list_item_append(&ctx->ipc_pipe->list, &ctx->ipc->comp_list); + + /* Step: Create host copier with HDA host output gateway */ + union ipc4_connector_node_id host_node_id = { .f = { + .dma_type = ipc4_hda_host_output_class, + .v_index = 0 + }}; + ctx->host_comp = create_copier(copier_module_id, host_instance_id, pipeline_id, + host_node_id); + zassert_not_null(ctx->host_comp, "host copier creation failed"); + + /* Assign pipeline to host component */ + ctx->host_comp->pipeline = ctx->p; + ctx->host_comp->ipc_config.type = SOF_COMP_HOST; + + LOG_INF("host copier created, comp_id = 0x%x", ctx->host_comp->ipc_config.id); + + /* Step: Create link copier with HDA link output gateway */ + union ipc4_connector_node_id link_node_id = { .f = { + .dma_type = ipc4_hda_link_output_class, + .v_index = 0 + }}; + ctx->dai_comp = create_copier(copier_module_id, dai_instance_id, pipeline_id, + link_node_id); + zassert_not_null(ctx->dai_comp, "DAI copier creation failed"); + + /* Assign pipeline to DAI component */ + ctx->dai_comp->pipeline = ctx->p; + ctx->dai_comp->ipc_config.type = SOF_COMP_DAI; + + LOG_INF("DAI copier created, comp_id = 0x%x", ctx->dai_comp->ipc_config.id); + + /* Step: Allocate a buffer to connect host -> DAI */ + ctx->buf = buffer_alloc(ctx->alloc, 384, 0, 0, false); + zassert_not_null(ctx->buf, "buffer allocation failed"); + + struct k_thread *task_thread; + + /* Create a user-space thread to execute pipeline operations */ + k_thread_create(&ppl_user_thread, ppl_user_stack, PPL_USER_STACKSIZE, + pipeline_user_thread, ctx, NULL, NULL, + -1, K_USER, K_FOREVER); + + /* Add thread to LL memory domain so it can access pipeline memory */ + k_mem_domain_add_thread(zephyr_ll_mem_domain(), &ppl_user_thread); + + user_grant_dai_access_all(&ppl_user_thread); + user_grant_dma_access_all(&ppl_user_thread); + user_access_to_mailbox(zephyr_ll_mem_domain(), &ppl_user_thread); + user_ll_grant_access(&ppl_user_thread, ctx->ipc_pipe->core); + + task_thread = scheduler_init_context(task); + zassert_not_null(task_thread); + + k_thread_start(&ppl_user_thread); + + LOG_INF("user thread started, waiting for completion"); + + k_thread_join(&ppl_user_thread, K_FOREVER); + + /* Step: Clean up - reset, disconnect, free buffer, free components, free pipeline */ + /* Reset pipeline to bring components back to COMP_STATE_READY, + * required before ipc_comp_free() which rejects non-READY components. + */ + ret = pipeline_reset(ctx->p, ctx->host_comp); + zassert_equal(ret, 0, "pipeline reset failed"); + + pipeline_disconnect(ctx->host_comp, ctx->buf, PPL_CONN_DIR_COMP_TO_BUFFER); + pipeline_disconnect(ctx->dai_comp, ctx->buf, PPL_CONN_DIR_BUFFER_TO_COMP); + + buffer_free(ctx->buf); + + /* Free components through IPC to properly remove from IPC device list */ + ret = ipc_comp_free(ctx->ipc, ctx->host_comp->ipc_config.id); + zassert_equal(ret, 0, "host comp free failed"); + + ret = ipc_comp_free(ctx->ipc, ctx->dai_comp->ipc_config.id); + zassert_equal(ret, 0, "DAI comp free failed"); + + /* Unregister pipeline from IPC component list */ + list_item_del(&ctx->ipc_pipe->list); + rfree(ctx->ipc_pipe); + + ret = pipeline_free(ctx->p); + zassert_equal(ret, 0, "pipeline free failed"); + + schedule_free(0); + + ret = schedule_task_free(task); + zassert_equal(ret, 0); + + sof_heap_free(heap, ctx->alloc); + sof_heap_free(heap, ctx); + + test_ll_task_free(task); + k_mem_domain_remove_partition(zephyr_ll_mem_domain(), &userspace_ll_part); + + LOG_INF("two component pipeline test complete"); +} + +ZTEST(userspace_ll, pipeline_two_components_user) +{ + pipeline_two_components(); +} + ZTEST_SUITE(userspace_ll, NULL, NULL, NULL, NULL, NULL); /** From 4ff1636e2dbdb248f0fb381298bb398fa9a73bbd Mon Sep 17 00:00:00 2001 From: Kai Vehmanen Date: Thu, 19 Feb 2026 16:25:31 +0200 Subject: [PATCH 043/108] (---section WIP mandatory changes START) From c8075aed0aa5bbb6260fc7269996b5ee285f1249 Mon Sep 17 00:00:00 2001 From: Kai Vehmanen Date: Thu, 2 Jul 2026 18:14:11 +0300 Subject: [PATCH 044/108] audio: pipeline: enable position reporting for user-space pipelines Place the pipeline position lookup table in the sysuser memory partition and replace k_spinlock with a dynamically allocated k_mutex when CONFIG_SOF_USERSPACE_LL is enabled. Spinlocks disable interrupts which is a privileged operation unavailable from user-mode threads. The mutex pointer is stored in a separate APP_SYSUSER_BSS variable outside the SHARED_DATA struct so Zephyr's kernel object tracking can recognize it for syscall verification. Move pipeline_posn_init() from task_main_start() to primary_core_init() before platform_init(), so the mutex is allocated before ipc_user_init() grants thread access to it. In pipeline_posn_get(), bypass the sof_get() kernel singleton and access the shared structure directly when running in user-space. Grant the ipc_user_init thread access to the pipeline position mutex via new pipeline_posn_grant_access() helper. Signed-off-by: Kai Vehmanen --- src/audio/pipeline/pipeline-graph.c | 57 +++++++++++++++++++++++++++-- src/include/sof/audio/pipeline.h | 8 ++++ src/init/init.c | 6 +++ src/ipc/ipc-common.c | 1 + zephyr/wrapper.c | 3 -- 5 files changed, 68 insertions(+), 7 deletions(-) diff --git a/src/audio/pipeline/pipeline-graph.c b/src/audio/pipeline/pipeline-graph.c index 0395a5791c42..efe4abfb44f5 100644 --- a/src/audio/pipeline/pipeline-graph.c +++ b/src/audio/pipeline/pipeline-graph.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -44,10 +45,20 @@ DECLARE_TR_CTX(pipe_tr, SOF_UUID(pipe_uuid), LOG_LEVEL_INFO); /* lookup table to determine busy/free pipeline metadata objects */ struct pipeline_posn { bool posn_offset[PPL_POSN_OFFSETS]; /**< available offsets */ +#ifndef CONFIG_SOF_USERSPACE_LL struct k_spinlock lock; /**< lock mechanism */ +#endif }; /* the pipeline position lookup table */ -static SHARED_DATA struct pipeline_posn pipeline_posn_shared; +static APP_SYSUSER_BSS SHARED_DATA struct pipeline_posn pipeline_posn_shared; + +#ifdef CONFIG_SOF_USERSPACE_LL +/* Mutex pointer in user-accessible partition so user-space threads + * can read the pointer for syscalls. Kept outside the SHARED_DATA + * struct to avoid kernel object tracking issues. + */ +static APP_SYSUSER_BSS struct k_mutex *pipeline_posn_lock; +#endif /** * \brief Retrieves pipeline position structure. @@ -55,7 +66,11 @@ static SHARED_DATA struct pipeline_posn pipeline_posn_shared; */ static inline struct pipeline_posn *pipeline_posn_get(void) { +#ifdef CONFIG_SOF_USERSPACE_LL + return &pipeline_posn_shared; +#else return sof_get()->pipeline_posn; +#endif } /** @@ -68,9 +83,14 @@ static inline int pipeline_posn_offset_get(uint32_t *posn_offset) struct pipeline_posn *pipeline_posn = pipeline_posn_get(); int ret = -EINVAL; uint32_t i; + +#ifdef CONFIG_SOF_USERSPACE_LL + k_mutex_lock(pipeline_posn_lock, K_FOREVER); +#else k_spinlock_key_t key; key = k_spin_lock(&pipeline_posn->lock); +#endif for (i = 0; i < PPL_POSN_OFFSETS; ++i) { if (!pipeline_posn->posn_offset[i]) { @@ -81,8 +101,11 @@ static inline int pipeline_posn_offset_get(uint32_t *posn_offset) } } - +#ifdef CONFIG_SOF_USERSPACE_LL + k_mutex_unlock(pipeline_posn_lock); +#else k_spin_unlock(&pipeline_posn->lock, key); +#endif return ret; } @@ -95,21 +118,42 @@ static inline void pipeline_posn_offset_put(uint32_t posn_offset) { struct pipeline_posn *pipeline_posn = pipeline_posn_get(); int i = posn_offset / sizeof(struct sof_ipc_stream_posn); + +#ifdef CONFIG_SOF_USERSPACE_LL + k_mutex_lock(pipeline_posn_lock, K_FOREVER); + pipeline_posn->posn_offset[i] = false; + k_mutex_unlock(pipeline_posn_lock); +#else k_spinlock_key_t key; key = k_spin_lock(&pipeline_posn->lock); - pipeline_posn->posn_offset[i] = false; - k_spin_unlock(&pipeline_posn->lock, key); +#endif } void pipeline_posn_init(struct sof *sof) { sof->pipeline_posn = &pipeline_posn_shared; +#ifdef CONFIG_SOF_USERSPACE_LL + pipeline_posn_lock = k_object_alloc(K_OBJ_MUTEX); + if (!pipeline_posn_lock) { + pipe_cl_err("pipeline posn mutex alloc failed"); + k_panic(); + } + k_mutex_init(pipeline_posn_lock); +#else k_spinlock_init(&sof->pipeline_posn->lock); +#endif } +#ifdef CONFIG_SOF_USERSPACE_LL +void pipeline_posn_grant_access(struct k_thread *thread) +{ + k_thread_access_grant(thread, pipeline_posn_lock); +} +#endif + /* create new pipeline - returns pipeline id or negative error */ struct pipeline *pipeline_new(struct k_heap *heap, uint32_t pipeline_id, uint32_t priority, uint32_t comp_id, struct create_pipeline_params *pparams) @@ -140,12 +184,17 @@ struct pipeline *pipeline_new(struct k_heap *heap, uint32_t pipeline_id, uint32_ p->pipeline_id = pipeline_id; p->status = COMP_STATE_INIT; p->trigger.cmd = COMP_TRIGGER_NO_ACTION; + +#ifdef CONFIG_SOF_USERSPACE_LL + LOG_WRN("pipeline trace settings cannot be copied"); +#else ret = memcpy_s(&p->tctx, sizeof(struct tr_ctx), &pipe_tr, sizeof(struct tr_ctx)); if (ret < 0) { pipe_err(p, "failed to copy trace settings"); goto free; } +#endif ret = pipeline_posn_offset_get(&p->posn_offset); if (ret < 0) { diff --git a/src/include/sof/audio/pipeline.h b/src/include/sof/audio/pipeline.h index 913a569c208c..ff456fbceb7d 100644 --- a/src/include/sof/audio/pipeline.h +++ b/src/include/sof/audio/pipeline.h @@ -206,6 +206,14 @@ int pipeline_complete(struct pipeline *p, struct comp_dev *source, */ void pipeline_posn_init(struct sof *sof); +#ifdef CONFIG_SOF_USERSPACE_LL +/** + * \brief Grants user-space thread access to pipeline position mutex. + * \param[in] thread Thread to grant access to. + */ +void pipeline_posn_grant_access(struct k_thread *thread); +#endif + /** * \brief Resets the pipeline and free runtime resources. * \param[in] p pipeline. diff --git a/src/init/init.c b/src/init/init.c index 55e7c7e31ba9..57f41b940bc2 100644 --- a/src/init/init.c +++ b/src/init/init.c @@ -32,6 +32,7 @@ #include #include #include +#include #include #if CONFIG_IPC_MAJOR_4 #include @@ -248,6 +249,11 @@ __cold static int primary_core_init(int argc, char *argv[], struct sof *sof) zephyr_ll_user_resources_init(); #endif + /* init pipeline position offsets - must be before platform_init() + * which calls ipc_init() -> ipc_user_init() that needs the posn mutex. + */ + pipeline_posn_init(sof); + /* init the platform */ if (platform_init(sof) < 0) sof_panic(SOF_IPC_PANIC_PLATFORM); diff --git a/src/ipc/ipc-common.c b/src/ipc/ipc-common.c index 51e4477b2191..26ae77493656 100644 --- a/src/ipc/ipc-common.c +++ b/src/ipc/ipc-common.c @@ -666,6 +666,7 @@ __cold int ipc_user_init(void) user_grant_dma_access_all(&ipc_user_thread); user_access_to_mailbox(zephyr_ll_mem_domain(), &ipc_user_thread); user_ll_grant_access(&ipc_user_thread, PLATFORM_PRIMARY_CORE_ID); + pipeline_posn_grant_access(&ipc_user_thread); k_mem_domain_add_thread(zephyr_ll_mem_domain(), &ipc_user_thread); k_thread_cpu_pin(&ipc_user_thread, PLATFORM_PRIMARY_CORE_ID); diff --git a/zephyr/wrapper.c b/zephyr/wrapper.c index e929135ce851..29b8cbdf82fc 100644 --- a/zephyr/wrapper.c +++ b/zephyr/wrapper.c @@ -177,9 +177,6 @@ int task_main_start(struct sof *sof) /* init default audio components */ sys_comp_init(sof); - /* init pipeline position offsets */ - pipeline_posn_init(sof); - return 0; } From 2ea1e4977ac29c74ff32e251552b7d4d2851a913 Mon Sep 17 00:00:00 2001 From: Kai Vehmanen Date: Fri, 5 Jun 2026 15:30:33 +0300 Subject: [PATCH 045/108] schedule: zephyr_ll: fix use-after-free when freeing an active user-space task zephyr_ll_task_sched_free() frees an active (RUNNING/RESCHEDULE) task by setting pdata->freeing and waiting on pdata->sem for the scheduler thread to remove the task from its run list before the memory is released. Under CONFIG_SOF_USERSPACE_LL this function runs in kernel context while pdata->sem is a sys_sem allocated on the user heap. sys_sem_take() returns -EINVAL immediately when called from kernel context, so the wait is a no-op: pdata is freed (and the struct task is subsequently freed by pipeline_free()) while the task is still linked in sch->tasks with n_tasks != 0 and the scheduling domain handler still set. Because n_tasks is non-zero, schedule_free() does not stop the LL timer, and the next timer tick runs zephyr_ll_run() over the dangling task, dereferencing freed memory and taking a load/store-privilege exception (EXCCAUSE 26) in the user-space LL thread. Stop relying on the cross-privilege semaphore handshake in this path. When the task must be waited for, mark it cancelled so that, should it actually be mid-execution on the scheduler's temporary list, it is removed via the cancel path without re-running task->run() on resources the caller may already have freed. If the task is still linked on the run list, the scheduler thread is provably not executing it (a running task is moved off sch->tasks with the lock dropped), so remove it directly and skip the wait. This guarantees the task is delisted (n_tasks -> 0, handler -> NULL) before pdata is freed, eliminating both the dangling list entry and the stray timer wakeups. Verified on PTL with the standalone user-space LL boot tests: the userspace_ll suite, including pipeline_two_components_user, now passes without the fatal exception at teardown. Signed-off-by: Kai Vehmanen --- src/schedule/zephyr_ll.c | 41 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/src/schedule/zephyr_ll.c b/src/schedule/zephyr_ll.c index 83d8bb99d1a6..d6b0672ddf35 100644 --- a/src/schedule/zephyr_ll.c +++ b/src/schedule/zephyr_ll.c @@ -158,6 +158,20 @@ static void zephyr_ll_task_done(struct zephyr_ll *sch, domain_unregister(sch->ll_domain, task, --sch->n_tasks); } +#if CONFIG_SOF_USERSPACE_LL +/* The caller must hold the scheduler lock */ +static bool zephyr_ll_task_on_run_list(struct zephyr_ll *sch, struct task *task) +{ + struct list_item *list; + + list_for_item(list, &sch->tasks) + if (list == &task->list) + return true; + + return false; +} +#endif + /* The caller must hold the lock and possibly disable interrupts */ static void zephyr_ll_task_insert_unlocked(struct zephyr_ll *sch, struct task *task) { @@ -509,6 +523,33 @@ static int zephyr_ll_task_free(void *data, struct task *task) pdata->freeing = true; +#if CONFIG_SOF_USERSPACE_LL + /* + * Under CONFIG_SOF_USERSPACE_LL this function runs in kernel context + * while the scheduler itself runs in a user-mode thread. The pdata->sem + * handshake used below cannot synchronise across that privilege boundary + * (sys_sem_take() returns -EINVAL when called from kernel context), so + * relying on it would free an active task while it is still linked in + * sch->tasks, leaving the scheduler to dereference freed memory on the + * next timer tick. + * + * Instead remove the task directly here. Mark it cancelled first so that, + * in the unlikely event it is currently mid-execution on the scheduler's + * temporary list, it is removed via the cancel path without re-running + * task->run() on resources the caller (e.g. pipeline_free()) may already + * have freed. If the task is still linked on the run list, the scheduler + * thread is not executing it (a running task is moved off sch->tasks with + * the lock dropped), so it is safe to remove it now and skip the wait. + */ + if (must_wait) { + task->state = SOF_TASK_STATE_CANCEL; + if (zephyr_ll_task_on_run_list(sch, task)) { + zephyr_ll_task_done(sch, task); + must_wait = false; + } + } +#endif + zephyr_ll_unlock(sch, &flags); if (must_wait) From 0ed5fd526764560c7a0159e7ef0dd908cab9c96d Mon Sep 17 00:00:00 2001 From: Kai Vehmanen Date: Thu, 16 Apr 2026 15:12:56 +0300 Subject: [PATCH 046/108] WIP: ipc: expose coldrodata to IPC user thread If SOF is built with CONFIG_SOF_USERSPACE_LL, the IPC user handled will require access to coldrodata sections to initialize audio modules. This logic is not required for LLEXT modules, which have existing code to add access to coldrodata (and other sections). This commit is needed for builds where LLEXT is not used. Signed-off-by: Kai Vehmanen --- src/ipc/ipc-common.c | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/src/ipc/ipc-common.c b/src/ipc/ipc-common.c index 26ae77493656..bcffb8490fd3 100644 --- a/src/ipc/ipc-common.c +++ b/src/ipc/ipc-common.c @@ -645,6 +645,40 @@ __cold int ipc_user_init(void) ret = k_mem_domain_add_partition(zephyr_ll_mem_domain(), &ipc_context_part); + /* + * Grant user-space access to .cold (execute) and .coldrodata (read) + * sections in IMR. The prepare path walks component code that may + * reference __cold functions and __cold_rodata data. + */ +#ifdef CONFIG_COLD_STORE_EXECUTE_DRAM + { + extern char __cold_start[], __cold_end[]; + extern char __coldrodata_start[]; + extern char _imr_end[]; + struct k_mem_partition cold_part; + + cold_part.start = (uintptr_t)__cold_start; + cold_part.size = ALIGN_UP((uintptr_t)__cold_end - + (uintptr_t)__cold_start, + CONFIG_MMU_PAGE_SIZE); + cold_part.attr = K_MEM_PARTITION_P_RX_U_RX; + ret = k_mem_domain_add_partition(zephyr_ll_mem_domain(), + &cold_part); + if (ret < 0) + LOG_WRN("cold text partition add failed: %d", ret); + + cold_part.start = (uintptr_t)__coldrodata_start; + cold_part.size = ALIGN_UP((uintptr_t)_imr_end - + (uintptr_t)__coldrodata_start, + CONFIG_MMU_PAGE_SIZE); + cold_part.attr = K_MEM_PARTITION_P_RO_U_RO; + ret = k_mem_domain_add_partition(zephyr_ll_mem_domain(), + &cold_part); + if (ret < 0) + LOG_WRN("cold rodata partition add failed: %d", ret); + } +#endif + k_sem_init(ipc_user->sem, 0, 1); /* Allocate kernel objects for the user-space thread */ From dcb5cf7de0a16704149aeb123021ae14b6945f95 Mon Sep 17 00:00:00 2001 From: Kai Vehmanen Date: Thu, 26 Feb 2026 17:14:39 +0200 Subject: [PATCH 047/108] HACK: audio: collection of feature limitations to run LL in user This is a set of temporary changes to audio code to remove calls to privileged interfaces that are not mandatory to run simple audio tests. These need proper solutions to be able to run all use-cases in user LL version. Signed-off-by: Kai Vehmanen --- src/audio/pipeline/pipeline-graph.c | 6 +++++- zephyr/include/sof/lib/cpu.h | 4 ++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/audio/pipeline/pipeline-graph.c b/src/audio/pipeline/pipeline-graph.c index efe4abfb44f5..5f0560720af4 100644 --- a/src/audio/pipeline/pipeline-graph.c +++ b/src/audio/pipeline/pipeline-graph.c @@ -572,6 +572,10 @@ struct comp_dev *pipeline_get_dai_comp(uint32_t pipeline_id, int dir) */ struct comp_dev *pipeline_get_dai_comp_latency(uint32_t pipeline_id, uint32_t *latency) { +#ifdef CONFIG_SOF_USERSPACE_LL + LOG_WRN("latency cannot be computed in user-space pipelines!"); + *latency = 0; +#else struct ipc_comp_dev *ipc_sink; struct ipc_comp_dev *ipc_source; struct comp_dev *source; @@ -639,7 +643,7 @@ struct comp_dev *pipeline_get_dai_comp_latency(uint32_t pipeline_id, uint32_t *l /* Get a next sink component */ ipc_sink = ipc_get_ppl_sink_comp(ipc, source->pipeline->pipeline_id); } - +#endif return NULL; } EXPORT_SYMBOL(pipeline_get_dai_comp_latency); diff --git a/zephyr/include/sof/lib/cpu.h b/zephyr/include/sof/lib/cpu.h index c23405e85121..533cb29f3602 100644 --- a/zephyr/include/sof/lib/cpu.h +++ b/zephyr/include/sof/lib/cpu.h @@ -55,7 +55,11 @@ static inline bool cpu_is_primary(int id) static inline bool cpu_is_me(int id) { +#ifdef CONFIG_SOF_USERSPACE_LL + return true; +#else return id == cpu_get_id(); +#endif } int cpu_enable_core(int id); From d87f5632d1fe2fce9be4ca753f5dc56ed19e900f Mon Sep 17 00:00:00 2001 From: Kai Vehmanen Date: Thu, 2 Jul 2026 18:31:55 +0300 Subject: [PATCH 048/108] Revert "module: generic: remove MEM_API_CHECK_THREAD debug mechanism" This reverts commit 768faad56199b52585c42f181011013102b1e164. --- src/audio/module_adapter/Kconfig | 11 +++++++++ src/audio/module_adapter/module/generic.c | 24 +++++++++++++++++++ .../sof/audio/module_adapter/module/generic.h | 7 ++++++ 3 files changed, 42 insertions(+) diff --git a/src/audio/module_adapter/Kconfig b/src/audio/module_adapter/Kconfig index fe8425ae5467..d80e42583b60 100644 --- a/src/audio/module_adapter/Kconfig +++ b/src/audio/module_adapter/Kconfig @@ -13,6 +13,17 @@ menu "Processing modules" containers to allocate at once is selected by this config option. + config MODULE_MEMORY_API_DEBUG + bool "Turn on memory API thread safety checks" + default y if DEBUG + help + The Module Memory API structures are not protected + by locks. This is because the initialization, + allocation, and freeing of resources should always + be done in the same thread. This option adds an + assert to make sure no other thread makes such + operations. + config CADENCE_CODEC bool "Cadence codec" help diff --git a/src/audio/module_adapter/module/generic.c b/src/audio/module_adapter/module/generic.c index ee1b2df92829..83ac6eb46cae 100644 --- a/src/audio/module_adapter/module/generic.c +++ b/src/audio/module_adapter/module/generic.c @@ -26,6 +26,16 @@ #include #endif +/* The __ZEPHYR__ condition is to keep cmocka tests working */ +#if CONFIG_MODULE_MEMORY_API_DEBUG && defined(__ZEPHYR__) +#define MEM_API_CHECK_THREAD(res) do { \ + if ((res)->rsrc_mngr != k_current_get()) \ + LOG_WRN("mngr %p != cur %p", (res)->rsrc_mngr, k_current_get()); \ +} while (0) +#else +#define MEM_API_CHECK_THREAD(res) +#endif + LOG_MODULE_DECLARE(module_adapter, CONFIG_SOF_LOG_LEVEL); int module_load_config(struct comp_dev *dev, const void *cfg, size_t size) @@ -114,6 +124,9 @@ int module_init(struct processing_module *mod) return -EIO; } +#if CONFIG_MODULE_MEMORY_API_DEBUG && defined(__ZEPHYR__) + mod->priv.resources.rsrc_mngr = k_current_get(); +#endif /* Now we can proceed with module specific initialization */ #if CONFIG_SOF_USERSPACE_APPLICATION if (mod->dev->ipc_config.proc_domain == COMP_PROCESSING_DOMAIN_DP) @@ -179,6 +192,8 @@ void *z_impl_mod_balloc_align(struct processing_module *mod, size_t size, size_t struct module_resources *res = &mod->priv.resources; struct module_resource *container; + MEM_API_CHECK_THREAD(res); + k_mutex_lock(&res->lock, K_FOREVER); container = container_get(mod); @@ -235,6 +250,8 @@ void *z_impl_mod_alloc_ext(struct processing_module *mod, uint32_t flags, size_t struct module_resources *res = &mod->priv.resources; struct module_resource *container; + MEM_API_CHECK_THREAD(res); + k_mutex_lock(&res->lock, K_FOREVER); container = container_get(mod); @@ -288,6 +305,8 @@ struct comp_data_blob_handler *z_impl_mod_data_blob_handler_new(struct processin struct comp_data_blob_handler *bhp; struct module_resource *container; + MEM_API_CHECK_THREAD(res); + k_mutex_lock(&res->lock, K_FOREVER); container = container_get(mod); @@ -328,6 +347,8 @@ const void *z_impl_mod_fast_get(struct processing_module *mod, const void * cons struct module_resource *container; const void *ptr; + MEM_API_CHECK_THREAD(res); + k_mutex_lock(&res->lock, K_FOREVER); container = container_get(mod); @@ -419,6 +440,7 @@ int z_impl_mod_free(struct processing_module *mod, const void *ptr) { struct module_resources *res = &mod->priv.resources; + MEM_API_CHECK_THREAD(res); if (!ptr) return 0; @@ -738,6 +760,8 @@ void mod_free_all(struct processing_module *mod) { struct module_resources *res = &mod->priv.resources; + MEM_API_CHECK_THREAD(res); + /* Free all contents found in used containers */ struct mod_res_cb_arg cb_arg = {mod, NULL}; diff --git a/src/include/sof/audio/module_adapter/module/generic.h b/src/include/sof/audio/module_adapter/module/generic.h index bfc4b19969c5..e943cf0ade0a 100644 --- a/src/include/sof/audio/module_adapter/module/generic.h +++ b/src/include/sof/audio/module_adapter/module/generic.h @@ -22,6 +22,10 @@ #include #include "module_interface.h" +/* The __ZEPHYR__ condition is to keep cmocka tests working */ +#if CONFIG_MODULE_MEMORY_API_DEBUG && defined(__ZEPHYR__) +#include +#endif #include /* @@ -131,6 +135,9 @@ struct module_resources { size_t heap_usage; size_t heap_high_water_mark; struct mod_alloc_ctx *alloc; +#if CONFIG_MODULE_MEMORY_API_DEBUG && defined(__ZEPHYR__) + k_tid_t rsrc_mngr; +#endif }; enum mod_resource_type { From 70358c5df5e850a6ad416940b57a2f173b591dbe Mon Sep 17 00:00:00 2001 From: Kai Vehmanen Date: Thu, 2 Jul 2026 18:31:58 +0300 Subject: [PATCH 049/108] Revert "module: generic: add mutex to protect module_resources" This reverts commit 7bb70f5bfff9620c9ae11eed1945902d34009f1c. --- src/audio/module_adapter/module/generic.c | 44 +++---------------- .../sof/audio/module_adapter/module/generic.h | 2 - 2 files changed, 5 insertions(+), 41 deletions(-) diff --git a/src/audio/module_adapter/module/generic.c b/src/audio/module_adapter/module/generic.c index 83ac6eb46cae..e1bd0813b097 100644 --- a/src/audio/module_adapter/module/generic.c +++ b/src/audio/module_adapter/module/generic.c @@ -12,7 +12,6 @@ */ #include -#include #include #include #include @@ -89,7 +88,6 @@ void mod_resource_init(struct processing_module *mod) struct module_resources *res = &mod->priv.resources; /* Init memory list */ - k_mutex_init(&res->lock); list_init(&res->objpool.list); res->objpool.heap = res->alloc->heap; res->objpool.vreg = res->alloc->vreg; @@ -194,18 +192,13 @@ void *z_impl_mod_balloc_align(struct processing_module *mod, size_t size, size_t MEM_API_CHECK_THREAD(res); - k_mutex_lock(&res->lock, K_FOREVER); - container = container_get(mod); - if (!container) { - k_mutex_unlock(&res->lock); + if (!container) return NULL; - } if (!size) { comp_err(mod->dev, "requested allocation of 0 bytes."); container_put(mod, container); - k_mutex_unlock(&res->lock); return NULL; } @@ -217,7 +210,6 @@ void *z_impl_mod_balloc_align(struct processing_module *mod, size_t size, size_t comp_err(mod->dev, "Failed to alloc %zu bytes %zu alignment for comp %#x.", size, alignment, dev_comp_id(mod->dev)); container_put(mod, container); - k_mutex_unlock(&res->lock); return NULL; } /* Store reference to allocated memory */ @@ -229,7 +221,6 @@ void *z_impl_mod_balloc_align(struct processing_module *mod, size_t size, size_t if (res->heap_usage > res->heap_high_water_mark) res->heap_high_water_mark = res->heap_usage; - k_mutex_unlock(&res->lock); return ptr; } EXPORT_SYMBOL(z_impl_mod_balloc_align); @@ -252,18 +243,13 @@ void *z_impl_mod_alloc_ext(struct processing_module *mod, uint32_t flags, size_t MEM_API_CHECK_THREAD(res); - k_mutex_lock(&res->lock, K_FOREVER); - container = container_get(mod); - if (!container) { - k_mutex_unlock(&res->lock); + if (!container) return NULL; - } if (!size) { comp_err(mod->dev, "requested allocation of 0 bytes."); container_put(mod, container); - k_mutex_unlock(&res->lock); return NULL; } @@ -274,7 +260,6 @@ void *z_impl_mod_alloc_ext(struct processing_module *mod, uint32_t flags, size_t comp_err(mod->dev, "Failed to alloc %zu bytes %zu alignment for comp %#x.", size, alignment, dev_comp_id(mod->dev)); container_put(mod, container); - k_mutex_unlock(&res->lock); return NULL; } /* Store reference to allocated memory */ @@ -286,7 +271,6 @@ void *z_impl_mod_alloc_ext(struct processing_module *mod, uint32_t flags, size_t if (res->heap_usage > res->heap_high_water_mark) res->heap_high_water_mark = res->heap_usage; - k_mutex_unlock(&res->lock); return ptr; } EXPORT_SYMBOL(z_impl_mod_alloc_ext); @@ -301,24 +285,19 @@ EXPORT_SYMBOL(z_impl_mod_alloc_ext); #if CONFIG_COMP_BLOB struct comp_data_blob_handler *z_impl_mod_data_blob_handler_new(struct processing_module *mod) { - struct module_resources *res = &mod->priv.resources; + struct module_resources * __maybe_unused res = &mod->priv.resources; struct comp_data_blob_handler *bhp; struct module_resource *container; MEM_API_CHECK_THREAD(res); - k_mutex_lock(&res->lock, K_FOREVER); - container = container_get(mod); - if (!container) { - k_mutex_unlock(&res->lock); + if (!container) return NULL; - } bhp = comp_data_blob_handler_new_ext(mod->dev, false, NULL, NULL); if (!bhp) { container_put(mod, container); - k_mutex_unlock(&res->lock); return NULL; } @@ -326,7 +305,6 @@ struct comp_data_blob_handler *z_impl_mod_data_blob_handler_new(struct processin container->size = 0; container->type = MOD_RES_BLOB_HANDLER; - k_mutex_unlock(&res->lock); return bhp; } EXPORT_SYMBOL(z_impl_mod_data_blob_handler_new); @@ -349,18 +327,13 @@ const void *z_impl_mod_fast_get(struct processing_module *mod, const void * cons MEM_API_CHECK_THREAD(res); - k_mutex_lock(&res->lock, K_FOREVER); - container = container_get(mod); - if (!container) { - k_mutex_unlock(&res->lock); + if (!container) return NULL; - } ptr = fast_get(res->alloc, dram_ptr, size); if (!ptr) { container_put(mod, container); - k_mutex_unlock(&res->lock); return NULL; } @@ -368,7 +341,6 @@ const void *z_impl_mod_fast_get(struct processing_module *mod, const void * cons container->size = 0; container->type = MOD_RES_FAST_GET; - k_mutex_unlock(&res->lock); return ptr; } EXPORT_SYMBOL(z_impl_mod_fast_get); @@ -446,12 +418,8 @@ int z_impl_mod_free(struct processing_module *mod, const void *ptr) /* Find which container holds this memory */ struct mod_res_cb_arg cb_arg = {mod, ptr}; - - k_mutex_lock(&res->lock, K_FOREVER); int ret = objpool_iterate(&res->objpool, mod_res_free, &cb_arg); - k_mutex_unlock(&res->lock); - if (ret < 0) comp_err(mod->dev, "error: could not find memory pointed by %p", ptr); @@ -765,10 +733,8 @@ void mod_free_all(struct processing_module *mod) /* Free all contents found in used containers */ struct mod_res_cb_arg cb_arg = {mod, NULL}; - k_mutex_lock(&res->lock, K_FOREVER); objpool_iterate(&res->objpool, mod_res_free, &cb_arg); objpool_prune(&res->objpool); - k_mutex_unlock(&res->lock); /* Make sure resource lists and accounting are reset */ mod_resource_init(mod); diff --git a/src/include/sof/audio/module_adapter/module/generic.h b/src/include/sof/audio/module_adapter/module/generic.h index e943cf0ade0a..5450b07aff7f 100644 --- a/src/include/sof/audio/module_adapter/module/generic.h +++ b/src/include/sof/audio/module_adapter/module/generic.h @@ -13,7 +13,6 @@ #ifndef __SOF_AUDIO_MODULE_GENERIC__ #define __SOF_AUDIO_MODULE_GENERIC__ -#include #include #include #include @@ -130,7 +129,6 @@ struct module_param { * when the module unloads. */ struct module_resources { - struct k_mutex lock; struct objpool_head objpool; size_t heap_usage; size_t heap_high_water_mark; From 5bef04210061a90d5575d3480ccbce9995ad55b5 Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Thu, 21 May 2026 14:13:46 +0200 Subject: [PATCH 050/108] ipc: only add a partition if it's non-empty The .coldrodata partition can be empty, avoid a failure in such cases. Signed-off-by: Guennadi Liakhovetski --- src/ipc/ipc-common.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/ipc/ipc-common.c b/src/ipc/ipc-common.c index bcffb8490fd3..c61351e1d478 100644 --- a/src/ipc/ipc-common.c +++ b/src/ipc/ipc-common.c @@ -672,10 +672,13 @@ __cold int ipc_user_init(void) (uintptr_t)__coldrodata_start, CONFIG_MMU_PAGE_SIZE); cold_part.attr = K_MEM_PARTITION_P_RO_U_RO; - ret = k_mem_domain_add_partition(zephyr_ll_mem_domain(), - &cold_part); - if (ret < 0) - LOG_WRN("cold rodata partition add failed: %d", ret); + if (cold_part.size && cold_part.start) { + ret = k_mem_domain_add_partition(zephyr_ll_mem_domain(), + &cold_part); + if (ret < 0) + LOG_WRN("cold rodata partition %#zx @ %#lx add failed: %d", cold_part.size, + cold_part.start, ret); + } } #endif From 54dbe709aef1ea3fb9358359ce4913d03cc8d637 Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Thu, 21 May 2026 16:55:15 +0200 Subject: [PATCH 051/108] audio: pipeline: add a missing header Add a missing header for the zephyr_ll_(un)lock_sched() functions. Signed-off-by: Guennadi Liakhovetski --- src/audio/pipeline/pipeline-graph.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/audio/pipeline/pipeline-graph.c b/src/audio/pipeline/pipeline-graph.c index 5f0560720af4..1eff4ebc58e1 100644 --- a/src/audio/pipeline/pipeline-graph.c +++ b/src/audio/pipeline/pipeline-graph.c @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include From 7d8bd456d080c7e1724cba57bb7aa06d7c4f290c Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Fri, 22 May 2026 10:53:50 +0200 Subject: [PATCH 052/108] lib-manager: add a syscall to handle LLEXT-related work Extract a privileged LLEXT-related part from lib_manager_module_create() into a separate function to be called from kernel context. At the same time lib_manager_mod_free_priv() already executes privileged operations, to make it callable in userspace convert lib_manager_free_module() to a system call. Signed-off-by: Guennadi Liakhovetski --- src/include/sof/lib_manager.h | 14 ++++ src/library_manager/lib_manager.c | 105 ++++++++++++++++++++---------- zephyr/CMakeLists.txt | 1 + 3 files changed, 87 insertions(+), 33 deletions(-) diff --git a/src/include/sof/lib_manager.h b/src/include/sof/lib_manager.h index 29c226eb61a7..8796c61c052f 100644 --- a/src/include/sof/lib_manager.h +++ b/src/include/sof/lib_manager.h @@ -217,6 +217,18 @@ void lib_manager_get_instance_bss_address(uint32_t instance_id, */ int lib_manager_load_library(uint32_t dma_id, uint32_t lib_id, uint32_t type); +/* + * \brief Allocate the module and start the agent if needed + */ +int lib_manager_mod_create_priv(const struct comp_driver *drv, + const struct comp_ipc_config *config, + const void *spec, void **adapter_priv, + struct userspace_context **userspace, + const struct module_interface **ops); + +#include +__syscall int lib_manager_free_module(const uint32_t component_id); + /* * \brief Initialize message * @@ -250,4 +262,6 @@ void lib_notif_msg_send(struct ipc_msg *msg); */ void lib_notif_msg_clean(bool leave_one_handle); +#include + #endif /* __SOF_LIB_MANAGER_H__ */ diff --git a/src/library_manager/lib_manager.c b/src/library_manager/lib_manager.c index 19c317ed1089..2e631747d03f 100644 --- a/src/library_manager/lib_manager.c +++ b/src/library_manager/lib_manager.c @@ -415,7 +415,7 @@ static uintptr_t lib_manager_allocate_module(const struct sof_man_fw_desc *const * * Function is responsible to free module resources in HP memory. */ -static int lib_manager_free_module(const uint32_t component_id) +int z_impl_lib_manager_free_module(const uint32_t component_id) { const struct sof_man_module *mod; const uint32_t module_id = IPC4_MOD_ID(component_id); @@ -462,7 +462,7 @@ static uintptr_t lib_manager_allocate_module(const struct sof_man_fw_desc *const return 0; } -static int lib_manager_free_module(const uint32_t component_id) +static int z_impl_lib_manager_free_module(const uint32_t component_id) { /* Since we cannot allocate the freeing is not considered to be an error */ tr_warn(&lib_manager_tr, "Dynamic module freeing is not supported"); @@ -642,34 +642,35 @@ static enum buildinfo_mod_type lib_manager_get_module_type(const struct sof_man_ } } -/* - * \brief Load module code, allocate its instance and create a module adapter component. - * \param[in] drv - component driver pointer. - * \param[in] config - component ipc descriptor pointer. - * \param[in] spec - passdowned data from driver. - * - * \return: a pointer to newly created module adapter component on success. NULL on error. - */ -static struct comp_dev *lib_manager_module_create(const struct comp_driver *drv, - const struct comp_ipc_config *config, - const void *spec) +static void lib_manager_mod_free_priv(const struct comp_driver *drv, + const struct comp_ipc_config *config, + struct userspace_context *userspace) +{ +#if CONFIG_SOF_USERSPACE_PROXY + if (userspace) + userspace_proxy_destroy(drv, userspace); +#endif /* CONFIG_SOF_USERSPACE_PROXY */ + lib_manager_free_module(config->id); +} + +int lib_manager_mod_create_priv(const struct comp_driver *drv, + const struct comp_ipc_config *config, + const void *spec, void **adapter_priv, + struct userspace_context **userspace, + const struct module_interface **ops) { const struct sof_man_fw_desc *const desc = lib_manager_get_library_manifest(config->id); const struct ipc_config_process *args = (const struct ipc_config_process *)spec; const uint32_t entry_index = LIB_MANAGER_GET_MODULE_INDEX(config->id); - struct userspace_context *userspace = NULL; - const struct module_interface *ops; const struct sof_man_module *mod; system_agent_start_fn agent; - void *adapter_priv = NULL; const void **agent_iface; - struct comp_dev *dev; int ret; #ifdef CONFIG_SOF_USERSPACE_PROXY if (drv->user_heap && config->proc_domain != COMP_PROCESSING_DOMAIN_DP) { tr_err(&lib_manager_tr, "Userspace supports only DP modules."); - return NULL; + return -EOPNOTSUPP; } #endif @@ -677,12 +678,12 @@ static struct comp_dev *lib_manager_module_create(const struct comp_driver *drv, if (!desc) { tr_err(&lib_manager_tr, "Error: Couldn't find loadable module with id %u.", config->id); - return NULL; + return -ENOENT; } if (entry_index >= desc->header.num_module_entries) { tr_err(&lib_manager_tr, "Entry index %u out of bounds.", entry_index); - return NULL; + return -EINVAL; } mod = (const struct sof_man_module *) @@ -693,53 +694,91 @@ static struct comp_dev *lib_manager_module_create(const struct comp_driver *drv, if (!module_entry_point) { tr_err(&lib_manager_tr, "lib_manager_allocate_module() failed!"); - return NULL; + return -ENOENT; } switch (lib_manager_get_module_type(desc, mod)) { case MOD_TYPE_LLEXT: agent = NULL; - ops = (const struct module_interface *)module_entry_point; + *ops = (const struct module_interface *)module_entry_point; agent_iface = NULL; break; case MOD_TYPE_LMDK: agent = &native_system_agent_start; - agent_iface = (const void **)&ops; + agent_iface = (const void **)ops; break; #if CONFIG_INTEL_MODULES case MOD_TYPE_IADK: agent = &system_agent_start; - ops = &processing_module_adapter_interface; - agent_iface = (const void **)&adapter_priv; + *ops = &processing_module_adapter_interface; + agent_iface = (const void **)adapter_priv; break; #endif case MOD_TYPE_INVALID: + default: + ret = -EINVAL; goto err; } if (agent || IS_ENABLED(CONFIG_SOF_USERSPACE_PROXY)) { /* At this point module resources are allocated and it is moved to L2 memory. */ ret = lib_manager_start_agent(drv, config, mod, args, module_entry_point, agent, - agent_iface, &userspace, &ops); + agent_iface, userspace, ops); if (ret) goto err; } - if (comp_set_adapter_ops(drv, ops) < 0) + ret = comp_set_adapter_ops(drv, *ops); + if (ret < 0) goto err; - dev = module_adapter_new_ext(drv, config, spec, adapter_priv, userspace, NULL); + return 0; + +err: + lib_manager_mod_free_priv(drv, config, *userspace); + return ret; +} + +#ifdef CONFIG_USERSPACE +#include + +static int z_vrfy_lib_manager_free_module(const uint32_t component_id) +{ + return z_impl_lib_manager_free_module(component_id); +} +#include + +#endif /* CONFIG_USERSPACE */ + +/* + * \brief Load module code, allocate its instance and create a module adapter component. + * \param[in] drv - component driver pointer. + * \param[in] config - component ipc descriptor pointer. + * \param[in] spec - passdowned data from driver. + * + * \return: a pointer to newly created module adapter component on success. NULL on error. + */ +static struct comp_dev *lib_manager_module_create(const struct comp_driver *drv, + const struct comp_ipc_config *config, + const void *spec) +{ + struct userspace_context *userspace = NULL; + const struct module_interface *ops; + void *adapter_priv = NULL; + struct comp_dev *dev; + int ret = lib_manager_mod_create_priv(drv, config, spec, &adapter_priv, &userspace, &ops); + + if (ret < 0) + return NULL; + + dev = module_adapter_new_ext(drv, config, spec, adapter_priv, userspace, ops); if (!dev) goto err; return dev; err: -#if CONFIG_SOF_USERSPACE_PROXY - if (userspace) - userspace_proxy_destroy(drv, userspace); -#endif /* CONFIG_SOF_USERSPACE_PROXY */ - lib_manager_free_module(config->id); + lib_manager_mod_free_priv(drv, config, userspace); return NULL; } diff --git a/zephyr/CMakeLists.txt b/zephyr/CMakeLists.txt index 4042d56ae546..482d7d675a9f 100644 --- a/zephyr/CMakeLists.txt +++ b/zephyr/CMakeLists.txt @@ -630,6 +630,7 @@ zephyr_library_sources_ifdef(CONFIG_SOF_USERSPACE_INTERFACE_ALLOC syscall/alloc. zephyr_syscall_header(${SOF_SRC_PATH}/include/sof/lib/vregion.h) zephyr_library_sources(syscall/vregion.c) zephyr_syscall_header(${SOF_SRC_PATH}/include/sof/lib/dai-zephyr.h) +zephyr_syscall_header(${SOF_SRC_PATH}/include/sof/lib_manager.h) zephyr_library_sources_ifdef(CONFIG_USERSPACE syscall/dai.c) zephyr_syscall_header(${SOF_SRC_PATH}/include/user/debug_stream_slot.h) From 549dc53ed4dd97d18198d2b961fc4748ffcbc75a Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Wed, 17 Jun 2026 12:48:38 +0200 Subject: [PATCH 053/108] audio: pipeline: add a missing header ll_schedule_domain.h is needed for user_ll_lock_sched() and user_ll_unlock_sched() Signed-off-by: Guennadi Liakhovetski --- src/audio/pipeline/pipeline-schedule.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/audio/pipeline/pipeline-schedule.c b/src/audio/pipeline/pipeline-schedule.c index 3612b63df08a..4015c7f02518 100644 --- a/src/audio/pipeline/pipeline-schedule.c +++ b/src/audio/pipeline/pipeline-schedule.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include From 342f499b58f13675f8a348ee13ed71c7449589ca Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Wed, 17 Jun 2026 12:53:37 +0200 Subject: [PATCH 054/108] ipc: ipc4: type-cast a pointer instead of memcpy() Use a pointer type-cast instead of copying a structure. Signed-off-by: Guennadi Liakhovetski --- src/ipc/ipc4/handler-user.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/ipc/ipc4/handler-user.c b/src/ipc/ipc4/handler-user.c index 8a390cea621f..3d9bbaf0cbb8 100644 --- a/src/ipc/ipc4/handler-user.c +++ b/src/ipc/ipc4/handler-user.c @@ -1631,10 +1631,10 @@ __cold int ipc4_user_process_module_message(struct ipc4_message_request *ipc4, case SOF_IPC4_MOD_LARGE_CONFIG_SET: #ifdef CONFIG_SOF_USERSPACE_LL { - struct ipc4_module_large_config config; + const struct ipc4_module_large_config *config = + (const struct ipc4_module_large_config *)ipc4; - memcpy_s(&config, sizeof(config), ipc4, sizeof(*ipc4)); - if (config.primary.r.module_id) { + if (config->primary.r.module_id) { ret = ipc_user_forward_cmd(ipc4); } else { /* Base firmware: keep in kernel (IMR access) */ From 8fd45353250fe9188d2fe5391c964e40856da025 Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Wed, 17 Jun 2026 12:54:38 +0200 Subject: [PATCH 055/108] ipc: ipc4: fix a logging format Use "%p" to log a pointer. Signed-off-by: Guennadi Liakhovetski --- src/ipc/ipc4/helper.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ipc/ipc4/helper.c b/src/ipc/ipc4/helper.c index 2b9312e03dba..06461672baa8 100644 --- a/src/ipc/ipc4/helper.c +++ b/src/ipc/ipc4/helper.c @@ -443,7 +443,7 @@ __cold static int ipc4_create_pipeline(struct ipc4_pipeline_create *pipe_desc, assert_can_be_cold(); - LOG_INF("pipe_desc %x, instance %u", pipe_desc, pipe_desc->primary.r.instance_id); + LOG_INF("pipe_desc %p, instance %u", pipe_desc, pipe_desc->primary.r.instance_id); /* check whether pipeline id is already taken or in use */ ipc_pipe = ipc_get_pipeline_by_id(ipc, pipe_desc->primary.r.instance_id); From a03e8f45dbb23421be770b3a4a2c64e446f9bda8 Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Wed, 17 Jun 2026 12:55:13 +0200 Subject: [PATCH 056/108] schedule: LL: limit scope of two functions scheduler_get_task_info_ll() and zephyr_ll_domain() are only needed when CONFIG_SOF_USERSPACE_LL=y Signed-off-by: Guennadi Liakhovetski --- src/schedule/zephyr_ll.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/schedule/zephyr_ll.c b/src/schedule/zephyr_ll.c index d6b0672ddf35..15360ea98c6c 100644 --- a/src/schedule/zephyr_ll.c +++ b/src/schedule/zephyr_ll.c @@ -807,6 +807,7 @@ __cold int zephyr_ll_scheduler_init(struct ll_schedule_domain *domain) return 0; } +#if CONFIG_SOF_USERSPACE_LL void scheduler_get_task_info_ll(struct scheduler_props *scheduler_props, uint32_t *data_off_size) { @@ -827,3 +828,4 @@ struct ll_schedule_domain *zephyr_ll_domain(void) return ll_sch->ll_domain; } +#endif From 3a9dde3b2d790902aac4161c1ba1230595356887 Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Fri, 22 May 2026 13:48:14 +0200 Subject: [PATCH 057/108] llext: add a function to map libraries to userspace Map library data in DRAM to the LL memory domain but only with kernel access. This is needed for LLEXT ELF linking. Signed-off-by: Guennadi Liakhovetski --- src/include/sof/lib_manager.h | 1 + src/include/sof/llext_manager.h | 2 ++ src/library_manager/llext_manager.c | 32 ++++++++++++++++++++++++ src/library_manager/llext_manager_dram.c | 2 ++ 4 files changed, 37 insertions(+) diff --git a/src/include/sof/lib_manager.h b/src/include/sof/lib_manager.h index 8796c61c052f..5f0efcc29a4b 100644 --- a/src/include/sof/lib_manager.h +++ b/src/include/sof/lib_manager.h @@ -122,6 +122,7 @@ struct lib_manager_mod_ctx { void *base_addr; /* library cold storage address (e.g. DRAM) */ unsigned int n_mod; struct lib_manager_module *mod; + bool user_mapped; }; struct ext_library { diff --git a/src/include/sof/llext_manager.h b/src/include/sof/llext_manager.h index 525fa50b1506..080a58d623a4 100644 --- a/src/include/sof/llext_manager.h +++ b/src/include/sof/llext_manager.h @@ -32,6 +32,7 @@ int llext_manager_add_library(uint32_t module_id); int llext_manager_add_domain(const uint32_t component_id, struct k_mem_domain *domain); int llext_manager_rm_domain(const uint32_t component_id, struct k_mem_domain *domain); +int llext_manager_map_lib(uint32_t comp_id); bool comp_is_llext(struct comp_dev *comp); #else @@ -40,6 +41,7 @@ bool comp_is_llext(struct comp_dev *comp); #define llext_manager_free_module(component_id) 0 #define llext_manager_add_library(module_id) 0 #define llext_manager_add_domain(component_id, domain) 0 +#define llext_manager_map_lib(component_id) 0 #define comp_is_llext(comp) false #endif diff --git a/src/library_manager/llext_manager.c b/src/library_manager/llext_manager.c index 4c1e4f02d5b5..55e47a1f2bd0 100644 --- a/src/library_manager/llext_manager.c +++ b/src/library_manager/llext_manager.c @@ -748,6 +748,38 @@ uintptr_t llext_manager_allocate_module(const struct comp_ipc_config *ipc_config } #ifdef CONFIG_USERSPACE +#ifdef CONFIG_SOF_USERSPACE_LL +int llext_manager_map_lib(uint32_t comp_id) +{ + struct lib_manager_mod_ctx *ctx = lib_manager_get_mod_ctx(comp_id); + + if (!ctx || ctx->user_mapped) + return 0; + + const struct sof_man_fw_desc *const desc = lib_manager_get_library_manifest(comp_id); + + if (!desc) + /* Built-in driver */ + return 0; + + size_t preload_size = desc->header.preload_page_count * CONFIG_MM_DRV_PAGE_SIZE; + struct k_mem_partition lib_part = { + .start = (uintptr_t)desc, + .size = preload_size, + .attr = K_MEM_PARTITION_P_RW_U_NA | XTENSA_MMU_CACHED_WB, + }; + + int ret = k_mem_domain_add_partition(zephyr_ll_mem_domain(), &lib_part); + + if (ret < 0) + return ret; + + ctx->user_mapped = true; + + return 0; +} +#endif + static int llext_manager_add_partition(struct k_mem_domain *domain, uintptr_t addr, size_t size, k_mem_partition_attr_t attr) diff --git a/src/library_manager/llext_manager_dram.c b/src/library_manager/llext_manager_dram.c index 2f6cff2b3501..f3db841b521e 100644 --- a/src/library_manager/llext_manager_dram.c +++ b/src/library_manager/llext_manager_dram.c @@ -210,6 +210,8 @@ int llext_manager_restore_from_dram(void) /* Restore the library context */ *ctx = lib_manager_dram.ctx[j++]; + /* It will have to be mapped again, reset the flag */ + ctx->user_mapped = false; /* Allocate and restore all the modules in the library */ struct lib_manager_module *mod = rmalloc(SOF_MEM_FLAG_KERNEL | SOF_MEM_FLAG_COHERENT, From 8d537481dcd2af914c2f026fa43315cb6db51685 Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Fri, 22 May 2026 15:03:28 +0200 Subject: [PATCH 058/108] lib-manager: use user heap for the driver object When CONFIG_SOF_USERSPACE_USE_DRIVER_HEAP isn't selected, dynamically allocated driver objects should still be accessible to the userspace. Signed-off-by: Guennadi Liakhovetski --- src/library_manager/lib_manager.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/library_manager/lib_manager.c b/src/library_manager/lib_manager.c index 2e631747d03f..dd30c3db73a9 100644 --- a/src/library_manager/lib_manager.c +++ b/src/library_manager/lib_manager.c @@ -858,6 +858,8 @@ int lib_manager_register_module(const uint32_t component_id) goto cleanup; } } +#else + drv_heap = sof_sys_user_heap_get(); #endif /* CONFIG_SOF_USERSPACE_USE_DRIVER_HEAP */ drv = sof_heap_alloc(drv_heap, SOF_MEM_FLAG_KERNEL | SOF_MEM_FLAG_COHERENT, From c39d5b5f7134e125b0d886ac653b298aae42d593 Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Fri, 22 May 2026 15:37:31 +0200 Subject: [PATCH 059/108] llext: map modules for userspace LL When loading and linking LLEXT modules map them automatically for the LL memory domain, unless they belong to the DP domain. Signed-off-by: Guennadi Liakhovetski --- src/include/sof/lib_manager.h | 1 + src/library_manager/llext_manager.c | 58 +++++++++++++++++++++++++---- 2 files changed, 51 insertions(+), 8 deletions(-) diff --git a/src/include/sof/lib_manager.h b/src/include/sof/lib_manager.h index 5f0efcc29a4b..1056656191c7 100644 --- a/src/include/sof/lib_manager.h +++ b/src/include/sof/lib_manager.h @@ -115,6 +115,7 @@ struct lib_manager_module { struct llext_buf_loader *ebl; /* Zephyr loadable extension buffer loader */ unsigned int n_dependent; /* For auxiliary modules: number of dependents */ bool mapped; + bool domain_dp; struct lib_manager_segment_desc segment[LIB_MANAGER_N_SEGMENTS]; }; diff --git a/src/library_manager/llext_manager.c b/src/library_manager/llext_manager.c index 55e47a1f2bd0..439aad9a4cb5 100644 --- a/src/library_manager/llext_manager.c +++ b/src/library_manager/llext_manager.c @@ -10,6 +10,7 @@ */ #include +#include #include #include #include @@ -23,6 +24,7 @@ #include #include #include +#include #include #include @@ -199,6 +201,11 @@ static void llext_manager_unmap_detached_sections(const struct llext_loader *ldr #endif } +static int llext_manager_add_mod_domain(struct lib_manager_module *mctx, + struct k_mem_domain *domain); +static int llext_manager_rm_mod_domain(struct lib_manager_module *mctx, + struct k_mem_domain *domain); + static int llext_manager_load_module(struct lib_manager_module *mctx) { /* Executable code (.text) */ @@ -292,8 +299,18 @@ static int llext_manager_load_module(struct lib_manager_module *mctx) memset((__sparse_force void *)bss_addr, 0, bss_size); mctx->mapped = true; - return 0; + if (!mctx->domain_dp) { + ret = llext_manager_add_mod_domain(mctx, zephyr_ll_mem_domain()); + if (ret < 0) { + tr_err(&lib_manager_tr, "failed to add domain: %d", ret); + goto e_data; + } + } + return 0; +e_data: + if (data_size) + llext_manager_align_unmap(va_base_data, data_size); e_rodata: if (rodata_size) llext_manager_align_unmap(va_base_rodata, rodata_size); @@ -353,6 +370,9 @@ static int llext_manager_unload_module(struct lib_manager_module *mctx) mctx->mapped = false; + if (!mctx->domain_dp) + llext_manager_rm_mod_domain(mctx, zephyr_ll_mem_domain()); + return err; } @@ -496,6 +516,7 @@ static int llext_manager_mod_init(struct lib_manager_mod_ctx *ctx, if (mod_array[i].segment[LIB_MANAGER_TEXT].file_offset != offs) { offs = mod_array[i].segment[LIB_MANAGER_TEXT].file_offset; ctx->mod[n_mod].mapped = false; + ctx->mod[n_mod].domain_dp = false; ctx->mod[n_mod].llext = NULL; ctx->mod[n_mod].ebl = NULL; ctx->mod[n_mod].n_dependent = 0; @@ -738,6 +759,8 @@ uintptr_t llext_manager_allocate_module(const struct comp_ipc_config *ipc_config dep_ctx[i] = dep; } + /* Avoid mapping DP modules to the LL domain */ + mctx->domain_dp = ipc_config->proc_domain == COMP_PROCESSING_DOMAIN_DP; /* Map executable code and data */ ret = llext_manager_load_module(mctx); if (ret < 0) @@ -810,13 +833,8 @@ static int llext_manager_rm_partition(struct k_mem_domain *domain, return k_mem_domain_remove_partition(domain, &part); } -int llext_manager_add_domain(const uint32_t component_id, struct k_mem_domain *domain) +static int llext_manager_add_mod_domain(struct lib_manager_module *mctx, struct k_mem_domain *domain) { - const uint32_t module_id = IPC4_MOD_ID(component_id); - struct lib_manager_mod_ctx *ctx = lib_manager_get_mod_ctx(module_id); - const uint32_t entry_index = LIB_MANAGER_GET_MODULE_INDEX(module_id); - const unsigned int mod_idx = llext_manager_mod_find(ctx, entry_index); - struct lib_manager_module *mctx = ctx->mod + mod_idx; const struct llext *ext = mctx->llext; const struct llext_loader *ldr = &mctx->ebl->loader; @@ -832,6 +850,13 @@ int llext_manager_add_domain(const uint32_t component_id, struct k_mem_domain *d uintptr_t va_base_data = mctx->segment[LIB_MANAGER_DATA].addr; size_t data_size = mctx->segment[LIB_MANAGER_DATA].size; + /* + * Add to domain on first load: for "normal" modules use_count == 1, + * for dependencies use_count == 2 and n_dependent == 1 + */ + if (ext->use_count > 1 && mctx->n_dependent != 1) + return 0; + int ret = llext_manager_add_partition(domain, va_base_text, text_size, K_MEM_PARTITION_P_RX_U_RX | XTENSA_MMU_CACHED_WB); @@ -946,7 +971,7 @@ int llext_manager_add_domain(const uint32_t component_id, struct k_mem_domain *d return ret; } -int llext_manager_rm_domain(const uint32_t component_id, struct k_mem_domain *domain) +int llext_manager_add_domain(const uint32_t component_id, struct k_mem_domain *domain) { const uint32_t module_id = IPC4_MOD_ID(component_id); struct lib_manager_mod_ctx *ctx = lib_manager_get_mod_ctx(module_id); @@ -954,6 +979,12 @@ int llext_manager_rm_domain(const uint32_t component_id, struct k_mem_domain *do const unsigned int mod_idx = llext_manager_mod_find(ctx, entry_index); struct lib_manager_module *mctx = ctx->mod + mod_idx; + /* FIXME: handle dependencies */ + return llext_manager_add_mod_domain(mctx, domain); +} + +static int llext_manager_rm_mod_domain(struct lib_manager_module *mctx, struct k_mem_domain *domain) +{ /* Executable code (.text) */ uintptr_t va_base_text = mctx->segment[LIB_MANAGER_TEXT].addr; size_t text_size = mctx->segment[LIB_MANAGER_TEXT].size; @@ -1020,6 +1051,17 @@ int llext_manager_rm_domain(const uint32_t component_id, struct k_mem_domain *do return ret; } + +int llext_manager_rm_domain(const uint32_t component_id, struct k_mem_domain *domain) +{ + const uint32_t module_id = IPC4_MOD_ID(component_id); + struct lib_manager_mod_ctx *ctx = lib_manager_get_mod_ctx(module_id); + const uint32_t entry_index = LIB_MANAGER_GET_MODULE_INDEX(module_id); + const unsigned int mod_idx = llext_manager_mod_find(ctx, entry_index); + struct lib_manager_module *mctx = ctx->mod + mod_idx; + + return llext_manager_rm_mod_domain(mctx, domain); +} #endif int llext_manager_free_module(const uint32_t component_id) From 157e8f212f7705f598eedcf765981cc329c859a9 Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Fri, 22 May 2026 16:01:09 +0200 Subject: [PATCH 060/108] ipc: enable syscalls for userspace always Even if userspace LL is disabled but generic userspace is enabled, IPC syscalls can be enabled. Signed-off-by: Guennadi Liakhovetski --- src/include/ipc4/handler.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/include/ipc4/handler.h b/src/include/ipc4/handler.h index 6073e4dc83c0..9e2c96c5ba28 100644 --- a/src/include/ipc4/handler.h +++ b/src/include/ipc4/handler.h @@ -68,7 +68,7 @@ int ipc4_set_pipeline_state(struct ipc4_message_request *ipc4); */ void ipc_compound_msg_done(uint32_t msg_id, int error); -#if defined(__ZEPHYR__) && defined(CONFIG_SOF_USERSPACE_LL) +#if defined(__ZEPHYR__) && defined(CONFIG_USERSPACE) /** * \brief Increment the IPC compound message pre-start counter. * @param[in] msg_id IPC message ID. From 0bc156908e55fa7b1e04ab586a064b587ea76f2b Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Fri, 22 May 2026 16:04:46 +0200 Subject: [PATCH 061/108] ipc: remove driver copy Drivers are now accessible to userspace LL, remove now superfluous copies. Signed-off-by: Guennadi Liakhovetski --- src/ipc/ipc-common.c | 23 +++++------------------ src/ipc/ipc4/handler-user.c | 28 ++++------------------------ 2 files changed, 9 insertions(+), 42 deletions(-) diff --git a/src/ipc/ipc-common.c b/src/ipc/ipc-common.c index c61351e1d478..cb91f957c78e 100644 --- a/src/ipc/ipc-common.c +++ b/src/ipc/ipc-common.c @@ -519,23 +519,17 @@ static void ipc_user_thread_fn(void *p1, void *p2, void *p3) * with kernel privileges. * * init_drv = original kernel pointer - * init_drv_data = user-accessible copy */ - const struct comp_driver *orig_drv = - ipc_user->init_drv; - const struct comp_driver *drv_copy = - (const struct comp_driver *) - ipc_user->init_drv_data; + const struct comp_driver *drv = ipc_user->init_drv; ipc_user->init_drv = NULL; - if (!orig_drv) { + if (!drv) { ipc_user->result = IPC4_MOD_NOT_INITIALIZED; break; } - struct comp_dev *dev = - comp_new_ipc4_user(&msg, drv_copy); + struct comp_dev *dev = comp_new_ipc4_user(&msg, drv); if (!dev) { ipc_user->result = @@ -543,19 +537,12 @@ static void ipc_user_thread_fn(void *p1, void *p2, void *p3) break; } - /* Restore original kernel driver pointer. - * comp_init() set dev->drv to the copy; - * runtime code expects the canonical - * kernel address. - */ - dev->drv = orig_drv; - - ipc_user->result = - ipc4_add_comp_dev(dev); + ipc_user->result = ipc4_add_comp_dev(dev); if (ipc_user->result != IPC4_SUCCESS) break; comp_update_ibs_obs_cpc(dev); + ipc_user->result = 0; break; } diff --git a/src/ipc/ipc4/handler-user.c b/src/ipc/ipc4/handler-user.c index 3d9bbaf0cbb8..a275777ce15a 100644 --- a/src/ipc/ipc4/handler-user.c +++ b/src/ipc/ipc4/handler-user.c @@ -1530,10 +1530,6 @@ __cold int ipc4_user_process_module_message(struct ipc4_message_request *ipc4, */ struct ipc4_module_init_instance mi; - BUILD_ASSERT(sizeof(struct comp_driver) + sizeof(struct tr_ctx) <= - sizeof(((struct ipc_user *)0)->init_drv_data), - "ipc_user.init_drv_data too small for driver copy"); - memcpy_s(&mi, sizeof(mi), ipc4, sizeof(*ipc4)); if (!cpu_is_me(mi.extension.r.core_id)) { ret = ipc4_init_module_instance(ipc4); @@ -1541,32 +1537,16 @@ __cold int ipc4_user_process_module_message(struct ipc4_message_request *ipc4, struct ipc *ipc = ipc_get(); uint32_t comp_id = IPC4_COMP_ID(mi.primary.r.module_id, mi.primary.r.instance_id); - const struct comp_driver *drv = ipc4_get_comp_drv( - IPC4_MOD_ID(comp_id)); + const struct comp_driver *drv = ipc4_get_comp_drv(IPC4_MOD_ID(comp_id)); if (!drv) { ret = IPC4_MOD_NOT_INITIALIZED; } else { struct ipc_user *pdata = ipc->ipc_user_pdata; - /* Copy comp_driver and tr_ctx into - * user-accessible ipc_user buffer — - * originals are in kernel .rodata/.data - * and not readable from user mode. - */ - struct comp_driver *drv_copy = - (struct comp_driver *)pdata->init_drv_data; - struct tr_ctx *tctx_copy = - (struct tr_ctx *)(pdata->init_drv_data + - sizeof(struct comp_driver)); - - memcpy_s(drv_copy, sizeof(*drv_copy), - drv, sizeof(*drv)); - if (drv->tctx) { - memcpy_s(tctx_copy, sizeof(*tctx_copy), - drv->tctx, sizeof(*drv->tctx)); - drv_copy->tctx = tctx_copy; - } + ret = llext_manager_map_lib(comp_id); + if (ret < 0) + break; pdata->init_drv = drv; ret = ipc_user_forward_cmd(ipc4); From 5340283188c2d08918c3de6d0835c63f269ab7c4 Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Fri, 22 May 2026 16:14:08 +0200 Subject: [PATCH 062/108] schedule: ll: userspace: enable LLEXT LLEXT is now working with userspace LL and can be enabled. Signed-off-by: Guennadi Liakhovetski --- app/overlays/ptl/ll_userspace_overlay.conf | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/app/overlays/ptl/ll_userspace_overlay.conf b/app/overlays/ptl/ll_userspace_overlay.conf index 8888a66f4c24..057e5f1a9dbb 100644 --- a/app/overlays/ptl/ll_userspace_overlay.conf +++ b/app/overlays/ptl/ll_userspace_overlay.conf @@ -24,11 +24,6 @@ CONFIG_COLD_STORE_EXECUTE_DEBUG=n CONFIG_SOF_TELEMETRY_PERFORMANCE_MEASUREMENTS=n CONFIG_SOF_TELEMETRY_IO_PERFORMANCE_MEASUREMENTS=n -# disable loadable modules (hits privilege issues in user-space now) -CONFIG_LLEXT_STORAGE_WRITABLE=n -CONFIG_LLEXT_EXPERIMENTAL=n -CONFIG_MODULES=n - # some of current boot tests interfere with user-space setup CONFIG_SOF_BOOT_TEST_ALLOWED=n @@ -36,3 +31,5 @@ CONFIG_SOF_BOOT_TEST_ALLOWED=n CONFIG_CROSS_CORE_STREAM=n CONFIG_INTEL_ADSP_MIC_PRIVACY=n CONFIG_XRUN_NOTIFICATIONS_ENABLE=n + +CONFIG_MAX_THREAD_BYTES=4 From f1af4318acbf7f86fc8af45e2af7ae08da6d493e Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Wed, 10 Jun 2026 11:55:19 +0200 Subject: [PATCH 063/108] audio: pipeline: remove unneeded warning "Trace context" isn't used any more, no need to warn about it. Signed-off-by: Guennadi Liakhovetski --- src/audio/pipeline/pipeline-graph.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/audio/pipeline/pipeline-graph.c b/src/audio/pipeline/pipeline-graph.c index 1eff4ebc58e1..4a3c2047731e 100644 --- a/src/audio/pipeline/pipeline-graph.c +++ b/src/audio/pipeline/pipeline-graph.c @@ -186,9 +186,7 @@ struct pipeline *pipeline_new(struct k_heap *heap, uint32_t pipeline_id, uint32_ p->status = COMP_STATE_INIT; p->trigger.cmd = COMP_TRIGGER_NO_ACTION; -#ifdef CONFIG_SOF_USERSPACE_LL - LOG_WRN("pipeline trace settings cannot be copied"); -#else +#ifndef CONFIG_SOF_USERSPACE_LL ret = memcpy_s(&p->tctx, sizeof(struct tr_ctx), &pipe_tr, sizeof(struct tr_ctx)); if (ret < 0) { From e25919c019d6bdbbb5ea2dbc660dc6012f20292c Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Wed, 10 Jun 2026 14:48:50 +0200 Subject: [PATCH 064/108] ipc: eliminate multiple memcpy() calls Eliminate multiple instances of IPC4 data copying, use simple type- casts instead. This removes stack objects and replaces run-time copying with compile-time pointer substitution. Signed-off-by: Guennadi Liakhovetski --- src/include/sof/audio/component_ext.h | 2 +- src/include/sof/ipc/topology.h | 4 +- src/ipc/ipc4/handler-user.c | 245 +++++++++++--------------- src/ipc/ipc4/helper.c | 6 +- 4 files changed, 112 insertions(+), 145 deletions(-) diff --git a/src/include/sof/audio/component_ext.h b/src/include/sof/audio/component_ext.h index 8532664488df..cb114add7f94 100644 --- a/src/include/sof/audio/component_ext.h +++ b/src/include/sof/audio/component_ext.h @@ -37,7 +37,7 @@ struct comp_driver_list { #if CONFIG_IPC_MAJOR_3 struct comp_dev *comp_new(struct sof_ipc_comp *comp); #elif CONFIG_IPC_MAJOR_4 -struct comp_dev *comp_new_ipc4(struct ipc4_module_init_instance *module_init); +struct comp_dev *comp_new_ipc4(const struct ipc4_module_init_instance *module_init); #endif /** See comp_ops::free */ diff --git a/src/include/sof/ipc/topology.h b/src/include/sof/ipc/topology.h index a76c299aa60e..ceabc7006203 100644 --- a/src/include/sof/ipc/topology.h +++ b/src/include/sof/ipc/topology.h @@ -56,8 +56,8 @@ struct comp_driver; struct comp_dev *comp_new_ipc4_user(struct ipc4_message_request *ipc4, const struct comp_driver *drv); #endif -int ipc4_chain_manager_create(struct ipc4_chain_dma *cdma); -int ipc4_chain_dma_state(struct comp_dev *dev, struct ipc4_chain_dma *cdma); +int ipc4_chain_manager_create(const struct ipc4_chain_dma *cdma); +int ipc4_chain_dma_state(struct comp_dev *dev, const struct ipc4_chain_dma *cdma); int ipc4_create_chain_dma(struct ipc *ipc, struct ipc4_chain_dma *cdma); int ipc4_trigger_chain_dma(struct ipc *ipc, struct ipc4_chain_dma *cdma, bool *delay); int ipc4_process_on_core(uint32_t core, bool blocking); diff --git a/src/ipc/ipc4/handler-user.c b/src/ipc/ipc4/handler-user.c index a275777ce15a..16ee823c1adb 100644 --- a/src/ipc/ipc4/handler-user.c +++ b/src/ipc/ipc4/handler-user.c @@ -583,15 +583,11 @@ __cold static int ipc4_process_chain_dma(struct ipc4_message_request *ipc4) #if CONFIG_COMP_CHAIN_DMA struct ipc_comp_dev *cdma_comp; struct ipc *ipc = ipc_get(); - struct ipc4_chain_dma cdma; + const struct ipc4_chain_dma *cdma = (const struct ipc4_chain_dma *)ipc4; int comp_id; int ret; - ret = memcpy_s(&cdma, sizeof(cdma), ipc4, sizeof(*ipc4)); - if (ret < 0) - return IPC4_FAILURE; - - comp_id = IPC4_COMP_ID(cdma.primary.r.host_dma_id + IPC4_MAX_MODULE_COUNT, 0); + comp_id = IPC4_COMP_ID(cdma->primary.r.host_dma_id + IPC4_MAX_MODULE_COUNT, 0); cdma_comp = ipc_get_comp_by_id(ipc, comp_id); if (!cdma_comp) { @@ -599,10 +595,10 @@ __cold static int ipc4_process_chain_dma(struct ipc4_message_request *ipc4) * Nothing to do when the chainDMA is not allocated and asked to * be freed */ - if (!cdma.primary.r.allocate && !cdma.primary.r.enable) + if (!cdma->primary.r.allocate && !cdma->primary.r.enable) return IPC4_SUCCESS; - ret = ipc4_chain_manager_create(&cdma); + ret = ipc4_chain_manager_create(cdma); if (ret < 0) return IPC4_FAILURE; @@ -611,7 +607,7 @@ __cold static int ipc4_process_chain_dma(struct ipc4_message_request *ipc4) return IPC4_FAILURE; } - ret = ipc4_chain_dma_state(cdma_comp->cd, &cdma); + ret = ipc4_chain_dma_state(cdma_comp->cd, cdma); if (ret < 0) { comp_free(cdma_comp->cd); list_item_del(&cdma_comp->list); @@ -622,7 +618,7 @@ __cold static int ipc4_process_chain_dma(struct ipc4_message_request *ipc4) return IPC4_SUCCESS; } - ret = ipc4_chain_dma_state(cdma_comp->cd, &cdma); + ret = ipc4_chain_dma_state(cdma_comp->cd, cdma); if (ret < 0) return IPC4_INVALID_CHAIN_STATE_TRANSITION; @@ -761,31 +757,25 @@ int ipc4_user_process_glb_message(struct ipc4_message_request *ipc4, __cold static int ipc4_init_module_instance(struct ipc4_message_request *ipc4) { - struct ipc4_module_init_instance module_init; + const struct ipc4_module_init_instance *module_init = + (const struct ipc4_module_init_instance *)ipc4; struct comp_dev *dev; assert_can_be_cold(); - /* we only need the common header here, all we have from the IPC */ - int ret = memcpy_s(&module_init, sizeof(module_init), ipc4, sizeof(*ipc4)); - - if (ret < 0) - return IPC4_FAILURE; - - tr_dbg(&ipc_tr, - "%x : %x", - (uint32_t)module_init.primary.r.module_id, - (uint32_t)module_init.primary.r.instance_id); + tr_dbg(&ipc_tr, "%x : %x", + (uint32_t)module_init->primary.r.module_id, + (uint32_t)module_init->primary.r.instance_id); /* Pass IPC to target core */ - if (!cpu_is_me(module_init.extension.r.core_id)) - return ipc4_process_on_core(module_init.extension.r.core_id, false); + if (!cpu_is_me(module_init->extension.r.core_id)) + return ipc4_process_on_core(module_init->extension.r.core_id, false); - dev = comp_new_ipc4(&module_init); + dev = comp_new_ipc4(module_init); if (!dev) { ipc_cmd_err(&ipc_tr, "error: failed to init module %x : %x", - (uint32_t)module_init.primary.r.module_id, - (uint32_t)module_init.primary.r.instance_id); + (uint32_t)module_init->primary.r.module_id, + (uint32_t)module_init->primary.r.instance_id); return IPC4_MOD_NOT_INITIALIZED; } @@ -794,40 +784,30 @@ __cold static int ipc4_init_module_instance(struct ipc4_message_request *ipc4) __cold static int ipc4_bind_module_instance(struct ipc4_message_request *ipc4) { - struct ipc4_module_bind_unbind bu; + struct ipc4_module_bind_unbind *bu = (struct ipc4_module_bind_unbind *)ipc4; struct ipc *ipc = ipc_get(); assert_can_be_cold(); - int ret = memcpy_s(&bu, sizeof(bu), ipc4, sizeof(*ipc4)); - - if (ret < 0) - return IPC4_FAILURE; - tr_dbg(&ipc_tr, "%x : %x with %x : %x", - (uint32_t)bu.primary.r.module_id, (uint32_t)bu.primary.r.instance_id, - (uint32_t)bu.extension.r.dst_module_id, (uint32_t)bu.extension.r.dst_instance_id); + (uint32_t)bu->primary.r.module_id, (uint32_t)bu->primary.r.instance_id, + (uint32_t)bu->extension.r.dst_module_id, (uint32_t)bu->extension.r.dst_instance_id); - return ipc_comp_connect(ipc, (ipc_pipe_comp_connect *)&bu); + return ipc_comp_connect(ipc, (ipc_pipe_comp_connect *)bu); } __cold static int ipc4_unbind_module_instance(struct ipc4_message_request *ipc4) { - struct ipc4_module_bind_unbind bu; + struct ipc4_module_bind_unbind *bu = (struct ipc4_module_bind_unbind *)ipc4; struct ipc *ipc = ipc_get(); assert_can_be_cold(); - int ret = memcpy_s(&bu, sizeof(bu), ipc4, sizeof(*ipc4)); - - if (ret < 0) - return IPC4_FAILURE; - tr_dbg(&ipc_tr, "%x : %x with %x : %x", - (uint32_t)bu.primary.r.module_id, (uint32_t)bu.primary.r.instance_id, - (uint32_t)bu.extension.r.dst_module_id, (uint32_t)bu.extension.r.dst_instance_id); + (uint32_t)bu->primary.r.module_id, (uint32_t)bu->primary.r.instance_id, + (uint32_t)bu->extension.r.dst_module_id, (uint32_t)bu->extension.r.dst_instance_id); - return ipc_comp_disconnect(ipc, (ipc_pipe_comp_connect *)&bu); + return ipc_comp_disconnect(ipc, (ipc_pipe_comp_connect *)bu); } /** @@ -1045,30 +1025,27 @@ __cold int ipc4_process_large_config_get(struct ipc4_message_request *ipc4, void **reply_tx_data) { struct ipc4_module_large_config_reply reply; - struct ipc4_module_large_config config; + const struct ipc4_module_large_config *config = + (const struct ipc4_module_large_config *)ipc4; char *data = ipc_get()->comp_data; const struct comp_driver *drv; struct comp_dev *dev = NULL; uint32_t data_offset; + int ret; assert_can_be_cold(); - int ret = memcpy_s(&config, sizeof(config), ipc4, sizeof(*ipc4)); - - if (ret < 0) - return IPC4_FAILURE; - tr_dbg(&ipc_tr, "%x : %x", - (uint32_t)config.primary.r.module_id, (uint32_t)config.primary.r.instance_id); + (uint32_t)config->primary.r.module_id, (uint32_t)config->primary.r.instance_id); /* get component dev for non-basefw since there is no * component dev for basefw */ - if (config.primary.r.module_id) { + if (config->primary.r.module_id) { uint32_t comp_id; - comp_id = IPC4_COMP_ID(config.primary.r.module_id, - config.primary.r.instance_id); + comp_id = IPC4_COMP_ID(config->primary.r.module_id, + config->primary.r.instance_id); dev = ipc4_get_comp_dev(comp_id); if (!dev) return IPC4_MOD_INVALID_ID; @@ -1079,7 +1056,7 @@ __cold int ipc4_process_large_config_get(struct ipc4_message_request *ipc4, * non-userspace path retains ipc4_process_on_core() */ } else { - drv = ipc4_get_comp_drv(config.primary.r.module_id); + drv = ipc4_get_comp_drv(config->primary.r.module_id); } if (!drv) @@ -1088,16 +1065,16 @@ __cold int ipc4_process_large_config_get(struct ipc4_message_request *ipc4, if (!drv->ops.get_large_config) return IPC4_INVALID_REQUEST; - data_offset = config.extension.r.data_off_size; + data_offset = config->extension.r.data_off_size; /* check for vendor param first */ - if (config.extension.r.large_param_id == VENDOR_CONFIG_PARAM) { + if (config->extension.r.large_param_id == VENDOR_CONFIG_PARAM) { /* For now only vendor_config case uses payload from hostbox */ dcache_invalidate_region((__sparse_force void __sparse_cache *)MAILBOX_HOSTBOX_BASE, - config.extension.r.data_off_size); + config->extension.r.data_off_size); ret = ipc4_get_vendor_config_module_instance(dev, drv, - config.extension.r.init_block, - config.extension.r.final_block, + config->extension.r.init_block, + config->extension.r.final_block, &data_offset, data, (const char *)MAILBOX_HOSTBOX_BASE); @@ -1105,12 +1082,12 @@ __cold int ipc4_process_large_config_get(struct ipc4_message_request *ipc4, #if CONFIG_LIBRARY data += sizeof(reply); #endif - ipc4_prepare_for_kcontrol_get(dev, config.extension.r.large_param_id, + ipc4_prepare_for_kcontrol_get(dev, config->extension.r.large_param_id, data, data_offset); - ret = drv->ops.get_large_config(dev, config.extension.r.large_param_id, - config.extension.r.init_block, - config.extension.r.final_block, + ret = drv->ops.get_large_config(dev, config->extension.r.large_param_id, + config->extension.r.init_block, + config->extension.r.final_block, &data_offset, data); } @@ -1119,11 +1096,11 @@ __cold int ipc4_process_large_config_get(struct ipc4_message_request *ipc4, ret = IPC4_MOD_INVALID_ID; /* Copy host config and overwrite */ - reply.extension.dat = config.extension.dat; + reply.extension.dat = config->extension.dat; reply.extension.r.data_off_size = data_offset; /* The last block, no more data */ - if (!config.extension.r.final_block && data_offset < SOF_IPC_MSG_MAX_SIZE) + if (!config->extension.r.final_block && data_offset < SOF_IPC_MSG_MAX_SIZE) reply.extension.r.final_block = 1; /* Indicate last block if error occurs */ @@ -1144,30 +1121,27 @@ __cold int ipc4_process_large_config_get(struct ipc4_message_request *ipc4, __cold static int ipc4_get_large_config_module_instance(struct ipc4_message_request *ipc4) { struct ipc4_module_large_config_reply reply; - struct ipc4_module_large_config config; + const struct ipc4_module_large_config *config = + (const struct ipc4_module_large_config *)ipc4; char *data = ipc_get()->comp_data; const struct comp_driver *drv; struct comp_dev *dev = NULL; uint32_t data_offset; + int ret; assert_can_be_cold(); - int ret = memcpy_s(&config, sizeof(config), ipc4, sizeof(*ipc4)); - - if (ret < 0) - return IPC4_FAILURE; - tr_dbg(&ipc_tr, "%x : %x", - (uint32_t)config.primary.r.module_id, (uint32_t)config.primary.r.instance_id); + (uint32_t)config->primary.r.module_id, (uint32_t)config->primary.r.instance_id); /* get component dev for non-basefw since there is no * component dev for basefw */ - if (config.primary.r.module_id) { + if (config->primary.r.module_id) { uint32_t comp_id; - comp_id = IPC4_COMP_ID(config.primary.r.module_id, - config.primary.r.instance_id); + comp_id = IPC4_COMP_ID(config->primary.r.module_id, + config->primary.r.instance_id); dev = ipc4_get_comp_dev(comp_id); if (!dev) return IPC4_INVALID_RESOURCE_ID; @@ -1179,10 +1153,10 @@ __cold static int ipc4_get_large_config_module_instance(struct ipc4_message_requ return ipc4_process_on_core(dev->ipc_config.core, false); } else { /* BaseFW module has only 0th instance */ - if (config.primary.r.instance_id) + if (config->primary.r.instance_id) return IPC4_INVALID_RESOURCE_ID; - drv = ipc4_get_comp_drv(config.primary.r.module_id); + drv = ipc4_get_comp_drv(config->primary.r.module_id); } if (!drv) @@ -1191,10 +1165,10 @@ __cold static int ipc4_get_large_config_module_instance(struct ipc4_message_requ if (!drv->ops.get_large_config) return IPC4_INVALID_REQUEST; - data_offset = config.extension.r.data_off_size; + data_offset = config->extension.r.data_off_size; /* check for vendor param first */ - if (config.extension.r.large_param_id == VENDOR_CONFIG_PARAM) { + if (config->extension.r.large_param_id == VENDOR_CONFIG_PARAM) { /* data_off_size is a 20-bit host-controlled field, so it can * claim far more than the hostbox can physically hold. */ @@ -1205,23 +1179,22 @@ __cold static int ipc4_get_large_config_module_instance(struct ipc4_message_requ } /* For now only vendor_config case uses payload from hostbox */ dcache_invalidate_region((__sparse_force void __sparse_cache *)MAILBOX_HOSTBOX_BASE, - config.extension.r.data_off_size); + config->extension.r.data_off_size); ret = ipc4_get_vendor_config_module_instance(dev, drv, - config.extension.r.init_block, - config.extension.r.final_block, - &data_offset, - data, + config->extension.r.init_block, + config->extension.r.final_block, + &data_offset, data, (const char *)MAILBOX_HOSTBOX_BASE); } else { #if CONFIG_LIBRARY data += sizeof(reply); #endif - ipc4_prepare_for_kcontrol_get(dev, config.extension.r.large_param_id, + ipc4_prepare_for_kcontrol_get(dev, config->extension.r.large_param_id, data, data_offset); - ret = drv->ops.get_large_config(dev, config.extension.r.large_param_id, - config.extension.r.init_block, - config.extension.r.final_block, + ret = drv->ops.get_large_config(dev, config->extension.r.large_param_id, + config->extension.r.init_block, + config->extension.r.final_block, &data_offset, data); } @@ -1230,11 +1203,11 @@ __cold static int ipc4_get_large_config_module_instance(struct ipc4_message_requ ret = IPC4_INVALID_RESOURCE_ID; /* Copy host config and overwrite */ - reply.extension.dat = config.extension.dat; + reply.extension.dat = config->extension.dat; reply.extension.r.data_off_size = data_offset; /* The last block, no more data */ - if (!config.extension.r.final_block && data_offset < SOF_IPC_MSG_MAX_SIZE) + if (!config->extension.r.final_block && data_offset < SOF_IPC_MSG_MAX_SIZE) reply.extension.r.final_block = 1; /* Indicate last block if error occurs */ @@ -1335,26 +1308,23 @@ __cold static int ipc4_set_vendor_config_module_instance(struct comp_dev *dev, __cold int ipc4_process_large_config_set(struct ipc4_message_request *ipc4) { - struct ipc4_module_large_config config; + const struct ipc4_module_large_config *config = + (const struct ipc4_module_large_config *)ipc4; struct comp_dev *dev = NULL; const struct comp_driver *drv; + int ret; assert_can_be_cold(); - int ret = memcpy_s(&config, sizeof(config), ipc4, sizeof(*ipc4)); - - if (ret < 0) - return IPC4_FAILURE; - dcache_invalidate_region((__sparse_force void __sparse_cache *)MAILBOX_HOSTBOX_BASE, - config.extension.r.data_off_size); + config->extension.r.data_off_size); tr_dbg(&ipc_tr, "%x : %x", - (uint32_t)config.primary.r.module_id, (uint32_t)config.primary.r.instance_id); + (uint32_t)config->primary.r.module_id, (uint32_t)config->primary.r.instance_id); - if (config.primary.r.module_id) { + if (config->primary.r.module_id) { uint32_t comp_id; - comp_id = IPC4_COMP_ID(config.primary.r.module_id, config.primary.r.instance_id); + comp_id = IPC4_COMP_ID(config->primary.r.module_id, config->primary.r.instance_id); dev = ipc4_get_comp_dev(comp_id); if (!dev) return IPC4_MOD_INVALID_ID; @@ -1365,7 +1335,7 @@ __cold int ipc4_process_large_config_set(struct ipc4_message_request *ipc4) * non-userspace path retains ipc4_process_on_core() */ } else { - drv = ipc4_get_comp_drv(config.primary.r.module_id); + drv = ipc4_get_comp_drv(config->primary.r.module_id); } if (!drv) @@ -1375,13 +1345,13 @@ __cold int ipc4_process_large_config_set(struct ipc4_message_request *ipc4) return IPC4_INVALID_REQUEST; /* check for vendor param first */ - if (config.extension.r.large_param_id == VENDOR_CONFIG_PARAM) { + if (config->extension.r.large_param_id == VENDOR_CONFIG_PARAM) { ret = ipc4_set_vendor_config_module_instance(dev, drv, - (uint32_t)config.primary.r.module_id, - (uint32_t)config.primary.r.instance_id, - config.extension.r.init_block, - config.extension.r.final_block, - config.extension.r.data_off_size, + (uint32_t)config->primary.r.module_id, + (uint32_t)config->primary.r.instance_id, + config->extension.r.init_block, + config->extension.r.final_block, + config->extension.r.data_off_size, (const char *)MAILBOX_HOSTBOX_BASE); } else { #if CONFIG_LIBRARY @@ -1390,13 +1360,13 @@ __cold int ipc4_process_large_config_set(struct ipc4_message_request *ipc4) #else const char *data = (const char *)MAILBOX_HOSTBOX_BASE; #endif - ret = drv->ops.set_large_config(dev, config.extension.r.large_param_id, - config.extension.r.init_block, config.extension.r.final_block, - config.extension.r.data_off_size, data); + ret = drv->ops.set_large_config(dev, config->extension.r.large_param_id, + config->extension.r.init_block, config->extension.r.final_block, + config->extension.r.data_off_size, data); if (ret < 0) { ipc_cmd_err(&ipc_tr, "failed to set large_config_module_instance %x : %x", - (uint32_t)config.primary.r.module_id, - (uint32_t)config.primary.r.instance_id); + (uint32_t)config->primary.r.module_id, + (uint32_t)config->primary.r.instance_id); ret = IPC4_INVALID_RESOURCE_ID; } } @@ -1406,26 +1376,23 @@ __cold int ipc4_process_large_config_set(struct ipc4_message_request *ipc4) __cold static int ipc4_set_large_config_module_instance(struct ipc4_message_request *ipc4) { - struct ipc4_module_large_config config; + const struct ipc4_module_large_config *config = + (const struct ipc4_module_large_config *)ipc4; struct comp_dev *dev = NULL; const struct comp_driver *drv; + int ret; assert_can_be_cold(); - int ret = memcpy_s(&config, sizeof(config), ipc4, sizeof(*ipc4)); - - if (ret < 0) - return IPC4_FAILURE; - dcache_invalidate_region((__sparse_force void __sparse_cache *)MAILBOX_HOSTBOX_BASE, - config.extension.r.data_off_size); + config->extension.r.data_off_size); tr_dbg(&ipc_tr, "%x : %x", - (uint32_t)config.primary.r.module_id, (uint32_t)config.primary.r.instance_id); + (uint32_t)config->primary.r.module_id, (uint32_t)config->primary.r.instance_id); - if (config.primary.r.module_id) { + if (config->primary.r.module_id) { uint32_t comp_id; - comp_id = IPC4_COMP_ID(config.primary.r.module_id, config.primary.r.instance_id); + comp_id = IPC4_COMP_ID(config->primary.r.module_id, config->primary.r.instance_id); dev = ipc4_get_comp_dev(comp_id); if (!dev) return IPC4_INVALID_RESOURCE_ID; @@ -1437,10 +1404,10 @@ __cold static int ipc4_set_large_config_module_instance(struct ipc4_message_requ return ipc4_process_on_core(dev->ipc_config.core, false); } else { /* BaseFW module has only 0th instance */ - if (config.primary.r.instance_id) + if (config->primary.r.instance_id) return IPC4_INVALID_RESOURCE_ID; - drv = ipc4_get_comp_drv(config.primary.r.module_id); + drv = ipc4_get_comp_drv(config->primary.r.module_id); } if (!drv) @@ -1450,13 +1417,13 @@ __cold static int ipc4_set_large_config_module_instance(struct ipc4_message_requ return IPC4_INVALID_REQUEST; /* check for vendor param first */ - if (config.extension.r.large_param_id == VENDOR_CONFIG_PARAM) { + if (config->extension.r.large_param_id == VENDOR_CONFIG_PARAM) { ret = ipc4_set_vendor_config_module_instance(dev, drv, - (uint32_t)config.primary.r.module_id, - (uint32_t)config.primary.r.instance_id, - config.extension.r.init_block, - config.extension.r.final_block, - config.extension.r.data_off_size, + (uint32_t)config->primary.r.module_id, + (uint32_t)config->primary.r.instance_id, + config->extension.r.init_block, + config->extension.r.final_block, + config->extension.r.data_off_size, (const char *)MAILBOX_HOSTBOX_BASE); } else { #if CONFIG_LIBRARY @@ -1465,13 +1432,13 @@ __cold static int ipc4_set_large_config_module_instance(struct ipc4_message_requ #else const char *data = (const char *)MAILBOX_HOSTBOX_BASE; #endif - ret = drv->ops.set_large_config(dev, config.extension.r.large_param_id, - config.extension.r.init_block, config.extension.r.final_block, - config.extension.r.data_off_size, data); + ret = drv->ops.set_large_config(dev, config->extension.r.large_param_id, + config->extension.r.init_block, config->extension.r.final_block, + config->extension.r.data_off_size, data); if (ret < 0) { ipc_cmd_err(&ipc_tr, "failed to set large_config_module_instance %x : %x", - (uint32_t)config.primary.r.module_id, - (uint32_t)config.primary.r.instance_id); + (uint32_t)config->primary.r.module_id, + (uint32_t)config->primary.r.instance_id); ret = IPC4_INVALID_RESOURCE_ID; } } @@ -1582,10 +1549,10 @@ __cold int ipc4_user_process_module_message(struct ipc4_message_request *ipc4, case SOF_IPC4_MOD_LARGE_CONFIG_GET: #ifdef CONFIG_SOF_USERSPACE_LL { - struct ipc4_module_large_config config; + const struct ipc4_module_large_config *config = + (const struct ipc4_module_large_config *)ipc4; - memcpy_s(&config, sizeof(config), ipc4, sizeof(*ipc4)); - if (config.primary.r.module_id) { + if (config->primary.r.module_id) { /* Module case: forward to user thread */ ret = ipc_user_forward_cmd(ipc4); if (!ret) { diff --git a/src/ipc/ipc4/helper.c b/src/ipc/ipc4/helper.c index 06461672baa8..b12ab9b228c5 100644 --- a/src/ipc/ipc4/helper.c +++ b/src/ipc/ipc4/helper.c @@ -110,7 +110,7 @@ __cold static inline char *ipc4_get_comp_new_data(void) #endif /* Only called from ipc4_init_module_instance(), which is __cold */ -__cold struct comp_dev *comp_new_ipc4(struct ipc4_module_init_instance *module_init) +__cold struct comp_dev *comp_new_ipc4(const struct ipc4_module_init_instance *module_init) { struct comp_ipc_config ipc_config; const struct comp_driver *drv; @@ -1134,7 +1134,7 @@ __cold int ipc_comp_disconnect(struct ipc *ipc, ipc_pipe_comp_connect *_connect) #if CONFIG_COMP_CHAIN_DMA /* Only called from ipc4_process_chain_dma(), which is __cold */ -__cold int ipc4_chain_manager_create(struct ipc4_chain_dma *cdma) +__cold int ipc4_chain_manager_create(const struct ipc4_chain_dma *cdma) { const struct sof_uuid uuid = SOF_REG_UUID(chain_dma); const struct comp_driver *drv; @@ -1161,7 +1161,7 @@ __cold int ipc4_chain_manager_create(struct ipc4_chain_dma *cdma) } /* Only called from ipc4_process_chain_dma(), which is __cold */ -__cold int ipc4_chain_dma_state(struct comp_dev *dev, struct ipc4_chain_dma *cdma) +__cold int ipc4_chain_dma_state(struct comp_dev *dev, const struct ipc4_chain_dma *cdma) { const bool allocate = cdma->primary.r.allocate; const bool enable = cdma->primary.r.enable; From ffb4fc6d494fcb950cb537a5d341e1bc7e0e347d Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Wed, 10 Jun 2026 15:57:14 +0200 Subject: [PATCH 065/108] ipc: allocate userspace IPC thread dynamically Prepare for multi-core support: allocate the IPC thread dynamically and extract thread initialisation into a separate function. Signed-off-by: Guennadi Liakhovetski --- src/ipc/ipc-common.c | 79 ++++++++++++++++++++++++++++++-------------- 1 file changed, 55 insertions(+), 24 deletions(-) diff --git a/src/ipc/ipc-common.c b/src/ipc/ipc-common.c index cb91f957c78e..2db7a9aea31b 100644 --- a/src/ipc/ipc-common.c +++ b/src/ipc/ipc-common.c @@ -394,7 +394,6 @@ void ipc_schedule_process(struct ipc *ipc) #define IPC_USER_EVENT_CMD BIT(0) #define IPC_USER_EVENT_STOP BIT(1) -static struct k_thread ipc_user_thread; static K_THREAD_STACK_DEFINE(ipc_user_stack, CONFIG_SOF_IPC_USER_THREAD_STACK_SIZE); /** @@ -617,6 +616,55 @@ static void ipc_user_thread_fn(void *p1, void *p2, void *p3) } } +__cold static int ipc_user_init_thread(struct ipc_user *ipc_user) +{ + char thread_name[] = "ll_user0"; + int ret; + + assert_can_be_cold(); + + /* Allocate kernel objects for the user-space thread */ + ipc_user->event = k_object_alloc(K_OBJ_EVENT); + if (!ipc_user->event) { + LOG_ERR("user IPC event alloc failed"); + return -ENOMEM; + } + k_event_init(ipc_user->event); + + ipc_user->thread = k_object_alloc(K_OBJ_THREAD); + if (!ipc_user->thread) { + LOG_ERR("user IPC thread alloc failed"); + ret = -ENOMEM; + goto e_event; + } + + k_thread_create(ipc_user->thread, ipc_user_stack, + CONFIG_SOF_IPC_USER_THREAD_STACK_SIZE, + ipc_user_thread_fn, ipc_user, NULL, NULL, + -1, K_USER, K_FOREVER); + + k_thread_cpu_pin(ipc_user->thread, PLATFORM_PRIMARY_CORE_ID); + k_thread_name_set(ipc_user->thread, thread_name); + + /* + * Each userspace IPC thread must be able to wait on its private event + * and signal completion on the primary core semaphore + */ + k_thread_access_grant(ipc_user->thread, ipc_user->sem, ipc_user->event); + user_grant_dai_access_all(ipc_user->thread); + user_grant_dma_access_all(ipc_user->thread); + pipeline_posn_grant_access(ipc_user->thread); + k_mem_domain_add_thread(zephyr_ll_mem_domain(), ipc_user->thread); + user_ll_grant_access(ipc_user->thread, PLATFORM_PRIMARY_CORE_ID); + + return 0; + +e_event: + k_object_free(ipc_user->event); + + return ret; +} + __cold int ipc_user_init(void) { struct ipc *ipc = ipc_get(); @@ -671,35 +719,18 @@ __cold int ipc_user_init(void) k_sem_init(ipc_user->sem, 0, 1); - /* Allocate kernel objects for the user-space thread */ - ipc_user->event = k_object_alloc(K_OBJ_EVENT); - if (!ipc_user->event) { - LOG_ERR("user IPC event alloc failed"); + ret = ipc_user_init_thread(ipc_user); + if (ret < 0) { + LOG_ERR("user IPC thread initialization failed"); k_panic(); } - k_event_init(ipc_user->event); - - k_thread_create(&ipc_user_thread, ipc_user_stack, - CONFIG_SOF_IPC_USER_THREAD_STACK_SIZE, - ipc_user_thread_fn, ipc_user, NULL, NULL, - -1, K_USER, K_FOREVER); - - ipc_user->thread = &ipc_user_thread; - k_thread_access_grant(&ipc_user_thread, ipc_user->sem, ipc_user->event); - user_grant_dai_access_all(&ipc_user_thread); - user_grant_dma_access_all(&ipc_user_thread); - user_access_to_mailbox(zephyr_ll_mem_domain(), &ipc_user_thread); - user_ll_grant_access(&ipc_user_thread, PLATFORM_PRIMARY_CORE_ID); - pipeline_posn_grant_access(&ipc_user_thread); - k_mem_domain_add_thread(zephyr_ll_mem_domain(), &ipc_user_thread); - k_thread_cpu_pin(&ipc_user_thread, PLATFORM_PRIMARY_CORE_ID); - k_thread_name_set(&ipc_user_thread, "ipc_user"); + user_access_to_mailbox(zephyr_ll_mem_domain(), ipc_user->thread); /* Store references in ipc struct so kernel handler can forward commands */ ipc->ipc_user_pdata = ipc_user; - k_thread_start(&ipc_user_thread); + k_thread_start(ipc_user->thread); struct task *task = zephyr_ll_task_alloc(); schedule_task_init_ll(task, SOF_UUID(ipc_uuid), SOF_SCHEDULE_LL_TIMER, @@ -710,7 +741,7 @@ __cold int ipc_user_init(void) * Needed so user-space dai_common_new() can call * k_thread_access_grant(audio_thread, dai_mutex) from user context. */ - k_thread_access_grant(&ipc_user_thread, ipc_user->audio_thread); + k_thread_access_grant(ipc_user->thread, ipc_user->audio_thread); /* Wait for user thread startup — consumes the initial k_sem_give from thread */ k_sem_take(ipc->ipc_user_pdata->sem, K_FOREVER); From a43a7ada28a73a89751cdb15e934b17244a6f6ba Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Wed, 10 Jun 2026 16:03:36 +0200 Subject: [PATCH 066/108] ipc: allocate userspace context uncached Userspace IPC context is global, allocate it uncached. Signed-off-by: Guennadi Liakhovetski --- src/ipc/ipc-common.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/ipc/ipc-common.c b/src/ipc/ipc-common.c index 2db7a9aea31b..b30749eb1e54 100644 --- a/src/ipc/ipc-common.c +++ b/src/ipc/ipc-common.c @@ -668,7 +668,8 @@ __cold static int ipc_user_init_thread(struct ipc_user *ipc_user) __cold int ipc_user_init(void) { struct ipc *ipc = ipc_get(); - struct ipc_user *ipc_user = sof_heap_alloc(sof_sys_user_heap_get(), SOF_MEM_FLAG_USER, + struct ipc_user *ipc_user = sof_heap_alloc(sof_sys_user_heap_get(), + SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, sizeof(*ipc_user), 0); int ret; From 2eee873b8f49b485afe3280358976974a8bd8f6c Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Thu, 18 Jun 2026 17:26:55 +0200 Subject: [PATCH 067/108] schedule: ll: use correct core Use current core when calling scheduler_get_data_for_core(). Signed-off-by: Guennadi Liakhovetski --- src/schedule/zephyr_ll.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/schedule/zephyr_ll.c b/src/schedule/zephyr_ll.c index 15360ea98c6c..e7c97837bcee 100644 --- a/src/schedule/zephyr_ll.c +++ b/src/schedule/zephyr_ll.c @@ -814,7 +814,8 @@ void scheduler_get_task_info_ll(struct scheduler_props *scheduler_props, uint32_t flags; scheduler_props->processing_domain = COMP_PROCESSING_DOMAIN_LL; - struct zephyr_ll *ll_sch = (struct zephyr_ll *)scheduler_get_data_for_core(SOF_SCHEDULE_LL_TIMER, 0); + struct zephyr_ll *ll_sch = (struct zephyr_ll *)scheduler_get_data_for_core( + SOF_SCHEDULE_LL_TIMER, cpu_get_id()); zephyr_ll_lock(ll_sch, &flags); scheduler_get_task_info(scheduler_props, data_off_size, &ll_sch->tasks); @@ -824,7 +825,7 @@ void scheduler_get_task_info_ll(struct scheduler_props *scheduler_props, /* Return a pointer to the LL scheduler timer domain */ struct ll_schedule_domain *zephyr_ll_domain(void) { - struct zephyr_ll *ll_sch = scheduler_get_data_for_core(SOF_SCHEDULE_LL_TIMER, 0); + struct zephyr_ll *ll_sch = scheduler_get_data_for_core(SOF_SCHEDULE_LL_TIMER, cpu_get_id()); return ll_sch->ll_domain; } From 892aaef6238003263dfb627ad36da00b8943daed Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Mon, 15 Jun 2026 15:16:30 +0200 Subject: [PATCH 068/108] userspace: ll: ipc: increase IPC processing timeout Sometimes 10ms aren't enough for userspace IPC processing, increase it to 20ms. Signed-off-by: Guennadi Liakhovetski --- src/ipc/ipc-common.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ipc/ipc-common.c b/src/ipc/ipc-common.c index b30749eb1e54..38c5f8386041 100644 --- a/src/ipc/ipc-common.c +++ b/src/ipc/ipc-common.c @@ -431,7 +431,7 @@ int ipc_user_forward_cmd(struct ipc4_message_request *ipc4) k_event_set(pdata->event, IPC_USER_EVENT_CMD); /* Wait for user thread to complete */ - ret = k_sem_take(pdata->sem, K_MSEC(10)); + ret = k_sem_take(pdata->sem, K_MSEC(100)); if (ret) { LOG_ERR("IPC user: sem error %d\n", ret); return ret; From 2cd4abcf2ea842f3545fe1e54f0e7c6e42e02697 Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Mon, 22 Jun 2026 12:04:57 +0200 Subject: [PATCH 069/108] ipc: ipc4: extract pipeline ID detection into a function The SOF_IPC4_GLB_SET_PIPELINE_STATE IPC can apply to one or to multiple pipelines. Extract pipeline ID detection into a function to be re-used with userspace IPC processing. Signed-off-by: Guennadi Liakhovetski --- src/ipc/ipc4/handler-user.c | 69 +++++++++++++++++++++++-------------- 1 file changed, 44 insertions(+), 25 deletions(-) diff --git a/src/ipc/ipc4/handler-user.c b/src/ipc/ipc4/handler-user.c index 16ee823c1adb..399cb5c55713 100644 --- a/src/ipc/ipc4/handler-user.c +++ b/src/ipc/ipc4/handler-user.c @@ -428,6 +428,44 @@ __cold const struct ipc4_pipeline_set_state_data *ipc4_get_pipeline_data_wrapper return ipc4_get_pipeline_data(); } +static int ipc4_pipeline_id_get(struct ipc4_message_request *ipc4, + struct ipc4_pipeline_set_state *state, + const uint32_t **ppl_id, unsigned int *ppl_count) +{ + if (!state->extension.r.multi_ppl) { + if (ppl_count) + *ppl_count = 1; + if (ppl_id) + *ppl_id = NULL; + return state->primary.r.ppl_id; + } + + const struct ipc4_pipeline_set_state_data *ppl_data = ipc4_get_pipeline_data(); + unsigned int cnt = ppl_data->pipelines_count; + + /* + * pipelines_count is read straight from the host-provided + * mailbox payload, so cap it at what the mailbox can + * physically hold. Anything larger means the host promised + * more ppl_id[] entries than fit in MAILBOX_HOSTBOX, and + * dereferencing the flex array would read out of bounds. + */ + if (cnt > (MAILBOX_HOSTBOX_SIZE - sizeof(struct ipc4_pipeline_set_state_data)) / + sizeof(uint32_t)) { + ipc_cmd_err(&ipc_tr, "ipc: pipelines_count %u exceeds mailbox bound", + cnt); + return -EINVAL; + } + dcache_invalidate_region((__sparse_force void __sparse_cache *)ppl_data->ppl_id, + sizeof(int) * cnt); + if (ppl_count) + *ppl_count = cnt; + if (ppl_id) + *ppl_id = ppl_data->ppl_id; + + return ppl_data->ppl_id[0]; +} + /** * \brief Process SET_PIPELINE_STATE IPC4 message (prepare + trigger phases). * @param[in] ipc4 IPC4 message request. @@ -440,7 +478,7 @@ int ipc4_set_pipeline_state(struct ipc4_message_request *ipc4) struct ipc_comp_dev *ppl_icd; struct ipc *ipc = ipc_get(); uint32_t cmd, ppl_count; - uint32_t id = 0; + int id; const uint32_t *ppl_id; bool use_idc = false; uint32_t idx; @@ -452,31 +490,12 @@ int ipc4_set_pipeline_state(struct ipc4_message_request *ipc4) cmd = state.primary.r.ppl_state; ppl_data = ipc4_get_pipeline_data(); - if (state.extension.r.multi_ppl) { - ppl_count = ppl_data->pipelines_count; - /* - * pipelines_count is read straight from the host-provided - * mailbox payload, so cap it at what the mailbox can - * physically hold. Anything larger means the host promised - * more ppl_id[] entries than fit in MAILBOX_HOSTBOX, and - * dereferencing the flex array would read out of bounds. - */ - if (ppl_count > (MAILBOX_HOSTBOX_SIZE - - sizeof(struct ipc4_pipeline_set_state_data)) / - sizeof(uint32_t)) { - ipc_cmd_err(&ipc_tr, - "ipc: pipelines_count %u exceeds mailbox bound", - ppl_count); - return IPC4_ERROR_INVALID_PARAM; - } - ppl_id = ppl_data->ppl_id; - dcache_invalidate_region((__sparse_force void __sparse_cache *)ppl_id, - sizeof(int) * ppl_count); - } else { - ppl_count = 1; - id = state.primary.r.ppl_id; + id = ipc4_pipeline_id_get(ipc4, &state, &ppl_id, &ppl_count); + if (id < 0) + return IPC4_ERROR_INVALID_PARAM; + + if (ppl_count == 1) ppl_id = &id; - } for (i = 0; i < ppl_count; i++) { ppl_icd = ipc_get_comp_by_ppl_id(ipc, COMP_TYPE_PIPELINE, From 4fc8d160ed9279cb08744d3cfdf1fd3774e58bc4 Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Mon, 22 Jun 2026 12:24:14 +0200 Subject: [PATCH 070/108] ipc: allocate userspace LL context uncached Userspace LL allocation context can be used by all cores, allocate it as uncached. Signed-off-by: Guennadi Liakhovetski --- src/ipc/ipc-common.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/ipc/ipc-common.c b/src/ipc/ipc-common.c index 38c5f8386041..54530ab82702 100644 --- a/src/ipc/ipc-common.c +++ b/src/ipc/ipc-common.c @@ -665,7 +665,8 @@ __cold static int ipc_user_init_thread(struct ipc_user *ipc_user) return ret; } -__cold int ipc_user_init(void) +/* Primary core only */ +__cold static int ipc_user_init(void) { struct ipc *ipc = ipc_get(); struct ipc_user *ipc_user = sof_heap_alloc(sof_sys_user_heap_get(), @@ -770,7 +771,8 @@ __cold int ipc_init(struct sof *sof) ipc = ipc_get(); memset(ipc, 0, sizeof(*ipc)); - ipc->ll_alloc = sof_heap_alloc(heap, SOF_MEM_FLAG_USER, sizeof(*ipc->ll_alloc), 0); + ipc->ll_alloc = sof_heap_alloc(heap, SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, + sizeof(*ipc->ll_alloc), 0); if (!ipc->ll_alloc) { tr_err(&ipc_tr, "Unable to allocate IPC ll_alloc"); return -ENOMEM; From a9cbb55b42385eb537fb294cd6154606195ad5f0 Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Wed, 10 Jun 2026 16:18:53 +0200 Subject: [PATCH 071/108] schedule: ll: enable multicore userspace Make scheduling LL thread and synchronisation objects per-core and forward IPCs and scheduling events accordingly. Signed-off-by: Guennadi Liakhovetski --- src/include/sof/ipc/common.h | 10 +-- src/init/init.c | 22 +++---- src/ipc/ipc-common.c | 124 ++++++++++++++++++++++++++--------- src/ipc/ipc4/handler-user.c | 120 +++++++++++++++++++++++---------- 4 files changed, 193 insertions(+), 83 deletions(-) diff --git a/src/include/sof/ipc/common.h b/src/include/sof/ipc/common.h index fc749f0b33a9..66658e4be1a8 100644 --- a/src/include/sof/ipc/common.h +++ b/src/include/sof/ipc/common.h @@ -56,9 +56,9 @@ extern struct tr_ctx ipc_tr; #define IPC_TASK_POWERDOWN BIT(3) struct ipc_user { - struct k_thread *thread; + struct k_thread *thread[CONFIG_CORE_COUNT]; struct k_sem *sem; - struct k_event *event; + struct k_event *event[CONFIG_CORE_COUNT]; /** @brief Copy of IPC4 message primary word forwarded to user thread */ uint32_t ipc_msg_pri; /** @brief Copy of IPC4 message extension word forwarded to user thread */ @@ -72,9 +72,10 @@ struct ipc_user { /** @brief Reply TX data pointer from user thread (e.g. LARGE_CONFIG_GET result) */ void *reply_tx_data; struct ipc *ipc; - struct k_thread *audio_thread; + struct k_thread *audio_thread[CONFIG_CORE_COUNT]; /** @brief Original kernel driver pointer for restoring dev->drv after create */ const struct comp_driver *init_drv; + bool init_needed[CONFIG_CORE_COUNT]; /** * @brief User-accessible copy of comp_driver + tr_ctx for create(). * @@ -353,7 +354,8 @@ struct ipc4_message_request; * @param ipc4 Pointer to the IPC4 message request * @return Result from user thread processing */ -int ipc_user_forward_cmd(struct ipc4_message_request *ipc4); +int ipc_user_forward_cmd(struct ipc4_message_request *ipc4, unsigned int core); +int ipc_user_init_secondary(unsigned int core); #endif #endif /* __SOF_DRIVERS_IPC_H__ */ diff --git a/src/init/init.c b/src/init/init.c index 57f41b940bc2..7fc6a057b63c 100644 --- a/src/init/init.c +++ b/src/init/init.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -114,6 +115,7 @@ static inline int secondary_core_restore(void) { return 0; }; __cold int secondary_core_init(struct sof *sof) { + unsigned int core = cpu_get_id(); int err; struct ll_schedule_domain *dma_domain; @@ -138,24 +140,18 @@ __cold int secondary_core_init(struct sof *sof) if (dma_domain) scheduler_init_ll(dma_domain); +#if CONFIG_SOF_USERSPACE_LL + err = ipc_user_init_secondary(core); + if (err < 0) + return err; +#endif + #if CONFIG_ZEPHYR_DP_SCHEDULER err = scheduler_dp_init(); if (err < 0) return err; #endif /* CONFIG_ZEPHYR_DP_SCHEDULER */ -#if CONFIG_SOF_USERSPACE_LL - /* Create domain thread for this secondary core's LL scheduler */ - { - struct task *task = zephyr_ll_task_alloc(); - - schedule_task_init_ll(task, SOF_UUID(sec_core_init_uuid), - SOF_SCHEDULE_LL_TIMER, - 0, NULL, NULL, cpu_get_id(), 0); - scheduler_init_context(task); - } -#endif - /* initialize IDC mechanism */ trace_point(TRACE_BOOT_PLATFORM_IDC); err = idc_init(); @@ -168,7 +164,7 @@ __cold int secondary_core_init(struct sof *sof) return err; #endif #if CONFIG_KCPS_DYNAMIC_CLOCK_CONTROL - err = core_kcps_adjust(cpu_get_id(), SECONDARY_CORE_BASE_CPS_USAGE); + err = core_kcps_adjust(core, SECONDARY_CORE_BASE_CPS_USAGE); if (err < 0) return err; #endif diff --git a/src/ipc/ipc-common.c b/src/ipc/ipc-common.c index 54530ab82702..3d36c77ae798 100644 --- a/src/ipc/ipc-common.c +++ b/src/ipc/ipc-common.c @@ -394,7 +394,8 @@ void ipc_schedule_process(struct ipc *ipc) #define IPC_USER_EVENT_CMD BIT(0) #define IPC_USER_EVENT_STOP BIT(1) -static K_THREAD_STACK_DEFINE(ipc_user_stack, CONFIG_SOF_IPC_USER_THREAD_STACK_SIZE); +static K_THREAD_STACK_ARRAY_DEFINE(ipc_user_stack, CONFIG_CORE_COUNT, + CONFIG_SOF_IPC_USER_THREAD_STACK_SIZE); /** * @brief Forward an IPC4 command to the user-space thread. @@ -408,7 +409,7 @@ static K_THREAD_STACK_DEFINE(ipc_user_stack, CONFIG_SOF_IPC_USER_THREAD_STACK_SI * @param ipc4 Pointer to the IPC4 message request * @return Result from user thread processing */ -int ipc_user_forward_cmd(struct ipc4_message_request *ipc4) +int ipc_user_forward_cmd(struct ipc4_message_request *ipc4, unsigned int core) { struct ipc *ipc = ipc_get(); struct ipc_user *pdata = ipc->ipc_user_pdata; @@ -422,13 +423,22 @@ int ipc_user_forward_cmd(struct ipc4_message_request *ipc4) pdata->ipc_msg_ext = ipc4->extension.dat; pdata->ipc = ipc; + /* + * Forwarding the first IPC to this core, wait for its userspace IPC + * thread to start + */ + if (pdata->init_needed[core]) { + pdata->init_needed[core] = false; + k_sem_take(pdata->sem, K_FOREVER); + } + /* Prevent host completion until user thread finishes */ key = k_spin_lock(&ipc->lock); ipc->task_mask |= IPC_TASK_IN_THREAD; k_spin_unlock(&ipc->lock, key); /* Wake the user thread */ - k_event_set(pdata->event, IPC_USER_EVENT_CMD); + k_event_set(pdata->event[core], IPC_USER_EVENT_CMD); /* Wait for user thread to complete */ ret = k_sem_take(pdata->sem, K_MSEC(100)); @@ -453,8 +463,8 @@ int ipc_user_forward_cmd(struct ipc4_message_request *ipc4) static void ipc_user_thread_fn(void *p1, void *p2, void *p3) { struct ipc_user *ipc_user = p1; + unsigned int core = POINTER_TO_UINT(p2); - ARG_UNUSED(p2); ARG_UNUSED(p3); __ASSERT(k_is_user_context(), "expected user context"); @@ -464,7 +474,7 @@ static void ipc_user_thread_fn(void *p1, void *p2, void *p3) LOG_INF("IPC user-space thread started"); for (;;) { - uint32_t mask = k_event_wait_safe(ipc_user->event, + uint32_t mask = k_event_wait_safe(ipc_user->event[core], IPC_USER_EVENT_CMD | IPC_USER_EVENT_STOP, false, K_MSEC(5000)); @@ -616,7 +626,7 @@ static void ipc_user_thread_fn(void *p1, void *p2, void *p3) } } -__cold static int ipc_user_init_thread(struct ipc_user *ipc_user) +__cold static int ipc_user_init_thread(struct ipc_user *ipc_user, unsigned int core) { char thread_name[] = "ll_user0"; int ret; @@ -624,47 +634,84 @@ __cold static int ipc_user_init_thread(struct ipc_user *ipc_user) assert_can_be_cold(); /* Allocate kernel objects for the user-space thread */ - ipc_user->event = k_object_alloc(K_OBJ_EVENT); - if (!ipc_user->event) { + ipc_user->event[core] = k_object_alloc(K_OBJ_EVENT); + if (!ipc_user->event[core]) { LOG_ERR("user IPC event alloc failed"); return -ENOMEM; } - k_event_init(ipc_user->event); + k_event_init(ipc_user->event[core]); - ipc_user->thread = k_object_alloc(K_OBJ_THREAD); - if (!ipc_user->thread) { + ipc_user->thread[core] = k_object_alloc(K_OBJ_THREAD); + if (!ipc_user->thread[core]) { LOG_ERR("user IPC thread alloc failed"); ret = -ENOMEM; goto e_event; } - k_thread_create(ipc_user->thread, ipc_user_stack, + k_thread_create(ipc_user->thread[core], ipc_user_stack[core], CONFIG_SOF_IPC_USER_THREAD_STACK_SIZE, - ipc_user_thread_fn, ipc_user, NULL, NULL, + ipc_user_thread_fn, ipc_user, UINT_TO_POINTER(core), NULL, -1, K_USER, K_FOREVER); - k_thread_cpu_pin(ipc_user->thread, PLATFORM_PRIMARY_CORE_ID); - k_thread_name_set(ipc_user->thread, thread_name); + k_thread_cpu_pin(ipc_user->thread[core], core); + thread_name[sizeof(thread_name) - 2] = '0' + core; + k_thread_name_set(ipc_user->thread[core], thread_name); /* * Each userspace IPC thread must be able to wait on its private event * and signal completion on the primary core semaphore */ - k_thread_access_grant(ipc_user->thread, ipc_user->sem, ipc_user->event); - user_grant_dai_access_all(ipc_user->thread); - user_grant_dma_access_all(ipc_user->thread); - pipeline_posn_grant_access(ipc_user->thread); - k_mem_domain_add_thread(zephyr_ll_mem_domain(), ipc_user->thread); - user_ll_grant_access(ipc_user->thread, PLATFORM_PRIMARY_CORE_ID); + k_thread_access_grant(ipc_user->thread[core], ipc_user->sem, ipc_user->event[core]); + user_grant_dai_access_all(ipc_user->thread[core]); + user_grant_dma_access_all(ipc_user->thread[core]); + pipeline_posn_grant_access(ipc_user->thread[core]); + k_mem_domain_add_thread(zephyr_ll_mem_domain(), ipc_user->thread[core]); + user_ll_grant_access(ipc_user->thread[core], core); return 0; e_event: - k_object_free(ipc_user->event); + k_object_free(ipc_user->event[core]); return ret; } +__cold int ipc_user_init_secondary(unsigned int core) +{ + struct ipc *ipc = ipc_get(); + struct ipc_user *ipc_user = ipc->ipc_user_pdata; + int ret = ipc_user_init_thread(ipc_user, core); + + if (ret < 0) + return ret; + + assert_can_be_cold(); + + k_thread_start(ipc_user->thread[core]); + + struct task *task = zephyr_ll_task_alloc(); + + if (!task) { + LOG_ERR("user LL task allocation failed"); + k_panic(); + } + + schedule_task_init_ll(task, SOF_UUID(ipc_uuid), SOF_SCHEDULE_LL_TIMER, + 0, NULL, NULL, core, 0); + + ipc_user->audio_thread[core] = scheduler_init_context(task); + if (!ipc_user->audio_thread[core]) { + LOG_ERR("user LL thread init failed"); + k_panic(); + } + + k_thread_access_grant(ipc_user->thread[core], ipc_user->audio_thread[core]); + ipc_user->init_needed[core] = true; + + /* Wait for user thread startup — consumes the initial k_sem_give from thread */ + return 0; +} + /* Primary core only */ __cold static int ipc_user_init(void) { @@ -674,6 +721,8 @@ __cold static int ipc_user_init(void) sizeof(*ipc_user), 0); int ret; + assert_can_be_cold(); + ipc_user->sem = k_object_alloc(K_OBJ_SEM); if (!ipc_user->sem) { LOG_ERR("user IPC sem alloc failed"); @@ -713,40 +762,51 @@ __cold static int ipc_user_init(void) ret = k_mem_domain_add_partition(zephyr_ll_mem_domain(), &cold_part); if (ret < 0) - LOG_WRN("cold rodata partition %#zx @ %#lx add failed: %d", cold_part.size, - cold_part.start, ret); + LOG_WRN("cold rodata partition %#zx @ %#lx add failed: %d", + cold_part.size, cold_part.start, ret); } } #endif k_sem_init(ipc_user->sem, 0, 1); - ret = ipc_user_init_thread(ipc_user); + ret = ipc_user_init_thread(ipc_user, PLATFORM_PRIMARY_CORE_ID); if (ret < 0) { LOG_ERR("user IPC thread initialization failed"); k_panic(); } - user_access_to_mailbox(zephyr_ll_mem_domain(), ipc_user->thread); + user_access_to_mailbox(zephyr_ll_mem_domain(), ipc_user->thread[PLATFORM_PRIMARY_CORE_ID]); /* Store references in ipc struct so kernel handler can forward commands */ ipc->ipc_user_pdata = ipc_user; - k_thread_start(ipc_user->thread); - struct task *task = zephyr_ll_task_alloc(); + + if (!task) { + LOG_ERR("task allocation failed"); + k_panic(); + } + schedule_task_init_ll(task, SOF_UUID(ipc_uuid), SOF_SCHEDULE_LL_TIMER, - 0, NULL, NULL, cpu_get_id(), 0); - ipc_user->audio_thread = scheduler_init_context(task); + 0, NULL, NULL, PLATFORM_PRIMARY_CORE_ID, 0); + ipc_user->audio_thread[PLATFORM_PRIMARY_CORE_ID] = scheduler_init_context(task); + if (!ipc_user->audio_thread[PLATFORM_PRIMARY_CORE_ID]) { + LOG_ERR("user LL thread init failed"); + k_panic(); + } /* Grant ipc_user thread permission on the audio thread object. * Needed so user-space dai_common_new() can call * k_thread_access_grant(audio_thread, dai_mutex) from user context. */ - k_thread_access_grant(ipc_user->thread, ipc_user->audio_thread); + k_thread_access_grant(ipc_user->thread[PLATFORM_PRIMARY_CORE_ID], + ipc_user->audio_thread[PLATFORM_PRIMARY_CORE_ID]); + + k_thread_start(ipc_user->thread[PLATFORM_PRIMARY_CORE_ID]); /* Wait for user thread startup — consumes the initial k_sem_give from thread */ - k_sem_take(ipc->ipc_user_pdata->sem, K_FOREVER); + k_sem_take(ipc_user->sem, K_FOREVER); return 0; } diff --git a/src/ipc/ipc4/handler-user.c b/src/ipc/ipc4/handler-user.c index 399cb5c55713..7906701fe5f7 100644 --- a/src/ipc/ipc4/handler-user.c +++ b/src/ipc/ipc4/handler-user.c @@ -90,7 +90,31 @@ static inline const struct ipc4_pipeline_set_state_data *ipc4_get_pipeline_data( /* * Global IPC Operations. */ -#ifndef CONFIG_SOF_USERSPACE_LL +#ifdef CONFIG_SOF_USERSPACE_LL +/* + * Determine the target core for an IPC4 module message. + * Falls back to current core when no component is bound yet. + */ +static unsigned int ipc4_user_target_core_module(struct ipc4_message_request *ipc4) +{ + /* + * Also works for struct ipc4_module_large_config, struct ipc4_module_bind_unbind, + * struct ipc4_module_delete_instance + */ + struct ipc4_module_config *config = (struct ipc4_module_config *)ipc4; + uint32_t module_id = config->primary.r.module_id; + + if (module_id) { + uint32_t instance_id = config->primary.r.instance_id; + struct comp_dev *dev = ipc4_get_comp_dev(IPC4_COMP_ID(module_id, instance_id)); + + if (dev) + return dev->ipc_config.core; + } + + return cpu_get_id(); +} +#else __cold static int ipc4_new_pipeline(struct ipc4_message_request *ipc4) { struct ipc *ipc = ipc_get(); @@ -99,9 +123,7 @@ __cold static int ipc4_new_pipeline(struct ipc4_message_request *ipc4) return ipc_pipeline_new(ipc, (ipc_pipe_new *)ipc4); } -#endif -#ifndef CONFIG_SOF_USERSPACE_LL __cold static int ipc4_delete_pipeline(struct ipc4_message_request *ipc4) { struct ipc4_pipeline_delete *pipe; @@ -686,6 +708,7 @@ static int ipc_glb_gdb_debug(struct ipc4_message_request *ipc4) int ipc4_user_process_glb_message(struct ipc4_message_request *ipc4, struct ipc_msg *reply) { + struct ipc *ipc = ipc_get(); uint32_t type; int ret; @@ -710,21 +733,56 @@ int ipc4_user_process_glb_message(struct ipc4_message_request *ipc4, case SOF_IPC4_GLB_CREATE_PIPELINE: /* Implementation in progress: forward only CREATE_PIPELINE for now */ #ifdef CONFIG_SOF_USERSPACE_LL - ret = ipc_user_forward_cmd(ipc4); + { + const struct ipc4_pipeline_create *create = + (const struct ipc4_pipeline_create *)ipc4; + + ret = ipc_user_forward_cmd(ipc4, create->extension.r.core_id); + } #else ret = ipc4_new_pipeline(ipc4); #endif break; case SOF_IPC4_GLB_DELETE_PIPELINE: #ifdef CONFIG_SOF_USERSPACE_LL - ret = ipc_user_forward_cmd(ipc4); + { + const struct ipc4_pipeline_delete *del = (const struct ipc4_pipeline_delete *)ipc4; + struct ipc_comp_dev *ppl = ipc_get_comp_by_ppl_id(ipc, COMP_TYPE_PIPELINE, + del->primary.r.instance_id, + IPC_COMP_ALL); + + if (!ppl) { + ret = IPC4_INVALID_RESOURCE_ID; + break; + } + ret = ipc_user_forward_cmd(ipc4, ppl->core); + } #else ret = ipc4_delete_pipeline(ipc4); #endif break; case SOF_IPC4_GLB_SET_PIPELINE_STATE: #ifdef CONFIG_SOF_USERSPACE_LL - ret = ipc_user_forward_cmd(ipc4); + { + struct ipc4_pipeline_set_state state = { + .primary.dat = ipc4->primary.dat, + .extension.dat = ipc4->extension.dat, + }; + int id = ipc4_pipeline_id_get(ipc4, &state, NULL, NULL); + if (id < 0) { + ret = IPC4_INVALID_RESOURCE_ID; + break; + } + + struct ipc_comp_dev *ppl = ipc_get_comp_by_ppl_id(ipc, COMP_TYPE_PIPELINE, id, + IPC_COMP_ALL); + + if (!ppl) { + ret = IPC4_INVALID_RESOURCE_ID; + break; + } + ret = ipc_user_forward_cmd(ipc4, ppl->core); + } #else ret = ipc4_set_pipeline_state(ipc4); #endif @@ -1512,32 +1570,26 @@ __cold int ipc4_user_process_module_message(struct ipc4_message_request *ipc4, * access to IMR manifest and driver list in kernel memory). * Component creation (drv->ops.create) runs in user thread * so untrusted module code does not execute in kernel context. - * Cross-core creation stays fully in kernel. */ - struct ipc4_module_init_instance mi; + struct ipc4_module_init_instance *mi = (struct ipc4_module_init_instance *)ipc4; + struct ipc *ipc = ipc_get(); + uint32_t comp_id = IPC4_COMP_ID(mi->primary.r.module_id, + mi->primary.r.instance_id); + const struct comp_driver *drv = ipc4_get_comp_drv(IPC4_MOD_ID(comp_id)); - memcpy_s(&mi, sizeof(mi), ipc4, sizeof(*ipc4)); - if (!cpu_is_me(mi.extension.r.core_id)) { - ret = ipc4_init_module_instance(ipc4); - } else { - struct ipc *ipc = ipc_get(); - uint32_t comp_id = IPC4_COMP_ID(mi.primary.r.module_id, - mi.primary.r.instance_id); - const struct comp_driver *drv = ipc4_get_comp_drv(IPC4_MOD_ID(comp_id)); + if (!drv) { + ret = IPC4_MOD_NOT_INITIALIZED; + break; + } - if (!drv) { - ret = IPC4_MOD_NOT_INITIALIZED; - } else { - struct ipc_user *pdata = ipc->ipc_user_pdata; + struct ipc_user *pdata = ipc->ipc_user_pdata; - ret = llext_manager_map_lib(comp_id); - if (ret < 0) - break; + ret = llext_manager_map_lib(comp_id); + if (ret < 0) + break; - pdata->init_drv = drv; - ret = ipc_user_forward_cmd(ipc4); - } - } + pdata->init_drv = drv; + ret = ipc_user_forward_cmd(ipc4, mi->extension.r.core_id); } #else ret = ipc4_init_module_instance(ipc4); @@ -1546,7 +1598,7 @@ __cold int ipc4_user_process_module_message(struct ipc4_message_request *ipc4, case SOF_IPC4_MOD_CONFIG_GET: #ifdef CONFIG_SOF_USERSPACE_LL /* Forward to user thread for privilege-separated execution */ - ret = ipc_user_forward_cmd(ipc4); + ret = ipc_user_forward_cmd(ipc4, ipc4_user_target_core_module(ipc4)); if (!ret) { struct ipc *ipc = ipc_get(); struct ipc_user *pdata = ipc->ipc_user_pdata; @@ -1560,7 +1612,7 @@ __cold int ipc4_user_process_module_message(struct ipc4_message_request *ipc4, case SOF_IPC4_MOD_CONFIG_SET: #ifdef CONFIG_SOF_USERSPACE_LL /* Forward to user thread for privilege-separated execution */ - ret = ipc_user_forward_cmd(ipc4); + ret = ipc_user_forward_cmd(ipc4, ipc4_user_target_core_module(ipc4)); #else ret = ipc4_set_get_config_module_instance(ipc4, true); #endif @@ -1573,7 +1625,7 @@ __cold int ipc4_user_process_module_message(struct ipc4_message_request *ipc4, if (config->primary.r.module_id) { /* Module case: forward to user thread */ - ret = ipc_user_forward_cmd(ipc4); + ret = ipc_user_forward_cmd(ipc4, ipc4_user_target_core_module(ipc4)); if (!ret) { struct ipc *ipc = ipc_get(); struct ipc_user *pdata = ipc->ipc_user_pdata; @@ -1601,7 +1653,7 @@ __cold int ipc4_user_process_module_message(struct ipc4_message_request *ipc4, (const struct ipc4_module_large_config *)ipc4; if (config->primary.r.module_id) { - ret = ipc_user_forward_cmd(ipc4); + ret = ipc_user_forward_cmd(ipc4, ipc4_user_target_core_module(ipc4)); } else { /* Base firmware: keep in kernel (IMR access) */ ret = ipc4_set_large_config_module_instance(ipc4); @@ -1613,21 +1665,21 @@ __cold int ipc4_user_process_module_message(struct ipc4_message_request *ipc4, break; case SOF_IPC4_MOD_BIND: #ifdef CONFIG_SOF_USERSPACE_LL - ret = ipc_user_forward_cmd(ipc4); + ret = ipc_user_forward_cmd(ipc4, ipc4_user_target_core_module(ipc4)); #else ret = ipc4_bind_module_instance(ipc4); #endif break; case SOF_IPC4_MOD_UNBIND: #ifdef CONFIG_SOF_USERSPACE_LL - ret = ipc_user_forward_cmd(ipc4); + ret = ipc_user_forward_cmd(ipc4, ipc4_user_target_core_module(ipc4)); #else ret = ipc4_unbind_module_instance(ipc4); #endif break; case SOF_IPC4_MOD_DELETE_INSTANCE: #ifdef CONFIG_SOF_USERSPACE_LL - ret = ipc_user_forward_cmd(ipc4); + ret = ipc_user_forward_cmd(ipc4, ipc4_user_target_core_module(ipc4)); #else ret = ipc4_delete_module_instance(ipc4); #endif From 9ae86307dce35750cbb1b059bddf25c61dbefbc5 Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Thu, 2 Jul 2026 10:29:22 +0200 Subject: [PATCH 072/108] audio: add driver list access debugging The global driver list should never be accessed from the audio context. Add an assertion for that. Signed-off-by: Guennadi Liakhovetski --- src/audio/component.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/audio/component.c b/src/audio/component.c index 62cbdac689f7..46a92a58aa67 100644 --- a/src/audio/component.c +++ b/src/audio/component.c @@ -41,6 +41,7 @@ static APP_SYSUSER_BSS SHARED_DATA struct comp_driver_list cd; #ifdef CONFIG_SOF_USERSPACE_LL struct comp_driver_list *comp_drivers_get(void) { + assert(!ll_sch_is_current()); return &cd; } EXPORT_SYMBOL(comp_drivers_get); From ab235b3589f0cf86ded5be6bbf5d0864223a15f9 Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Thu, 2 Jul 2026 10:47:55 +0200 Subject: [PATCH 073/108] schedule: ll: remove a redundant type-cast scheduler_get_data_for_core() returns void pointer, no need to type-cast it to other pointer types. Signed-off-by: Guennadi Liakhovetski --- src/schedule/zephyr_ll.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/schedule/zephyr_ll.c b/src/schedule/zephyr_ll.c index e7c97837bcee..0e128c011465 100644 --- a/src/schedule/zephyr_ll.c +++ b/src/schedule/zephyr_ll.c @@ -814,8 +814,7 @@ void scheduler_get_task_info_ll(struct scheduler_props *scheduler_props, uint32_t flags; scheduler_props->processing_domain = COMP_PROCESSING_DOMAIN_LL; - struct zephyr_ll *ll_sch = (struct zephyr_ll *)scheduler_get_data_for_core( - SOF_SCHEDULE_LL_TIMER, cpu_get_id()); + struct zephyr_ll *ll_sch = scheduler_get_data_for_core(SOF_SCHEDULE_LL_TIMER, cpu_get_id()); zephyr_ll_lock(ll_sch, &flags); scheduler_get_task_info(scheduler_props, data_off_size, &ll_sch->tasks); From 58798ac9745ef69cc60b6a7de73f6bd4a195c621 Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Thu, 2 Jul 2026 11:04:37 +0200 Subject: [PATCH 074/108] userspace: use syscalls for kernel builds too syscall-related code can be used when CONFIG_USERSPACE is undefined too. Signed-off-by: Guennadi Liakhovetski --- src/include/ipc4/handler.h | 8 +++----- src/include/sof/ipc/common.h | 27 --------------------------- src/include/sof/ipc/ipc_reply.h | 8 +++----- 3 files changed, 6 insertions(+), 37 deletions(-) diff --git a/src/include/ipc4/handler.h b/src/include/ipc4/handler.h index 9e2c96c5ba28..0ed298dd9ad1 100644 --- a/src/include/ipc4/handler.h +++ b/src/include/ipc4/handler.h @@ -68,7 +68,7 @@ int ipc4_set_pipeline_state(struct ipc4_message_request *ipc4); */ void ipc_compound_msg_done(uint32_t msg_id, int error); -#if defined(__ZEPHYR__) && defined(CONFIG_USERSPACE) +#if defined(__ZEPHYR__) && defined(CONFIG_SOF_FULL_ZEPHYR_APPLICATION) /** * \brief Increment the IPC compound message pre-start counter. * @param[in] msg_id IPC message ID. @@ -88,6 +88,8 @@ __syscall void ipc_compound_post_start(uint32_t msg_id, int ret, bool delayed); * @return 0 on success, error code otherwise on timeout. */ __syscall int ipc_wait_for_compound_msg(void); + +#include #else void z_impl_ipc_compound_pre_start(int msg_id); #define ipc_compound_pre_start z_impl_ipc_compound_pre_start @@ -97,8 +99,4 @@ int z_impl_ipc_wait_for_compound_msg(void); #define ipc_wait_for_compound_msg z_impl_ipc_wait_for_compound_msg #endif -#if defined(__ZEPHYR__) && defined(CONFIG_SOF_USERSPACE_LL) -#include -#endif - #endif /* __SOF_IPC4_HANDLER_H__ */ diff --git a/src/include/sof/ipc/common.h b/src/include/sof/ipc/common.h index 66658e4be1a8..87edc75abef1 100644 --- a/src/include/sof/ipc/common.h +++ b/src/include/sof/ipc/common.h @@ -229,33 +229,6 @@ int ipc4_user_process_module_message(struct ipc4_message_request *ipc4, struct i */ int ipc4_user_process_glb_message(struct ipc4_message_request *ipc4, struct ipc_msg *reply); -/* - * When CONFIG_SOF_USERSPACE_LL is enabled, compound message functions are - * declared as syscalls in ipc4/handler.h — do not re-declare here with - * external linkage as that conflicts with the static inline syscall wrappers. - */ -#if !(defined(__ZEPHYR__) && defined(CONFIG_SOF_USERSPACE_LL)) -/** - * \brief Increment the IPC compound message pre-start counter. - * @param[in] msg_id IPC message ID. - */ -void ipc_compound_pre_start(int msg_id); - -/** - * \brief Decrement the IPC compound message pre-start counter on return value status. - * @param[in] msg_id IPC message ID. - * @param[in] ret Return value of the IPC command. - * @param[in] delayed True if the reply is delayed. - */ -void ipc_compound_post_start(uint32_t msg_id, int ret, bool delayed); - -/** - * \brief Wait for the IPC compound message to complete. - * @return 0 on success, error code otherwise on timeout. - */ -int ipc_wait_for_compound_msg(void); -#endif /* !CONFIG_SOF_USERSPACE_LL */ - /** * \brief Complete the IPC compound message. * @param[in] msg_id IPC message ID. diff --git a/src/include/sof/ipc/ipc_reply.h b/src/include/sof/ipc/ipc_reply.h index 756fd535ca30..915b6261c6bd 100644 --- a/src/include/sof/ipc/ipc_reply.h +++ b/src/include/sof/ipc/ipc_reply.h @@ -14,15 +14,13 @@ struct sof_ipc_reply; * \brief reply to an IPC message. * @param[in] reply pointer to the reply structure. */ -#if defined(__ZEPHYR__) && defined(CONFIG_SOF_USERSPACE_LL) +#if defined(__ZEPHYR__) && defined(CONFIG_SOF_FULL_ZEPHYR_APPLICATION) __syscall void ipc_msg_reply(struct sof_ipc_reply *reply); + +#include #else void z_impl_ipc_msg_reply(struct sof_ipc_reply *reply); #define ipc_msg_reply z_impl_ipc_msg_reply #endif -#if defined(__ZEPHYR__) && defined(CONFIG_SOF_USERSPACE_LL) -#include -#endif - #endif /* __SOF_IPC_IPC_REPLY_H__ */ From 9b75e7ec85e346786e2813363211f36facffbba6 Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Thu, 2 Jul 2026 12:41:55 +0200 Subject: [PATCH 075/108] init: fix an "unused variable" compiler warning Define the "core" variable only when it is used. Signed-off-by: Guennadi Liakhovetski --- src/include/sof/lib/vregion.h | 10 ---------- src/init/init.c | 2 ++ 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/src/include/sof/lib/vregion.h b/src/include/sof/lib/vregion.h index 93233bd2caf6..cadf2464968a 100644 --- a/src/include/sof/lib/vregion.h +++ b/src/include/sof/lib/vregion.h @@ -84,15 +84,11 @@ struct vregion *vregion_put(struct vregion *vr); */ __syscall void *vregion_alloc(struct vregion *vr, size_t size); -void *z_impl_vregion_alloc(struct vregion *vr, size_t size); - /** * @brief like vregion_alloc() but allocates coherent memory */ __syscall void *vregion_alloc_coherent(struct vregion *vr, size_t size); -void *z_impl_vregion_alloc_coherent(struct vregion *vr, size_t size); - /** * @brief Allocate aligned memory from the specified virtual region. * @@ -106,15 +102,11 @@ void *z_impl_vregion_alloc_coherent(struct vregion *vr, size_t size); */ __syscall void *vregion_alloc_align(struct vregion *vr, size_t size, size_t alignment); -void *z_impl_vregion_alloc_align(struct vregion *vr, size_t size, size_t alignment); - /** * @brief like vregion_alloc_align() but allocates coherent memory */ __syscall void *vregion_alloc_coherent_align(struct vregion *vr, size_t size, size_t alignment); -void *z_impl_vregion_alloc_coherent_align(struct vregion *vr, size_t size, size_t alignment); - /** * @brief Free memory allocated from the specified virtual region. * @@ -125,8 +117,6 @@ void *z_impl_vregion_alloc_coherent_align(struct vregion *vr, size_t size, size_ */ __syscall void vregion_free(struct vregion *vr, void *ptr); -void z_impl_vregion_free(struct vregion *vr, void *ptr); - /** * @brief Log virtual region memory usage. * diff --git a/src/init/init.c b/src/init/init.c index 7fc6a057b63c..8605723a96ff 100644 --- a/src/init/init.c +++ b/src/init/init.c @@ -115,7 +115,9 @@ static inline int secondary_core_restore(void) { return 0; }; __cold int secondary_core_init(struct sof *sof) { +#if CONFIG_SOF_USERSPACE_LL || CONFIG_KCPS_DYNAMIC_CLOCK_CONTROL unsigned int core = cpu_get_id(); +#endif int err; struct ll_schedule_domain *dma_domain; From 05baa8f1b2b5075e132b9604a98cc6dacb9d9c65 Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Thu, 2 Jul 2026 12:48:31 +0200 Subject: [PATCH 076/108] ipc: fix an "unused variable" compiler warning Define the "ipc" variable only for cases, when it's used. Signed-off-by: Guennadi Liakhovetski --- src/ipc/ipc4/handler-user.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/ipc/ipc4/handler-user.c b/src/ipc/ipc4/handler-user.c index 7906701fe5f7..7bde99389eda 100644 --- a/src/ipc/ipc4/handler-user.c +++ b/src/ipc/ipc4/handler-user.c @@ -708,7 +708,9 @@ static int ipc_glb_gdb_debug(struct ipc4_message_request *ipc4) int ipc4_user_process_glb_message(struct ipc4_message_request *ipc4, struct ipc_msg *reply) { +#ifdef CONFIG_SOF_USERSPACE_LL struct ipc *ipc = ipc_get(); +#endif uint32_t type; int ret; From 2633e300b531747e33e759b0c7dcb1c9bc6fb9ba Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Thu, 2 Jul 2026 12:54:31 +0200 Subject: [PATCH 077/108] ipc: add the DP case to module initialization ipc4_init_module_instance() should be called when CONFIG_SOF_USERSPACE_LL isn't selected but also when initializing a DP module. Signed-off-by: Guennadi Liakhovetski --- src/ipc/ipc4/handler-user.c | 50 +++++++++++++++++++------------------ src/ipc/ipc4/helper.c | 28 ++++++++------------- 2 files changed, 37 insertions(+), 41 deletions(-) diff --git a/src/ipc/ipc4/handler-user.c b/src/ipc/ipc4/handler-user.c index 7bde99389eda..b02360b4e9d1 100644 --- a/src/ipc/ipc4/handler-user.c +++ b/src/ipc/ipc4/handler-user.c @@ -1556,6 +1556,7 @@ __cold static int ipc4_delete_module_instance(struct ipc4_message_request *ipc4) __cold int ipc4_user_process_module_message(struct ipc4_message_request *ipc4, struct ipc_msg *reply) { + struct ipc4_module_init_instance *mi; uint32_t type; int ret; @@ -1566,36 +1567,37 @@ __cold int ipc4_user_process_module_message(struct ipc4_message_request *ipc4, switch (type) { case SOF_IPC4_MOD_INIT_INSTANCE: + mi = (struct ipc4_module_init_instance *)ipc4; + + if (mi->extension.r.proc_domain || !IS_ENABLED(CONFIG_SOF_USERSPACE_LL)) { + ret = ipc4_init_module_instance(ipc4); + } else { #ifdef CONFIG_SOF_USERSPACE_LL - { - /* User-space init: kernel does driver lookup only (requires - * access to IMR manifest and driver list in kernel memory). - * Component creation (drv->ops.create) runs in user thread - * so untrusted module code does not execute in kernel context. - */ - struct ipc4_module_init_instance *mi = (struct ipc4_module_init_instance *)ipc4; - struct ipc *ipc = ipc_get(); - uint32_t comp_id = IPC4_COMP_ID(mi->primary.r.module_id, - mi->primary.r.instance_id); - const struct comp_driver *drv = ipc4_get_comp_drv(IPC4_MOD_ID(comp_id)); + /* User-space init: kernel does driver lookup only (requires + * access to IMR manifest and driver list in kernel memory). + * Component creation (drv->ops.create) runs in user thread + * so untrusted module code does not execute in kernel context. + */ + struct ipc *ipc = ipc_get(); + uint32_t comp_id = IPC4_COMP_ID(mi->primary.r.module_id, + mi->primary.r.instance_id); + const struct comp_driver *drv = ipc4_get_comp_drv(IPC4_MOD_ID(comp_id)); - if (!drv) { - ret = IPC4_MOD_NOT_INITIALIZED; - break; - } + if (!drv) { + ret = IPC4_MOD_NOT_INITIALIZED; + break; + } - struct ipc_user *pdata = ipc->ipc_user_pdata; + ret = llext_manager_map_lib(comp_id); + if (ret < 0) + break; - ret = llext_manager_map_lib(comp_id); - if (ret < 0) - break; + struct ipc_user *pdata = ipc->ipc_user_pdata; - pdata->init_drv = drv; - ret = ipc_user_forward_cmd(ipc4, mi->extension.r.core_id); - } -#else - ret = ipc4_init_module_instance(ipc4); + pdata->init_drv = drv; + ret = ipc_user_forward_cmd(ipc4, mi->extension.r.core_id); #endif + } break; case SOF_IPC4_MOD_CONFIG_GET: #ifdef CONFIG_SOF_USERSPACE_LL diff --git a/src/ipc/ipc4/helper.c b/src/ipc/ipc4/helper.c index b12ab9b228c5..2e146f80d478 100644 --- a/src/ipc/ipc4/helper.c +++ b/src/ipc/ipc4/helper.c @@ -161,18 +161,16 @@ __cold struct comp_dev *comp_new_ipc4(const struct ipc4_module_init_instance *mo if (!drv) return NULL; -#if CONFIG_ZEPHYR_DP_SCHEDULER - if (module_init->extension.r.proc_domain) - ipc_config.proc_domain = COMP_PROCESSING_DOMAIN_DP; - else + if (!module_init->extension.r.proc_domain) { ipc_config.proc_domain = COMP_PROCESSING_DOMAIN_LL; -#else /* CONFIG_ZEPHYR_DP_SCHEDULER */ - if (module_init->extension.r.proc_domain) { - tr_err(&ipc_tr, "ipc: DP scheduling is disabled, cannot create comp 0x%x", comp_id); + } else if (IS_ENABLED(CONFIG_ZEPHYR_DP_SCHEDULER)) { + ipc_config.proc_domain = COMP_PROCESSING_DOMAIN_DP; + } else { + tr_err(&ipc_tr, + "ipc: DP scheduling is disabled, cannot create comp 0x%x", + comp_id); return NULL; } - ipc_config.proc_domain = COMP_PROCESSING_DOMAIN_LL; -#endif /* CONFIG_ZEPHYR_DP_SCHEDULER */ if (drv->type == SOF_COMP_MODULE_ADAPTER) { const struct ipc_config_process spec = { @@ -272,20 +270,16 @@ __cold struct comp_dev *comp_new_ipc4_user(struct ipc4_message_request *ipc4, #endif data = ipc4_get_comp_new_data(); -#if CONFIG_ZEPHYR_DP_SCHEDULER - if (module_init.extension.r.proc_domain) - ipc_config.proc_domain = COMP_PROCESSING_DOMAIN_DP; - else + if (!module_init.extension.r.proc_domain) { ipc_config.proc_domain = COMP_PROCESSING_DOMAIN_LL; -#else - if (module_init.extension.r.proc_domain) { + } else if (IS_ENABLED(CONFIG_ZEPHYR_DP_SCHEDULER)) { + ipc_config.proc_domain = COMP_PROCESSING_DOMAIN_DP; + } else { tr_err(&ipc_tr, "ipc: DP scheduling is disabled, cannot create comp 0x%x", comp_id); return NULL; } - ipc_config.proc_domain = COMP_PROCESSING_DOMAIN_LL; -#endif if (drv->type == SOF_COMP_MODULE_ADAPTER) { const struct ipc_config_process spec = { From d586943f84b832a74412d477c03328ab817a05f0 Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Thu, 2 Jul 2026 13:52:52 +0200 Subject: [PATCH 078/108] llext: with userspace let LL thread access DP modules too If LL runs in userspace, it needs access to loaded LLEXT modules, running in DP more too. Signed-off-by: Guennadi Liakhovetski --- src/library_manager/llext_manager.c | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/src/library_manager/llext_manager.c b/src/library_manager/llext_manager.c index 439aad9a4cb5..0defc94a7d1f 100644 --- a/src/library_manager/llext_manager.c +++ b/src/library_manager/llext_manager.c @@ -299,18 +299,20 @@ static int llext_manager_load_module(struct lib_manager_module *mctx) memset((__sparse_force void *)bss_addr, 0, bss_size); mctx->mapped = true; - if (!mctx->domain_dp) { - ret = llext_manager_add_mod_domain(mctx, zephyr_ll_mem_domain()); - if (ret < 0) { - tr_err(&lib_manager_tr, "failed to add domain: %d", ret); - goto e_data; - } +#ifdef CONFIG_SOF_USERSPACE_LL + ret = llext_manager_add_mod_domain(mctx, zephyr_ll_mem_domain()); + if (ret < 0) { + tr_err(&lib_manager_tr, "failed to add domain: %d", ret); + goto e_data; } +#endif return 0; +#ifdef CONFIG_SOF_USERSPACE_LL e_data: if (data_size) llext_manager_align_unmap(va_base_data, data_size); +#endif e_rodata: if (rodata_size) llext_manager_align_unmap(va_base_rodata, rodata_size); @@ -370,8 +372,9 @@ static int llext_manager_unload_module(struct lib_manager_module *mctx) mctx->mapped = false; - if (!mctx->domain_dp) - llext_manager_rm_mod_domain(mctx, zephyr_ll_mem_domain()); +#ifdef CONFIG_SOF_USERSPACE_LL + llext_manager_rm_mod_domain(mctx, zephyr_ll_mem_domain()); +#endif return err; } From 6d513fe7196dd6096d5cdd725bfd4a623fae6e19 Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Thu, 2 Jul 2026 17:40:13 +0200 Subject: [PATCH 079/108] userspace: fix cpu_is_me() for kernel context In kernel context cpu_is_me() should return a correct result for correct multicore support. Signed-off-by: Guennadi Liakhovetski --- zephyr/include/sof/lib/cpu.h | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/zephyr/include/sof/lib/cpu.h b/zephyr/include/sof/lib/cpu.h index 533cb29f3602..b0d7c7e054ca 100644 --- a/zephyr/include/sof/lib/cpu.h +++ b/zephyr/include/sof/lib/cpu.h @@ -55,11 +55,7 @@ static inline bool cpu_is_primary(int id) static inline bool cpu_is_me(int id) { -#ifdef CONFIG_SOF_USERSPACE_LL - return true; -#else - return id == cpu_get_id(); -#endif + return k_is_user_context() || id == cpu_get_id(); } int cpu_enable_core(int id); From 6478e58b9655022aadc0a0000379139a523f1a63 Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Mon, 6 Jul 2026 16:58:35 +0200 Subject: [PATCH 080/108] audio: module-adapter: add two system calls Add two syscall functions to allocate and map, and to unmap vregion for userspace modules. For now only used for DP modules. Signed-off-by: Guennadi Liakhovetski --- src/audio/buffers/comp_buffer.c | 4 +- src/audio/module_adapter/module_adapter.c | 102 ++++++++++++++++-- .../sof/audio/module_adapter/module/generic.h | 7 ++ zephyr/include/rtos/alloc.h | 2 + 4 files changed, 105 insertions(+), 10 deletions(-) diff --git a/src/audio/buffers/comp_buffer.c b/src/audio/buffers/comp_buffer.c index 8a3d44133d4b..f64b74add169 100644 --- a/src/audio/buffers/comp_buffer.c +++ b/src/audio/buffers/comp_buffer.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -165,8 +166,7 @@ static void comp_buffer_free(struct sof_audio_buffer *audio_buffer) if (alloc && alloc->vreg) { vregion_free(alloc->vreg, buffer); - if (!vregion_put(alloc->vreg)) - rfree(alloc); + module_adapter_vreg_free(alloc); } else { sof_heap_free(alloc ? alloc->heap : NULL, buffer); } diff --git a/src/audio/module_adapter/module_adapter.c b/src/audio/module_adapter/module_adapter.c index 53e8c75c42e2..f46709abb1b1 100644 --- a/src/audio/module_adapter/module_adapter.c +++ b/src/audio/module_adapter/module_adapter.c @@ -58,14 +58,80 @@ struct comp_dev *module_adapter_new(const struct comp_driver *drv, #define PAGE_SZ HOST_PAGE_SIZE #endif -static struct vregion *module_adapter_dp_heap_new(const struct comp_ipc_config *config, - size_t *heap_size) +struct vregion *z_impl_module_adapter_vreg_new(const struct comp_ipc_config *config, + uintptr_t *vreg_start, size_t *vreg_size) { /* src-lite with 8 channels has been seen allocating 14k in one go */ /* FIXME: the size will be derived from configuration */ const size_t buf_size = 28 * 1024; + struct vregion *vr = vregion_create(buf_size); - return vregion_create(buf_size); + if (!vr) + return NULL; + +#ifdef CONFIG_SOF_USERSPACE_LL + vregion_mem_info(vr, vreg_size, vreg_start); + + /* + * In the userspace LL case allocations are also performed by the + * userspace IPC thread, which is also the one, executing this syscall + */ + struct k_mem_partition part = { + .start = *vreg_start, + .size = *vreg_size, + .attr = K_MEM_PARTITION_P_RW_U_RW | XTENSA_MMU_CACHED_WB, + }; + int ret = k_mem_domain_add_partition(zephyr_ll_mem_domain(), &part); + + if (ret < 0) { + vregion_put(vr); + return NULL; + } + + part.start = (uintptr_t)sys_cache_uncached_ptr_get((void *)part.start); + part.attr = K_MEM_PARTITION_P_RW_U_RW; + + ret = k_mem_domain_add_partition(zephyr_ll_mem_domain(), &part); + if (ret < 0) { + vregion_put(vr); + return NULL; + } +#else + ARG_UNUSED(vreg_start); + ARG_UNUSED(vreg_size); +#endif + + return vr; +} + +void z_impl_module_adapter_vreg_unmap(const struct mod_alloc_ctx *alloc) +{ +#ifdef CONFIG_SOF_USERSPACE_LL + struct k_mem_partition part = { + .start = alloc->vreg_start, + .size = alloc->vreg_size, + .attr = K_MEM_PARTITION_P_RW_U_RW | XTENSA_MMU_CACHED_WB, + }; + + k_mem_domain_remove_partition(zephyr_ll_mem_domain(), &part); + + part.start = (uintptr_t)sys_cache_uncached_ptr_get((void *)part.start); + part.attr = K_MEM_PARTITION_P_RW_U_RW; + + k_mem_domain_remove_partition(zephyr_ll_mem_domain(), &part); +#else + ARG_UNUSED(alloc); +#endif +} + +void module_adapter_vreg_free(struct mod_alloc_ctx *alloc) +{ + if (vregion_put(alloc->vreg)) + return; + + module_adapter_vreg_unmap(alloc); + + sof_heap_free(alloc->heap, alloc); } static struct processing_module *module_adapter_mem_alloc(const struct comp_driver *drv, @@ -84,11 +150,12 @@ static struct processing_module *module_adapter_mem_alloc(const struct comp_driv */ uint32_t flags = config->proc_domain == COMP_PROCESSING_DOMAIN_DP ? SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT : SOF_MEM_FLAG_USER; - size_t heap_size; + size_t vreg_size; + uintptr_t vreg_start; if (config->proc_domain == COMP_PROCESSING_DOMAIN_DP && IS_ENABLED(CONFIG_SOF_VREGIONS) && IS_ENABLED(CONFIG_USERSPACE) && !IS_ENABLED(CONFIG_SOF_USERSPACE_USE_DRIVER_HEAP)) { - mod_vreg = module_adapter_dp_heap_new(config, &heap_size); + mod_vreg = module_adapter_vreg_new(config, &vreg_start, &vreg_size); if (!mod_vreg) { comp_cl_err(drv, "Failed to allocate DP module heap / vregion"); return NULL; @@ -101,7 +168,6 @@ static struct processing_module *module_adapter_mem_alloc(const struct comp_driv #else mod_heap = drv->user_heap; #endif - heap_size = 0; mod_vreg = NULL; } @@ -125,6 +191,8 @@ static struct processing_module *module_adapter_mem_alloc(const struct comp_driv memset(mod, 0, sizeof(*mod)); alloc->heap = mod_heap; alloc->vreg = mod_vreg; + alloc->vreg_start = vreg_start; + alloc->vreg_size = vreg_size; mod->priv.resources.alloc = alloc; mod_resource_init(mod); @@ -165,6 +233,25 @@ static struct processing_module *module_adapter_mem_alloc(const struct comp_driv return NULL; } +#ifdef CONFIG_USERSPACE +#include +struct vregion *z_vrfy_module_adapter_vreg_new(const struct comp_ipc_config *config, + uintptr_t *vreg_start, size_t *vreg_size) +{ + K_OOPS(K_SYSCALL_MEMORY_WRITE(vreg_start, sizeof(*vreg_start))); + K_OOPS(K_SYSCALL_MEMORY_WRITE(vreg_size, sizeof(*vreg_size))); + K_OOPS(K_SYSCALL_MEMORY_READ(config, sizeof(*config))); + return z_impl_module_adapter_vreg_new(config, vreg_start, vreg_size); +} +#include +void z_vrfy_module_adapter_vreg_unmap(const struct mod_alloc_ctx *alloc) +{ + K_OOPS(K_SYSCALL_MEMORY_READ(alloc, sizeof(*alloc))); + z_impl_module_adapter_vreg_unmap(alloc); +} +#include +#endif + static void module_adapter_mem_free(struct processing_module *mod) { struct mod_alloc_ctx *alloc = mod->priv.resources.alloc; @@ -182,8 +269,7 @@ static void module_adapter_mem_free(struct processing_module *mod) vregion_free(mod_vreg, mod->dev); vregion_free(mod_vreg, mod); - if (!vregion_put(mod_vreg)) - sof_heap_free(alloc->heap, alloc); + module_adapter_vreg_free(alloc); } else { sof_heap_free(mod_heap, mod->dev); sof_heap_free(mod_heap, mod); diff --git a/src/include/sof/audio/module_adapter/module/generic.h b/src/include/sof/audio/module_adapter/module/generic.h index 5450b07aff7f..b7476fc4f1a0 100644 --- a/src/include/sof/audio/module_adapter/module/generic.h +++ b/src/include/sof/audio/module_adapter/module/generic.h @@ -199,11 +199,18 @@ void *z_impl_mod_balloc_align(struct processing_module *mod, size_t size, size_t #endif void mod_resource_init(struct processing_module *mod); void mod_heap_info(struct processing_module *mod, size_t *size, uintptr_t *start); +void module_adapter_vreg_free(struct mod_alloc_ctx *alloc); #if defined(__ZEPHYR__) && defined(CONFIG_SOF_FULL_ZEPHYR_APPLICATION) +__syscall struct vregion *module_adapter_vreg_new(const struct comp_ipc_config *config, + uintptr_t *vreg_start, size_t *vreg_size); +__syscall void module_adapter_vreg_unmap(const struct mod_alloc_ctx *alloc); __syscall void *mod_alloc_ext(struct processing_module *mod, uint32_t flags, size_t size, size_t alignment); __syscall int mod_free(struct processing_module *mod, const void *ptr); #else +struct vregion *z_impl_module_adapter_vreg_new(const struct comp_ipc_config *config, + uintptr_t *vreg_start, size_t *vreg_size); +void z_impl_module_adapter_vreg_unmap(const struct mod_alloc_ctx *alloc); void *z_impl_mod_alloc_ext(struct processing_module *mod, uint32_t flags, size_t size, size_t alignment); int z_impl_mod_free(struct processing_module *mod, const void *ptr); diff --git a/zephyr/include/rtos/alloc.h b/zephyr/include/rtos/alloc.h index b727d1e700fb..d512ccdff2ef 100644 --- a/zephyr/include/rtos/alloc.h +++ b/zephyr/include/rtos/alloc.h @@ -167,6 +167,8 @@ size_t get_shared_buffer_heap_size(void); struct mod_alloc_ctx { struct k_heap *heap; struct vregion *vreg; + uintptr_t vreg_start; + size_t vreg_size; }; /** From 0d94483d208d11d5782e889e331d31363c82c240 Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Tue, 7 Jul 2026 10:50:58 +0200 Subject: [PATCH 081/108] vregion: make 3 vregion API functions syscalls vregion_get(), vregion_put() and vregion_set_interim() should also be callable from the userspace. Make them syscalls. Also remove redundant symbol exporting since the vregion API shouldn't be used directly by LLEXT modules. Signed-off-by: Guennadi Liakhovetski --- src/include/sof/lib/vregion.h | 12 +++--- zephyr/lib/vregion.c | 15 +++---- zephyr/syscall/vregion.c | 78 ++++++++++++++++++++--------------- 3 files changed, 55 insertions(+), 50 deletions(-) diff --git a/src/include/sof/lib/vregion.h b/src/include/sof/lib/vregion.h index cadf2464968a..ada3cb1e475e 100644 --- a/src/include/sof/lib/vregion.h +++ b/src/include/sof/lib/vregion.h @@ -51,7 +51,7 @@ struct vregion *vregion_create(size_t memsize); * * @param[in] vr Pointer to the virtual region instance. */ -void vregion_set_interim(struct vregion *vr); +__syscall void vregion_set_interim(struct vregion *vr); /** * @brief Increment virtual region's user count. @@ -62,7 +62,7 @@ void vregion_set_interim(struct vregion *vr); * @param[in] vr Pointer to the virtual region instance to release. * @return struct vregion* Pointer to the virtual region instance. */ -struct vregion *vregion_get(struct vregion *vr); +__syscall struct vregion *vregion_get(struct vregion *vr); /** * @brief Decrement virtual region's user count or destroy it. @@ -73,7 +73,7 @@ struct vregion *vregion_get(struct vregion *vr); * @param[in] vr Pointer to the virtual region instance to release. * @return struct vregion* Pointer to the virtual region instance or NULL if it has been destroyed. */ -struct vregion *vregion_put(struct vregion *vr); +__syscall struct vregion *vregion_put(struct vregion *vr); /** * @brief Allocate memory from the specified virtual region. @@ -133,6 +133,8 @@ void vregion_info(struct vregion *vr); */ void vregion_mem_info(struct vregion *vr, size_t *size, uintptr_t *start); +#include + #else /* CONFIG_SOF_VREGIONS */ struct vregion { @@ -183,8 +185,4 @@ static inline void vregion_mem_info(struct vregion *vr, size_t *size, uintptr_t } #endif -#if CONFIG_SOF_VREGIONS -#include -#endif - #endif /* __SOF_LIB_VREGION_H__ */ diff --git a/zephyr/lib/vregion.c b/zephyr/lib/vregion.c index ff7d3ef0f98d..b7f1a74eb8a5 100644 --- a/zephyr/lib/vregion.c +++ b/zephyr/lib/vregion.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include @@ -161,12 +162,12 @@ struct vregion *vregion_create(size_t memsize) /* log the new vregion */ LOG_INF("new at base %p size %#zx pages %u metadata at %p", - (void *)vr->base, total_size, pages, (void *)vr); + (void *)vregion_base, total_size, pages, (void *)vr); return vr; } -struct vregion *vregion_get(struct vregion *vr) +struct vregion *z_impl_vregion_get(struct vregion *vr) { if (!vr) return NULL; @@ -184,7 +185,7 @@ struct vregion *vregion_get(struct vregion *vr) * @param[in] vr Pointer to the virtual region instance to release. * @return struct vregion* Pointer to the virtual region instance or NULL if it has been destroyed. */ -struct vregion *vregion_put(struct vregion *vr) +struct vregion *z_impl_vregion_put(struct vregion *vr) { unsigned int use_count; @@ -259,7 +260,7 @@ static void interim_heap_init(struct vregion *vr) vr->lifetime.used = (uint8_t *)vr->lifetime.ptr - (uint8_t *)vr->lifetime.base; } -void vregion_set_interim(struct vregion *vr) +void z_impl_vregion_set_interim(struct vregion *vr) { if (!vr) return; @@ -390,7 +391,6 @@ void z_impl_vregion_free(struct vregion *vr, void *ptr) k_mutex_unlock(&vr->lock); } -EXPORT_SYMBOL(z_impl_vregion_free); /** * @brief Allocate memory from the virtual region. @@ -430,7 +430,6 @@ void *z_impl_vregion_alloc_align(struct vregion *vr, return p; } -EXPORT_SYMBOL(z_impl_vregion_alloc_align); /** * @brief Allocate memory from the virtual region. @@ -442,7 +441,6 @@ void *z_impl_vregion_alloc(struct vregion *vr, size_t size) { return z_impl_vregion_alloc_align(vr, size, 0); } -EXPORT_SYMBOL(z_impl_vregion_alloc); void *z_impl_vregion_alloc_coherent(struct vregion *vr, size_t size) { @@ -457,7 +455,6 @@ void *z_impl_vregion_alloc_coherent(struct vregion *vr, size_t size) return sys_cache_uncached_ptr_get(p); } -EXPORT_SYMBOL(z_impl_vregion_alloc_coherent); void *z_impl_vregion_alloc_coherent_align(struct vregion *vr, size_t size, size_t alignment) { @@ -474,7 +471,6 @@ void *z_impl_vregion_alloc_coherent_align(struct vregion *vr, size_t size, size_ return sys_cache_uncached_ptr_get(p); } -EXPORT_SYMBOL(z_impl_vregion_alloc_coherent_align); /** * @brief Log virtual region memory usage. @@ -491,7 +487,6 @@ void vregion_info(struct vregion *vr) LOG_INF("lifetime used %#zx free count %d", vr->lifetime.used, vr->lifetime.free_count); } -EXPORT_SYMBOL(vregion_info); void vregion_mem_info(struct vregion *vr, size_t *size, uintptr_t *start) { diff --git a/zephyr/syscall/vregion.c b/zephyr/syscall/vregion.c index 70fb038eba05..027c69394a18 100644 --- a/zephyr/syscall/vregion.c +++ b/zephyr/syscall/vregion.c @@ -6,8 +6,11 @@ #include #include -static inline void *z_vrfy_vregion_alloc(struct vregion *vr, size_t size) +static bool vregion_verify(struct vregion *vr) { + if (!vr) + return false; + size_t vr_size = 0; uintptr_t vr_start; @@ -15,59 +18,68 @@ static inline void *z_vrfy_vregion_alloc(struct vregion *vr, size_t size) if (vr_size) K_OOPS(K_SYSCALL_MEMORY_WRITE((void *)vr_start, vr_size)); - return z_impl_vregion_alloc(vr, size); + return true; +} + +static inline void *z_vrfy_vregion_alloc(struct vregion *vr, size_t size) +{ + if (vregion_verify(vr)) + return z_impl_vregion_alloc(vr, size); + return NULL; } #include static inline void *z_vrfy_vregion_alloc_coherent(struct vregion *vr, size_t size) { - size_t vr_size = 0; - uintptr_t vr_start; - - vregion_mem_info(vr, &vr_size, &vr_start); - if (vr_size) - K_OOPS(K_SYSCALL_MEMORY_WRITE((void *)vr_start, vr_size)); - - return z_impl_vregion_alloc_coherent(vr, size); + if (vregion_verify(vr)) + return z_impl_vregion_alloc_coherent(vr, size); + return NULL; } #include static inline void *z_vrfy_vregion_alloc_align(struct vregion *vr, size_t size, size_t alignment) { - size_t vr_size = 0; - uintptr_t vr_start; - - vregion_mem_info(vr, &vr_size, &vr_start); - if (vr_size) - K_OOPS(K_SYSCALL_MEMORY_WRITE((void *)vr_start, vr_size)); - - return z_impl_vregion_alloc_align(vr, size, alignment); + if (vregion_verify(vr)) + return z_impl_vregion_alloc_align(vr, size, alignment); + return NULL; } #include static inline void *z_vrfy_vregion_alloc_coherent_align(struct vregion *vr, size_t size, size_t alignment) { - size_t vr_size = 0; - uintptr_t vr_start; - - vregion_mem_info(vr, &vr_size, &vr_start); - if (vr_size) - K_OOPS(K_SYSCALL_MEMORY_WRITE((void *)vr_start, vr_size)); - - return z_impl_vregion_alloc_coherent_align(vr, size, alignment); + if (vregion_verify(vr)) + return z_impl_vregion_alloc_coherent_align(vr, size, alignment); + return NULL; } #include static inline void z_vrfy_vregion_free(struct vregion *vr, void *ptr) { - size_t vr_size = 0; - uintptr_t vr_start; + if (vregion_verify(vr)) + z_impl_vregion_free(vr, ptr); +} +#include - vregion_mem_info(vr, &vr_size, &vr_start); - if (vr_size) - K_OOPS(K_SYSCALL_MEMORY_WRITE((void *)vr_start, vr_size)); +struct vregion *z_vrfy_vregion_get(struct vregion *vr) +{ + if (vregion_verify(vr)) + return z_impl_vregion_get(vr); + return NULL; +} +#include - z_impl_vregion_free(vr, ptr); +struct vregion *z_vrfy_vregion_put(struct vregion *vr) +{ + if (vregion_verify(vr)) + return z_impl_vregion_put(vr); + return NULL; } -#include +#include + +void z_vrfy_vregion_set_interim(struct vregion *vr) +{ + if (vregion_verify(vr)) + z_impl_vregion_set_interim(vr); +} +#include From 0741aeead7e88dbf67a83f2db3247b677d4aaea7 Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Tue, 7 Jul 2026 12:01:52 +0200 Subject: [PATCH 082/108] schedule: dp: call directly instead of a notifier scheduler_dp_ll_tick() is currently registered as a notifier callback, but it's always triggered deterministically, always with the same-core-only flag, which leads to it being called immediately. So the notifier only adds a layer of indirection and reduces clarity. Replace it with a direct function call. Signed-off-by: Guennadi Liakhovetski --- src/include/sof/schedule/dp_schedule.h | 1 + src/schedule/zephyr_dp_schedule.c | 31 +++++++------------ src/schedule/zephyr_dp_schedule.h | 2 +- src/schedule/zephyr_dp_schedule_application.c | 4 +-- src/schedule/zephyr_dp_schedule_thread.c | 4 +-- src/schedule/zephyr_ll.c | 8 ++--- 6 files changed, 19 insertions(+), 31 deletions(-) diff --git a/src/include/sof/schedule/dp_schedule.h b/src/include/sof/schedule/dp_schedule.h index 2267d676fb8a..d4225d741930 100644 --- a/src/include/sof/schedule/dp_schedule.h +++ b/src/include/sof/schedule/dp_schedule.h @@ -78,6 +78,7 @@ int scheduler_dp_task_init(struct task **task, uint16_t core, size_t stack_size, uint32_t options); +void scheduler_dp_ll_tick(void); /** * \brief Extract information about scheduler's tasks diff --git a/src/schedule/zephyr_dp_schedule.c b/src/schedule/zephyr_dp_schedule.c index 25fa8b319457..7beb739d1b09 100644 --- a/src/schedule/zephyr_dp_schedule.c +++ b/src/schedule/zephyr_dp_schedule.c @@ -22,7 +22,6 @@ #include #include #include -#include #include #include "zephyr_dp_schedule.h" @@ -223,19 +222,19 @@ static enum task_state scheduler_dp_ll_tick_dummy(void *data) * needed 1.2ms for processing - but the example would be too complicated) */ -void scheduler_dp_ll_tick(void *receiver_data, enum notify_id event_type, void *caller_data) +void scheduler_dp_ll_tick(void) { - (void)receiver_data; - (void)event_type; - (void)caller_data; unsigned int lock_key; struct scheduler_dp_data *dp_sch = scheduler_get_data(SOF_SCHEDULE_DP); + if (!dp_sch) + return; + /* remember current timestamp as "NOW" */ dp_sch->last_ll_tick_timestamp = k_cycle_get_32(); lock_key = scheduler_dp_lock(cpu_get_id()); - scheduler_dp_recalculate(dp_sch, event_type == NOTIFIER_ID_LL_POST_RUN); + scheduler_dp_recalculate(dp_sch); scheduler_dp_unlock(lock_key); } @@ -347,10 +346,9 @@ static struct scheduler_ops schedule_dp_ops = { .schedule_task_free = scheduler_dp_task_free, }; +/* Runs on each core */ __cold int scheduler_dp_init(void) { - int ret; - assert_can_be_cold(); struct scheduler_dp_data *dp_sch = rzalloc(SOF_MEM_FLAG_KERNEL, @@ -364,18 +362,11 @@ __cold int scheduler_dp_init(void) scheduler_init(SOF_SCHEDULE_DP, &schedule_dp_ops, dp_sch); /* init src of DP tick */ - ret = schedule_task_init_ll(&dp_sch->ll_tick_src, - SOF_UUID(dp_sched_uuid), - SOF_SCHEDULE_LL_TIMER, - 0, scheduler_dp_ll_tick_dummy, dp_sch, - cpu_get_id(), 0); - - if (ret) - return ret; - - notifier_register(NULL, NULL, NOTIFIER_ID_LL_POST_RUN, scheduler_dp_ll_tick, 0); - - return 0; + return schedule_task_init_ll(&dp_sch->ll_tick_src, + SOF_UUID(dp_sched_uuid), + SOF_SCHEDULE_LL_TIMER, + 0, scheduler_dp_ll_tick_dummy, dp_sch, + cpu_get_id(), 0); } void scheduler_get_task_info_dp(struct scheduler_props *scheduler_props, uint32_t *data_off_size) diff --git a/src/schedule/zephyr_dp_schedule.h b/src/schedule/zephyr_dp_schedule.h index c4f37fc812f2..694bb541f87e 100644 --- a/src/schedule/zephyr_dp_schedule.h +++ b/src/schedule/zephyr_dp_schedule.h @@ -52,7 +52,7 @@ struct task_dp_pdata { #endif }; -void scheduler_dp_recalculate(struct scheduler_dp_data *dp_sch, bool is_ll_post_run); +void scheduler_dp_recalculate(struct scheduler_dp_data *dp_sch); void dp_thread_fn(void *p1, void *p2, void *p3); unsigned int scheduler_dp_lock(uint16_t core); void scheduler_dp_unlock(unsigned int key); diff --git a/src/schedule/zephyr_dp_schedule_application.c b/src/schedule/zephyr_dp_schedule_application.c index e00972bd467b..187f37db7409 100644 --- a/src/schedule/zephyr_dp_schedule_application.c +++ b/src/schedule/zephyr_dp_schedule_application.c @@ -197,7 +197,7 @@ int scheduler_dp_thread_ipc(struct processing_module *pmod, unsigned int cmd, /* Go through all DP tasks and recalculate their readiness and deadlines * NOT REENTRANT, called with scheduler_dp_lock() held */ -void scheduler_dp_recalculate(struct scheduler_dp_data *dp_sch, bool is_ll_post_run) +void scheduler_dp_recalculate(struct scheduler_dp_data *dp_sch) { struct list_item *tlist; struct task *curr_task; @@ -210,7 +210,7 @@ void scheduler_dp_recalculate(struct scheduler_dp_data *dp_sch, bool is_ll_post_ bool trigger_task = false; /* decrease number of LL ticks/cycles left till the module reaches its deadline */ - if (mod->dp_startup_delay && is_ll_post_run && pdata->ll_cycles_to_start) { + if (mod->dp_startup_delay && pdata->ll_cycles_to_start) { pdata->ll_cycles_to_start--; if (!pdata->ll_cycles_to_start) /* delayed start complete, clear startup delay flag. diff --git a/src/schedule/zephyr_dp_schedule_thread.c b/src/schedule/zephyr_dp_schedule_thread.c index 7fbb3b6a5c21..6cda7f6f2d9a 100644 --- a/src/schedule/zephyr_dp_schedule_thread.c +++ b/src/schedule/zephyr_dp_schedule_thread.c @@ -30,7 +30,7 @@ extern struct tr_ctx dp_tr; /* Go through all DP tasks and recalculate their readiness and deadlines * NOT REENTRANT, should be called with scheduler_dp_lock() */ -void scheduler_dp_recalculate(struct scheduler_dp_data *dp_sch, bool is_ll_post_run) +void scheduler_dp_recalculate(struct scheduler_dp_data *dp_sch) { struct list_item *tlist; struct task *curr_task; @@ -43,7 +43,7 @@ void scheduler_dp_recalculate(struct scheduler_dp_data *dp_sch, bool is_ll_post_ bool trigger_task = false; /* decrease number of LL ticks/cycles left till the module reaches its deadline */ - if (mod->dp_startup_delay && is_ll_post_run && pdata->ll_cycles_to_start) { + if (mod->dp_startup_delay && pdata->ll_cycles_to_start) { pdata->ll_cycles_to_start--; if (!pdata->ll_cycles_to_start) /* delayed start complete, clear startup delay flag. diff --git a/src/schedule/zephyr_ll.c b/src/schedule/zephyr_ll.c index 0e128c011465..ede25670a74a 100644 --- a/src/schedule/zephyr_ll.c +++ b/src/schedule/zephyr_ll.c @@ -10,7 +10,7 @@ #include #include #include -#include +#include #include #include #include @@ -344,11 +344,7 @@ static void zephyr_ll_run(void *data) zephyr_ll_unlock(sch, &flags); -#ifndef CONFIG_SOF_USERSPACE_LL - /* TODO: to be replaced with direct function calls */ - notifier_event(sch, NOTIFIER_ID_LL_POST_RUN, - NOTIFIER_TARGET_CORE_LOCAL, NULL, 0); -#endif + scheduler_dp_ll_tick(); } static void schedule_ll_callback(void *data) From 4f1891df533cf3395a95403e3da2ee2180e5ea3f Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Tue, 7 Jul 2026 12:45:00 +0200 Subject: [PATCH 083/108] schedule: userspace: DP: make scheduler_dp_ll_tick() a syscall scheduler_dp_ll_tick(() has to recalculate DP deadlines and reschedule DP threads. Make it a syscall to be able to call it from the userspace LL scheduler. Signed-off-by: Guennadi Liakhovetski --- src/include/sof/schedule/dp_schedule.h | 6 +++++- src/schedule/zephyr_dp_schedule.c | 17 +++++++++++++++-- src/schedule/zephyr_ll.c | 2 +- zephyr/CMakeLists.txt | 1 + 4 files changed, 22 insertions(+), 4 deletions(-) diff --git a/src/include/sof/schedule/dp_schedule.h b/src/include/sof/schedule/dp_schedule.h index d4225d741930..16be82ec4f76 100644 --- a/src/include/sof/schedule/dp_schedule.h +++ b/src/include/sof/schedule/dp_schedule.h @@ -78,7 +78,11 @@ int scheduler_dp_task_init(struct task **task, uint16_t core, size_t stack_size, uint32_t options); -void scheduler_dp_ll_tick(void); + +#if defined(__ZEPHYR__) && CONFIG_SOF_FULL_ZEPHYR_APPLICATION +__syscall void scheduler_dp_ll_tick(unsigned int core); +#include +#endif /** * \brief Extract information about scheduler's tasks diff --git a/src/schedule/zephyr_dp_schedule.c b/src/schedule/zephyr_dp_schedule.c index 7beb739d1b09..bdce0c589b92 100644 --- a/src/schedule/zephyr_dp_schedule.c +++ b/src/schedule/zephyr_dp_schedule.c @@ -222,10 +222,14 @@ static enum task_state scheduler_dp_ll_tick_dummy(void *data) * needed 1.2ms for processing - but the example would be too complicated) */ -void scheduler_dp_ll_tick(void) +void z_impl_scheduler_dp_ll_tick(unsigned int core) { unsigned int lock_key; +#ifndef CONFIG_SOF_USERSPACE_LL struct scheduler_dp_data *dp_sch = scheduler_get_data(SOF_SCHEDULE_DP); +#else + struct scheduler_dp_data *dp_sch = scheduler_get_data_for_core(SOF_SCHEDULE_DP, core); +#endif if (!dp_sch) return; @@ -233,11 +237,20 @@ void scheduler_dp_ll_tick(void) /* remember current timestamp as "NOW" */ dp_sch->last_ll_tick_timestamp = k_cycle_get_32(); - lock_key = scheduler_dp_lock(cpu_get_id()); + lock_key = scheduler_dp_lock(core); scheduler_dp_recalculate(dp_sch); scheduler_dp_unlock(lock_key); } +#ifdef CONFIG_USERSPACE +#include +void z_vrfy_scheduler_dp_ll_tick(unsigned int core) +{ + z_impl_scheduler_dp_ll_tick(core); +} +#include +#endif + #if CONFIG_SOF_USERSPACE_APPLICATION static int scheduler_dp_task_cancel(void *data, struct task *task) { diff --git a/src/schedule/zephyr_ll.c b/src/schedule/zephyr_ll.c index ede25670a74a..17cbbe6b2137 100644 --- a/src/schedule/zephyr_ll.c +++ b/src/schedule/zephyr_ll.c @@ -344,7 +344,7 @@ static void zephyr_ll_run(void *data) zephyr_ll_unlock(sch, &flags); - scheduler_dp_ll_tick(); + scheduler_dp_ll_tick(sch->core); } static void schedule_ll_callback(void *data) diff --git a/zephyr/CMakeLists.txt b/zephyr/CMakeLists.txt index 482d7d675a9f..caf8a12beb15 100644 --- a/zephyr/CMakeLists.txt +++ b/zephyr/CMakeLists.txt @@ -631,6 +631,7 @@ zephyr_syscall_header(${SOF_SRC_PATH}/include/sof/lib/vregion.h) zephyr_library_sources(syscall/vregion.c) zephyr_syscall_header(${SOF_SRC_PATH}/include/sof/lib/dai-zephyr.h) zephyr_syscall_header(${SOF_SRC_PATH}/include/sof/lib_manager.h) +zephyr_syscall_header(${SOF_SRC_PATH}/include/sof/schedule/dp_schedule.h) zephyr_library_sources_ifdef(CONFIG_USERSPACE syscall/dai.c) zephyr_syscall_header(${SOF_SRC_PATH}/include/user/debug_stream_slot.h) From 1ff13d9737afb27e72c8b8d0e21a9e31528d55aa Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Tue, 7 Jul 2026 12:53:50 +0200 Subject: [PATCH 084/108] schedule: (cosmetic) clarify arch_*schedulers_get() Both arch_schedulers_get() and arch_user_schedulers_get() should only be called from the privileged context. Signed-off-by: Guennadi Liakhovetski --- zephyr/schedule.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zephyr/schedule.c b/zephyr/schedule.c index afc53995610f..3874b28ff1f6 100644 --- a/zephyr/schedule.c +++ b/zephyr/schedule.c @@ -33,8 +33,7 @@ static APP_SYSUSER_BSS struct schedulers *_u_schedulers[CONFIG_CORE_COUNT]; */ struct schedulers **arch_schedulers_get(void) { -#if CONFIG_SOF_USERSPACE_LL - /* user-space callers must use arch_user_schedulers_get() */ +#ifdef __ZEPHYR__ assert(!k_is_user_context()); #endif return _k_schedulers + cpu_get_id(); @@ -53,6 +52,7 @@ EXPORT_SYMBOL(arch_schedulers_get); struct schedulers **arch_user_schedulers_get(void) { #ifdef CONFIG_SOF_USERSPACE_LL + assert(!k_is_user_context()); return _u_schedulers + cpu_get_id(); #else return NULL; From 27a5a64735ec276eee233422a3ca9113cc2edbf4 Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Tue, 7 Jul 2026 17:03:02 +0200 Subject: [PATCH 085/108] schedule: dp: userspace: make scheduler_dp_internal_free() a syscall Make scheduler_dp_internal_free() a syscall in the "application" DP implementation. Signed-off-by: Guennadi Liakhovetski --- src/include/sof/schedule/dp_schedule.h | 6 +++ src/schedule/zephyr_dp_schedule.h | 4 -- src/schedule/zephyr_dp_schedule_application.c | 41 ++++++++++++++++++- 3 files changed, 46 insertions(+), 5 deletions(-) diff --git a/src/include/sof/schedule/dp_schedule.h b/src/include/sof/schedule/dp_schedule.h index 16be82ec4f76..d4463ba2c7ec 100644 --- a/src/include/sof/schedule/dp_schedule.h +++ b/src/include/sof/schedule/dp_schedule.h @@ -80,6 +80,12 @@ int scheduler_dp_task_init(struct task **task, uint32_t options); #if defined(__ZEPHYR__) && CONFIG_SOF_FULL_ZEPHYR_APPLICATION +#if CONFIG_SOF_USERSPACE_APPLICATION +__syscall void scheduler_dp_internal_free(struct task *task); +#else +void scheduler_dp_internal_free(struct task *task); +#endif + __syscall void scheduler_dp_ll_tick(unsigned int core); #include #endif diff --git a/src/schedule/zephyr_dp_schedule.h b/src/schedule/zephyr_dp_schedule.h index 694bb541f87e..2119e44c9dfc 100644 --- a/src/schedule/zephyr_dp_schedule.h +++ b/src/schedule/zephyr_dp_schedule.h @@ -57,7 +57,3 @@ void dp_thread_fn(void *p1, void *p2, void *p3); unsigned int scheduler_dp_lock(uint16_t core); void scheduler_dp_unlock(unsigned int key); void scheduler_dp_grant(k_tid_t thread_id, uint16_t core); -int scheduler_dp_task_init(struct task **task, const struct sof_uuid_entry *uid, - const struct task_ops *ops, struct processing_module *mod, - uint16_t core, size_t stack_size, uint32_t options); -void scheduler_dp_internal_free(struct task *task); diff --git a/src/schedule/zephyr_dp_schedule_application.c b/src/schedule/zephyr_dp_schedule_application.c index 187f37db7409..9aed4313882f 100644 --- a/src/schedule/zephyr_dp_schedule_application.c +++ b/src/schedule/zephyr_dp_schedule_application.c @@ -8,6 +8,7 @@ #include #include +#include #include #include #include @@ -400,7 +401,7 @@ struct scheduler_dp_task_memory { struct ipc4_flat flat; }; -void scheduler_dp_internal_free(struct task *task) +void z_impl_scheduler_dp_internal_free(struct task *task) { struct task_dp_pdata *pdata = task->priv_data; @@ -617,3 +618,41 @@ int scheduler_dp_task_init(struct task **task, const struct sof_uuid_entry *uid, mod_free(mod, task_memory); return ret; } + +#ifdef CONFIG_USERSPACE +#include + +static void scheduler_dp_mod_vrfy(struct processing_module *mod) +{ + K_OOPS(K_SYSCALL_MEMORY_WRITE(mod, sizeof(*mod))); + K_OOPS(K_SYSCALL_MEMORY_WRITE(mod->dev, sizeof(*mod->dev))); + K_OOPS(K_SYSCALL_MEMORY_READ(mod->dev->drv, sizeof(*mod->dev->drv))); + + struct mod_alloc_ctx *alloc = mod->priv.resources.alloc; + + assert(alloc); + if (alloc->heap) { + size_t h_size = 0; + uintptr_t h_start; + + mod_heap_info(mod, &h_size, &h_start); + if (h_size) + K_OOPS(K_SYSCALL_MEMORY_WRITE(h_start, h_size)); + } + if (alloc->vreg) + K_OOPS(K_SYSCALL_MEMORY_WRITE(alloc->vreg_start, alloc->vreg_size)); +} + +void z_vrfy_scheduler_dp_internal_free(struct task *task) +{ + K_OOPS(K_SYSCALL_MEMORY_WRITE(task, sizeof(*task))); + + struct task_dp_pdata *pdata = task->priv_data; + + K_OOPS(K_SYSCALL_OBJ(pdata->event, K_OBJ_EVENT)); + K_OOPS(K_SYSCALL_OBJ_INIT(pdata->thread, K_OBJ_THREAD)); + scheduler_dp_mod_vrfy(pdata->mod); + return z_impl_scheduler_dp_internal_free(task); +} +#include +#endif From df8141d952842cb7b7ccaa21597fc965be4209b3 Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Wed, 8 Jul 2026 15:22:17 +0200 Subject: [PATCH 086/108] schedule: dp: userspace grant IPC thread rights on DP assets In case of userspace LL scheduling the (also userspace) IPC thread needs access rights to DP assets like the thread itself and its stack and synchronisation primitives. Signed-off-by: Guennadi Liakhovetski --- src/include/sof/ipc/common.h | 5 +++++ src/ipc/ipc-common.c | 8 ++++++++ src/schedule/zephyr_dp_schedule_application.c | 14 +++++++++++--- 3 files changed, 24 insertions(+), 3 deletions(-) diff --git a/src/include/sof/ipc/common.h b/src/include/sof/ipc/common.h index 87edc75abef1..2a0b6aa57d72 100644 --- a/src/include/sof/ipc/common.h +++ b/src/include/sof/ipc/common.h @@ -331,4 +331,9 @@ int ipc_user_forward_cmd(struct ipc4_message_request *ipc4, unsigned int core); int ipc_user_init_secondary(unsigned int core); #endif +/** + * \brief get pointer to the userspace IPC thread for core + */ +struct k_thread *ipc_thread_user(unsigned int core); + #endif /* __SOF_DRIVERS_IPC_H__ */ diff --git a/src/ipc/ipc-common.c b/src/ipc/ipc-common.c index 3d36c77ae798..a6132c2c6bae 100644 --- a/src/ipc/ipc-common.c +++ b/src/ipc/ipc-common.c @@ -712,6 +712,14 @@ __cold int ipc_user_init_secondary(unsigned int core) return 0; } +struct k_thread *ipc_thread_user(unsigned int core) +{ + struct ipc *ipc = ipc_get(); + struct ipc_user *ipc_user = ipc->ipc_user_pdata; + + return ipc_user->thread[core]; +} + /* Primary core only */ __cold static int ipc_user_init(void) { diff --git a/src/schedule/zephyr_dp_schedule_application.c b/src/schedule/zephyr_dp_schedule_application.c index 9aed4313882f..ba96580d4a84 100644 --- a/src/schedule/zephyr_dp_schedule_application.c +++ b/src/schedule/zephyr_dp_schedule_application.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -156,6 +157,7 @@ int scheduler_dp_thread_ipc(struct processing_module *pmod, unsigned int cmd, const union scheduler_dp_thread_ipc_param *param) { struct task_dp_pdata *pdata = pmod->dev->task->priv_data; + unsigned int core = pmod->dev->task->core; int ret; if (!pmod) { @@ -165,14 +167,14 @@ int scheduler_dp_thread_ipc(struct processing_module *pmod, unsigned int cmd, if (cmd == SOF_IPC4_MOD_INIT_INSTANCE) { /* Wait for the DP thread to start */ - ret = k_sem_take(&dp_sync[pmod->dev->task->core], DP_THREAD_IPC_TIMEOUT); + ret = k_sem_take(&dp_sync[core], DP_THREAD_IPC_TIMEOUT); if (ret < 0) { tr_err(&dp_tr, "Failed waiting for DP thread to start: %d", ret); return ret; } } - unsigned int lock_key = scheduler_dp_lock(pmod->dev->task->core); + unsigned int lock_key = scheduler_dp_lock(core); /* IPCs are serialised */ pdata->flat->ret = -ENOSYS; @@ -185,7 +187,7 @@ int scheduler_dp_thread_ipc(struct processing_module *pmod, unsigned int cmd, if (!ret) { /* Wait for completion */ - ret = k_sem_take(&dp_sync[cpu_get_id()], DP_THREAD_IPC_TIMEOUT); + ret = k_sem_take(&dp_sync[core], DP_THREAD_IPC_TIMEOUT); if (ret < 0) tr_err(&dp_tr, "Failed waiting for DP thread: %d", ret); else @@ -529,6 +531,12 @@ int scheduler_dp_task_init(struct task **task, const struct sof_uuid_entry *uid, k_thread_access_grant(pdata->thread_id, pdata->event, &dp_sync[core]); scheduler_dp_grant(pdata->thread_id, core); +#if CONFIG_SOF_USERSPACE_LL + struct k_thread *thread_ipc = ipc_thread_user(core); + + k_thread_access_grant(thread_ipc, pdata->event, pdata->thread_id, p_stack, &dp_sync[core]); + scheduler_dp_grant(thread_ipc, core); +#endif struct k_mem_domain *mdom = objpool_alloc(&dp_mdom_head, sizeof(*mdom), SOF_MEM_FLAG_COHERENT); From c63182850113afd3b1ca5337eaa9f984f15b1402 Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Wed, 8 Jul 2026 16:08:02 +0200 Subject: [PATCH 087/108] schedule: ll: fix non-userspace build zephyr_ll_domain() and scheduler_get_task_info_ll() are needed in non-userspace builds too, make them universally available. Signed-off-by: Guennadi Liakhovetski --- src/schedule/zephyr_ll.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/schedule/zephyr_ll.c b/src/schedule/zephyr_ll.c index 17cbbe6b2137..61402b71c14b 100644 --- a/src/schedule/zephyr_ll.c +++ b/src/schedule/zephyr_ll.c @@ -803,14 +803,17 @@ __cold int zephyr_ll_scheduler_init(struct ll_schedule_domain *domain) return 0; } -#if CONFIG_SOF_USERSPACE_LL void scheduler_get_task_info_ll(struct scheduler_props *scheduler_props, uint32_t *data_off_size) { uint32_t flags; scheduler_props->processing_domain = COMP_PROCESSING_DOMAIN_LL; +#if CONFIG_SOF_USERSPACE_LL struct zephyr_ll *ll_sch = scheduler_get_data_for_core(SOF_SCHEDULE_LL_TIMER, cpu_get_id()); +#else + struct zephyr_ll *ll_sch = scheduler_get_data(SOF_SCHEDULE_LL_TIMER); +#endif zephyr_ll_lock(ll_sch, &flags); scheduler_get_task_info(scheduler_props, data_off_size, &ll_sch->tasks); @@ -820,8 +823,11 @@ void scheduler_get_task_info_ll(struct scheduler_props *scheduler_props, /* Return a pointer to the LL scheduler timer domain */ struct ll_schedule_domain *zephyr_ll_domain(void) { +#if CONFIG_SOF_USERSPACE_LL struct zephyr_ll *ll_sch = scheduler_get_data_for_core(SOF_SCHEDULE_LL_TIMER, cpu_get_id()); +#else + struct zephyr_ll *ll_sch = scheduler_get_data(SOF_SCHEDULE_LL_TIMER); +#endif - return ll_sch->ll_domain; + return ll_sch ? ll_sch->ll_domain : NULL; } -#endif From 6feb026c51530ba205ec66c952b51c4546316cc2 Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Wed, 8 Jul 2026 16:17:54 +0200 Subject: [PATCH 088/108] schedule: ll: userspace: grant the LL thread rights on DP The LL userspace thread has to interact with the DP one. Grant required rights. Signed-off-by: Guennadi Liakhovetski --- src/include/sof/schedule/ll_schedule_domain.h | 2 ++ src/schedule/zephyr_domain.c | 14 ++++++++++++++ src/schedule/zephyr_dp_schedule_application.c | 1 + src/schedule/zephyr_ll.c | 12 ++++++++++++ 4 files changed, 29 insertions(+) diff --git a/src/include/sof/schedule/ll_schedule_domain.h b/src/include/sof/schedule/ll_schedule_domain.h index f356566ca0aa..f45084ade521 100644 --- a/src/include/sof/schedule/ll_schedule_domain.h +++ b/src/include/sof/schedule/ll_schedule_domain.h @@ -329,6 +329,8 @@ struct ll_schedule_domain *zephyr_domain_init(int clk); #ifdef CONFIG_SOF_USERSPACE_LL struct k_thread *zephyr_domain_thread_tid(struct ll_schedule_domain *domain); struct k_mem_domain *zephyr_ll_mem_domain(void); +struct ll_schedule_domain *zephyr_ll_domain_for_core(unsigned int core); +struct k_thread *zephyr_ll_domain_thread(unsigned int core); #endif /* CONFIG_SOF_USERSPACE_LL */ #endif /* __ZEPHYR__ */ diff --git a/src/schedule/zephyr_domain.c b/src/schedule/zephyr_domain.c index 89defd4f49f9..81ef0e1b498f 100644 --- a/src/schedule/zephyr_domain.c +++ b/src/schedule/zephyr_domain.c @@ -498,6 +498,20 @@ struct k_thread *zephyr_domain_thread_tid(struct ll_schedule_domain *domain) return dt->ll_thread; } +struct k_thread *zephyr_ll_domain_thread(unsigned int core) +{ + struct ll_schedule_domain *ll_domain = zephyr_ll_domain_for_core(core); + + if (!ll_domain) + return NULL; + + struct zephyr_domain *zephyr_domain = ll_sch_domain_get_pdata(ll_domain); + + struct zephyr_domain_thread *dt = zephyr_domain->domain_thread + core; + + return dt->ll_thread; +} + #endif /* CONFIG_SOF_USERSPACE_LL */ #if CONFIG_CROSS_CORE_STREAM diff --git a/src/schedule/zephyr_dp_schedule_application.c b/src/schedule/zephyr_dp_schedule_application.c index ba96580d4a84..1f796c14cc3e 100644 --- a/src/schedule/zephyr_dp_schedule_application.c +++ b/src/schedule/zephyr_dp_schedule_application.c @@ -536,6 +536,7 @@ int scheduler_dp_task_init(struct task **task, const struct sof_uuid_entry *uid, k_thread_access_grant(thread_ipc, pdata->event, pdata->thread_id, p_stack, &dp_sync[core]); scheduler_dp_grant(thread_ipc, core); + scheduler_dp_grant(zephyr_ll_domain_thread(core), core); #endif struct k_mem_domain *mdom = objpool_alloc(&dp_mdom_head, sizeof(*mdom), diff --git a/src/schedule/zephyr_ll.c b/src/schedule/zephyr_ll.c index 61402b71c14b..c12e0cb36b52 100644 --- a/src/schedule/zephyr_ll.c +++ b/src/schedule/zephyr_ll.c @@ -831,3 +831,15 @@ struct ll_schedule_domain *zephyr_ll_domain(void) return ll_sch ? ll_sch->ll_domain : NULL; } + +/* Return a pointer to the LL scheduler timer domain */ +struct ll_schedule_domain *zephyr_ll_domain_for_core(unsigned int core) +{ +#if CONFIG_SOF_USERSPACE_LL + struct zephyr_ll *ll_sch = scheduler_get_data_for_core(SOF_SCHEDULE_LL_TIMER, core); +#else + struct zephyr_ll *ll_sch = scheduler_get_data(SOF_SCHEDULE_LL_TIMER); +#endif + + return ll_sch ? ll_sch->ll_domain : NULL; +} From 4017b09ef6852d816c94748fe32b751b1a6b70cd Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Thu, 9 Jul 2026 12:04:43 +0200 Subject: [PATCH 089/108] schedule: dp: make compatible with userspace LL DP scheduler operations, instance data and DP module memory have to be accessible to the userspace LL scheduler. Allocate dynamic data on the userspace heap and place static data in the userspace accessible ELF section. Signed-off-by: Guennadi Liakhovetski --- src/audio/module_adapter/module_adapter.c | 4 ++++ src/schedule/zephyr_dp_schedule.c | 6 +++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/audio/module_adapter/module_adapter.c b/src/audio/module_adapter/module_adapter.c index f46709abb1b1..a7499bb87498 100644 --- a/src/audio/module_adapter/module_adapter.c +++ b/src/audio/module_adapter/module_adapter.c @@ -160,7 +160,11 @@ static struct processing_module *module_adapter_mem_alloc(const struct comp_driv comp_cl_err(drv, "Failed to allocate DP module heap / vregion"); return NULL; } +#ifdef CONFIG_SOF_USERSPACE_LL + mod_heap = sof_sys_user_heap_get(); +#else mod_heap = NULL; +#endif } else { #ifdef CONFIG_SOF_USERSPACE_LL mod_heap = sof_sys_user_heap_get(); diff --git a/src/schedule/zephyr_dp_schedule.c b/src/schedule/zephyr_dp_schedule.c index bdce0c589b92..8b2b24f31349 100644 --- a/src/schedule/zephyr_dp_schedule.c +++ b/src/schedule/zephyr_dp_schedule.c @@ -349,7 +349,7 @@ static int scheduler_dp_task_shedule(void *data, struct task *task, uint64_t sta return 0; } -static struct scheduler_ops schedule_dp_ops = { +APP_SYSUSER_DATA static struct scheduler_ops schedule_dp_ops = { .schedule_task = scheduler_dp_task_shedule, #if CONFIG_SOF_USERSPACE_APPLICATION .schedule_task_cancel = scheduler_dp_task_cancel, @@ -364,8 +364,8 @@ __cold int scheduler_dp_init(void) { assert_can_be_cold(); - struct scheduler_dp_data *dp_sch = rzalloc(SOF_MEM_FLAG_KERNEL, - sizeof(struct scheduler_dp_data)); + struct scheduler_dp_data *dp_sch = sof_heap_alloc(sof_sys_user_heap_get(), + SOF_MEM_FLAG_KERNEL, sizeof(*dp_sch), 0); if (!dp_sch) return -ENOMEM; From 3dcb43ce8a84bd945176180643969fccdc2e2cb5 Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Thu, 9 Jul 2026 12:14:35 +0200 Subject: [PATCH 090/108] ipc: remove a duplicate header Remove duplicate ll_schedule_domain.h inclusion in ipc-common.c. Signed-off-by: Guennadi Liakhovetski --- src/ipc/ipc-common.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/ipc/ipc-common.c b/src/ipc/ipc-common.c index a6132c2c6bae..3a43afd91baa 100644 --- a/src/ipc/ipc-common.c +++ b/src/ipc/ipc-common.c @@ -44,7 +44,6 @@ #ifdef CONFIG_SOF_USERSPACE_LL #include -#include #include #include #include From 57188dfd00b48f03395a22ac0e47d8860b71eea1 Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Thu, 9 Jul 2026 12:16:24 +0200 Subject: [PATCH 091/108] ipc: add a comment to explain DP flow Switching to the userspace mode in DP and LL cases differs. Add a comment to explain that. Signed-off-by: Guennadi Liakhovetski --- src/ipc/ipc4/handler-user.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/ipc/ipc4/handler-user.c b/src/ipc/ipc4/handler-user.c index b02360b4e9d1..76ce4e6ef89b 100644 --- a/src/ipc/ipc4/handler-user.c +++ b/src/ipc/ipc4/handler-user.c @@ -1570,6 +1570,11 @@ __cold int ipc4_user_process_module_message(struct ipc4_message_request *ipc4, mi = (struct ipc4_module_init_instance *)ipc4; if (mi->extension.r.proc_domain || !IS_ENABLED(CONFIG_SOF_USERSPACE_LL)) { + /* + * DP module creation starts running in kernel mode and + * switches to userspace later via a scheduler_dp_thread_ipc() + * call in module_init(). + */ ret = ipc4_init_module_instance(ipc4); } else { #ifdef CONFIG_SOF_USERSPACE_LL From baa3237efd657fe42ed69b916cf2c430a81d93c9 Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Thu, 16 Jul 2026 14:14:10 +0200 Subject: [PATCH 092/108] schedule: fix regression with DRAM execution debugging Fix a NULL dereference when a __cold function is called before schedulers are initilised on that core and CONFIG_COLD_STORE_EXECUTE_DEBUG=y and assertions are enabled. Signed-off-by: Guennadi Liakhovetski --- src/include/sof/schedule/schedule.h | 3 +++ src/schedule/zephyr_domain.c | 3 ++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/include/sof/schedule/schedule.h b/src/include/sof/schedule/schedule.h index 940df24fc310..df45f38f8fa1 100644 --- a/src/include/sof/schedule/schedule.h +++ b/src/include/sof/schedule/schedule.h @@ -197,6 +197,9 @@ static inline void *scheduler_get_data(uint16_t type) struct schedule_data *sch; struct list_item *slist; + if (!schedulers) + return NULL; + list_for_item(slist, &schedulers->list) { sch = container_of(slist, struct schedule_data, list); if (type == sch->type) diff --git a/src/schedule/zephyr_domain.c b/src/schedule/zephyr_domain.c index 81ef0e1b498f..69b09a92e55f 100644 --- a/src/schedule/zephyr_domain.c +++ b/src/schedule/zephyr_domain.c @@ -623,7 +623,8 @@ struct ll_schedule_domain *zephyr_domain_init(int clk) /* Check if currently running in the LL scheduler thread context */ bool ll_sch_is_current(void) { - struct zephyr_domain *zephyr_domain = ll_sch_domain_get_pdata(zephyr_ll_domain()); + struct ll_schedule_domain *ll_dom = zephyr_ll_domain(); + struct zephyr_domain *zephyr_domain = ll_dom ? ll_sch_domain_get_pdata(ll_dom) : NULL; if (!zephyr_domain) return false; From 31e1e7a483d9594d8154a32ffe18e0d7c186700e Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Fri, 31 Jul 2026 14:35:38 +0200 Subject: [PATCH 093/108] llext: adjust for MMU flag application The .cold section contains executable code, while .coldrodata contains data. Therefore their MMU flags have to be different, particularly when mapping for userspace. Add a page-size alignment to account for such cases. Signed-off-by: Guennadi Liakhovetski --- scripts/llext_link_helper.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/llext_link_helper.py b/scripts/llext_link_helper.py index f10777c9918f..3b6c2d451e7c 100755 --- a/scripts/llext_link_helper.py +++ b/scripts/llext_link_helper.py @@ -144,8 +144,7 @@ def main(): # critical functions in a .cold ELF section, read-only data in a # .coldrodata ELF section, etc. When compiling and linking such # functions, an additional .cold.literal section is automatically - # created. Note, that for some reason the compiler also marks .cold as - # executable. + # created. # This script links those sections at address 0. We could hard-code # section names, but so far we choose to only link .text the "original" # way and all other executable sections we link at 0. For data sections @@ -168,6 +167,8 @@ def main(): dram_addr += section.header['sh_size'] + dram_addr = align_up(dram_addr, 0x1000) + for section in readonly_dram: s_alignment = section.header['sh_addralign'] s_name = section.name From dc9ed05d29d356b685dc51e95ab94d1341d8fd9b Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Mon, 3 Aug 2026 09:16:23 +0200 Subject: [PATCH 094/108] ipc: don't attempt to add missing cold sections Not all SOF builds have non-zero size .cold and .coldrodata sections even if cold data and execution are enabled. Only add respective memory regions to the userspace domain if they have non-zero size. Signed-off-by: Guennadi Liakhovetski --- src/ipc/ipc-common.c | 51 ++++++++++++++++++++++---------------------- 1 file changed, 26 insertions(+), 25 deletions(-) diff --git a/src/ipc/ipc-common.c b/src/ipc/ipc-common.c index 3a43afd91baa..f5eca61d8e77 100644 --- a/src/ipc/ipc-common.c +++ b/src/ipc/ipc-common.c @@ -744,34 +744,35 @@ __cold static int ipc_user_init(void) * reference __cold functions and __cold_rodata data. */ #ifdef CONFIG_COLD_STORE_EXECUTE_DRAM - { - extern char __cold_start[], __cold_end[]; - extern char __coldrodata_start[]; - extern char _imr_end[]; - struct k_mem_partition cold_part; - - cold_part.start = (uintptr_t)__cold_start; - cold_part.size = ALIGN_UP((uintptr_t)__cold_end - - (uintptr_t)__cold_start, - CONFIG_MMU_PAGE_SIZE); - cold_part.attr = K_MEM_PARTITION_P_RX_U_RX; - ret = k_mem_domain_add_partition(zephyr_ll_mem_domain(), - &cold_part); + extern char __cold_start[], __cold_end[]; + extern char __coldrodata_start[]; + extern char _imr_end[]; + + if (&__cold_end[0] > &__cold_start[0]) { + struct k_mem_partition cold_part = { + .start = (uintptr_t)__cold_start, + .size = ALIGN_UP((uintptr_t)__cold_end - (uintptr_t)__cold_start, + CONFIG_MMU_PAGE_SIZE), + .attr = K_MEM_PARTITION_P_RX_U_RX, + }; + + ret = k_mem_domain_add_partition(zephyr_ll_mem_domain(), &cold_part); if (ret < 0) LOG_WRN("cold text partition add failed: %d", ret); + } - cold_part.start = (uintptr_t)__coldrodata_start; - cold_part.size = ALIGN_UP((uintptr_t)_imr_end - - (uintptr_t)__coldrodata_start, - CONFIG_MMU_PAGE_SIZE); - cold_part.attr = K_MEM_PARTITION_P_RO_U_RO; - if (cold_part.size && cold_part.start) { - ret = k_mem_domain_add_partition(zephyr_ll_mem_domain(), - &cold_part); - if (ret < 0) - LOG_WRN("cold rodata partition %#zx @ %#lx add failed: %d", - cold_part.size, cold_part.start, ret); - } + if (&_imr_end[0] > &__coldrodata_start[0]) { + struct k_mem_partition cold_part = { + .start = (uintptr_t)__coldrodata_start, + .size = ALIGN_UP((uintptr_t)_imr_end - (uintptr_t)__coldrodata_start, + CONFIG_MMU_PAGE_SIZE), + .attr = K_MEM_PARTITION_P_RO_U_RO, + }; + + ret = k_mem_domain_add_partition(zephyr_ll_mem_domain(), &cold_part); + if (ret < 0) + LOG_WRN("cold rodata partition %#zx @ %#lx add failed: %d", + cold_part.size, cold_part.start, ret); } #endif From 8f3afdb6b2eb75c523dfbb25ec78f557c8fd4262 Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Mon, 3 Aug 2026 09:26:48 +0200 Subject: [PATCH 095/108] ipc: (cosmetic) compact several lines Join several needlessly broken lines. Signed-off-by: Guennadi Liakhovetski --- src/ipc/ipc-common.c | 8 +++----- src/ipc/ipc-helper.c | 3 +-- src/ipc/ipc4/helper.c | 9 +++------ 3 files changed, 7 insertions(+), 13 deletions(-) diff --git a/src/ipc/ipc-common.c b/src/ipc/ipc-common.c index f5eca61d8e77..fec6ff57a9b3 100644 --- a/src/ipc/ipc-common.c +++ b/src/ipc/ipc-common.c @@ -578,8 +578,7 @@ static void ipc_user_thread_fn(void *p1, void *p2, void *p3) break; case SOF_IPC4_MOD_LARGE_CONFIG_SET: ipc_user->result = - ipc4_process_large_config_set( - &msg); + ipc4_process_large_config_set(&msg); break; default: LOG_ERR("IPC user: unsupported module cmd type %d", @@ -599,9 +598,8 @@ static void ipc_user_thread_fn(void *p1, void *p2, void *p3) struct ipc4_pipeline_delete *pipe = (struct ipc4_pipeline_delete *)&msg; ipc_user->result = - ipc_pipeline_free( - ipc_user->ipc, - pipe->primary.r.instance_id); + ipc_pipeline_free(ipc_user->ipc, + pipe->primary.r.instance_id); break; } case SOF_IPC4_GLB_SET_PIPELINE_STATE: diff --git a/src/ipc/ipc-helper.c b/src/ipc/ipc-helper.c index cbd17d00da4c..4bbe6489175e 100644 --- a/src/ipc/ipc-helper.c +++ b/src/ipc/ipc-helper.c @@ -304,8 +304,7 @@ __cold int ipc_comp_free(struct ipc *ipc, uint32_t comp_id) /* check whether component exists */ icd = ipc_get_comp_by_id(ipc, comp_id); if (!icd) { - tr_err(&ipc_tr, "comp id: 0x%x is not found", - comp_id); + tr_err(&ipc_tr, "comp id: 0x%x is not found", comp_id); return -ENODEV; } diff --git a/src/ipc/ipc4/helper.c b/src/ipc/ipc4/helper.c index 2e146f80d478..48be6915918f 100644 --- a/src/ipc/ipc4/helper.c +++ b/src/ipc/ipc4/helper.c @@ -1291,13 +1291,10 @@ __cold static const struct comp_driver *ipc4_search_for_drv(const void *uuid) /* search driver list with UUID */ list_for_item(clist, &drivers->list) { - info = container_of(clist, struct comp_driver_info, - list); + info = container_of(clist, struct comp_driver_info, list); if (!memcmp(info->drv->uid, uuid, UUID_SIZE)) { - tr_dbg(&comp_tr, - "found type %d, uuid %pU", - info->drv->type, - info->drv->tctx->uuid_p); + tr_dbg(&comp_tr, "found type %d, uuid %pU", + info->drv->type, info->drv->tctx->uuid_p); drv = info->drv; break; } From 08744c7b0d86b1f98f04d4c1e87e8da9fd6eed58 Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Mon, 3 Aug 2026 09:46:24 +0200 Subject: [PATCH 096/108] build: fix gcc compilation Remove version.h and autoconf.h inclusion to fix gcc compilation. Also limit scope of a variable to avoid an "unused variable" warning. Signed-off-by: Guennadi Liakhovetski --- src/init/init.c | 1 - src/ipc/ipc-helper.c | 2 ++ src/ipc/ipc-zephyr.c | 2 -- zephyr/lib/alloc.c | 1 - zephyr/lib/cpu.c | 1 - zephyr/wrapper.c | 1 - 6 files changed, 2 insertions(+), 6 deletions(-) diff --git a/src/init/init.c b/src/init/init.c index 8605723a96ff..8451e49b0d15 100644 --- a/src/init/init.c +++ b/src/init/init.c @@ -43,7 +43,6 @@ #include #include #include -#include #endif #include diff --git a/src/ipc/ipc-helper.c b/src/ipc/ipc-helper.c index 4bbe6489175e..07c168b834dc 100644 --- a/src/ipc/ipc-helper.c +++ b/src/ipc/ipc-helper.c @@ -297,7 +297,9 @@ __cold int ipc_comp_free(struct ipc *ipc, uint32_t comp_id) struct comp_buffer *buffer; struct comp_buffer *safe; struct list_item *clist; +#ifndef CONFIG_SOF_USERSPACE_LL uint32_t flags; +#endif assert_can_be_cold(); diff --git a/src/ipc/ipc-zephyr.c b/src/ipc/ipc-zephyr.c index 66ada4ddaa05..e30f9ae739ea 100644 --- a/src/ipc/ipc-zephyr.c +++ b/src/ipc/ipc-zephyr.c @@ -9,8 +9,6 @@ // Andrey Borisovich // Adrian Warecki -#include - #include #include #ifdef CONFIG_INTEL_ADSP_IPC diff --git a/zephyr/lib/alloc.c b/zephyr/lib/alloc.c index 031b31673be8..9a92e896c7df 100644 --- a/zephyr/lib/alloc.c +++ b/zephyr/lib/alloc.c @@ -37,7 +37,6 @@ struct vmh_heap *virtual_buffers_heap; #include #include #include -#include #include #include diff --git a/zephyr/lib/cpu.c b/zephyr/lib/cpu.c index 94f004bd7356..59af03f5f70f 100644 --- a/zephyr/lib/cpu.c +++ b/zephyr/lib/cpu.c @@ -27,7 +27,6 @@ static uint32_t mic_disable_status; #include "../audio/copier/copier.h" /* Zephyr includes */ -#include #include #include #include diff --git a/zephyr/wrapper.c b/zephyr/wrapper.c index 29b8cbdf82fc..8c00f14de7bf 100644 --- a/zephyr/wrapper.c +++ b/zephyr/wrapper.c @@ -33,7 +33,6 @@ #include #include #include -#include #include #include #include From 2d7b90da3ccdefdd555757e768d91f14cd8ec2a2 Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Mon, 3 Aug 2026 09:51:16 +0200 Subject: [PATCH 097/108] (cosmetic) move variable definitions closer to usage Move variable definition closer to where they are used to clarify their scope. Signed-off-by: Guennadi Liakhovetski --- src/ipc/ipc-common.c | 4 ++-- zephyr/edf_schedule.c | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/ipc/ipc-common.c b/src/ipc/ipc-common.c index fec6ff57a9b3..3514ffb43f3f 100644 --- a/src/ipc/ipc-common.c +++ b/src/ipc/ipc-common.c @@ -884,12 +884,12 @@ __cold int ipc_init(struct sof *sof) #ifdef __ZEPHYR__ - struct k_thread *thread = &ipc->ipc_send_wq.thread; - k_work_queue_init(&ipc->ipc_send_wq); k_work_queue_start(&ipc->ipc_send_wq, ipc_send_wq_stack, K_THREAD_STACK_SIZEOF(ipc_send_wq_stack), 1, NULL); + struct k_thread *thread = ipc->ipc_send_wq.thread_id; + k_thread_suspend(thread); #ifdef CONFIG_SCHED_CPU_MASK diff --git a/zephyr/edf_schedule.c b/zephyr/edf_schedule.c index c15f36b7be58..9de172d8693f 100644 --- a/zephyr/edf_schedule.c +++ b/zephyr/edf_schedule.c @@ -102,8 +102,6 @@ static struct scheduler_ops schedule_edf_ops = { __cold int scheduler_init_edf(void) { - struct k_thread *thread = &edf_workq.thread; - assert_can_be_cold(); scheduler_init(SOF_SCHEDULE_EDF, &schedule_edf_ops, NULL); @@ -113,6 +111,8 @@ __cold int scheduler_init_edf(void) K_THREAD_STACK_SIZEOF(edf_workq_stack), CONFIG_EDF_THREAD_PRIORITY, NULL); + struct k_thread *thread = edf_workq.thread_id; + k_thread_suspend(thread); k_thread_heap_assign(thread, sof_sys_heap_get()); From b1fce505268fbf9d5b0bbde9c41144733ac729ad Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Mon, 3 Aug 2026 10:08:15 +0200 Subject: [PATCH 098/108] ipc: (cosmetic) reduce the numner of type-casts Change several pointers from "char *" to "unsigned char *" to reduce the number of type-casts. Signed-off-by: Guennadi Liakhovetski --- src/ipc/ipc4/helper.c | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/src/ipc/ipc4/helper.c b/src/ipc/ipc4/helper.c index 48be6915918f..6aca3fc84e14 100644 --- a/src/ipc/ipc4/helper.c +++ b/src/ipc/ipc4/helper.c @@ -88,24 +88,25 @@ void ipc_build_trace_posn(struct sof_ipc_dma_trace_posn *posn) } #if CONFIG_LIBRARY -static inline char *ipc4_get_comp_new_data(void) +static inline unsigned char *ipc4_get_comp_new_data(void) { struct ipc *ipc = ipc_get(); - char *data = (char *)ipc->comp_data + sizeof(struct ipc4_module_init_instance); + unsigned char *data = (unsigned char *)ipc->comp_data + + sizeof(struct ipc4_module_init_instance); return data; } -static const struct comp_driver *ipc4_library_get_comp_drv(char *data) +static const struct comp_driver *ipc4_library_get_comp_drv(unsigned char *data) { return ipc4_get_drv(data); } #else -__cold static inline char *ipc4_get_comp_new_data(void) +__cold static inline unsigned char *ipc4_get_comp_new_data(void) { assert_can_be_cold(); - return (char *)MAILBOX_HOSTBOX_BASE; + return (unsigned char *)MAILBOX_HOSTBOX_BASE; } #endif @@ -116,7 +117,7 @@ __cold struct comp_dev *comp_new_ipc4(const struct ipc4_module_init_instance *mo const struct comp_driver *drv; struct comp_dev *dev; uint32_t comp_id; - char *data; + unsigned char *data; assert_can_be_cold(); @@ -174,13 +175,13 @@ __cold struct comp_dev *comp_new_ipc4(const struct ipc4_module_init_instance *mo if (drv->type == SOF_COMP_MODULE_ADAPTER) { const struct ipc_config_process spec = { - .data = (const unsigned char *)data, + .data = data, .size = ipc_config.ipc_config_size, }; - dev = drv->ops.create(drv, &ipc_config, (const void *)&spec); + dev = drv->ops.create(drv, &ipc_config, &spec); } else { - dev = drv->ops.create(drv, &ipc_config, (const void *)data); + dev = drv->ops.create(drv, &ipc_config, data); } if (!dev) return NULL; @@ -225,7 +226,7 @@ __cold struct comp_dev *comp_new_ipc4_user(struct ipc4_message_request *ipc4, struct comp_ipc_config ipc_config; struct comp_dev *dev; uint32_t comp_id; - char *data; + unsigned char *data; int ret; assert_can_be_cold(); @@ -283,13 +284,13 @@ __cold struct comp_dev *comp_new_ipc4_user(struct ipc4_message_request *ipc4, if (drv->type == SOF_COMP_MODULE_ADAPTER) { const struct ipc_config_process spec = { - .data = (const unsigned char *)data, + .data = data, .size = ipc_config.ipc_config_size, }; - dev = drv->ops.create(drv, &ipc_config, (const void *)&spec); + dev = drv->ops.create(drv, &ipc_config, &spec); } else { - dev = drv->ops.create(drv, &ipc_config, (const void *)data); + dev = drv->ops.create(drv, &ipc_config, data); } if (!dev) return NULL; From 0bc3737fe5b5c7b9289b61c561ff5b5e8c27e2f7 Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Mon, 3 Aug 2026 10:18:58 +0200 Subject: [PATCH 099/108] component: use a local UUID copy for LLEXT instances The .uid field in struct comp_driver is used for driver identification using its unique UUID. However with LLEXT that UUID is located in DRAM, which makes access to it difficult from userspace threads. Make a local copy of it instead for reliable driver searching from different contexts. Signed-off-by: Guennadi Liakhovetski --- src/include/sof/audio/component.h | 1 + src/library_manager/lib_manager.c | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/include/sof/audio/component.h b/src/include/sof/audio/component.h index db8a69e63646..928590f43e2a 100644 --- a/src/include/sof/audio/component.h +++ b/src/include/sof/audio/component.h @@ -593,6 +593,7 @@ struct comp_driver { * Currently used by module_adapter. */ struct k_heap *user_heap; /**< Userspace heap */ + struct sof_uuid uid_cp; /**< UUID copy for LLEXT modules */ }; /** \brief Holds constant pointer to component driver */ diff --git a/src/library_manager/lib_manager.c b/src/library_manager/lib_manager.c index dd30c3db73a9..7492a225656d 100644 --- a/src/library_manager/lib_manager.c +++ b/src/library_manager/lib_manager.c @@ -799,7 +799,8 @@ static void lib_manager_module_free(struct comp_dev *dev) static void lib_manager_prepare_module_adapter(struct comp_driver *drv, const struct sof_uuid *uuid) { drv->type = SOF_COMP_MODULE_ADAPTER; - drv->uid = uuid; + drv->uid_cp = *uuid; + drv->uid = &drv->uid_cp; drv->tctx = &lib_manager_tr; drv->ops.create = lib_manager_module_create; drv->ops.prepare = module_adapter_prepare; From ea11914fa384e2d3f47a802cc02e309f5f442f43 Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Mon, 3 Aug 2026 10:48:07 +0200 Subject: [PATCH 100/108] ipc: ipc4: extract common code into a function Extract common initialisation code from comp_new_ipc4_user() and comp_new_ipc4() into a new function. Signed-off-by: Guennadi Liakhovetski --- src/ipc/ipc4/helper.c | 119 +++++++++++++++++------------------------- 1 file changed, 48 insertions(+), 71 deletions(-) diff --git a/src/ipc/ipc4/helper.c b/src/ipc/ipc4/helper.c index 6aca3fc84e14..d57bc492c196 100644 --- a/src/ipc/ipc4/helper.c +++ b/src/ipc/ipc4/helper.c @@ -110,41 +110,64 @@ __cold static inline unsigned char *ipc4_get_comp_new_data(void) } #endif -/* Only called from ipc4_init_module_instance(), which is __cold */ -__cold struct comp_dev *comp_new_ipc4(const struct ipc4_module_init_instance *module_init) +__cold static int ipc4_comp_new_config(struct comp_ipc_config *ipc_config, + const struct ipc4_module_init_instance *module_init) { - struct comp_ipc_config ipc_config; - const struct comp_driver *drv; - struct comp_dev *dev; - uint32_t comp_id; - unsigned char *data; + uint32_t comp_id = IPC4_COMP_ID(module_init->primary.r.module_id, + module_init->primary.r.instance_id); assert_can_be_cold(); - comp_id = IPC4_COMP_ID(module_init->primary.r.module_id, - module_init->primary.r.instance_id); - if (ipc4_get_comp_dev(comp_id)) { tr_err(&ipc_tr, "comp 0x%x exists", comp_id); - return NULL; + return -EEXIST; } if (module_init->extension.r.core_id >= CONFIG_CORE_COUNT) { tr_err(&ipc_tr, "ipc: comp->core = %u", (uint32_t)module_init->extension.r.core_id); - return NULL; + return -EINVAL; } - memset(&ipc_config, 0, sizeof(ipc_config)); - ipc_config.id = comp_id; - ipc_config.pipeline_id = module_init->extension.r.ppl_instance_id; - ipc_config.core = module_init->extension.r.core_id; - ipc_config.ipc_config_size = module_init->extension.r.param_block_size * sizeof(uint32_t); - ipc_config.ipc_extended_init = module_init->extension.r.extended_init; - if (ipc_config.ipc_config_size > MAILBOX_HOSTBOX_SIZE) { + memset(ipc_config, 0, sizeof(*ipc_config)); + ipc_config->id = comp_id; + ipc_config->pipeline_id = module_init->extension.r.ppl_instance_id; + ipc_config->core = module_init->extension.r.core_id; + ipc_config->ipc_config_size = module_init->extension.r.param_block_size * sizeof(uint32_t); + ipc_config->ipc_extended_init = module_init->extension.r.extended_init; + if (ipc_config->ipc_config_size > MAILBOX_HOSTBOX_SIZE) { tr_err(&ipc_tr, "IPC payload size %u too big for the message window", - ipc_config.ipc_config_size); - return NULL; + ipc_config->ipc_config_size); + return -ENOSPC; } + if (!module_init->extension.r.proc_domain) { + ipc_config->proc_domain = COMP_PROCESSING_DOMAIN_LL; + } else if (IS_ENABLED(CONFIG_ZEPHYR_DP_SCHEDULER)) { + ipc_config->proc_domain = COMP_PROCESSING_DOMAIN_DP; + } else { + tr_err(&ipc_tr, "ipc: DP scheduling is disabled, cannot create comp 0x%x", comp_id); + return -EINVAL; + } + + return 0; +} + +/* Only called from ipc4_init_module_instance(), which is __cold */ +__cold struct comp_dev *comp_new_ipc4(const struct ipc4_module_init_instance *module_init) +{ + struct comp_ipc_config ipc_config; + const struct comp_driver *drv; + struct comp_dev *dev; + unsigned char *data; + uint32_t comp_id = IPC4_COMP_ID(module_init->primary.r.module_id, + module_init->primary.r.instance_id); + + assert_can_be_cold(); + + int ret = ipc4_comp_new_config(&ipc_config, module_init); + + if (ret < 0) + return NULL; + #ifdef CONFIG_DCACHE_LINE_SIZE if (!IS_ENABLED(CONFIG_LIBRARY)) sys_cache_data_invd_range((__sparse_force void __sparse_cache *) @@ -162,17 +185,6 @@ __cold struct comp_dev *comp_new_ipc4(const struct ipc4_module_init_instance *mo if (!drv) return NULL; - if (!module_init->extension.r.proc_domain) { - ipc_config.proc_domain = COMP_PROCESSING_DOMAIN_LL; - } else if (IS_ENABLED(CONFIG_ZEPHYR_DP_SCHEDULER)) { - ipc_config.proc_domain = COMP_PROCESSING_DOMAIN_DP; - } else { - tr_err(&ipc_tr, - "ipc: DP scheduling is disabled, cannot create comp 0x%x", - comp_id); - return NULL; - } - if (drv->type == SOF_COMP_MODULE_ADAPTER) { const struct ipc_config_process spec = { .data = data, @@ -225,43 +237,19 @@ __cold struct comp_dev *comp_new_ipc4_user(struct ipc4_message_request *ipc4, struct ipc4_module_init_instance module_init; struct comp_ipc_config ipc_config; struct comp_dev *dev; - uint32_t comp_id; unsigned char *data; - int ret; assert_can_be_cold(); - ret = memcpy_s(&module_init, sizeof(module_init), ipc4, sizeof(*ipc4)); - if (ret < 0) - return NULL; - - comp_id = IPC4_COMP_ID(module_init.primary.r.module_id, - module_init.primary.r.instance_id); + int ret = memcpy_s(&module_init, sizeof(module_init), ipc4, sizeof(*ipc4)); - if (ipc4_get_comp_dev(comp_id)) { - tr_err(&ipc_tr, "comp 0x%x exists", comp_id); + if (ret < 0) return NULL; - } - if (module_init.extension.r.core_id >= CONFIG_CORE_COUNT) { - tr_err(&ipc_tr, "ipc: comp->core = %u", - (uint32_t)module_init.extension.r.core_id); + ret = ipc4_comp_new_config(&ipc_config, &module_init); + if (ret < 0) return NULL; - } - memset(&ipc_config, 0, sizeof(ipc_config)); - ipc_config.id = comp_id; - ipc_config.pipeline_id = module_init.extension.r.ppl_instance_id; - ipc_config.core = module_init.extension.r.core_id; - ipc_config.ipc_config_size = - module_init.extension.r.param_block_size * sizeof(uint32_t); - ipc_config.ipc_extended_init = module_init.extension.r.extended_init; - if (ipc_config.ipc_config_size > MAILBOX_HOSTBOX_SIZE) { - tr_err(&ipc_tr, - "IPC payload size %u too big for the message window", - ipc_config.ipc_config_size); - return NULL; - } #ifdef CONFIG_DCACHE_LINE_SIZE if (!IS_ENABLED(CONFIG_LIBRARY)) sys_cache_data_invd_range( @@ -271,17 +259,6 @@ __cold struct comp_dev *comp_new_ipc4_user(struct ipc4_message_request *ipc4, #endif data = ipc4_get_comp_new_data(); - if (!module_init.extension.r.proc_domain) { - ipc_config.proc_domain = COMP_PROCESSING_DOMAIN_LL; - } else if (IS_ENABLED(CONFIG_ZEPHYR_DP_SCHEDULER)) { - ipc_config.proc_domain = COMP_PROCESSING_DOMAIN_DP; - } else { - tr_err(&ipc_tr, - "ipc: DP scheduling is disabled, cannot create comp 0x%x", - comp_id); - return NULL; - } - if (drv->type == SOF_COMP_MODULE_ADAPTER) { const struct ipc_config_process spec = { .data = data, From 19531d0e4f2fa9b397e89a16ce30075f2f3251fe Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Mon, 3 Aug 2026 13:47:41 +0200 Subject: [PATCH 101/108] llext: add a check to llext_manager_mod_find() Add a check to llext_manager_mod_find() in case scanning the array reached the last element, that the index indeed is within that element's range. Return an error otherwise. Signed-off-by: Guennadi Liakhovetski --- src/include/sof/lib_manager.h | 1 + src/library_manager/llext_manager.c | 30 ++++++++++++++++++++++++----- 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/src/include/sof/lib_manager.h b/src/include/sof/lib_manager.h index 1056656191c7..4be9c28f5e34 100644 --- a/src/include/sof/lib_manager.h +++ b/src/include/sof/lib_manager.h @@ -114,6 +114,7 @@ struct lib_manager_module { struct llext *llext; /* Zephyr loadable extension context */ struct llext_buf_loader *ebl; /* Zephyr loadable extension buffer loader */ unsigned int n_dependent; /* For auxiliary modules: number of dependents */ + unsigned int n_mod; bool mapped; bool domain_dp; struct lib_manager_segment_desc segment[LIB_MANAGER_N_SEGMENTS]; diff --git a/src/library_manager/llext_manager.c b/src/library_manager/llext_manager.c index 0defc94a7d1f..dfebdd66375a 100644 --- a/src/library_manager/llext_manager.c +++ b/src/library_manager/llext_manager.c @@ -470,6 +470,7 @@ static int llext_manager_link(const char *name, if (ret >= 0) { llext_get_section_info(ldr, *llext, ret, &hdr, NULL, NULL); *mod_manifest = llext_peek(ldr, hdr->sh_offset); + mctx->n_mod = hdr->sh_size / sizeof(struct sof_man_module_manifest); } return *buildinfo && *mod_manifest ? 0 : -EPROTO; @@ -530,7 +531,7 @@ static int llext_manager_mod_init(struct lib_manager_mod_ctx *ctx, } /* Find a module context, containing the driver with the supplied index */ -static unsigned int llext_manager_mod_find(const struct lib_manager_mod_ctx *ctx, unsigned int idx) +static int llext_manager_mod_find(const struct lib_manager_mod_ctx *ctx, unsigned int idx) { unsigned int i; @@ -538,6 +539,9 @@ static unsigned int llext_manager_mod_find(const struct lib_manager_mod_ctx *ctx if (ctx->mod[i].start_idx > idx) break; + if (i == ctx->n_mod && ctx->mod[i].start_idx + ctx->mod[i].n_mod <= idx) + return -ENOENT; + return i - 1; } @@ -559,7 +563,11 @@ static int llext_manager_link_single(uint32_t module_id, const struct sof_man_fw return -EINVAL; } - unsigned int mod_ctx_idx = llext_manager_mod_find(ctx, entry_index); + int mod_ctx_idx = llext_manager_mod_find(ctx, entry_index); + + if (mod_ctx_idx < 0) + return mod_ctx_idx; + struct lib_manager_module *mctx = ctx->mod + mod_ctx_idx; size_t mod_size; int i, inst_idx; @@ -979,7 +987,11 @@ int llext_manager_add_domain(const uint32_t component_id, struct k_mem_domain *d const uint32_t module_id = IPC4_MOD_ID(component_id); struct lib_manager_mod_ctx *ctx = lib_manager_get_mod_ctx(module_id); const uint32_t entry_index = LIB_MANAGER_GET_MODULE_INDEX(module_id); - const unsigned int mod_idx = llext_manager_mod_find(ctx, entry_index); + const int mod_idx = llext_manager_mod_find(ctx, entry_index); + + if (mod_idx < 0) + return mod_idx; + struct lib_manager_module *mctx = ctx->mod + mod_idx; /* FIXME: handle dependencies */ @@ -1060,7 +1072,11 @@ int llext_manager_rm_domain(const uint32_t component_id, struct k_mem_domain *do const uint32_t module_id = IPC4_MOD_ID(component_id); struct lib_manager_mod_ctx *ctx = lib_manager_get_mod_ctx(module_id); const uint32_t entry_index = LIB_MANAGER_GET_MODULE_INDEX(module_id); - const unsigned int mod_idx = llext_manager_mod_find(ctx, entry_index); + const int mod_idx = llext_manager_mod_find(ctx, entry_index); + + if (mod_idx < 0) + return mod_idx; + struct lib_manager_module *mctx = ctx->mod + mod_idx; return llext_manager_rm_mod_domain(mctx, domain); @@ -1085,7 +1101,11 @@ int llext_manager_free_module(const uint32_t component_id) return -ENOENT; } - unsigned int mod_idx = llext_manager_mod_find(ctx, entry_index); + int mod_idx = llext_manager_mod_find(ctx, entry_index); + + if (mod_idx < 0) + return mod_idx; + struct lib_manager_module *mctx = ctx->mod + mod_idx; /* Protected by IPC serialization */ From 25af50833940981829952928da43b92edffbc385 Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Mon, 3 Aug 2026 14:01:11 +0200 Subject: [PATCH 102/108] llext: move 2 functions higher Move llext_manager_add_partition() and llext_manager_rm_partition() higher in the file for future use. No functional change. Signed-off-by: Guennadi Liakhovetski --- src/library_manager/llext_manager.c | 60 ++++++++++++++--------------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/src/library_manager/llext_manager.c b/src/library_manager/llext_manager.c index dfebdd66375a..4c2b546dba5b 100644 --- a/src/library_manager/llext_manager.c +++ b/src/library_manager/llext_manager.c @@ -168,6 +168,36 @@ static int llext_manager_load_data_from_storage(const struct sys_mm_drv_region * return ret; } +static int llext_manager_add_partition(struct k_mem_domain *domain, + uintptr_t addr, size_t size, + k_mem_partition_attr_t attr) +{ + size_t pre_pad_size = addr & (PAGE_SZ - 1); + struct k_mem_partition part = { + .start = addr - pre_pad_size, + .size = ALIGN_UP(pre_pad_size + size, PAGE_SZ), + .attr = attr, + }; + + tr_dbg(&lib_manager_tr, "add %#zx @ %lx partition", part.size, part.start); + return k_mem_domain_add_partition(domain, &part); +} + +static int llext_manager_rm_partition(struct k_mem_domain *domain, + uintptr_t addr, size_t size, + k_mem_partition_attr_t attr) +{ + size_t pre_pad_size = addr & (PAGE_SZ - 1); + struct k_mem_partition part = { + .start = addr - pre_pad_size, + .size = ALIGN_UP(pre_pad_size + size, PAGE_SZ), + .attr = attr, + }; + + tr_dbg(&lib_manager_tr, "remove %#zx @ %lx partition", part.size, part.start); + return k_mem_domain_remove_partition(domain, &part); +} + static void llext_manager_unmap_detached_sections(const struct llext_loader *ldr, const struct llext *ext, enum llext_mem region, @@ -814,36 +844,6 @@ int llext_manager_map_lib(uint32_t comp_id) } #endif -static int llext_manager_add_partition(struct k_mem_domain *domain, - uintptr_t addr, size_t size, - k_mem_partition_attr_t attr) -{ - size_t pre_pad_size = addr & (PAGE_SZ - 1); - struct k_mem_partition part = { - .start = addr - pre_pad_size, - .size = ALIGN_UP(pre_pad_size + size, PAGE_SZ), - .attr = attr, - }; - - tr_dbg(&lib_manager_tr, "add %#zx @ %lx partition", part.size, part.start); - return k_mem_domain_add_partition(domain, &part); -} - -static int llext_manager_rm_partition(struct k_mem_domain *domain, - uintptr_t addr, size_t size, - k_mem_partition_attr_t attr) -{ - size_t pre_pad_size = addr & (PAGE_SZ - 1); - struct k_mem_partition part = { - .start = addr - pre_pad_size, - .size = ALIGN_UP(pre_pad_size + size, PAGE_SZ), - .attr = attr, - }; - - tr_dbg(&lib_manager_tr, "remove %#zx @ %lx partition", part.size, part.start); - return k_mem_domain_remove_partition(domain, &part); -} - static int llext_manager_add_mod_domain(struct lib_manager_module *mctx, struct k_mem_domain *domain) { const struct llext *ext = mctx->llext; From b99676d0c05376c5bfb3f1dff3c63cb1ca259835 Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Mon, 3 Aug 2026 14:48:45 +0200 Subject: [PATCH 103/108] llext: don't access DRAM when freeing With userspace LL enabled module freeing runs in syscall contex on behalf of the userspace IPC thread. That thread doesn't have access to DRAM. Therefore we cannot call lib_manager_get_module_manifest() in that case. Use SRAM module data by calling llext_manager_mod_find() instead. Signed-off-by: Guennadi Liakhovetski --- src/include/sof/llext_manager.h | 3 +++ src/library_manager/lib_manager.c | 9 ++++++--- src/library_manager/llext_manager.c | 2 +- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/include/sof/llext_manager.h b/src/include/sof/llext_manager.h index 080a58d623a4..d83e86a33ebd 100644 --- a/src/include/sof/llext_manager.h +++ b/src/include/sof/llext_manager.h @@ -16,6 +16,7 @@ struct comp_dev; struct comp_driver; struct comp_ipc_config; struct k_mem_domain; +struct lib_manager_mod_ctx; static inline bool module_is_llext(const struct sof_man_module *mod) { @@ -34,6 +35,7 @@ int llext_manager_add_domain(const uint32_t component_id, struct k_mem_domain *d int llext_manager_rm_domain(const uint32_t component_id, struct k_mem_domain *domain); int llext_manager_map_lib(uint32_t comp_id); +int llext_manager_mod_find(const struct lib_manager_mod_ctx *ctx, unsigned int idx); bool comp_is_llext(struct comp_dev *comp); #else #define module_is_llext(mod) false @@ -42,6 +44,7 @@ bool comp_is_llext(struct comp_dev *comp); #define llext_manager_add_library(module_id) 0 #define llext_manager_add_domain(component_id, domain) 0 #define llext_manager_map_lib(component_id) 0 +#define llext_manager_mod_find(ctx, idx) -ENOENT #define comp_is_llext(comp) false #endif diff --git a/src/library_manager/lib_manager.c b/src/library_manager/lib_manager.c index 7492a225656d..c6744cc8bf76 100644 --- a/src/library_manager/lib_manager.c +++ b/src/library_manager/lib_manager.c @@ -423,15 +423,18 @@ int z_impl_lib_manager_free_module(const uint32_t component_id) tr_dbg(&lib_manager_tr, "mod_id: %#x", component_id); + const uint32_t entry_index = LIB_MANAGER_GET_MODULE_INDEX(module_id); + const struct lib_manager_mod_ctx *const ctx = lib_manager_get_mod_ctx(module_id); + + if (llext_manager_mod_find(ctx, entry_index) >= 0) + return llext_manager_free_module(component_id); + mod = lib_manager_get_module_manifest(module_id); if (!mod) { tr_err(&lib_manager_tr, "failed to get module descriptor"); return -EINVAL; } - if (module_is_llext(mod)) - return llext_manager_free_module(component_id); - ret = lib_manager_unload_module(mod); if (ret < 0) return ret; diff --git a/src/library_manager/llext_manager.c b/src/library_manager/llext_manager.c index 4c2b546dba5b..c6e753e8d91d 100644 --- a/src/library_manager/llext_manager.c +++ b/src/library_manager/llext_manager.c @@ -561,7 +561,7 @@ static int llext_manager_mod_init(struct lib_manager_mod_ctx *ctx, } /* Find a module context, containing the driver with the supplied index */ -static int llext_manager_mod_find(const struct lib_manager_mod_ctx *ctx, unsigned int idx) +int llext_manager_mod_find(const struct lib_manager_mod_ctx *ctx, unsigned int idx) { unsigned int i; From ee804372fba387015edfa3ae29a90cb89e986320 Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Mon, 3 Aug 2026 15:09:09 +0200 Subject: [PATCH 104/108] llext: temporarily map ELF headers for userspace when freeing llext_manager_unload_module() is called in a syscall context on behalf of the userspace IPC thread, so it doesn't have direct access to LLEXT module DRAM data. Map section headers temporarily for the duration of the function. Signed-off-by: Guennadi Liakhovetski --- src/library_manager/llext_manager.c | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/library_manager/llext_manager.c b/src/library_manager/llext_manager.c index c6e753e8d91d..72c74013e83c 100644 --- a/src/library_manager/llext_manager.c +++ b/src/library_manager/llext_manager.c @@ -376,6 +376,18 @@ static int llext_manager_unload_module(struct lib_manager_module *mctx) mctx->segment[LIB_MANAGER_BSS].size; int err = 0, ret; + unsigned int sect_cnt = llext_section_count(ext); + size_t total = sect_cnt * sizeof(elf_shdr_t); + const elf_shdr_t *shdr; + + ret = llext_get_section_info(ldr, ext, 0, &shdr, NULL, NULL); + if (ret < 0) + return ret; + + /* Temporarily map ELF section headers */ + llext_manager_add_partition(zephyr_ll_mem_domain(), (uintptr_t)shdr, total, + K_MEM_PARTITION_P_RW_U_NA | XTENSA_MMU_CACHED_WB); + llext_manager_unmap_detached_sections(ldr, ext, LLEXT_MEM_TEXT, va_base_text, text_size); ret = llext_manager_align_unmap(va_base_text, text_size); @@ -400,6 +412,9 @@ static int llext_manager_unload_module(struct lib_manager_module *mctx) if (ret < 0 && !err) err = ret; + llext_manager_rm_partition(zephyr_ll_mem_domain(), (uintptr_t)shdr, total, + K_MEM_PARTITION_P_RW_U_NA | XTENSA_MMU_CACHED_WB); + mctx->mapped = false; #ifdef CONFIG_SOF_USERSPACE_LL From 6fe6623353736c6cff370a27d7c323f91443da43 Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Mon, 3 Aug 2026 15:14:59 +0200 Subject: [PATCH 105/108] llext: avoid DRAM access in userspace when freeing llext_manager_free_module() cannot access DRAM when running with userspace LL enabled. Don't call lib_manager_get_library_manifest() to obtain the DRAM descriptor, needed to verify the entry index. The index is now verified by llext_manager_mod_find(). Signed-off-by: Guennadi Liakhovetski --- src/library_manager/llext_manager.c | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/library_manager/llext_manager.c b/src/library_manager/llext_manager.c index 72c74013e83c..0c25db05e841 100644 --- a/src/library_manager/llext_manager.c +++ b/src/library_manager/llext_manager.c @@ -1101,16 +1101,9 @@ int llext_manager_rm_domain(const uint32_t component_id, struct k_mem_domain *do int llext_manager_free_module(const uint32_t component_id) { const uint32_t module_id = IPC4_MOD_ID(component_id); - struct sof_man_fw_desc *desc = (struct sof_man_fw_desc *)lib_manager_get_library_manifest(module_id); struct lib_manager_mod_ctx *ctx = lib_manager_get_mod_ctx(module_id); uint32_t entry_index = LIB_MANAGER_GET_MODULE_INDEX(module_id); - if (entry_index >= desc->header.num_module_entries) { - tr_err(&lib_manager_tr, "Invalid driver index %u exceeds %d", - entry_index, desc->header.num_module_entries - 1); - return -ENOENT; - } - if (!ctx->mod) { tr_err(&lib_manager_tr, "NULL module array: ID %#x ctx %p", component_id, ctx); return -ENOENT; From 8d16a0dd9c91b46b8c0e9490709bbcad95a2fc8f Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Mon, 3 Aug 2026 16:04:10 +0200 Subject: [PATCH 106/108] userspace: perform library loading in kernel context When running in syscall context on behalf of a userspace thread dynamically mapped memory doesn't automatically become accessible. To make it accessible it has to be added to the thread memory domain. This is a problem for loadable modules with executable cold sections. To be able to execute them they have to be mapped to threads with the executable bit set. While for linking that memory has to be mapped writable. To solve the problem we perform linking from the kernel IPC context before forwarding to the userspace IPC thread. Signed-off-by: Guennadi Liakhovetski --- src/include/ipc4/handler.h | 8 ++++++++ src/ipc/ipc4/handler-user.c | 16 +++++++++------ src/ipc/ipc4/helper.c | 33 +++++++++++++++++++++++++++++++ src/library_manager/lib_manager.c | 14 +++++++++---- 4 files changed, 61 insertions(+), 10 deletions(-) diff --git a/src/include/ipc4/handler.h b/src/include/ipc4/handler.h index 0ed298dd9ad1..9e56dbce78e9 100644 --- a/src/include/ipc4/handler.h +++ b/src/include/ipc4/handler.h @@ -16,6 +16,14 @@ struct ipc4_message_request; */ int ipc4_user_process_module_message(struct ipc4_message_request *ipc4, struct ipc_msg *reply); +/** + * \brief Load a dynamically loadable module. + * @param[in] drv Component driver. + * @param[in] mi SOF_IPC4_MOD_INIT_INSTANCE data + */ +int ipc4_user_module_load(const struct comp_driver *drv, + const struct ipc4_module_init_instance *mi); + /** * @brief Process MOD_CONFIG_GET or MOD_CONFIG_SET in any execution context. * @param[in] ipc4 IPC4 message request. diff --git a/src/ipc/ipc4/handler-user.c b/src/ipc/ipc4/handler-user.c index 76ce4e6ef89b..6e0c0e73feeb 100644 --- a/src/ipc/ipc4/handler-user.c +++ b/src/ipc/ipc4/handler-user.c @@ -1583,7 +1583,6 @@ __cold int ipc4_user_process_module_message(struct ipc4_message_request *ipc4, * Component creation (drv->ops.create) runs in user thread * so untrusted module code does not execute in kernel context. */ - struct ipc *ipc = ipc_get(); uint32_t comp_id = IPC4_COMP_ID(mi->primary.r.module_id, mi->primary.r.instance_id); const struct comp_driver *drv = ipc4_get_comp_drv(IPC4_MOD_ID(comp_id)); @@ -1593,13 +1592,18 @@ __cold int ipc4_user_process_module_message(struct ipc4_message_request *ipc4, break; } - ret = llext_manager_map_lib(comp_id); - if (ret < 0) - break; + struct lib_manager_mod_ctx *ctx = lib_manager_get_mod_ctx(comp_id); - struct ipc_user *pdata = ipc->ipc_user_pdata; + if (ctx && !ctx->user_mapped && drv->type == SOF_COMP_MODULE_ADAPTER) { + int err = ipc4_user_module_load(drv, mi); + + if (err < 0) { + ret = IPC4_MOD_NOT_INITIALIZED; + break; + } + } - pdata->init_drv = drv; + ipc_get()->ipc_user_pdata->init_drv = drv; ret = ipc_user_forward_cmd(ipc4, mi->extension.r.core_id); #endif } diff --git a/src/ipc/ipc4/helper.c b/src/ipc/ipc4/helper.c index d57bc492c196..2b14657a506f 100644 --- a/src/ipc/ipc4/helper.c +++ b/src/ipc/ipc4/helper.c @@ -220,6 +220,39 @@ __cold struct comp_dev *comp_new_ipc4(const struct ipc4_module_init_instance *mo } #ifdef CONFIG_SOF_USERSPACE_LL + +int ipc4_user_module_load(const struct comp_driver *drv, + const struct ipc4_module_init_instance *mi) +{ + /* + * move a part to the kernel thread: + * the userspace IPC handling thread would call comp_new_ipc4_user() to + * then call library manager .create method lib_manager_module_create(). + * That one calls lib_manager_mod_create_priv(), then + * lib_manager_allocate_module() and eventually + * llext_manager_allocate_module() for LLEXT modules. + */ + struct comp_ipc_config ipc_config; + int ret = ipc4_comp_new_config(&ipc_config, mi); + + if (ret < 0) + return ret; + + const struct ipc_config_process spec = { + .data = ipc4_get_comp_new_data(), + .size = ipc_config.ipc_config_size, + }; + +#if CONFIG_DCACHE_LINE_SIZE && !CONFIG_LIBRARY + sys_cache_data_invd_range((__sparse_force void __sparse_cache *)spec.data, spec.size); +#endif + + struct userspace_context *userspace = NULL; + const struct module_interface *ops = NULL; + + return lib_manager_mod_create_priv(drv, &ipc_config, &spec, NULL, &userspace, &ops); +} + /** * comp_new_ipc4_user - Create component in user-space IPC thread context. * diff --git a/src/library_manager/lib_manager.c b/src/library_manager/lib_manager.c index c6744cc8bf76..bba4b97841d6 100644 --- a/src/library_manager/lib_manager.c +++ b/src/library_manager/lib_manager.c @@ -645,6 +645,7 @@ static enum buildinfo_mod_type lib_manager_get_module_type(const struct sof_man_ } } +/* Error path resource freeing */ static void lib_manager_mod_free_priv(const struct comp_driver *drv, const struct comp_ipc_config *config, struct userspace_context *userspace) @@ -766,13 +767,18 @@ static struct comp_dev *lib_manager_module_create(const struct comp_driver *drv, const void *spec) { struct userspace_context *userspace = NULL; - const struct module_interface *ops; + const struct module_interface *ops = NULL; void *adapter_priv = NULL; struct comp_dev *dev; - int ret = lib_manager_mod_create_priv(drv, config, spec, &adapter_priv, &userspace, &ops); - if (ret < 0) - return NULL; + if (config->proc_domain == COMP_PROCESSING_DOMAIN_DP || + !IS_ENABLED(CONFIG_SOF_USERSPACE_LL)) { + int ret = lib_manager_mod_create_priv(drv, config, spec, &adapter_priv, + &userspace, &ops); + + if (ret < 0) + return NULL; + } dev = module_adapter_new_ext(drv, config, spec, adapter_priv, userspace, ops); if (!dev) From bd4b1c4fa3eb1475c8493c14af16dd61702c9274 Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Mon, 3 Aug 2026 16:13:19 +0200 Subject: [PATCH 107/108] llext: remove llext_manager_map_lib() llext_manager_map_lib() isn't needed any more, remove it. Signed-off-by: Guennadi Liakhovetski --- src/include/sof/llext_manager.h | 2 -- src/library_manager/llext_manager.c | 32 ----------------------------- 2 files changed, 34 deletions(-) diff --git a/src/include/sof/llext_manager.h b/src/include/sof/llext_manager.h index d83e86a33ebd..1d5dc96c2562 100644 --- a/src/include/sof/llext_manager.h +++ b/src/include/sof/llext_manager.h @@ -33,7 +33,6 @@ int llext_manager_add_library(uint32_t module_id); int llext_manager_add_domain(const uint32_t component_id, struct k_mem_domain *domain); int llext_manager_rm_domain(const uint32_t component_id, struct k_mem_domain *domain); -int llext_manager_map_lib(uint32_t comp_id); int llext_manager_mod_find(const struct lib_manager_mod_ctx *ctx, unsigned int idx); bool comp_is_llext(struct comp_dev *comp); @@ -43,7 +42,6 @@ bool comp_is_llext(struct comp_dev *comp); #define llext_manager_free_module(component_id) 0 #define llext_manager_add_library(module_id) 0 #define llext_manager_add_domain(component_id, domain) 0 -#define llext_manager_map_lib(component_id) 0 #define llext_manager_mod_find(ctx, idx) -ENOENT #define comp_is_llext(comp) false #endif diff --git a/src/library_manager/llext_manager.c b/src/library_manager/llext_manager.c index 0c25db05e841..cc15c4380100 100644 --- a/src/library_manager/llext_manager.c +++ b/src/library_manager/llext_manager.c @@ -827,38 +827,6 @@ uintptr_t llext_manager_allocate_module(const struct comp_ipc_config *ipc_config } #ifdef CONFIG_USERSPACE -#ifdef CONFIG_SOF_USERSPACE_LL -int llext_manager_map_lib(uint32_t comp_id) -{ - struct lib_manager_mod_ctx *ctx = lib_manager_get_mod_ctx(comp_id); - - if (!ctx || ctx->user_mapped) - return 0; - - const struct sof_man_fw_desc *const desc = lib_manager_get_library_manifest(comp_id); - - if (!desc) - /* Built-in driver */ - return 0; - - size_t preload_size = desc->header.preload_page_count * CONFIG_MM_DRV_PAGE_SIZE; - struct k_mem_partition lib_part = { - .start = (uintptr_t)desc, - .size = preload_size, - .attr = K_MEM_PARTITION_P_RW_U_NA | XTENSA_MMU_CACHED_WB, - }; - - int ret = k_mem_domain_add_partition(zephyr_ll_mem_domain(), &lib_part); - - if (ret < 0) - return ret; - - ctx->user_mapped = true; - - return 0; -} -#endif - static int llext_manager_add_mod_domain(struct lib_manager_module *mctx, struct k_mem_domain *domain) { const struct llext *ext = mctx->llext; From c20c623c6b5a305a3fb66ee7e07298ec198cd28d Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Mon, 3 Aug 2026 16:14:09 +0200 Subject: [PATCH 108/108] userspace-ll: re-enable DRAM execution and data CONFIG_COLD_STORE_EXECUTE_DRAM can now be re-enabled for userspace LL. Signed-off-by: Guennadi Liakhovetski --- app/overlays/ptl/ll_userspace_overlay.conf | 1 - 1 file changed, 1 deletion(-) diff --git a/app/overlays/ptl/ll_userspace_overlay.conf b/app/overlays/ptl/ll_userspace_overlay.conf index 057e5f1a9dbb..83db063deb40 100644 --- a/app/overlays/ptl/ll_userspace_overlay.conf +++ b/app/overlays/ptl/ll_userspace_overlay.conf @@ -17,7 +17,6 @@ CONFIG_DAI_USERSPACE=y # problem with DSP panics due to illegal instruction hit in user-space if cold # store execution is enabled. Disable it for now until rootcause is found. -CONFIG_COLD_STORE_EXECUTE_DRAM=n CONFIG_COLD_STORE_EXECUTE_DEBUG=n # telemetry not yet user-space compatible