From 558805a76fd06cae73e78a25ead1fda30183bc49 Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Wed, 8 Apr 2026 15:12:59 +0200 Subject: [PATCH 01/15] audio: buffer: remove left-over function names from logs Remove several more function names from logging prints. Signed-off-by: Guennadi Liakhovetski --- src/audio/buffers/comp_buffer.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/audio/buffers/comp_buffer.c b/src/audio/buffers/comp_buffer.c index 0c28d9204cd1..e1e4c1867e85 100644 --- a/src/audio/buffers/comp_buffer.c +++ b/src/audio/buffers/comp_buffer.c @@ -146,7 +146,7 @@ static void comp_buffer_free(struct sof_audio_buffer *audio_buffer) struct comp_buffer *buffer = container_of(audio_buffer, struct comp_buffer, audio_buffer); - buf_dbg(buffer, "buffer_free()"); + buf_dbg(buffer, "entry"); #if CONFIG_PROBE if (buffer->probe_cb_free) @@ -198,7 +198,7 @@ static struct comp_buffer *buffer_alloc_struct(struct k_heap *heap, { struct comp_buffer *buffer; - tr_dbg(&buffer_tr, "buffer_alloc_struct()"); + tr_dbg(&buffer_tr, "entry"); /* allocate new buffer, but add coherent if shared with other cores */ if (is_shared) @@ -240,7 +240,7 @@ struct comp_buffer *buffer_alloc(struct k_heap *heap, size_t size, uint32_t flag struct comp_buffer *buffer; void *stream_addr; - tr_dbg(&buffer_tr, "buffer_alloc()"); + tr_dbg(&buffer_tr, "entry"); /* validate request */ if (size == 0) { @@ -310,7 +310,7 @@ struct comp_buffer *buffer_alloc_range(struct k_heap *heap, size_t preferred_siz void buffer_zero(struct comp_buffer *buffer) { - buf_dbg(buffer, "stream_zero()"); + buf_dbg(buffer, "entry"); CORE_CHECK_STRUCT(&buffer->audio_buffer); bzero(audio_stream_get_addr(&buffer->stream), audio_stream_get_size(&buffer->stream)); From 326ca4ca83ab44eea056868a19a045c8da1cdf4b Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Mon, 13 Apr 2026 17:06:19 +0200 Subject: [PATCH 02/15] base-fw: fix a compiler warning, make functions cold Fix an unused function compiler warning and make two functions, called only in "cold" context, "cold" themselves. Signed-off-by: Guennadi Liakhovetski --- src/audio/base_fw_intel.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/audio/base_fw_intel.c b/src/audio/base_fw_intel.c index 38bb7bd7aa4c..ec3a35a9b601 100644 --- a/src/audio/base_fw_intel.c +++ b/src/audio/base_fw_intel.c @@ -109,7 +109,8 @@ static const struct device *uaol_devs[] = { DT_FOREACH_STATUS_OKAY(intel_adsp_uaol, DEV_AND_COMMA) }; -static void tlv_value_set_uaol_caps(struct sof_tlv *tuple, uint32_t type) +#if !CONFIG_SOF_OS_LINUX_COMPAT_PRIORITY +__cold static void tlv_value_set_uaol_caps(struct sof_tlv *tuple, uint32_t type) { const size_t dev_count = ARRAY_SIZE(uaol_devs); struct uaol_capabilities dev_cap; @@ -118,6 +119,8 @@ static void tlv_value_set_uaol_caps(struct sof_tlv *tuple, uint32_t type) size_t i; int ret; + assert_can_be_cold(); + memset(caps, 0, caps_size); caps->link_count = dev_count; @@ -135,12 +138,15 @@ static void tlv_value_set_uaol_caps(struct sof_tlv *tuple, uint32_t type) tlv_value_set(tuple, type, caps_size, caps); } +#endif /* CONFIG_SOF_OS_LINUX_COMPAT_PRIORITY */ -static int uaol_stream_id_to_hda_link_stream_id(int uaol_stream_id) +__cold static int uaol_stream_id_to_hda_link_stream_id(int uaol_stream_id) { size_t dev_count = ARRAY_SIZE(uaol_devs); size_t i; + assert_can_be_cold(); + for (i = 0; i < dev_count; i++) { int hda_link_stream_id = uaol_get_mapped_hda_link_stream_id(uaol_devs[i], uaol_stream_id); From 61e2aa1a5568f0fe2f56fc7d2b312c12b3e03d9f Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Mon, 13 Apr 2026 17:13:37 +0200 Subject: [PATCH 03/15] audio: dai: allocate object on Zephyr heap No need to allocate a DMA configuration object on VMH, allocate it on the common SOF Zephyr heap. Signed-off-by: Guennadi Liakhovetski --- src/audio/dai-zephyr.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/audio/dai-zephyr.c b/src/audio/dai-zephyr.c index 6dda66899c26..80cba02eb8fc 100644 --- a/src/audio/dai-zephyr.c +++ b/src/audio/dai-zephyr.c @@ -869,7 +869,7 @@ static int dai_set_dma_config(struct dai_data *dd, struct comp_dev *dev) comp_dbg(dev, "entry"); - dma_cfg = rballoc(SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT | SOF_MEM_FLAG_DMA, + dma_cfg = rmalloc(SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT | SOF_MEM_FLAG_DMA, sizeof(struct dma_config)); if (!dma_cfg) { comp_err(dev, "dma_cfg allocation failed"); From 3f811ea318fa9251ab7b82fbe19c17c932b8f7d0 Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Mon, 13 Apr 2026 17:19:06 +0200 Subject: [PATCH 04/15] vregion: add coherent allocation functions Add vregion functions for coherent (uncached) memory allocations. This also requires cache-line size alignment where cached and uncached access can be mixed. Signed-off-by: Guennadi Liakhovetski --- zephyr/include/sof/lib/vregion.h | 20 ++++++++++++ zephyr/lib/vregion.c | 52 +++++++++++++++++++++++++++++--- 2 files changed, 68 insertions(+), 4 deletions(-) diff --git a/zephyr/include/sof/lib/vregion.h b/zephyr/include/sof/lib/vregion.h index 135052c51280..f85d028765b0 100644 --- a/zephyr/include/sof/lib/vregion.h +++ b/zephyr/include/sof/lib/vregion.h @@ -58,6 +58,11 @@ enum vregion_mem_type { */ void *vregion_alloc(struct vregion *vr, enum vregion_mem_type type, size_t size); +/** + * @brief like vregion_alloc() but allocates coherent memory + */ +void *vregion_alloc_coherent(struct vregion *vr, enum vregion_mem_type type, size_t size); + /** * @brief Allocate aligned memory from the specified virtual region. * @@ -72,6 +77,12 @@ void *vregion_alloc(struct vregion *vr, enum vregion_mem_type type, size_t size) void *vregion_alloc_align(struct vregion *vr, enum vregion_mem_type type, size_t size, size_t alignment); +/** + * @brief like vregion_alloc_align() but allocates coherent memory + */ +void *vregion_alloc_coherent_align(struct vregion *vr, enum vregion_mem_type type, + size_t size, size_t alignment); + /** * @brief Free memory allocated from the specified virtual region. * @@ -89,6 +100,15 @@ void vregion_free(struct vregion *vr, void *ptr); */ void vregion_info(struct vregion *vr); +/** + * @brief Get virtual region memory start and size. + * + * @param[in] vr Pointer to the virtual region instance. + * @param[in] size Pointer to size + * @param[in] start Pointer to start + */ +void vregion_mem_info(struct vregion *vr, size_t *size, uintptr_t *start); + #ifdef __cplusplus } #endif diff --git a/zephyr/lib/vregion.c b/zephyr/lib/vregion.c index 7ef9b52fe97e..f62809ee9ecd 100644 --- a/zephyr/lib/vregion.c +++ b/zephyr/lib/vregion.c @@ -145,8 +145,9 @@ struct vregion *vregion_create(size_t lifetime_size, size_t interim_size) vr->lifetime.base = vr->base + interim_size; /* set alloc ptr addresses for lifetime linear partitions */ - vr->lifetime.ptr = vr->lifetime.base + sizeof(*vr); /* skip vregion struct */ - vr->lifetime.used = sizeof(*vr); + vr->lifetime.ptr = vr->lifetime.base + + ALIGN_UP(sizeof(*vr), CONFIG_DCACHE_LINE_SIZE); /* skip vregion struct */ + vr->lifetime.used = ALIGN_UP(sizeof(*vr), CONFIG_DCACHE_LINE_SIZE); /* init interim heaps */ k_heap_init(&vr->interim.heap, vr->interim.heap.heap.init_mem, interim_size); @@ -274,6 +275,9 @@ void vregion_free(struct vregion *vr, void *ptr) if (!vr || !ptr) return; + if (sys_cache_is_ptr_uncached(ptr)) + ptr = sys_cache_cached_ptr_get(ptr); + /* check if pointer is in interim heap */ if (ptr >= (void *)vr->interim.heap.heap.init_mem && ptr < (void *)((uint8_t *)vr->interim.heap.heap.init_mem + @@ -309,8 +313,8 @@ void *vregion_alloc_align(struct vregion *vr, enum vregion_mem_type type, if (!vr || !size) return NULL; - if (!alignment) - alignment = 4; /* default align 4 bytes */ + if (alignment < PLATFORM_DCACHE_ALIGN) + alignment = PLATFORM_DCACHE_ALIGN; switch (type) { case VREGION_MEM_TYPE_INTERIM: @@ -337,6 +341,37 @@ void *vregion_alloc(struct vregion *vr, enum vregion_mem_type type, size_t size) } EXPORT_SYMBOL(vregion_alloc); +void *vregion_alloc_coherent(struct vregion *vr, enum vregion_mem_type type, size_t size) +{ + size = ALIGN_UP(size, CONFIG_DCACHE_LINE_SIZE); + + void *p = vregion_alloc_align(vr, type, size, CONFIG_DCACHE_LINE_SIZE); + + if (!p) + return NULL; + + sys_cache_data_invd_range(p, size); + + return sys_cache_uncached_ptr_get(p); +} + +void *vregion_alloc_coherent_align(struct vregion *vr, enum vregion_mem_type type, + size_t size, size_t alignment) +{ + if (alignment < CONFIG_DCACHE_LINE_SIZE) + alignment = CONFIG_DCACHE_LINE_SIZE; + size = ALIGN_UP(size, CONFIG_DCACHE_LINE_SIZE); + + void *p = vregion_alloc_align(vr, type, size, alignment); + + if (!p) + return NULL; + + sys_cache_data_invd_range(p, size); + + return sys_cache_uncached_ptr_get(p); +} + /** * @brief Log virtual region memory usage. * @@ -353,3 +388,12 @@ void vregion_info(struct vregion *vr) vr->lifetime.used, vr->lifetime.free_count); } EXPORT_SYMBOL(vregion_info); + +void vregion_mem_info(struct vregion *vr, size_t *size, uintptr_t *start) +{ + if (size) + *size = vr->size; + + if (start) + *start = (uintptr_t)vr->base; +} From 542138328744d11a7cc8e4acbd3bf7beff6fd435 Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Tue, 14 Apr 2026 10:38:14 +0200 Subject: [PATCH 05/15] vregion: add stubs Add stubs for the vregions API for when CONFIG_SOF_VREGIONS=n. Signed-off-by: Guennadi Liakhovetski --- zephyr/include/sof/lib/vregion.h | 64 +++++++++++++++++++++++++------- 1 file changed, 51 insertions(+), 13 deletions(-) diff --git a/zephyr/include/sof/lib/vregion.h b/zephyr/include/sof/lib/vregion.h index f85d028765b0..55c028797709 100644 --- a/zephyr/include/sof/lib/vregion.h +++ b/zephyr/include/sof/lib/vregion.h @@ -14,6 +14,21 @@ extern "C" { struct vregion; +/** + * @brief Memory types for virtual region allocations. + * Used to specify the type of memory allocation within a virtual region. + * + * @note + * - interim: allocation that can be freed i.e. get/set large config, kcontrols. + * - lifetime: allocation that cannot be freed i.e. init data, pipeline data. + */ +enum vregion_mem_type { + VREGION_MEM_TYPE_INTERIM, /* interim allocation that can be freed */ + VREGION_MEM_TYPE_LIFETIME, /* lifetime allocation */ +}; + +#if CONFIG_SOF_VREGIONS + /** * @brief Create a new virtual region instance. * @@ -35,19 +50,6 @@ struct vregion *vregion_create(size_t lifetime_size, size_t interim_size); */ void vregion_destroy(struct vregion *vr); -/** - * @brief Memory types for virtual region allocations. - * Used to specify the type of memory allocation within a virtual region. - * - * @note - * - interim: allocation that can be freed i.e. get/set large config, kcontrols. - * - lifetime: allocation that cannot be freed i.e. init data, pipeline data. - */ -enum vregion_mem_type { - VREGION_MEM_TYPE_INTERIM, /* interim allocation that can be freed */ - VREGION_MEM_TYPE_LIFETIME, /* lifetime allocation */ -}; - /** * @brief Allocate memory from the specified virtual region. * @@ -109,6 +111,42 @@ void vregion_info(struct vregion *vr); */ void vregion_mem_info(struct vregion *vr, size_t *size, uintptr_t *start); +#else /* CONFIG_SOF_VREGIONS */ + +static inline struct vregion *vregion_create(size_t lifetime_size, size_t interim_size) +{ + return NULL; +} +static inline void vregion_destroy(struct vregion *vr) {} +static inline void *vregion_alloc(struct vregion *vr, enum vregion_mem_type type, size_t size) +{ + return NULL; +} +static inline void *vregion_alloc_coherent(struct vregion *vr, enum vregion_mem_type type, + size_t size) +{ + return NULL; +} +static inline void *vregion_alloc_align(struct vregion *vr, enum vregion_mem_type type, + size_t size, size_t alignment) +{ + return NULL; +} +static inline void *vregion_alloc_coherent_align(struct vregion *vr, enum vregion_mem_type type, + size_t size, size_t alignment) +{ + return NULL; +} +static inline void vregion_free(struct vregion *vr, void *ptr) {} +static inline void vregion_info(struct vregion *vr) {} +static inline void vregion_mem_info(struct vregion *vr, size_t *size, uintptr_t *start) +{ + if (size) + *size = 0; +} + +#endif /* CONFIG_SOF_VREGIONS */ + #ifdef __cplusplus } #endif From 6d2176b2ec4c3782f2298640d6fe19c7019842ee Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Wed, 15 Apr 2026 09:41:48 +0200 Subject: [PATCH 06/15] vregion: move a header We want to have the vregion header accessible to userspace builds like the testbench too. The easiest way to achieve that is by moving it under src/include/... Signed-off-by: Guennadi Liakhovetski --- {zephyr => src}/include/sof/lib/vregion.h | 1 - 1 file changed, 1 deletion(-) rename {zephyr => src}/include/sof/lib/vregion.h (99%) diff --git a/zephyr/include/sof/lib/vregion.h b/src/include/sof/lib/vregion.h similarity index 99% rename from zephyr/include/sof/lib/vregion.h rename to src/include/sof/lib/vregion.h index 55c028797709..94f26178d0b8 100644 --- a/zephyr/include/sof/lib/vregion.h +++ b/src/include/sof/lib/vregion.h @@ -5,7 +5,6 @@ #ifndef __SOF_LIB_VREGION_H__ #define __SOF_LIB_VREGION_H__ -#include #include #ifdef __cplusplus From d89a4a0ef54e0cfd60f086380b0ab8e80aba8f17 Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Fri, 17 Apr 2026 10:58:14 +0200 Subject: [PATCH 07/15] vregion: add a reference count and locking Vregions can be used by multiple asynchronous users. Add a refcount and a lock. Signed-off-by: Guennadi Liakhovetski --- src/include/sof/lib/vregion.h | 45 +++++++++++++++++--- zephyr/lib/vregion.c | 77 +++++++++++++++++++++++++++-------- zephyr/test/vregion.c | 4 +- 3 files changed, 100 insertions(+), 26 deletions(-) diff --git a/src/include/sof/lib/vregion.h b/src/include/sof/lib/vregion.h index 94f26178d0b8..612443f5bc48 100644 --- a/src/include/sof/lib/vregion.h +++ b/src/include/sof/lib/vregion.h @@ -41,13 +41,26 @@ enum vregion_mem_type { struct vregion *vregion_create(size_t lifetime_size, size_t interim_size); /** - * @brief Destroy a virtual region instance. + * @brief Increment virtual region's user count. * - * Free all associated resources and deallocate the virtual region instance. + * The creator of the virtual region is its first user, for any additional users + * increment the region's use-count. * - * @param[in] vr Pointer to the virtual region instance to destroy. + * @param[in] vr Pointer to the virtual region instance to release. + * @return struct vregion* Pointer to the virtual region instance. */ -void vregion_destroy(struct vregion *vr); +struct vregion *vregion_get(struct vregion *vr); + +/** + * @brief Decrement virtual region's user count or destroy it. + * + * Decrement virtual region's user count, when it reaches 0 free all associated + * resources. + * + * @param[in] vr Pointer to the virtual region instance to release. + * @return struct vregion* Pointer to the virtual region instance or NULL if it has been destroyed. + */ +struct vregion *vregion_put(struct vregion *vr); /** * @brief Allocate memory from the specified virtual region. @@ -112,11 +125,31 @@ void vregion_mem_info(struct vregion *vr, size_t *size, uintptr_t *start); #else /* CONFIG_SOF_VREGIONS */ +#include + +struct vregion { + unsigned int use_count; +}; + static inline struct vregion *vregion_create(size_t lifetime_size, size_t interim_size) { - return NULL; + struct vregion *vr = rmalloc(0, sizeof(*vr)); + + vr->use_count = 1; + return vr; +} +static inline struct vregion *vregion_get(struct vregion *vr) +{ + if (vr) + vr->use_count++; + return vr; +} +static inline struct vregion *vregion_put(struct vregion *vr) +{ + if (vr && !--vr->use_count) + rfree(vr); + return vr; } -static inline void vregion_destroy(struct vregion *vr) {} static inline void *vregion_alloc(struct vregion *vr, enum vregion_mem_type type, size_t size) { return NULL; diff --git a/zephyr/lib/vregion.c b/zephyr/lib/vregion.c index f62809ee9ecd..84af0d0645e6 100644 --- a/zephyr/lib/vregion.c +++ b/zephyr/lib/vregion.c @@ -84,6 +84,8 @@ struct vregion { uint8_t *base; /* base address of entire region */ size_t size; /* size of whole region in bytes */ unsigned int pages; /* size of whole region in pages */ + struct k_mutex lock; /* protect vregion heaps and use-count */ + unsigned int use_count; /* interim heap */ struct interim_heap interim; /* interim heap */ @@ -152,6 +154,10 @@ struct vregion *vregion_create(size_t lifetime_size, size_t interim_size) /* init interim heaps */ k_heap_init(&vr->interim.heap, vr->interim.heap.heap.init_mem, interim_size); + k_mutex_init(&vr->lock); + /* The creator is the first user */ + vr->use_count = 1; + /* log the new vregion */ LOG_INF("new at base %p size %#zx pages %u struct embedded at %p", (void *)vr->base, total_size, pages, (void *)vr); @@ -161,20 +167,46 @@ struct vregion *vregion_create(size_t lifetime_size, size_t interim_size) return vr; } +struct vregion *vregion_get(struct vregion *vr) +{ + if (!vr) + return NULL; + + k_mutex_lock(&vr->lock, K_FOREVER); + vr->use_count++; + k_mutex_unlock(&vr->lock); + + return vr; +} + /** - * @brief Destroy a virtual region instance. + * @brief Decrement virtual region's user count or destroy it. * - * @param[in] vr Pointer to the virtual region instance to destroy. + * @param[in] vr Pointer to the virtual region instance to release. + * @return struct vregion* Pointer to the virtual region instance or NULL if it has been destroyed. */ -void vregion_destroy(struct vregion *vr) +struct vregion *vregion_put(struct vregion *vr) { + unsigned int use_count; + if (!vr) - return; + return NULL; + + k_mutex_lock(&vr->lock, K_FOREVER); + use_count = --vr->use_count; + k_mutex_unlock(&vr->lock); + + if (use_count) + return vr; + + /* Last user: nobody else can access the instance. */ /* log the vregion being destroyed */ LOG_DBG("destroy %p size %#zx pages %u", (void *)vr->base, vr->size, vr->pages); LOG_DBG(" lifetime used %zu free count %d", vr->lifetime.used, vr->lifetime.free_count); vpage_free(vr->base); + + return NULL; } /** @@ -275,25 +307,24 @@ void vregion_free(struct vregion *vr, void *ptr) if (!vr || !ptr) return; + k_mutex_lock(&vr->lock, K_FOREVER); + if (sys_cache_is_ptr_uncached(ptr)) ptr = sys_cache_cached_ptr_get(ptr); - /* check if pointer is in interim heap */ if (ptr >= (void *)vr->interim.heap.heap.init_mem && ptr < (void *)((uint8_t *)vr->interim.heap.heap.init_mem + - vr->interim.heap.heap.init_bytes)) { + vr->interim.heap.heap.init_bytes)) + /* pointer is in interim heap */ interim_free(&vr->interim, ptr); - return; - } - - /* check if pointer is in lifetime heap */ - if (ptr >= (void *)vr->lifetime.base && - ptr < (void *)(vr->lifetime.base + vr->lifetime.size)) { + else if (ptr >= (void *)vr->lifetime.base && + ptr < (void *)(vr->lifetime.base + vr->lifetime.size)) + /* pointer is in lifetime heap */ lifetime_free(&vr->lifetime, ptr); - return; - } + else + LOG_ERR("error: vregion free invalid pointer %p", ptr); - LOG_ERR("error: vregion free invalid pointer %p", ptr); + k_mutex_unlock(&vr->lock); } EXPORT_SYMBOL(vregion_free); @@ -310,21 +341,31 @@ EXPORT_SYMBOL(vregion_free); void *vregion_alloc_align(struct vregion *vr, enum vregion_mem_type type, size_t size, size_t alignment) { + void *p; + if (!vr || !size) return NULL; if (alignment < PLATFORM_DCACHE_ALIGN) alignment = PLATFORM_DCACHE_ALIGN; + k_mutex_lock(&vr->lock, K_FOREVER); + switch (type) { case VREGION_MEM_TYPE_INTERIM: - return interim_alloc(&vr->interim, size, alignment); + p = interim_alloc(&vr->interim, size, alignment); + break; case VREGION_MEM_TYPE_LIFETIME: - return lifetime_alloc(&vr->lifetime, size, alignment); + p = lifetime_alloc(&vr->lifetime, size, alignment); + break; default: LOG_ERR("error: invalid memory type %d", type); - return NULL; + p = NULL; } + + k_mutex_unlock(&vr->lock); + + return p; } EXPORT_SYMBOL(vregion_alloc_align); diff --git a/zephyr/test/vregion.c b/zephyr/test/vregion.c index 65586d690494..eb59f68a14d7 100644 --- a/zephyr/test/vregion.c +++ b/zephyr/test/vregion.c @@ -31,7 +31,7 @@ static void test_vreg_alloc_lifet(struct vregion *vreg) zassert_not_null(ptr); - void *ptr_align = vregion_alloc_align(vreg, VREGION_MEM_TYPE_LIFETIME, 2000, 16); + void *ptr_align = vregion_alloc_align(vreg, VREGION_MEM_TYPE_LIFETIME, 1600, 16); zassert_not_null(ptr_align); zassert_equal((uintptr_t)ptr_align & 15, 0); @@ -76,7 +76,7 @@ static void test_vreg_alloc_tmp(struct vregion *vreg) static void test_vreg_destroy(struct vregion *vreg) { vregion_info(vreg); - vregion_destroy(vreg); + vregion_put(vreg); } ZTEST(sof_boot, vregion) From 9a49a1a8025ce4c5a200f393c522a3e64061b29d Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Wed, 8 Apr 2026 16:15:54 +0200 Subject: [PATCH 08/15] audio: buffer: prepare to allocate on vregion Buffers, that are accessible to userspace DP modules, have to migrate to vregion together with the latter. Prepare them for the migration. Signed-off-by: Guennadi Liakhovetski --- src/audio/buffers/comp_buffer.c | 38 ++++++++------ src/audio/buffers/ring_buffer.c | 49 ++++++++++++++----- src/audio/module_adapter/module/generic.c | 22 ++++----- src/audio/module_adapter/module_adapter.c | 19 +++---- .../module_adapter/module_adapter_ipc4.c | 2 +- src/include/sof/audio/audio_buffer.h | 2 +- src/include/sof/audio/buffer.h | 10 ++-- src/include/sof/audio/component.h | 9 ++++ .../sof/audio/module_adapter/module/generic.h | 2 +- src/ipc/ipc-helper.c | 6 +-- src/ipc/ipc4/helper.c | 21 ++++---- 11 files changed, 110 insertions(+), 70 deletions(-) diff --git a/src/audio/buffers/comp_buffer.c b/src/audio/buffers/comp_buffer.c index e1e4c1867e85..9914009ba4aa 100644 --- a/src/audio/buffers/comp_buffer.c +++ b/src/audio/buffers/comp_buffer.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -153,15 +154,19 @@ static void comp_buffer_free(struct sof_audio_buffer *audio_buffer) buffer->probe_cb_free(buffer->probe_cb_arg); #endif - struct k_heap *heap = buffer->audio_buffer.heap; + struct mod_alloc_ctx *alloc = buffer->audio_buffer.alloc; rfree(buffer->stream.addr); - sof_heap_free(heap, buffer); - if (heap) { - struct dp_heap_user *mod_heap_user = container_of(heap, struct dp_heap_user, heap); + if (alloc && alloc->vreg) + vregion_free(alloc->vreg, buffer); + else + sof_heap_free(alloc ? alloc->heap : NULL, buffer); - if (!--mod_heap_user->client_count) - rfree(mod_heap_user); + if (alloc && alloc->client && !--alloc->client->client_count) { + rfree(alloc->client); + alloc->client = NULL; + /* NULL is allowed */ + vregion_put(alloc->vreg); } } @@ -192,7 +197,7 @@ static const struct audio_buffer_ops audio_buffer_ops = { .set_alignment_constants = comp_buffer_set_alignment_constants, }; -static struct comp_buffer *buffer_alloc_struct(struct k_heap *heap, +static struct comp_buffer *buffer_alloc_struct(struct mod_alloc_ctx *alloc, void *stream_addr, size_t size, uint32_t flags, bool is_shared) { @@ -204,7 +209,12 @@ static struct comp_buffer *buffer_alloc_struct(struct k_heap *heap, if (is_shared) flags |= SOF_MEM_FLAG_COHERENT; - buffer = sof_heap_alloc(heap, flags, sizeof(*buffer), 0); + if (!alloc || !alloc->vreg) + buffer = sof_heap_alloc(alloc ? alloc->heap : NULL, flags, sizeof(*buffer), 0); + else if (is_shared) + buffer = vregion_alloc_coherent(alloc->vreg, VREGION_MEM_TYPE_INTERIM, sizeof(*buffer)); + else + buffer = vregion_alloc(alloc->vreg, VREGION_MEM_TYPE_INTERIM, sizeof(*buffer)); if (!buffer) { tr_err(&buffer_tr, "could not alloc structure"); return NULL; @@ -226,7 +236,7 @@ static struct comp_buffer *buffer_alloc_struct(struct k_heap *heap, audio_stream_set_underrun(&buffer->stream, !!(flags & SOF_BUF_UNDERRUN_PERMITTED)); audio_stream_set_overrun(&buffer->stream, !!(flags & SOF_BUF_OVERRUN_PERMITTED)); - buffer->audio_buffer.heap = heap; + buffer->audio_buffer.alloc = alloc; comp_buffer_reset_source_list(buffer); comp_buffer_reset_sink_list(buffer); @@ -234,8 +244,8 @@ static struct comp_buffer *buffer_alloc_struct(struct k_heap *heap, return buffer; } -struct comp_buffer *buffer_alloc(struct k_heap *heap, size_t size, uint32_t flags, uint32_t align, - bool is_shared) +struct comp_buffer *buffer_alloc(struct mod_alloc_ctx *alloc, size_t size, uint32_t flags, + uint32_t align, bool is_shared) { struct comp_buffer *buffer; void *stream_addr; @@ -255,7 +265,7 @@ struct comp_buffer *buffer_alloc(struct k_heap *heap, size_t size, uint32_t flag return NULL; } - buffer = buffer_alloc_struct(heap, stream_addr, size, flags, is_shared); + buffer = buffer_alloc_struct(alloc, stream_addr, size, flags, is_shared); if (!buffer) { tr_err(&buffer_tr, "could not alloc buffer structure"); rfree(stream_addr); @@ -264,7 +274,7 @@ struct comp_buffer *buffer_alloc(struct k_heap *heap, size_t size, uint32_t flag return buffer; } -struct comp_buffer *buffer_alloc_range(struct k_heap *heap, size_t preferred_size, +struct comp_buffer *buffer_alloc_range(struct mod_alloc_ctx *alloc, size_t preferred_size, size_t minimum_size, uint32_t flags, uint32_t align, bool is_shared) { @@ -299,7 +309,7 @@ struct comp_buffer *buffer_alloc_range(struct k_heap *heap, size_t preferred_siz return NULL; } - buffer = buffer_alloc_struct(heap, stream_addr, size, flags, is_shared); + buffer = buffer_alloc_struct(alloc, stream_addr, size, flags, is_shared); if (!buffer) { tr_err(&buffer_tr, "could not alloc buffer structure"); rfree(stream_addr); diff --git a/src/audio/buffers/ring_buffer.c b/src/audio/buffers/ring_buffer.c index 11ebd736dfe4..07f3a7f9d95c 100644 --- a/src/audio/buffers/ring_buffer.c +++ b/src/audio/buffers/ring_buffer.c @@ -6,6 +6,7 @@ #include #include #include +#include #include #include @@ -96,9 +97,15 @@ static void ring_buffer_free(struct sof_audio_buffer *audio_buffer) struct ring_buffer *ring_buffer = container_of(audio_buffer, struct ring_buffer, audio_buffer); - - sof_heap_free(audio_buffer->heap, (__sparse_force void *)ring_buffer->_data_buffer); - sof_heap_free(audio_buffer->heap, ring_buffer); + struct mod_alloc_ctx *alloc = audio_buffer->alloc; + + if (alloc->vreg) { + vregion_free(alloc->vreg, (__sparse_force void *)ring_buffer->_data_buffer); + vregion_free(alloc->vreg, ring_buffer); + } else { + sof_heap_free(alloc->heap, (__sparse_force void *)ring_buffer->_data_buffer); + sof_heap_free(alloc->heap, ring_buffer); + } } static void ring_buffer_reset(struct sof_audio_buffer *audio_buffer) @@ -287,12 +294,19 @@ struct ring_buffer *ring_buffer_create(struct comp_dev *dev, size_t min_availabl uint32_t id) { struct ring_buffer *ring_buffer; - struct k_heap *heap = dev->mod->priv.resources.heap; + struct mod_alloc_ctx *alloc = &dev->mod->priv.resources.alloc; + struct k_heap *heap = alloc->heap; + struct vregion *vreg = alloc->vreg; int memory_flags = (is_shared ? SOF_MEM_FLAG_COHERENT : 0) | user_get_buffer_memory_region(dev->drv); /* allocate ring_buffer structure */ - ring_buffer = sof_heap_alloc(heap, memory_flags, sizeof(*ring_buffer), 0); + if (!vreg) + ring_buffer = sof_heap_alloc(heap, memory_flags, sizeof(*ring_buffer), 0); + else if (is_shared) + ring_buffer = vregion_alloc_coherent(vreg, VREGION_MEM_TYPE_INTERIM, sizeof(*ring_buffer)); + else + ring_buffer = vregion_alloc(vreg, VREGION_MEM_TYPE_INTERIM, sizeof(*ring_buffer)); if (!ring_buffer) return NULL; @@ -307,7 +321,8 @@ struct ring_buffer *ring_buffer_create(struct comp_dev *dev, size_t min_availabl audio_buffer_init(&ring_buffer->audio_buffer, BUFFER_TYPE_RING_BUFFER, is_shared, &ring_buffer_source_ops, &ring_buffer_sink_ops, &audio_buffer_ops, NULL); - ring_buffer->audio_buffer.heap = heap; + ring_buffer->audio_buffer.alloc = alloc; + ring_buffer->audio_buffer.alloc->heap = heap; /* set obs/ibs in sink/source interfaces */ sink_set_min_free_space(audio_buffer_get_sink(&ring_buffer->audio_buffer), @@ -364,12 +379,21 @@ struct ring_buffer *ring_buffer_create(struct comp_dev *dev, size_t min_availabl /* allocate data buffer - always in cached memory alias */ ring_buffer->data_buffer_size = ALIGN_UP(ring_buffer->data_buffer_size, PLATFORM_DCACHE_ALIGN); - ring_buffer->_data_buffer = (__sparse_force __sparse_cache void *)sof_heap_alloc(heap, - user_get_buffer_memory_region(dev->drv), - ring_buffer->data_buffer_size, PLATFORM_DCACHE_ALIGN); - if (!ring_buffer->_data_buffer) + + void *data_buf; + + if (vreg) + data_buf = vregion_alloc_align(vreg, VREGION_MEM_TYPE_INTERIM, ring_buffer->data_buffer_size, + PLATFORM_DCACHE_ALIGN); + else + data_buf = sof_heap_alloc(heap, user_get_buffer_memory_region(dev->drv), + ring_buffer->data_buffer_size, PLATFORM_DCACHE_ALIGN); + + if (!data_buf) goto err; + ring_buffer->_data_buffer = (__sparse_force __sparse_cache void *)data_buf; + tr_info(&ring_buffer_tr, "Ring buffer created, id: %u shared: %u min_available: %u min_free_space %u, size %u", id, ring_buffer_is_shared(ring_buffer), min_available, min_free_space, ring_buffer->data_buffer_size); @@ -378,6 +402,9 @@ struct ring_buffer *ring_buffer_create(struct comp_dev *dev, size_t min_availabl return ring_buffer; err: tr_err(&ring_buffer_tr, "Ring buffer creation failure"); - sof_heap_free(heap, ring_buffer); + if (vreg) + vregion_free(vreg, ring_buffer); + else + sof_heap_free(heap, ring_buffer); return NULL; } diff --git a/src/audio/module_adapter/module/generic.c b/src/audio/module_adapter/module/generic.c index 179d412b5a99..d94452855a4b 100644 --- a/src/audio/module_adapter/module/generic.c +++ b/src/audio/module_adapter/module/generic.c @@ -86,7 +86,7 @@ void mod_resource_init(struct processing_module *mod) /* Init memory list */ list_init(&md->resources.objpool.list); - md->resources.objpool.heap = md->resources.heap; + md->resources.objpool.heap = md->resources.alloc.heap; md->resources.heap_usage = 0; md->resources.heap_high_water_mark = 0; } @@ -159,10 +159,10 @@ void mod_heap_info(struct processing_module *mod, size_t *size, uintptr_t *start struct module_resources *res = &mod->priv.resources; if (size) - *size = res->heap->heap.init_bytes; + *size = res->alloc.heap->heap.init_bytes; if (start) - *start = (uintptr_t)container_of(res->heap, struct dp_heap_user, heap); + *start = (uintptr_t)res->alloc.client; } #endif @@ -195,7 +195,7 @@ void *mod_balloc_align(struct processing_module *mod, size_t size, size_t alignm } /* Allocate buffer memory for module */ - void *ptr = sof_heap_alloc(res->heap, SOF_MEM_FLAG_USER | SOF_MEM_FLAG_LARGE_BUFFER, + void *ptr = sof_heap_alloc(res->alloc.heap, SOF_MEM_FLAG_USER | SOF_MEM_FLAG_LARGE_BUFFER, size, alignment); if (!ptr) { @@ -246,7 +246,7 @@ void *z_impl_mod_alloc_ext(struct processing_module *mod, uint32_t flags, size_t } /* Allocate memory for module */ - void *ptr = sof_heap_alloc(res->heap, flags, size, alignment); + void *ptr = sof_heap_alloc(res->alloc.heap, flags, size, alignment); if (!ptr) { comp_err(mod->dev, "Failed to alloc %zu bytes %zu alignment for comp %#x.", @@ -323,7 +323,7 @@ const void *z_impl_mod_fast_get(struct processing_module *mod, const void * cons if (!container) return NULL; - ptr = fast_get(res->heap, dram_ptr, size); + ptr = fast_get(res->alloc.heap, dram_ptr, size); if (!ptr) { container_put(mod, container); return NULL; @@ -347,7 +347,7 @@ static int free_contents(struct processing_module *mod, struct module_resource * switch (container->type) { case MOD_RES_HEAP: - sof_heap_free(res->heap, container->ptr); + sof_heap_free(res->alloc.heap, container->ptr); res->heap_usage -= container->size; return 0; #if CONFIG_COMP_BLOB @@ -362,7 +362,7 @@ static int free_contents(struct processing_module *mod, struct module_resource * #else mdom = NULL; #endif - fast_put(res->heap, mdom, container->sram_ptr); + fast_put(res->alloc.heap, mdom, container->sram_ptr); return 0; #endif default: @@ -429,7 +429,7 @@ const void *z_vrfy_mod_fast_get(struct processing_module *mod, const void * cons struct module_resources *res = &mod->priv.resources; K_OOPS(K_SYSCALL_MEMORY_WRITE(mod, sizeof(*mod))); - K_OOPS(K_SYSCALL_MEMORY_WRITE(res->heap, sizeof(*res->heap))); + K_OOPS(K_SYSCALL_MEMORY_WRITE(res->alloc.heap, sizeof(*res->alloc.heap))); K_OOPS(K_SYSCALL_MEMORY_READ(dram_ptr, size)); return z_impl_mod_fast_get(mod, dram_ptr, size); @@ -443,7 +443,7 @@ void *z_vrfy_mod_alloc_ext(struct processing_module *mod, uint32_t flags, size_t struct module_resources *res = &mod->priv.resources; K_OOPS(K_SYSCALL_MEMORY_WRITE(mod, sizeof(*mod))); - K_OOPS(K_SYSCALL_MEMORY_WRITE(res->heap, sizeof(*res->heap))); + K_OOPS(K_SYSCALL_MEMORY_WRITE(res->alloc.heap, sizeof(*res->alloc.heap))); return z_impl_mod_alloc_ext(mod, flags, size, alignment); } @@ -454,7 +454,7 @@ int z_vrfy_mod_free(struct processing_module *mod, const void *ptr) struct module_resources *res = &mod->priv.resources; K_OOPS(K_SYSCALL_MEMORY_WRITE(mod, sizeof(*mod))); - K_OOPS(K_SYSCALL_MEMORY_WRITE(res->heap, sizeof(*res->heap))); + K_OOPS(K_SYSCALL_MEMORY_WRITE(res->alloc.heap, sizeof(*res->alloc.heap))); return z_impl_mod_free(mod, ptr); } diff --git a/src/audio/module_adapter/module_adapter.c b/src/audio/module_adapter/module_adapter.c index 9218b0df33ce..bc9a69605bf6 100644 --- a/src/audio/module_adapter/module_adapter.c +++ b/src/audio/module_adapter/module_adapter.c @@ -124,7 +124,8 @@ static struct processing_module *module_adapter_mem_alloc(const struct comp_driv } memset(mod, 0, sizeof(*mod)); - mod->priv.resources.heap = mod_heap; + mod->priv.resources.alloc.heap = mod_heap; + mod->priv.resources.alloc.client = mod_heap_user; mod_resource_init(mod); /* @@ -161,7 +162,7 @@ static struct processing_module *module_adapter_mem_alloc(const struct comp_driv static void module_adapter_mem_free(struct processing_module *mod) { - struct k_heap *mod_heap = mod->priv.resources.heap; + struct k_heap *mod_heap = mod->priv.resources.alloc.heap; unsigned int domain = mod->dev->ipc_config.proc_domain; /* @@ -611,8 +612,8 @@ int module_adapter_prepare(struct comp_dev *dev) if (list_is_empty(&mod->raw_data_buffers_list)) { for (i = 0; i < mod->num_of_sinks; i++) { /* allocate not shared buffer */ - struct comp_buffer *buffer = buffer_alloc(md->resources.heap, buff_size, - memory_flags, + struct comp_buffer *buffer = buffer_alloc(&md->resources.alloc, + buff_size, memory_flags, PLATFORM_DCACHE_ALIGN, BUFFER_USAGE_NOT_SHARED); uint32_t flags; @@ -623,13 +624,9 @@ int module_adapter_prepare(struct comp_dev *dev) goto free; } - if (md->resources.heap && md->resources.heap != dev->drv->user_heap) { - struct dp_heap_user *dp_user = container_of(md->resources.heap, - struct dp_heap_user, - heap); - - dp_user->client_count++; - } + if (md->resources.alloc.heap && + md->resources.alloc.heap != dev->drv->user_heap) + md->resources.alloc.client->client_count++; irq_local_disable(flags); list_item_prepend(&buffer->buffers_list, &mod->raw_data_buffers_list); diff --git a/src/audio/module_adapter/module_adapter_ipc4.c b/src/audio/module_adapter/module_adapter_ipc4.c index cdf535b1661e..f909f7dd876e 100644 --- a/src/audio/module_adapter/module_adapter_ipc4.c +++ b/src/audio/module_adapter/module_adapter_ipc4.c @@ -147,7 +147,7 @@ int module_adapter_init_data(struct comp_dev *dev, if (cfgsz == (sizeof(*cfg) + pinsz)) { dst->nb_input_pins = n_in; dst->nb_output_pins = n_out; - dst->input_pins = sof_heap_alloc(dev->mod->priv.resources.heap, + dst->input_pins = sof_heap_alloc(dev->mod->priv.resources.alloc.heap, SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, pinsz, 0); if (!dst->input_pins) diff --git a/src/include/sof/audio/audio_buffer.h b/src/include/sof/audio/audio_buffer.h index e627fc0494fa..2526a7696534 100644 --- a/src/include/sof/audio/audio_buffer.h +++ b/src/include/sof/audio/audio_buffer.h @@ -111,7 +111,7 @@ struct sof_audio_buffer { */ bool walking; /**< indicates if the buffer is being walked */ - struct k_heap *heap; + struct mod_alloc_ctx *alloc; }; #if CONFIG_PIPELINE_2_0 diff --git a/src/include/sof/audio/buffer.h b/src/include/sof/audio/buffer.h index 6eab812d8f28..b15381dd0f9a 100644 --- a/src/include/sof/audio/buffer.h +++ b/src/include/sof/audio/buffer.h @@ -217,15 +217,15 @@ struct buffer_cb_transact { buffer->cb_type = type; \ } while (0) -struct k_heap; +struct mod_alloc_ctx; /* pipeline buffer creation and destruction */ -struct comp_buffer *buffer_alloc(struct k_heap *heap, size_t size, uint32_t flags, uint32_t align, - bool is_shared); -struct comp_buffer *buffer_alloc_range(struct k_heap *heap, size_t preferred_size, +struct comp_buffer *buffer_alloc(struct mod_alloc_ctx *alloc, size_t size, uint32_t flags, + uint32_t align, bool is_shared); +struct comp_buffer *buffer_alloc_range(struct mod_alloc_ctx *alloc, size_t preferred_size, size_t minimum_size, uint32_t flags, uint32_t align, bool is_shared); -struct comp_buffer *buffer_new(struct k_heap *heap, const struct sof_ipc_buffer *desc, +struct comp_buffer *buffer_new(struct mod_alloc_ctx *alloc, const struct sof_ipc_buffer *desc, bool is_shared); int buffer_set_size(struct comp_buffer *buffer, uint32_t size, uint32_t alignment); diff --git a/src/include/sof/audio/component.h b/src/include/sof/audio/component.h index b6695b9cd312..bcae1ce05f72 100644 --- a/src/include/sof/audio/component.h +++ b/src/include/sof/audio/component.h @@ -578,6 +578,15 @@ struct comp_ops { uint64_t (*get_total_data_processed)(struct comp_dev *dev, uint32_t stream_no, bool input); }; +struct k_heap; +struct vregion; +struct dp_heap_user; +struct mod_alloc_ctx { + struct k_heap *heap; + struct vregion *vreg; + struct dp_heap_user *client; +}; + /** * Audio component base driver "class" * - used by all other component types. diff --git a/src/include/sof/audio/module_adapter/module/generic.h b/src/include/sof/audio/module_adapter/module/generic.h index 59f5f398bad5..fa30e746df12 100644 --- a/src/include/sof/audio/module_adapter/module/generic.h +++ b/src/include/sof/audio/module_adapter/module/generic.h @@ -132,7 +132,7 @@ struct module_resources { struct objpool_head objpool; size_t heap_usage; size_t heap_high_water_mark; - struct k_heap *heap; + struct mod_alloc_ctx alloc; #if CONFIG_MODULE_MEMORY_API_DEBUG && defined(__ZEPHYR__) k_tid_t rsrc_mngr; #endif diff --git a/src/ipc/ipc-helper.c b/src/ipc/ipc-helper.c index 2f685b551747..ad7b3771a16b 100644 --- a/src/ipc/ipc-helper.c +++ b/src/ipc/ipc-helper.c @@ -50,8 +50,8 @@ __cold static bool valid_ipc_buffer_desc(const struct sof_ipc_buffer *desc) } /* create a new component in the pipeline */ -__cold struct comp_buffer *buffer_new(struct k_heap *heap, const struct sof_ipc_buffer *desc, - bool is_shared) +__cold struct comp_buffer *buffer_new(struct mod_alloc_ctx *alloc, + const struct sof_ipc_buffer *desc, bool is_shared) { struct comp_buffer *buffer; uint32_t flags = desc->flags; @@ -79,7 +79,7 @@ __cold struct comp_buffer *buffer_new(struct k_heap *heap, const struct sof_ipc_ desc->caps, flags); /* allocate buffer */ - buffer = buffer_alloc(heap, desc->size, flags, PLATFORM_DCACHE_ALIGN, + buffer = buffer_alloc(alloc, desc->size, flags, PLATFORM_DCACHE_ALIGN, is_shared); if (buffer) { buffer->stream.runtime_stream_params.id = desc->comp.id; diff --git a/src/ipc/ipc4/helper.c b/src/ipc/ipc4/helper.c index 3404906b9771..9a31231bcb96 100644 --- a/src/ipc/ipc4/helper.c +++ b/src/ipc/ipc4/helper.c @@ -527,8 +527,8 @@ __cold int ipc_pipeline_free(struct ipc *ipc, uint32_t comp_id) } __cold static struct comp_buffer *ipc4_create_buffer(struct comp_dev *src, bool is_shared, - uint32_t buf_size, uint32_t src_queue, - uint32_t dst_queue, struct k_heap *heap) + uint32_t buf_size, uint32_t src_queue, + uint32_t dst_queue, struct mod_alloc_ctx *alloc) { struct sof_ipc_buffer ipc_buf; @@ -539,7 +539,7 @@ __cold static struct comp_buffer *ipc4_create_buffer(struct comp_dev *src, bool ipc_buf.comp.id = IPC4_COMP_ID(src_queue, dst_queue); ipc_buf.comp.pipeline_id = src->ipc_config.pipeline_id; ipc_buf.comp.core = cpu_get_id(); - return buffer_new(heap, &ipc_buf, is_shared); + return buffer_new(alloc, &ipc_buf, is_shared); } #if CONFIG_CROSS_CORE_STREAM @@ -640,7 +640,7 @@ __cold int ipc_comp_connect(struct ipc *ipc, ipc_pipe_comp_connect *_connect) return IPC4_INVALID_RESOURCE_ID; } - struct k_heap *dp_heap; + struct mod_alloc_ctx *alloc; #if CONFIG_ZEPHYR_DP_SCHEDULER if (source->ipc_config.proc_domain == COMP_PROCESSING_DOMAIN_DP && @@ -659,9 +659,9 @@ __cold int ipc_comp_connect(struct ipc *ipc, ipc_pipe_comp_connect *_connect) else dp = NULL; - dp_heap = dp && dp->mod ? dp->mod->priv.resources.heap : NULL; + alloc = dp && dp->mod ? &dp->mod->priv.resources.alloc : NULL; #else - dp_heap = NULL; + alloc = NULL; #endif /* CONFIG_ZEPHYR_DP_SCHEDULER */ bool cross_core_bind = source->ipc_config.core != sink->ipc_config.core; @@ -731,18 +731,15 @@ __cold int ipc_comp_connect(struct ipc *ipc, ipc_pipe_comp_connect *_connect) buf_size = ibs * 2; buffer = ipc4_create_buffer(source, cross_core_bind, buf_size, bu->extension.r.src_queue, - bu->extension.r.dst_queue, dp_heap); + bu->extension.r.dst_queue, alloc); if (!buffer) { tr_err(&ipc_tr, "failed to allocate buffer to bind %#x to %#x", src_id, sink_id); return IPC4_OUT_OF_MEMORY; } #if CONFIG_ZEPHYR_DP_SCHEDULER - if (dp_heap) { - struct dp_heap_user *dp_user = container_of(dp_heap, struct dp_heap_user, heap); - - dp_user->client_count++; - } + if (alloc && alloc->client) + alloc->client->client_count++; #endif /* From 070c235751f05d980af0693381019db74f26747d Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Fri, 17 Apr 2026 13:47:36 +0200 Subject: [PATCH 09/15] audio: module: simplify resource allocation context Merge struct mod_alloc_ctx and struct dp_heap_user. We still need a separately allocated allocation context to make sure it persists when the module adapter object is freed but some of the connected buffers still exist, so struct mod_alloc_ctx now becomes that separate context. Signed-off-by: Guennadi Liakhovetski --- src/audio/buffers/comp_buffer.c | 6 +- src/audio/buffers/ring_buffer.c | 2 +- src/audio/module_adapter/module/generic.c | 22 +++---- src/audio/module_adapter/module_adapter.c | 63 ++++++++++--------- .../module_adapter/module_adapter_ipc4.c | 2 +- src/include/sof/audio/component.h | 3 +- .../sof/audio/module_adapter/module/generic.h | 2 +- src/include/sof/schedule/dp_schedule.h | 6 -- src/ipc/ipc4/helper.c | 6 +- 9 files changed, 55 insertions(+), 57 deletions(-) diff --git a/src/audio/buffers/comp_buffer.c b/src/audio/buffers/comp_buffer.c index 9914009ba4aa..a66e83e5edf9 100644 --- a/src/audio/buffers/comp_buffer.c +++ b/src/audio/buffers/comp_buffer.c @@ -162,11 +162,9 @@ static void comp_buffer_free(struct sof_audio_buffer *audio_buffer) else sof_heap_free(alloc ? alloc->heap : NULL, buffer); - if (alloc && alloc->client && !--alloc->client->client_count) { - rfree(alloc->client); - alloc->client = NULL; - /* NULL is allowed */ + if (alloc && !--alloc->client_count) { vregion_put(alloc->vreg); + rfree(alloc); } } diff --git a/src/audio/buffers/ring_buffer.c b/src/audio/buffers/ring_buffer.c index 07f3a7f9d95c..fe67027df8db 100644 --- a/src/audio/buffers/ring_buffer.c +++ b/src/audio/buffers/ring_buffer.c @@ -294,7 +294,7 @@ struct ring_buffer *ring_buffer_create(struct comp_dev *dev, size_t min_availabl uint32_t id) { struct ring_buffer *ring_buffer; - struct mod_alloc_ctx *alloc = &dev->mod->priv.resources.alloc; + struct mod_alloc_ctx *alloc = dev->mod->priv.resources.alloc; struct k_heap *heap = alloc->heap; struct vregion *vreg = alloc->vreg; int memory_flags = (is_shared ? SOF_MEM_FLAG_COHERENT : 0) | diff --git a/src/audio/module_adapter/module/generic.c b/src/audio/module_adapter/module/generic.c index d94452855a4b..de25bdada10a 100644 --- a/src/audio/module_adapter/module/generic.c +++ b/src/audio/module_adapter/module/generic.c @@ -86,7 +86,7 @@ void mod_resource_init(struct processing_module *mod) /* Init memory list */ list_init(&md->resources.objpool.list); - md->resources.objpool.heap = md->resources.alloc.heap; + md->resources.objpool.heap = md->resources.alloc->heap; md->resources.heap_usage = 0; md->resources.heap_high_water_mark = 0; } @@ -159,10 +159,10 @@ void mod_heap_info(struct processing_module *mod, size_t *size, uintptr_t *start struct module_resources *res = &mod->priv.resources; if (size) - *size = res->alloc.heap->heap.init_bytes; + *size = res->alloc->heap->heap.init_bytes; if (start) - *start = (uintptr_t)res->alloc.client; + *start = (uintptr_t)res->alloc->heap; } #endif @@ -195,7 +195,7 @@ void *mod_balloc_align(struct processing_module *mod, size_t size, size_t alignm } /* Allocate buffer memory for module */ - void *ptr = sof_heap_alloc(res->alloc.heap, SOF_MEM_FLAG_USER | SOF_MEM_FLAG_LARGE_BUFFER, + void *ptr = sof_heap_alloc(res->alloc->heap, SOF_MEM_FLAG_USER | SOF_MEM_FLAG_LARGE_BUFFER, size, alignment); if (!ptr) { @@ -246,7 +246,7 @@ void *z_impl_mod_alloc_ext(struct processing_module *mod, uint32_t flags, size_t } /* Allocate memory for module */ - void *ptr = sof_heap_alloc(res->alloc.heap, flags, size, alignment); + void *ptr = sof_heap_alloc(res->alloc->heap, flags, size, alignment); if (!ptr) { comp_err(mod->dev, "Failed to alloc %zu bytes %zu alignment for comp %#x.", @@ -323,7 +323,7 @@ const void *z_impl_mod_fast_get(struct processing_module *mod, const void * cons if (!container) return NULL; - ptr = fast_get(res->alloc.heap, dram_ptr, size); + ptr = fast_get(res->alloc->heap, dram_ptr, size); if (!ptr) { container_put(mod, container); return NULL; @@ -347,7 +347,7 @@ static int free_contents(struct processing_module *mod, struct module_resource * switch (container->type) { case MOD_RES_HEAP: - sof_heap_free(res->alloc.heap, container->ptr); + sof_heap_free(res->alloc->heap, container->ptr); res->heap_usage -= container->size; return 0; #if CONFIG_COMP_BLOB @@ -362,7 +362,7 @@ static int free_contents(struct processing_module *mod, struct module_resource * #else mdom = NULL; #endif - fast_put(res->alloc.heap, mdom, container->sram_ptr); + fast_put(res->alloc->heap, mdom, container->sram_ptr); return 0; #endif default: @@ -429,7 +429,7 @@ const void *z_vrfy_mod_fast_get(struct processing_module *mod, const void * cons struct module_resources *res = &mod->priv.resources; K_OOPS(K_SYSCALL_MEMORY_WRITE(mod, sizeof(*mod))); - K_OOPS(K_SYSCALL_MEMORY_WRITE(res->alloc.heap, sizeof(*res->alloc.heap))); + K_OOPS(K_SYSCALL_MEMORY_WRITE(res->alloc->heap, sizeof(*res->alloc->heap))); K_OOPS(K_SYSCALL_MEMORY_READ(dram_ptr, size)); return z_impl_mod_fast_get(mod, dram_ptr, size); @@ -443,7 +443,7 @@ void *z_vrfy_mod_alloc_ext(struct processing_module *mod, uint32_t flags, size_t struct module_resources *res = &mod->priv.resources; K_OOPS(K_SYSCALL_MEMORY_WRITE(mod, sizeof(*mod))); - K_OOPS(K_SYSCALL_MEMORY_WRITE(res->alloc.heap, sizeof(*res->alloc.heap))); + K_OOPS(K_SYSCALL_MEMORY_WRITE(res->alloc->heap, sizeof(*res->alloc->heap))); return z_impl_mod_alloc_ext(mod, flags, size, alignment); } @@ -454,7 +454,7 @@ int z_vrfy_mod_free(struct processing_module *mod, const void *ptr) struct module_resources *res = &mod->priv.resources; K_OOPS(K_SYSCALL_MEMORY_WRITE(mod, sizeof(*mod))); - K_OOPS(K_SYSCALL_MEMORY_WRITE(res->alloc.heap, sizeof(*res->alloc.heap))); + K_OOPS(K_SYSCALL_MEMORY_WRITE(res->alloc->heap, sizeof(*res->alloc->heap))); return z_impl_mod_free(mod, ptr); } diff --git a/src/audio/module_adapter/module_adapter.c b/src/audio/module_adapter/module_adapter.c index bc9a69605bf6..b6c87e09a4f8 100644 --- a/src/audio/module_adapter/module_adapter.c +++ b/src/audio/module_adapter/module_adapter.c @@ -57,8 +57,8 @@ struct comp_dev *module_adapter_new(const struct comp_driver *drv, #define PAGE_SZ HOST_PAGE_SIZE #endif -static struct dp_heap_user *module_adapter_dp_heap_new(const struct comp_ipc_config *config, - size_t *heap_size) +static struct k_heap *module_adapter_dp_heap_new(const struct comp_ipc_config *config, + size_t *heap_size) { /* src-lite with 8 channels has been seen allocating 14k in one go */ /* FIXME: the size will be derived from configuration */ @@ -71,9 +71,8 @@ static struct dp_heap_user *module_adapter_dp_heap_new(const struct comp_ipc_con if (!mod_heap_mem) return NULL; - struct dp_heap_user *mod_heap_user = (struct dp_heap_user *)mod_heap_mem; - struct k_heap *mod_heap = &mod_heap_user->heap; - const size_t heap_prefix_size = ALIGN_UP(sizeof(*mod_heap_user), 4); + struct k_heap *mod_heap = (struct k_heap *)mod_heap_mem; + const size_t heap_prefix_size = ALIGN_UP(sizeof(*mod_heap), 4); void *mod_heap_buf = mod_heap_mem + heap_prefix_size; *heap_size = buf_size - heap_prefix_size; @@ -83,7 +82,7 @@ static struct dp_heap_user *module_adapter_dp_heap_new(const struct comp_ipc_con mod_heap->heap.init_bytes = *heap_size; #endif - return mod_heap_user; + return mod_heap; } static struct processing_module *module_adapter_mem_alloc(const struct comp_driver *drv, @@ -99,20 +98,17 @@ static struct processing_module *module_adapter_mem_alloc(const struct comp_driv */ uint32_t flags = config->proc_domain == COMP_PROCESSING_DOMAIN_DP ? SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT : SOF_MEM_FLAG_USER; - struct dp_heap_user *mod_heap_user; size_t heap_size; if (config->proc_domain == COMP_PROCESSING_DOMAIN_DP && IS_ENABLED(CONFIG_USERSPACE) && !IS_ENABLED(CONFIG_SOF_USERSPACE_USE_DRIVER_HEAP)) { - mod_heap_user = module_adapter_dp_heap_new(config, &heap_size); - if (!mod_heap_user) { + mod_heap = module_adapter_dp_heap_new(config, &heap_size); + if (!mod_heap) { comp_cl_err(drv, "Failed to allocate DP module heap"); return NULL; } - mod_heap = &mod_heap_user->heap; } else { mod_heap = drv->user_heap; - mod_heap_user = NULL; heap_size = 0; } @@ -123,9 +119,16 @@ static struct processing_module *module_adapter_mem_alloc(const struct comp_driv goto emod; } + struct mod_alloc_ctx *alloc = rmalloc(flags, sizeof(*alloc)); + + if (!alloc) + goto ealloc; + memset(mod, 0, sizeof(*mod)); - mod->priv.resources.alloc.heap = mod_heap; - mod->priv.resources.alloc.client = mod_heap_user; + alloc->heap = mod_heap; + alloc->vreg = NULL; + alloc->client_count = 0; + mod->priv.resources.alloc = alloc; mod_resource_init(mod); /* @@ -138,7 +141,7 @@ static struct processing_module *module_adapter_mem_alloc(const struct comp_driv if (!dev) { comp_cl_err(drv, "failed to allocate memory for comp_dev"); - goto err; + goto edev; } memset(dev, 0, sizeof(*dev)); @@ -147,22 +150,25 @@ static struct processing_module *module_adapter_mem_alloc(const struct comp_driv mod->dev = dev; dev->mod = mod; - if (mod_heap_user) - mod_heap_user->client_count++; + if (config->proc_domain == COMP_PROCESSING_DOMAIN_DP) + alloc->client_count++; return mod; -err: +edev: + rfree(alloc); +ealloc: sof_heap_free(mod_heap, mod); emod: - rfree(mod_heap_user); + rfree(mod_heap); return NULL; } static void module_adapter_mem_free(struct processing_module *mod) { - struct k_heap *mod_heap = mod->priv.resources.alloc.heap; + struct mod_alloc_ctx *alloc = mod->priv.resources.alloc; + struct k_heap *mod_heap = alloc->heap; unsigned int domain = mod->dev->ipc_config.proc_domain; /* @@ -175,11 +181,12 @@ static void module_adapter_mem_free(struct processing_module *mod) sof_heap_free(mod_heap, mod->dev); sof_heap_free(mod_heap, mod); if (domain == COMP_PROCESSING_DOMAIN_DP) { - struct dp_heap_user *mod_heap_user = container_of(mod_heap, struct dp_heap_user, - heap); - - if (mod_heap && !--mod_heap_user->client_count) - rfree(mod_heap_user); + if (!IS_ENABLED(CONFIG_SOF_USERSPACE_USE_DRIVER_HEAP)) + rfree(mod_heap); + if (!--alloc->client_count) + rfree(alloc); + } else { + rfree(alloc); } } @@ -612,7 +619,7 @@ int module_adapter_prepare(struct comp_dev *dev) if (list_is_empty(&mod->raw_data_buffers_list)) { for (i = 0; i < mod->num_of_sinks; i++) { /* allocate not shared buffer */ - struct comp_buffer *buffer = buffer_alloc(&md->resources.alloc, + struct comp_buffer *buffer = buffer_alloc(md->resources.alloc, buff_size, memory_flags, PLATFORM_DCACHE_ALIGN, BUFFER_USAGE_NOT_SHARED); @@ -624,9 +631,9 @@ int module_adapter_prepare(struct comp_dev *dev) goto free; } - if (md->resources.alloc.heap && - md->resources.alloc.heap != dev->drv->user_heap) - md->resources.alloc.client->client_count++; + if (md->resources.alloc->heap && + md->resources.alloc->heap != dev->drv->user_heap) + md->resources.alloc->client_count++; irq_local_disable(flags); list_item_prepend(&buffer->buffers_list, &mod->raw_data_buffers_list); diff --git a/src/audio/module_adapter/module_adapter_ipc4.c b/src/audio/module_adapter/module_adapter_ipc4.c index f909f7dd876e..092f93314bac 100644 --- a/src/audio/module_adapter/module_adapter_ipc4.c +++ b/src/audio/module_adapter/module_adapter_ipc4.c @@ -147,7 +147,7 @@ int module_adapter_init_data(struct comp_dev *dev, if (cfgsz == (sizeof(*cfg) + pinsz)) { dst->nb_input_pins = n_in; dst->nb_output_pins = n_out; - dst->input_pins = sof_heap_alloc(dev->mod->priv.resources.alloc.heap, + dst->input_pins = sof_heap_alloc(dev->mod->priv.resources.alloc->heap, SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, pinsz, 0); if (!dst->input_pins) diff --git a/src/include/sof/audio/component.h b/src/include/sof/audio/component.h index bcae1ce05f72..2df7e6a9bd42 100644 --- a/src/include/sof/audio/component.h +++ b/src/include/sof/audio/component.h @@ -580,11 +580,10 @@ struct comp_ops { struct k_heap; struct vregion; -struct dp_heap_user; struct mod_alloc_ctx { struct k_heap *heap; struct vregion *vreg; - struct dp_heap_user *client; + unsigned int client_count; }; /** diff --git a/src/include/sof/audio/module_adapter/module/generic.h b/src/include/sof/audio/module_adapter/module/generic.h index fa30e746df12..91bdce96b1c7 100644 --- a/src/include/sof/audio/module_adapter/module/generic.h +++ b/src/include/sof/audio/module_adapter/module/generic.h @@ -132,7 +132,7 @@ struct module_resources { struct objpool_head objpool; size_t heap_usage; size_t heap_high_water_mark; - struct mod_alloc_ctx alloc; + struct mod_alloc_ctx *alloc; #if CONFIG_MODULE_MEMORY_API_DEBUG && defined(__ZEPHYR__) k_tid_t rsrc_mngr; #endif diff --git a/src/include/sof/schedule/dp_schedule.h b/src/include/sof/schedule/dp_schedule.h index 37b8f1fc3f2c..2267d676fb8a 100644 --- a/src/include/sof/schedule/dp_schedule.h +++ b/src/include/sof/schedule/dp_schedule.h @@ -119,12 +119,6 @@ union scheduler_dp_thread_ipc_param { } pipeline_state; }; -struct dp_heap_user { - struct k_heap heap; - /* So far relying on linear processing of serialized IPCs, but might need protection */ - unsigned int client_count; /* devices and buffers */ -}; - #if CONFIG_ZEPHYR_DP_SCHEDULER int scheduler_dp_thread_ipc(struct processing_module *pmod, unsigned int cmd, const union scheduler_dp_thread_ipc_param *param); diff --git a/src/ipc/ipc4/helper.c b/src/ipc/ipc4/helper.c index 9a31231bcb96..d6cf1c24561f 100644 --- a/src/ipc/ipc4/helper.c +++ b/src/ipc/ipc4/helper.c @@ -659,7 +659,7 @@ __cold int ipc_comp_connect(struct ipc *ipc, ipc_pipe_comp_connect *_connect) else dp = NULL; - alloc = dp && dp->mod ? &dp->mod->priv.resources.alloc : NULL; + alloc = dp && dp->mod ? dp->mod->priv.resources.alloc : NULL; #else alloc = NULL; #endif /* CONFIG_ZEPHYR_DP_SCHEDULER */ @@ -738,8 +738,8 @@ __cold int ipc_comp_connect(struct ipc *ipc, ipc_pipe_comp_connect *_connect) } #if CONFIG_ZEPHYR_DP_SCHEDULER - if (alloc && alloc->client) - alloc->client->client_count++; + if (alloc) + alloc->client_count++; #endif /* From ea9edfba2aba338b65b5ad308b83e9be0042efd0 Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Tue, 14 Apr 2026 15:04:45 +0200 Subject: [PATCH 10/15] audio: module: (cosmetic) simplify pointer dereferencing Simplify pointers in mod_resource_init(). Signed-off-by: Guennadi Liakhovetski --- src/audio/module_adapter/module/generic.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/audio/module_adapter/module/generic.c b/src/audio/module_adapter/module/generic.c index de25bdada10a..60513ff0edb7 100644 --- a/src/audio/module_adapter/module/generic.c +++ b/src/audio/module_adapter/module/generic.c @@ -82,13 +82,13 @@ int module_load_config(struct comp_dev *dev, const void *cfg, size_t size) void mod_resource_init(struct processing_module *mod) { - struct module_data *md = &mod->priv; + struct module_resources *res = &mod->priv.resources; /* Init memory list */ - list_init(&md->resources.objpool.list); - md->resources.objpool.heap = md->resources.alloc->heap; - md->resources.heap_usage = 0; - md->resources.heap_high_water_mark = 0; + list_init(&res->objpool.list); + res->objpool.heap = res->alloc->heap; + res->heap_usage = 0; + res->heap_high_water_mark = 0; } int module_init(struct processing_module *mod) From e0e4a1f2caa6f0c0971a9880a1ee7d4655fab647 Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Wed, 29 Apr 2026 12:43:30 +0200 Subject: [PATCH 11/15] uuid: move the uuid header from buffer.h to trace.h uuid.h isn't needed in buffer.h. However just removing it from there breaks compilation because of multiple trace context definitions like DECLARE_TR_CTX(comp_tr, SOF_UUID(component_uuid), LOG_LEVEL_INFO); in files, not explicitly including uuid.h. To make a simple fix move the header to trace.h because that is anyway needed for all such lines. Signed-off-by: Guennadi Liakhovetski --- src/include/sof/audio/buffer.h | 1 - src/include/sof/trace/trace.h | 4 ++++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/include/sof/audio/buffer.h b/src/include/sof/audio/buffer.h index b15381dd0f9a..6e6b8a9caef8 100644 --- a/src/include/sof/audio/buffer.h +++ b/src/include/sof/audio/buffer.h @@ -16,7 +16,6 @@ #include #include #include -#include #include #include #include diff --git a/src/include/sof/trace/trace.h b/src/include/sof/trace/trace.h index ce8a12ed83c2..fc3aa324f847 100644 --- a/src/include/sof/trace/trace.h +++ b/src/include/sof/trace/trace.h @@ -31,6 +31,9 @@ #endif #include +#if CONFIG_ZEPHYR_LOG || CONFIG_LIBRARY || CONFIG_ARCH_POSIX_LIBFUZZER +#include +#endif #include #include @@ -128,6 +131,7 @@ static inline void mtrace_printf(int log_level, const char *format_str, ...) /** * Trace context. */ +struct sof_uuid_entry; struct tr_ctx { const struct sof_uuid_entry *uuid_p; /**< UUID pointer, use SOF_UUID() to init */ uint32_t level; /**< Default log level */ From c6c943e418986804db13d6bbd6711ff72c2cc134 Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Wed, 29 Apr 2026 12:50:07 +0200 Subject: [PATCH 12/15] dma: only include sof_dma.h when needed sof_dma.h is only used when actually building SOF. Put it under a proprocessor conditional to fix including the header in twister builds. Signed-off-by: Guennadi Liakhovetski --- zephyr/include/sof/lib/dma.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zephyr/include/sof/lib/dma.h b/zephyr/include/sof/lib/dma.h index b13f3c25221b..8af0d9f35391 100644 --- a/zephyr/include/sof/lib/dma.h +++ b/zephyr/include/sof/lib/dma.h @@ -270,9 +270,9 @@ int dmac_init(struct sof *sof); * Need to use sof_dma.h to avoid "syscalls/dma.h" name conflict * with Zephyr autogenerated headers for syscall support. */ +#ifdef CONFIG_ZEPHYR_NATIVE_DRIVERS #include "sof_dma.h" - -#ifndef CONFIG_ZEPHYR_NATIVE_DRIVERS +#else #include "dma-legacy.h" #endif /* !CONFIG_ZEPHYR_NATIVE_DRIVERS */ From dd5ea94804fdcd5d1bd8101b16dce4a711869f62 Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Wed, 29 Apr 2026 09:30:25 +0200 Subject: [PATCH 13/15] fast-get: switch to module allocation context fast-get has to choose which allocation method to use, it needs access to the full module allocation context. Signed-off-by: Guennadi Liakhovetski --- src/audio/module_adapter/module/generic.c | 4 ++-- src/include/sof/lib/fast-get.h | 12 +++++++----- zephyr/lib/fast-get.c | 7 +++++-- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/src/audio/module_adapter/module/generic.c b/src/audio/module_adapter/module/generic.c index 60513ff0edb7..ff7a88495c20 100644 --- a/src/audio/module_adapter/module/generic.c +++ b/src/audio/module_adapter/module/generic.c @@ -323,7 +323,7 @@ const void *z_impl_mod_fast_get(struct processing_module *mod, const void * cons if (!container) return NULL; - ptr = fast_get(res->alloc->heap, dram_ptr, size); + ptr = fast_get(res->alloc, dram_ptr, size); if (!ptr) { container_put(mod, container); return NULL; @@ -362,7 +362,7 @@ static int free_contents(struct processing_module *mod, struct module_resource * #else mdom = NULL; #endif - fast_put(res->alloc->heap, mdom, container->sram_ptr); + fast_put(res->alloc, mdom, container->sram_ptr); return 0; #endif default: diff --git a/src/include/sof/lib/fast-get.h b/src/include/sof/lib/fast-get.h index 41f6e9d98150..05f098a752cc 100644 --- a/src/include/sof/lib/fast-get.h +++ b/src/include/sof/lib/fast-get.h @@ -10,8 +10,8 @@ #include -struct k_heap; struct k_mem_domain; +struct mod_alloc_ctx; /* * When built for SOF, fast_get() and fast_put() are only needed when DRAM @@ -25,14 +25,16 @@ struct k_mem_domain; #if (CONFIG_COLD_STORE_EXECUTE_DRAM && \ (CONFIG_LLEXT_TYPE_ELF_RELOCATABLE || !defined(LL_EXTENSION_BUILD))) || \ !CONFIG_SOF_FULL_ZEPHYR_APPLICATION -const void *fast_get(struct k_heap *heap, const void * const dram_ptr, size_t size); -void fast_put(struct k_heap *heap, struct k_mem_domain *mdom, const void *sram_ptr); +const void *fast_get(struct mod_alloc_ctx *alloc, const void * const dram_ptr, size_t size); +void fast_put(struct mod_alloc_ctx *alloc, struct k_mem_domain *mdom, const void *sram_ptr); #else -static inline const void *fast_get(struct k_heap *heap, const void * const dram_ptr, size_t size) +static inline const void *fast_get(struct mod_alloc_ctx *alloc, const void * const dram_ptr, + size_t size) { return dram_ptr; } -static inline void fast_put(struct k_heap *heap, struct k_mem_domain *mdom, const void *sram_ptr) {} +static inline void fast_put(struct mod_alloc_ctx *alloc, struct k_mem_domain *mdom, + const void *sram_ptr) {} #endif #endif /* __SOF_LIB_FAST_GET_H__ */ diff --git a/zephyr/lib/fast-get.c b/zephyr/lib/fast-get.c index edb490df0e06..6bc3cf7ce610 100644 --- a/zephyr/lib/fast-get.c +++ b/zephyr/lib/fast-get.c @@ -8,6 +8,7 @@ #include #include +#include #include #include #include @@ -125,8 +126,9 @@ static int fast_get_access_grant(struct k_mem_domain *mdom, void *addr, size_t s } #endif /* CONFIG_USERSPACE */ -const void *fast_get(struct k_heap *heap, const void *dram_ptr, size_t size) +const void *fast_get(struct mod_alloc_ctx *alloc, const void *dram_ptr, size_t size) { + struct k_heap *heap = alloc ? alloc->heap : NULL; #if CONFIG_USERSPACE bool current_is_userspace = thread_is_userspace(k_current_get()); #endif @@ -266,8 +268,9 @@ static struct sof_fast_get_entry *fast_put_find_entry(struct sof_fast_get_data * return NULL; } -void fast_put(struct k_heap *heap, struct k_mem_domain *mdom, const void *sram_ptr) +void fast_put(struct mod_alloc_ctx *alloc, struct k_mem_domain *mdom, const void *sram_ptr) { + struct k_heap *heap = alloc ? alloc->heap : NULL; struct sof_fast_get_data *data = &fast_get_data; struct sof_fast_get_entry *entry; k_spinlock_key_t key; From 0de748bb88702ffa6a2540e0977968d71b37c8f2 Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Wed, 29 Apr 2026 09:52:00 +0200 Subject: [PATCH 14/15] fast-get: add support for vregion allocations When userspace modules use fast-get to access smaller buffers, their copies should be allocated on module's vregion, if available, not on the global SOF heap. Signed-off-by: Guennadi Liakhovetski --- zephyr/lib/fast-get.c | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/zephyr/lib/fast-get.c b/zephyr/lib/fast-get.c index 6bc3cf7ce610..e0850c381a33 100644 --- a/zephyr/lib/fast-get.c +++ b/zephyr/lib/fast-get.c @@ -10,6 +10,7 @@ #include #include +#include #include #include #include @@ -217,12 +218,12 @@ const void *fast_get(struct mod_alloc_ctx *alloc, const void *dram_ptr, size_t s goto out; } - /* - * If a userspace threads is the first user to fast-get the buffer, an - * SRAM copy will be allocated on its own heap, so it will have access - * to it - */ - ret = sof_heap_alloc(heap, alloc_flags, alloc_size, alloc_align); + if (alloc && alloc->vreg && size <= FAST_GET_MAX_COPY_SIZE) + /* A userspace allocation, that won't be shared */ + ret = vregion_alloc_align(alloc->vreg, VREGION_MEM_TYPE_INTERIM, alloc_size, + alloc_align); + else + ret = sof_heap_alloc(heap, alloc_flags, alloc_size, alloc_align); if (!ret) goto out; @@ -286,7 +287,10 @@ void fast_put(struct mod_alloc_ctx *alloc, struct k_mem_domain *mdom, const void if (!entry->refcount) { LOG_DBG("freeing buffer %p", sram_ptr); - sof_heap_free(heap, entry->sram_ptr); + if (alloc && alloc->vreg && entry->size <= FAST_GET_MAX_COPY_SIZE) + vregion_free(alloc->vreg, entry->sram_ptr); + else + sof_heap_free(heap, entry->sram_ptr); } #if CONFIG_USERSPACE From 9d97d233f174dadb29821e3a6e9c5ec6fdc2d4f5 Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Tue, 14 Apr 2026 15:12:18 +0200 Subject: [PATCH 15/15] userspace: application: move to vregion Modules, running in userspace while using the "application" DP implementation, are moved to vregion for all their private allocations. This has the advantage of not depending on build-time configured VMH buffers and of faster lifetime allocations. Signed-off-by: Guennadi Liakhovetski --- src/audio/buffers/comp_buffer.c | 10 +-- src/audio/module_adapter/module/generic.c | 51 ++++++++--- src/audio/module_adapter/module_adapter.c | 86 +++++++++---------- src/include/sof/audio/component.h | 1 - src/include/sof/objpool.h | 2 + src/ipc/ipc4/helper.c | 3 +- src/lib/objpool.c | 21 ++++- src/schedule/zephyr_dp_schedule_application.c | 4 +- zephyr/Kconfig | 1 + 9 files changed, 111 insertions(+), 68 deletions(-) diff --git a/src/audio/buffers/comp_buffer.c b/src/audio/buffers/comp_buffer.c index a66e83e5edf9..425b192603ab 100644 --- a/src/audio/buffers/comp_buffer.c +++ b/src/audio/buffers/comp_buffer.c @@ -157,14 +157,12 @@ static void comp_buffer_free(struct sof_audio_buffer *audio_buffer) struct mod_alloc_ctx *alloc = buffer->audio_buffer.alloc; rfree(buffer->stream.addr); - if (alloc && alloc->vreg) + if (alloc && alloc->vreg) { vregion_free(alloc->vreg, buffer); - else + if (!vregion_put(alloc->vreg)) + rfree(alloc); + } else { sof_heap_free(alloc ? alloc->heap : NULL, buffer); - - if (alloc && !--alloc->client_count) { - vregion_put(alloc->vreg); - rfree(alloc); } } diff --git a/src/audio/module_adapter/module/generic.c b/src/audio/module_adapter/module/generic.c index ff7a88495c20..2989bceb160b 100644 --- a/src/audio/module_adapter/module/generic.c +++ b/src/audio/module_adapter/module/generic.c @@ -17,6 +17,7 @@ #include #include #include +#include #include #if CONFIG_IPC_MAJOR_4 #include @@ -87,6 +88,7 @@ void mod_resource_init(struct processing_module *mod) /* Init memory list */ list_init(&res->objpool.list); res->objpool.heap = res->alloc->heap; + res->objpool.vreg = res->alloc->vreg; res->heap_usage = 0; res->heap_high_water_mark = 0; } @@ -158,11 +160,15 @@ void mod_heap_info(struct processing_module *mod, size_t *size, uintptr_t *start { struct module_resources *res = &mod->priv.resources; - if (size) - *size = res->alloc->heap->heap.init_bytes; + if (res->alloc->vreg) { + vregion_mem_info(res->alloc->vreg, size, start); + } else if (res->alloc->heap) { + if (size) + *size = res->alloc->heap->heap.init_bytes; - if (start) - *start = (uintptr_t)res->alloc->heap; + if (start) + *start = (uintptr_t)res->alloc->heap->heap.init_mem; + } } #endif @@ -246,7 +252,16 @@ void *z_impl_mod_alloc_ext(struct processing_module *mod, uint32_t flags, size_t } /* Allocate memory for module */ - void *ptr = sof_heap_alloc(res->alloc->heap, flags, size, alignment); + void *ptr; + + if (!res->alloc->vreg) + ptr = sof_heap_alloc(res->alloc->heap, flags, size, alignment); + else if (flags & SOF_MEM_FLAG_COHERENT) + ptr = vregion_alloc_coherent_align(res->alloc->vreg, VREGION_MEM_TYPE_INTERIM, + size, alignment); + else + ptr = vregion_alloc_align(res->alloc->vreg, VREGION_MEM_TYPE_INTERIM, + size, alignment); if (!ptr) { comp_err(mod->dev, "Failed to alloc %zu bytes %zu alignment for comp %#x.", @@ -347,7 +362,10 @@ static int free_contents(struct processing_module *mod, struct module_resource * switch (container->type) { case MOD_RES_HEAP: - sof_heap_free(res->alloc->heap, container->ptr); + if (res->alloc->vreg) + vregion_free(res->alloc->vreg, container->ptr); + else + sof_heap_free(res->alloc->heap, container->ptr); res->heap_usage -= container->size; return 0; #if CONFIG_COMP_BLOB @@ -426,10 +444,13 @@ EXPORT_SYMBOL(z_impl_mod_free); const void *z_vrfy_mod_fast_get(struct processing_module *mod, const void * const dram_ptr, size_t size) { - struct module_resources *res = &mod->priv.resources; + size_t h_size = 0; + uintptr_t h_start; K_OOPS(K_SYSCALL_MEMORY_WRITE(mod, sizeof(*mod))); - K_OOPS(K_SYSCALL_MEMORY_WRITE(res->alloc->heap, sizeof(*res->alloc->heap))); + mod_heap_info(mod, &h_size, &h_start); + if (h_size) + K_OOPS(K_SYSCALL_MEMORY_WRITE(h_start, h_size)); K_OOPS(K_SYSCALL_MEMORY_READ(dram_ptr, size)); return z_impl_mod_fast_get(mod, dram_ptr, size); @@ -440,10 +461,13 @@ const void *z_vrfy_mod_fast_get(struct processing_module *mod, const void * cons void *z_vrfy_mod_alloc_ext(struct processing_module *mod, uint32_t flags, size_t size, size_t alignment) { - struct module_resources *res = &mod->priv.resources; + size_t h_size = 0; + uintptr_t h_start; K_OOPS(K_SYSCALL_MEMORY_WRITE(mod, sizeof(*mod))); - K_OOPS(K_SYSCALL_MEMORY_WRITE(res->alloc->heap, sizeof(*res->alloc->heap))); + mod_heap_info(mod, &h_size, &h_start); + if (h_size) + K_OOPS(K_SYSCALL_MEMORY_WRITE(h_start, h_size)); return z_impl_mod_alloc_ext(mod, flags, size, alignment); } @@ -451,10 +475,13 @@ void *z_vrfy_mod_alloc_ext(struct processing_module *mod, uint32_t flags, size_t int z_vrfy_mod_free(struct processing_module *mod, const void *ptr) { - struct module_resources *res = &mod->priv.resources; + size_t h_size = 0; + uintptr_t h_start; K_OOPS(K_SYSCALL_MEMORY_WRITE(mod, sizeof(*mod))); - K_OOPS(K_SYSCALL_MEMORY_WRITE(res->alloc->heap, sizeof(*res->alloc->heap))); + mod_heap_info(mod, &h_size, &h_start); + if (h_size) + K_OOPS(K_SYSCALL_MEMORY_WRITE(h_start, h_size)); return z_impl_mod_free(mod, ptr); } diff --git a/src/audio/module_adapter/module_adapter.c b/src/audio/module_adapter/module_adapter.c index b6c87e09a4f8..772af199f0c1 100644 --- a/src/audio/module_adapter/module_adapter.c +++ b/src/audio/module_adapter/module_adapter.c @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -57,38 +58,29 @@ struct comp_dev *module_adapter_new(const struct comp_driver *drv, #define PAGE_SZ HOST_PAGE_SIZE #endif -static struct k_heap *module_adapter_dp_heap_new(const struct comp_ipc_config *config, - size_t *heap_size) +static struct vregion *module_adapter_dp_heap_new(const struct comp_ipc_config *config, + size_t *heap_size) { /* src-lite with 8 channels has been seen allocating 14k in one go */ /* FIXME: the size will be derived from configuration */ const size_t buf_size = 20 * 1024; - /* Keep uncached to match the default SOF heap! */ - uint8_t *mod_heap_mem = rballoc_align(SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, - buf_size, PAGE_SZ); - - if (!mod_heap_mem) - return NULL; - - struct k_heap *mod_heap = (struct k_heap *)mod_heap_mem; - const size_t heap_prefix_size = ALIGN_UP(sizeof(*mod_heap), 4); - void *mod_heap_buf = mod_heap_mem + heap_prefix_size; - - *heap_size = buf_size - heap_prefix_size; - k_heap_init(mod_heap, mod_heap_buf, *heap_size); -#ifdef __ZEPHYR__ - mod_heap->heap.init_mem = mod_heap_buf; - mod_heap->heap.init_bytes = *heap_size; -#endif - - return mod_heap; + /* + * A 1-to-1 replacement of the original heap implementation would be to + * have "lifetime size" equal to 0. But (1) this is invalid for + * vregion_create() and (2) we gradually move objects, that are simple + * to move to the lifetime buffer. Make it 1k for the beginning. + */ + return vregion_create(4096, buf_size - 4096); } static struct processing_module *module_adapter_mem_alloc(const struct comp_driver *drv, const struct comp_ipc_config *config) { struct k_heap *mod_heap; + struct vregion *mod_vreg; + struct processing_module *mod; + struct comp_dev *dev; /* * For DP shared modules the struct processing_module object must be * accessible from all cores. Unfortunately at this point there's no @@ -102,17 +94,24 @@ static struct processing_module *module_adapter_mem_alloc(const struct comp_driv if (config->proc_domain == COMP_PROCESSING_DOMAIN_DP && IS_ENABLED(CONFIG_USERSPACE) && !IS_ENABLED(CONFIG_SOF_USERSPACE_USE_DRIVER_HEAP)) { - mod_heap = module_adapter_dp_heap_new(config, &heap_size); - if (!mod_heap) { - comp_cl_err(drv, "Failed to allocate DP module heap"); + mod_vreg = module_adapter_dp_heap_new(config, &heap_size); + if (!mod_vreg) { + comp_cl_err(drv, "Failed to allocate DP module heap / vregion"); return NULL; } + mod_heap = NULL; } else { mod_heap = drv->user_heap; heap_size = 0; + mod_vreg = NULL; } - struct processing_module *mod = sof_heap_alloc(mod_heap, flags, sizeof(*mod), 0); + if (!mod_vreg) + mod = sof_heap_alloc(mod_heap, flags, sizeof(*mod), 0); + else if (flags & SOF_MEM_FLAG_COHERENT) + mod = vregion_alloc_coherent(mod_vreg, VREGION_MEM_TYPE_LIFETIME, sizeof(*mod)); + else + mod = vregion_alloc(mod_vreg, VREGION_MEM_TYPE_LIFETIME, sizeof(*mod)); if (!mod) { comp_cl_err(drv, "failed to allocate memory for module"); @@ -126,8 +125,7 @@ static struct processing_module *module_adapter_mem_alloc(const struct comp_driv memset(mod, 0, sizeof(*mod)); alloc->heap = mod_heap; - alloc->vreg = NULL; - alloc->client_count = 0; + alloc->vreg = mod_vreg; mod->priv.resources.alloc = alloc; mod_resource_init(mod); @@ -137,7 +135,10 @@ static struct processing_module *module_adapter_mem_alloc(const struct comp_driv * then it can be cached. Effectively it can be only cached in * single-core configurations. */ - struct comp_dev *dev = sof_heap_alloc(mod_heap, SOF_MEM_FLAG_COHERENT, sizeof(*dev), 0); + if (mod_vreg) + dev = vregion_alloc_coherent(mod_vreg, VREGION_MEM_TYPE_LIFETIME, sizeof(*dev)); + else + dev = sof_heap_alloc(mod_heap, SOF_MEM_FLAG_COHERENT, sizeof(*dev), 0); if (!dev) { comp_cl_err(drv, "failed to allocate memory for comp_dev"); @@ -150,17 +151,17 @@ static struct processing_module *module_adapter_mem_alloc(const struct comp_driv mod->dev = dev; dev->mod = mod; - if (config->proc_domain == COMP_PROCESSING_DOMAIN_DP) - alloc->client_count++; - return mod; edev: rfree(alloc); ealloc: - sof_heap_free(mod_heap, mod); + if (mod_vreg) + vregion_free(mod_vreg, mod); + else + sof_heap_free(mod_heap, mod); emod: - rfree(mod_heap); + vregion_put(mod_vreg); return NULL; } @@ -169,7 +170,6 @@ static void module_adapter_mem_free(struct processing_module *mod) { struct mod_alloc_ctx *alloc = mod->priv.resources.alloc; struct k_heap *mod_heap = alloc->heap; - unsigned int domain = mod->dev->ipc_config.proc_domain; /* * In principle it shouldn't even be needed to free individual objects @@ -178,14 +178,16 @@ static void module_adapter_mem_free(struct processing_module *mod) #if CONFIG_IPC_MAJOR_4 sof_heap_free(mod_heap, mod->priv.cfg.input_pins); #endif - sof_heap_free(mod_heap, mod->dev); - sof_heap_free(mod_heap, mod); - if (domain == COMP_PROCESSING_DOMAIN_DP) { - if (!IS_ENABLED(CONFIG_SOF_USERSPACE_USE_DRIVER_HEAP)) - rfree(mod_heap); - if (!--alloc->client_count) + if (alloc->vreg) { + struct vregion *mod_vreg = alloc->vreg; + + vregion_free(mod_vreg, mod->dev); + vregion_free(mod_vreg, mod); + if (!vregion_put(mod_vreg)) rfree(alloc); } else { + sof_heap_free(mod_heap, mod->dev); + sof_heap_free(mod_heap, mod); rfree(alloc); } } @@ -631,9 +633,7 @@ int module_adapter_prepare(struct comp_dev *dev) goto free; } - if (md->resources.alloc->heap && - md->resources.alloc->heap != dev->drv->user_heap) - md->resources.alloc->client_count++; + vregion_get(md->resources.alloc->vreg); irq_local_disable(flags); list_item_prepend(&buffer->buffers_list, &mod->raw_data_buffers_list); diff --git a/src/include/sof/audio/component.h b/src/include/sof/audio/component.h index 2df7e6a9bd42..c0e737a55b9b 100644 --- a/src/include/sof/audio/component.h +++ b/src/include/sof/audio/component.h @@ -583,7 +583,6 @@ struct vregion; struct mod_alloc_ctx { struct k_heap *heap; struct vregion *vreg; - unsigned int client_count; }; /** diff --git a/src/include/sof/objpool.h b/src/include/sof/objpool.h index 697a19cd2336..0821fec8786b 100644 --- a/src/include/sof/objpool.h +++ b/src/include/sof/objpool.h @@ -12,9 +12,11 @@ #include #include +struct vregion; struct k_heap; struct objpool_head { struct list_item list; + struct vregion *vreg; struct k_heap *heap; uint32_t flags; }; diff --git a/src/ipc/ipc4/helper.c b/src/ipc/ipc4/helper.c index d6cf1c24561f..a81a75312f13 100644 --- a/src/ipc/ipc4/helper.c +++ b/src/ipc/ipc4/helper.c @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -739,7 +740,7 @@ __cold int ipc_comp_connect(struct ipc *ipc, ipc_pipe_comp_connect *_connect) #if CONFIG_ZEPHYR_DP_SCHEDULER if (alloc) - alloc->client_count++; + vregion_get(alloc->vreg); #endif /* diff --git a/src/lib/objpool.c b/src/lib/objpool.c index 3df78103e13c..6925e6f7070a 100644 --- a/src/lib/objpool.c +++ b/src/lib/objpool.c @@ -7,6 +7,7 @@ #include #include #include +#include #include #include @@ -37,8 +38,17 @@ static int objpool_add(struct objpool_head *head, unsigned int n, size_t size, u if (!head->heap) head->heap = sof_sys_heap_get(); - struct objpool *pobjpool = sof_heap_alloc(head->heap, flags, - aligned_size + sizeof(*pobjpool), 0); + struct objpool *pobjpool; + + if (!head->vreg) + pobjpool = sof_heap_alloc(head->heap, flags, + aligned_size + sizeof(*pobjpool), 0); + else if (flags & SOF_MEM_FLAG_COHERENT) + pobjpool = vregion_alloc_coherent(head->vreg, VREGION_MEM_TYPE_INTERIM, + aligned_size + sizeof(*pobjpool)); + else + pobjpool = vregion_alloc(head->vreg, VREGION_MEM_TYPE_INTERIM, + aligned_size + sizeof(*pobjpool)); if (!pobjpool) return -ENOMEM; @@ -150,8 +160,13 @@ void objpool_prune(struct objpool_head *head) struct list_item *next, *tmp; list_for_item_safe(next, tmp, &head->list) { + struct objpool *pool = container_of(next, struct objpool, list); + list_item_del(next); - sof_heap_free(head->heap, container_of(next, struct objpool, list)); + if (head->vreg) + vregion_free(head->vreg, pool); + else + sof_heap_free(head->heap, pool); } } diff --git a/src/schedule/zephyr_dp_schedule_application.c b/src/schedule/zephyr_dp_schedule_application.c index 35dd072040c4..099ffb7c9a5d 100644 --- a/src/schedule/zephyr_dp_schedule_application.c +++ b/src/schedule/zephyr_dp_schedule_application.c @@ -526,12 +526,12 @@ int scheduler_dp_task_init(struct task **task, const struct sof_uuid_entry *uid, /* Module heap partition */ mod_heap_info(mod, &size, &start); pdata->mpart[SOF_DP_PART_HEAP] = (struct k_mem_partition){ - .start = start, + .start = (uintptr_t)sys_cache_uncached_ptr_get((void *)start), .size = size, .attr = K_MEM_PARTITION_P_RW_U_RW, }; pdata->mpart[SOF_DP_PART_HEAP_CACHE] = (struct k_mem_partition){ - .start = (uintptr_t)sys_cache_cached_ptr_get((void *)start), + .start = start, .size = size, .attr = K_MEM_PARTITION_P_RW_U_RW | XTENSA_MMU_CACHED_WB, }; diff --git a/zephyr/Kconfig b/zephyr/Kconfig index f3bc736ee6d0..f1d1896c4234 100644 --- a/zephyr/Kconfig +++ b/zephyr/Kconfig @@ -143,6 +143,7 @@ config SOF_USERSPACE_PROXY_WORKER_STACK_SIZE config SOF_USERSPACE_APPLICATION bool default USERSPACE && !SOF_USERSPACE_PROXY + depends on SOF_VREGIONS help Not manually settable. This is effectively a shortcut to replace numerous checks for (CONFIG_USERSPACE && !CONFIG_SOF_USERSPACE_PROXY)