dp: convert to user-space

Run DP threads in user-space. Move all the respective memory and
kobjects to a dedicated memory domain. Work around Zephyr inability
to remove memory domains on Xtensa.

Signed-off-by: Guennadi Liakhovetski <[email protected]>

Author: lyakh

app/boards/intel_adsp_ace30_ptl.conf

@@ -42,6 +42,7 @@ CONFIG_LLEXT=y
 CONFIG_LLEXT_STORAGE_WRITABLE=y
 CONFIG_LLEXT_EXPERIMENTAL=y
 CONFIG_MODULES=y
+CONFIG_USERSPACE=y
 
 # Zephyr / device drivers
 CONFIG_DAI_INIT_PRIORITY=70

app/prj.conf

@@ -48,3 +48,9 @@ CONFIG_SCHED_CPU_MASK=y
 CONFIG_SYS_CLOCK_TICKS_PER_SEC=15000
 CONFIG_DAI=y
 CONFIG_HEAP_MEM_POOL_SIZE=2048
+
+# Support dynamic thread stack allocation
+CONFIG_DYNAMIC_THREAD=y
+CONFIG_DYNAMIC_THREAD_ALLOC=y
+CONFIG_DYNAMIC_THREAD_PREFER_ALLOC=y
+CONFIG_MAX_THREAD_BYTES=3

src/audio/buffers/comp_buffer.c

@@ -18,6 +18,7 @@
 #include <rtos/cache.h>
 #include <sof/lib/notifier.h>
 #include <sof/list.h>
+#include <sof/schedule/dp_schedule.h>
 #include <rtos/spinlock.h>
 #include <rtos/symbol.h>
 #include <ipc/topology.h>
@@ -158,8 +159,16 @@ static void comp_buffer_free(struct sof_audio_buffer *audio_buffer)
 	/* In case some listeners didn't unregister from buffer's callbacks */
 	notifier_unregister_all(NULL, buffer);
 
+	struct k_heap *heap = buffer->audio_buffer.heap;
+
 	rfree(buffer->stream.addr);
-	sof_heap_free(buffer->audio_buffer.heap, buffer);
+	sof_heap_free(heap, buffer);
+	if (heap) {
+		struct dp_heap_user *mod_heap_user = container_of(heap, struct dp_heap_user, heap);
+
+		if (!--mod_heap_user->client_count)
+			rfree(mod_heap_user);
+	}
 }
 
 APP_TASK_DATA static const struct source_ops comp_buffer_source_ops = {

src/audio/module_adapter/module/generic.c

@@ -78,7 +78,7 @@ int module_load_config(struct comp_dev *dev, const void *cfg, size_t size)
 	return ret;
 }
 
-static void mod_resource_init(struct processing_module *mod)
+void mod_resource_init(struct processing_module *mod)
 {
 	struct module_data *md = &mod->priv;
 	/* Init memory list */
@@ -116,7 +116,6 @@ int module_init(struct processing_module *mod)
 		return -EIO;
 	}
 
-	mod_resource_init(mod);
 #if CONFIG_MODULE_MEMORY_API_DEBUG && defined(__ZEPHYR__)
 	mod->priv.resources.rsrc_mngr = k_current_get();
 #endif
@@ -181,6 +180,17 @@ static void container_put(struct processing_module *mod, struct module_resource
 	list_item_append(&container->list, &res->free_cont_list);
 }
 
+void mod_heap_info(struct processing_module *mod, size_t *size, uintptr_t *start)
+{
+	struct module_resources *res = &mod->priv.resources;
+
+	if (size)
+		*size = res->heap_size;
+
+	if (start)
+		*start = (uintptr_t)container_of(res->heap, struct dp_heap_user, heap);
+}
+
 /**
  * Allocates aligned buffer memory block for module.
  * @param mod		Pointer to the module this memory block is allocated for.
@@ -416,6 +426,7 @@ int z_impl_mod_free(struct processing_module *mod, const void *ptr)
 EXPORT_SYMBOL(z_impl_mod_free);
 
 #ifdef CONFIG_USERSPACE
+#include <zephyr/internal/syscall_handler.h>
 const void *z_vrfy_mod_fast_get(struct processing_module *mod, const void * const dram_ptr,
 				size_t size)
 {

src/audio/module_adapter/module_adapter.c

@@ -58,24 +58,27 @@ struct comp_dev *module_adapter_new(const struct comp_driver *drv,
 #define PAGE_SZ HOST_PAGE_SIZE
 #endif
 
-static struct k_heap *module_adapter_dp_heap_new(const struct comp_ipc_config *config)
+static struct k_heap *module_adapter_dp_heap_new(const struct comp_ipc_config *config,
+						 size_t *heap_size)
 {
 	/* src-lite with 8 channels has been seen allocating 14k in one go */
 	/* FIXME: the size will be derived from configuration */
-	const size_t heap_size = 20 * 1024;
+	const size_t buf_size = 20 * 1024;
 
 	/* Keep uncached to match the default SOF heap! */
 	uint8_t *mod_heap_mem = rballoc_align(SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT,
-					      heap_size, PAGE_SZ);
+					      buf_size, PAGE_SZ);
 
 	if (!mod_heap_mem)
 		return NULL;
 
-	struct k_heap *mod_heap = (struct k_heap *)mod_heap_mem;
-	const size_t heap_prefix_size = ALIGN_UP(sizeof(*mod_heap), 8);
+	struct dp_heap_user *mod_heap_user = (struct dp_heap_user *)mod_heap_mem;
+	struct k_heap *mod_heap = &mod_heap_user->heap;
+	const size_t heap_prefix_size = ALIGN_UP(sizeof(*mod_heap_user), 4);
 	void *mod_heap_buf = mod_heap_mem + heap_prefix_size;
 
-	k_heap_init(mod_heap, mod_heap_buf, heap_size - heap_prefix_size);
+	*heap_size = buf_size - heap_prefix_size;
+	k_heap_init(mod_heap, mod_heap_buf, *heap_size);
 
 	return mod_heap;
 }
@@ -93,16 +96,21 @@ static struct processing_module *module_adapter_mem_alloc(const struct comp_driv
 	 */
 	uint32_t flags = config->proc_domain == COMP_PROCESSING_DOMAIN_DP ?
 		SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT : SOF_MEM_FLAG_USER;
+	struct dp_heap_user *mod_heap_user;
+	size_t heap_size;
 
 	if (config->proc_domain == COMP_PROCESSING_DOMAIN_DP && IS_ENABLED(CONFIG_USERSPACE) &&
 	    !IS_ENABLED(CONFIG_SOF_USERSPACE_USE_DRIVER_HEAP)) {
-		mod_heap = module_adapter_dp_heap_new(config);
+		mod_heap = module_adapter_dp_heap_new(config, &heap_size);
 		if (!mod_heap) {
 			comp_cl_err(drv, "Failed to allocate DP module heap");
 			return NULL;
 		}
+		mod_heap_user = container_of(mod_heap, struct dp_heap_user, heap);
 	} else {
 		mod_heap = drv->user_heap;
+		mod_heap_user = NULL;
+		heap_size = 0;
 	}
 
 	struct processing_module *mod = sof_heap_alloc(mod_heap, flags, sizeof(*mod), 0);
@@ -114,6 +122,8 @@ static struct processing_module *module_adapter_mem_alloc(const struct comp_driv
 
 	memset(mod, 0, sizeof(*mod));
 	mod->priv.resources.heap = mod_heap;
+	mod->priv.resources.heap_size = heap_size;
+	mod_resource_init(mod);
 
 	/*
 	 * Would be difficult to optimize the allocation to use cache. Only if
@@ -134,26 +144,37 @@ static struct processing_module *module_adapter_mem_alloc(const struct comp_driv
 	mod->dev = dev;
 	dev->mod = mod;
 
+	if (mod_heap_user)
+		mod_heap_user->client_count++;
+
 	return mod;
 
 err:
 	sof_heap_free(mod_heap, mod);
 emod:
-	if (mod_heap != drv->user_heap)
-		rfree(mod_heap);
+	rfree(mod_heap_user);
 
 	return NULL;
 }
 
 static void module_adapter_mem_free(struct processing_module *mod)
 {
 	struct k_heap *mod_heap = mod->priv.resources.heap;
+	struct dp_heap_user *mod_heap_user = container_of(mod_heap, struct dp_heap_user, heap);
 
+	/*
+	 * In principle it shouldn't even be needed to free individual objects
+	 * on the module heap since we're freeing the heap itself too
+	 */
 #if CONFIG_IPC_MAJOR_4
 	sof_heap_free(mod_heap, mod->priv.cfg.input_pins);
 #endif
+	list_item_del(&mod->dev->bsource_list);
+	list_item_del(&mod->dev->bsink_list);
 	sof_heap_free(mod_heap, mod->dev);
 	sof_heap_free(mod_heap, mod);
+	if (!mod_heap || !--mod_heap_user->client_count)
+		rfree(mod_heap_user);
 }
 
 /*
@@ -1374,11 +1395,16 @@ void module_adapter_free(struct comp_dev *dev)
 	comp_dbg(dev, "start");
 
 	if (dev->task) {
-		/* Run DP module's .free() method in its thread context */
+		/*
+		 * Run DP module's .free() method in its thread context.
+		 * Unlike with other IPCs we first run module's .free() in
+		 * thread context, then cancel the thread, and then execute
+		 * final clean up
+		 */
 #if CONFIG_IPC_MAJOR_4
 		scheduler_dp_thread_ipc(mod, SOF_IPC4_MOD_DELETE_INSTANCE, NULL);
 #endif
-		schedule_task_cancel(dev->task);
+		schedule_task_free(dev->task);
 	}
 
 	ret = module_free(mod);

src/include/sof/audio/component_ext.h

@@ -44,8 +44,6 @@ struct comp_dev *comp_new_ipc4(struct ipc4_module_init_instance *module_init);
 /** See comp_ops::free */
 static inline void comp_free(struct comp_dev *dev)
 {
-	struct task *task = dev->is_shared ||
-		dev->ipc_config.proc_domain == COMP_PROCESSING_DOMAIN_DP ? dev->task : NULL;
 	const struct comp_driver *drv = dev->drv;
 
 	assert(drv->ops.free);
@@ -55,12 +53,6 @@ static inline void comp_free(struct comp_dev *dev)
 	 * be freed after this.
 	 */
 	drv->ops.free(dev);
-
-	/* free task if shared component or DP task*/
-	if (task) {
-		schedule_task_free(task);
-		sof_heap_free(drv->user_heap, task);
-	}
 }
 
 /**

src/include/sof/audio/module_adapter/module/generic.h

@@ -134,6 +134,7 @@ struct module_resources {
 	size_t heap_usage;
 	size_t heap_high_water_mark;
 	struct k_heap *heap;
+	size_t heap_size;
 #if CONFIG_MODULE_MEMORY_API_DEBUG && defined(__ZEPHYR__)
 	k_tid_t rsrc_mngr;
 #endif
@@ -191,9 +192,19 @@ struct module_processing_data {
 int module_load_config(struct comp_dev *dev, const void *cfg, size_t size);
 int module_init(struct processing_module *mod);
 void *mod_balloc_align(struct processing_module *mod, size_t size, size_t alignment);
+void mod_resource_init(struct processing_module *mod);
+void mod_heap_info(struct processing_module *mod, size_t *size, uintptr_t *start);
+#if defined(__ZEPHYR__) && defined(CONFIG_SOF_FULL_ZEPHYR_APPLICATION)
 __syscall void *mod_alloc_ext(struct processing_module *mod, uint32_t flags, size_t size,
 			      size_t alignment);
 __syscall int mod_free(struct processing_module *mod, const void *ptr);
+#else
+void *z_impl_mod_alloc_ext(struct processing_module *mod, uint32_t flags, size_t size,
+			   size_t alignment);
+int z_impl_mod_free(struct processing_module *mod, const void *ptr);
+#define mod_alloc_ext z_impl_mod_alloc_ext
+#define mod_free z_impl_mod_free
+#endif
 
 /**
  * Allocates aligned memory block for module.
@@ -234,17 +245,21 @@ struct comp_data_blob_handler *mod_data_blob_handler_new(struct processing_modul
 void mod_data_blob_handler_free(struct processing_module *mod, struct comp_data_blob_handler *dbh);
 #endif
 #if CONFIG_FAST_GET
+#if defined(__ZEPHYR__) && defined(CONFIG_SOF_FULL_ZEPHYR_APPLICATION)
 __syscall const void *mod_fast_get(struct processing_module *mod, const void * const dram_ptr,
 				   size_t size);
+#else
+const void *z_impl_mod_fast_get(struct processing_module *mod, const void * const dram_ptr,
+				size_t size);
+#define mod_fast_get z_impl_mod_fast_get
+#endif
 void mod_fast_put(struct processing_module *mod, const void *sram_ptr);
 #endif
 void mod_free_all(struct processing_module *mod);
 int module_prepare(struct processing_module *mod,
 		   struct sof_source **sources, int num_of_sources,
 		   struct sof_sink **sinks, int num_of_sinks);
 
-#include <zephyr/syscalls/generic.h>
-
 static inline
 bool generic_module_is_ready_to_process(struct processing_module *mod,
 					struct sof_source **sources,
@@ -454,4 +469,8 @@ static inline uint32_t module_get_lpt(struct processing_module *mod)
 	return mod->dev->period;
 }
 
+#if defined(__ZEPHYR__) && defined(CONFIG_SOF_FULL_ZEPHYR_APPLICATION)
+#include <zephyr/syscalls/generic.h>
+#endif
+
 #endif /* __SOF_AUDIO_MODULE_GENERIC__ */

src/include/sof/lib/fast-get.h

@@ -12,7 +12,7 @@
 
 struct k_heap;
 
-#if defined(__ZEPHYR__) && defined(CONFIG_SOF)
+#if defined(__ZEPHYR__) && defined(CONFIG_SOF_FULL_ZEPHYR_APPLICATION)
 #include <zephyr/toolchain.h>
 
 __syscall const void *fast_get(struct k_heap *heap, const void * const dram_ptr, size_t size);

src/include/sof/schedule/dp_schedule.h

@@ -103,12 +103,18 @@ union scheduler_dp_thread_ipc_param {
 	} pipeline_state;
 };
 
+struct dp_heap_user {
+	struct k_heap heap;
+	unsigned int client_count;	/* devices and buffers */
+	// lock;			/* protect the counter */
+};
+
 #if CONFIG_ZEPHYR_DP_SCHEDULER
-int scheduler_dp_thread_ipc(struct processing_module *pmod, enum sof_ipc4_module_type cmd,
+int scheduler_dp_thread_ipc(struct processing_module *pmod, unsigned int cmd,
 			    union scheduler_dp_thread_ipc_param *param);
 #else
 static inline int scheduler_dp_thread_ipc(struct processing_module *pmod,
-					  enum sof_ipc4_module_type cmd,
+					  unsigned int cmd,
 					  union scheduler_dp_thread_ipc_param *param)
 {
 	return 0;

src/ipc/ipc4/helper.c

@@ -23,6 +23,7 @@
 #include <sof/lib/memory.h>
 #include <sof/list.h>
 #include <sof/platform.h>
+#include <sof/schedule/dp_schedule.h>
 #include <sof/schedule/ll_schedule_domain.h>
 #include <rtos/symbol.h>
 #include <rtos/wait.h>
@@ -600,6 +601,14 @@ __cold int ipc_comp_connect(struct ipc *ipc, ipc_pipe_comp_connect *_connect)
 		return IPC4_OUT_OF_MEMORY;
 	}
 
+#if CONFIG_ZEPHYR_DP_SCHEDULER
+	if (dp_heap) {
+		struct dp_heap_user *dp_user = container_of(dp_heap, struct dp_heap_user, heap);
+
+		dp_user->client_count++;
+	}
+#endif
+
 	/*
 	 * set min_free_space and min_available in sink/src api of created buffer.
 	 * buffer is connected like: