diff --git a/Kconfig.sof b/Kconfig.sof index be5bdc2f35b0..2c18ede53362 100644 --- a/Kconfig.sof +++ b/Kconfig.sof @@ -130,6 +130,13 @@ config COLD_STORE_EXECUTE_DRAM option to enable this feature to save SRAM and to speed up SRAM copying of performance-critical data and code. + +config COLD_STORE_EXECUTE_DEBUG + bool "Enable checks for cold code on hot paths" + help + This enables an assert_can_be_cold() check, which causes an exception + if called in the LL task context and assert() evaluation is enabled. + config FAST_GET bool "Enable simple refcounting DRAM data copier" default n diff --git a/app/debug_overlay.conf b/app/debug_overlay.conf index 3827976086d8..d020447ef02d 100644 --- a/app/debug_overlay.conf +++ b/app/debug_overlay.conf @@ -15,3 +15,5 @@ CONFIG_DAI_VERBOSE_GLITCH_WARNINGS=y # CONFIG_SYS_HEAP_VALIDATE=y # CONFIG_SPIN_VALIDATE=y # CONFIG_SPIN_LOCK_TIME_LIMIT=50000 + +CONFIG_COLD_STORE_EXECUTE_DEBUG=y diff --git a/posix/include/sof/lib/memory.h b/posix/include/sof/lib/memory.h index 9a784ec7f19d..219ebf2bc76c 100644 --- a/posix/include/sof/lib/memory.h +++ b/posix/include/sof/lib/memory.h @@ -18,4 +18,6 @@ #define __cold_rodata #endif +#define assert_can_be_cold() do {} while (0) + #endif /* __SOF_LIB_MEMORY_H__ */ diff --git a/src/audio/drc/drc.c b/src/audio/drc/drc.c index deaa5b1d8ad4..6b473f000bea 100644 --- a/src/audio/drc/drc.c +++ b/src/audio/drc/drc.c @@ -151,6 +151,8 @@ __cold static int drc_init(struct processing_module *mod) size_t bs = cfg->size; int ret; + assert_can_be_cold(); + comp_info(dev, "drc_init()"); /* Check first before proceeding with dev and cd that coefficients @@ -203,6 +205,8 @@ __cold static int drc_free(struct processing_module *mod) { struct drc_comp_data *cd = module_get_private_data(mod); + assert_can_be_cold(); + comp_data_blob_handler_free(cd->model_handler); rfree(cd); return 0; @@ -216,6 +220,8 @@ __cold static int drc_set_config(struct processing_module *mod, uint32_t param_i struct drc_comp_data *cd = module_get_private_data(mod); struct comp_dev *dev = mod->dev; + assert_can_be_cold(); + comp_dbg(dev, "drc_set_config()"); #if CONFIG_IPC_MAJOR_4 @@ -253,6 +259,8 @@ __cold static int drc_get_config(struct processing_module *mod, struct sof_ipc_ctrl_data *cdata = (struct sof_ipc_ctrl_data *)fragment; struct drc_comp_data *cd = module_get_private_data(mod); + assert_can_be_cold(); + comp_info(mod->dev, "drc_get_config()"); return comp_data_blob_get_cmd(cd->model_handler, cdata, fragment_size); diff --git a/src/audio/multiband_drc/multiband_drc.c b/src/audio/multiband_drc/multiband_drc.c index 62432357b4e4..3bb41ad775fb 100644 --- a/src/audio/multiband_drc/multiband_drc.c +++ b/src/audio/multiband_drc/multiband_drc.c @@ -285,6 +285,8 @@ __cold static int multiband_drc_free(struct processing_module *mod) { struct multiband_drc_comp_data *cd = module_get_private_data(mod); + assert_can_be_cold(); + comp_info(mod->dev, "multiband_drc_free()"); comp_data_blob_handler_free(cd->model_handler); @@ -301,6 +303,8 @@ __cold static int multiband_drc_set_config(struct processing_module *mod, uint32 { struct comp_dev *dev = mod->dev; + assert_can_be_cold(); + comp_dbg(dev, "multiband_drc_set_config()"); return multiband_drc_set_ipc_config(mod, param_id, @@ -313,6 +317,8 @@ __cold static int multiband_drc_get_config(struct processing_module *mod, { struct sof_ipc_ctrl_data *cdata = (struct sof_ipc_ctrl_data *)fragment; + assert_can_be_cold(); + comp_dbg(mod->dev, "multiband_drc_get_config()"); return multiband_drc_get_ipc_config(mod, cdata, fragment_size); @@ -364,6 +370,8 @@ __cold static int multiband_drc_prepare(struct processing_module *mod, int rate; int ret = 0; + assert_can_be_cold(); + comp_info(dev, "multiband_drc_prepare()"); ret = multiband_drc_params(mod); @@ -403,6 +411,8 @@ __cold static int multiband_drc_reset(struct processing_module *mod) { struct multiband_drc_comp_data *cd = module_get_private_data(mod); + assert_can_be_cold(); + comp_info(mod->dev, "multiband_drc_reset()"); multiband_drc_reset_state(&cd->state); diff --git a/src/audio/src/src.c b/src/audio/src/src.c index 5061d47abb22..9eeb93507033 100644 --- a/src/audio/src/src.c +++ b/src/audio/src/src.c @@ -42,6 +42,8 @@ __cold static int src_prepare(struct processing_module *mod, struct src_param *a = &cd->param; int ret; + assert_can_be_cold(); + comp_info(mod->dev, "src_prepare()"); if (num_of_sources != 1 || num_of_sinks != 1) diff --git a/src/audio/src/src_common.c b/src/audio/src/src_common.c index 7b8fc99dc5c0..9605a3b8bb87 100644 --- a/src/audio/src/src_common.c +++ b/src/audio/src/src_common.c @@ -54,6 +54,8 @@ __cold static int src_buffer_lengths(struct comp_dev *dev, struct comp_data *cd, int source_frames; int r1, n; + assert_can_be_cold(); + a = &cd->param; fs_in = cd->source_rate; fs_out = cd->sink_rate; @@ -192,6 +194,8 @@ __cold static int src_polyphase_init(struct polyphase_src *src, struct src_param int n_stages; int ret; + assert_can_be_cold(); + if (p->idx_in < 0 || p->idx_out < 0) return -EINVAL; @@ -400,6 +404,8 @@ __cold static int src_verify_params(struct processing_module *mod) struct comp_dev *dev = mod->dev; int ret; + assert_can_be_cold(); + comp_dbg(dev, "src_verify_params()"); /* check whether params->rate (received from driver) are equal @@ -485,6 +491,8 @@ __cold int src_params_general(struct processing_module *mod, int n; int err; + assert_can_be_cold(); + comp_info(dev, "src_params()"); err = src_set_params(mod, sink); @@ -581,6 +589,8 @@ __cold int src_param_set(struct comp_dev *dev, struct comp_data *cd) int fs_in = cd->source_rate; int fs_out = cd->sink_rate; + assert_can_be_cold(); + a->idx_in = src_find_fs(a->in_fs, a->num_in_fs, fs_in); a->idx_out = src_find_fs(a->out_fs, a->num_out_fs, fs_out); @@ -607,6 +617,8 @@ __cold int src_allocate_copy_stages(struct comp_dev *dev, struct src_param *prm, size_t tap_size = sizeof(int32_t); #endif + assert_can_be_cold(); + stage_dst = rmalloc(SOF_MEM_ZONE_RUNTIME, 0, SOF_MEM_CAPS_RAM, 2 * sizeof(*stage_dst)); if (!stage_dst) { @@ -672,12 +684,16 @@ __cold int src_set_config(struct processing_module *mod, uint32_t config_id, const uint8_t *fragment, size_t fragment_size, uint8_t *response, size_t response_size) { + assert_can_be_cold(); + return -EINVAL; } __cold int src_get_config(struct processing_module *mod, uint32_t config_id, uint32_t *data_offset_size, uint8_t *fragment, size_t fragment_size) { + assert_can_be_cold(); + return -EINVAL; } @@ -685,6 +701,8 @@ __cold int src_reset(struct processing_module *mod) { struct comp_data *cd = module_get_private_data(mod); + assert_can_be_cold(); + comp_info(mod->dev, "src_reset()"); cd->src_func = src_fallback; @@ -697,6 +715,8 @@ __cold int src_free(struct processing_module *mod) { struct comp_data *cd = module_get_private_data(mod); + assert_can_be_cold(); + comp_info(mod->dev, "src_free()"); /* Free dynamically reserved buffers for SRC algorithm */ diff --git a/src/audio/src/src_ipc4.c b/src/audio/src/src_ipc4.c index dab696a992cf..db32bc228838 100644 --- a/src/audio/src/src_ipc4.c +++ b/src/audio/src/src_ipc4.c @@ -86,6 +86,8 @@ __cold int src_set_params(struct processing_module *mod, struct sof_sink *sink) struct comp_dev *dev = mod->dev; int ret; + assert_can_be_cold(); + src_params = *params; src_params.channels = mod->priv.cfg.base_cfg.audio_fmt.channels_count; src_params.buffer_fmt = mod->priv.cfg.base_cfg.audio_fmt.interleaving_style; @@ -124,6 +126,8 @@ __cold void src_get_source_sink_params(struct comp_dev *dev, struct sof_source * struct comp_data *cd = module_get_private_data(mod); enum sof_ipc_frame frame_fmt, valid_fmt; + assert_can_be_cold(); + /* convert IPC4 config to format used by the module */ audio_stream_fmt_conversion(cd->ipc_config.base.audio_fmt.depth, cd->ipc_config.base.audio_fmt.valid_bit_depth, @@ -144,6 +148,8 @@ __cold int src_prepare_general(struct processing_module *mod, struct comp_dev *dev = mod->dev; int ret = 0; + assert_can_be_cold(); + /* set align requirements */ src_set_alignment(source, sink); @@ -187,6 +193,8 @@ __cold int src_init(struct processing_module *mod) struct comp_dev *dev = mod->dev; struct comp_data *cd = NULL; + assert_can_be_cold(); + comp_dbg(dev, "src_init()"); if (!cfg->init_data || cfg->size != sizeof(cd->ipc_config)) { diff --git a/src/debug/tester/tester_simple_dram_test.c b/src/debug/tester/tester_simple_dram_test.c index 5114c66725c2..5cf1917de3cd 100644 --- a/src/debug/tester/tester_simple_dram_test.c +++ b/src/debug/tester/tester_simple_dram_test.c @@ -36,6 +36,8 @@ __cold static int simple_dram_test_case_init(struct processing_module *mod, void struct tester_module_simple_dram_test_data *data = rzalloc(0, 0, SOF_MEM_CAPS_L3, sizeof(*data)); + assert_can_be_cold(); + if (!data) return -ENOMEM; @@ -63,6 +65,8 @@ __cold static int simple_dram_test_case_process(void *ctx, struct processing_mod { struct tester_module_simple_dram_test_data *data = ctx; + assert_can_be_cold(); + /* copy every second cycle */ *do_copy = data->do_copy_data; data->do_copy_data = !data->do_copy_data; @@ -72,6 +76,8 @@ __cold static int simple_dram_test_case_process(void *ctx, struct processing_mod __cold static int simple_dram_test_free(void *ctx, struct processing_module *mod) { + assert_can_be_cold(); + rfree(ctx); return 0; } diff --git a/src/include/sof/schedule/ll_schedule_domain.h b/src/include/sof/schedule/ll_schedule_domain.h index 9765df5ef0bc..dbc32656dd2f 100644 --- a/src/include/sof/schedule/ll_schedule_domain.h +++ b/src/include/sof/schedule/ll_schedule_domain.h @@ -240,6 +240,7 @@ struct ll_schedule_domain *zephyr_dma_domain_init(struct dma *dma_array, uint32_t num_dma, int clk); #endif /* CONFIG_DMA_DOMAIN */ +struct ll_schedule_domain *zephyr_ll_domain(void); struct ll_schedule_domain *zephyr_domain_init(int clk); #define timer_domain_init(timer, clk) zephyr_domain_init(clk) #endif diff --git a/src/ipc/ipc4/handler.c b/src/ipc/ipc4/handler.c index 3fb49696f343..5ea024430ec7 100644 --- a/src/ipc/ipc4/handler.c +++ b/src/ipc/ipc4/handler.c @@ -886,6 +886,8 @@ __cold static int ipc4_init_module_instance(struct ipc4_message_request *ipc4) if (ret < 0) return IPC4_FAILURE; + assert_can_be_cold(); + tr_dbg(&ipc_tr, "ipc4_init_module_instance %x : %x", (uint32_t)module_init.primary.r.module_id, diff --git a/src/ipc/ipc4/helper.c b/src/ipc/ipc4/helper.c index eca6acd17413..8605a9e37ecd 100644 --- a/src/ipc/ipc4/helper.c +++ b/src/ipc/ipc4/helper.c @@ -110,6 +110,8 @@ __cold struct comp_dev *comp_new_ipc4(struct ipc4_module_init_instance *module_i uint32_t comp_id; char *data; + assert_can_be_cold(); + comp_id = IPC4_COMP_ID(module_init->primary.r.module_id, module_init->primary.r.instance_id); diff --git a/src/schedule/zephyr_domain.c b/src/schedule/zephyr_domain.c index 0c052f611ca4..f4925e35b11a 100644 --- a/src/schedule/zephyr_domain.c +++ b/src/schedule/zephyr_domain.c @@ -6,6 +6,7 @@ #include #include +#include #include #include #include @@ -321,3 +322,17 @@ struct ll_schedule_domain *zephyr_domain_init(int clk) return domain; } + +/* Check if currently running in the LL scheduler thread context */ +bool ll_sch_is_current(void) +{ + struct zephyr_domain *zephyr_domain = ll_sch_domain_get_pdata(zephyr_ll_domain()); + + if (!zephyr_domain) + return false; + + struct zephyr_domain_thread *dt = zephyr_domain->domain_thread + cpu_get_id(); + + return k_current_get() == &dt->ll_thread; +} +EXPORT_SYMBOL(ll_sch_is_current); diff --git a/src/schedule/zephyr_ll.c b/src/schedule/zephyr_ll.c index e455421ef040..8ce37c2dfddc 100644 --- a/src/schedule/zephyr_ll.c +++ b/src/schedule/zephyr_ll.c @@ -555,3 +555,11 @@ void scheduler_get_task_info_ll(struct scheduler_props *scheduler_props, scheduler_get_task_info(scheduler_props, data_off_size, &ll_sch->tasks); zephyr_ll_unlock(ll_sch, &flags); } + +/* Return a pointer to the LL scheduler timer domain */ +struct ll_schedule_domain *zephyr_ll_domain(void) +{ + struct zephyr_ll *ll_sch = scheduler_get_data(SOF_SCHEDULE_LL_TIMER); + + return ll_sch->ll_domain; +} diff --git a/xtos/include/sof/lib/memory.h b/xtos/include/sof/lib/memory.h index 9a784ec7f19d..219ebf2bc76c 100644 --- a/xtos/include/sof/lib/memory.h +++ b/xtos/include/sof/lib/memory.h @@ -18,4 +18,6 @@ #define __cold_rodata #endif +#define assert_can_be_cold() do {} while (0) + #endif /* __SOF_LIB_MEMORY_H__ */ diff --git a/zephyr/include/sof/lib/memory.h b/zephyr/include/sof/lib/memory.h index f58686a92981..b7d709b5276b 100644 --- a/zephyr/include/sof/lib/memory.h +++ b/zephyr/include/sof/lib/memory.h @@ -16,4 +16,21 @@ #define __cold_rodata #endif +#if CONFIG_COLD_STORE_EXECUTE_DEBUG +#include + +#ifdef __ZEPHYR__ +bool ll_sch_is_current(void); +#else +#define ll_sch_is_current() false +#endif + +static inline void assert_can_be_cold(void) +{ + assert(!ll_sch_is_current()); +} +#else +#define assert_can_be_cold() do {} while (0) +#endif + #endif /* __SOF_LIB_MEMORY_H__ */