Heap refinement Part3 -- make full use of HP SRAM banks

Kconfig.xtos-dbg

@@ -12,3 +12,12 @@ config DEBUG_BLOCK_FREE
  already freed block of memory. Enabling this feature increases
  number of memory writes and reads, due to checks for memory patterns
  that may be performed on allocation and deallocation.
+
+config DEBUG_FORCE_COHERENT_BUFFER
+	bool "Force the allocator to allocate coherent buffer only"
+	default n
+	help
+ Select if we want to force the allocator to return coherent/uncached
+ buffer only.
+ This should be selected for debug purpose only, as accessing buffer
+ without caching it will reduce the read/write performance.

src/arch/xtensa/lib/cpu.c

@@ -24,6 +24,7 @@
 #include <sof/schedule/schedule.h>
 #include <sof/sof.h>
 #include <sof/trace/trace.h>
+#include <ipc/topology.h>
 #include <ipc/trace.h>
 #include <user/trace.h>
 #include <xtos-structs.h>
@@ -46,7 +47,7 @@ static void alloc_shared_secondary_cores_objects(void)
 {
 	uint8_t *dynamic_vectors;
 
-	dynamic_vectors = rzalloc(SOF_MEM_ZONE_RUNTIME_SHARED, 0, 0, SOF_DYNAMIC_VECTORS_SIZE);
+	dynamic_vectors = rballoc(SOF_MEM_FLAG_COHERENT, 0, SOF_DYNAMIC_VECTORS_SIZE);
 	if (dynamic_vectors == NULL)
  panic(SOF_IPC_PANIC_MEM);
 

src/audio/buffer.c

@@ -31,21 +31,20 @@ struct comp_buffer *buffer_alloc(uint32_t size, uint32_t caps, uint32_t align)
 	tr_dbg(&buffer_tr, "buffer_alloc()");
 
 	/* validate request */
-	if (size == 0 || size > HEAP_BUFFER_SIZE) {
+	if (size == 0) {
  tr_err(&buffer_tr, "buffer_alloc(): new size = %u is invalid",
  size);
  return NULL;
 	}
 
 	/* allocate new buffer */
-	buffer = rzalloc(SOF_MEM_ZONE_RUNTIME_SHARED, 0, SOF_MEM_CAPS_RAM,
- sizeof(*buffer));
+	buffer = rballoc(SOF_MEM_FLAG_COHERENT, SOF_MEM_CAPS_RAM, sizeof(*buffer));
 	if (!buffer) {
  tr_err(&buffer_tr, "buffer_alloc(): could not alloc structure");
  return NULL;
 	}
 
-	buffer->lock = rzalloc(SOF_MEM_ZONE_RUNTIME_SHARED, 0, SOF_MEM_CAPS_RAM,
+	buffer->lock = rballoc(SOF_MEM_FLAG_COHERENT, SOF_MEM_CAPS_RAM,
  sizeof(*buffer->lock));
 	if (!buffer->lock) {
  rfree(buffer);
@@ -85,7 +84,7 @@ int buffer_set_size(struct comp_buffer *buffer, uint32_t size)
 	void *new_ptr = NULL;
 
 	/* validate request */
-	if (size == 0 || size > HEAP_BUFFER_SIZE) {
+	if (size == 0) {
  buf_err(buffer, "resize size = %u is invalid", size);
  return -EINVAL;
 	}

src/audio/dai.c

@@ -164,7 +164,7 @@ static struct comp_dev *dai_new(const struct comp_driver *drv,
  return NULL;
 	dev->ipc_config = *config;
 
-	dd = rzalloc(SOF_MEM_ZONE_RUNTIME_SHARED, 0, SOF_MEM_CAPS_RAM, sizeof(*dd));
+	dd = rballoc(SOF_MEM_FLAG_COHERENT, SOF_MEM_CAPS_RAM, sizeof(*dd));
 	if (!dd) {
  rfree(dev);
  return NULL;

src/audio/mux/mux.c

@@ -173,7 +173,8 @@ static struct comp_dev *mux_new(const struct comp_driver *drv,
  return NULL;
 	dev->ipc_config = *config;
 
-	cd = rzalloc(SOF_MEM_ZONE_RUNTIME, 0, SOF_MEM_CAPS_RAM,
+	/* allocate quite a big buffer, use rballoc to make sure it will succeed */
+	cd = rballoc(0, SOF_MEM_CAPS_RAM,
  sizeof(*cd) + MUX_MAX_STREAMS * sizeof(struct mux_stream_data));
 	if (!cd) {
  rfree(dev);

src/drivers/dw/dma.c

@@ -437,9 +437,6 @@ static int dw_dma_stop(struct dma_chan_data *channel)
  lli->ctrl_hi &= ~DW_CTLH_DONE(1);
  lli++;
 	}
-
-	dcache_writeback_region(dw_chan->lli,
- sizeof(struct dw_lli) * channel->desc_count);
 #endif
 
 	/* disable linear link position */
@@ -557,9 +554,9 @@ static int dw_dma_set_config(struct dma_chan_data *channel,
  if (dw_chan->lli)
  rfree(dw_chan->lli);
 
- dw_chan->lli = rballoc_align(0, SOF_MEM_CAPS_RAM | SOF_MEM_CAPS_DMA,
-	sizeof(struct dw_lli) * channel->desc_count,
-	PLATFORM_DCACHE_ALIGN);
+ dw_chan->lli = rballoc(SOF_MEM_FLAG_COHERENT,
+ SOF_MEM_CAPS_RAM | SOF_MEM_CAPS_DMA,
+ sizeof(struct dw_lli) * channel->desc_count);
  if (!dw_chan->lli) {
  tr_err(&dwdma_tr, "dw_dma_set_config(): dma %d channel %d lli alloc failed",
  channel->dma->plat_data.id,
@@ -767,10 +764,6 @@ static int dw_dma_set_config(struct dma_chan_data *channel,
 #endif
 	}
 
-	/* write back descriptors so DMA engine can read them directly */
-	dcache_writeback_region(dw_chan->lli,
- sizeof(struct dw_lli) * channel->desc_count);
-
 	channel->status = COMP_STATE_PREPARE;
 	dw_chan->lli_current = dw_chan->lli;
 
@@ -810,8 +803,9 @@ static void dw_dma_verify_transfer(struct dma_chan_data *channel,
 #if defined __ZEPHYR__
 	int i;
 #else
-	struct dw_lli *lli = platform_dw_dma_lli_get(dw_chan->lli_current);
+	struct dw_lli *lli = dw_chan->lli_current;
 #endif
+
 	switch (next->status) {
 	case DMA_CB_STATUS_END:
  channel->status = COMP_STATE_PREPARE;
@@ -823,20 +817,14 @@ static void dw_dma_verify_transfer(struct dma_chan_data *channel,
  * sure the cache is coherent between DSP and DMAC.
  */
 #if defined __ZEPHYR__
- dcache_invalidate_region(dw_chan->lli,
- sizeof(struct dw_lli) * channel->desc_count);
-
  for (i = 0; i < channel->desc_count; i++)
  dw_chan->lli[i].ctrl_hi &= ~DW_CTLH_DONE(1);
-
- dcache_writeback_region(dw_chan->lli,
- sizeof(struct dw_lli) * channel->desc_count);
 #else
  while (lli->ctrl_hi & DW_CTLH_DONE(1)) {
  lli->ctrl_hi &= ~DW_CTLH_DONE(1);
  dw_chan->lli_current =
  (struct dw_lli *)dw_chan->lli_current->llp;
- lli = platform_dw_dma_lli_get(dw_chan->lli_current);
+ lli = dw_chan->lli_current;
  }
 #endif
  break;
@@ -966,9 +954,8 @@ static int dw_dma_probe(struct dma *dma)
 	pm_runtime_get_sync(DW_DMAC_CLK, dma->plat_data.id);
 
 	/* allocate dma channels */
-	dma->chan = rzalloc(SOF_MEM_ZONE_RUNTIME_SHARED, 0, SOF_MEM_CAPS_RAM,
+	dma->chan = rballoc(SOF_MEM_FLAG_COHERENT, SOF_MEM_CAPS_RAM,
  sizeof(struct dma_chan_data) * dma->plat_data.channels);
-
 	if (!dma->chan) {
  tr_err(&dwdma_tr, "dw_dma_probe(): dma %d allocaction of channels failed",
  dma->plat_data.id);
@@ -987,9 +974,7 @@ static int dw_dma_probe(struct dma *dma)
  chan->index = i;
  chan->core = DMA_CORE_INVALID;
 
- dw_chan = rzalloc(SOF_MEM_ZONE_RUNTIME_SHARED, 0, SOF_MEM_CAPS_RAM,
- sizeof(*dw_chan));
-
+ dw_chan = rballoc(SOF_MEM_FLAG_COHERENT, SOF_MEM_CAPS_RAM, sizeof(*dw_chan));
  if (!dw_chan) {
  tr_err(&dwdma_tr, "dw_dma_probe(): dma %d allocaction of channel %d private data failed",
  dma->plat_data.id, i);

src/drivers/generic/dummy-dma.c

@@ -435,7 +435,7 @@ static int dummy_dma_probe(struct dma *dma)
  return -EEXIST; /* already created */
 	}
 
-	dma->chan = rzalloc(SOF_MEM_ZONE_RUNTIME_SHARED, 0, SOF_MEM_CAPS_RAM,
+	dma->chan = rballoc(SOF_MEM_FLAG_COHERENT, SOF_MEM_CAPS_RAM,
  dma->plat_data.channels * sizeof(dma->chan[0]));
 	if (!dma->chan) {
  tr_err(&ddma_tr, "dummy-dmac %d: Out of memory!",

src/drivers/imx/esai.c

@@ -394,7 +394,7 @@ static int esai_probe(struct dai *dai)
  dai_err(dai, "ESAI: Repeated probe, skipping");
  return -EEXIST;
 	}
-	pdata = rzalloc(SOF_MEM_ZONE_RUNTIME_SHARED, 0, SOF_MEM_CAPS_RAM,sizeof(*pdata));
+	pdata = rballoc(SOF_MEM_FLAG_COHERENT, SOF_MEM_CAPS_RAM,sizeof(*pdata));
 	if (!pdata) {
  dai_err(dai, "ESAI probe failure, out of memory");
  return -ENOMEM;

src/drivers/imx/sai.c

@@ -367,7 +367,7 @@ static int sai_probe(struct dai *dai)
 	dai_info(dai, "SAI: sai_probe");
 
 	/* allocate private data */
-	sai = rzalloc(SOF_MEM_ZONE_RUNTIME_SHARED, 0, SOF_MEM_CAPS_RAM, sizeof(*sai));
+	sai = rballoc(SOF_MEM_FLAG_COHERENT, SOF_MEM_CAPS_RAM, sizeof(*sai));
 	if (!sai) {
  dai_err(dai, "sai_probe(): alloc failed");
  return -ENOMEM;

src/drivers/intel/alh.c

@@ -117,7 +117,7 @@ static int alh_probe(struct dai *dai)
 	if (dai_get_drvdata(dai))
  return -EEXIST;
 
-	alh = rzalloc(SOF_MEM_ZONE_RUNTIME_SHARED, 0, SOF_MEM_CAPS_RAM, sizeof(*alh));
+	alh = rballoc(SOF_MEM_FLAG_COHERENT, SOF_MEM_CAPS_RAM, sizeof(*alh));
 	if (!alh) {
  dai_err(dai, "alh_probe() error: alloc failed");
  return -ENOMEM;

src/drivers/intel/baytrail/ssp.c

@@ -617,7 +617,7 @@ static int ssp_probe(struct dai *dai)
 	struct ssp_pdata *ssp;
 
 	/* allocate private data */
-	ssp = rzalloc(SOF_MEM_ZONE_SYS_SHARED, 0, SOF_MEM_CAPS_RAM, sizeof(*ssp));
+	ssp = rballoc(SOF_MEM_FLAG_COHERENT, SOF_MEM_CAPS_RAM, sizeof(*ssp));
 	dai_set_drvdata(dai, ssp);
 
 	ssp->state[DAI_DIR_PLAYBACK] = COMP_STATE_READY;

src/drivers/intel/dmic/dmic.c

@@ -554,7 +554,7 @@ static int dmic_probe(struct dai *dai)
 	if (dai_get_drvdata(dai))
  return -EEXIST; /* already created */
 
-	dmic = rzalloc(SOF_MEM_ZONE_RUNTIME_SHARED, 0, SOF_MEM_CAPS_RAM, sizeof(*dmic));
+	dmic = rballoc(SOF_MEM_FLAG_COHERENT, SOF_MEM_CAPS_RAM, sizeof(*dmic));
 	if (!dmic) {
  dai_err(dai, "dmic_probe(): alloc failed");
  return -ENOMEM;

src/drivers/intel/haswell/ssp.c

@@ -540,7 +540,7 @@ static int ssp_probe(struct dai *dai)
 	struct ssp_pdata *ssp;
 
 	/* allocate private data */
-	ssp = rzalloc(SOF_MEM_ZONE_SYS_SHARED, 0, SOF_MEM_CAPS_RAM, sizeof(*ssp));
+	ssp = rballoc(SOF_MEM_FLAG_COHERENT, SOF_MEM_CAPS_RAM, sizeof(*ssp));
 	dai_set_drvdata(dai, ssp);
 
 	ssp->state[DAI_DIR_PLAYBACK] = COMP_STATE_READY;

src/drivers/intel/hda/hda-dma.c

@@ -792,9 +792,8 @@ static int hda_dma_probe(struct dma *dma)
 	if (dma->chan)
  return -EEXIST; /* already created */
 
-	dma->chan = rzalloc(SOF_MEM_ZONE_RUNTIME_SHARED, 0, SOF_MEM_CAPS_RAM,
+	dma->chan = rballoc(SOF_MEM_FLAG_COHERENT, SOF_MEM_CAPS_RAM,
  sizeof(struct dma_chan_data) * dma->plat_data.channels);
-
 	if (!dma->chan) {
  tr_err(&hdma_tr, "hda-dmac: %d channels alloc failed",
  dma->plat_data.id);
@@ -810,8 +809,7 @@ static int hda_dma_probe(struct dma *dma)
  chan->status = COMP_STATE_INIT;
  chan->core = DMA_CORE_INVALID;
 
- hda_chan = rzalloc(SOF_MEM_ZONE_RUNTIME_SHARED, 0,
- SOF_MEM_CAPS_RAM,
+ hda_chan = rballoc(SOF_MEM_FLAG_COHERENT, SOF_MEM_CAPS_RAM,
  sizeof(struct hda_chan_data));
  if (!hda_chan) {
  tr_err(&hdma_tr, "hda-dma: %d channel %d private data alloc failed",

src/drivers/intel/hda/hda.c

@@ -71,7 +71,7 @@ static int hda_probe(struct dai *dai)
 	if (dai_get_drvdata(dai))
  return -EEXIST;
 
-	hda = rzalloc(SOF_MEM_ZONE_RUNTIME_SHARED, 0, SOF_MEM_CAPS_RAM, sizeof(*hda));
+	hda = rballoc(SOF_MEM_FLAG_COHERENT, SOF_MEM_CAPS_RAM, sizeof(*hda));
 	if (!hda) {
  dai_err(dai, "hda_probe() error: alloc failed");
  return -ENOMEM;

src/drivers/intel/ssp/ssp.c

@@ -1132,7 +1132,7 @@ static int ssp_probe(struct dai *dai)
  return -EEXIST; /* already created */
 
 	/* allocate private data */
-	ssp = rzalloc(SOF_MEM_ZONE_RUNTIME_SHARED, 0, SOF_MEM_CAPS_RAM, sizeof(*ssp));
+	ssp = rballoc(SOF_MEM_FLAG_COHERENT, SOF_MEM_CAPS_RAM, sizeof(*ssp));
 	if (!ssp) {
  dai_err(dai, "ssp_probe(): alloc failed");
  return -ENOMEM;

src/drivers/interrupt.c

@@ -72,7 +72,7 @@ int interrupt_cascade_register(const struct irq_cascade_tmpl *tmpl)
 
 	}
 
-	*cascade = rzalloc(SOF_MEM_ZONE_SYS_SHARED, 0, SOF_MEM_CAPS_RAM, sizeof(**cascade));
+	*cascade = rballoc(SOF_MEM_FLAG_COHERENT, SOF_MEM_CAPS_RAM, sizeof(**cascade));
 
 	spinlock_init(&(*cascade)->lock);
 
@@ -204,7 +204,7 @@ static int irq_register_child(struct irq_cascade_desc *cascade, int irq,
  /* init child from run-time, may be registered and unregistered
  * many times at run-time
  */
- child = rzalloc(SOF_MEM_ZONE_RUNTIME_SHARED, 0, SOF_MEM_CAPS_RAM,
+ child = rballoc(SOF_MEM_FLAG_COHERENT, SOF_MEM_CAPS_RAM,
  sizeof(struct irq_desc));
  if (!child) {
  ret = -ENOMEM;

src/drivers/mediatek/mt8195/afe-drv.c

@@ -352,7 +352,7 @@ int afe_probe(struct mtk_base_afe *afe)
 	tr_dbg(&afedrv_tr, "afe_base:0x%x\n", afe->base);
 	/* TODO how to get the memif number, how to sync with dmac lib */
 	afe->memifs_size = platform->memif_size;
-	afe->memif = rzalloc(SOF_MEM_ZONE_RUNTIME_SHARED, 0, SOF_MEM_CAPS_RAM,
+	afe->memif = rballoc(SOF_MEM_FLAG_COHERENT, SOF_MEM_CAPS_RAM,
  sizeof(struct mtk_base_afe_memif) * afe->memifs_size);
 	if (!afe->memif)
  return -ENOMEM;
@@ -362,14 +362,14 @@ int afe_probe(struct mtk_base_afe *afe)
 
 	/* TODO how to get the dai number, how to sync with dai lib*/
 	afe->dais_size = platform->dais_size;
-	afe->dais = rzalloc(SOF_MEM_ZONE_RUNTIME_SHARED, 0, SOF_MEM_CAPS_RAM,
+	afe->dais = rballoc(SOF_MEM_FLAG_COHERENT, SOF_MEM_CAPS_RAM,
  sizeof(struct mtk_base_afe_dai) * afe->dais_size);
 	if (!afe->dais)
  goto err_alloc_memif;
 
 	/* TODO how to get the irq number */
 	afe->irqs_size = platform->irqs_size;
-	afe->irqs = rzalloc(SOF_MEM_ZONE_RUNTIME_SHARED, 0, SOF_MEM_CAPS_RAM,
+	afe->irqs = rballoc(SOF_MEM_FLAG_COHERENT, SOF_MEM_CAPS_RAM,
  sizeof(struct mtk_base_afe_irq) * afe->irqs_size);
 	if (!afe->irqs)
  goto err_alloc_dais;

src/include/sof/ipc/msg.h

@@ -45,11 +45,11 @@ static inline struct ipc_msg *ipc_msg_init(uint32_t header, uint32_t size)
 {
 	struct ipc_msg *msg;
 
-	msg = rzalloc(SOF_MEM_ZONE_RUNTIME_SHARED, 0, SOF_MEM_CAPS_RAM, sizeof(*msg));
+	msg = rballoc(SOF_MEM_FLAG_COHERENT, SOF_MEM_CAPS_RAM, sizeof(*msg));
 	if (!msg)
  return NULL;
 
-	msg->tx_data = rzalloc(SOF_MEM_ZONE_RUNTIME_SHARED, 0, SOF_MEM_CAPS_RAM, size);
+	msg->tx_data = rballoc(SOF_MEM_FLAG_COHERENT, SOF_MEM_CAPS_RAM, size);
 	if (!msg->tx_data) {
  rfree(msg);
  return NULL;

src/include/sof/lib/alloc.h

@@ -44,21 +44,13 @@
  *
  * 4) Buffer Zone. Largest heap zone intended for audio buffers.
  *
- * 5) Runtime Shared Zone. Similar to Runtime Zone, but content may be used and
- * fred from any enabled core.
- *
- * 6) System Shared Zone. Similar to System Zone, but content may be used from
- * any enabled core.
- *
  * See platform/memory.h for heap size configuration and mappings.
  */
 enum mem_zone {
 	SOF_MEM_ZONE_SYS = 0, /**< System zone */
 	SOF_MEM_ZONE_SYS_RUNTIME,	/**< System-runtime zone */
 	SOF_MEM_ZONE_RUNTIME, /**< Runtime zone */
 	SOF_MEM_ZONE_BUFFER, /**< Buffer zone */
-	SOF_MEM_ZONE_RUNTIME_SHARED,	/**< Runtime shared zone */
-	SOF_MEM_ZONE_SYS_SHARED,	/**< System shared zone */
 };
 
 /** \name Heap zone flags
@@ -67,6 +59,8 @@ enum mem_zone {
 
 /** \brief Indicates that original content should not be copied by realloc. */
 #define SOF_MEM_FLAG_NO_COPY	BIT(1)
+/** \brief Indicates that if we should return uncached address. */
+#define SOF_MEM_FLAG_COHERENT	BIT(2)
 
 /** @} */
 
@@ -92,9 +86,12 @@ void *rmalloc(enum mem_zone zone, uint32_t flags, uint32_t caps, size_t bytes);
 void *rzalloc(enum mem_zone zone, uint32_t flags, uint32_t caps, size_t bytes);
 
 /**
- * Allocates memory block from SOF_MEM_ZONE_BUFFER.
+ * The recommended helper for buffer allocation.
+ * It allocates memory block in the buffer zone, and return either cached
+ * or uncached address, according the requirement from the flag
  * @param flags Flags, see SOF_MEM_FLAG_...
- * @param caps Capabilities, see SOF_MEM_CAPS_...
+ * @param caps Capabilities, e.g. if _CAPS_DMA is specified, allocate
+ * buffer from uncacheable memory. See SOF_MEM_CAPS_...
  * @param bytes Size in bytes.
  * @param alignment Alignment in bytes.
  * @return Pointer to the allocated memory or NULL if failed.
@@ -113,7 +110,8 @@ static inline void *rballoc(uint32_t flags, uint32_t caps, size_t bytes)
 /**
  * Changes size of the memory block allocated from SOF_MEM_ZONE_BUFFER.
  * @param ptr Address of the block to resize.
- * @param flags Flags, see SOF_MEM_FLAG_...
+ * @param flags Flags, return uncached address if SOF_MEM_FLAG_COHERENT
+ * is specified, see SOF_MEM_FLAG_...
  * @param caps Capabilities, see SOF_MEM_CAPS_...
  * @param bytes New size in bytes.
  * @param old_bytes Old size in bytes.