aboutsummaryrefslogtreecommitdiffstats
path: root/drivers/dma/shdma.c
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/dma/shdma.c')
-rw-r--r--drivers/dma/shdma.c211
1 files changed, 145 insertions, 66 deletions
diff --git a/drivers/dma/shdma.c b/drivers/dma/shdma.c
index 0283300..81809c2 100644
--- a/drivers/dma/shdma.c
+++ b/drivers/dma/shdma.c
@@ -70,12 +70,36 @@ static u32 sh_dmae_readl(struct sh_dmae_chan *sh_dc, u32 reg)
static u16 dmaor_read(struct sh_dmae_device *shdev)
{
- return __raw_readw(shdev->chan_reg + DMAOR / sizeof(u32));
+ u32 __iomem *addr = shdev->chan_reg + DMAOR / sizeof(u32);
+
+ if (shdev->pdata->dmaor_is_32bit)
+ return __raw_readl(addr);
+ else
+ return __raw_readw(addr);
}
static void dmaor_write(struct sh_dmae_device *shdev, u16 data)
{
- __raw_writew(data, shdev->chan_reg + DMAOR / sizeof(u32));
+ u32 __iomem *addr = shdev->chan_reg + DMAOR / sizeof(u32);
+
+ if (shdev->pdata->dmaor_is_32bit)
+ __raw_writel(data, addr);
+ else
+ __raw_writew(data, addr);
+}
+
+static void chcr_write(struct sh_dmae_chan *sh_dc, u32 data)
+{
+ struct sh_dmae_device *shdev = to_sh_dev(sh_dc);
+
+ __raw_writel(data, sh_dc->base + shdev->chcr_offset / sizeof(u32));
+}
+
+static u32 chcr_read(struct sh_dmae_chan *sh_dc)
+{
+ struct sh_dmae_device *shdev = to_sh_dev(sh_dc);
+
+ return __raw_readl(sh_dc->base + shdev->chcr_offset / sizeof(u32));
}
/*
@@ -120,7 +144,7 @@ static int sh_dmae_rst(struct sh_dmae_device *shdev)
static bool dmae_is_busy(struct sh_dmae_chan *sh_chan)
{
- u32 chcr = sh_dmae_readl(sh_chan, CHCR);
+ u32 chcr = chcr_read(sh_chan);
if ((chcr & (CHCR_DE | CHCR_TE)) == CHCR_DE)
return true; /* working */
@@ -130,8 +154,7 @@ static bool dmae_is_busy(struct sh_dmae_chan *sh_chan)
static unsigned int calc_xmit_shift(struct sh_dmae_chan *sh_chan, u32 chcr)
{
- struct sh_dmae_device *shdev = container_of(sh_chan->common.device,
- struct sh_dmae_device, common);
+ struct sh_dmae_device *shdev = to_sh_dev(sh_chan);
struct sh_dmae_pdata *pdata = shdev->pdata;
int cnt = ((chcr & pdata->ts_low_mask) >> pdata->ts_low_shift) |
((chcr & pdata->ts_high_mask) >> pdata->ts_high_shift);
@@ -144,8 +167,7 @@ static unsigned int calc_xmit_shift(struct sh_dmae_chan *sh_chan, u32 chcr)
static u32 log2size_to_chcr(struct sh_dmae_chan *sh_chan, int l2size)
{
- struct sh_dmae_device *shdev = container_of(sh_chan->common.device,
- struct sh_dmae_device, common);
+ struct sh_dmae_device *shdev = to_sh_dev(sh_chan);
struct sh_dmae_pdata *pdata = shdev->pdata;
int i;
@@ -169,18 +191,23 @@ static void dmae_set_reg(struct sh_dmae_chan *sh_chan, struct sh_dmae_regs *hw)
static void dmae_start(struct sh_dmae_chan *sh_chan)
{
- u32 chcr = sh_dmae_readl(sh_chan, CHCR);
+ struct sh_dmae_device *shdev = to_sh_dev(sh_chan);
+ u32 chcr = chcr_read(sh_chan);
+
+ if (shdev->pdata->needs_tend_set)
+ sh_dmae_writel(sh_chan, 0xFFFFFFFF, TEND);
- chcr |= CHCR_DE | CHCR_IE;
- sh_dmae_writel(sh_chan, chcr & ~CHCR_TE, CHCR);
+ chcr |= CHCR_DE | shdev->chcr_ie_bit;
+ chcr_write(sh_chan, chcr & ~CHCR_TE);
}
static void dmae_halt(struct sh_dmae_chan *sh_chan)
{
- u32 chcr = sh_dmae_readl(sh_chan, CHCR);
+ struct sh_dmae_device *shdev = to_sh_dev(sh_chan);
+ u32 chcr = chcr_read(sh_chan);
- chcr &= ~(CHCR_DE | CHCR_TE | CHCR_IE);
- sh_dmae_writel(sh_chan, chcr, CHCR);
+ chcr &= ~(CHCR_DE | CHCR_TE | shdev->chcr_ie_bit);
+ chcr_write(sh_chan, chcr);
}
static void dmae_init(struct sh_dmae_chan *sh_chan)
@@ -192,7 +219,7 @@ static void dmae_init(struct sh_dmae_chan *sh_chan)
u32 chcr = DM_INC | SM_INC | 0x400 | log2size_to_chcr(sh_chan,
LOG2_DEFAULT_XFER_SIZE);
sh_chan->xmit_shift = calc_xmit_shift(sh_chan, chcr);
- sh_dmae_writel(sh_chan, chcr, CHCR);
+ chcr_write(sh_chan, chcr);
}
static int dmae_set_chcr(struct sh_dmae_chan *sh_chan, u32 val)
@@ -202,23 +229,25 @@ static int dmae_set_chcr(struct sh_dmae_chan *sh_chan, u32 val)
return -EBUSY;
sh_chan->xmit_shift = calc_xmit_shift(sh_chan, val);
- sh_dmae_writel(sh_chan, val, CHCR);
+ chcr_write(sh_chan, val);
return 0;
}
static int dmae_set_dmars(struct sh_dmae_chan *sh_chan, u16 val)
{
- struct sh_dmae_device *shdev = container_of(sh_chan->common.device,
- struct sh_dmae_device, common);
+ struct sh_dmae_device *shdev = to_sh_dev(sh_chan);
struct sh_dmae_pdata *pdata = shdev->pdata;
const struct sh_dmae_channel *chan_pdata = &pdata->channel[sh_chan->id];
u16 __iomem *addr = shdev->dmars;
- int shift = chan_pdata->dmars_bit;
+ unsigned int shift = chan_pdata->dmars_bit;
if (dmae_is_busy(sh_chan))
return -EBUSY;
+ if (pdata->no_dmars)
+ return 0;
+
/* in the case of a missing DMARS resource use first memory window */
if (!addr)
addr = (u16 __iomem *)shdev->chan_reg;
@@ -230,14 +259,23 @@ static int dmae_set_dmars(struct sh_dmae_chan *sh_chan, u16 val)
return 0;
}
+static void sh_chan_xfer_ld_queue(struct sh_dmae_chan *sh_chan);
+
static dma_cookie_t sh_dmae_tx_submit(struct dma_async_tx_descriptor *tx)
{
struct sh_desc *desc = tx_to_sh_desc(tx), *chunk, *last = desc, *c;
struct sh_dmae_chan *sh_chan = to_sh_chan(tx->chan);
+ struct sh_dmae_slave *param = tx->chan->private;
dma_async_tx_callback callback = tx->callback;
dma_cookie_t cookie;
+ bool power_up;
- spin_lock_bh(&sh_chan->desc_lock);
+ spin_lock_irq(&sh_chan->desc_lock);
+
+ if (list_empty(&sh_chan->ld_queue))
+ power_up = true;
+ else
+ power_up = false;
cookie = sh_chan->common.cookie;
cookie++;
@@ -273,7 +311,38 @@ static dma_cookie_t sh_dmae_tx_submit(struct dma_async_tx_descriptor *tx)
tx->cookie, &last->async_tx, sh_chan->id,
desc->hw.sar, desc->hw.tcr, desc->hw.dar);
- spin_unlock_bh(&sh_chan->desc_lock);
+ if (power_up) {
+ sh_chan->pm_state = DMAE_PM_BUSY;
+
+ pm_runtime_get(sh_chan->dev);
+
+ spin_unlock_irq(&sh_chan->desc_lock);
+
+ pm_runtime_barrier(sh_chan->dev);
+
+ spin_lock_irq(&sh_chan->desc_lock);
+
+ /* Have we been reset, while waiting? */
+ if (sh_chan->pm_state != DMAE_PM_ESTABLISHED) {
+ dev_dbg(sh_chan->dev, "Bring up channel %d\n",
+ sh_chan->id);
+ if (param) {
+ const struct sh_dmae_slave_config *cfg =
+ param->config;
+
+ dmae_set_dmars(sh_chan, cfg->mid_rid);
+ dmae_set_chcr(sh_chan, cfg->chcr);
+ } else {
+ dmae_init(sh_chan);
+ }
+
+ if (sh_chan->pm_state == DMAE_PM_PENDING)
+ sh_chan_xfer_ld_queue(sh_chan);
+ sh_chan->pm_state = DMAE_PM_ESTABLISHED;
+ }
+ }
+
+ spin_unlock_irq(&sh_chan->desc_lock);
return cookie;
}
@@ -296,9 +365,7 @@ static struct sh_desc *sh_dmae_get_desc(struct sh_dmae_chan *sh_chan)
static const struct sh_dmae_slave_config *sh_dmae_find_slave(
struct sh_dmae_chan *sh_chan, struct sh_dmae_slave *param)
{
- struct dma_device *dma_dev = sh_chan->common.device;
- struct sh_dmae_device *shdev = container_of(dma_dev,
- struct sh_dmae_device, common);
+ struct sh_dmae_device *shdev = to_sh_dev(sh_chan);
struct sh_dmae_pdata *pdata = shdev->pdata;
int i;
@@ -319,8 +386,6 @@ static int sh_dmae_alloc_chan_resources(struct dma_chan *chan)
struct sh_dmae_slave *param = chan->private;
int ret;
- pm_runtime_get_sync(sh_chan->dev);
-
/*
* This relies on the guarantee from dmaengine that alloc_chan_resources
* never runs concurrently with itself or free_chan_resources.
@@ -340,31 +405,20 @@ static int sh_dmae_alloc_chan_resources(struct dma_chan *chan)
}
param->config = cfg;
-
- dmae_set_dmars(sh_chan, cfg->mid_rid);
- dmae_set_chcr(sh_chan, cfg->chcr);
- } else {
- dmae_init(sh_chan);
}
- spin_lock_bh(&sh_chan->desc_lock);
while (sh_chan->descs_allocated < NR_DESCS_PER_CHANNEL) {
- spin_unlock_bh(&sh_chan->desc_lock);
desc = kzalloc(sizeof(struct sh_desc), GFP_KERNEL);
- if (!desc) {
- spin_lock_bh(&sh_chan->desc_lock);
+ if (!desc)
break;
- }
dma_async_tx_descriptor_init(&desc->async_tx,
&sh_chan->common);
desc->async_tx.tx_submit = sh_dmae_tx_submit;
desc->mark = DESC_IDLE;
- spin_lock_bh(&sh_chan->desc_lock);
list_add(&desc->node, &sh_chan->ld_free);
sh_chan->descs_allocated++;
}
- spin_unlock_bh(&sh_chan->desc_lock);
if (!sh_chan->descs_allocated) {
ret = -ENOMEM;
@@ -378,7 +432,7 @@ edescalloc:
clear_bit(param->slave_id, sh_dmae_slave_used);
etestused:
efindslave:
- pm_runtime_put(sh_chan->dev);
+ chan->private = NULL;
return ret;
}
@@ -390,7 +444,6 @@ static void sh_dmae_free_chan_resources(struct dma_chan *chan)
struct sh_dmae_chan *sh_chan = to_sh_chan(chan);
struct sh_desc *desc, *_desc;
LIST_HEAD(list);
- int descs = sh_chan->descs_allocated;
/* Protect against ISR */
spin_lock_irq(&sh_chan->desc_lock);
@@ -410,15 +463,12 @@ static void sh_dmae_free_chan_resources(struct dma_chan *chan)
chan->private = NULL;
}
- spin_lock_bh(&sh_chan->desc_lock);
+ spin_lock_irq(&sh_chan->desc_lock);
list_splice_init(&sh_chan->ld_free, &list);
sh_chan->descs_allocated = 0;
- spin_unlock_bh(&sh_chan->desc_lock);
-
- if (descs > 0)
- pm_runtime_put(sh_chan->dev);
+ spin_unlock_irq(&sh_chan->desc_lock);
list_for_each_entry_safe(desc, _desc, &list, node)
kfree(desc);
@@ -507,6 +557,7 @@ static struct dma_async_tx_descriptor *sh_dmae_prep_sg(struct sh_dmae_chan *sh_c
struct sh_desc *first = NULL, *new = NULL /* compiler... */;
LIST_HEAD(tx_list);
int chunks = 0;
+ unsigned long irq_flags;
int i;
if (!sg_len)
@@ -517,7 +568,7 @@ static struct dma_async_tx_descriptor *sh_dmae_prep_sg(struct sh_dmae_chan *sh_c
(SH_DMA_TCR_MAX + 1);
/* Have to lock the whole loop to protect against concurrent release */
- spin_lock_bh(&sh_chan->desc_lock);
+ spin_lock_irqsave(&sh_chan->desc_lock, irq_flags);
/*
* Chaining:
@@ -563,7 +614,7 @@ static struct dma_async_tx_descriptor *sh_dmae_prep_sg(struct sh_dmae_chan *sh_c
/* Put them back on the free list, so, they don't get lost */
list_splice_tail(&tx_list, &sh_chan->ld_free);
- spin_unlock_bh(&sh_chan->desc_lock);
+ spin_unlock_irqrestore(&sh_chan->desc_lock, irq_flags);
return &first->async_tx;
@@ -572,7 +623,7 @@ err_get_desc:
new->mark = DESC_IDLE;
list_splice(&tx_list, &sh_chan->ld_free);
- spin_unlock_bh(&sh_chan->desc_lock);
+ spin_unlock_irqrestore(&sh_chan->desc_lock, irq_flags);
return NULL;
}
@@ -634,6 +685,7 @@ static int sh_dmae_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd,
unsigned long arg)
{
struct sh_dmae_chan *sh_chan = to_sh_chan(chan);
+ unsigned long flags;
/* Only supports DMA_TERMINATE_ALL */
if (cmd != DMA_TERMINATE_ALL)
@@ -642,7 +694,7 @@ static int sh_dmae_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd,
if (!chan)
return -EINVAL;
- spin_lock_bh(&sh_chan->desc_lock);
+ spin_lock_irqsave(&sh_chan->desc_lock, flags);
dmae_halt(sh_chan);
if (!list_empty(&sh_chan->ld_queue)) {
@@ -651,9 +703,8 @@ static int sh_dmae_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd,
struct sh_desc, node);
desc->partial = (desc->hw.tcr - sh_dmae_readl(sh_chan, TCR)) <<
sh_chan->xmit_shift;
-
}
- spin_unlock_bh(&sh_chan->desc_lock);
+ spin_unlock_irqrestore(&sh_chan->desc_lock, flags);
sh_dmae_chan_ld_cleanup(sh_chan, true);
@@ -668,8 +719,9 @@ static dma_async_tx_callback __ld_cleanup(struct sh_dmae_chan *sh_chan, bool all
dma_cookie_t cookie = 0;
dma_async_tx_callback callback = NULL;
void *param = NULL;
+ unsigned long flags;
- spin_lock_bh(&sh_chan->desc_lock);
+ spin_lock_irqsave(&sh_chan->desc_lock, flags);
list_for_each_entry_safe(desc, _desc, &sh_chan->ld_queue, node) {
struct dma_async_tx_descriptor *tx = &desc->async_tx;
@@ -735,7 +787,13 @@ static dma_async_tx_callback __ld_cleanup(struct sh_dmae_chan *sh_chan, bool all
async_tx_test_ack(&desc->async_tx)) || all) {
/* Remove from ld_queue list */
desc->mark = DESC_IDLE;
+
list_move(&desc->node, &sh_chan->ld_free);
+
+ if (list_empty(&sh_chan->ld_queue)) {
+ dev_dbg(sh_chan->dev, "Bring down channel %d\n", sh_chan->id);
+ pm_runtime_put(sh_chan->dev);
+ }
}
}
@@ -746,7 +804,7 @@ static dma_async_tx_callback __ld_cleanup(struct sh_dmae_chan *sh_chan, bool all
*/
sh_chan->completed_cookie = sh_chan->common.cookie;
- spin_unlock_bh(&sh_chan->desc_lock);
+ spin_unlock_irqrestore(&sh_chan->desc_lock, flags);
if (callback)
callback(param);
@@ -765,16 +823,14 @@ static void sh_dmae_chan_ld_cleanup(struct sh_dmae_chan *sh_chan, bool all)
;
}
+/* Called under spin_lock_irq(&sh_chan->desc_lock) */
static void sh_chan_xfer_ld_queue(struct sh_dmae_chan *sh_chan)
{
struct sh_desc *desc;
- spin_lock_bh(&sh_chan->desc_lock);
/* DMA work check */
- if (dmae_is_busy(sh_chan)) {
- spin_unlock_bh(&sh_chan->desc_lock);
+ if (dmae_is_busy(sh_chan))
return;
- }
/* Find the first not transferred descriptor */
list_for_each_entry(desc, &sh_chan->ld_queue, node)
@@ -787,14 +843,18 @@ static void sh_chan_xfer_ld_queue(struct sh_dmae_chan *sh_chan)
dmae_start(sh_chan);
break;
}
-
- spin_unlock_bh(&sh_chan->desc_lock);
}
static void sh_dmae_memcpy_issue_pending(struct dma_chan *chan)
{
struct sh_dmae_chan *sh_chan = to_sh_chan(chan);
- sh_chan_xfer_ld_queue(sh_chan);
+
+ spin_lock_irq(&sh_chan->desc_lock);
+ if (sh_chan->pm_state == DMAE_PM_ESTABLISHED)
+ sh_chan_xfer_ld_queue(sh_chan);
+ else
+ sh_chan->pm_state = DMAE_PM_PENDING;
+ spin_unlock_irq(&sh_chan->desc_lock);
}
static enum dma_status sh_dmae_tx_status(struct dma_chan *chan,
@@ -805,6 +865,7 @@ static enum dma_status sh_dmae_tx_status(struct dma_chan *chan,
dma_cookie_t last_used;
dma_cookie_t last_complete;
enum dma_status status;
+ unsigned long flags;
sh_dmae_chan_ld_cleanup(sh_chan, false);
@@ -815,7 +876,7 @@ static enum dma_status sh_dmae_tx_status(struct dma_chan *chan,
BUG_ON(last_complete < 0);
dma_set_tx_state(txstate, last_complete, last_used, 0);
- spin_lock_bh(&sh_chan->desc_lock);
+ spin_lock_irqsave(&sh_chan->desc_lock, flags);
status = dma_async_is_complete(cookie, last_complete, last_used);
@@ -833,7 +894,7 @@ static enum dma_status sh_dmae_tx_status(struct dma_chan *chan,
}
}
- spin_unlock_bh(&sh_chan->desc_lock);
+ spin_unlock_irqrestore(&sh_chan->desc_lock, flags);
return status;
}
@@ -846,7 +907,7 @@ static irqreturn_t sh_dmae_interrupt(int irq, void *data)
spin_lock(&sh_chan->desc_lock);
- chcr = sh_dmae_readl(sh_chan, CHCR);
+ chcr = chcr_read(sh_chan);
if (chcr & CHCR_TE) {
/* DMA stop */
@@ -886,6 +947,12 @@ static bool sh_dmae_reset(struct sh_dmae_device *shdev)
list_splice_init(&sh_chan->ld_queue, &dl);
+ if (!list_empty(&dl)) {
+ dev_dbg(sh_chan->dev, "Bring down channel %d\n", sh_chan->id);
+ pm_runtime_put(sh_chan->dev);
+ }
+ sh_chan->pm_state = DMAE_PM_ESTABLISHED;
+
spin_unlock(&sh_chan->desc_lock);
/* Complete all */
@@ -926,7 +993,7 @@ static void dmae_do_tasklet(unsigned long data)
u32 sar_buf = sh_dmae_readl(sh_chan, SAR);
u32 dar_buf = sh_dmae_readl(sh_chan, DAR);
- spin_lock(&sh_chan->desc_lock);
+ spin_lock_irq(&sh_chan->desc_lock);
list_for_each_entry(desc, &sh_chan->ld_queue, node) {
if (desc->mark == DESC_SUBMITTED &&
((desc->direction == DMA_FROM_DEVICE &&
@@ -939,10 +1006,10 @@ static void dmae_do_tasklet(unsigned long data)
break;
}
}
- spin_unlock(&sh_chan->desc_lock);
-
/* Next desc */
sh_chan_xfer_ld_queue(sh_chan);
+ spin_unlock_irq(&sh_chan->desc_lock);
+
sh_dmae_chan_ld_cleanup(sh_chan, false);
}
@@ -1010,7 +1077,9 @@ static int __devinit sh_dmae_chan_probe(struct sh_dmae_device *shdev, int id,
return -ENOMEM;
}
- /* copy struct dma_device */
+ new_sh_chan->pm_state = DMAE_PM_ESTABLISHED;
+
+ /* reference struct dma_device */
new_sh_chan->common.device = &shdev->common;
new_sh_chan->dev = shdev->common.dev;
@@ -1144,6 +1213,16 @@ static int __init sh_dmae_probe(struct platform_device *pdev)
/* platform data */
shdev->pdata = pdata;
+ if (pdata->chcr_offset)
+ shdev->chcr_offset = pdata->chcr_offset;
+ else
+ shdev->chcr_offset = CHCR;
+
+ if (pdata->chcr_ie_bit)
+ shdev->chcr_ie_bit = pdata->chcr_ie_bit;
+ else
+ shdev->chcr_ie_bit = CHCR_IE;
+
platform_set_drvdata(pdev, shdev);
pm_runtime_enable(&pdev->dev);