Kernel/linux
1
0
Fork 0
mirror of https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git synced 2025-01-12 00:00:00 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge branch 'pci/controller/dwc'

Browse Source 
- Release previously-requested DW eDMA IRQs if request_irq() fails (Serge
  Semin)

- Convert DW eDMA linked-list (ll) and data target (dt) from CPU-relative
  addresses to PCI bus addresses (Serge Semin)

- Fix missing src/dst address for interleaved transfers (Serge Semin)

- Enforce the DW eDMA restriction that interleaved transfers must increment
  src and dst addresses (Serge Semin)

- Fix some invalid interleaved transfer semantics (Serge Semin)

- Convert CPU-relative addresses to PCI bus addresses for eDMA engine
  (Serge Semin)

- Drop chancnt initialization from dw-edma-core, since it is managed by the
  dmaengine core, e.g., in dma_async_device_channel_register() (Serge Semin)

- Clean up bogus casting of debugfs_entries.reg addresses (Serge Semin)

- Ignore debugfs file/directory creation errors (Serge Semin)

- Allocate debugfs entries from the heap to prepare for multi-eDMA
  platforms (Serge Semin)

- Simplify and rework register accessors to remove another obstacle to
  multi-eDMA platforms (Serge Semin)

- Consolidate eDMA read/write channels in a single dma_device to simplify,
  better reflect the hardware design, and avoid a debugfs complaint (Serge
  Semin)

- Move eDMA-specific debugfs nodes into existing dmaengine subdirectory
  (Serge Semin)

- Fix a readq_ch() truncation from 64 to 32 bits (Serge Semin)

- Use existing readq()/writeq rather than hand-coding new ones (Serge
  Semin)

- Drop unnecessary data target region allocation in favor of existing
  dw_edma_chip members (Serge Semin)

- Use parent device in eDMA controller name to prepare for multi-eDMA
  platforms (Serge Semin)

- In addition to the existing MMIO accessors for linked list entries, add
  support for ioremapped entries for use by eDMA in Root Ports or local
  Endpoints (Serge Semin)

- Convert DW_EDMA_PCIE so it depends on DW_EDMA instead of selecting it
  (Serge Semin)

- Allow DWC drivers to set streaming DMA masks larger than 32 bits;
  previously both streaming and coherent DMA were limited to 32 bits
  because some PCI devices only support coherent 32-bit DMA for MSI (Serge
  Semin)

- Set 64-bit streaming and coherent DMA mask for the bt1 driver (Serge
  Semin)

- Add DW Root Port and Endpoint controller support for eDMA (Serge Semin)

* pci/controller/dwc:
  PCI: dwc: Add Root Port and Endpoint controller eDMA engine support
  PCI: bt1: Set 64-bit DMA mask
  PCI: dwc: Restrict only coherent DMA mask for MSI address allocation
  dmaengine: dw-edma: Prepare dw_edma_probe() for builtin callers
  dmaengine: dw-edma: Depend on DW_EDMA instead of selecting it
  dmaengine: dw-edma: Add mem-mapped LL-entries support
  dmaengine: dw-edma: Skip cleanup procedure if no private data found
  dmaengine: dw-edma: Replace chip ID number with device name
  dmaengine: dw-edma: Drop DT-region allocation
  dmaengine: dw-edma: Use non-atomic io-64 methods
  dmaengine: dw-edma: Fix readq_ch() return value truncation
  dmaengine: dw-edma: Use DMA engine device debugfs subdirectory
  dmaengine: dw-edma: Join read/write channels into a single device
  dmaengine: dw-edma: Move eDMA data pointer to debugfs node descriptor
  dmaengine: dw-edma: Simplify debugfs context CSRs init procedure
  dmaengine: dw-edma: Rename debugfs dentry variables to 'dent'
  dmaengine: dw-edma: Convert debugfs descs to being heap-allocated
  dmaengine: dw-edma: Add dw_edma prefix to debugfs nodes descriptor
  dmaengine: dw-edma: Stop checking debugfs_create_*() return value
  dmaengine: dw-edma: Drop unnecessary debugfs reg casts
  dmaengine: dw-edma: Drop chancnt initialization
  dmaengine: dw-edma: Add PCI bus address getter to the remote EP glue driver
  dmaengine: dw-edma: Add CPU to PCI bus address translation
  dmaengine: dw-edma: Fix invalid interleaved xfers semantics
  dmaengine: dw-edma: Don't permit non-inc interleaved xfers
  dmaengine: dw-edma: Fix missing src/dst address of interleaved xfers
  dmaengine: dw-edma: Convert ll/dt phys address to PCI bus/DMA address
  dmaengine: dw-edma: Release requested IRQs on failure
  dmaengine: Fix dma_slave_config.dst_addr description
This commit is contained in:
Bjorn Helgaas 2023-02-22 13:47:29 -06:00
commit 5256d49380
15 changed files with 642 additions and 385 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
drivers/dma/dw-edma/Kconfig
View File

@ -9,11 +9,14 @@ config DW_EDMA
Support the Synopsys DesignWare eDMA controller, normally
implemented on endpoints SoCs.
if DW_EDMA
config DW_EDMA_PCIE
tristate "Synopsys DesignWare eDMA PCIe driver"
depends on PCI && PCI_MSI
select DW_EDMA
help
Provides a glue-logic between the Synopsys DesignWare
eDMA controller and an endpoint PCIe device. This also serves
as a reference design to whom desires to use this IP.
endif # DW_EDMA

198
drivers/dma/dw-edma/dw-edma-core.c
View File

@ -39,6 +39,17 @@ struct dw_edma_desc *vd2dw_edma_desc(struct virt_dma_desc *vd)
return container_of(vd, struct dw_edma_desc, vd);
}
static inline
u64 dw_edma_get_pci_address(struct dw_edma_chan *chan, phys_addr_t cpu_addr)
{
struct dw_edma_chip *chip = chan->dw->chip;
if (chip->ops->pci_address)
return chip->ops->pci_address(chip->dev, cpu_addr);
return cpu_addr;
}
static struct dw_edma_burst *dw_edma_alloc_burst(struct dw_edma_chunk *chunk)
{
struct dw_edma_burst *burst;
@ -197,6 +208,24 @@ static void dw_edma_start_transfer(struct dw_edma_chan *chan)
desc->chunks_alloc--;
}
static void dw_edma_device_caps(struct dma_chan *dchan,
struct dma_slave_caps *caps)
{
struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan);
if (chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL) {
if (chan->dir == EDMA_DIR_READ)
caps->directions = BIT(DMA_DEV_TO_MEM);
else
caps->directions = BIT(DMA_MEM_TO_DEV);
} else {
if (chan->dir == EDMA_DIR_WRITE)
caps->directions = BIT(DMA_DEV_TO_MEM);
else
caps->directions = BIT(DMA_MEM_TO_DEV);
}
}
static int dw_edma_device_config(struct dma_chan *dchan,
struct dma_slave_config *config)
{
@ -327,11 +356,12 @@ dw_edma_device_transfer(struct dw_edma_transfer *xfer)
{
struct dw_edma_chan *chan = dchan2dw_edma_chan(xfer->dchan);
enum dma_transfer_direction dir = xfer->direction;
phys_addr_t src_addr, dst_addr;
struct scatterlist *sg = NULL;
struct dw_edma_chunk *chunk;
struct dw_edma_burst *burst;
struct dw_edma_desc *desc;
u64 src_addr, dst_addr;
size_t fsz = 0;
u32 cnt = 0;
int i;
@ -381,9 +411,9 @@ dw_edma_device_transfer(struct dw_edma_transfer *xfer)
if (xfer->xfer.sg.len < 1)
return NULL;
} else if (xfer->type == EDMA_XFER_INTERLEAVED) {
if (!xfer->xfer.il->numf)
if (!xfer->xfer.il->numf || xfer->xfer.il->frame_size < 1)
return NULL;
if (xfer->xfer.il->numf > 0 && xfer->xfer.il->frame_size > 0)
if (!xfer->xfer.il->src_inc || !xfer->xfer.il->dst_inc)
return NULL;
} else {
return NULL;
@ -405,16 +435,19 @@ dw_edma_device_transfer(struct dw_edma_transfer *xfer)
dst_addr = chan->config.dst_addr;
}
if (dir == DMA_DEV_TO_MEM)
src_addr = dw_edma_get_pci_address(chan, (phys_addr_t)src_addr);
else
dst_addr = dw_edma_get_pci_address(chan, (phys_addr_t)dst_addr);
if (xfer->type == EDMA_XFER_CYCLIC) {
cnt = xfer->xfer.cyclic.cnt;
} else if (xfer->type == EDMA_XFER_SCATTER_GATHER) {
cnt = xfer->xfer.sg.len;
sg = xfer->xfer.sg.sgl;
} else if (xfer->type == EDMA_XFER_INTERLEAVED) {
if (xfer->xfer.il->numf > 0)
cnt = xfer->xfer.il->numf;
else
cnt = xfer->xfer.il->frame_size;
cnt = xfer->xfer.il->numf * xfer->xfer.il->frame_size;
fsz = xfer->xfer.il->frame_size;
}
for (i = 0; i < cnt; i++) {
@ -436,7 +469,7 @@ dw_edma_device_transfer(struct dw_edma_transfer *xfer)
else if (xfer->type == EDMA_XFER_SCATTER_GATHER)
burst->sz = sg_dma_len(sg);
else if (xfer->type == EDMA_XFER_INTERLEAVED)
burst->sz = xfer->xfer.il->sgl[i].size;
burst->sz = xfer->xfer.il->sgl[i % fsz].size;
chunk->ll_region.sz += burst->sz;
desc->alloc_sz += burst->sz;
@ -455,6 +488,8 @@ dw_edma_device_transfer(struct dw_edma_transfer *xfer)
* and destination addresses are increased
* by the same portion (data length)
*/
} else if (xfer->type == EDMA_XFER_INTERLEAVED) {
burst->dar = dst_addr;
}
} else {
burst->dar = dst_addr;
@ -470,25 +505,24 @@ dw_edma_device_transfer(struct dw_edma_transfer *xfer)
* and destination addresses are increased
* by the same portion (data length)
*/
} else if (xfer->type == EDMA_XFER_INTERLEAVED) {
burst->sar = src_addr;
}
}
if (xfer->type == EDMA_XFER_SCATTER_GATHER) {
sg = sg_next(sg);
} else if (xfer->type == EDMA_XFER_INTERLEAVED &&
xfer->xfer.il->frame_size > 0) {
} else if (xfer->type == EDMA_XFER_INTERLEAVED) {
struct dma_interleaved_template *il = xfer->xfer.il;
struct data_chunk *dc = &il->sgl[i];
struct data_chunk *dc = &il->sgl[i % fsz];
if (il->src_sgl) {
src_addr += burst->sz;
src_addr += burst->sz;
if (il->src_sgl)
src_addr += dmaengine_get_src_icg(il, dc);
}
if (il->dst_sgl) {
dst_addr += burst->sz;
dst_addr += burst->sz;
if (il->dst_sgl)
dst_addr += dmaengine_get_dst_icg(il, dc);
}
}
}
@ -701,92 +735,76 @@ static void dw_edma_free_chan_resources(struct dma_chan *dchan)
}
}
static int dw_edma_channel_setup(struct dw_edma *dw, bool write,
u32 wr_alloc, u32 rd_alloc)
static int dw_edma_channel_setup(struct dw_edma *dw, u32 wr_alloc, u32 rd_alloc)
{
struct dw_edma_chip *chip = dw->chip;
struct dw_edma_region *dt_region;
struct device *dev = chip->dev;
struct dw_edma_chan *chan;
struct dw_edma_irq *irq;
struct dma_device *dma;
u32 alloc, off_alloc;
u32 i, j, cnt;
int err = 0;
u32 i, ch_cnt;
u32 pos;
if (write) {
i = 0;
cnt = dw->wr_ch_cnt;
dma = &dw->wr_edma;
alloc = wr_alloc;
off_alloc = 0;
} else {
i = dw->wr_ch_cnt;
cnt = dw->rd_ch_cnt;
dma = &dw->rd_edma;
alloc = rd_alloc;
off_alloc = wr_alloc;
}
ch_cnt = dw->wr_ch_cnt + dw->rd_ch_cnt;
dma = &dw->dma;
INIT_LIST_HEAD(&dma->channels);
for (j = 0; (alloc || dw->nr_irqs == 1) && j < cnt; j++, i++) {
for (i = 0; i < ch_cnt; i++) {
chan = &dw->chan[i];
dt_region = devm_kzalloc(dev, sizeof(*dt_region), GFP_KERNEL);
if (!dt_region)
return -ENOMEM;
chan->vc.chan.private = dt_region;
chan->dw = dw;
chan->id = j;
chan->dir = write ? EDMA_DIR_WRITE : EDMA_DIR_READ;
if (i < dw->wr_ch_cnt) {
chan->id = i;
chan->dir = EDMA_DIR_WRITE;
} else {
chan->id = i - dw->wr_ch_cnt;
chan->dir = EDMA_DIR_READ;
}
chan->configured = false;
chan->request = EDMA_REQ_NONE;
chan->status = EDMA_ST_IDLE;
if (write)
chan->ll_max = (chip->ll_region_wr[j].sz / EDMA_LL_SZ);
if (chan->dir == EDMA_DIR_WRITE)
chan->ll_max = (chip->ll_region_wr[chan->id].sz / EDMA_LL_SZ);
else
chan->ll_max = (chip->ll_region_rd[j].sz / EDMA_LL_SZ);
chan->ll_max = (chip->ll_region_rd[chan->id].sz / EDMA_LL_SZ);
chan->ll_max -= 1;
dev_vdbg(dev, "L. List:\tChannel %s[%u] max_cnt=%u\n",
write ? "write" : "read", j, chan->ll_max);
chan->dir == EDMA_DIR_WRITE ? "write" : "read",
chan->id, chan->ll_max);
if (dw->nr_irqs == 1)
pos = 0;
else if (chan->dir == EDMA_DIR_WRITE)
pos = chan->id % wr_alloc;
else
pos = off_alloc + (j % alloc);
pos = wr_alloc + chan->id % rd_alloc;
irq = &dw->irq[pos];
if (write)
irq->wr_mask |= BIT(j);
if (chan->dir == EDMA_DIR_WRITE)
irq->wr_mask |= BIT(chan->id);
else
irq->rd_mask |= BIT(j);
irq->rd_mask |= BIT(chan->id);
irq->dw = dw;
memcpy(&chan->msi, &irq->msi, sizeof(chan->msi));
dev_vdbg(dev, "MSI:\t\tChannel %s[%u] addr=0x%.8x%.8x, data=0x%.8x\n",
write ? "write" : "read", j,
chan->dir == EDMA_DIR_WRITE ? "write" : "read", chan->id,
chan->msi.address_hi, chan->msi.address_lo,
chan->msi.data);
chan->vc.desc_free = vchan_free_desc;
vchan_init(&chan->vc, dma);
chan->vc.chan.private = chan->dir == EDMA_DIR_WRITE ?
&dw->chip->dt_region_wr[chan->id] :
&dw->chip->dt_region_rd[chan->id];
if (write) {
dt_region->paddr = chip->dt_region_wr[j].paddr;
dt_region->vaddr = chip->dt_region_wr[j].vaddr;
dt_region->sz = chip->dt_region_wr[j].sz;
} else {
dt_region->paddr = chip->dt_region_rd[j].paddr;
dt_region->vaddr = chip->dt_region_rd[j].vaddr;
dt_region->sz = chip->dt_region_rd[j].sz;
}
vchan_init(&chan->vc, dma);
dw_edma_v0_core_device_config(chan);
}
@ -797,16 +815,16 @@ static int dw_edma_channel_setup(struct dw_edma *dw, bool write,
dma_cap_set(DMA_CYCLIC, dma->cap_mask);
dma_cap_set(DMA_PRIVATE, dma->cap_mask);
dma_cap_set(DMA_INTERLEAVE, dma->cap_mask);
dma->directions = BIT(write ? DMA_DEV_TO_MEM : DMA_MEM_TO_DEV);
dma->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
dma->src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);
dma->dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);
dma->residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR;
dma->chancnt = cnt;
/* Set DMA channel callbacks */
dma->dev = chip->dev;
dma->device_alloc_chan_resources = dw_edma_alloc_chan_resources;
dma->device_free_chan_resources = dw_edma_free_chan_resources;
dma->device_caps = dw_edma_device_caps;
dma->device_config = dw_edma_device_config;
dma->device_pause = dw_edma_device_pause;
dma->device_resume = dw_edma_device_resume;
@ -820,9 +838,7 @@ static int dw_edma_channel_setup(struct dw_edma *dw, bool write,
dma_set_max_seg_size(dma->dev, U32_MAX);
/* Register DMA device */
err = dma_async_device_register(dma);
return err;
return dma_async_device_register(dma);
}
static inline void dw_edma_dec_irq_alloc(int *nr_irqs, u32 *alloc, u16 cnt)
@ -893,10 +909,8 @@ static int dw_edma_irq_request(struct dw_edma *dw,
dw_edma_interrupt_read,
IRQF_SHARED, dw->name,
&dw->irq[i]);
if (err) {
dw->nr_irqs = i;
return err;
}
if (err)
goto err_irq_free;
if (irq_get_msi_desc(irq))
get_cached_msi_msg(irq, &dw->irq[i].msi);
@ -905,6 +919,14 @@ static int dw_edma_irq_request(struct dw_edma *dw,
dw->nr_irqs = i;
}
return 0;
err_irq_free:
for (i--; i >= 0; i--) {
irq = chip->ops->irq_vector(dev, i);
free_irq(irq, &dw->irq[i]);
}
return err;
}
@ -951,7 +973,8 @@ int dw_edma_probe(struct dw_edma_chip *chip)
if (!dw->chan)
return -ENOMEM;
snprintf(dw->name, sizeof(dw->name), "dw-edma-core:%d", chip->id);
snprintf(dw->name, sizeof(dw->name), "dw-edma-core:%s",
dev_name(chip->dev));
/* Disable eDMA, only to establish the ideal initial conditions */
dw_edma_v0_core_off(dw);
@ -961,13 +984,8 @@ int dw_edma_probe(struct dw_edma_chip *chip)
if (err)
return err;
/* Setup write channels */
err = dw_edma_channel_setup(dw, true, wr_alloc, rd_alloc);
if (err)
goto err_irq_free;
/* Setup read channels */
err = dw_edma_channel_setup(dw, false, wr_alloc, rd_alloc);
/* Setup write/read channels */
err = dw_edma_channel_setup(dw, wr_alloc, rd_alloc);
if (err)
goto err_irq_free;
@ -993,6 +1011,10 @@ int dw_edma_remove(struct dw_edma_chip *chip)
struct dw_edma *dw = chip->dw;
int i;
/* Skip removal if no private data found */
if (!dw)
return -ENODEV;
/* Disable eDMA */
dw_edma_v0_core_off(dw);
@ -1001,23 +1023,13 @@ int dw_edma_remove(struct dw_edma_chip *chip)
free_irq(chip->ops->irq_vector(dev, i), &dw->irq[i]);
/* Deregister eDMA device */
dma_async_device_unregister(&dw->wr_edma);
list_for_each_entry_safe(chan, _chan, &dw->wr_edma.channels,
dma_async_device_unregister(&dw->dma);
list_for_each_entry_safe(chan, _chan, &dw->dma.channels,
vc.chan.device_node) {
tasklet_kill(&chan->vc.task);
list_del(&chan->vc.chan.device_node);
}
dma_async_device_unregister(&dw->rd_edma);
list_for_each_entry_safe(chan, _chan, &dw->rd_edma.channels,
vc.chan.device_node) {
tasklet_kill(&chan->vc.task);
list_del(&chan->vc.chan.device_node);
}
/* Turn debugfs off */
dw_edma_v0_core_debugfs_off(dw);
return 0;
}
EXPORT_SYMBOL_GPL(dw_edma_remove);

10
drivers/dma/dw-edma/dw-edma-core.h
View File

@ -96,12 +96,11 @@ struct dw_edma_irq {
};
struct dw_edma {
char name[20];
char name[32];
struct dma_device dma;
struct dma_device wr_edma;
u16 wr_ch_cnt;
struct dma_device rd_edma;
u16 rd_ch_cnt;
struct dw_edma_irq *irq;
@ -112,9 +111,6 @@ struct dw_edma {
raw_spinlock_t lock; /* Only for legacy */
struct dw_edma_chip *chip;
#ifdef CONFIG_DEBUG_FS
struct dentry *debugfs;
#endif /* CONFIG_DEBUG_FS */
};
struct dw_edma_sg {

56
drivers/dma/dw-edma/dw-edma-pcie.c
View File

@ -95,8 +95,23 @@ static int dw_edma_pcie_irq_vector(struct device *dev, unsigned int nr)
return pci_irq_vector(to_pci_dev(dev), nr);
}
static u64 dw_edma_pcie_address(struct device *dev, phys_addr_t cpu_addr)
{
struct pci_dev *pdev = to_pci_dev(dev);
struct pci_bus_region region;
struct resource res = {
.flags = IORESOURCE_MEM,
.start = cpu_addr,
.end = cpu_addr,
};
pcibios_resource_to_bus(pdev->bus, &region, &res);
return region.start;
}
static const struct dw_edma_core_ops dw_edma_pcie_core_ops = {
.irq_vector = dw_edma_pcie_irq_vector,
.pci_address = dw_edma_pcie_address,
};
static void dw_edma_pcie_get_vsec_dma_data(struct pci_dev *pdev,
@ -207,7 +222,6 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev,
/* Data structure initialization */
chip->dev = dev;
chip->id = pdev->devfn;
chip->mf = vsec_data.mf;
chip->nr_irqs = nr_irqs;
@ -226,21 +240,21 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev,
struct dw_edma_block *ll_block = &vsec_data.ll_wr[i];
struct dw_edma_block *dt_block = &vsec_data.dt_wr[i];
ll_region->vaddr = pcim_iomap_table(pdev)[ll_block->bar];
if (!ll_region->vaddr)
ll_region->vaddr.io = pcim_iomap_table(pdev)[ll_block->bar];
if (!ll_region->vaddr.io)
return -ENOMEM;
ll_region->vaddr += ll_block->off;
ll_region->paddr = pdev->resource[ll_block->bar].start;
ll_region->vaddr.io += ll_block->off;
ll_region->paddr = pci_bus_address(pdev, ll_block->bar);
ll_region->paddr += ll_block->off;
ll_region->sz = ll_block->sz;
dt_region->vaddr = pcim_iomap_table(pdev)[dt_block->bar];
if (!dt_region->vaddr)
dt_region->vaddr.io = pcim_iomap_table(pdev)[dt_block->bar];
if (!dt_region->vaddr.io)
return -ENOMEM;
dt_region->vaddr += dt_block->off;
dt_region->paddr = pdev->resource[dt_block->bar].start;
dt_region->vaddr.io += dt_block->off;
dt_region->paddr = pci_bus_address(pdev, dt_block->bar);
dt_region->paddr += dt_block->off;
dt_region->sz = dt_block->sz;
}
@ -251,21 +265,21 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev,
struct dw_edma_block *ll_block = &vsec_data.ll_rd[i];
struct dw_edma_block *dt_block = &vsec_data.dt_rd[i];
ll_region->vaddr = pcim_iomap_table(pdev)[ll_block->bar];
if (!ll_region->vaddr)
ll_region->vaddr.io = pcim_iomap_table(pdev)[ll_block->bar];
if (!ll_region->vaddr.io)
return -ENOMEM;
ll_region->vaddr += ll_block->off;
ll_region->paddr = pdev->resource[ll_block->bar].start;
ll_region->vaddr.io += ll_block->off;
ll_region->paddr = pci_bus_address(pdev, ll_block->bar);
ll_region->paddr += ll_block->off;
ll_region->sz = ll_block->sz;
dt_region->vaddr = pcim_iomap_table(pdev)[dt_block->bar];
if (!dt_region->vaddr)
dt_region->vaddr.io = pcim_iomap_table(pdev)[dt_block->bar];
if (!dt_region->vaddr.io)
return -ENOMEM;
dt_region->vaddr += dt_block->off;
dt_region->paddr = pdev->resource[dt_block->bar].start;
dt_region->vaddr.io += dt_block->off;
dt_region->paddr = pci_bus_address(pdev, dt_block->bar);
dt_region->paddr += dt_block->off;
dt_region->sz = dt_block->sz;
}
@ -289,24 +303,24 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev,
pci_dbg(pdev, "L. List:\tWRITE CH%.2u, BAR=%u, off=0x%.8lx, sz=0x%zx bytes, addr(v=%p, p=%pa)\n",
i, vsec_data.ll_wr[i].bar,
vsec_data.ll_wr[i].off, chip->ll_region_wr[i].sz,
chip->ll_region_wr[i].vaddr, &chip->ll_region_wr[i].paddr);
chip->ll_region_wr[i].vaddr.io, &chip->ll_region_wr[i].paddr);
pci_dbg(pdev, "Data:\tWRITE CH%.2u, BAR=%u, off=0x%.8lx, sz=0x%zx bytes, addr(v=%p, p=%pa)\n",
i, vsec_data.dt_wr[i].bar,
vsec_data.dt_wr[i].off, chip->dt_region_wr[i].sz,
chip->dt_region_wr[i].vaddr, &chip->dt_region_wr[i].paddr);
chip->dt_region_wr[i].vaddr.io, &chip->dt_region_wr[i].paddr);
}
for (i = 0; i < chip->ll_rd_cnt; i++) {
pci_dbg(pdev, "L. List:\tREAD CH%.2u, BAR=%u, off=0x%.8lx, sz=0x%zx bytes, addr(v=%p, p=%pa)\n",
i, vsec_data.ll_rd[i].bar,
vsec_data.ll_rd[i].off, chip->ll_region_rd[i].sz,
chip->ll_region_rd[i].vaddr, &chip->ll_region_rd[i].paddr);
chip->ll_region_rd[i].vaddr.io, &chip->ll_region_rd[i].paddr);
pci_dbg(pdev, "Data:\tREAD CH%.2u, BAR=%u, off=0x%.8lx, sz=0x%zx bytes, addr(v=%p, p=%pa)\n",
i, vsec_data.dt_rd[i].bar,
vsec_data.dt_rd[i].off, chip->dt_region_rd[i].sz,
chip->dt_region_rd[i].vaddr, &chip->dt_region_rd[i].paddr);
chip->dt_region_rd[i].vaddr.io, &chip->dt_region_rd[i].paddr);
}
pci_dbg(pdev, "Nr. IRQs:\t%u\n", chip->nr_irqs);

100
drivers/dma/dw-edma/dw-edma-v0-core.c
View File

@ -8,6 +8,8 @@
#include <linux/bitfield.h>
#include <linux/io-64-nonatomic-lo-hi.h>
#include "dw-edma-core.h"
#include "dw-edma-v0-core.h"
#include "dw-edma-v0-regs.h"
@ -53,8 +55,6 @@ static inline struct dw_edma_v0_regs __iomem *__dw_regs(struct dw_edma *dw)
SET_32(dw, rd_##name, value); \
} while (0)
#ifdef CONFIG_64BIT
#define SET_64(dw, name, value) \
writeq(value, &(__dw_regs(dw)->name))
@ -80,8 +80,6 @@ static inline struct dw_edma_v0_regs __iomem *__dw_regs(struct dw_edma *dw)
SET_64(dw, rd_##name, value); \
} while (0)
#endif /* CONFIG_64BIT */
#define SET_COMPAT(dw, name, value) \
writel(value, &(__dw_regs(dw)->type.unroll.name))
@ -161,11 +159,6 @@ static inline u32 readl_ch(struct dw_edma *dw, enum dw_edma_dir dir, u16 ch,
#define GET_CH_32(dw, dir, ch, name) \
readl_ch(dw, dir, ch, &(__dw_ch_regs(dw, dir, ch)->name))
#define SET_LL_32(ll, value) \
writel(value, ll)
#ifdef CONFIG_64BIT
static inline void writeq_ch(struct dw_edma *dw, enum dw_edma_dir dir, u16 ch,
u64 value, void __iomem *addr)
{
@ -192,7 +185,7 @@ static inline void writeq_ch(struct dw_edma *dw, enum dw_edma_dir dir, u16 ch,
static inline u64 readq_ch(struct dw_edma *dw, enum dw_edma_dir dir, u16 ch,
const void __iomem *addr)
{
u32 value;
u64 value;
if (dw->chip->mf == EDMA_MF_EDMA_LEGACY) {
u32 viewport_sel;
@ -222,11 +215,6 @@ static inline u64 readq_ch(struct dw_edma *dw, enum dw_edma_dir dir, u16 ch,
#define GET_CH_64(dw, dir, ch, name) \
readq_ch(dw, dir, ch, &(__dw_ch_regs(dw, dir, ch)->name))
#define SET_LL_64(ll, value) \
writeq(value, ll)
#endif /* CONFIG_64BIT */
/* eDMA management callbacks */
void dw_edma_v0_core_off(struct dw_edma *dw)
{
@ -298,17 +286,53 @@ u32 dw_edma_v0_core_status_abort_int(struct dw_edma *dw, enum dw_edma_dir dir)
GET_RW_32(dw, dir, int_status));
}
static void dw_edma_v0_write_ll_data(struct dw_edma_chunk *chunk, int i,
u32 control, u32 size, u64 sar, u64 dar)
{
ptrdiff_t ofs = i * sizeof(struct dw_edma_v0_lli);
if (chunk->chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL) {
struct dw_edma_v0_lli *lli = chunk->ll_region.vaddr.mem + ofs;
lli->control = control;
lli->transfer_size = size;
lli->sar.reg = sar;
lli->dar.reg = dar;
} else {
struct dw_edma_v0_lli __iomem *lli = chunk->ll_region.vaddr.io + ofs;
writel(control, &lli->control);
writel(size, &lli->transfer_size);
writeq(sar, &lli->sar.reg);
writeq(dar, &lli->dar.reg);
}
}
static void dw_edma_v0_write_ll_link(struct dw_edma_chunk *chunk,
int i, u32 control, u64 pointer)
{
ptrdiff_t ofs = i * sizeof(struct dw_edma_v0_lli);
if (chunk->chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL) {
struct dw_edma_v0_llp *llp = chunk->ll_region.vaddr.mem + ofs;
llp->control = control;
llp->llp.reg = pointer;
} else {
struct dw_edma_v0_llp __iomem *llp = chunk->ll_region.vaddr.io + ofs;
writel(control, &llp->control);
writeq(pointer, &llp->llp.reg);
}
}
static void dw_edma_v0_core_write_chunk(struct dw_edma_chunk *chunk)
{
struct dw_edma_burst *child;
struct dw_edma_chan *chan = chunk->chan;
struct dw_edma_v0_lli __iomem *lli;
struct dw_edma_v0_llp __iomem *llp;
u32 control = 0, i = 0;
int j;
lli = chunk->ll_region.vaddr;
if (chunk->cb)
control = DW_EDMA_V0_CB;
@ -320,41 +344,16 @@ static void dw_edma_v0_core_write_chunk(struct dw_edma_chunk *chunk)
if (!(chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL))
control |= DW_EDMA_V0_RIE;
}
/* Channel control */
SET_LL_32(&lli[i].control, control);
/* Transfer size */
SET_LL_32(&lli[i].transfer_size, child->sz);
/* SAR */
#ifdef CONFIG_64BIT
SET_LL_64(&lli[i].sar.reg, child->sar);
#else /* CONFIG_64BIT */
SET_LL_32(&lli[i].sar.lsb, lower_32_bits(child->sar));
SET_LL_32(&lli[i].sar.msb, upper_32_bits(child->sar));
#endif /* CONFIG_64BIT */
/* DAR */
#ifdef CONFIG_64BIT
SET_LL_64(&lli[i].dar.reg, child->dar);
#else /* CONFIG_64BIT */
SET_LL_32(&lli[i].dar.lsb, lower_32_bits(child->dar));
SET_LL_32(&lli[i].dar.msb, upper_32_bits(child->dar));
#endif /* CONFIG_64BIT */
i++;
dw_edma_v0_write_ll_data(chunk, i++, control, child->sz,
child->sar, child->dar);
}
llp = (void __iomem *)&lli[i];
control = DW_EDMA_V0_LLP | DW_EDMA_V0_TCB;
if (!chunk->cb)
control |= DW_EDMA_V0_CB;
/* Channel control */
SET_LL_32(&llp->control, control);
/* Linked list */
#ifdef CONFIG_64BIT
SET_LL_64(&llp->llp.reg, chunk->ll_region.paddr);
#else /* CONFIG_64BIT */
SET_LL_32(&llp->llp.lsb, lower_32_bits(chunk->ll_region.paddr));
SET_LL_32(&llp->llp.msb, upper_32_bits(chunk->ll_region.paddr));
#endif /* CONFIG_64BIT */
dw_edma_v0_write_ll_link(chunk, i, control, chunk->ll_region.paddr);
}
void dw_edma_v0_core_start(struct dw_edma_chunk *chunk, bool first)
@ -504,8 +503,3 @@ void dw_edma_v0_core_debugfs_on(struct dw_edma *dw)
{
dw_edma_v0_debugfs_on(dw);
}
void dw_edma_v0_core_debugfs_off(struct dw_edma *dw)
{
dw_edma_v0_debugfs_off(dw);
}

1
drivers/dma/dw-edma/dw-edma-v0-core.h
View File

@ -23,6 +23,5 @@ void dw_edma_v0_core_start(struct dw_edma_chunk *chunk, bool first);
int dw_edma_v0_core_device_config(struct dw_edma_chan *chan);
/* eDMA debug fs callbacks */
void dw_edma_v0_core_debugfs_on(struct dw_edma *dw);
void dw_edma_v0_core_debugfs_off(struct dw_edma *dw);
#endif /* _DW_EDMA_V0_CORE_H */

368
drivers/dma/dw-edma/dw-edma-v0-debugfs.c
View File

@ -13,76 +13,79 @@
#include "dw-edma-v0-regs.h"
#include "dw-edma-core.h"
#define REGS_ADDR(name) \
((void __force *)&regs->name)
#define REGISTER(name) \
{ #name, REGS_ADDR(name) }
#define REGS_ADDR(dw, name) \
({ \
struct dw_edma_v0_regs __iomem *__regs = (dw)->chip->reg_base; \
\
(void __iomem *)&__regs->name; \
})
#define WR_REGISTER(name) \
{ #name, REGS_ADDR(wr_##name) }
#define RD_REGISTER(name) \
{ #name, REGS_ADDR(rd_##name) }
#define REGS_CH_ADDR(dw, name, _dir, _ch) \
({ \
struct dw_edma_v0_ch_regs __iomem *__ch_regs; \
\
if ((dw)->chip->mf == EDMA_MF_EDMA_LEGACY) \
__ch_regs = REGS_ADDR(dw, type.legacy.ch); \
else if (_dir == EDMA_DIR_READ) \
__ch_regs = REGS_ADDR(dw, type.unroll.ch[_ch].rd); \
else \
__ch_regs = REGS_ADDR(dw, type.unroll.ch[_ch].wr); \
\
(void __iomem *)&__ch_regs->name; \
})
#define WR_REGISTER_LEGACY(name) \
{ #name, REGS_ADDR(type.legacy.wr_##name) }
#define REGISTER(dw, name) \
{ dw, #name, REGS_ADDR(dw, name) }
#define CTX_REGISTER(dw, name, dir, ch) \
{ dw, #name, REGS_CH_ADDR(dw, name, dir, ch), dir, ch }
#define WR_REGISTER(dw, name) \
{ dw, #name, REGS_ADDR(dw, wr_##name) }
#define RD_REGISTER(dw, name) \
{ dw, #name, REGS_ADDR(dw, rd_##name) }
#define WR_REGISTER_LEGACY(dw, name) \
{ dw, #name, REGS_ADDR(dw, type.legacy.wr_##name) }
#define RD_REGISTER_LEGACY(name) \
{ #name, REGS_ADDR(type.legacy.rd_##name) }
{ dw, #name, REGS_ADDR(dw, type.legacy.rd_##name) }
#define WR_REGISTER_UNROLL(name) \
{ #name, REGS_ADDR(type.unroll.wr_##name) }
#define RD_REGISTER_UNROLL(name) \
{ #name, REGS_ADDR(type.unroll.rd_##name) }
#define WR_REGISTER_UNROLL(dw, name) \
{ dw, #name, REGS_ADDR(dw, type.unroll.wr_##name) }
#define RD_REGISTER_UNROLL(dw, name) \
{ dw, #name, REGS_ADDR(dw, type.unroll.rd_##name) }
#define WRITE_STR "write"
#define READ_STR "read"
#define CHANNEL_STR "channel"
#define REGISTERS_STR "registers"
static struct dw_edma *dw;
static struct dw_edma_v0_regs __iomem *regs;
static struct {
void __iomem *start;
void __iomem *end;
} lim[2][EDMA_V0_MAX_NR_CH];
struct debugfs_entries {
struct dw_edma_debugfs_entry {
struct dw_edma *dw;
const char *name;
dma_addr_t *reg;
void __iomem *reg;
enum dw_edma_dir dir;
u16 ch;
};
static int dw_edma_debugfs_u32_get(void *data, u64 *val)
{
void __iomem *reg = (void __force __iomem *)data;
struct dw_edma_debugfs_entry *entry = data;
struct dw_edma *dw = entry->dw;
void __iomem *reg = entry->reg;
if (dw->chip->mf == EDMA_MF_EDMA_LEGACY &&
reg >= (void __iomem *)&regs->type.legacy.ch) {
void __iomem *ptr = &regs->type.legacy.ch;
u32 viewport_sel = 0;
reg >= REGS_ADDR(dw, type.legacy.ch)) {
unsigned long flags;
u16 ch;
u32 viewport_sel;
for (ch = 0; ch < dw->wr_ch_cnt; ch++)
if (lim[0][ch].start >= reg && reg < lim[0][ch].end) {
ptr += (reg - lim[0][ch].start);
goto legacy_sel_wr;
}
for (ch = 0; ch < dw->rd_ch_cnt; ch++)
if (lim[1][ch].start >= reg && reg < lim[1][ch].end) {
ptr += (reg - lim[1][ch].start);
goto legacy_sel_rd;
}
return 0;
legacy_sel_rd:
viewport_sel = BIT(31);
legacy_sel_wr:
viewport_sel |= FIELD_PREP(EDMA_V0_VIEWPORT_MASK, ch);
viewport_sel = entry->dir == EDMA_DIR_READ ? BIT(31) : 0;
viewport_sel |= FIELD_PREP(EDMA_V0_VIEWPORT_MASK, entry->ch);
raw_spin_lock_irqsave(&dw->lock, flags);
writel(viewport_sel, &regs->type.legacy.viewport_sel);
*val = readl(ptr);
writel(viewport_sel, REGS_ADDR(dw, type.legacy.viewport_sel));
*val = readl(reg);
raw_spin_unlock_irqrestore(&dw->lock, flags);
} else {
@ -93,222 +96,197 @@ legacy_sel_wr:
}
DEFINE_DEBUGFS_ATTRIBUTE(fops_x32, dw_edma_debugfs_u32_get, NULL, "0x%08llx\n");
static void dw_edma_debugfs_create_x32(const struct debugfs_entries entries[],
int nr_entries, struct dentry *dir)
static void dw_edma_debugfs_create_x32(struct dw_edma *dw,
const struct dw_edma_debugfs_entry ini[],
int nr_entries, struct dentry *dent)
{
struct dw_edma_debugfs_entry *entries;
int i;
entries = devm_kcalloc(dw->chip->dev, nr_entries, sizeof(*entries),
GFP_KERNEL);
if (!entries)
return;
for (i = 0; i < nr_entries; i++) {
if (!debugfs_create_file_unsafe(entries[i].name, 0444, dir,
entries[i].reg, &fops_x32))
break;
entries[i] = ini[i];
debugfs_create_file_unsafe(entries[i].name, 0444, dent,
&entries[i], &fops_x32);
}
}
static void dw_edma_debugfs_regs_ch(struct dw_edma_v0_ch_regs __iomem *regs,
struct dentry *dir)
static void dw_edma_debugfs_regs_ch(struct dw_edma *dw, enum dw_edma_dir dir,
u16 ch, struct dentry *dent)
{
int nr_entries;
const struct debugfs_entries debugfs_regs[] = {
REGISTER(ch_control1),
REGISTER(ch_control2),
REGISTER(transfer_size),
REGISTER(sar.lsb),
REGISTER(sar.msb),
REGISTER(dar.lsb),
REGISTER(dar.msb),
REGISTER(llp.lsb),
REGISTER(llp.msb),
struct dw_edma_debugfs_entry debugfs_regs[] = {
CTX_REGISTER(dw, ch_control1, dir, ch),
CTX_REGISTER(dw, ch_control2, dir, ch),
CTX_REGISTER(dw, transfer_size, dir, ch),
CTX_REGISTER(dw, sar.lsb, dir, ch),
CTX_REGISTER(dw, sar.msb, dir, ch),
CTX_REGISTER(dw, dar.lsb, dir, ch),
CTX_REGISTER(dw, dar.msb, dir, ch),
CTX_REGISTER(dw, llp.lsb, dir, ch),
CTX_REGISTER(dw, llp.msb, dir, ch),
};
int nr_entries;
nr_entries = ARRAY_SIZE(debugfs_regs);
dw_edma_debugfs_create_x32(debugfs_regs, nr_entries, dir);
dw_edma_debugfs_create_x32(dw, debugfs_regs, nr_entries, dent);
}
static void dw_edma_debugfs_regs_wr(struct dentry *dir)
static noinline_for_stack void
dw_edma_debugfs_regs_wr(struct dw_edma *dw, struct dentry *dent)
{
const struct debugfs_entries debugfs_regs[] = {
const struct dw_edma_debugfs_entry debugfs_regs[] = {
/* eDMA global registers */
WR_REGISTER(engine_en),
WR_REGISTER(doorbell),
WR_REGISTER(ch_arb_weight.lsb),
WR_REGISTER(ch_arb_weight.msb),
WR_REGISTER(dw, engine_en),
WR_REGISTER(dw, doorbell),
WR_REGISTER(dw, ch_arb_weight.lsb),
WR_REGISTER(dw, ch_arb_weight.msb),
/* eDMA interrupts registers */
WR_REGISTER(int_status),
WR_REGISTER(int_mask),
WR_REGISTER(int_clear),
WR_REGISTER(err_status),
WR_REGISTER(done_imwr.lsb),
WR_REGISTER(done_imwr.msb),
WR_REGISTER(abort_imwr.lsb),
WR_REGISTER(abort_imwr.msb),
WR_REGISTER(ch01_imwr_data),
WR_REGISTER(ch23_imwr_data),
WR_REGISTER(ch45_imwr_data),
WR_REGISTER(ch67_imwr_data),
WR_REGISTER(linked_list_err_en),
WR_REGISTER(dw, int_status),
WR_REGISTER(dw, int_mask),
WR_REGISTER(dw, int_clear),
WR_REGISTER(dw, err_status),
WR_REGISTER(dw, done_imwr.lsb),
WR_REGISTER(dw, done_imwr.msb),
WR_REGISTER(dw, abort_imwr.lsb),
WR_REGISTER(dw, abort_imwr.msb),
WR_REGISTER(dw, ch01_imwr_data),
WR_REGISTER(dw, ch23_imwr_data),
WR_REGISTER(dw, ch45_imwr_data),
WR_REGISTER(dw, ch67_imwr_data),
WR_REGISTER(dw, linked_list_err_en),
};
const struct debugfs_entries debugfs_unroll_regs[] = {
const struct dw_edma_debugfs_entry debugfs_unroll_regs[] = {
/* eDMA channel context grouping */
WR_REGISTER_UNROLL(engine_chgroup),
WR_REGISTER_UNROLL(engine_hshake_cnt.lsb),
WR_REGISTER_UNROLL(engine_hshake_cnt.msb),
WR_REGISTER_UNROLL(ch0_pwr_en),
WR_REGISTER_UNROLL(ch1_pwr_en),
WR_REGISTER_UNROLL(ch2_pwr_en),
WR_REGISTER_UNROLL(ch3_pwr_en),
WR_REGISTER_UNROLL(ch4_pwr_en),
WR_REGISTER_UNROLL(ch5_pwr_en),
WR_REGISTER_UNROLL(ch6_pwr_en),
WR_REGISTER_UNROLL(ch7_pwr_en),
WR_REGISTER_UNROLL(dw, engine_chgroup),
WR_REGISTER_UNROLL(dw, engine_hshake_cnt.lsb),
WR_REGISTER_UNROLL(dw, engine_hshake_cnt.msb),
WR_REGISTER_UNROLL(dw, ch0_pwr_en),
WR_REGISTER_UNROLL(dw, ch1_pwr_en),
WR_REGISTER_UNROLL(dw, ch2_pwr_en),
WR_REGISTER_UNROLL(dw, ch3_pwr_en),
WR_REGISTER_UNROLL(dw, ch4_pwr_en),
WR_REGISTER_UNROLL(dw, ch5_pwr_en),
WR_REGISTER_UNROLL(dw, ch6_pwr_en),
WR_REGISTER_UNROLL(dw, ch7_pwr_en),
};
struct dentry *regs_dir, *ch_dir;
struct dentry *regs_dent, *ch_dent;
int nr_entries, i;
char name[16];
regs_dir = debugfs_create_dir(WRITE_STR, dir);
if (!regs_dir)
return;
regs_dent = debugfs_create_dir(WRITE_STR, dent);
nr_entries = ARRAY_SIZE(debugfs_regs);
dw_edma_debugfs_create_x32(debugfs_regs, nr_entries, regs_dir);
dw_edma_debugfs_create_x32(dw, debugfs_regs, nr_entries, regs_dent);
if (dw->chip->mf == EDMA_MF_HDMA_COMPAT) {
nr_entries = ARRAY_SIZE(debugfs_unroll_regs);
dw_edma_debugfs_create_x32(debugfs_unroll_regs, nr_entries,
regs_dir);
dw_edma_debugfs_create_x32(dw, debugfs_unroll_regs, nr_entries,
regs_dent);
}
for (i = 0; i < dw->wr_ch_cnt; i++) {
snprintf(name, sizeof(name), "%s:%d", CHANNEL_STR, i);
ch_dir = debugfs_create_dir(name, regs_dir);
if (!ch_dir)
return;
ch_dent = debugfs_create_dir(name, regs_dent);
dw_edma_debugfs_regs_ch(&regs->type.unroll.ch[i].wr, ch_dir);
lim[0][i].start = &regs->type.unroll.ch[i].wr;
lim[0][i].end = &regs->type.unroll.ch[i].padding_1[0];
dw_edma_debugfs_regs_ch(dw, EDMA_DIR_WRITE, i, ch_dent);
}
}
static void dw_edma_debugfs_regs_rd(struct dentry *dir)
static noinline_for_stack void dw_edma_debugfs_regs_rd(struct dw_edma *dw,
struct dentry *dent)
{
const struct debugfs_entries debugfs_regs[] = {
const struct dw_edma_debugfs_entry debugfs_regs[] = {
/* eDMA global registers */
RD_REGISTER(engine_en),
RD_REGISTER(doorbell),
RD_REGISTER(ch_arb_weight.lsb),
RD_REGISTER(ch_arb_weight.msb),
RD_REGISTER(dw, engine_en),
RD_REGISTER(dw, doorbell),
RD_REGISTER(dw, ch_arb_weight.lsb),
RD_REGISTER(dw, ch_arb_weight.msb),
/* eDMA interrupts registers */
RD_REGISTER(int_status),
RD_REGISTER(int_mask),
RD_REGISTER(int_clear),
RD_REGISTER(err_status.lsb),
RD_REGISTER(err_status.msb),
RD_REGISTER(linked_list_err_en),
RD_REGISTER(done_imwr.lsb),
RD_REGISTER(done_imwr.msb),
RD_REGISTER(abort_imwr.lsb),
RD_REGISTER(abort_imwr.msb),
RD_REGISTER(ch01_imwr_data),
RD_REGISTER(ch23_imwr_data),
RD_REGISTER(ch45_imwr_data),
RD_REGISTER(ch67_imwr_data),
RD_REGISTER(dw, int_status),
RD_REGISTER(dw, int_mask),
RD_REGISTER(dw, int_clear),
RD_REGISTER(dw, err_status.lsb),
RD_REGISTER(dw, err_status.msb),
RD_REGISTER(dw, linked_list_err_en),
RD_REGISTER(dw, done_imwr.lsb),
RD_REGISTER(dw, done_imwr.msb),
RD_REGISTER(dw, abort_imwr.lsb),
RD_REGISTER(dw, abort_imwr.msb),
RD_REGISTER(dw, ch01_imwr_data),
RD_REGISTER(dw, ch23_imwr_data),
RD_REGISTER(dw, ch45_imwr_data),
RD_REGISTER(dw, ch67_imwr_data),
};
const struct debugfs_entries debugfs_unroll_regs[] = {
const struct dw_edma_debugfs_entry debugfs_unroll_regs[] = {
/* eDMA channel context grouping */
RD_REGISTER_UNROLL(engine_chgroup),
RD_REGISTER_UNROLL(engine_hshake_cnt.lsb),
RD_REGISTER_UNROLL(engine_hshake_cnt.msb),
RD_REGISTER_UNROLL(ch0_pwr_en),
RD_REGISTER_UNROLL(ch1_pwr_en),
RD_REGISTER_UNROLL(ch2_pwr_en),
RD_REGISTER_UNROLL(ch3_pwr_en),
RD_REGISTER_UNROLL(ch4_pwr_en),
RD_REGISTER_UNROLL(ch5_pwr_en),
RD_REGISTER_UNROLL(ch6_pwr_en),
RD_REGISTER_UNROLL(ch7_pwr_en),
RD_REGISTER_UNROLL(dw, engine_chgroup),
RD_REGISTER_UNROLL(dw, engine_hshake_cnt.lsb),
RD_REGISTER_UNROLL(dw, engine_hshake_cnt.msb),
RD_REGISTER_UNROLL(dw, ch0_pwr_en),
RD_REGISTER_UNROLL(dw, ch1_pwr_en),
RD_REGISTER_UNROLL(dw, ch2_pwr_en),
RD_REGISTER_UNROLL(dw, ch3_pwr_en),
RD_REGISTER_UNROLL(dw, ch4_pwr_en),
RD_REGISTER_UNROLL(dw, ch5_pwr_en),
RD_REGISTER_UNROLL(dw, ch6_pwr_en),
RD_REGISTER_UNROLL(dw, ch7_pwr_en),
};
struct dentry *regs_dir, *ch_dir;
struct dentry *regs_dent, *ch_dent;
int nr_entries, i;
char name[16];
regs_dir = debugfs_create_dir(READ_STR, dir);
if (!regs_dir)
return;
regs_dent = debugfs_create_dir(READ_STR, dent);
nr_entries = ARRAY_SIZE(debugfs_regs);
dw_edma_debugfs_create_x32(debugfs_regs, nr_entries, regs_dir);
dw_edma_debugfs_create_x32(dw, debugfs_regs, nr_entries, regs_dent);
if (dw->chip->mf == EDMA_MF_HDMA_COMPAT) {
nr_entries = ARRAY_SIZE(debugfs_unroll_regs);
dw_edma_debugfs_create_x32(debugfs_unroll_regs, nr_entries,
regs_dir);
dw_edma_debugfs_create_x32(dw, debugfs_unroll_regs, nr_entries,
regs_dent);
}
for (i = 0; i < dw->rd_ch_cnt; i++) {
snprintf(name, sizeof(name), "%s:%d", CHANNEL_STR, i);
ch_dir = debugfs_create_dir(name, regs_dir);
if (!ch_dir)
return;
ch_dent = debugfs_create_dir(name, regs_dent);
dw_edma_debugfs_regs_ch(&regs->type.unroll.ch[i].rd, ch_dir);
lim[1][i].start = &regs->type.unroll.ch[i].rd;
lim[1][i].end = &regs->type.unroll.ch[i].padding_2[0];
dw_edma_debugfs_regs_ch(dw, EDMA_DIR_READ, i, ch_dent);
}
}
static void dw_edma_debugfs_regs(void)
static void dw_edma_debugfs_regs(struct dw_edma *dw)
{
const struct debugfs_entries debugfs_regs[] = {
REGISTER(ctrl_data_arb_prior),
REGISTER(ctrl),
const struct dw_edma_debugfs_entry debugfs_regs[] = {
REGISTER(dw, ctrl_data_arb_prior),
REGISTER(dw, ctrl),
};
struct dentry *regs_dir;
struct dentry *regs_dent;
int nr_entries;
regs_dir = debugfs_create_dir(REGISTERS_STR, dw->debugfs);
if (!regs_dir)
return;
regs_dent = debugfs_create_dir(REGISTERS_STR, dw->dma.dbg_dev_root);
nr_entries = ARRAY_SIZE(debugfs_regs);
dw_edma_debugfs_create_x32(debugfs_regs, nr_entries, regs_dir);
dw_edma_debugfs_create_x32(dw, debugfs_regs, nr_entries, regs_dent);
dw_edma_debugfs_regs_wr(regs_dir);
dw_edma_debugfs_regs_rd(regs_dir);
dw_edma_debugfs_regs_wr(dw, regs_dent);
dw_edma_debugfs_regs_rd(dw, regs_dent);
}
void dw_edma_v0_debugfs_on(struct dw_edma *_dw)
void dw_edma_v0_debugfs_on(struct dw_edma *dw)
{
dw = _dw;
if (!dw)
if (!debugfs_initialized())
return;
regs = dw->chip->reg_base;
if (!regs)
return;
debugfs_create_u32("mf", 0444, dw->dma.dbg_dev_root, &dw->chip->mf);
debugfs_create_u16("wr_ch_cnt", 0444, dw->dma.dbg_dev_root, &dw->wr_ch_cnt);
debugfs_create_u16("rd_ch_cnt", 0444, dw->dma.dbg_dev_root, &dw->rd_ch_cnt);
dw->debugfs = debugfs_create_dir(dw->name, NULL);
if (!dw->debugfs)
return;
debugfs_create_u32("mf", 0444, dw->debugfs, &dw->chip->mf);
debugfs_create_u16("wr_ch_cnt", 0444, dw->debugfs, &dw->wr_ch_cnt);
debugfs_create_u16("rd_ch_cnt", 0444, dw->debugfs, &dw->rd_ch_cnt);
dw_edma_debugfs_regs();
}
void dw_edma_v0_debugfs_off(struct dw_edma *_dw)
{
dw = _dw;
if (!dw)
return;
debugfs_remove_recursive(dw->debugfs);
dw->debugfs = NULL;
dw_edma_debugfs_regs(dw);
}

5
drivers/dma/dw-edma/dw-edma-v0-debugfs.h
View File

@ -13,15 +13,10 @@
#ifdef CONFIG_DEBUG_FS
void dw_edma_v0_debugfs_on(struct dw_edma *dw);
void dw_edma_v0_debugfs_off(struct dw_edma *dw);
#else
static inline void dw_edma_v0_debugfs_on(struct dw_edma *dw)
{
}
static inline void dw_edma_v0_debugfs_off(struct dw_edma *dw)
{
}
#endif /* CONFIG_DEBUG_FS */
#endif /* _DW_EDMA_V0_DEBUG_FS_H */

4
drivers/pci/controller/dwc/pcie-bt1.c
View File

@ -583,6 +583,10 @@ static int bt1_pcie_add_port(struct bt1_pcie *btpci)
struct device *dev = &btpci->pdev->dev;
int ret;
ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64));
if (ret)
return ret;
btpci->dw.version = DW_PCIE_VER_460A;
btpci->dw.dev = dev;
btpci->dw.ops = &bt1_pcie_ops;

12
drivers/pci/controller/dwc/pcie-designware-ep.c
View File

@ -612,8 +612,11 @@ int dw_pcie_ep_raise_msix_irq(struct dw_pcie_ep *ep, u8 func_no,
void dw_pcie_ep_exit(struct dw_pcie_ep *ep)
{
struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
struct pci_epc *epc = ep->epc;
dw_pcie_edma_remove(pci);
pci_epc_mem_free_addr(epc, ep->msi_mem_phys, ep->msi_mem,
epc->mem->window.page_size);
@ -785,6 +788,10 @@ int dw_pcie_ep_init(struct dw_pcie_ep *ep)
goto err_exit_epc_mem;
}
ret = dw_pcie_edma_detect(pci);
if (ret)
goto err_free_epc_mem;
if (ep->ops->get_features) {
epc_features = ep->ops->get_features(ep);
if (epc_features->core_init_notifier)
@ -793,10 +800,13 @@ int dw_pcie_ep_init(struct dw_pcie_ep *ep)
ret = dw_pcie_ep_init_complete(ep);
if (ret)
goto err_free_epc_mem;
goto err_remove_edma;
return 0;
err_remove_edma:
dw_pcie_edma_remove(pci);
err_free_epc_mem:
pci_epc_mem_free_addr(epc, ep->msi_mem_phys, ep->msi_mem,
epc->mem->window.page_size);

25
drivers/pci/controller/dwc/pcie-designware-host.c
View File

@ -366,7 +366,17 @@ static int dw_pcie_msi_host_init(struct dw_pcie_rp *pp)
dw_chained_msi_isr, pp);
}
ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32));
/*
* Even though the iMSI-RX Module supports 64-bit addresses some
* peripheral PCIe devices may lack 64-bit message support. In
* order not to miss MSI TLPs from those devices the MSI target
* address has to be within the lowest 4GB.
*
* Note until there is a better alternative found the reservation is
* done by allocating from the artificially limited DMA-coherent
* memory.
*/
ret = dma_set_coherent_mask(dev, DMA_BIT_MASK(32));
if (ret)
dev_warn(dev, "Failed to set DMA mask to 32-bit. Devices with only 32-bit MSI support may not work properly\n");
@ -467,14 +477,18 @@ int dw_pcie_host_init(struct dw_pcie_rp *pp)
dw_pcie_iatu_detect(pci);
ret = dw_pcie_setup_rc(pp);
ret = dw_pcie_edma_detect(pci);
if (ret)
goto err_free_msi;
ret = dw_pcie_setup_rc(pp);
if (ret)
goto err_remove_edma;
if (!dw_pcie_link_up(pci)) {
ret = dw_pcie_start_link(pci);
if (ret)
goto err_free_msi;
goto err_remove_edma;
}
/* Ignore errors, the link may come up later */
@ -491,6 +505,9 @@ int dw_pcie_host_init(struct dw_pcie_rp *pp)
err_stop_link:
dw_pcie_stop_link(pci);
err_remove_edma:
dw_pcie_edma_remove(pci);
err_free_msi:
if (pp->has_msi_ctrl)
dw_pcie_free_msi(pp);
@ -512,6 +529,8 @@ void dw_pcie_host_deinit(struct dw_pcie_rp *pp)
dw_pcie_stop_link(pci);
dw_pcie_edma_remove(pci);
if (pp->has_msi_ctrl)
dw_pcie_free_msi(pp);

195
drivers/pci/controller/dwc/pcie-designware.c
View File

@ -12,6 +12,7 @@
#include <linux/bitops.h>
#include <linux/clk.h>
#include <linux/delay.h>
#include <linux/dma/edma.h>
#include <linux/gpio/consumer.h>
#include <linux/ioport.h>
#include <linux/of.h>
@ -142,6 +143,18 @@ int dw_pcie_get_resources(struct dw_pcie *pci)
if (!pci->atu_size)
pci->atu_size = SZ_4K;
/* eDMA region can be mapped to a custom base address */
if (!pci->edma.reg_base) {
res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "dma");
if (res) {
pci->edma.reg_base = devm_ioremap_resource(pci->dev, res);
if (IS_ERR(pci->edma.reg_base))
return PTR_ERR(pci->edma.reg_base);
} else if (pci->atu_size >= 2 * DEFAULT_DBI_DMA_OFFSET) {
pci->edma.reg_base = pci->atu_base + DEFAULT_DBI_DMA_OFFSET;
}
}
/* LLDD is supposed to manually switch the clocks and resets state */
if (dw_pcie_cap_is(pci, REQ_RES)) {
ret = dw_pcie_get_clocks(pci);
@ -782,6 +795,188 @@ void dw_pcie_iatu_detect(struct dw_pcie *pci)
pci->region_align / SZ_1K, (pci->region_limit + 1) / SZ_1G);
}
static u32 dw_pcie_readl_dma(struct dw_pcie *pci, u32 reg)
{
u32 val = 0;
int ret;
if (pci->ops && pci->ops->read_dbi)
return pci->ops->read_dbi(pci, pci->edma.reg_base, reg, 4);
ret = dw_pcie_read(pci->edma.reg_base + reg, 4, &val);
if (ret)
dev_err(pci->dev, "Read DMA address failed\n");
return val;
}
static int dw_pcie_edma_irq_vector(struct device *dev, unsigned int nr)
{
struct platform_device *pdev = to_platform_device(dev);
char name[6];
int ret;
if (nr >= EDMA_MAX_WR_CH + EDMA_MAX_RD_CH)
return -EINVAL;
ret = platform_get_irq_byname_optional(pdev, "dma");
if (ret > 0)
return ret;
snprintf(name, sizeof(name), "dma%u", nr);
return platform_get_irq_byname_optional(pdev, name);
}
static struct dw_edma_core_ops dw_pcie_edma_ops = {
.irq_vector = dw_pcie_edma_irq_vector,
};
static int dw_pcie_edma_find_chip(struct dw_pcie *pci)
{
u32 val;
/*
* Indirect eDMA CSRs access has been completely removed since v5.40a
* thus no space is now reserved for the eDMA channels viewport and
* former DMA CTRL register is no longer fixed to FFs.
*/
if (dw_pcie_ver_is_ge(pci, 540A))
val = 0xFFFFFFFF;
else
val = dw_pcie_readl_dbi(pci, PCIE_DMA_VIEWPORT_BASE + PCIE_DMA_CTRL);
if (val == 0xFFFFFFFF && pci->edma.reg_base) {
pci->edma.mf = EDMA_MF_EDMA_UNROLL;
val = dw_pcie_readl_dma(pci, PCIE_DMA_CTRL);
} else if (val != 0xFFFFFFFF) {
pci->edma.mf = EDMA_MF_EDMA_LEGACY;
pci->edma.reg_base = pci->dbi_base + PCIE_DMA_VIEWPORT_BASE;
} else {
return -ENODEV;
}
pci->edma.dev = pci->dev;
if (!pci->edma.ops)
pci->edma.ops = &dw_pcie_edma_ops;
pci->edma.flags |= DW_EDMA_CHIP_LOCAL;
pci->edma.ll_wr_cnt = FIELD_GET(PCIE_DMA_NUM_WR_CHAN, val);
pci->edma.ll_rd_cnt = FIELD_GET(PCIE_DMA_NUM_RD_CHAN, val);
/* Sanity check the channels count if the mapping was incorrect */
if (!pci->edma.ll_wr_cnt || pci->edma.ll_wr_cnt > EDMA_MAX_WR_CH ||
!pci->edma.ll_rd_cnt || pci->edma.ll_rd_cnt > EDMA_MAX_RD_CH)
return -EINVAL;
return 0;
}
static int dw_pcie_edma_irq_verify(struct dw_pcie *pci)
{
struct platform_device *pdev = to_platform_device(pci->dev);
u16 ch_cnt = pci->edma.ll_wr_cnt + pci->edma.ll_rd_cnt;
char name[6];
int ret;
if (pci->edma.nr_irqs == 1)
return 0;
else if (pci->edma.nr_irqs > 1)
return pci->edma.nr_irqs != ch_cnt ? -EINVAL : 0;
ret = platform_get_irq_byname_optional(pdev, "dma");
if (ret > 0) {
pci->edma.nr_irqs = 1;
return 0;
}
for (; pci->edma.nr_irqs < ch_cnt; pci->edma.nr_irqs++) {
snprintf(name, sizeof(name), "dma%d", pci->edma.nr_irqs);
ret = platform_get_irq_byname_optional(pdev, name);
if (ret <= 0)
return -EINVAL;
}
return 0;
}
static int dw_pcie_edma_ll_alloc(struct dw_pcie *pci)
{
struct dw_edma_region *ll;
dma_addr_t paddr;
int i;
for (i = 0; i < pci->edma.ll_wr_cnt; i++) {
ll = &pci->edma.ll_region_wr[i];
ll->sz = DMA_LLP_MEM_SIZE;
ll->vaddr.mem = dmam_alloc_coherent(pci->dev, ll->sz,
&paddr, GFP_KERNEL);
if (!ll->vaddr.mem)
return -ENOMEM;
ll->paddr = paddr;
}
for (i = 0; i < pci->edma.ll_rd_cnt; i++) {
ll = &pci->edma.ll_region_rd[i];
ll->sz = DMA_LLP_MEM_SIZE;
ll->vaddr.mem = dmam_alloc_coherent(pci->dev, ll->sz,
&paddr, GFP_KERNEL);
if (!ll->vaddr.mem)
return -ENOMEM;
ll->paddr = paddr;
}
return 0;
}
int dw_pcie_edma_detect(struct dw_pcie *pci)
{
int ret;
/* Don't fail if no eDMA was found (for the backward compatibility) */
ret = dw_pcie_edma_find_chip(pci);
if (ret)
return 0;
/* Don't fail on the IRQs verification (for the backward compatibility) */
ret = dw_pcie_edma_irq_verify(pci);
if (ret) {
dev_err(pci->dev, "Invalid eDMA IRQs found\n");
return 0;
}
ret = dw_pcie_edma_ll_alloc(pci);
if (ret) {
dev_err(pci->dev, "Couldn't allocate LLP memory\n");
return ret;
}
/* Don't fail if the DW eDMA driver can't find the device */
ret = dw_edma_probe(&pci->edma);
if (ret && ret != -ENODEV) {
dev_err(pci->dev, "Couldn't register eDMA device\n");
return ret;
}
dev_info(pci->dev, "eDMA: unroll %s, %hu wr, %hu rd\n",
pci->edma.mf == EDMA_MF_EDMA_UNROLL ? "T" : "F",
pci->edma.ll_wr_cnt, pci->edma.ll_rd_cnt);
return 0;
}
void dw_pcie_edma_remove(struct dw_pcie *pci)
{
dw_edma_remove(&pci->edma);
}
void dw_pcie_setup(struct dw_pcie *pci)
{
u32 val;

21
drivers/pci/controller/dwc/pcie-designware.h
View File

@ -15,6 +15,7 @@
#include <linux/bitops.h>
#include <linux/clk.h>
#include <linux/dma-mapping.h>
#include <linux/dma/edma.h>
#include <linux/gpio/consumer.h>
#include <linux/irq.h>
#include <linux/msi.h>
@ -31,6 +32,7 @@
#define DW_PCIE_VER_480A 0x3438302a
#define DW_PCIE_VER_490A 0x3439302a
#define DW_PCIE_VER_520A 0x3532302a
#define DW_PCIE_VER_540A 0x3534302a
#define __dw_pcie_ver_cmp(_pci, _ver, _op) \
((_pci)->version _op DW_PCIE_VER_ ## _ver)
@ -167,6 +169,18 @@
#define PCIE_MSIX_DOORBELL 0x948
#define PCIE_MSIX_DOORBELL_PF_SHIFT 24
/*
* eDMA CSRs. DW PCIe IP-core v4.70a and older had the eDMA registers accessible
* over the Port Logic registers space. Afterwards the unrolled mapping was
* introduced so eDMA and iATU could be accessed via a dedicated registers
* space.
*/
#define PCIE_DMA_VIEWPORT_BASE 0x970
#define PCIE_DMA_UNROLL_BASE 0x80000
#define PCIE_DMA_CTRL 0x008
#define PCIE_DMA_NUM_WR_CHAN GENMASK(3, 0)
#define PCIE_DMA_NUM_RD_CHAN GENMASK(19, 16)
#define PCIE_PL_CHK_REG_CONTROL_STATUS 0xB20
#define PCIE_PL_CHK_REG_CHK_REG_START BIT(0)
#define PCIE_PL_CHK_REG_CHK_REG_CONTINUOUS BIT(1)
@ -215,6 +229,7 @@
* this offset, if atu_base not set.
*/
#define DEFAULT_DBI_ATU_OFFSET (0x3 << 20)
#define DEFAULT_DBI_DMA_OFFSET PCIE_DMA_UNROLL_BASE
#define MAX_MSI_IRQS 256
#define MAX_MSI_IRQS_PER_CTRL 32
@ -226,6 +241,9 @@
#define MAX_IATU_IN 256
#define MAX_IATU_OUT 256
/* Default eDMA LLP memory size */
#define DMA_LLP_MEM_SIZE PAGE_SIZE
struct dw_pcie;
struct dw_pcie_rp;
struct dw_pcie_ep;
@ -369,6 +387,7 @@ struct dw_pcie {
int num_lanes;
int link_gen;
u8 n_fts[2];
struct dw_edma_chip edma;
struct clk_bulk_data app_clks[DW_PCIE_NUM_APP_CLKS];
struct clk_bulk_data core_clks[DW_PCIE_NUM_CORE_CLKS];
struct reset_control_bulk_data app_rsts[DW_PCIE_NUM_APP_RSTS];
@ -408,6 +427,8 @@ int dw_pcie_prog_ep_inbound_atu(struct dw_pcie *pci, u8 func_no, int index,
void dw_pcie_disable_atu(struct dw_pcie *pci, u32 dir, int index);
void dw_pcie_setup(struct dw_pcie *pci);
void dw_pcie_iatu_detect(struct dw_pcie *pci);
int dw_pcie_edma_detect(struct dw_pcie *pci);
void dw_pcie_edma_remove(struct dw_pcie *pci);
static inline void dw_pcie_writel_dbi(struct dw_pcie *pci, u32 reg, u32 val)
{

25
include/linux/dma/edma.h
View File

@ -18,13 +18,31 @@
struct dw_edma;
struct dw_edma_region {
phys_addr_t paddr;
void __iomem *vaddr;
u64 paddr;
union {
void *mem;
void __iomem *io;
} vaddr;
size_t sz;
};
/**
* struct dw_edma_core_ops - platform-specific eDMA methods
* @irq_vector: Get IRQ number of the passed eDMA channel. Note the
* method accepts the channel id in the end-to-end
* numbering with the eDMA write channels being placed
* first in the row.
* @pci_address: Get PCIe bus address corresponding to the passed CPU
* address. Note there is no need in specifying this
* function if the address translation is performed by
* the DW PCIe RP/EP controller with the DW eDMA device in
* subject and DMA_BYPASS isn't set for all the outbound
* iATU windows. That will be done by the controller
* automatically.
*/
struct dw_edma_core_ops {
int (*irq_vector)(struct device *dev, unsigned int nr);
u64 (*pci_address)(struct device *dev, phys_addr_t cpu_addr);
};
enum dw_edma_map_format {
@ -61,7 +79,6 @@ enum dw_edma_chip_flags {
*/
struct dw_edma_chip {
struct device *dev;
int id;
int nr_irqs;
const struct dw_edma_core_ops *ops;
u32 flags;
@ -84,7 +101,7 @@ struct dw_edma_chip {
};
/* Export to the platform drivers */
#if IS_ENABLED(CONFIG_DW_EDMA)
#if IS_REACHABLE(CONFIG_DW_EDMA)
int dw_edma_probe(struct dw_edma_chip *chip);
int dw_edma_remove(struct dw_edma_chip *chip);
#else

2
include/linux/dmaengine.h
View File

@ -394,7 +394,7 @@ enum dma_slave_buswidth {
* should be read (RX), if the source is memory this argument is
* ignored.
* @dst_addr: this is the physical address where DMA slave data
* should be written (TX), if the source is memory this argument
* should be written (TX), if the destination is memory this argument
* is ignored.
* @src_addr_width: this is the width in bytes of the source (RX)
* register where DMA data shall be read. If the source