diff options
author | codeworkx <daniel.hillenbrand@codeworkx.de> | 2012-06-02 13:09:29 +0200 |
---|---|---|
committer | codeworkx <daniel.hillenbrand@codeworkx.de> | 2012-06-02 13:09:29 +0200 |
commit | c6da2cfeb05178a11c6d062a06f8078150ee492f (patch) | |
tree | f3b4021d252c52d6463a9b3c1bb7245e399b009c /drivers/iommu | |
parent | c6d7c4dbff353eac7919342ae6b3299a378160a6 (diff) | |
download | kernel_samsung_smdk4412-c6da2cfeb05178a11c6d062a06f8078150ee492f.zip kernel_samsung_smdk4412-c6da2cfeb05178a11c6d062a06f8078150ee492f.tar.gz kernel_samsung_smdk4412-c6da2cfeb05178a11c6d062a06f8078150ee492f.tar.bz2 |
samsung update 1
Diffstat (limited to 'drivers/iommu')
-rw-r--r-- | drivers/iommu/Kconfig | 161 | ||||
-rw-r--r-- | drivers/iommu/Makefile | 12 | ||||
-rw-r--r-- | drivers/iommu/exynos-iommu.c | 1054 | ||||
-rw-r--r-- | drivers/iommu/exynos-iovmm.c | 510 | ||||
-rw-r--r-- | drivers/iommu/iommu.c | 343 |
5 files changed, 2080 insertions, 0 deletions
diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig new file mode 100644 index 0000000..9eccee4 --- /dev/null +++ b/drivers/iommu/Kconfig @@ -0,0 +1,161 @@ +# IOMMU_API always gets selected by whoever wants it. +config IOMMU_API + bool + +menuconfig IOMMU_SUPPORT + bool "IOMMU Hardware Support" + default y + ---help--- + Say Y here if you want to compile device drivers for IO Memory + Management Units into the kernel. These devices usually allow to + remap DMA requests and/or remap interrupts from other devices on the + system. + +if IOMMU_SUPPORT + +# MSM IOMMU support +config MSM_IOMMU + bool "MSM IOMMU Support" + depends on ARCH_MSM8X60 || ARCH_MSM8960 + select IOMMU_API + help + Support for the IOMMUs found on certain Qualcomm SOCs. + These IOMMUs allow virtualization of the address space used by most + cores within the multimedia subsystem. + + If unsure, say N here. + +config IOMMU_PGTABLES_L2 + def_bool y + depends on MSM_IOMMU && MMU && SMP && CPU_DCACHE_DISABLE=n + +# AMD IOMMU support +config AMD_IOMMU + bool "AMD IOMMU support" + select SWIOTLB + select PCI_MSI + select PCI_ATS + select PCI_PRI + select PCI_PASID + select IOMMU_API + depends on X86_64 && PCI && ACPI + ---help--- + With this option you can enable support for AMD IOMMU hardware in + your system. An IOMMU is a hardware component which provides + remapping of DMA memory accesses from devices. With an AMD IOMMU you + can isolate the the DMA memory of different devices and protect the + system from misbehaving device drivers or hardware. + + You can find out if your system has an AMD IOMMU if you look into + your BIOS for an option to enable it or if you have an IVRS ACPI + table. + +config AMD_IOMMU_STATS + bool "Export AMD IOMMU statistics to debugfs" + depends on AMD_IOMMU + select DEBUG_FS + ---help--- + This option enables code in the AMD IOMMU driver to collect various + statistics about whats happening in the driver and exports that + information to userspace via debugfs. + If unsure, say N. + +config AMD_IOMMU_V2 + tristate "AMD IOMMU Version 2 driver (EXPERIMENTAL)" + depends on AMD_IOMMU && PROFILING && EXPERIMENTAL + select MMU_NOTIFIER + ---help--- + This option enables support for the AMD IOMMUv2 features of the IOMMU + hardware. Select this option if you want to use devices that support + the the PCI PRI and PASID interface. + +# Intel IOMMU support +config DMAR_TABLE + bool + +config INTEL_IOMMU + bool "Support for Intel IOMMU using DMA Remapping Devices" + depends on PCI_MSI && ACPI && (X86 || IA64_GENERIC) + select IOMMU_API + select DMAR_TABLE + help + DMA remapping (DMAR) devices support enables independent address + translations for Direct Memory Access (DMA) from devices. + These DMA remapping devices are reported via ACPI tables + and include PCI device scope covered by these DMA + remapping devices. + +config INTEL_IOMMU_DEFAULT_ON + def_bool y + prompt "Enable Intel DMA Remapping Devices by default" + depends on INTEL_IOMMU + help + Selecting this option will enable a DMAR device at boot time if + one is found. If this option is not selected, DMAR support can + be enabled by passing intel_iommu=on to the kernel. + +config INTEL_IOMMU_BROKEN_GFX_WA + bool "Workaround broken graphics drivers (going away soon)" + depends on INTEL_IOMMU && BROKEN && X86 + ---help--- + Current Graphics drivers tend to use physical address + for DMA and avoid using DMA APIs. Setting this config + option permits the IOMMU driver to set a unity map for + all the OS-visible memory. Hence the driver can continue + to use physical addresses for DMA, at least until this + option is removed in the 2.6.32 kernel. + +config INTEL_IOMMU_FLOPPY_WA + def_bool y + depends on INTEL_IOMMU && X86 + ---help--- + Floppy disk drivers are known to bypass DMA API calls + thereby failing to work when IOMMU is enabled. This + workaround will setup a 1:1 mapping for the first + 16MiB to make floppy (an ISA device) work. + +config IRQ_REMAP + bool "Support for Interrupt Remapping (EXPERIMENTAL)" + depends on X86_64 && X86_IO_APIC && PCI_MSI && ACPI && EXPERIMENTAL + select DMAR_TABLE + ---help--- + Supports Interrupt remapping for IO-APIC and MSI devices. + To use x2apic mode in the CPU's which support x2APIC enhancements or + to support platforms with CPU's having > 8 bit APIC ID, say Y. + +config EXYNOS_IOMMU + bool "Exynos IOMMU Support" + depends on ARCH_EXYNOS && EXYNOS_DEV_SYSMMU + select IOMMU_API + help + Support for the IOMMU(System MMU) of Samsung Exynos application + processor family. This enables H/W multimedia accellerators to see + non-linear physical memory chunks as a linear memory in their + address spaces + + If unsure, say N here. +config EXYNOS_IOVMM + bool "IO Virtual Memory Manager for Exynos IOMMUs" + select GENERIC_ALLOCATOR + depends on EXYNOS_IOMMU + default y + help + Supporting the users of Exynos IOMMU for allocating and mapping + an IO virtual memory region with a physical memory region + and managing the allocated virtual memory regions. + +config EXYNOS_IOVMM_ALIGN64K + bool "Let I/O memory region 64KB aligned" + default y + depends on EXYNOS_IOVMM && ARCH_EXYNOS5 + +config EXYNOS_IOMMU_DEBUG + bool "Debugging log for Exynos IOMMU" + depends on EXYNOS_IOMMU + help + Select this to see the detailed log message that shows what + happens in the IOMMU driver + + Say N unless you need kernel log message for IOMMU debugging + +endif # IOMMU_SUPPORT diff --git a/drivers/iommu/Makefile b/drivers/iommu/Makefile new file mode 100644 index 0000000..b6fcc3e --- /dev/null +++ b/drivers/iommu/Makefile @@ -0,0 +1,12 @@ +obj-$(CONFIG_IOMMU_API) += iommu.o +obj-$(CONFIG_MSM_IOMMU) += msm_iommu.o msm_iommu_dev.o +obj-$(CONFIG_AMD_IOMMU) += amd_iommu.o amd_iommu_init.o +obj-$(CONFIG_AMD_IOMMU_V2) += amd_iommu_v2.o +obj-$(CONFIG_DMAR_TABLE) += dmar.o +obj-$(CONFIG_INTEL_IOMMU) += iova.o intel-iommu.o +obj-$(CONFIG_IRQ_REMAP) += intr_remapping.o +obj-$(CONFIG_OMAP_IOMMU) += omap-iommu.o +obj-$(CONFIG_OMAP_IOVMM) += omap-iovmm.o +obj-$(CONFIG_OMAP_IOMMU_DEBUG) += omap-iommu-debug.o +obj-$(CONFIG_EXYNOS_IOMMU) += exynos-iommu.o +obj-$(CONFIG_EXYNOS_IOVMM) += exynos-iovmm.o diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c new file mode 100644 index 0000000..e080220 --- /dev/null +++ b/drivers/iommu/exynos-iommu.c @@ -0,0 +1,1054 @@ +/* linux/drivers/iommu/exynos_iommu.c + * + * Copyright (c) 2011 Samsung Electronics Co., Ltd. + * http://www.samsung.com + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#ifdef CONFIG_EXYNOS_IOMMU_DEBUG +#define DEBUG +#endif + +#include <linux/io.h> +#include <linux/interrupt.h> +#include <linux/platform_device.h> +#include <linux/slab.h> +#include <linux/pm_runtime.h> +#include <linux/clk.h> +#include <linux/err.h> +#include <linux/mm.h> +#include <linux/iommu.h> +#include <linux/errno.h> +#include <linux/list.h> +#include <linux/memblock.h> + +#include <asm/cacheflush.h> +#include <asm/pgtable.h> + +#include <plat/sysmmu.h> + +#include <mach/sysmmu.h> + +/* We does not consider super section mapping (16MB) */ +#define SECT_ORDER 20 +#define LPAGE_ORDER 16 +#define SPAGE_ORDER 12 + +#define SECT_SIZE (1 << SECT_ORDER) +#define LPAGE_SIZE (1 << LPAGE_ORDER) +#define SPAGE_SIZE (1 << SPAGE_ORDER) + +#define SECT_MASK (~(SECT_SIZE - 1)) +#define LPAGE_MASK (~(LPAGE_SIZE - 1)) +#define SPAGE_MASK (~(SPAGE_SIZE - 1)) + +#define lv1ent_fault(sent) (((*(sent) & 3) == 0) || ((*(sent) & 3) == 3)) +#define lv1ent_page(sent) ((*(sent) & 3) == 1) +#define lv1ent_section(sent) ((*(sent) & 3) == 2) + +#define lv2ent_fault(pent) ((*(pent) & 3) == 0) +#define lv2ent_small(pent) ((*(pent) & 2) == 2) +#define lv2ent_large(pent) ((*(pent) & 3) == 1) + +#define section_phys(sent) (*(sent) & SECT_MASK) +#define section_offs(iova) ((iova) & 0xFFFFF) +#define lpage_phys(pent) (*(pent) & LPAGE_MASK) +#define lpage_offs(iova) ((iova) & 0xFFFF) +#define spage_phys(pent) (*(pent) & SPAGE_MASK) +#define spage_offs(iova) ((iova) & 0xFFF) + +#define lv1ent_offset(iova) ((iova) >> SECT_ORDER) +#define lv2ent_offset(iova) (((iova) & 0xFF000) >> SPAGE_ORDER) + +#define NUM_LV1ENTRIES 4096 +#define NUM_LV2ENTRIES 256 + +#define LV2TABLE_SIZE (NUM_LV2ENTRIES * sizeof(long)) + +#define SPAGES_PER_LPAGE (LPAGE_SIZE / SPAGE_SIZE) + +#define lv2table_base(sent) (*(sent) & 0xFFFFFC00) + +#define mk_lv1ent_sect(pa) ((pa) | 2) +#define mk_lv1ent_page(pa) ((pa) | 1) +#define mk_lv2ent_lpage(pa) ((pa) | 1) +#define mk_lv2ent_spage(pa) ((pa) | 2) + +#define CTRL_ENABLE 0x5 +#define CTRL_BLOCK 0x7 +#define CTRL_DISABLE 0x0 + +#define REG_MMU_CTRL 0x000 +#define REG_MMU_CFG 0x004 +#define REG_MMU_STATUS 0x008 +#define REG_MMU_FLUSH 0x00C +#define REG_MMU_FLUSH_ENTRY 0x010 +#define REG_PT_BASE_ADDR 0x014 +#define REG_INT_STATUS 0x018 +#define REG_INT_CLEAR 0x01C + +#define REG_PAGE_FAULT_ADDR 0x024 +#define REG_AW_FAULT_ADDR 0x028 +#define REG_AR_FAULT_ADDR 0x02C +#define REG_DEFAULT_SLAVE_ADDR 0x030 + +#define REG_MMU_VERSION 0x034 + +#define REG_PB0_SADDR 0x04C +#define REG_PB0_EADDR 0x050 +#define REG_PB1_SADDR 0x054 +#define REG_PB1_EADDR 0x058 + +static unsigned long *section_entry(unsigned long *pgtable, unsigned long iova) +{ + return pgtable + lv1ent_offset(iova); +} + +static unsigned long *page_entry(unsigned long *sent, unsigned long iova) +{ + return (unsigned long *)__va(lv2table_base(sent)) + lv2ent_offset(iova); +} + +static unsigned short fault_reg_offset[SYSMMU_FAULTS_NUM] = { + REG_PAGE_FAULT_ADDR, + REG_AR_FAULT_ADDR, + REG_AW_FAULT_ADDR, + REG_DEFAULT_SLAVE_ADDR, + REG_AR_FAULT_ADDR, + REG_AR_FAULT_ADDR, + REG_AW_FAULT_ADDR, + REG_AW_FAULT_ADDR +}; + +static char *sysmmu_fault_name[SYSMMU_FAULTS_NUM] = { + "PAGE FAULT", + "AR MULTI-HIT FAULT", + "AW MULTI-HIT FAULT", + "BUS ERROR", + "AR SECURITY PROTECTION FAULT", + "AR ACCESS PROTECTION FAULT", + "AW SECURITY PROTECTION FAULT", + "AW ACCESS PROTECTION FAULT", + "UNKNOWN FAULT" +}; + +struct exynos_iommu_domain { + struct list_head clients; /* list of sysmmu_drvdata.node */ + unsigned long *pgtable; /* lv1 page table, 16KB */ + short *lv2entcnt; /* free lv2 entry counter for each section */ + spinlock_t lock; /* lock for this structure */ + spinlock_t pgtablelock; /* lock for modifying page table @ pgtable */ +}; + +struct sysmmu_drvdata { + struct list_head node; /* entry of exynos_iommu_domain.clients */ + struct device *sysmmu; /* System MMU's device descriptor */ + struct device *dev; /* Owner of system MMU */ + char *dbgname; + int nsfrs; + void __iomem **sfrbases; + struct clk *clk[2]; + int activations; + rwlock_t lock; + struct iommu_domain *domain; + sysmmu_fault_handler_t fault_handler; + unsigned long pgtable; +}; + +static bool set_sysmmu_active(struct sysmmu_drvdata *data) +{ + /* return true if the System MMU was not active previously + and it needs to be initialized */ + return ++data->activations == 1; +} + +static bool set_sysmmu_inactive(struct sysmmu_drvdata *data) +{ + /* return true if the System MMU is needed to be disabled */ + BUG_ON(data->activations < 1); + return --data->activations == 0; +} + +static bool is_sysmmu_active(struct sysmmu_drvdata *data) +{ + return data->activations > 0; +} + +static void sysmmu_unblock(void __iomem *sfrbase) +{ + __raw_writel(CTRL_ENABLE, sfrbase + REG_MMU_CTRL); +} + +static bool sysmmu_block(void __iomem *sfrbase) +{ + int i = 120; + + __raw_writel(CTRL_BLOCK, sfrbase + REG_MMU_CTRL); + while ((i > 0) && !(__raw_readl(sfrbase + REG_MMU_STATUS) & 1)) + --i; + + if (!(__raw_readl(sfrbase + REG_MMU_STATUS) & 1)) { + sysmmu_unblock(sfrbase); + return false; + } + + return true; +} + +static void __sysmmu_tlb_invalidate(void __iomem *sfrbase) +{ + __raw_writel(0x1, sfrbase + REG_MMU_FLUSH); +} + +static void __sysmmu_tlb_invalidate_entry(void __iomem *sfrbase, + unsigned long iova) +{ + __raw_writel((iova & SPAGE_MASK) | 1, sfrbase + REG_MMU_FLUSH_ENTRY); +} + +static void __sysmmu_set_ptbase(void __iomem *sfrbase, + unsigned long pgd) +{ + __raw_writel(0x1, sfrbase + REG_MMU_CFG); /* 16KB LV1, LRU */ + __raw_writel(pgd, sfrbase + REG_PT_BASE_ADDR); + + __sysmmu_tlb_invalidate(sfrbase); +} + +static void __sysmmu_set_prefbuf(void __iomem *sfrbase, unsigned long base, + unsigned long size, int idx) +{ + __raw_writel(base, sfrbase + REG_PB0_SADDR + idx * 8); + __raw_writel(size - 1 + base, sfrbase + REG_PB0_EADDR + idx * 8); +} + +void exynos_sysmmu_set_prefbuf(struct device *dev, + unsigned long base0, unsigned long size0, + unsigned long base1, unsigned long size1) +{ + struct sysmmu_drvdata *data = dev_get_drvdata(dev->archdata.iommu); + unsigned long flags; + int i; + + BUG_ON((base0 + size0) <= base0); + BUG_ON((size1 > 0) && ((base1 + size1) <= base1)); + + read_lock_irqsave(&data->lock, flags); + if (!is_sysmmu_active(data)) + goto finish; + + for (i = 0; i < data->nsfrs; i++) { + if ((readl(data->sfrbases[i] + REG_MMU_VERSION) >> 28) == 3) { + if (!sysmmu_block(data->sfrbases[i])) + continue; + + if (size1 == 0) { + if (size0 <= SZ_128K) { + base1 = base0; + size1 = size0; + } else { + size1 = size0 - + ALIGN(size0 / 2, SZ_64K); + size0 = size0 - size1; + base1 = base0 + size0; + } + } + + __sysmmu_set_prefbuf( + data->sfrbases[i], base0, size0, 0); + __sysmmu_set_prefbuf( + data->sfrbases[i], base1, size1, 1); + + sysmmu_unblock(data->sfrbases[i]); + } + } +finish: + read_unlock_irqrestore(&data->lock, flags); +} + +static void __set_fault_handler(struct sysmmu_drvdata *data, + sysmmu_fault_handler_t handler) +{ + unsigned long flags; + + write_lock_irqsave(&data->lock, flags); + data->fault_handler = handler; + write_unlock_irqrestore(&data->lock, flags); +} + +void exynos_sysmmu_set_fault_handler(struct device *dev, + sysmmu_fault_handler_t handler) +{ + struct sysmmu_drvdata *data = dev_get_drvdata(dev->archdata.iommu); + + __set_fault_handler(data, handler); +} + +static int default_fault_handler(enum exynos_sysmmu_inttype itype, + unsigned long pgtable_base, unsigned long fault_addr) +{ + unsigned long *ent; + + if ((itype >= SYSMMU_FAULTS_NUM) || (itype < SYSMMU_PAGEFAULT)) + itype = SYSMMU_FAULT_UNKNOWN; + + pr_err("%s occured at 0x%lx(Page table base: 0x%lx)\n", + sysmmu_fault_name[itype], fault_addr, pgtable_base); + + ent = section_entry(__va(pgtable_base), fault_addr); + pr_err("\tLv1 entry: 0x%lx\n", *ent); + + if (lv1ent_page(ent)) { + ent = page_entry(ent, fault_addr); + pr_err("\t Lv2 entry: 0x%lx\n", *ent); + } + + pr_err("Generating Kernel OOPS... because it is unrecoverable.\n"); + + BUG(); + + return 0; +} + +static irqreturn_t exynos_sysmmu_irq(int irq, void *dev_id) +{ + /* SYSMMU is in blocked when interrupt occurred. */ + struct sysmmu_drvdata *data = dev_id; + struct resource *irqres; + struct platform_device *pdev; + enum exynos_sysmmu_inttype itype; + unsigned long addr = -1; + + int i, ret = -ENOSYS; + + read_lock(&data->lock); + + WARN_ON(!is_sysmmu_active(data)); + + pdev = to_platform_device(data->sysmmu); + for (i = 0; i < (pdev->num_resources / 2); i++) { + irqres = platform_get_resource(pdev, IORESOURCE_IRQ, i); + if (irqres && ((int)irqres->start == irq)) + break; + } + + if (i == pdev->num_resources) { + itype = SYSMMU_FAULT_UNKNOWN; + } else { + itype = (enum exynos_sysmmu_inttype) + __ffs(__raw_readl(data->sfrbases[i] + REG_INT_STATUS)); + if (WARN_ON(!((itype >= 0) && (itype < SYSMMU_FAULT_UNKNOWN)))) + itype = SYSMMU_FAULT_UNKNOWN; + else + addr = __raw_readl( + data->sfrbases[i] + fault_reg_offset[itype]); + } + + if (data->domain) + ret = report_iommu_fault(data->domain, data->dev, + addr, itype); + + if ((ret == -ENOSYS) && data->fault_handler) { + unsigned long base = data->pgtable; + if (itype != SYSMMU_FAULT_UNKNOWN) + base = __raw_readl( + data->sfrbases[i] + REG_PT_BASE_ADDR); + ret = data->fault_handler(itype, base, addr); + } + + if (!ret && (itype != SYSMMU_FAULT_UNKNOWN)) + __raw_writel(1 << itype, data->sfrbases[i] + REG_INT_CLEAR); + else + dev_dbg(data->sysmmu, "(%s) %s is not handled.\n", + data->dbgname, sysmmu_fault_name[itype]); + + if (itype != SYSMMU_FAULT_UNKNOWN) + sysmmu_unblock(data->sfrbases[i]); + + read_unlock(&data->lock); + + return IRQ_HANDLED; +} + +static bool __exynos_sysmmu_disable(struct sysmmu_drvdata *data) +{ + unsigned long flags; + bool disabled = false; + int i; + + write_lock_irqsave(&data->lock, flags); + + if (!set_sysmmu_inactive(data)) + goto finish; + + for (i = 0; i < data->nsfrs; i++) + __raw_writel(CTRL_DISABLE, data->sfrbases[i] + REG_MMU_CTRL); + + if (data->clk[1]) + clk_disable(data->clk[1]); + if (data->clk[0]) + clk_disable(data->clk[0]); + + disabled = true; + data->pgtable = 0; + data->domain = NULL; +finish: + write_unlock_irqrestore(&data->lock, flags); + + if (disabled) + dev_dbg(data->sysmmu, "(%s) Disabled\n", data->dbgname); + else + dev_dbg(data->sysmmu, "(%s) %d times left to be disabled\n", + data->dbgname, data->activations); + + return disabled; +} + +/* __exynos_sysmmu_enable: Enables System MMU + * + * returns -error if an error occurred and System MMU is not enabled, + * 0 if the System MMU has been just enabled and 1 if System MMU was already + * enabled before. + */ +static int __exynos_sysmmu_enable(struct sysmmu_drvdata *data, + unsigned long pgtable, struct iommu_domain *domain) +{ + int i, ret = 0; + unsigned long flags; + + write_lock_irqsave(&data->lock, flags); + + if (!set_sysmmu_active(data)) { + if (WARN_ON(pgtable != data->pgtable)) { + ret = -EBUSY; + set_sysmmu_inactive(data); + } else { + ret = 1; + } + + dev_dbg(data->sysmmu, "(%s) Already enabled\n", data->dbgname); + goto finish; + } + + if (data->clk[0]) + clk_enable(data->clk[0]); + if (data->clk[1]) + clk_enable(data->clk[1]); + + data->pgtable = pgtable; + + for (i = 0; i < data->nsfrs; i++) { + __sysmmu_set_ptbase(data->sfrbases[i], pgtable); + + if ((readl(data->sfrbases[i] + REG_MMU_VERSION) >> 28) == 3) { + /* System MMU version is 3.x */ + __raw_writel((1 << 12) | (2 << 28), + data->sfrbases[i] + REG_MMU_CFG); + __sysmmu_set_prefbuf(data->sfrbases[i], 0, -1, 0); + __sysmmu_set_prefbuf(data->sfrbases[i], 0, -1, 1); + } + + __raw_writel(CTRL_ENABLE, data->sfrbases[i] + REG_MMU_CTRL); + } + + data->domain = domain; + + dev_dbg(data->sysmmu, "(%s) Enabled\n", data->dbgname); +finish: + write_unlock_irqrestore(&data->lock, flags); + + return ret; +} + +int exynos_sysmmu_enable(struct device *dev, unsigned long pgtable) +{ + struct sysmmu_drvdata *data = dev_get_drvdata(dev->archdata.iommu); + int ret; + + BUG_ON(!memblock_is_memory(pgtable)); + + ret = pm_runtime_get_sync(data->sysmmu); + if (ret < 0) { + dev_dbg(data->sysmmu, "(%s) Failed to enable\n", data->dbgname); + return ret; + } + + ret = __exynos_sysmmu_enable(data, pgtable, NULL); + if (WARN_ON(ret < 0)) { + pm_runtime_put(data->sysmmu); + dev_err(data->sysmmu, + "(%s) Already enabled with page table %#lx\n", + data->dbgname, data->pgtable); + } else { + data->dev = dev; + } + + return ret; +} + +bool exynos_sysmmu_disable(struct device *dev) +{ + struct sysmmu_drvdata *data = dev_get_drvdata(dev->archdata.iommu); + bool disabled; + + disabled = __exynos_sysmmu_disable(data); + pm_runtime_put(data->sysmmu); + + return disabled; +} + +static void sysmmu_tlb_invalidate_entry(struct device *dev, unsigned long iova) +{ + unsigned long flags; + struct sysmmu_drvdata *data = dev_get_drvdata(dev->archdata.iommu); + + read_lock_irqsave(&data->lock, flags); + + if (is_sysmmu_active(data)) { + int i; + for (i = 0; i < data->nsfrs; i++) { + if (sysmmu_block(data->sfrbases[i])) { + __sysmmu_tlb_invalidate_entry( + data->sfrbases[i], iova); + sysmmu_unblock(data->sfrbases[i]); + } + } + } else { + dev_dbg(data->sysmmu, + "(%s) Disabled. Skipping invalidating TLB.\n", + data->dbgname); + } + + read_unlock_irqrestore(&data->lock, flags); +} + +void exynos_sysmmu_tlb_invalidate(struct device *dev) +{ + unsigned long flags; + struct sysmmu_drvdata *data = dev_get_drvdata(dev->archdata.iommu); + + read_lock_irqsave(&data->lock, flags); + + if (is_sysmmu_active(data)) { + int i; + for (i = 0; i < data->nsfrs; i++) { + if (sysmmu_block(data->sfrbases[i])) { + __sysmmu_tlb_invalidate(data->sfrbases[i]); + sysmmu_unblock(data->sfrbases[i]); + } + } + } else { + dev_dbg(data->sysmmu, + "(%s) Disabled. Skipping invalidating TLB.\n", + data->dbgname); + } + + read_unlock_irqrestore(&data->lock, flags); +} + +static int exynos_sysmmu_probe(struct platform_device *pdev) +{ + int i, ret; + struct device *dev; + struct sysmmu_drvdata *data; + + dev = &pdev->dev; + + data = kzalloc(sizeof(*data), GFP_KERNEL); + if (!data) { + dev_dbg(dev, "Not enough memory\n"); + ret = -ENOMEM; + goto err_alloc; + } + + ret = dev_set_drvdata(dev, data); + if (ret) { + dev_dbg(dev, "Unabled to initialize driver data\n"); + goto err_init; + } + + data->nsfrs = pdev->num_resources / 2; + data->sfrbases = kmalloc(sizeof(*data->sfrbases) * data->nsfrs, + GFP_KERNEL); + if (data->sfrbases == NULL) { + dev_dbg(dev, "Not enough memory\n"); + ret = -ENOMEM; + goto err_init; + } + + for (i = 0; i < data->nsfrs; i++) { + struct resource *res; + res = platform_get_resource(pdev, IORESOURCE_MEM, i); + if (!res) { + dev_dbg(dev, "Unable to find IOMEM region\n"); + ret = -ENOENT; + goto err_res; + } + + data->sfrbases[i] = ioremap(res->start, resource_size(res)); + if (!data->sfrbases[i]) { + dev_dbg(dev, "Unable to map IOMEM @ PA:%#x\n", + res->start); + ret = -ENOENT; + goto err_res; + } + } + + for (i = 0; i < data->nsfrs; i++) { + ret = platform_get_irq(pdev, i); + if (ret <= 0) { + dev_dbg(dev, "Unable to find IRQ resource\n"); + goto err_irq; + } + + ret = request_irq(ret, exynos_sysmmu_irq, 0, + dev_name(dev), data); + if (ret) { + dev_dbg(dev, "Unabled to register interrupt handler\n"); + goto err_irq; + } + } + + if (dev_get_platdata(dev)) { + char *deli, *beg; + struct sysmmu_platform_data *platdata = dev_get_platdata(dev); + + beg = platdata->clockname; + + for (deli = beg; (*deli != '\0') && (*deli != ','); deli++) + /* NOTHING */; + + if (*deli == '\0') + deli = NULL; + else + *deli = '\0'; + + data->clk[0] = clk_get(dev, beg); + if (IS_ERR(data->clk[0])) { + data->clk[0] = NULL; + dev_dbg(dev, "No clock descriptor registered\n"); + } + + if (data->clk[0] && deli) { + *deli = ','; + data->clk[1] = clk_get(dev, deli + 1); + if (IS_ERR(data->clk[1])) + data->clk[1] = NULL; + } + + data->dbgname = platdata->dbgname; + } + + data->sysmmu = dev; + rwlock_init(&data->lock); + INIT_LIST_HEAD(&data->node); + + __set_fault_handler(data, &default_fault_handler); + + if (dev->parent) + pm_runtime_enable(dev); + + dev_dbg(dev, "(%s) Initialized\n", data->dbgname); + return 0; +err_irq: + while (i-- > 0) { + int irq; + + irq = platform_get_irq(pdev, i); + free_irq(irq, data); + } +err_res: + while (data->nsfrs-- > 0) + iounmap(data->sfrbases[data->nsfrs]); + kfree(data->sfrbases); +err_init: + kfree(data); +err_alloc: + dev_err(dev, "Failed to initialize\n"); + return ret; +} + +static struct platform_driver exynos_sysmmu_driver = { + .probe = exynos_sysmmu_probe, + .driver = { + .owner = THIS_MODULE, + .name = "exynos-sysmmu", + } +}; + +static inline void pgtable_flush(void *vastart, void *vaend) +{ + dmac_flush_range(vastart, vaend); + outer_flush_range(virt_to_phys(vastart), + virt_to_phys(vaend)); +} + +static int exynos_iommu_domain_init(struct iommu_domain *domain) +{ + struct exynos_iommu_domain *priv; + + priv = kzalloc(sizeof(*priv), GFP_KERNEL); + if (!priv) + return -ENOMEM; + + priv->pgtable = (unsigned long *)__get_free_pages( + GFP_KERNEL | __GFP_ZERO, 2); + if (!priv->pgtable) + goto err_pgtable; + + priv->lv2entcnt = (short *)__get_free_pages( + GFP_KERNEL | __GFP_ZERO, 1); + if (!priv->lv2entcnt) + goto err_counter; + + pgtable_flush(priv->pgtable, priv->pgtable + NUM_LV1ENTRIES); + + spin_lock_init(&priv->lock); + spin_lock_init(&priv->pgtablelock); + INIT_LIST_HEAD(&priv->clients); + + domain->priv = priv; + return 0; + +err_counter: + free_pages((unsigned long)priv->pgtable, 2); +err_pgtable: + kfree(priv); + return -ENOMEM; +} + +static void exynos_iommu_domain_destroy(struct iommu_domain *domain) +{ + struct exynos_iommu_domain *priv = domain->priv; + struct sysmmu_drvdata *data; + unsigned long flags; + int i; + + WARN_ON(!list_empty(&priv->clients)); + + spin_lock_irqsave(&priv->lock, flags); + + list_for_each_entry(data, &priv->clients, node) { + while (!exynos_sysmmu_disable(data->dev)) + ; /* until System MMU is actually disabled */ + } + + spin_unlock_irqrestore(&priv->lock, flags); + + for (i = 0; i < NUM_LV1ENTRIES; i++) + if (lv1ent_page(priv->pgtable + i)) + kfree(__va(lv2table_base(priv->pgtable + i))); + + free_pages((unsigned long)priv->pgtable, 2); + free_pages((unsigned long)priv->lv2entcnt, 1); + kfree(domain->priv); + domain->priv = NULL; +} + +static int exynos_iommu_attach_device(struct iommu_domain *domain, + struct device *dev) +{ + struct sysmmu_drvdata *data = dev_get_drvdata(dev->archdata.iommu); + struct exynos_iommu_domain *priv = domain->priv; + unsigned long flags; + int ret; + + ret = pm_runtime_get_sync(data->sysmmu); + if (ret < 0) + return ret; + + ret = 0; + + spin_lock_irqsave(&priv->lock, flags); + + ret = __exynos_sysmmu_enable(data, __pa(priv->pgtable), domain); + + if (ret == 0) { + /* 'data->node' must not be appeared in priv->clients */ + BUG_ON(!list_empty(&data->node)); + data->dev = dev; + list_add_tail(&data->node, &priv->clients); + } + + spin_unlock_irqrestore(&priv->lock, flags); + + if (ret < 0) { + dev_err(dev, "%s: Failed to attach IOMMU with pgtable %#lx\n", + __func__, __pa(priv->pgtable)); + pm_runtime_put(data->sysmmu); + } else if (ret > 0) { + dev_dbg(dev, "%s: IOMMU with pgtable 0x%lx already attached\n", + __func__, __pa(priv->pgtable)); + } else { + dev_dbg(dev, "%s: Attached new IOMMU with pgtable 0x%lx\n", + __func__, __pa(priv->pgtable)); + } + + return ret; +} + +static void exynos_iommu_detach_device(struct iommu_domain *domain, + struct device *dev) +{ + struct sysmmu_drvdata *data = dev_get_drvdata(dev->archdata.iommu); + struct exynos_iommu_domain *priv = domain->priv; + struct list_head *pos; + unsigned long flags; + bool found = false; + + spin_lock_irqsave(&priv->lock, flags); + + list_for_each(pos, &priv->clients) { + if (list_entry(pos, struct sysmmu_drvdata, node) == data) { + found = true; + break; + } + } + + if (!found) + goto finish; + + if (__exynos_sysmmu_disable(data)) { + dev_dbg(dev, "%s: Detached IOMMU with pgtable %#lx\n", + __func__, __pa(priv->pgtable)); + list_del(&data->node); + INIT_LIST_HEAD(&data->node); + + } else { + dev_dbg(dev, "%s: Detaching IOMMU with pgtable %#lx delayed", + __func__, __pa(priv->pgtable)); + } + +finish: + spin_unlock_irqrestore(&priv->lock, flags); + + if (found) + pm_runtime_put(data->sysmmu); +} + +static unsigned long *alloc_lv2entry(unsigned long *sent, unsigned long iova, + short *pgcounter) +{ + if (lv1ent_fault(sent)) { + unsigned long *pent; + + pent = kzalloc(LV2TABLE_SIZE, GFP_ATOMIC); + BUG_ON((unsigned long)pent & (LV2TABLE_SIZE - 1)); + if (!pent) + return NULL; + + *sent = mk_lv1ent_page(__pa(pent)); + *pgcounter = NUM_LV2ENTRIES; + pgtable_flush(pent, pent + NUM_LV2ENTRIES); + pgtable_flush(sent, sent + 1); + } + + return page_entry(sent, iova); +} + +static int lv1set_section(unsigned long *sent, phys_addr_t paddr, short *pgcnt) +{ + if (lv1ent_section(sent)) + return -EADDRINUSE; + + if (lv1ent_page(sent)) { + if (*pgcnt != NUM_LV2ENTRIES) + return -EADDRINUSE; + + kfree(page_entry(sent, 0)); + + *pgcnt = 0; + } + + *sent = mk_lv1ent_sect(paddr); + + pgtable_flush(sent, sent + 1); + + return 0; +} + +static int lv2set_page(unsigned long *pent, phys_addr_t paddr, size_t size, + short *pgcnt) +{ + if (size == SPAGE_SIZE) { + if (!lv2ent_fault(pent)) + return -EADDRINUSE; + + *pent = mk_lv2ent_spage(paddr); + pgtable_flush(pent, pent + 1); + *pgcnt -= 1; + } else { /* size == LPAGE_SIZE */ + int i; + for (i = 0; i < SPAGES_PER_LPAGE; i++, pent++) { + if (!lv2ent_fault(pent)) { + memset(pent, 0, sizeof(*pent) * i); + return -EADDRINUSE; + } + + *pent = mk_lv2ent_lpage(paddr); + } + pgtable_flush(pent - SPAGES_PER_LPAGE, pent); + *pgcnt -= SPAGES_PER_LPAGE; + } + + return 0; +} + +static int exynos_iommu_map(struct iommu_domain *domain, unsigned long iova, + phys_addr_t paddr, size_t size, int prot) +{ + struct exynos_iommu_domain *priv = domain->priv; + unsigned long *entry; + unsigned long flags; + int ret = -ENOMEM; + + BUG_ON(priv->pgtable == NULL); + + spin_lock_irqsave(&priv->pgtablelock, flags); + + entry = section_entry(priv->pgtable, iova); + + if (size == SECT_SIZE) { + ret = lv1set_section(entry, paddr, + &priv->lv2entcnt[lv1ent_offset(iova)]); + } else { + unsigned long *pent; + + pent = alloc_lv2entry(entry, iova, + &priv->lv2entcnt[lv1ent_offset(iova)]); + + if (!pent) + ret = -ENOMEM; + else + ret = lv2set_page(pent, paddr, size, + &priv->lv2entcnt[lv1ent_offset(iova)]); + } + + if (ret) { + pr_debug("%s: Failed to map iova 0x%lx/0x%x bytes\n", + __func__, iova, size); + } + + spin_unlock_irqrestore(&priv->pgtablelock, flags); + + return ret; +} + +static size_t exynos_iommu_unmap(struct iommu_domain *domain, + unsigned long iova, size_t size) +{ + struct exynos_iommu_domain *priv = domain->priv; + struct sysmmu_drvdata *data; + unsigned long flags; + unsigned long *ent; + + BUG_ON(priv->pgtable == NULL); + + spin_lock_irqsave(&priv->pgtablelock, flags); + + ent = section_entry(priv->pgtable, iova); + + if (lv1ent_section(ent)) { + BUG_ON(size < SECT_SIZE); + + *ent = 0; + pgtable_flush(ent, ent + 1); + size = SECT_SIZE; + goto done; + } + + if (unlikely(lv1ent_fault(ent))) { + if (size > SECT_SIZE) + size = SECT_SIZE; + goto done; + } + + /* lv1ent_page(sent) == true here */ + + ent = page_entry(ent, iova); + + if (unlikely(lv2ent_fault(ent))) { + size = SPAGE_SIZE; + goto done; + } + + if (lv2ent_small(ent)) { + *ent = 0; + size = SPAGE_SIZE; + priv->lv2entcnt[lv1ent_offset(iova)] += 1; + goto done; + } + + /* lv1ent_large(ent) == true here */ + BUG_ON(size < LPAGE_SIZE); + + memset(ent, 0, sizeof(*ent) * SPAGES_PER_LPAGE); + + size = LPAGE_SIZE; + priv->lv2entcnt[lv1ent_offset(iova)] += SPAGES_PER_LPAGE; +done: + spin_unlock_irqrestore(&priv->pgtablelock, flags); + + spin_lock_irqsave(&priv->lock, flags); + list_for_each_entry(data, &priv->clients, node) + sysmmu_tlb_invalidate_entry(data->dev, iova); + spin_unlock_irqrestore(&priv->lock, flags); + + + return size; +} + +static phys_addr_t exynos_iommu_iova_to_phys(struct iommu_domain *domain, + unsigned long iova) +{ + struct exynos_iommu_domain *priv = domain->priv; + unsigned long *entry; + unsigned long flags; + phys_addr_t phys = 0; + + spin_lock_irqsave(&priv->pgtablelock, flags); + + entry = section_entry(priv->pgtable, iova); + + if (lv1ent_section(entry)) { + phys = section_phys(entry) + section_offs(iova); + } else if (lv1ent_page(entry)) { + entry = page_entry(entry, iova); + + if (lv2ent_large(entry)) + phys = lpage_phys(entry) + lpage_offs(iova); + else if (lv2ent_small(entry)) + phys = spage_phys(entry) + spage_offs(iova); + } + + spin_unlock_irqrestore(&priv->pgtablelock, flags); + + return phys; +} + +static struct iommu_ops exynos_iommu_ops = { + .domain_init = &exynos_iommu_domain_init, + .domain_destroy = &exynos_iommu_domain_destroy, + .attach_dev = &exynos_iommu_attach_device, + .detach_dev = &exynos_iommu_detach_device, + .map = &exynos_iommu_map, + .unmap = &exynos_iommu_unmap, + .iova_to_phys = &exynos_iommu_iova_to_phys, + .pgsize_bitmap = SECT_SIZE | LPAGE_SIZE | SPAGE_SIZE, +}; + +static int __init exynos_iommu_init(void) +{ + int ret; + + ret = platform_driver_register(&exynos_sysmmu_driver); + + if (ret == 0) + bus_set_iommu(&platform_bus_type, &exynos_iommu_ops); + + return ret; +} +subsys_initcall(exynos_iommu_init); diff --git a/drivers/iommu/exynos-iovmm.c b/drivers/iommu/exynos-iovmm.c new file mode 100644 index 0000000..bc78722 --- /dev/null +++ b/drivers/iommu/exynos-iovmm.c @@ -0,0 +1,510 @@ +/* linux/arch/arm/plat-s5p/s5p_iovmm.c + * + * Copyright (c) 2011 Samsung Electronics Co., Ltd. + * http://www.samsung.com + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#ifdef CONFIG_EXYNOS_IOMMU_DEBUG +#define DEBUG +#endif + +#include <linux/slab.h> +#include <linux/scatterlist.h> +#include <linux/device.h> +#include <linux/list.h> +#include <linux/iommu.h> +#include <linux/genalloc.h> +#include <linux/err.h> +#include <linux/spinlock.h> +#include <linux/platform_device.h> +#include <linux/atomic.h> +#include <linux/rculist.h> + +#include <plat/iovmm.h> + +#define IOVA_START 0xC0000000 +#define IOVM_SIZE (SZ_1G - SZ_4K) /* last 4K is for error values */ + +struct s5p_vm_region { + struct list_head node; + dma_addr_t start; + size_t size; +}; + +struct s5p_iovmm { + struct list_head node; /* element of s5p_iovmm_list */ + struct rcu_head rcu; + struct iommu_domain *domain; + struct device *dev; + struct gen_pool *vmm_pool; + struct list_head regions_list; /* list of s5p_vm_region */ + atomic_t activations; + int num_setup; + spinlock_t lock; +}; + +static LIST_HEAD(s5p_iovmm_list); + +static struct s5p_iovmm *find_iovmm(struct device *dev) +{ + struct s5p_iovmm *vmm; + + list_for_each_entry(vmm, &s5p_iovmm_list, node) + if ((vmm->dev == dev) && (vmm->num_setup > 0)) + return vmm; + + return NULL; +} + +static struct s5p_vm_region *find_region(struct s5p_iovmm *vmm, dma_addr_t iova) +{ + struct s5p_vm_region *region; + + list_for_each_entry(region, &vmm->regions_list, node) + if (region->start == iova) + return region; + + return NULL; +} + +int iovmm_setup(struct device *dev) +{ + struct s5p_iovmm *vmm = NULL; + struct list_head *pos; + int ret; + + list_for_each(pos, &s5p_iovmm_list) { + vmm = list_entry(pos, struct s5p_iovmm, node); + if (vmm->dev == dev) { + struct s5p_iovmm *rcu_vmm; + + rcu_vmm = kmalloc(sizeof(*rcu_vmm), GFP_KERNEL); + if (rcu_vmm == NULL) + return -ENOMEM; + + memcpy(rcu_vmm, vmm, sizeof(*vmm)); + rcu_vmm->num_setup++; + list_replace_rcu(&vmm->node, &rcu_vmm->node); + + kfree(vmm); + + return 0; + } + } + + vmm = kzalloc(sizeof(*vmm), GFP_KERNEL); + if (!vmm) { + ret = -ENOMEM; + goto err_setup_alloc; + } + + vmm->vmm_pool = gen_pool_create(PAGE_SHIFT, -1); + if (!vmm->vmm_pool) { + ret = -ENOMEM; + goto err_setup_genalloc; + } + + /* (1GB - 4KB) addr space from 0xC0000000 */ + ret = gen_pool_add(vmm->vmm_pool, IOVA_START, IOVM_SIZE, -1); + if (ret) + goto err_setup_domain; + + vmm->domain = iommu_domain_alloc(&platform_bus_type); + if (!vmm->domain) { + ret = -ENOMEM; + goto err_setup_domain; + } + + vmm->dev = dev; + vmm->num_setup = 1; + + spin_lock_init(&vmm->lock); + + INIT_LIST_HEAD(&vmm->node); + INIT_LIST_HEAD(&vmm->regions_list); + atomic_set(&vmm->activations, 0); + + list_add_rcu(&vmm->node, &s5p_iovmm_list); + + dev_dbg(dev, "IOVMM: Created %#x B IOVMM from %#x.\n", + IOVM_SIZE, IOVA_START); + + return 0; +err_setup_domain: + gen_pool_destroy(vmm->vmm_pool); +err_setup_genalloc: + kfree(vmm); +err_setup_alloc: + dev_dbg(dev, "IOVMM: Failed to create IOVMM (%d)\n", ret); + return ret; +} + +static void iovmm_destroy(struct rcu_head *rcu) +{ + struct s5p_iovmm *vmm = container_of(rcu, struct s5p_iovmm, rcu); + struct list_head *pos, *tmp; + + while (WARN_ON(atomic_dec_return(&vmm->activations) > 0)) + iommu_detach_device(vmm->domain, vmm->dev); + + iommu_domain_free(vmm->domain); + + WARN_ON(!list_empty(&vmm->regions_list)); + + list_for_each_safe(pos, tmp, &vmm->regions_list) { + struct s5p_vm_region *region; + + region = list_entry(pos, struct s5p_vm_region, node); + + /* No need to unmap the region because + * iommu_domain_free() frees the page table */ + gen_pool_free(vmm->vmm_pool, region->start, + region->size); + + kfree(list_entry(pos, struct s5p_vm_region, node)); + } + + gen_pool_destroy(vmm->vmm_pool); + + dev_dbg(vmm->dev, "IOVMM: Removed IOVMM\n"); + + kfree(vmm); +} + +void iovmm_cleanup(struct device *dev) +{ + struct s5p_iovmm *vmm, *n; + + list_for_each_entry_safe(vmm, n, &s5p_iovmm_list, node) { + if (vmm->dev == dev) { + struct s5p_iovmm *rcu_vmm = NULL; + + while (rcu_vmm == NULL) /* should success */ + rcu_vmm = kmalloc(sizeof(*rcu_vmm), GFP_ATOMIC); + + memcpy(rcu_vmm, vmm, sizeof(*vmm)); + rcu_vmm->num_setup--; + list_replace_rcu(&vmm->node, &rcu_vmm->node); + + kfree(vmm); + + if (rcu_vmm->num_setup == 0) { + list_del_rcu(&rcu_vmm->node); + call_rcu(&rcu_vmm->rcu, iovmm_destroy); + } + + return; + } + } + + WARN(true, "%s: No IOVMM exist for %s\n", __func__, dev_name(dev)); +} + +int iovmm_activate(struct device *dev) +{ + struct s5p_iovmm *vmm; + int ret = 0; + + rcu_read_lock(); + + vmm = find_iovmm(dev); + if (WARN_ON(!vmm)) { + rcu_read_unlock(); + return -EINVAL; + } + + ret = iommu_attach_device(vmm->domain, vmm->dev); + if (!ret) + atomic_inc(&vmm->activations); + + rcu_read_unlock(); + + return ret; +} + +void iovmm_deactivate(struct device *dev) +{ + struct s5p_iovmm *vmm; + + rcu_read_lock(); + + vmm = find_iovmm(dev); + if (WARN_ON(!vmm)) { + rcu_read_unlock(); + return; + } + + iommu_detach_device(vmm->domain, vmm->dev); + + atomic_add_unless(&vmm->activations, -1, 0); + + rcu_read_unlock(); +} + +dma_addr_t iovmm_map(struct device *dev, struct scatterlist *sg, off_t offset, + size_t size) +{ + off_t start_off; + dma_addr_t addr, start = 0; + size_t mapped_size = 0; + struct s5p_vm_region *region; + struct s5p_iovmm *vmm; + int order; + unsigned long flags; +#ifdef CONFIG_EXYNOS_IOVMM_ALIGN64K + size_t iova_size = 0; +#endif + + rcu_read_lock(); + + vmm = find_iovmm(dev); + if (WARN_ON(!vmm)) + goto err_map_nomem; + + for (; sg_dma_len(sg) < offset; sg = sg_next(sg)) + offset -= sg_dma_len(sg); + + start_off = offset_in_page(sg_phys(sg) + offset); + size = PAGE_ALIGN(size + start_off); + + order = __fls(min_t(size_t, size, SZ_1M)); + + region = kmalloc(sizeof(*region), GFP_KERNEL); + if (!region) + goto err_map_nomem; + + INIT_LIST_HEAD(®ion->node); + + spin_lock_irqsave(&vmm->lock, flags); + +#ifdef CONFIG_EXYNOS_IOVMM_ALIGN64K + iova_size = ALIGN(size, SZ_64K); + start = (dma_addr_t)gen_pool_alloc_aligned(vmm->vmm_pool, iova_size, + order); +#else + start = (dma_addr_t)gen_pool_alloc_aligned(vmm->vmm_pool, size, order); +#endif + if (!start) + goto err_map_nomem_lock; + + addr = start; + do { + phys_addr_t phys; + size_t len; + + phys = sg_phys(sg); + len = sg_dma_len(sg); + + if (offset > 0) { + len -= offset; + phys += offset; + offset = 0; + } + + if (offset_in_page(phys)) { + len += offset_in_page(phys); + phys = round_down(phys, PAGE_SIZE); + } + + len = PAGE_ALIGN(len); + + if (len > (size - mapped_size)) + len = size - mapped_size; + + if (iommu_map(vmm->domain, addr, phys, len, 0)) + break; + + addr += len; + mapped_size += len; + } while ((sg = sg_next(sg)) && (mapped_size < size)); + + BUG_ON(mapped_size > size); + + if (mapped_size < size) + goto err_map_map; + +#ifdef CONFIG_EXYNOS_IOVMM_ALIGN64K + if (iova_size != size) { + addr = start + size; + size = iova_size; + + for (; addr < start + size; addr += PAGE_SIZE) { + if (iommu_map(vmm->domain, addr, + page_to_phys(ZERO_PAGE(0)), PAGE_SIZE, 0)) + goto err_map_map; + + mapped_size += PAGE_SIZE; + } + } +#endif + + region->start = start + start_off; + region->size = size; + + list_add(®ion->node, &vmm->regions_list); + + spin_unlock_irqrestore(&vmm->lock, flags); + + dev_dbg(dev, "IOVMM: Allocated VM region @ %#x/%#X bytes.\n", + region->start, region->size); + + rcu_read_unlock(); + + return region->start; + +err_map_map: + iommu_unmap(vmm->domain, start, mapped_size); + gen_pool_free(vmm->vmm_pool, start, size); +err_map_nomem_lock: + spin_unlock_irqrestore(&vmm->lock, flags); + kfree(region); +err_map_nomem: + dev_dbg(dev, "IOVMM: Failed to allocated VM region for %#x bytes.\n", + size); + rcu_read_unlock(); + + return (dma_addr_t)0; +} + +void iovmm_unmap(struct device *dev, dma_addr_t iova) +{ + struct s5p_vm_region *region; + struct s5p_iovmm *vmm; + unsigned long flags; + size_t unmapped_size; + + rcu_read_lock(); + + vmm = find_iovmm(dev); + + if (WARN_ON(!vmm)) { + rcu_read_unlock(); + return; + } + + spin_lock_irqsave(&vmm->lock, flags); + + region = find_region(vmm, iova); + if (WARN_ON(!region)) + goto err_region_not_found; + + region->start = round_down(region->start, PAGE_SIZE); + + gen_pool_free(vmm->vmm_pool, region->start, region->size); + list_del(®ion->node); + + unmapped_size = iommu_unmap(vmm->domain, region->start, region->size); + + WARN_ON(unmapped_size != region->size); + dev_dbg(dev, "IOVMM: Unmapped %#x bytes from %#x.\n", + unmapped_size, region->start); + + kfree(region); +err_region_not_found: + spin_unlock_irqrestore(&vmm->lock, flags); + + rcu_read_unlock(); +} + +int iovmm_map_oto(struct device *dev, phys_addr_t phys, size_t size) +{ + struct s5p_vm_region *region; + struct s5p_iovmm *vmm; + unsigned long flags; + int ret; + + rcu_read_lock(); + + vmm = find_iovmm(dev); + if (WARN_ON(!vmm)) { + ret = -EINVAL; + goto err_map_nomem; + } + + region = kmalloc(sizeof(*region), GFP_KERNEL); + if (!region) { + ret = -ENOMEM; + goto err_map_nomem; + } + + if (WARN_ON((phys + size) >= IOVA_START)) { + dev_err(dev, + "Unable to create one to one mapping for %#x @ %#x\n", + size, phys); + ret = -EINVAL; + goto err_out_of_memory; + } + + if (WARN_ON(phys & ~PAGE_MASK)) + phys = round_down(phys, PAGE_SIZE); + + spin_lock_irqsave(&vmm->lock, flags); + + ret = iommu_map(vmm->domain, (dma_addr_t)phys, phys, size, 0); + if (ret < 0) + goto err_map_failed; + + region->start = (dma_addr_t)phys; + region->size = size; + INIT_LIST_HEAD(®ion->node); + + list_add(®ion->node, &vmm->regions_list); + + spin_unlock_irqrestore(&vmm->lock, flags); + + rcu_read_unlock(); + + return 0; + +err_map_failed: + spin_unlock_irqrestore(&vmm->lock, flags); +err_out_of_memory: + kfree(region); +err_map_nomem: + rcu_read_unlock(); + + return ret; +} + +void iovmm_unmap_oto(struct device *dev, phys_addr_t phys) +{ + struct s5p_vm_region *region; + struct s5p_iovmm *vmm; + unsigned long flags; + size_t unmapped_size; + + rcu_read_lock(); + + vmm = find_iovmm(dev); + if (WARN_ON(!vmm)) { + rcu_read_unlock(); + return; + } + + if (WARN_ON(phys & ~PAGE_MASK)) + phys = round_down(phys, PAGE_SIZE); + + spin_lock_irqsave(&vmm->lock, flags); + + region = find_region(vmm, (dma_addr_t)phys); + if (WARN_ON(!region)) + goto err_region_not_found; + + list_del(®ion->node); + + unmapped_size = iommu_unmap(vmm->domain, region->start, region->size); + + WARN_ON(unmapped_size != region->size); + dev_dbg(dev, "IOVMM: Unmapped %#x bytes from %#x.\n", + unmapped_size, region->start); + + kfree(region); +err_region_not_found: + spin_unlock_irqrestore(&vmm->lock, flags); + + rcu_read_unlock(); +} diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c new file mode 100644 index 0000000..2198b2d --- /dev/null +++ b/drivers/iommu/iommu.c @@ -0,0 +1,343 @@ +/* + * Copyright (C) 2007-2008 Advanced Micro Devices, Inc. + * Author: Joerg Roedel <joerg.roedel@amd.com> + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 as published + * by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#define pr_fmt(fmt) "%s: " fmt, __func__ + +#include <linux/device.h> +#include <linux/kernel.h> +#include <linux/bug.h> +#include <linux/types.h> +#include <linux/module.h> +#include <linux/slab.h> +#include <linux/errno.h> +#include <linux/iommu.h> + +static ssize_t show_iommu_group(struct device *dev, + struct device_attribute *attr, char *buf) +{ + unsigned int groupid; + + if (iommu_device_group(dev, &groupid)) + return 0; + + return sprintf(buf, "%u", groupid); +} +static DEVICE_ATTR(iommu_group, S_IRUGO, show_iommu_group, NULL); + +static int add_iommu_group(struct device *dev, void *data) +{ + unsigned int groupid; + + if (iommu_device_group(dev, &groupid) == 0) + return device_create_file(dev, &dev_attr_iommu_group); + + return 0; +} + +static int remove_iommu_group(struct device *dev) +{ + unsigned int groupid; + + if (iommu_device_group(dev, &groupid) == 0) + device_remove_file(dev, &dev_attr_iommu_group); + + return 0; +} + +static int iommu_device_notifier(struct notifier_block *nb, + unsigned long action, void *data) +{ + struct device *dev = data; + + if (action == BUS_NOTIFY_ADD_DEVICE) + return add_iommu_group(dev, NULL); + else if (action == BUS_NOTIFY_DEL_DEVICE) + return remove_iommu_group(dev); + + return 0; +} + +static struct notifier_block iommu_device_nb = { + .notifier_call = iommu_device_notifier, +}; + +static void iommu_bus_init(struct bus_type *bus, struct iommu_ops *ops) +{ + bus_register_notifier(bus, &iommu_device_nb); + bus_for_each_dev(bus, NULL, NULL, add_iommu_group); +} + +/** + * bus_set_iommu - set iommu-callbacks for the bus + * @bus: bus. + * @ops: the callbacks provided by the iommu-driver + * + * This function is called by an iommu driver to set the iommu methods + * used for a particular bus. Drivers for devices on that bus can use + * the iommu-api after these ops are registered. + * This special function is needed because IOMMUs are usually devices on + * the bus itself, so the iommu drivers are not initialized when the bus + * is set up. With this function the iommu-driver can set the iommu-ops + * afterwards. + */ +int bus_set_iommu(struct bus_type *bus, struct iommu_ops *ops) +{ + if (bus->iommu_ops != NULL) + return -EBUSY; + + bus->iommu_ops = ops; + + /* Do IOMMU specific setup for this bus-type */ + iommu_bus_init(bus, ops); + + return 0; +} +EXPORT_SYMBOL_GPL(bus_set_iommu); + +bool iommu_present(struct bus_type *bus) +{ + return bus->iommu_ops != NULL; +} +EXPORT_SYMBOL_GPL(iommu_present); + +/** + * iommu_set_fault_handler() - set a fault handler for an iommu domain + * @domain: iommu domain + * @handler: fault handler + * + * This function should be used by IOMMU users which want to be notified + * whenever an IOMMU fault happens. + * + * The fault handler itself should return 0 on success, and an appropriate + * error code otherwise. + */ +void iommu_set_fault_handler(struct iommu_domain *domain, + iommu_fault_handler_t handler) +{ + BUG_ON(!domain); + + domain->handler = handler; +} +EXPORT_SYMBOL_GPL(iommu_set_fault_handler); + +struct iommu_domain *iommu_domain_alloc(struct bus_type *bus) +{ + struct iommu_domain *domain; + int ret; + + if (bus == NULL || bus->iommu_ops == NULL) + return NULL; + + domain = kzalloc(sizeof(*domain), GFP_KERNEL); + if (!domain) + return NULL; + + domain->ops = bus->iommu_ops; + + ret = domain->ops->domain_init(domain); + if (ret) + goto out_free; + + return domain; + +out_free: + kfree(domain); + + return NULL; +} +EXPORT_SYMBOL_GPL(iommu_domain_alloc); + +void iommu_domain_free(struct iommu_domain *domain) +{ + if (likely(domain->ops->domain_destroy != NULL)) + domain->ops->domain_destroy(domain); + + kfree(domain); +} +EXPORT_SYMBOL_GPL(iommu_domain_free); + +int iommu_attach_device(struct iommu_domain *domain, struct device *dev) +{ + if (unlikely(domain->ops->attach_dev == NULL)) + return -ENODEV; + + return domain->ops->attach_dev(domain, dev); +} +EXPORT_SYMBOL_GPL(iommu_attach_device); + +void iommu_detach_device(struct iommu_domain *domain, struct device *dev) +{ + if (unlikely(domain->ops->detach_dev == NULL)) + return; + + domain->ops->detach_dev(domain, dev); +} +EXPORT_SYMBOL_GPL(iommu_detach_device); + +phys_addr_t iommu_iova_to_phys(struct iommu_domain *domain, + unsigned long iova) +{ + if (unlikely(domain->ops->iova_to_phys == NULL)) + return 0; + + return domain->ops->iova_to_phys(domain, iova); +} +EXPORT_SYMBOL_GPL(iommu_iova_to_phys); + +int iommu_domain_has_cap(struct iommu_domain *domain, + unsigned long cap) +{ + if (unlikely(domain->ops->domain_has_cap == NULL)) + return 0; + + return domain->ops->domain_has_cap(domain, cap); +} +EXPORT_SYMBOL_GPL(iommu_domain_has_cap); + +int iommu_map(struct iommu_domain *domain, unsigned long iova, + phys_addr_t paddr, size_t size, int prot) +{ + unsigned long orig_iova = iova; + unsigned int min_pagesz; + size_t orig_size = size; + int ret = 0; + + if (unlikely(domain->ops->map == NULL)) + return -ENODEV; + + /* find out the minimum page size supported */ + min_pagesz = 1 << __ffs(domain->ops->pgsize_bitmap); + + /* + * both the virtual address and the physical one, as well as + * the size of the mapping, must be aligned (at least) to the + * size of the smallest page supported by the hardware + */ + if (!IS_ALIGNED(iova | paddr | size, min_pagesz)) { + pr_err("unaligned: iova 0x%lx pa 0x%lx size 0x%lx min_pagesz " + "0x%x\n", iova, (unsigned long)paddr, + (unsigned long)size, min_pagesz); + return -EINVAL; + } + + pr_debug("map: iova 0x%lx pa 0x%lx size 0x%lx\n", iova, + (unsigned long)paddr, (unsigned long)size); + + while (size) { + unsigned long pgsize, addr_merge = iova | paddr; + unsigned int pgsize_idx; + + /* Max page size that still fits into 'size' */ + pgsize_idx = __fls(size); + + /* need to consider alignment requirements ? */ + if (likely(addr_merge)) { + /* Max page size allowed by both iova and paddr */ + unsigned int align_pgsize_idx = __ffs(addr_merge); + + pgsize_idx = min(pgsize_idx, align_pgsize_idx); + } + + /* build a mask of acceptable page sizes */ + pgsize = (1UL << (pgsize_idx + 1)) - 1; + + /* throw away page sizes not supported by the hardware */ + pgsize &= domain->ops->pgsize_bitmap; + + /* make sure we're still sane */ + BUG_ON(!pgsize); + + /* pick the biggest page */ + pgsize_idx = __fls(pgsize); + pgsize = 1UL << pgsize_idx; + + pr_debug("mapping: iova 0x%lx pa 0x%lx pgsize %lu\n", iova, + (unsigned long)paddr, pgsize); + + ret = domain->ops->map(domain, iova, paddr, pgsize, prot); + if (ret) + break; + + iova += pgsize; + paddr += pgsize; + size -= pgsize; + } + + /* unroll mapping in case something went wrong */ + if (ret) + iommu_unmap(domain, orig_iova, orig_size - size); + + return ret; +} +EXPORT_SYMBOL_GPL(iommu_map); + +size_t iommu_unmap(struct iommu_domain *domain, unsigned long iova, size_t size) +{ + size_t unmapped_page, unmapped = 0; + unsigned int min_pagesz; + + if (unlikely(domain->ops->unmap == NULL)) + return -ENODEV; + + /* find out the minimum page size supported */ + min_pagesz = 1 << __ffs(domain->ops->pgsize_bitmap); + + /* + * The virtual address, as well as the size of the mapping, must be + * aligned (at least) to the size of the smallest page supported + * by the hardware + */ + if (!IS_ALIGNED(iova | size, min_pagesz)) { + pr_err("unaligned: iova 0x%lx size 0x%lx min_pagesz 0x%x\n", + iova, (unsigned long)size, min_pagesz); + return -EINVAL; + } + + pr_debug("unmap this: iova 0x%lx size 0x%lx\n", iova, + (unsigned long)size); + + /* + * Keep iterating until we either unmap 'size' bytes (or more) + * or we hit an area that isn't mapped. + */ + while (unmapped < size) { + size_t left = size - unmapped; + + unmapped_page = domain->ops->unmap(domain, iova, left); + if (!unmapped_page) + break; + + pr_debug("unmapped: iova 0x%lx size %lx\n", iova, + (unsigned long)unmapped_page); + + iova += unmapped_page; + unmapped += unmapped_page; + } + + return unmapped; +} +EXPORT_SYMBOL_GPL(iommu_unmap); + +int iommu_device_group(struct device *dev, unsigned int *groupid) +{ + if (iommu_present(dev->bus) && dev->bus->iommu_ops->device_group) + return dev->bus->iommu_ops->device_group(dev, groupid); + + return -ENODEV; +} +EXPORT_SYMBOL_GPL(iommu_device_group); |