Thomas Gleixner | d2912cb | 2019-06-04 10:11:33 +0200 | [diff] [blame^] | 1 | // SPDX-License-Identifier: GPL-2.0-only |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 2 | /* |
| 3 | * omap iommu: tlb and pagetable primitives |
| 4 | * |
Hiroshi DOYU | c127c7d | 2010-02-15 10:03:32 -0800 | [diff] [blame] | 5 | * Copyright (C) 2008-2010 Nokia Corporation |
Suman Anna | 9d5018d | 2017-09-05 17:56:18 -0500 | [diff] [blame] | 6 | * Copyright (C) 2013-2017 Texas Instruments Incorporated - http://www.ti.com/ |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 7 | * |
| 8 | * Written by Hiroshi DOYU <Hiroshi.DOYU@nokia.com>, |
| 9 | * Paul Mundt and Toshihiro Kobayashi |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 10 | */ |
| 11 | |
Josue Albarran | bfee0cf | 2017-07-28 15:49:14 -0500 | [diff] [blame] | 12 | #include <linux/dma-mapping.h> |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 13 | #include <linux/err.h> |
Tejun Heo | 5a0e3ad | 2010-03-24 17:04:11 +0900 | [diff] [blame] | 14 | #include <linux/slab.h> |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 15 | #include <linux/interrupt.h> |
| 16 | #include <linux/ioport.h> |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 17 | #include <linux/platform_device.h> |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 18 | #include <linux/iommu.h> |
Tony Lindgren | c8d35c8 | 2012-11-02 12:24:03 -0700 | [diff] [blame] | 19 | #include <linux/omap-iommu.h> |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 20 | #include <linux/mutex.h> |
| 21 | #include <linux/spinlock.h> |
Tony Lindgren | ed1c7de | 2012-11-02 12:24:06 -0700 | [diff] [blame] | 22 | #include <linux/io.h> |
Omar Ramirez Luna | ebf7cda | 2012-11-19 19:05:51 -0600 | [diff] [blame] | 23 | #include <linux/pm_runtime.h> |
Florian Vaussard | 3c92748 | 2014-02-28 14:42:36 -0600 | [diff] [blame] | 24 | #include <linux/of.h> |
| 25 | #include <linux/of_iommu.h> |
| 26 | #include <linux/of_irq.h> |
Suman Anna | 7d68277 | 2014-09-04 17:27:30 -0500 | [diff] [blame] | 27 | #include <linux/of_platform.h> |
Suman Anna | 3ca9299 | 2015-10-02 18:02:44 -0500 | [diff] [blame] | 28 | #include <linux/regmap.h> |
| 29 | #include <linux/mfd/syscon.h> |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 30 | |
Tony Lindgren | 2ab7c84 | 2012-11-02 12:24:14 -0700 | [diff] [blame] | 31 | #include <linux/platform_data/iommu-omap.h> |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 32 | |
Ido Yariv | 2f7702a | 2012-11-02 12:24:00 -0700 | [diff] [blame] | 33 | #include "omap-iopgtable.h" |
Tony Lindgren | ed1c7de | 2012-11-02 12:24:06 -0700 | [diff] [blame] | 34 | #include "omap-iommu.h" |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 35 | |
Joerg Roedel | 01611fe | 2017-04-12 00:21:30 -0500 | [diff] [blame] | 36 | static const struct iommu_ops omap_iommu_ops; |
| 37 | |
Suman Anna | 5acc97d | 2014-03-17 20:31:34 -0500 | [diff] [blame] | 38 | #define to_iommu(dev) \ |
| 39 | ((struct omap_iommu *)platform_get_drvdata(to_platform_device(dev))) |
| 40 | |
Ohad Ben-Cohen | 66bc8cf | 2011-11-10 11:32:27 +0200 | [diff] [blame] | 41 | /* bitmap of the page sizes currently supported */ |
| 42 | #define OMAP_IOMMU_PGSIZES (SZ_4K | SZ_64K | SZ_1M | SZ_16M) |
| 43 | |
Ido Yariv | 7bd9e25 | 2012-11-02 12:24:09 -0700 | [diff] [blame] | 44 | #define MMU_LOCK_BASE_SHIFT 10 |
| 45 | #define MMU_LOCK_BASE_MASK (0x1f << MMU_LOCK_BASE_SHIFT) |
| 46 | #define MMU_LOCK_BASE(x) \ |
| 47 | ((x & MMU_LOCK_BASE_MASK) >> MMU_LOCK_BASE_SHIFT) |
| 48 | |
| 49 | #define MMU_LOCK_VICT_SHIFT 4 |
| 50 | #define MMU_LOCK_VICT_MASK (0x1f << MMU_LOCK_VICT_SHIFT) |
| 51 | #define MMU_LOCK_VICT(x) \ |
| 52 | ((x & MMU_LOCK_VICT_MASK) >> MMU_LOCK_VICT_SHIFT) |
| 53 | |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 54 | static struct platform_driver omap_iommu_driver; |
| 55 | static struct kmem_cache *iopte_cachep; |
| 56 | |
| 57 | /** |
Joerg Roedel | 8cf851e | 2015-03-26 13:43:09 +0100 | [diff] [blame] | 58 | * to_omap_domain - Get struct omap_iommu_domain from generic iommu_domain |
| 59 | * @dom: generic iommu domain handle |
| 60 | **/ |
| 61 | static struct omap_iommu_domain *to_omap_domain(struct iommu_domain *dom) |
| 62 | { |
| 63 | return container_of(dom, struct omap_iommu_domain, domain); |
| 64 | } |
| 65 | |
| 66 | /** |
Ohad Ben-Cohen | 6c32df4 | 2011-08-17 22:57:56 +0300 | [diff] [blame] | 67 | * omap_iommu_save_ctx - Save registers for pm off-mode support |
Ohad Ben-Cohen | fabdbca | 2011-10-11 00:18:33 +0200 | [diff] [blame] | 68 | * @dev: client device |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 69 | **/ |
Ohad Ben-Cohen | fabdbca | 2011-10-11 00:18:33 +0200 | [diff] [blame] | 70 | void omap_iommu_save_ctx(struct device *dev) |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 71 | { |
Suman Anna | 9d5018d | 2017-09-05 17:56:18 -0500 | [diff] [blame] | 72 | struct omap_iommu_arch_data *arch_data = dev->archdata.iommu; |
| 73 | struct omap_iommu *obj; |
| 74 | u32 *p; |
Suman Anna | bd4396f | 2014-10-22 17:22:27 -0500 | [diff] [blame] | 75 | int i; |
Ohad Ben-Cohen | fabdbca | 2011-10-11 00:18:33 +0200 | [diff] [blame] | 76 | |
Suman Anna | 9d5018d | 2017-09-05 17:56:18 -0500 | [diff] [blame] | 77 | if (!arch_data) |
| 78 | return; |
| 79 | |
| 80 | while (arch_data->iommu_dev) { |
| 81 | obj = arch_data->iommu_dev; |
| 82 | p = obj->ctx; |
| 83 | for (i = 0; i < (MMU_REG_SIZE / sizeof(u32)); i++) { |
| 84 | p[i] = iommu_read_reg(obj, i * sizeof(u32)); |
| 85 | dev_dbg(obj->dev, "%s\t[%02d] %08x\n", __func__, i, |
| 86 | p[i]); |
| 87 | } |
| 88 | arch_data++; |
Suman Anna | bd4396f | 2014-10-22 17:22:27 -0500 | [diff] [blame] | 89 | } |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 90 | } |
Ohad Ben-Cohen | 6c32df4 | 2011-08-17 22:57:56 +0300 | [diff] [blame] | 91 | EXPORT_SYMBOL_GPL(omap_iommu_save_ctx); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 92 | |
| 93 | /** |
Ohad Ben-Cohen | 6c32df4 | 2011-08-17 22:57:56 +0300 | [diff] [blame] | 94 | * omap_iommu_restore_ctx - Restore registers for pm off-mode support |
Ohad Ben-Cohen | fabdbca | 2011-10-11 00:18:33 +0200 | [diff] [blame] | 95 | * @dev: client device |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 96 | **/ |
Ohad Ben-Cohen | fabdbca | 2011-10-11 00:18:33 +0200 | [diff] [blame] | 97 | void omap_iommu_restore_ctx(struct device *dev) |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 98 | { |
Suman Anna | 9d5018d | 2017-09-05 17:56:18 -0500 | [diff] [blame] | 99 | struct omap_iommu_arch_data *arch_data = dev->archdata.iommu; |
| 100 | struct omap_iommu *obj; |
| 101 | u32 *p; |
Suman Anna | bd4396f | 2014-10-22 17:22:27 -0500 | [diff] [blame] | 102 | int i; |
Ohad Ben-Cohen | fabdbca | 2011-10-11 00:18:33 +0200 | [diff] [blame] | 103 | |
Suman Anna | 9d5018d | 2017-09-05 17:56:18 -0500 | [diff] [blame] | 104 | if (!arch_data) |
| 105 | return; |
| 106 | |
| 107 | while (arch_data->iommu_dev) { |
| 108 | obj = arch_data->iommu_dev; |
| 109 | p = obj->ctx; |
| 110 | for (i = 0; i < (MMU_REG_SIZE / sizeof(u32)); i++) { |
| 111 | iommu_write_reg(obj, p[i], i * sizeof(u32)); |
| 112 | dev_dbg(obj->dev, "%s\t[%02d] %08x\n", __func__, i, |
| 113 | p[i]); |
| 114 | } |
| 115 | arch_data++; |
Suman Anna | bd4396f | 2014-10-22 17:22:27 -0500 | [diff] [blame] | 116 | } |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 117 | } |
Ohad Ben-Cohen | 6c32df4 | 2011-08-17 22:57:56 +0300 | [diff] [blame] | 118 | EXPORT_SYMBOL_GPL(omap_iommu_restore_ctx); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 119 | |
Suman Anna | 3ca9299 | 2015-10-02 18:02:44 -0500 | [diff] [blame] | 120 | static void dra7_cfg_dspsys_mmu(struct omap_iommu *obj, bool enable) |
| 121 | { |
| 122 | u32 val, mask; |
| 123 | |
| 124 | if (!obj->syscfg) |
| 125 | return; |
| 126 | |
| 127 | mask = (1 << (obj->id * DSP_SYS_MMU_CONFIG_EN_SHIFT)); |
| 128 | val = enable ? mask : 0; |
| 129 | regmap_update_bits(obj->syscfg, DSP_SYS_MMU_CONFIG, mask, val); |
| 130 | } |
| 131 | |
Suman Anna | bd4396f | 2014-10-22 17:22:27 -0500 | [diff] [blame] | 132 | static void __iommu_set_twl(struct omap_iommu *obj, bool on) |
| 133 | { |
| 134 | u32 l = iommu_read_reg(obj, MMU_CNTL); |
| 135 | |
| 136 | if (on) |
| 137 | iommu_write_reg(obj, MMU_IRQ_TWL_MASK, MMU_IRQENABLE); |
| 138 | else |
| 139 | iommu_write_reg(obj, MMU_IRQ_TLB_MISS_MASK, MMU_IRQENABLE); |
| 140 | |
| 141 | l &= ~MMU_CNTL_MASK; |
| 142 | if (on) |
| 143 | l |= (MMU_CNTL_MMU_EN | MMU_CNTL_TWL_EN); |
| 144 | else |
| 145 | l |= (MMU_CNTL_MMU_EN); |
| 146 | |
| 147 | iommu_write_reg(obj, l, MMU_CNTL); |
| 148 | } |
| 149 | |
| 150 | static int omap2_iommu_enable(struct omap_iommu *obj) |
| 151 | { |
| 152 | u32 l, pa; |
| 153 | |
| 154 | if (!obj->iopgd || !IS_ALIGNED((u32)obj->iopgd, SZ_16K)) |
| 155 | return -EINVAL; |
| 156 | |
| 157 | pa = virt_to_phys(obj->iopgd); |
| 158 | if (!IS_ALIGNED(pa, SZ_16K)) |
| 159 | return -EINVAL; |
| 160 | |
| 161 | l = iommu_read_reg(obj, MMU_REVISION); |
| 162 | dev_info(obj->dev, "%s: version %d.%d\n", obj->name, |
| 163 | (l >> 4) & 0xf, l & 0xf); |
| 164 | |
| 165 | iommu_write_reg(obj, pa, MMU_TTB); |
| 166 | |
Suman Anna | 3ca9299 | 2015-10-02 18:02:44 -0500 | [diff] [blame] | 167 | dra7_cfg_dspsys_mmu(obj, true); |
| 168 | |
Suman Anna | bd4396f | 2014-10-22 17:22:27 -0500 | [diff] [blame] | 169 | if (obj->has_bus_err_back) |
| 170 | iommu_write_reg(obj, MMU_GP_REG_BUS_ERR_BACK_EN, MMU_GP_REG); |
| 171 | |
| 172 | __iommu_set_twl(obj, true); |
| 173 | |
| 174 | return 0; |
| 175 | } |
| 176 | |
| 177 | static void omap2_iommu_disable(struct omap_iommu *obj) |
| 178 | { |
| 179 | u32 l = iommu_read_reg(obj, MMU_CNTL); |
| 180 | |
| 181 | l &= ~MMU_CNTL_MASK; |
| 182 | iommu_write_reg(obj, l, MMU_CNTL); |
Suman Anna | 3ca9299 | 2015-10-02 18:02:44 -0500 | [diff] [blame] | 183 | dra7_cfg_dspsys_mmu(obj, false); |
Suman Anna | bd4396f | 2014-10-22 17:22:27 -0500 | [diff] [blame] | 184 | |
| 185 | dev_dbg(obj->dev, "%s is shutting down\n", obj->name); |
| 186 | } |
| 187 | |
Ohad Ben-Cohen | 6c32df4 | 2011-08-17 22:57:56 +0300 | [diff] [blame] | 188 | static int iommu_enable(struct omap_iommu *obj) |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 189 | { |
| 190 | int err; |
Omar Ramirez Luna | 72b15b6 | 2012-11-19 19:05:50 -0600 | [diff] [blame] | 191 | struct platform_device *pdev = to_platform_device(obj->dev); |
Kiran Padwal | 99cb9ae | 2014-10-30 11:59:47 +0530 | [diff] [blame] | 192 | struct iommu_platform_data *pdata = dev_get_platdata(&pdev->dev); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 193 | |
Florian Vaussard | 90e569c | 2014-02-28 14:42:34 -0600 | [diff] [blame] | 194 | if (pdata && pdata->deassert_reset) { |
Omar Ramirez Luna | 72b15b6 | 2012-11-19 19:05:50 -0600 | [diff] [blame] | 195 | err = pdata->deassert_reset(pdev, pdata->reset_name); |
| 196 | if (err) { |
| 197 | dev_err(obj->dev, "deassert_reset failed: %d\n", err); |
| 198 | return err; |
| 199 | } |
| 200 | } |
| 201 | |
Omar Ramirez Luna | ebf7cda | 2012-11-19 19:05:51 -0600 | [diff] [blame] | 202 | pm_runtime_get_sync(obj->dev); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 203 | |
Suman Anna | bd4396f | 2014-10-22 17:22:27 -0500 | [diff] [blame] | 204 | err = omap2_iommu_enable(obj); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 205 | |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 206 | return err; |
| 207 | } |
| 208 | |
Ohad Ben-Cohen | 6c32df4 | 2011-08-17 22:57:56 +0300 | [diff] [blame] | 209 | static void iommu_disable(struct omap_iommu *obj) |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 210 | { |
Omar Ramirez Luna | 72b15b6 | 2012-11-19 19:05:50 -0600 | [diff] [blame] | 211 | struct platform_device *pdev = to_platform_device(obj->dev); |
Kiran Padwal | 99cb9ae | 2014-10-30 11:59:47 +0530 | [diff] [blame] | 212 | struct iommu_platform_data *pdata = dev_get_platdata(&pdev->dev); |
Omar Ramirez Luna | 72b15b6 | 2012-11-19 19:05:50 -0600 | [diff] [blame] | 213 | |
Suman Anna | bd4396f | 2014-10-22 17:22:27 -0500 | [diff] [blame] | 214 | omap2_iommu_disable(obj); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 215 | |
Omar Ramirez Luna | ebf7cda | 2012-11-19 19:05:51 -0600 | [diff] [blame] | 216 | pm_runtime_put_sync(obj->dev); |
Omar Ramirez Luna | 72b15b6 | 2012-11-19 19:05:50 -0600 | [diff] [blame] | 217 | |
Florian Vaussard | 90e569c | 2014-02-28 14:42:34 -0600 | [diff] [blame] | 218 | if (pdata && pdata->assert_reset) |
Omar Ramirez Luna | 72b15b6 | 2012-11-19 19:05:50 -0600 | [diff] [blame] | 219 | pdata->assert_reset(pdev, pdata->reset_name); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 220 | } |
| 221 | |
| 222 | /* |
| 223 | * TLB operations |
| 224 | */ |
Ohad Ben-Cohen | e1f2381 | 2011-08-16 14:58:14 +0300 | [diff] [blame] | 225 | static u32 iotlb_cr_to_virt(struct cr_regs *cr) |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 226 | { |
Suman Anna | bd4396f | 2014-10-22 17:22:27 -0500 | [diff] [blame] | 227 | u32 page_size = cr->cam & MMU_CAM_PGSZ_MASK; |
| 228 | u32 mask = get_cam_va_mask(cr->cam & page_size); |
| 229 | |
| 230 | return cr->cam & mask; |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 231 | } |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 232 | |
| 233 | static u32 get_iopte_attr(struct iotlb_entry *e) |
| 234 | { |
Suman Anna | bd4396f | 2014-10-22 17:22:27 -0500 | [diff] [blame] | 235 | u32 attr; |
| 236 | |
| 237 | attr = e->mixed << 5; |
| 238 | attr |= e->endian; |
| 239 | attr |= e->elsz >> 3; |
| 240 | attr <<= (((e->pgsz == MMU_CAM_PGSZ_4K) || |
| 241 | (e->pgsz == MMU_CAM_PGSZ_64K)) ? 0 : 6); |
| 242 | return attr; |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 243 | } |
| 244 | |
Ohad Ben-Cohen | 6c32df4 | 2011-08-17 22:57:56 +0300 | [diff] [blame] | 245 | static u32 iommu_report_fault(struct omap_iommu *obj, u32 *da) |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 246 | { |
Suman Anna | bd4396f | 2014-10-22 17:22:27 -0500 | [diff] [blame] | 247 | u32 status, fault_addr; |
| 248 | |
| 249 | status = iommu_read_reg(obj, MMU_IRQSTATUS); |
| 250 | status &= MMU_IRQ_MASK; |
| 251 | if (!status) { |
| 252 | *da = 0; |
| 253 | return 0; |
| 254 | } |
| 255 | |
| 256 | fault_addr = iommu_read_reg(obj, MMU_FAULT_AD); |
| 257 | *da = fault_addr; |
| 258 | |
| 259 | iommu_write_reg(obj, status, MMU_IRQSTATUS); |
| 260 | |
| 261 | return status; |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 262 | } |
| 263 | |
Suman Anna | 69c2c19 | 2015-07-20 17:33:25 -0500 | [diff] [blame] | 264 | void iotlb_lock_get(struct omap_iommu *obj, struct iotlb_lock *l) |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 265 | { |
| 266 | u32 val; |
| 267 | |
| 268 | val = iommu_read_reg(obj, MMU_LOCK); |
| 269 | |
| 270 | l->base = MMU_LOCK_BASE(val); |
| 271 | l->vict = MMU_LOCK_VICT(val); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 272 | } |
| 273 | |
Suman Anna | 69c2c19 | 2015-07-20 17:33:25 -0500 | [diff] [blame] | 274 | void iotlb_lock_set(struct omap_iommu *obj, struct iotlb_lock *l) |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 275 | { |
| 276 | u32 val; |
| 277 | |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 278 | val = (l->base << MMU_LOCK_BASE_SHIFT); |
| 279 | val |= (l->vict << MMU_LOCK_VICT_SHIFT); |
| 280 | |
| 281 | iommu_write_reg(obj, val, MMU_LOCK); |
| 282 | } |
| 283 | |
Ohad Ben-Cohen | 6c32df4 | 2011-08-17 22:57:56 +0300 | [diff] [blame] | 284 | static void iotlb_read_cr(struct omap_iommu *obj, struct cr_regs *cr) |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 285 | { |
Suman Anna | bd4396f | 2014-10-22 17:22:27 -0500 | [diff] [blame] | 286 | cr->cam = iommu_read_reg(obj, MMU_READ_CAM); |
| 287 | cr->ram = iommu_read_reg(obj, MMU_READ_RAM); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 288 | } |
| 289 | |
Ohad Ben-Cohen | 6c32df4 | 2011-08-17 22:57:56 +0300 | [diff] [blame] | 290 | static void iotlb_load_cr(struct omap_iommu *obj, struct cr_regs *cr) |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 291 | { |
Suman Anna | bd4396f | 2014-10-22 17:22:27 -0500 | [diff] [blame] | 292 | iommu_write_reg(obj, cr->cam | MMU_CAM_V, MMU_CAM); |
| 293 | iommu_write_reg(obj, cr->ram, MMU_RAM); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 294 | |
| 295 | iommu_write_reg(obj, 1, MMU_FLUSH_ENTRY); |
| 296 | iommu_write_reg(obj, 1, MMU_LD_TLB); |
| 297 | } |
| 298 | |
Hiroshi DOYU | 37c2836 | 2010-04-27 05:37:12 +0000 | [diff] [blame] | 299 | /* only used in iotlb iteration for-loop */ |
Suman Anna | 69c2c19 | 2015-07-20 17:33:25 -0500 | [diff] [blame] | 300 | struct cr_regs __iotlb_read_cr(struct omap_iommu *obj, int n) |
Hiroshi DOYU | 37c2836 | 2010-04-27 05:37:12 +0000 | [diff] [blame] | 301 | { |
| 302 | struct cr_regs cr; |
| 303 | struct iotlb_lock l; |
| 304 | |
| 305 | iotlb_lock_get(obj, &l); |
| 306 | l.vict = n; |
| 307 | iotlb_lock_set(obj, &l); |
| 308 | iotlb_read_cr(obj, &cr); |
| 309 | |
| 310 | return cr; |
| 311 | } |
| 312 | |
Suman Anna | bd4396f | 2014-10-22 17:22:27 -0500 | [diff] [blame] | 313 | #ifdef PREFETCH_IOTLB |
| 314 | static struct cr_regs *iotlb_alloc_cr(struct omap_iommu *obj, |
| 315 | struct iotlb_entry *e) |
| 316 | { |
| 317 | struct cr_regs *cr; |
| 318 | |
| 319 | if (!e) |
| 320 | return NULL; |
| 321 | |
| 322 | if (e->da & ~(get_cam_va_mask(e->pgsz))) { |
| 323 | dev_err(obj->dev, "%s:\twrong alignment: %08x\n", __func__, |
| 324 | e->da); |
| 325 | return ERR_PTR(-EINVAL); |
| 326 | } |
| 327 | |
| 328 | cr = kmalloc(sizeof(*cr), GFP_KERNEL); |
| 329 | if (!cr) |
| 330 | return ERR_PTR(-ENOMEM); |
| 331 | |
| 332 | cr->cam = (e->da & MMU_CAM_VATAG_MASK) | e->prsvd | e->pgsz | e->valid; |
| 333 | cr->ram = e->pa | e->endian | e->elsz | e->mixed; |
| 334 | |
| 335 | return cr; |
| 336 | } |
| 337 | |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 338 | /** |
| 339 | * load_iotlb_entry - Set an iommu tlb entry |
| 340 | * @obj: target iommu |
| 341 | * @e: an iommu tlb entry info |
| 342 | **/ |
Ohad Ben-Cohen | 6c32df4 | 2011-08-17 22:57:56 +0300 | [diff] [blame] | 343 | static int load_iotlb_entry(struct omap_iommu *obj, struct iotlb_entry *e) |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 344 | { |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 345 | int err = 0; |
| 346 | struct iotlb_lock l; |
| 347 | struct cr_regs *cr; |
| 348 | |
| 349 | if (!obj || !obj->nr_tlb_entries || !e) |
| 350 | return -EINVAL; |
| 351 | |
Omar Ramirez Luna | ebf7cda | 2012-11-19 19:05:51 -0600 | [diff] [blame] | 352 | pm_runtime_get_sync(obj->dev); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 353 | |
Kanigeri, Hari | be6d802 | 2010-04-22 23:26:11 +0000 | [diff] [blame] | 354 | iotlb_lock_get(obj, &l); |
| 355 | if (l.base == obj->nr_tlb_entries) { |
| 356 | dev_warn(obj->dev, "%s: preserve entries full\n", __func__); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 357 | err = -EBUSY; |
| 358 | goto out; |
| 359 | } |
Kanigeri, Hari | be6d802 | 2010-04-22 23:26:11 +0000 | [diff] [blame] | 360 | if (!e->prsvd) { |
Hiroshi DOYU | 37c2836 | 2010-04-27 05:37:12 +0000 | [diff] [blame] | 361 | int i; |
| 362 | struct cr_regs tmp; |
Kanigeri, Hari | be6d802 | 2010-04-22 23:26:11 +0000 | [diff] [blame] | 363 | |
Hiroshi DOYU | 37c2836 | 2010-04-27 05:37:12 +0000 | [diff] [blame] | 364 | for_each_iotlb_cr(obj, obj->nr_tlb_entries, i, tmp) |
Kanigeri, Hari | be6d802 | 2010-04-22 23:26:11 +0000 | [diff] [blame] | 365 | if (!iotlb_cr_valid(&tmp)) |
| 366 | break; |
Hiroshi DOYU | 37c2836 | 2010-04-27 05:37:12 +0000 | [diff] [blame] | 367 | |
Kanigeri, Hari | be6d802 | 2010-04-22 23:26:11 +0000 | [diff] [blame] | 368 | if (i == obj->nr_tlb_entries) { |
| 369 | dev_dbg(obj->dev, "%s: full: no entry\n", __func__); |
| 370 | err = -EBUSY; |
| 371 | goto out; |
| 372 | } |
Hiroshi DOYU | 37c2836 | 2010-04-27 05:37:12 +0000 | [diff] [blame] | 373 | |
| 374 | iotlb_lock_get(obj, &l); |
Kanigeri, Hari | be6d802 | 2010-04-22 23:26:11 +0000 | [diff] [blame] | 375 | } else { |
| 376 | l.vict = l.base; |
| 377 | iotlb_lock_set(obj, &l); |
| 378 | } |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 379 | |
| 380 | cr = iotlb_alloc_cr(obj, e); |
| 381 | if (IS_ERR(cr)) { |
Omar Ramirez Luna | ebf7cda | 2012-11-19 19:05:51 -0600 | [diff] [blame] | 382 | pm_runtime_put_sync(obj->dev); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 383 | return PTR_ERR(cr); |
| 384 | } |
| 385 | |
| 386 | iotlb_load_cr(obj, cr); |
| 387 | kfree(cr); |
| 388 | |
Kanigeri, Hari | be6d802 | 2010-04-22 23:26:11 +0000 | [diff] [blame] | 389 | if (e->prsvd) |
| 390 | l.base++; |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 391 | /* increment victim for next tlb load */ |
| 392 | if (++l.vict == obj->nr_tlb_entries) |
Kanigeri, Hari | be6d802 | 2010-04-22 23:26:11 +0000 | [diff] [blame] | 393 | l.vict = l.base; |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 394 | iotlb_lock_set(obj, &l); |
| 395 | out: |
Omar Ramirez Luna | ebf7cda | 2012-11-19 19:05:51 -0600 | [diff] [blame] | 396 | pm_runtime_put_sync(obj->dev); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 397 | return err; |
| 398 | } |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 399 | |
Ohad Ben-Cohen | 5da14a4 | 2011-08-16 15:19:10 +0300 | [diff] [blame] | 400 | #else /* !PREFETCH_IOTLB */ |
| 401 | |
Ohad Ben-Cohen | 6c32df4 | 2011-08-17 22:57:56 +0300 | [diff] [blame] | 402 | static int load_iotlb_entry(struct omap_iommu *obj, struct iotlb_entry *e) |
Ohad Ben-Cohen | 5da14a4 | 2011-08-16 15:19:10 +0300 | [diff] [blame] | 403 | { |
| 404 | return 0; |
| 405 | } |
| 406 | |
| 407 | #endif /* !PREFETCH_IOTLB */ |
| 408 | |
Ohad Ben-Cohen | 6c32df4 | 2011-08-17 22:57:56 +0300 | [diff] [blame] | 409 | static int prefetch_iotlb_entry(struct omap_iommu *obj, struct iotlb_entry *e) |
Ohad Ben-Cohen | 5da14a4 | 2011-08-16 15:19:10 +0300 | [diff] [blame] | 410 | { |
| 411 | return load_iotlb_entry(obj, e); |
| 412 | } |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 413 | |
| 414 | /** |
| 415 | * flush_iotlb_page - Clear an iommu tlb entry |
| 416 | * @obj: target iommu |
| 417 | * @da: iommu device virtual address |
| 418 | * |
| 419 | * Clear an iommu tlb entry which includes 'da' address. |
| 420 | **/ |
Ohad Ben-Cohen | 6c32df4 | 2011-08-17 22:57:56 +0300 | [diff] [blame] | 421 | static void flush_iotlb_page(struct omap_iommu *obj, u32 da) |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 422 | { |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 423 | int i; |
Hiroshi DOYU | 37c2836 | 2010-04-27 05:37:12 +0000 | [diff] [blame] | 424 | struct cr_regs cr; |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 425 | |
Omar Ramirez Luna | ebf7cda | 2012-11-19 19:05:51 -0600 | [diff] [blame] | 426 | pm_runtime_get_sync(obj->dev); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 427 | |
Hiroshi DOYU | 37c2836 | 2010-04-27 05:37:12 +0000 | [diff] [blame] | 428 | for_each_iotlb_cr(obj, obj->nr_tlb_entries, i, cr) { |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 429 | u32 start; |
| 430 | size_t bytes; |
| 431 | |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 432 | if (!iotlb_cr_valid(&cr)) |
| 433 | continue; |
| 434 | |
| 435 | start = iotlb_cr_to_virt(&cr); |
| 436 | bytes = iopgsz_to_bytes(cr.cam & 3); |
| 437 | |
| 438 | if ((start <= da) && (da < start + bytes)) { |
| 439 | dev_dbg(obj->dev, "%s: %08x<=%08x(%x)\n", |
| 440 | __func__, start, da, bytes); |
Hari Kanigeri | 0fa035e | 2010-08-20 13:50:18 +0000 | [diff] [blame] | 441 | iotlb_load_cr(obj, &cr); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 442 | iommu_write_reg(obj, 1, MMU_FLUSH_ENTRY); |
Laurent Pinchart | f7129a0 | 2014-03-07 23:47:03 +0100 | [diff] [blame] | 443 | break; |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 444 | } |
| 445 | } |
Omar Ramirez Luna | ebf7cda | 2012-11-19 19:05:51 -0600 | [diff] [blame] | 446 | pm_runtime_put_sync(obj->dev); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 447 | |
| 448 | if (i == obj->nr_tlb_entries) |
| 449 | dev_dbg(obj->dev, "%s: no page for %08x\n", __func__, da); |
| 450 | } |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 451 | |
| 452 | /** |
| 453 | * flush_iotlb_all - Clear all iommu tlb entries |
| 454 | * @obj: target iommu |
| 455 | **/ |
Ohad Ben-Cohen | 6c32df4 | 2011-08-17 22:57:56 +0300 | [diff] [blame] | 456 | static void flush_iotlb_all(struct omap_iommu *obj) |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 457 | { |
| 458 | struct iotlb_lock l; |
| 459 | |
Omar Ramirez Luna | ebf7cda | 2012-11-19 19:05:51 -0600 | [diff] [blame] | 460 | pm_runtime_get_sync(obj->dev); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 461 | |
| 462 | l.base = 0; |
| 463 | l.vict = 0; |
| 464 | iotlb_lock_set(obj, &l); |
| 465 | |
| 466 | iommu_write_reg(obj, 1, MMU_GFLUSH); |
| 467 | |
Omar Ramirez Luna | ebf7cda | 2012-11-19 19:05:51 -0600 | [diff] [blame] | 468 | pm_runtime_put_sync(obj->dev); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 469 | } |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 470 | |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 471 | /* |
| 472 | * H/W pagetable operations |
| 473 | */ |
Josue Albarran | bfee0cf | 2017-07-28 15:49:14 -0500 | [diff] [blame] | 474 | static void flush_iopte_range(struct device *dev, dma_addr_t dma, |
| 475 | unsigned long offset, int num_entries) |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 476 | { |
Josue Albarran | bfee0cf | 2017-07-28 15:49:14 -0500 | [diff] [blame] | 477 | size_t size = num_entries * sizeof(u32); |
| 478 | |
| 479 | dma_sync_single_range_for_device(dev, dma, offset, size, DMA_TO_DEVICE); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 480 | } |
| 481 | |
Josue Albarran | bfee0cf | 2017-07-28 15:49:14 -0500 | [diff] [blame] | 482 | static void iopte_free(struct omap_iommu *obj, u32 *iopte, bool dma_valid) |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 483 | { |
Josue Albarran | bfee0cf | 2017-07-28 15:49:14 -0500 | [diff] [blame] | 484 | dma_addr_t pt_dma; |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 485 | |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 486 | /* Note: freed iopte's must be clean ready for re-use */ |
Josue Albarran | bfee0cf | 2017-07-28 15:49:14 -0500 | [diff] [blame] | 487 | if (iopte) { |
| 488 | if (dma_valid) { |
| 489 | pt_dma = virt_to_phys(iopte); |
| 490 | dma_unmap_single(obj->dev, pt_dma, IOPTE_TABLE_SIZE, |
| 491 | DMA_TO_DEVICE); |
| 492 | } |
| 493 | |
Zhouyi Zhou | e28045a | 2014-03-05 18:20:19 +0800 | [diff] [blame] | 494 | kmem_cache_free(iopte_cachep, iopte); |
Josue Albarran | bfee0cf | 2017-07-28 15:49:14 -0500 | [diff] [blame] | 495 | } |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 496 | } |
| 497 | |
Josue Albarran | bfee0cf | 2017-07-28 15:49:14 -0500 | [diff] [blame] | 498 | static u32 *iopte_alloc(struct omap_iommu *obj, u32 *iopgd, |
| 499 | dma_addr_t *pt_dma, u32 da) |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 500 | { |
| 501 | u32 *iopte; |
Josue Albarran | bfee0cf | 2017-07-28 15:49:14 -0500 | [diff] [blame] | 502 | unsigned long offset = iopgd_index(da) * sizeof(da); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 503 | |
| 504 | /* a table has already existed */ |
| 505 | if (*iopgd) |
| 506 | goto pte_ready; |
| 507 | |
| 508 | /* |
| 509 | * do the allocation outside the page table lock |
| 510 | */ |
| 511 | spin_unlock(&obj->page_table_lock); |
| 512 | iopte = kmem_cache_zalloc(iopte_cachep, GFP_KERNEL); |
| 513 | spin_lock(&obj->page_table_lock); |
| 514 | |
| 515 | if (!*iopgd) { |
| 516 | if (!iopte) |
| 517 | return ERR_PTR(-ENOMEM); |
| 518 | |
Josue Albarran | bfee0cf | 2017-07-28 15:49:14 -0500 | [diff] [blame] | 519 | *pt_dma = dma_map_single(obj->dev, iopte, IOPTE_TABLE_SIZE, |
| 520 | DMA_TO_DEVICE); |
| 521 | if (dma_mapping_error(obj->dev, *pt_dma)) { |
| 522 | dev_err(obj->dev, "DMA map error for L2 table\n"); |
| 523 | iopte_free(obj, iopte, false); |
| 524 | return ERR_PTR(-ENOMEM); |
| 525 | } |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 526 | |
Josue Albarran | bfee0cf | 2017-07-28 15:49:14 -0500 | [diff] [blame] | 527 | /* |
| 528 | * we rely on dma address and the physical address to be |
| 529 | * the same for mapping the L2 table |
| 530 | */ |
| 531 | if (WARN_ON(*pt_dma != virt_to_phys(iopte))) { |
| 532 | dev_err(obj->dev, "DMA translation error for L2 table\n"); |
| 533 | dma_unmap_single(obj->dev, *pt_dma, IOPTE_TABLE_SIZE, |
| 534 | DMA_TO_DEVICE); |
| 535 | iopte_free(obj, iopte, false); |
| 536 | return ERR_PTR(-ENOMEM); |
| 537 | } |
| 538 | |
| 539 | *iopgd = virt_to_phys(iopte) | IOPGD_TABLE; |
| 540 | |
| 541 | flush_iopte_range(obj->dev, obj->pd_dma, offset, 1); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 542 | dev_vdbg(obj->dev, "%s: a new pte:%p\n", __func__, iopte); |
| 543 | } else { |
| 544 | /* We raced, free the reduniovant table */ |
Josue Albarran | bfee0cf | 2017-07-28 15:49:14 -0500 | [diff] [blame] | 545 | iopte_free(obj, iopte, false); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 546 | } |
| 547 | |
| 548 | pte_ready: |
| 549 | iopte = iopte_offset(iopgd, da); |
Ralf Goebel | 04c532a | 2018-08-06 17:00:36 +0200 | [diff] [blame] | 550 | *pt_dma = iopgd_page_paddr(iopgd); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 551 | dev_vdbg(obj->dev, |
| 552 | "%s: da:%08x pgd:%p *pgd:%08x pte:%p *pte:%08x\n", |
| 553 | __func__, da, iopgd, *iopgd, iopte, *iopte); |
| 554 | |
| 555 | return iopte; |
| 556 | } |
| 557 | |
Ohad Ben-Cohen | 6c32df4 | 2011-08-17 22:57:56 +0300 | [diff] [blame] | 558 | static int iopgd_alloc_section(struct omap_iommu *obj, u32 da, u32 pa, u32 prot) |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 559 | { |
| 560 | u32 *iopgd = iopgd_offset(obj, da); |
Josue Albarran | bfee0cf | 2017-07-28 15:49:14 -0500 | [diff] [blame] | 561 | unsigned long offset = iopgd_index(da) * sizeof(da); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 562 | |
Hiroshi DOYU | 4abb761 | 2010-05-06 18:24:04 +0300 | [diff] [blame] | 563 | if ((da | pa) & ~IOSECTION_MASK) { |
| 564 | dev_err(obj->dev, "%s: %08x:%08x should aligned on %08lx\n", |
| 565 | __func__, da, pa, IOSECTION_SIZE); |
| 566 | return -EINVAL; |
| 567 | } |
| 568 | |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 569 | *iopgd = (pa & IOSECTION_MASK) | prot | IOPGD_SECTION; |
Josue Albarran | bfee0cf | 2017-07-28 15:49:14 -0500 | [diff] [blame] | 570 | flush_iopte_range(obj->dev, obj->pd_dma, offset, 1); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 571 | return 0; |
| 572 | } |
| 573 | |
Ohad Ben-Cohen | 6c32df4 | 2011-08-17 22:57:56 +0300 | [diff] [blame] | 574 | static int iopgd_alloc_super(struct omap_iommu *obj, u32 da, u32 pa, u32 prot) |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 575 | { |
| 576 | u32 *iopgd = iopgd_offset(obj, da); |
Josue Albarran | bfee0cf | 2017-07-28 15:49:14 -0500 | [diff] [blame] | 577 | unsigned long offset = iopgd_index(da) * sizeof(da); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 578 | int i; |
| 579 | |
Hiroshi DOYU | 4abb761 | 2010-05-06 18:24:04 +0300 | [diff] [blame] | 580 | if ((da | pa) & ~IOSUPER_MASK) { |
| 581 | dev_err(obj->dev, "%s: %08x:%08x should aligned on %08lx\n", |
| 582 | __func__, da, pa, IOSUPER_SIZE); |
| 583 | return -EINVAL; |
| 584 | } |
| 585 | |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 586 | for (i = 0; i < 16; i++) |
| 587 | *(iopgd + i) = (pa & IOSUPER_MASK) | prot | IOPGD_SUPER; |
Josue Albarran | bfee0cf | 2017-07-28 15:49:14 -0500 | [diff] [blame] | 588 | flush_iopte_range(obj->dev, obj->pd_dma, offset, 16); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 589 | return 0; |
| 590 | } |
| 591 | |
Ohad Ben-Cohen | 6c32df4 | 2011-08-17 22:57:56 +0300 | [diff] [blame] | 592 | static int iopte_alloc_page(struct omap_iommu *obj, u32 da, u32 pa, u32 prot) |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 593 | { |
| 594 | u32 *iopgd = iopgd_offset(obj, da); |
Josue Albarran | bfee0cf | 2017-07-28 15:49:14 -0500 | [diff] [blame] | 595 | dma_addr_t pt_dma; |
| 596 | u32 *iopte = iopte_alloc(obj, iopgd, &pt_dma, da); |
| 597 | unsigned long offset = iopte_index(da) * sizeof(da); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 598 | |
| 599 | if (IS_ERR(iopte)) |
| 600 | return PTR_ERR(iopte); |
| 601 | |
| 602 | *iopte = (pa & IOPAGE_MASK) | prot | IOPTE_SMALL; |
Josue Albarran | bfee0cf | 2017-07-28 15:49:14 -0500 | [diff] [blame] | 603 | flush_iopte_range(obj->dev, pt_dma, offset, 1); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 604 | |
| 605 | dev_vdbg(obj->dev, "%s: da:%08x pa:%08x pte:%p *pte:%08x\n", |
| 606 | __func__, da, pa, iopte, *iopte); |
| 607 | |
| 608 | return 0; |
| 609 | } |
| 610 | |
Ohad Ben-Cohen | 6c32df4 | 2011-08-17 22:57:56 +0300 | [diff] [blame] | 611 | static int iopte_alloc_large(struct omap_iommu *obj, u32 da, u32 pa, u32 prot) |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 612 | { |
| 613 | u32 *iopgd = iopgd_offset(obj, da); |
Josue Albarran | bfee0cf | 2017-07-28 15:49:14 -0500 | [diff] [blame] | 614 | dma_addr_t pt_dma; |
| 615 | u32 *iopte = iopte_alloc(obj, iopgd, &pt_dma, da); |
| 616 | unsigned long offset = iopte_index(da) * sizeof(da); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 617 | int i; |
| 618 | |
Hiroshi DOYU | 4abb761 | 2010-05-06 18:24:04 +0300 | [diff] [blame] | 619 | if ((da | pa) & ~IOLARGE_MASK) { |
| 620 | dev_err(obj->dev, "%s: %08x:%08x should aligned on %08lx\n", |
| 621 | __func__, da, pa, IOLARGE_SIZE); |
| 622 | return -EINVAL; |
| 623 | } |
| 624 | |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 625 | if (IS_ERR(iopte)) |
| 626 | return PTR_ERR(iopte); |
| 627 | |
| 628 | for (i = 0; i < 16; i++) |
| 629 | *(iopte + i) = (pa & IOLARGE_MASK) | prot | IOPTE_LARGE; |
Josue Albarran | bfee0cf | 2017-07-28 15:49:14 -0500 | [diff] [blame] | 630 | flush_iopte_range(obj->dev, pt_dma, offset, 16); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 631 | return 0; |
| 632 | } |
| 633 | |
Ohad Ben-Cohen | 6c32df4 | 2011-08-17 22:57:56 +0300 | [diff] [blame] | 634 | static int |
| 635 | iopgtable_store_entry_core(struct omap_iommu *obj, struct iotlb_entry *e) |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 636 | { |
Ohad Ben-Cohen | 6c32df4 | 2011-08-17 22:57:56 +0300 | [diff] [blame] | 637 | int (*fn)(struct omap_iommu *, u32, u32, u32); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 638 | u32 prot; |
| 639 | int err; |
| 640 | |
| 641 | if (!obj || !e) |
| 642 | return -EINVAL; |
| 643 | |
| 644 | switch (e->pgsz) { |
| 645 | case MMU_CAM_PGSZ_16M: |
| 646 | fn = iopgd_alloc_super; |
| 647 | break; |
| 648 | case MMU_CAM_PGSZ_1M: |
| 649 | fn = iopgd_alloc_section; |
| 650 | break; |
| 651 | case MMU_CAM_PGSZ_64K: |
| 652 | fn = iopte_alloc_large; |
| 653 | break; |
| 654 | case MMU_CAM_PGSZ_4K: |
| 655 | fn = iopte_alloc_page; |
| 656 | break; |
| 657 | default: |
| 658 | fn = NULL; |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 659 | break; |
| 660 | } |
| 661 | |
Suman Anna | 7c1ab60 | 2016-04-04 17:46:19 -0500 | [diff] [blame] | 662 | if (WARN_ON(!fn)) |
| 663 | return -EINVAL; |
| 664 | |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 665 | prot = get_iopte_attr(e); |
| 666 | |
| 667 | spin_lock(&obj->page_table_lock); |
| 668 | err = fn(obj, e->da, e->pa, prot); |
| 669 | spin_unlock(&obj->page_table_lock); |
| 670 | |
| 671 | return err; |
| 672 | } |
| 673 | |
| 674 | /** |
Ohad Ben-Cohen | 6c32df4 | 2011-08-17 22:57:56 +0300 | [diff] [blame] | 675 | * omap_iopgtable_store_entry - Make an iommu pte entry |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 676 | * @obj: target iommu |
| 677 | * @e: an iommu tlb entry info |
| 678 | **/ |
Suman Anna | 4899a56 | 2014-10-22 17:22:32 -0500 | [diff] [blame] | 679 | static int |
| 680 | omap_iopgtable_store_entry(struct omap_iommu *obj, struct iotlb_entry *e) |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 681 | { |
| 682 | int err; |
| 683 | |
| 684 | flush_iotlb_page(obj, e->da); |
| 685 | err = iopgtable_store_entry_core(obj, e); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 686 | if (!err) |
Ohad Ben-Cohen | 5da14a4 | 2011-08-16 15:19:10 +0300 | [diff] [blame] | 687 | prefetch_iotlb_entry(obj, e); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 688 | return err; |
| 689 | } |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 690 | |
| 691 | /** |
| 692 | * iopgtable_lookup_entry - Lookup an iommu pte entry |
| 693 | * @obj: target iommu |
| 694 | * @da: iommu device virtual address |
| 695 | * @ppgd: iommu pgd entry pointer to be returned |
| 696 | * @ppte: iommu pte entry pointer to be returned |
| 697 | **/ |
Ohad Ben-Cohen | e1f2381 | 2011-08-16 14:58:14 +0300 | [diff] [blame] | 698 | static void |
| 699 | iopgtable_lookup_entry(struct omap_iommu *obj, u32 da, u32 **ppgd, u32 **ppte) |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 700 | { |
| 701 | u32 *iopgd, *iopte = NULL; |
| 702 | |
| 703 | iopgd = iopgd_offset(obj, da); |
| 704 | if (!*iopgd) |
| 705 | goto out; |
| 706 | |
Hiroshi DOYU | a1a5445 | 2010-05-13 09:45:35 +0300 | [diff] [blame] | 707 | if (iopgd_is_table(*iopgd)) |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 708 | iopte = iopte_offset(iopgd, da); |
| 709 | out: |
| 710 | *ppgd = iopgd; |
| 711 | *ppte = iopte; |
| 712 | } |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 713 | |
Ohad Ben-Cohen | 6c32df4 | 2011-08-17 22:57:56 +0300 | [diff] [blame] | 714 | static size_t iopgtable_clear_entry_core(struct omap_iommu *obj, u32 da) |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 715 | { |
| 716 | size_t bytes; |
| 717 | u32 *iopgd = iopgd_offset(obj, da); |
| 718 | int nent = 1; |
Josue Albarran | bfee0cf | 2017-07-28 15:49:14 -0500 | [diff] [blame] | 719 | dma_addr_t pt_dma; |
| 720 | unsigned long pd_offset = iopgd_index(da) * sizeof(da); |
| 721 | unsigned long pt_offset = iopte_index(da) * sizeof(da); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 722 | |
| 723 | if (!*iopgd) |
| 724 | return 0; |
| 725 | |
Hiroshi DOYU | a1a5445 | 2010-05-13 09:45:35 +0300 | [diff] [blame] | 726 | if (iopgd_is_table(*iopgd)) { |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 727 | int i; |
| 728 | u32 *iopte = iopte_offset(iopgd, da); |
| 729 | |
| 730 | bytes = IOPTE_SIZE; |
| 731 | if (*iopte & IOPTE_LARGE) { |
| 732 | nent *= 16; |
| 733 | /* rewind to the 1st entry */ |
Hiroshi DOYU | c127c7d | 2010-02-15 10:03:32 -0800 | [diff] [blame] | 734 | iopte = iopte_offset(iopgd, (da & IOLARGE_MASK)); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 735 | } |
| 736 | bytes *= nent; |
| 737 | memset(iopte, 0, nent * sizeof(*iopte)); |
Ralf Goebel | 04c532a | 2018-08-06 17:00:36 +0200 | [diff] [blame] | 738 | pt_dma = iopgd_page_paddr(iopgd); |
Josue Albarran | bfee0cf | 2017-07-28 15:49:14 -0500 | [diff] [blame] | 739 | flush_iopte_range(obj->dev, pt_dma, pt_offset, nent); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 740 | |
| 741 | /* |
| 742 | * do table walk to check if this table is necessary or not |
| 743 | */ |
| 744 | iopte = iopte_offset(iopgd, 0); |
| 745 | for (i = 0; i < PTRS_PER_IOPTE; i++) |
| 746 | if (iopte[i]) |
| 747 | goto out; |
| 748 | |
Josue Albarran | bfee0cf | 2017-07-28 15:49:14 -0500 | [diff] [blame] | 749 | iopte_free(obj, iopte, true); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 750 | nent = 1; /* for the next L1 entry */ |
| 751 | } else { |
| 752 | bytes = IOPGD_SIZE; |
Hiroshi DOYU | dcc730d | 2009-10-22 14:46:32 -0700 | [diff] [blame] | 753 | if ((*iopgd & IOPGD_SUPER) == IOPGD_SUPER) { |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 754 | nent *= 16; |
| 755 | /* rewind to the 1st entry */ |
Hiroshi DOYU | 8d33ea5 | 2010-02-15 10:03:32 -0800 | [diff] [blame] | 756 | iopgd = iopgd_offset(obj, (da & IOSUPER_MASK)); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 757 | } |
| 758 | bytes *= nent; |
| 759 | } |
| 760 | memset(iopgd, 0, nent * sizeof(*iopgd)); |
Josue Albarran | bfee0cf | 2017-07-28 15:49:14 -0500 | [diff] [blame] | 761 | flush_iopte_range(obj->dev, obj->pd_dma, pd_offset, nent); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 762 | out: |
| 763 | return bytes; |
| 764 | } |
| 765 | |
| 766 | /** |
| 767 | * iopgtable_clear_entry - Remove an iommu pte entry |
| 768 | * @obj: target iommu |
| 769 | * @da: iommu device virtual address |
| 770 | **/ |
Ohad Ben-Cohen | 6c32df4 | 2011-08-17 22:57:56 +0300 | [diff] [blame] | 771 | static size_t iopgtable_clear_entry(struct omap_iommu *obj, u32 da) |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 772 | { |
| 773 | size_t bytes; |
| 774 | |
| 775 | spin_lock(&obj->page_table_lock); |
| 776 | |
| 777 | bytes = iopgtable_clear_entry_core(obj, da); |
| 778 | flush_iotlb_page(obj, da); |
| 779 | |
| 780 | spin_unlock(&obj->page_table_lock); |
| 781 | |
| 782 | return bytes; |
| 783 | } |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 784 | |
Ohad Ben-Cohen | 6c32df4 | 2011-08-17 22:57:56 +0300 | [diff] [blame] | 785 | static void iopgtable_clear_entry_all(struct omap_iommu *obj) |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 786 | { |
Josue Albarran | bfee0cf | 2017-07-28 15:49:14 -0500 | [diff] [blame] | 787 | unsigned long offset; |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 788 | int i; |
| 789 | |
| 790 | spin_lock(&obj->page_table_lock); |
| 791 | |
| 792 | for (i = 0; i < PTRS_PER_IOPGD; i++) { |
| 793 | u32 da; |
| 794 | u32 *iopgd; |
| 795 | |
| 796 | da = i << IOPGD_SHIFT; |
| 797 | iopgd = iopgd_offset(obj, da); |
Josue Albarran | bfee0cf | 2017-07-28 15:49:14 -0500 | [diff] [blame] | 798 | offset = iopgd_index(da) * sizeof(da); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 799 | |
| 800 | if (!*iopgd) |
| 801 | continue; |
| 802 | |
Hiroshi DOYU | a1a5445 | 2010-05-13 09:45:35 +0300 | [diff] [blame] | 803 | if (iopgd_is_table(*iopgd)) |
Josue Albarran | bfee0cf | 2017-07-28 15:49:14 -0500 | [diff] [blame] | 804 | iopte_free(obj, iopte_offset(iopgd, 0), true); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 805 | |
| 806 | *iopgd = 0; |
Josue Albarran | bfee0cf | 2017-07-28 15:49:14 -0500 | [diff] [blame] | 807 | flush_iopte_range(obj->dev, obj->pd_dma, offset, 1); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 808 | } |
| 809 | |
| 810 | flush_iotlb_all(obj); |
| 811 | |
| 812 | spin_unlock(&obj->page_table_lock); |
| 813 | } |
| 814 | |
| 815 | /* |
| 816 | * Device IOMMU generic operations |
| 817 | */ |
| 818 | static irqreturn_t iommu_fault_handler(int irq, void *data) |
| 819 | { |
David Cohen | d594f1f | 2011-02-16 19:35:51 +0000 | [diff] [blame] | 820 | u32 da, errs; |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 821 | u32 *iopgd, *iopte; |
Ohad Ben-Cohen | 6c32df4 | 2011-08-17 22:57:56 +0300 | [diff] [blame] | 822 | struct omap_iommu *obj = data; |
Ohad Ben-Cohen | e7f10f0 | 2011-09-13 15:26:29 -0400 | [diff] [blame] | 823 | struct iommu_domain *domain = obj->domain; |
Joerg Roedel | 8cf851e | 2015-03-26 13:43:09 +0100 | [diff] [blame] | 824 | struct omap_iommu_domain *omap_domain = to_omap_domain(domain); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 825 | |
Suman Anna | 0d36428 | 2017-09-05 17:56:17 -0500 | [diff] [blame] | 826 | if (!omap_domain->dev) |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 827 | return IRQ_NONE; |
| 828 | |
David Cohen | d594f1f | 2011-02-16 19:35:51 +0000 | [diff] [blame] | 829 | errs = iommu_report_fault(obj, &da); |
Laurent Pinchart | c56b2dd | 2011-05-10 16:56:46 +0200 | [diff] [blame] | 830 | if (errs == 0) |
| 831 | return IRQ_HANDLED; |
David Cohen | d594f1f | 2011-02-16 19:35:51 +0000 | [diff] [blame] | 832 | |
| 833 | /* Fault callback or TLB/PTE Dynamic loading */ |
Ohad Ben-Cohen | e7f10f0 | 2011-09-13 15:26:29 -0400 | [diff] [blame] | 834 | if (!report_iommu_fault(domain, obj->dev, da, 0)) |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 835 | return IRQ_HANDLED; |
| 836 | |
Fernando Guzman Lugo | 159d3e3 | 2017-07-28 15:49:13 -0500 | [diff] [blame] | 837 | iommu_write_reg(obj, 0, MMU_IRQENABLE); |
Hiroshi DOYU | 37b2981 | 2010-05-24 02:01:52 +0000 | [diff] [blame] | 838 | |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 839 | iopgd = iopgd_offset(obj, da); |
| 840 | |
Hiroshi DOYU | a1a5445 | 2010-05-13 09:45:35 +0300 | [diff] [blame] | 841 | if (!iopgd_is_table(*iopgd)) { |
Suman Anna | b6c2e09 | 2013-05-30 18:10:59 -0500 | [diff] [blame] | 842 | dev_err(obj->dev, "%s: errs:0x%08x da:0x%08x pgd:0x%p *pgd:px%08x\n", |
Suman Anna | 5835b6a | 2015-07-20 17:33:32 -0500 | [diff] [blame] | 843 | obj->name, errs, da, iopgd, *iopgd); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 844 | return IRQ_NONE; |
| 845 | } |
| 846 | |
| 847 | iopte = iopte_offset(iopgd, da); |
| 848 | |
Suman Anna | b6c2e09 | 2013-05-30 18:10:59 -0500 | [diff] [blame] | 849 | dev_err(obj->dev, "%s: errs:0x%08x da:0x%08x pgd:0x%p *pgd:0x%08x pte:0x%p *pte:0x%08x\n", |
Suman Anna | 5835b6a | 2015-07-20 17:33:32 -0500 | [diff] [blame] | 850 | obj->name, errs, da, iopgd, *iopgd, iopte, *iopte); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 851 | |
| 852 | return IRQ_NONE; |
| 853 | } |
| 854 | |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 855 | /** |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 856 | * omap_iommu_attach() - attach iommu device to an iommu domain |
Joerg Roedel | ede1c2e | 2017-04-12 00:21:29 -0500 | [diff] [blame] | 857 | * @obj: target omap iommu device |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 858 | * @iopgd: page table |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 859 | **/ |
Joerg Roedel | ede1c2e | 2017-04-12 00:21:29 -0500 | [diff] [blame] | 860 | static int omap_iommu_attach(struct omap_iommu *obj, u32 *iopgd) |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 861 | { |
Suman Anna | 7ee08b9e | 2014-02-28 14:42:33 -0600 | [diff] [blame] | 862 | int err; |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 863 | |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 864 | spin_lock(&obj->iommu_lock); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 865 | |
Josue Albarran | bfee0cf | 2017-07-28 15:49:14 -0500 | [diff] [blame] | 866 | obj->pd_dma = dma_map_single(obj->dev, iopgd, IOPGD_TABLE_SIZE, |
| 867 | DMA_TO_DEVICE); |
| 868 | if (dma_mapping_error(obj->dev, obj->pd_dma)) { |
| 869 | dev_err(obj->dev, "DMA map error for L1 table\n"); |
| 870 | err = -ENOMEM; |
| 871 | goto out_err; |
| 872 | } |
| 873 | |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 874 | obj->iopgd = iopgd; |
| 875 | err = iommu_enable(obj); |
| 876 | if (err) |
Josue Albarran | bfee0cf | 2017-07-28 15:49:14 -0500 | [diff] [blame] | 877 | goto out_err; |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 878 | flush_iotlb_all(obj); |
| 879 | |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 880 | spin_unlock(&obj->iommu_lock); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 881 | |
| 882 | dev_dbg(obj->dev, "%s: %s\n", __func__, obj->name); |
Joerg Roedel | ede1c2e | 2017-04-12 00:21:29 -0500 | [diff] [blame] | 883 | |
| 884 | return 0; |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 885 | |
Josue Albarran | bfee0cf | 2017-07-28 15:49:14 -0500 | [diff] [blame] | 886 | out_err: |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 887 | spin_unlock(&obj->iommu_lock); |
Joerg Roedel | ede1c2e | 2017-04-12 00:21:29 -0500 | [diff] [blame] | 888 | |
| 889 | return err; |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 890 | } |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 891 | |
| 892 | /** |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 893 | * omap_iommu_detach - release iommu device |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 894 | * @obj: target iommu |
| 895 | **/ |
Ohad Ben-Cohen | 6c32df4 | 2011-08-17 22:57:56 +0300 | [diff] [blame] | 896 | static void omap_iommu_detach(struct omap_iommu *obj) |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 897 | { |
Roel Kluin | acf9d46 | 2010-01-08 10:29:05 -0800 | [diff] [blame] | 898 | if (!obj || IS_ERR(obj)) |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 899 | return; |
| 900 | |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 901 | spin_lock(&obj->iommu_lock); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 902 | |
Josue Albarran | bfee0cf | 2017-07-28 15:49:14 -0500 | [diff] [blame] | 903 | dma_unmap_single(obj->dev, obj->pd_dma, IOPGD_TABLE_SIZE, |
| 904 | DMA_TO_DEVICE); |
Suman Anna | 2088ecb | 2014-10-22 17:22:19 -0500 | [diff] [blame] | 905 | iommu_disable(obj); |
Josue Albarran | bfee0cf | 2017-07-28 15:49:14 -0500 | [diff] [blame] | 906 | obj->pd_dma = 0; |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 907 | obj->iopgd = NULL; |
| 908 | |
| 909 | spin_unlock(&obj->iommu_lock); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 910 | |
| 911 | dev_dbg(obj->dev, "%s: %s\n", __func__, obj->name); |
| 912 | } |
David Cohen | d594f1f | 2011-02-16 19:35:51 +0000 | [diff] [blame] | 913 | |
Suman Anna | 9d5018d | 2017-09-05 17:56:18 -0500 | [diff] [blame] | 914 | static bool omap_iommu_can_register(struct platform_device *pdev) |
| 915 | { |
| 916 | struct device_node *np = pdev->dev.of_node; |
| 917 | |
| 918 | if (!of_device_is_compatible(np, "ti,dra7-dsp-iommu")) |
| 919 | return true; |
| 920 | |
| 921 | /* |
| 922 | * restrict IOMMU core registration only for processor-port MDMA MMUs |
| 923 | * on DRA7 DSPs |
| 924 | */ |
| 925 | if ((!strcmp(dev_name(&pdev->dev), "40d01000.mmu")) || |
| 926 | (!strcmp(dev_name(&pdev->dev), "41501000.mmu"))) |
| 927 | return true; |
| 928 | |
| 929 | return false; |
| 930 | } |
| 931 | |
Suman Anna | 3ca9299 | 2015-10-02 18:02:44 -0500 | [diff] [blame] | 932 | static int omap_iommu_dra7_get_dsp_system_cfg(struct platform_device *pdev, |
| 933 | struct omap_iommu *obj) |
| 934 | { |
| 935 | struct device_node *np = pdev->dev.of_node; |
| 936 | int ret; |
| 937 | |
| 938 | if (!of_device_is_compatible(np, "ti,dra7-dsp-iommu")) |
| 939 | return 0; |
| 940 | |
| 941 | if (!of_property_read_bool(np, "ti,syscon-mmuconfig")) { |
| 942 | dev_err(&pdev->dev, "ti,syscon-mmuconfig property is missing\n"); |
| 943 | return -EINVAL; |
| 944 | } |
| 945 | |
| 946 | obj->syscfg = |
| 947 | syscon_regmap_lookup_by_phandle(np, "ti,syscon-mmuconfig"); |
| 948 | if (IS_ERR(obj->syscfg)) { |
| 949 | /* can fail with -EPROBE_DEFER */ |
| 950 | ret = PTR_ERR(obj->syscfg); |
| 951 | return ret; |
| 952 | } |
| 953 | |
| 954 | if (of_property_read_u32_index(np, "ti,syscon-mmuconfig", 1, |
| 955 | &obj->id)) { |
| 956 | dev_err(&pdev->dev, "couldn't get the IOMMU instance id within subsystem\n"); |
| 957 | return -EINVAL; |
| 958 | } |
| 959 | |
| 960 | if (obj->id != 0 && obj->id != 1) { |
| 961 | dev_err(&pdev->dev, "invalid IOMMU instance id\n"); |
| 962 | return -EINVAL; |
| 963 | } |
| 964 | |
| 965 | return 0; |
| 966 | } |
| 967 | |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 968 | /* |
| 969 | * OMAP Device MMU(IOMMU) detection |
| 970 | */ |
Greg Kroah-Hartman | d34d651 | 2012-12-21 15:05:21 -0800 | [diff] [blame] | 971 | static int omap_iommu_probe(struct platform_device *pdev) |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 972 | { |
| 973 | int err = -ENODEV; |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 974 | int irq; |
Ohad Ben-Cohen | 6c32df4 | 2011-08-17 22:57:56 +0300 | [diff] [blame] | 975 | struct omap_iommu *obj; |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 976 | struct resource *res; |
Florian Vaussard | 3c92748 | 2014-02-28 14:42:36 -0600 | [diff] [blame] | 977 | struct device_node *of = pdev->dev.of_node; |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 978 | |
Suman Anna | 49a57ef | 2017-04-12 00:21:27 -0500 | [diff] [blame] | 979 | if (!of) { |
| 980 | pr_err("%s: only DT-based devices are supported\n", __func__); |
| 981 | return -ENODEV; |
| 982 | } |
| 983 | |
Suman Anna | f129b3d | 2014-02-28 14:42:32 -0600 | [diff] [blame] | 984 | obj = devm_kzalloc(&pdev->dev, sizeof(*obj) + MMU_REG_SIZE, GFP_KERNEL); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 985 | if (!obj) |
| 986 | return -ENOMEM; |
| 987 | |
Suman Anna | 49a57ef | 2017-04-12 00:21:27 -0500 | [diff] [blame] | 988 | obj->name = dev_name(&pdev->dev); |
| 989 | obj->nr_tlb_entries = 32; |
| 990 | err = of_property_read_u32(of, "ti,#tlb-entries", &obj->nr_tlb_entries); |
| 991 | if (err && err != -EINVAL) |
| 992 | return err; |
| 993 | if (obj->nr_tlb_entries != 32 && obj->nr_tlb_entries != 8) |
| 994 | return -EINVAL; |
| 995 | if (of_find_property(of, "ti,iommu-bus-err-back", NULL)) |
| 996 | obj->has_bus_err_back = MMU_GP_REG_BUS_ERR_BACK_EN; |
Florian Vaussard | 3c92748 | 2014-02-28 14:42:36 -0600 | [diff] [blame] | 997 | |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 998 | obj->dev = &pdev->dev; |
| 999 | obj->ctx = (void *)obj + sizeof(*obj); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 1000 | |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 1001 | spin_lock_init(&obj->iommu_lock); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 1002 | spin_lock_init(&obj->page_table_lock); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 1003 | |
| 1004 | res = platform_get_resource(pdev, IORESOURCE_MEM, 0); |
Suman Anna | f129b3d | 2014-02-28 14:42:32 -0600 | [diff] [blame] | 1005 | obj->regbase = devm_ioremap_resource(obj->dev, res); |
| 1006 | if (IS_ERR(obj->regbase)) |
| 1007 | return PTR_ERR(obj->regbase); |
Aaro Koskinen | da4a0f7 | 2011-03-14 12:28:32 +0000 | [diff] [blame] | 1008 | |
Suman Anna | 3ca9299 | 2015-10-02 18:02:44 -0500 | [diff] [blame] | 1009 | err = omap_iommu_dra7_get_dsp_system_cfg(pdev, obj); |
| 1010 | if (err) |
| 1011 | return err; |
| 1012 | |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 1013 | irq = platform_get_irq(pdev, 0); |
Suman Anna | f129b3d | 2014-02-28 14:42:32 -0600 | [diff] [blame] | 1014 | if (irq < 0) |
| 1015 | return -ENODEV; |
| 1016 | |
| 1017 | err = devm_request_irq(obj->dev, irq, iommu_fault_handler, IRQF_SHARED, |
| 1018 | dev_name(obj->dev), obj); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 1019 | if (err < 0) |
Suman Anna | f129b3d | 2014-02-28 14:42:32 -0600 | [diff] [blame] | 1020 | return err; |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 1021 | platform_set_drvdata(pdev, obj); |
| 1022 | |
Suman Anna | 9d5018d | 2017-09-05 17:56:18 -0500 | [diff] [blame] | 1023 | if (omap_iommu_can_register(pdev)) { |
| 1024 | obj->group = iommu_group_alloc(); |
| 1025 | if (IS_ERR(obj->group)) |
| 1026 | return PTR_ERR(obj->group); |
Joerg Roedel | 28ae1e3 | 2017-04-12 00:21:31 -0500 | [diff] [blame] | 1027 | |
Suman Anna | 9d5018d | 2017-09-05 17:56:18 -0500 | [diff] [blame] | 1028 | err = iommu_device_sysfs_add(&obj->iommu, obj->dev, NULL, |
| 1029 | obj->name); |
| 1030 | if (err) |
| 1031 | goto out_group; |
Joerg Roedel | 01611fe | 2017-04-12 00:21:30 -0500 | [diff] [blame] | 1032 | |
Suman Anna | 9d5018d | 2017-09-05 17:56:18 -0500 | [diff] [blame] | 1033 | iommu_device_set_ops(&obj->iommu, &omap_iommu_ops); |
Joerg Roedel | 01611fe | 2017-04-12 00:21:30 -0500 | [diff] [blame] | 1034 | |
Suman Anna | 9d5018d | 2017-09-05 17:56:18 -0500 | [diff] [blame] | 1035 | err = iommu_device_register(&obj->iommu); |
| 1036 | if (err) |
| 1037 | goto out_sysfs; |
| 1038 | } |
Joerg Roedel | 01611fe | 2017-04-12 00:21:30 -0500 | [diff] [blame] | 1039 | |
Omar Ramirez Luna | ebf7cda | 2012-11-19 19:05:51 -0600 | [diff] [blame] | 1040 | pm_runtime_irq_safe(obj->dev); |
| 1041 | pm_runtime_enable(obj->dev); |
| 1042 | |
Suman Anna | 61c7535 | 2014-10-22 17:22:30 -0500 | [diff] [blame] | 1043 | omap_iommu_debugfs_add(obj); |
| 1044 | |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 1045 | dev_info(&pdev->dev, "%s registered\n", obj->name); |
Joerg Roedel | 28ae1e3 | 2017-04-12 00:21:31 -0500 | [diff] [blame] | 1046 | |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 1047 | return 0; |
Joerg Roedel | 01611fe | 2017-04-12 00:21:30 -0500 | [diff] [blame] | 1048 | |
| 1049 | out_sysfs: |
| 1050 | iommu_device_sysfs_remove(&obj->iommu); |
Joerg Roedel | 28ae1e3 | 2017-04-12 00:21:31 -0500 | [diff] [blame] | 1051 | out_group: |
| 1052 | iommu_group_put(obj->group); |
Joerg Roedel | 01611fe | 2017-04-12 00:21:30 -0500 | [diff] [blame] | 1053 | return err; |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 1054 | } |
| 1055 | |
Greg Kroah-Hartman | d34d651 | 2012-12-21 15:05:21 -0800 | [diff] [blame] | 1056 | static int omap_iommu_remove(struct platform_device *pdev) |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 1057 | { |
Ohad Ben-Cohen | 6c32df4 | 2011-08-17 22:57:56 +0300 | [diff] [blame] | 1058 | struct omap_iommu *obj = platform_get_drvdata(pdev); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 1059 | |
Suman Anna | 9d5018d | 2017-09-05 17:56:18 -0500 | [diff] [blame] | 1060 | if (obj->group) { |
| 1061 | iommu_group_put(obj->group); |
| 1062 | obj->group = NULL; |
Joerg Roedel | 28ae1e3 | 2017-04-12 00:21:31 -0500 | [diff] [blame] | 1063 | |
Suman Anna | 9d5018d | 2017-09-05 17:56:18 -0500 | [diff] [blame] | 1064 | iommu_device_sysfs_remove(&obj->iommu); |
| 1065 | iommu_device_unregister(&obj->iommu); |
| 1066 | } |
Joerg Roedel | 01611fe | 2017-04-12 00:21:30 -0500 | [diff] [blame] | 1067 | |
Suman Anna | 61c7535 | 2014-10-22 17:22:30 -0500 | [diff] [blame] | 1068 | omap_iommu_debugfs_remove(obj); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 1069 | |
Omar Ramirez Luna | ebf7cda | 2012-11-19 19:05:51 -0600 | [diff] [blame] | 1070 | pm_runtime_disable(obj->dev); |
| 1071 | |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 1072 | dev_info(&pdev->dev, "%s removed\n", obj->name); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 1073 | return 0; |
| 1074 | } |
| 1075 | |
Kiran Padwal | d943b0f | 2014-09-11 19:07:36 +0530 | [diff] [blame] | 1076 | static const struct of_device_id omap_iommu_of_match[] = { |
Florian Vaussard | 3c92748 | 2014-02-28 14:42:36 -0600 | [diff] [blame] | 1077 | { .compatible = "ti,omap2-iommu" }, |
| 1078 | { .compatible = "ti,omap4-iommu" }, |
| 1079 | { .compatible = "ti,dra7-iommu" }, |
Suman Anna | 3ca9299 | 2015-10-02 18:02:44 -0500 | [diff] [blame] | 1080 | { .compatible = "ti,dra7-dsp-iommu" }, |
Florian Vaussard | 3c92748 | 2014-02-28 14:42:36 -0600 | [diff] [blame] | 1081 | {}, |
| 1082 | }; |
Florian Vaussard | 3c92748 | 2014-02-28 14:42:36 -0600 | [diff] [blame] | 1083 | |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 1084 | static struct platform_driver omap_iommu_driver = { |
| 1085 | .probe = omap_iommu_probe, |
Greg Kroah-Hartman | d34d651 | 2012-12-21 15:05:21 -0800 | [diff] [blame] | 1086 | .remove = omap_iommu_remove, |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 1087 | .driver = { |
| 1088 | .name = "omap-iommu", |
Florian Vaussard | 3c92748 | 2014-02-28 14:42:36 -0600 | [diff] [blame] | 1089 | .of_match_table = of_match_ptr(omap_iommu_of_match), |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 1090 | }, |
| 1091 | }; |
| 1092 | |
Laurent Pinchart | 286f600 | 2014-03-08 00:44:38 +0100 | [diff] [blame] | 1093 | static u32 iotlb_init_entry(struct iotlb_entry *e, u32 da, u32 pa, int pgsz) |
Tony Lindgren | ed1c7de | 2012-11-02 12:24:06 -0700 | [diff] [blame] | 1094 | { |
| 1095 | memset(e, 0, sizeof(*e)); |
| 1096 | |
| 1097 | e->da = da; |
| 1098 | e->pa = pa; |
Suman Anna | d760e3e | 2014-03-17 20:31:32 -0500 | [diff] [blame] | 1099 | e->valid = MMU_CAM_V; |
Laurent Pinchart | 286f600 | 2014-03-08 00:44:38 +0100 | [diff] [blame] | 1100 | e->pgsz = pgsz; |
| 1101 | e->endian = MMU_RAM_ENDIAN_LITTLE; |
| 1102 | e->elsz = MMU_RAM_ELSZ_8; |
| 1103 | e->mixed = 0; |
Tony Lindgren | ed1c7de | 2012-11-02 12:24:06 -0700 | [diff] [blame] | 1104 | |
| 1105 | return iopgsz_to_bytes(e->pgsz); |
| 1106 | } |
| 1107 | |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 1108 | static int omap_iommu_map(struct iommu_domain *domain, unsigned long da, |
Suman Anna | 5835b6a | 2015-07-20 17:33:32 -0500 | [diff] [blame] | 1109 | phys_addr_t pa, size_t bytes, int prot) |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 1110 | { |
Joerg Roedel | 8cf851e | 2015-03-26 13:43:09 +0100 | [diff] [blame] | 1111 | struct omap_iommu_domain *omap_domain = to_omap_domain(domain); |
Suman Anna | 9d5018d | 2017-09-05 17:56:18 -0500 | [diff] [blame] | 1112 | struct device *dev = omap_domain->dev; |
| 1113 | struct omap_iommu_device *iommu; |
| 1114 | struct omap_iommu *oiommu; |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 1115 | struct iotlb_entry e; |
| 1116 | int omap_pgsz; |
Suman Anna | 9d5018d | 2017-09-05 17:56:18 -0500 | [diff] [blame] | 1117 | u32 ret = -EINVAL; |
| 1118 | int i; |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 1119 | |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 1120 | omap_pgsz = bytes_to_iopgsz(bytes); |
| 1121 | if (omap_pgsz < 0) { |
| 1122 | dev_err(dev, "invalid size to map: %d\n", bytes); |
| 1123 | return -EINVAL; |
| 1124 | } |
| 1125 | |
Joerg Roedel | 1d7f449 | 2015-01-22 14:42:06 +0100 | [diff] [blame] | 1126 | dev_dbg(dev, "mapping da 0x%lx to pa %pa size 0x%x\n", da, &pa, bytes); |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 1127 | |
Laurent Pinchart | 286f600 | 2014-03-08 00:44:38 +0100 | [diff] [blame] | 1128 | iotlb_init_entry(&e, da, pa, omap_pgsz); |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 1129 | |
Suman Anna | 9d5018d | 2017-09-05 17:56:18 -0500 | [diff] [blame] | 1130 | iommu = omap_domain->iommus; |
| 1131 | for (i = 0; i < omap_domain->num_iommus; i++, iommu++) { |
| 1132 | oiommu = iommu->iommu_dev; |
| 1133 | ret = omap_iopgtable_store_entry(oiommu, &e); |
| 1134 | if (ret) { |
| 1135 | dev_err(dev, "omap_iopgtable_store_entry failed: %d\n", |
| 1136 | ret); |
| 1137 | break; |
| 1138 | } |
| 1139 | } |
| 1140 | |
| 1141 | if (ret) { |
| 1142 | while (i--) { |
| 1143 | iommu--; |
| 1144 | oiommu = iommu->iommu_dev; |
| 1145 | iopgtable_clear_entry(oiommu, da); |
| 1146 | } |
| 1147 | } |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 1148 | |
Ohad Ben-Cohen | b4550d4 | 2011-09-02 13:32:31 -0400 | [diff] [blame] | 1149 | return ret; |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 1150 | } |
| 1151 | |
Ohad Ben-Cohen | 5009065 | 2011-11-10 11:32:25 +0200 | [diff] [blame] | 1152 | static size_t omap_iommu_unmap(struct iommu_domain *domain, unsigned long da, |
Suman Anna | 5835b6a | 2015-07-20 17:33:32 -0500 | [diff] [blame] | 1153 | size_t size) |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 1154 | { |
Joerg Roedel | 8cf851e | 2015-03-26 13:43:09 +0100 | [diff] [blame] | 1155 | struct omap_iommu_domain *omap_domain = to_omap_domain(domain); |
Suman Anna | 9d5018d | 2017-09-05 17:56:18 -0500 | [diff] [blame] | 1156 | struct device *dev = omap_domain->dev; |
| 1157 | struct omap_iommu_device *iommu; |
| 1158 | struct omap_iommu *oiommu; |
| 1159 | bool error = false; |
| 1160 | size_t bytes = 0; |
| 1161 | int i; |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 1162 | |
Ohad Ben-Cohen | 5009065 | 2011-11-10 11:32:25 +0200 | [diff] [blame] | 1163 | dev_dbg(dev, "unmapping da 0x%lx size %u\n", da, size); |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 1164 | |
Suman Anna | 9d5018d | 2017-09-05 17:56:18 -0500 | [diff] [blame] | 1165 | iommu = omap_domain->iommus; |
| 1166 | for (i = 0; i < omap_domain->num_iommus; i++, iommu++) { |
| 1167 | oiommu = iommu->iommu_dev; |
| 1168 | bytes = iopgtable_clear_entry(oiommu, da); |
| 1169 | if (!bytes) |
| 1170 | error = true; |
| 1171 | } |
| 1172 | |
| 1173 | /* |
| 1174 | * simplify return - we are only checking if any of the iommus |
| 1175 | * reported an error, but not if all of them are unmapping the |
| 1176 | * same number of entries. This should not occur due to the |
| 1177 | * mirror programming. |
| 1178 | */ |
| 1179 | return error ? 0 : bytes; |
| 1180 | } |
| 1181 | |
| 1182 | static int omap_iommu_count(struct device *dev) |
| 1183 | { |
| 1184 | struct omap_iommu_arch_data *arch_data = dev->archdata.iommu; |
| 1185 | int count = 0; |
| 1186 | |
| 1187 | while (arch_data->iommu_dev) { |
| 1188 | count++; |
| 1189 | arch_data++; |
| 1190 | } |
| 1191 | |
| 1192 | return count; |
| 1193 | } |
| 1194 | |
| 1195 | /* caller should call cleanup if this function fails */ |
| 1196 | static int omap_iommu_attach_init(struct device *dev, |
| 1197 | struct omap_iommu_domain *odomain) |
| 1198 | { |
| 1199 | struct omap_iommu_device *iommu; |
| 1200 | int i; |
| 1201 | |
| 1202 | odomain->num_iommus = omap_iommu_count(dev); |
| 1203 | if (!odomain->num_iommus) |
| 1204 | return -EINVAL; |
| 1205 | |
| 1206 | odomain->iommus = kcalloc(odomain->num_iommus, sizeof(*iommu), |
| 1207 | GFP_ATOMIC); |
| 1208 | if (!odomain->iommus) |
| 1209 | return -ENOMEM; |
| 1210 | |
| 1211 | iommu = odomain->iommus; |
| 1212 | for (i = 0; i < odomain->num_iommus; i++, iommu++) { |
| 1213 | iommu->pgtable = kzalloc(IOPGD_TABLE_SIZE, GFP_ATOMIC); |
| 1214 | if (!iommu->pgtable) |
| 1215 | return -ENOMEM; |
| 1216 | |
| 1217 | /* |
| 1218 | * should never fail, but please keep this around to ensure |
| 1219 | * we keep the hardware happy |
| 1220 | */ |
| 1221 | if (WARN_ON(!IS_ALIGNED((long)iommu->pgtable, |
| 1222 | IOPGD_TABLE_SIZE))) |
| 1223 | return -EINVAL; |
| 1224 | } |
| 1225 | |
| 1226 | return 0; |
| 1227 | } |
| 1228 | |
| 1229 | static void omap_iommu_detach_fini(struct omap_iommu_domain *odomain) |
| 1230 | { |
| 1231 | int i; |
| 1232 | struct omap_iommu_device *iommu = odomain->iommus; |
| 1233 | |
| 1234 | for (i = 0; iommu && i < odomain->num_iommus; i++, iommu++) |
| 1235 | kfree(iommu->pgtable); |
| 1236 | |
| 1237 | kfree(odomain->iommus); |
| 1238 | odomain->num_iommus = 0; |
| 1239 | odomain->iommus = NULL; |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 1240 | } |
| 1241 | |
| 1242 | static int |
| 1243 | omap_iommu_attach_dev(struct iommu_domain *domain, struct device *dev) |
| 1244 | { |
Joerg Roedel | 8cf851e | 2015-03-26 13:43:09 +0100 | [diff] [blame] | 1245 | struct omap_iommu_domain *omap_domain = to_omap_domain(domain); |
Ohad Ben-Cohen | fabdbca | 2011-10-11 00:18:33 +0200 | [diff] [blame] | 1246 | struct omap_iommu_arch_data *arch_data = dev->archdata.iommu; |
Suman Anna | 9d5018d | 2017-09-05 17:56:18 -0500 | [diff] [blame] | 1247 | struct omap_iommu_device *iommu; |
Joerg Roedel | ede1c2e | 2017-04-12 00:21:29 -0500 | [diff] [blame] | 1248 | struct omap_iommu *oiommu; |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 1249 | int ret = 0; |
Suman Anna | 9d5018d | 2017-09-05 17:56:18 -0500 | [diff] [blame] | 1250 | int i; |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 1251 | |
Joerg Roedel | ede1c2e | 2017-04-12 00:21:29 -0500 | [diff] [blame] | 1252 | if (!arch_data || !arch_data->iommu_dev) { |
Suman Anna | e3f595b | 2014-09-04 17:27:29 -0500 | [diff] [blame] | 1253 | dev_err(dev, "device doesn't have an associated iommu\n"); |
| 1254 | return -EINVAL; |
| 1255 | } |
| 1256 | |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 1257 | spin_lock(&omap_domain->lock); |
| 1258 | |
Suman Anna | 0d36428 | 2017-09-05 17:56:17 -0500 | [diff] [blame] | 1259 | /* only a single client device can be attached to a domain */ |
| 1260 | if (omap_domain->dev) { |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 1261 | dev_err(dev, "iommu domain is already attached\n"); |
| 1262 | ret = -EBUSY; |
| 1263 | goto out; |
| 1264 | } |
| 1265 | |
Suman Anna | 9d5018d | 2017-09-05 17:56:18 -0500 | [diff] [blame] | 1266 | ret = omap_iommu_attach_init(dev, omap_domain); |
Joerg Roedel | ede1c2e | 2017-04-12 00:21:29 -0500 | [diff] [blame] | 1267 | if (ret) { |
Suman Anna | 9d5018d | 2017-09-05 17:56:18 -0500 | [diff] [blame] | 1268 | dev_err(dev, "failed to allocate required iommu data %d\n", |
| 1269 | ret); |
| 1270 | goto init_fail; |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 1271 | } |
| 1272 | |
Suman Anna | 9d5018d | 2017-09-05 17:56:18 -0500 | [diff] [blame] | 1273 | iommu = omap_domain->iommus; |
| 1274 | for (i = 0; i < omap_domain->num_iommus; i++, iommu++, arch_data++) { |
| 1275 | /* configure and enable the omap iommu */ |
| 1276 | oiommu = arch_data->iommu_dev; |
| 1277 | ret = omap_iommu_attach(oiommu, iommu->pgtable); |
| 1278 | if (ret) { |
| 1279 | dev_err(dev, "can't get omap iommu: %d\n", ret); |
| 1280 | goto attach_fail; |
| 1281 | } |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 1282 | |
Suman Anna | 9d5018d | 2017-09-05 17:56:18 -0500 | [diff] [blame] | 1283 | oiommu->domain = domain; |
| 1284 | iommu->iommu_dev = oiommu; |
| 1285 | } |
| 1286 | |
| 1287 | omap_domain->dev = dev; |
| 1288 | |
| 1289 | goto out; |
| 1290 | |
| 1291 | attach_fail: |
| 1292 | while (i--) { |
| 1293 | iommu--; |
| 1294 | arch_data--; |
| 1295 | oiommu = iommu->iommu_dev; |
| 1296 | omap_iommu_detach(oiommu); |
| 1297 | iommu->iommu_dev = NULL; |
| 1298 | oiommu->domain = NULL; |
| 1299 | } |
| 1300 | init_fail: |
| 1301 | omap_iommu_detach_fini(omap_domain); |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 1302 | out: |
| 1303 | spin_unlock(&omap_domain->lock); |
| 1304 | return ret; |
| 1305 | } |
| 1306 | |
Omar Ramirez Luna | 803b527 | 2012-04-18 13:09:41 -0500 | [diff] [blame] | 1307 | static void _omap_iommu_detach_dev(struct omap_iommu_domain *omap_domain, |
Suman Anna | 5835b6a | 2015-07-20 17:33:32 -0500 | [diff] [blame] | 1308 | struct device *dev) |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 1309 | { |
Suman Anna | 9d5018d | 2017-09-05 17:56:18 -0500 | [diff] [blame] | 1310 | struct omap_iommu_arch_data *arch_data = dev->archdata.iommu; |
| 1311 | struct omap_iommu_device *iommu = omap_domain->iommus; |
| 1312 | struct omap_iommu *oiommu; |
| 1313 | int i; |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 1314 | |
Suman Anna | 0d36428 | 2017-09-05 17:56:17 -0500 | [diff] [blame] | 1315 | if (!omap_domain->dev) { |
| 1316 | dev_err(dev, "domain has no attached device\n"); |
| 1317 | return; |
| 1318 | } |
| 1319 | |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 1320 | /* only a single device is supported per domain for now */ |
Suman Anna | 0d36428 | 2017-09-05 17:56:17 -0500 | [diff] [blame] | 1321 | if (omap_domain->dev != dev) { |
| 1322 | dev_err(dev, "invalid attached device\n"); |
Omar Ramirez Luna | 803b527 | 2012-04-18 13:09:41 -0500 | [diff] [blame] | 1323 | return; |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 1324 | } |
| 1325 | |
Suman Anna | 9d5018d | 2017-09-05 17:56:18 -0500 | [diff] [blame] | 1326 | /* |
| 1327 | * cleanup in the reverse order of attachment - this addresses |
| 1328 | * any h/w dependencies between multiple instances, if any |
| 1329 | */ |
| 1330 | iommu += (omap_domain->num_iommus - 1); |
| 1331 | arch_data += (omap_domain->num_iommus - 1); |
| 1332 | for (i = 0; i < omap_domain->num_iommus; i++, iommu--, arch_data--) { |
| 1333 | oiommu = iommu->iommu_dev; |
| 1334 | iopgtable_clear_entry_all(oiommu); |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 1335 | |
Suman Anna | 9d5018d | 2017-09-05 17:56:18 -0500 | [diff] [blame] | 1336 | omap_iommu_detach(oiommu); |
| 1337 | iommu->iommu_dev = NULL; |
| 1338 | oiommu->domain = NULL; |
| 1339 | } |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 1340 | |
Suman Anna | 9d5018d | 2017-09-05 17:56:18 -0500 | [diff] [blame] | 1341 | omap_iommu_detach_fini(omap_domain); |
| 1342 | |
Omar Ramirez Luna | 803b527 | 2012-04-18 13:09:41 -0500 | [diff] [blame] | 1343 | omap_domain->dev = NULL; |
| 1344 | } |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 1345 | |
Omar Ramirez Luna | 803b527 | 2012-04-18 13:09:41 -0500 | [diff] [blame] | 1346 | static void omap_iommu_detach_dev(struct iommu_domain *domain, |
Suman Anna | 5835b6a | 2015-07-20 17:33:32 -0500 | [diff] [blame] | 1347 | struct device *dev) |
Omar Ramirez Luna | 803b527 | 2012-04-18 13:09:41 -0500 | [diff] [blame] | 1348 | { |
Joerg Roedel | 8cf851e | 2015-03-26 13:43:09 +0100 | [diff] [blame] | 1349 | struct omap_iommu_domain *omap_domain = to_omap_domain(domain); |
Omar Ramirez Luna | 803b527 | 2012-04-18 13:09:41 -0500 | [diff] [blame] | 1350 | |
| 1351 | spin_lock(&omap_domain->lock); |
| 1352 | _omap_iommu_detach_dev(omap_domain, dev); |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 1353 | spin_unlock(&omap_domain->lock); |
| 1354 | } |
| 1355 | |
Joerg Roedel | 8cf851e | 2015-03-26 13:43:09 +0100 | [diff] [blame] | 1356 | static struct iommu_domain *omap_iommu_domain_alloc(unsigned type) |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 1357 | { |
| 1358 | struct omap_iommu_domain *omap_domain; |
| 1359 | |
Joerg Roedel | 8cf851e | 2015-03-26 13:43:09 +0100 | [diff] [blame] | 1360 | if (type != IOMMU_DOMAIN_UNMANAGED) |
| 1361 | return NULL; |
| 1362 | |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 1363 | omap_domain = kzalloc(sizeof(*omap_domain), GFP_KERNEL); |
Suman Anna | 99ee98d | 2015-07-20 17:33:29 -0500 | [diff] [blame] | 1364 | if (!omap_domain) |
Suman Anna | 9d5018d | 2017-09-05 17:56:18 -0500 | [diff] [blame] | 1365 | return NULL; |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 1366 | |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 1367 | spin_lock_init(&omap_domain->lock); |
| 1368 | |
Joerg Roedel | 8cf851e | 2015-03-26 13:43:09 +0100 | [diff] [blame] | 1369 | omap_domain->domain.geometry.aperture_start = 0; |
| 1370 | omap_domain->domain.geometry.aperture_end = (1ULL << 32) - 1; |
| 1371 | omap_domain->domain.geometry.force_aperture = true; |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 1372 | |
Joerg Roedel | 8cf851e | 2015-03-26 13:43:09 +0100 | [diff] [blame] | 1373 | return &omap_domain->domain; |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 1374 | } |
| 1375 | |
Joerg Roedel | 8cf851e | 2015-03-26 13:43:09 +0100 | [diff] [blame] | 1376 | static void omap_iommu_domain_free(struct iommu_domain *domain) |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 1377 | { |
Joerg Roedel | 8cf851e | 2015-03-26 13:43:09 +0100 | [diff] [blame] | 1378 | struct omap_iommu_domain *omap_domain = to_omap_domain(domain); |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 1379 | |
Omar Ramirez Luna | 803b527 | 2012-04-18 13:09:41 -0500 | [diff] [blame] | 1380 | /* |
| 1381 | * An iommu device is still attached |
| 1382 | * (currently, only one device can be attached) ? |
| 1383 | */ |
Suman Anna | 0d36428 | 2017-09-05 17:56:17 -0500 | [diff] [blame] | 1384 | if (omap_domain->dev) |
Omar Ramirez Luna | 803b527 | 2012-04-18 13:09:41 -0500 | [diff] [blame] | 1385 | _omap_iommu_detach_dev(omap_domain, omap_domain->dev); |
| 1386 | |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 1387 | kfree(omap_domain); |
| 1388 | } |
| 1389 | |
| 1390 | static phys_addr_t omap_iommu_iova_to_phys(struct iommu_domain *domain, |
Suman Anna | 5835b6a | 2015-07-20 17:33:32 -0500 | [diff] [blame] | 1391 | dma_addr_t da) |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 1392 | { |
Joerg Roedel | 8cf851e | 2015-03-26 13:43:09 +0100 | [diff] [blame] | 1393 | struct omap_iommu_domain *omap_domain = to_omap_domain(domain); |
Suman Anna | 9d5018d | 2017-09-05 17:56:18 -0500 | [diff] [blame] | 1394 | struct omap_iommu_device *iommu = omap_domain->iommus; |
| 1395 | struct omap_iommu *oiommu = iommu->iommu_dev; |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 1396 | struct device *dev = oiommu->dev; |
| 1397 | u32 *pgd, *pte; |
| 1398 | phys_addr_t ret = 0; |
| 1399 | |
Suman Anna | 9d5018d | 2017-09-05 17:56:18 -0500 | [diff] [blame] | 1400 | /* |
| 1401 | * all the iommus within the domain will have identical programming, |
| 1402 | * so perform the lookup using just the first iommu |
| 1403 | */ |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 1404 | iopgtable_lookup_entry(oiommu, da, &pgd, &pte); |
| 1405 | |
| 1406 | if (pte) { |
| 1407 | if (iopte_is_small(*pte)) |
| 1408 | ret = omap_iommu_translate(*pte, da, IOPTE_MASK); |
| 1409 | else if (iopte_is_large(*pte)) |
| 1410 | ret = omap_iommu_translate(*pte, da, IOLARGE_MASK); |
| 1411 | else |
Suman Anna | 2abfcfb | 2013-05-30 18:10:38 -0500 | [diff] [blame] | 1412 | dev_err(dev, "bogus pte 0x%x, da 0x%llx", *pte, |
Suman Anna | 5835b6a | 2015-07-20 17:33:32 -0500 | [diff] [blame] | 1413 | (unsigned long long)da); |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 1414 | } else { |
| 1415 | if (iopgd_is_section(*pgd)) |
| 1416 | ret = omap_iommu_translate(*pgd, da, IOSECTION_MASK); |
| 1417 | else if (iopgd_is_super(*pgd)) |
| 1418 | ret = omap_iommu_translate(*pgd, da, IOSUPER_MASK); |
| 1419 | else |
Suman Anna | 2abfcfb | 2013-05-30 18:10:38 -0500 | [diff] [blame] | 1420 | dev_err(dev, "bogus pgd 0x%x, da 0x%llx", *pgd, |
Suman Anna | 5835b6a | 2015-07-20 17:33:32 -0500 | [diff] [blame] | 1421 | (unsigned long long)da); |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 1422 | } |
| 1423 | |
| 1424 | return ret; |
| 1425 | } |
| 1426 | |
Laurent Pinchart | 07a0203 | 2014-02-28 14:42:38 -0600 | [diff] [blame] | 1427 | static int omap_iommu_add_device(struct device *dev) |
| 1428 | { |
Suman Anna | 9d5018d | 2017-09-05 17:56:18 -0500 | [diff] [blame] | 1429 | struct omap_iommu_arch_data *arch_data, *tmp; |
Joerg Roedel | ede1c2e | 2017-04-12 00:21:29 -0500 | [diff] [blame] | 1430 | struct omap_iommu *oiommu; |
Joerg Roedel | 28ae1e3 | 2017-04-12 00:21:31 -0500 | [diff] [blame] | 1431 | struct iommu_group *group; |
Laurent Pinchart | 07a0203 | 2014-02-28 14:42:38 -0600 | [diff] [blame] | 1432 | struct device_node *np; |
Suman Anna | 7d68277 | 2014-09-04 17:27:30 -0500 | [diff] [blame] | 1433 | struct platform_device *pdev; |
Suman Anna | 9d5018d | 2017-09-05 17:56:18 -0500 | [diff] [blame] | 1434 | int num_iommus, i; |
Joerg Roedel | 01611fe | 2017-04-12 00:21:30 -0500 | [diff] [blame] | 1435 | int ret; |
Laurent Pinchart | 07a0203 | 2014-02-28 14:42:38 -0600 | [diff] [blame] | 1436 | |
| 1437 | /* |
| 1438 | * Allocate the archdata iommu structure for DT-based devices. |
| 1439 | * |
| 1440 | * TODO: Simplify this when removing non-DT support completely from the |
| 1441 | * IOMMU users. |
| 1442 | */ |
| 1443 | if (!dev->of_node) |
| 1444 | return 0; |
| 1445 | |
Suman Anna | 9d5018d | 2017-09-05 17:56:18 -0500 | [diff] [blame] | 1446 | /* |
| 1447 | * retrieve the count of IOMMU nodes using phandle size as element size |
| 1448 | * since #iommu-cells = 0 for OMAP |
| 1449 | */ |
| 1450 | num_iommus = of_property_count_elems_of_size(dev->of_node, "iommus", |
| 1451 | sizeof(phandle)); |
| 1452 | if (num_iommus < 0) |
Laurent Pinchart | 07a0203 | 2014-02-28 14:42:38 -0600 | [diff] [blame] | 1453 | return 0; |
| 1454 | |
Kees Cook | 6396bb2 | 2018-06-12 14:03:40 -0700 | [diff] [blame] | 1455 | arch_data = kcalloc(num_iommus + 1, sizeof(*arch_data), GFP_KERNEL); |
Suman Anna | 9d5018d | 2017-09-05 17:56:18 -0500 | [diff] [blame] | 1456 | if (!arch_data) |
Laurent Pinchart | 07a0203 | 2014-02-28 14:42:38 -0600 | [diff] [blame] | 1457 | return -ENOMEM; |
Suman Anna | 9d5018d | 2017-09-05 17:56:18 -0500 | [diff] [blame] | 1458 | |
| 1459 | for (i = 0, tmp = arch_data; i < num_iommus; i++, tmp++) { |
| 1460 | np = of_parse_phandle(dev->of_node, "iommus", i); |
| 1461 | if (!np) { |
| 1462 | kfree(arch_data); |
| 1463 | return -EINVAL; |
| 1464 | } |
| 1465 | |
| 1466 | pdev = of_find_device_by_node(np); |
| 1467 | if (WARN_ON(!pdev)) { |
| 1468 | of_node_put(np); |
| 1469 | kfree(arch_data); |
| 1470 | return -EINVAL; |
| 1471 | } |
| 1472 | |
| 1473 | oiommu = platform_get_drvdata(pdev); |
| 1474 | if (!oiommu) { |
| 1475 | of_node_put(np); |
| 1476 | kfree(arch_data); |
| 1477 | return -EINVAL; |
| 1478 | } |
| 1479 | |
| 1480 | tmp->iommu_dev = oiommu; |
| 1481 | |
| 1482 | of_node_put(np); |
Laurent Pinchart | 07a0203 | 2014-02-28 14:42:38 -0600 | [diff] [blame] | 1483 | } |
| 1484 | |
Suman Anna | 9d5018d | 2017-09-05 17:56:18 -0500 | [diff] [blame] | 1485 | /* |
| 1486 | * use the first IOMMU alone for the sysfs device linking. |
| 1487 | * TODO: Evaluate if a single iommu_group needs to be |
| 1488 | * maintained for both IOMMUs |
| 1489 | */ |
| 1490 | oiommu = arch_data->iommu_dev; |
Joerg Roedel | 01611fe | 2017-04-12 00:21:30 -0500 | [diff] [blame] | 1491 | ret = iommu_device_link(&oiommu->iommu, dev); |
| 1492 | if (ret) { |
| 1493 | kfree(arch_data); |
Joerg Roedel | 01611fe | 2017-04-12 00:21:30 -0500 | [diff] [blame] | 1494 | return ret; |
| 1495 | } |
| 1496 | |
Laurent Pinchart | 07a0203 | 2014-02-28 14:42:38 -0600 | [diff] [blame] | 1497 | dev->archdata.iommu = arch_data; |
| 1498 | |
Joerg Roedel | 28ae1e3 | 2017-04-12 00:21:31 -0500 | [diff] [blame] | 1499 | /* |
| 1500 | * IOMMU group initialization calls into omap_iommu_device_group, which |
| 1501 | * needs a valid dev->archdata.iommu pointer |
| 1502 | */ |
| 1503 | group = iommu_group_get_for_dev(dev); |
| 1504 | if (IS_ERR(group)) { |
| 1505 | iommu_device_unlink(&oiommu->iommu, dev); |
| 1506 | dev->archdata.iommu = NULL; |
| 1507 | kfree(arch_data); |
| 1508 | return PTR_ERR(group); |
| 1509 | } |
| 1510 | iommu_group_put(group); |
| 1511 | |
Laurent Pinchart | 07a0203 | 2014-02-28 14:42:38 -0600 | [diff] [blame] | 1512 | return 0; |
| 1513 | } |
| 1514 | |
| 1515 | static void omap_iommu_remove_device(struct device *dev) |
| 1516 | { |
| 1517 | struct omap_iommu_arch_data *arch_data = dev->archdata.iommu; |
| 1518 | |
| 1519 | if (!dev->of_node || !arch_data) |
| 1520 | return; |
| 1521 | |
Joerg Roedel | 01611fe | 2017-04-12 00:21:30 -0500 | [diff] [blame] | 1522 | iommu_device_unlink(&arch_data->iommu_dev->iommu, dev); |
Joerg Roedel | 28ae1e3 | 2017-04-12 00:21:31 -0500 | [diff] [blame] | 1523 | iommu_group_remove_device(dev); |
Joerg Roedel | 01611fe | 2017-04-12 00:21:30 -0500 | [diff] [blame] | 1524 | |
Joerg Roedel | ede1c2e | 2017-04-12 00:21:29 -0500 | [diff] [blame] | 1525 | dev->archdata.iommu = NULL; |
Laurent Pinchart | 07a0203 | 2014-02-28 14:42:38 -0600 | [diff] [blame] | 1526 | kfree(arch_data); |
Joerg Roedel | 01611fe | 2017-04-12 00:21:30 -0500 | [diff] [blame] | 1527 | |
Laurent Pinchart | 07a0203 | 2014-02-28 14:42:38 -0600 | [diff] [blame] | 1528 | } |
| 1529 | |
Joerg Roedel | 28ae1e3 | 2017-04-12 00:21:31 -0500 | [diff] [blame] | 1530 | static struct iommu_group *omap_iommu_device_group(struct device *dev) |
| 1531 | { |
| 1532 | struct omap_iommu_arch_data *arch_data = dev->archdata.iommu; |
Joerg Roedel | 8faf5e5 | 2017-06-28 12:50:16 +0200 | [diff] [blame] | 1533 | struct iommu_group *group = ERR_PTR(-EINVAL); |
Joerg Roedel | 28ae1e3 | 2017-04-12 00:21:31 -0500 | [diff] [blame] | 1534 | |
| 1535 | if (arch_data->iommu_dev) |
Jeffy Chen | b6d57f1 | 2018-03-01 19:22:08 +0800 | [diff] [blame] | 1536 | group = iommu_group_ref_get(arch_data->iommu_dev->group); |
Joerg Roedel | 28ae1e3 | 2017-04-12 00:21:31 -0500 | [diff] [blame] | 1537 | |
| 1538 | return group; |
| 1539 | } |
| 1540 | |
Thierry Reding | b22f643 | 2014-06-27 09:03:12 +0200 | [diff] [blame] | 1541 | static const struct iommu_ops omap_iommu_ops = { |
Joerg Roedel | 8cf851e | 2015-03-26 13:43:09 +0100 | [diff] [blame] | 1542 | .domain_alloc = omap_iommu_domain_alloc, |
| 1543 | .domain_free = omap_iommu_domain_free, |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 1544 | .attach_dev = omap_iommu_attach_dev, |
| 1545 | .detach_dev = omap_iommu_detach_dev, |
| 1546 | .map = omap_iommu_map, |
| 1547 | .unmap = omap_iommu_unmap, |
| 1548 | .iova_to_phys = omap_iommu_iova_to_phys, |
Laurent Pinchart | 07a0203 | 2014-02-28 14:42:38 -0600 | [diff] [blame] | 1549 | .add_device = omap_iommu_add_device, |
| 1550 | .remove_device = omap_iommu_remove_device, |
Joerg Roedel | 28ae1e3 | 2017-04-12 00:21:31 -0500 | [diff] [blame] | 1551 | .device_group = omap_iommu_device_group, |
Ohad Ben-Cohen | 66bc8cf | 2011-11-10 11:32:27 +0200 | [diff] [blame] | 1552 | .pgsize_bitmap = OMAP_IOMMU_PGSIZES, |
Ohad Ben-Cohen | f626b52 | 2011-06-02 01:46:12 +0300 | [diff] [blame] | 1553 | }; |
| 1554 | |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 1555 | static int __init omap_iommu_init(void) |
| 1556 | { |
| 1557 | struct kmem_cache *p; |
| 1558 | const unsigned long flags = SLAB_HWCACHE_ALIGN; |
| 1559 | size_t align = 1 << 10; /* L2 pagetable alignement */ |
Thierry Reding | f938aab | 2015-02-06 11:44:06 +0100 | [diff] [blame] | 1560 | struct device_node *np; |
Suman Anna | abaa7e5 | 2017-04-12 00:21:26 -0500 | [diff] [blame] | 1561 | int ret; |
Thierry Reding | f938aab | 2015-02-06 11:44:06 +0100 | [diff] [blame] | 1562 | |
| 1563 | np = of_find_matching_node(NULL, omap_iommu_of_match); |
| 1564 | if (!np) |
| 1565 | return 0; |
| 1566 | |
| 1567 | of_node_put(np); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 1568 | |
| 1569 | p = kmem_cache_create("iopte_cache", IOPTE_TABLE_SIZE, align, flags, |
Josue Albarran | bfee0cf | 2017-07-28 15:49:14 -0500 | [diff] [blame] | 1570 | NULL); |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 1571 | if (!p) |
| 1572 | return -ENOMEM; |
| 1573 | iopte_cachep = p; |
| 1574 | |
Suman Anna | 61c7535 | 2014-10-22 17:22:30 -0500 | [diff] [blame] | 1575 | omap_iommu_debugfs_init(); |
| 1576 | |
Suman Anna | abaa7e5 | 2017-04-12 00:21:26 -0500 | [diff] [blame] | 1577 | ret = platform_driver_register(&omap_iommu_driver); |
| 1578 | if (ret) { |
| 1579 | pr_err("%s: failed to register driver\n", __func__); |
| 1580 | goto fail_driver; |
| 1581 | } |
| 1582 | |
| 1583 | ret = bus_set_iommu(&platform_bus_type, &omap_iommu_ops); |
| 1584 | if (ret) |
| 1585 | goto fail_bus; |
| 1586 | |
| 1587 | return 0; |
| 1588 | |
| 1589 | fail_bus: |
| 1590 | platform_driver_unregister(&omap_iommu_driver); |
| 1591 | fail_driver: |
| 1592 | kmem_cache_destroy(iopte_cachep); |
| 1593 | return ret; |
Hiroshi DOYU | a9dcad5 | 2009-01-26 15:13:40 +0200 | [diff] [blame] | 1594 | } |
Ohad Ben-Cohen | 435792d | 2012-02-26 12:14:14 +0200 | [diff] [blame] | 1595 | subsys_initcall(omap_iommu_init); |
Suman Anna | 0cdbf72 | 2015-07-20 17:33:24 -0500 | [diff] [blame] | 1596 | /* must be ready before omap3isp is probed */ |