// SPDX-License-Identifier: GPL-2.0 /* * iommu.c: Generic sw64 IOMMU support * * This is designed and tested for 3231. If there are no changes in hardware * in later chips, then it should work just as well. * */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "sunway_iommu.h" #define MAX_DOMAIN_NUM 65536 #define IOVA_PFN(addr) ((addr) >> PAGE_SHIFT) #define SW64_32BIT_DMA_LIMIT (0xe0000000 - 1) #define SW64_64BIT_DMA_LIMIT ((1UL << 41) - 1) #define SW64_BAR_ADDRESS (IO_BASE | PCI_BASE) #define SW64_IOMMU_PGSIZES (((1ULL) << PAGE_SHIFT) \ | ((1ULL) << PAGE_8M_SHIFT) \ | ((1ULL) << PAGE_512M_SHIFT) \ | ((1ULL) << PAGE_8G_SHIFT)) #define IDENTMAP_ALL ((1U) << 0) #define DMA_MASK64 ((1U) << 1) #define PTE_VALID 0x8000000000000000UL #define LAST_STAGE 0x100UL #define PTE_GRN_8M 0x10UL #define PTE_GRN_512M 0x20UL #define PTE_GRN_8G 0x30UL #define PTE_WRITEE 0x2UL #define PTE_READE 0x1UL #define PTE_RWE 0x3UL #define PTE_FLAGS_MASK 0x8000000000000133UL #define PAGE_8G_OFFSET_MASK ((1UL << PAGE_8G_SHIFT) - 1) #define PAGE_512M_OFFSET_MASK ((1UL << PAGE_512M_SHIFT) - 1) #define PAGE_8M_OFFSET_MASK ((1UL << PAGE_8M_SHIFT) - 1) /* IOMMU Exceptional Status */ enum exceptype { DTE_LEVEL1 = 0x0, DTE_LEVEL2, PTE_LEVEL1, PTE_LEVEL2, PTE_LEVEL3, UNAUTHORIZED_ACCESS, ILLEGAL_RESPONSE, DTE_LEVEL1_VAL, DTE_LEVEL2_VAL, PTE_LEVEL1_VAL, PTE_LEVEL2_VAL, PTE_LEVEL3_VAL, }; u64 iommu_enable_cmd; /* default IOMMU boot param: 0 */ unsigned long *sunway_iommu_domain_bitmap; static DEFINE_SPINLOCK(domain_bitmap_lock); static DEFINE_SPINLOCK(sunway_iommu_device_table_lock); spinlock_t sunway_domain_lock; static LLIST_HEAD(dev_data_list); LIST_HEAD(sunway_domain_list); struct dma_domain { struct sunway_iommu_domain sdomain; struct iova_domain iovad; }; const struct iommu_ops sunway_iommu_ops; static const struct dma_map_ops sunway_dma_ops; /* flush helpers */ static void piu_flush_all(struct pci_controller *hose) { write_piu_ior0(hose->node, hose->index, DTLB_FLUSHALL, 0); write_piu_ior0(hose->node, hose->index, PTLB_FLUSHALL, 0); write_piu_ior0(hose->node, hose->index, PCACHE_FLUSHALL, 0); } void flush_pcache_by_addr(struct sunway_iommu_domain *sdomain, unsigned long flush_addr) { struct pci_controller *hose; struct sunway_iommu_dev *sdev; list_for_each_entry(sdev, &sdomain->dev_list, list) { hose = pci_bus_to_pci_controller(sdev->pdev->bus); flush_addr = __pa(flush_addr); /* Set memory bar here */ mb(); write_piu_ior0(hose->node, hose->index, PCACHE_FLUSHPADDR, flush_addr); } } void flush_ptlb_by_addr(struct sunway_iommu_domain *sdomain, unsigned long flush_addr) { struct pci_controller *hose; struct sunway_iommu_dev *sdev; struct pci_dev *pdev; list_for_each_entry(sdev, &sdomain->dev_list, list) { pdev = sdev->pdev; hose = pci_bus_to_pci_controller(pdev->bus); flush_addr = (pdev->bus->number << 8) | pdev->devfn | (flush_addr << 16); write_piu_ior0(hose->node, hose->index, PTLB_FLUSHVADDR, flush_addr); } } /* domain helpers */ static struct sunway_iommu_domain *to_sunway_domain(struct iommu_domain *dom) { return container_of(dom, struct sunway_iommu_domain, domain); } static struct dma_domain *to_dma_domain(struct sunway_iommu_domain *sdomain) { return container_of(sdomain, struct dma_domain, sdomain); } static void add_domain_to_list(struct sunway_iommu_domain *sdomain) { unsigned long flags; spin_lock_irqsave(&sunway_domain_lock, flags); list_add(&sdomain->list, &sunway_domain_list); spin_unlock_irqrestore(&sunway_domain_lock, flags); } static void del_domain_from_list(struct sunway_iommu_domain *sdomain) { unsigned long flags; spin_lock_irqsave(&sunway_domain_lock, flags); list_del(&sdomain->list); spin_unlock_irqrestore(&sunway_domain_lock, flags); } static void free_pagetable(struct sunway_iommu_domain *sdomain) { unsigned long *l2_pte, *l3_pte; unsigned long l2_pte_val, l3_pte_val; int l2_index, l3_index, ptes_one_page; l2_pte = sdomain->pt_root; if (!l2_pte) return; ptes_one_page = PAGE_SIZE/sizeof(unsigned long); for (l2_index = 0; l2_index < ptes_one_page; l2_index++, l2_pte++) { l2_pte_val = *l2_pte; if ((l2_pte_val & SW64_IOMMU_ENTRY_VALID) == 0) continue; l2_pte_val &= ~(SW64_IOMMU_ENTRY_VALID) & PAGE_MASK; l2_pte_val |= PAGE_OFFSET; l3_pte = (unsigned long *)l2_pte_val; for (l3_index = 0; l3_index < ptes_one_page; l3_index++, l3_pte++) { l3_pte_val = *l3_pte; if ((l3_pte_val & SW64_IOMMU_ENTRY_VALID) == 0) continue; l3_pte_val &= ~(SW64_IOMMU_ENTRY_VALID) & PAGE_MASK; l3_pte_val |= PAGE_OFFSET; free_page(l3_pte_val); } free_page(l2_pte_val); } free_page((unsigned long)sdomain->pt_root); } static void domain_id_free(int id) { spin_lock(&domain_bitmap_lock); if (id > 0) __clear_bit(id, sunway_iommu_domain_bitmap); spin_unlock(&domain_bitmap_lock); } static void dma_domain_free(struct dma_domain *dma_dom) { if (!dma_dom) return; del_domain_from_list(&dma_dom->sdomain); put_iova_domain(&dma_dom->iovad); free_pagetable(&dma_dom->sdomain); if (dma_dom->sdomain.id) domain_id_free(dma_dom->sdomain.id); kfree(dma_dom); } static void sunway_domain_free(struct sunway_iommu_domain *sdomain) { if (!sdomain) return; del_domain_from_list(sdomain); if (sdomain->id) domain_id_free(sdomain->id); kfree(sdomain); } static u16 sunway_domain_id_alloc(void) { int id; spin_lock(&domain_bitmap_lock); id = find_first_zero_bit(sunway_iommu_domain_bitmap, MAX_DOMAIN_NUM); if (id > 0 && id < MAX_DOMAIN_NUM) __set_bit(id, sunway_iommu_domain_bitmap); else id = 0; spin_unlock(&domain_bitmap_lock); return id; } static int sunway_domain_init(struct sunway_iommu_domain *sdomain) { spin_lock_init(&sdomain->lock); mutex_init(&sdomain->api_lock); sdomain->id = sunway_domain_id_alloc(); if (!sdomain->id) return -ENOMEM; INIT_LIST_HEAD(&sdomain->dev_list); return 1; } static struct sunway_iommu_domain *sunway_domain_alloc(void) { struct sunway_iommu_domain *sdomain; sdomain = kzalloc(sizeof(struct sunway_iommu_domain), GFP_KERNEL); if (!sdomain) return NULL; if (!sunway_domain_init(sdomain)) { kfree(sdomain); return NULL; } add_domain_to_list(sdomain); return sdomain; } static struct dma_domain *dma_domain_alloc(void) { struct dma_domain *dma_dom; struct page; dma_dom = kzalloc(sizeof(struct dma_domain), GFP_KERNEL); if (!dma_dom) return NULL; sunway_domain_init(&dma_dom->sdomain); dma_dom->sdomain.type = IOMMU_DOMAIN_DMA; init_iova_domain(&dma_dom->iovad, PAGE_SIZE, IOVA_PFN(SW64_DMA_START)); reserve_iova(&dma_dom->iovad, (0xe0000000UL >> PAGE_SHIFT), (0x100000000UL >> PAGE_SHIFT)); add_domain_to_list(&dma_dom->sdomain); return dma_dom; } static void device_flush_all(struct sunway_iommu_dev *sdata) { struct pci_controller *hose = pci_bus_to_pci_controller(sdata->pdev->bus); if (hose == NULL) return; write_piu_ior0(hose->node, hose->index, DTLB_FLUSHDEV, sdata->devid); write_piu_ior0(hose->node, hose->index, PTLB_FLUSHDEV, sdata->devid); write_piu_ior0(hose->node, hose->index, PCACHE_FLUSHDEV, sdata->devid); } /* iommu_ops device attach/unattach helpers */ static void set_dte_entry(struct sunway_iommu_dev *sdev, struct sunway_iommu_domain *sdomain) { struct sunway_iommu *iommu; struct pci_dev *pdev; struct page *dt_page, *pt_page; unsigned long *dte_l1, *dte_l2; unsigned long dte_l1_val, dte_l2_base, dte_l2_val; pdev = sdev->pdev; if (pdev->hdr_type == PCI_HEADER_TYPE_BRIDGE) return; sdev->devid = PCI_DEVID(pdev->bus->number, pdev->devfn); iommu = sdev->iommu; dte_l1 = iommu->iommu_dtbr + (pdev->bus->number); dte_l1_val = *dte_l1; if (!dte_l1_val) { /* Alloc a new level-2 device table page */ dt_page = alloc_pages_node(iommu->node, GFP_KERNEL | __GFP_ZERO, get_order(PAGE_SIZE)); if (!dt_page) { pr_err("Allocating a new level-2 device table page failed.\n"); return; } dte_l2_base = (unsigned long)page_address(dt_page); dte_l1_val = (__pa(dte_l2_base) & PAGE_MASK) | SW64_IOMMU_ENTRY_VALID; *dte_l1 = dte_l1_val; } if (!sdomain->pt_root) { pt_page = alloc_pages_node(iommu->node, GFP_KERNEL | __GFP_ZERO, 0); if (!pt_page) { pr_err("Allocating pt_root failed!\n"); return; } sdomain->pt_root = page_address(pt_page); } dte_l2 = __va(dte_l1_val & ~(SW64_IOMMU_ENTRY_VALID) & PAGE_MASK) + (pdev->devfn << 3); dte_l2_val = (__pa(sdomain->pt_root) & PAGE_MASK) | SW64_IOMMU_ENTRY_VALID; if (sdomain->type == IOMMU_DOMAIN_IDENTITY) { dte_l2_val |= 0x1; sdev->passthrough = IDENTMAP_ALL; } *dte_l2 = dte_l2_val; device_flush_all(sdev); } static void do_attach(struct sunway_iommu_dev *sdev_data, struct sunway_iommu_domain *sdomain) { sdev_data->domain = sdomain; list_add(&sdev_data->list, &sdomain->dev_list); sdomain->dev_cnt++; set_dte_entry(sdev_data, sdomain); pr_debug("iommu: device %d add to domain: %d\n", sdev_data->devid, sdomain->id); } static void do_detach(struct sunway_iommu_dev *sdev_data) { struct sunway_iommu_domain *sdomain = sdev_data->domain; sdev_data->domain = NULL; list_del(&sdev_data->list); device_flush_all(sdev_data); sdomain->dev_cnt--; pr_debug("iommu: device %d detached from domain %d\n", sdev_data->devid, sdomain->id); } static int __attach_device(struct sunway_iommu_dev *sdev_data, struct sunway_iommu_domain *sdomain) { int ret; spin_lock(&sdomain->lock); ret = -EBUSY; if (sdev_data->domain != NULL) goto out_unlock; do_attach(sdev_data, sdomain); ret = 0; out_unlock: spin_unlock(&sdomain->lock); return ret; } static void __detach_device(struct sunway_iommu_dev *sunway_dev_data) { struct sunway_iommu_domain *domain; domain = sunway_dev_data->domain; spin_lock(&domain->lock); do_detach(sunway_dev_data); spin_unlock(&domain->lock); } static int attach_device(struct device *dev, struct sunway_iommu_domain *sdomain) { struct sunway_iommu_dev *sdev; unsigned long flags; int ret; sdev = dev_iommu_priv_get(dev); spin_lock_irqsave(&sunway_iommu_device_table_lock, flags); ret = __attach_device(sdev, sdomain); spin_unlock_irqrestore(&sunway_iommu_device_table_lock, flags); return ret; } static void detach_device(struct device *dev) { struct sunway_iommu_domain *sunway_domain; struct sunway_iommu_dev *sdev; unsigned long flags; sdev = dev_iommu_priv_get(dev); sunway_domain = sdev->domain; if (WARN_ON(!sdev->domain)) return; spin_lock_irqsave(&sunway_iommu_device_table_lock, flags); __detach_device(sdev); spin_unlock_irqrestore(&sunway_iommu_device_table_lock, flags); if (!dev_is_pci(dev)) return; } static struct sunway_iommu_dev *search_dev_data(u16 devid) { struct sunway_iommu_dev *sdev_data; struct llist_node *node; if (llist_empty(&dev_data_list)) return NULL; node = dev_data_list.first; llist_for_each_entry(sdev_data, node, dev_data_list) { if (sdev_data->devid == devid) return sdev_data; } return NULL; } /* dma_ops helpers*/ static struct sunway_iommu_domain *get_sunway_domain(struct device *dev) { struct sunway_iommu_domain *sdomain; struct iommu_domain *domain; struct pci_dev *pdev; struct sunway_iommu_dev *sdev; pdev = to_pci_dev(dev); if (!pdev) return ERR_PTR(-ENODEV); sdev = dev_iommu_priv_get(dev); sdomain = sdev->domain; if (sdomain == NULL) { domain = iommu_get_domain_for_dev(dev); sdomain = to_sunway_domain(domain); attach_device(dev, sdomain); } if (sdomain == NULL) return ERR_PTR(-EBUSY); return sdomain; } /********************************************************************** * * Following functions describe IOMMU init ops * **********************************************************************/ static struct sunway_iommu *sunway_iommu_early_init(struct pci_controller *hose) { struct sunway_iommu *iommu; struct page *page; unsigned long base; hose->pci_iommu = kzalloc(sizeof(struct sunway_iommu), GFP_KERNEL); if (!hose->pci_iommu) return 0; iommu = hose->pci_iommu; spin_lock_init(&iommu->dt_lock); iommu->node = hose->node; if (!node_online(hose->node)) iommu->node = -1; page = alloc_pages_node(iommu->node, __GFP_ZERO, get_order(PAGE_SIZE)); if (!page) { pr_err("Allocating a new iommu_dtbr page failed.\n"); kfree(hose->pci_iommu); return NULL; } iommu->iommu_dtbr = page_address(page); iommu->hose_pt = hose; iommu->index = hose->index; iommu->enabled = true; base = __pa(iommu->iommu_dtbr) & PAGE_MASK; write_piu_ior0(hose->node, hose->index, DTBASEADDR, base); return iommu; } unsigned long fetch_dte(struct sunway_iommu *iommu, unsigned long devid, enum exceptype type) { unsigned long *dte_l1, *dte_l2; unsigned long dte_l1_val, dte_l2_val; if (!iommu) return 0; dte_l1 = iommu->iommu_dtbr + (devid >> 8); if (type == DTE_LEVEL1) return (unsigned long)dte_l1; dte_l1_val = *dte_l1; if (type == DTE_LEVEL1_VAL) return dte_l1_val; dte_l1_val &= (~(SW64_IOMMU_ENTRY_VALID)) & (PAGE_MASK); dte_l1_val |= PAGE_OFFSET; dte_l2 = (unsigned long *)(dte_l1_val + ((devid & 0xff) << 3)); if (type == DTE_LEVEL2) return (unsigned long)dte_l2; dte_l2_val = *dte_l2; if (type == DTE_LEVEL2_VAL) return dte_l2_val; return dte_l2_val; } unsigned long fetch_pte(struct sunway_iommu_domain *sdomain, dma_addr_t iova, enum exceptype type) { unsigned long iova_pfn; unsigned long pte_l1_val, pte_l2_val, pte_l3_val; unsigned long *pte_l1, *pte_l2, *pte_l3; unsigned long pte_root; unsigned long offset; if (!sdomain) return -EINVAL; pte_root = __pa(sdomain->pt_root) & PAGE_MASK; iova_pfn = iova >> PAGE_SHIFT; pte_root = ((pte_root) & (~(SW64_IOMMU_ENTRY_VALID)) & (PAGE_MASK)); pte_root |= PAGE_OFFSET; offset = ((iova_pfn >> 20) & SW64_IOMMU_LEVEL1_OFFSET) << 3; pte_l1 = (unsigned long *)(pte_root + offset); if (type == PTE_LEVEL1) return (unsigned long)pte_l1; pte_l1_val = *pte_l1; if (type == PTE_LEVEL1_VAL) return pte_l1_val; pte_l1_val &= (~(SW64_IOMMU_ENTRY_VALID)) & (PAGE_MASK); pte_l1_val |= PAGE_OFFSET; offset = ((iova_pfn >> 10) & SW64_IOMMU_LEVEL2_OFFSET) << 3; pte_l2 = (unsigned long *)(pte_l1_val + offset); if (type == PTE_LEVEL2) return (unsigned long)pte_l2; pte_l2_val = *pte_l2; if (type == PTE_LEVEL2_VAL) return pte_l2_val; pte_l2_val &= (~(SW64_IOMMU_ENTRY_VALID)) & (PAGE_MASK); pte_l2_val |= PAGE_OFFSET; offset = (iova_pfn & SW64_IOMMU_LEVEL3_OFFSET) << 3; pte_l3 = (unsigned long *)(pte_l2_val + offset); if (type == PTE_LEVEL3) return (unsigned long)pte_l3; pte_l3_val = *pte_l3; if (type == PTE_LEVEL3_VAL) return pte_l3_val; return pte_l3_val; } /* IOMMU Interrupt handle */ irqreturn_t iommu_interrupt(int irq, void *dev) { struct pci_controller *hose = (struct pci_controller *)dev; struct sunway_iommu_domain *sdomain; struct sunway_iommu_dev *sdev; unsigned long iommu_status; unsigned long type; unsigned long devid, dva; iommu_status = read_piu_ior0(hose->node, hose->index, IOMMUEXCPT_STATUS); if (!(iommu_status >> 63)) return IRQ_NONE; type = (iommu_status >> 58) & 0xf; devid = (iommu_status >> 36) & 0xffff; dva = ((iommu_status & 0xffffffff) >> 3) << 13; pr_info("%s, iommu_status = %#lx, devid %#lx, dva %#lx, ", __func__, iommu_status, devid, dva); sdev = search_dev_data(devid); if (sdev == NULL) { pr_info("no such dev!!!\n"); iommu_status &= ~(1UL << 62); write_piu_ior0(hose->node, hose->index, IOMMUEXCPT_STATUS, iommu_status); return IRQ_HANDLED; } sdomain = sdev->domain; switch (type) { case DTE_LEVEL1: pr_info("invalid level1 dte, addr:%#lx, val:%#lx\n", fetch_dte(hose->pci_iommu, devid, DTE_LEVEL1), fetch_dte(hose->pci_iommu, devid, DTE_LEVEL1_VAL)); break; case DTE_LEVEL2: pr_info("invalid level2 dte, addr:%#lx, val:%#lx\n", fetch_dte(hose->pci_iommu, devid, DTE_LEVEL2), fetch_dte(hose->pci_iommu, devid, DTE_LEVEL2_VAL)); break; case PTE_LEVEL1: pr_info("invalid level1 pte, addr: %#lx, val:%#lx\n", fetch_pte(sdomain, dva, PTE_LEVEL1), fetch_pte(sdomain, dva, PTE_LEVEL1_VAL)); iommu_status &= ~(1UL << 62); write_piu_ior0(hose->node, hose->index, IOMMUEXCPT_STATUS, iommu_status); break; case PTE_LEVEL2: pr_info("invalid level2 pte, addr: %#lx, val: %#lx\n", fetch_pte(sdomain, dva, PTE_LEVEL2), fetch_pte(sdomain, dva, PTE_LEVEL2_VAL)); iommu_status &= ~(1UL << 62); write_piu_ior0(hose->node, hose->index, IOMMUEXCPT_STATUS, iommu_status); break; case PTE_LEVEL3: pr_info("invalid level3 pte, addr: %#lx, val: %#lx\n", fetch_pte(sdomain, dva, PTE_LEVEL3), fetch_pte(sdomain, dva, PTE_LEVEL3_VAL)); iommu_status &= ~(1UL << 62); write_piu_ior0(hose->node, hose->index, IOMMUEXCPT_STATUS, iommu_status); break; default: pr_info("iommu exception type %ld\n", type); break; } return IRQ_HANDLED; } struct irqaction iommu_irqaction = { .handler = iommu_interrupt, .flags = IRQF_SHARED | IRQF_NO_THREAD, .name = "sunway_iommu", }; void sunway_enable_iommu_func(struct pci_controller *hose) { unsigned int iommu_irq, err; unsigned long iommu_conf, iommu_ctrl; iommu_irq = hose->int_irq; pr_debug("%s node %ld rc %ld iommu_irq %d\n", __func__, hose->node, hose->index, iommu_irq); err = request_irq(iommu_irq, iommu_interrupt, IRQF_SHARED, "sunway_iommu", hose); if (err < 0) pr_info("sw iommu request irq failed!\n"); iommu_ctrl = (1UL << 63) | (0x100UL << 10); write_piu_ior0(hose->node, hose->index, IOMMUEXCPT_CTRL, iommu_ctrl); iommu_conf = read_piu_ior0(hose->node, hose->index, PIUCONFIG0); iommu_conf = iommu_conf | (0x3 << 7); write_piu_ior0(hose->node, hose->index, PIUCONFIG0, iommu_conf); write_piu_ior0(hose->node, hose->index, TIMEOUT_CONFIG, 0xf); iommu_conf = read_piu_ior0(hose->node, hose->index, PIUCONFIG0); pr_debug("SW arch configure node %ld hose-%ld iommu_conf = %#lx\n", hose->node, hose->index, iommu_conf); } static bool is_iommu_enable(struct pci_controller *hose) { u64 rc_mask = 0x1; rc_mask <<= (8 * hose->node + hose->index); if (iommu_enable_cmd & rc_mask) return true; return false; } /* iommu cpu syscore ops */ static int iommu_cpu_suspend(void) { return 0; } static void iommu_cpu_resume(void) { } struct syscore_ops iommu_cpu_syscore_ops = { .suspend = iommu_cpu_suspend, .resume = iommu_cpu_resume, }; static struct iommu_domain *sunway_iommu_domain_alloc(unsigned int type); static int sunway_iommu_init(void) { struct pci_controller *hose; struct sunway_iommu *iommu; int ret; int iommu_index = 0; sunway_iommu_domain_bitmap = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, get_order(MAX_DOMAIN_NUM / 8)); if (sunway_iommu_domain_bitmap == NULL) return 0; __set_bit(0, sunway_iommu_domain_bitmap); /* Do the loop */ for (hose = hose_head; hose; hose = hose->next) { if (!is_iommu_enable(hose)) { hose->iommu_enable = false; continue; } iommu = sunway_iommu_early_init(hose); if (!iommu) { pr_err("Allocating sunway_iommu failed\n"); hose->iommu_enable = false; continue; } iommu_device_sysfs_add(&iommu->iommu, NULL, NULL, "%d", iommu_index); iommu_device_set_ops(&iommu->iommu, &sunway_iommu_ops); iommu_device_register(&iommu->iommu); iommu_index++; sunway_enable_iommu_func(hose); hose->iommu_enable = true; } ret = iova_cache_get(); if (ret) return ret; ret = bus_set_iommu(&pci_bus_type, &sunway_iommu_ops); if (ret) return ret; for (hose = hose_head; hose; hose = hose->next) if (hose->iommu_enable) piu_flush_all(hose); register_syscore_ops(&iommu_cpu_syscore_ops); return 1; } subsys_initcall_sync(sunway_iommu_init); /******************************************************************************* * * DMA OPS Functions * ******************************************************************************/ struct sunway_iommu *get_first_iommu_from_domain(struct sunway_iommu_domain *sdomain) { struct sunway_iommu *iommu; struct sunway_iommu_dev *entry; entry = list_first_entry(&sdomain->dev_list, struct sunway_iommu_dev, list); iommu = entry->iommu; return iommu; } static unsigned long sunway_iommu_unmap_page(struct sunway_iommu_domain *sunway_domain, unsigned long iova, unsigned long page_size) { unsigned long offset, iova_pfn; unsigned long *pte_base, *pte; unsigned long grn; int level, current_level; int tmp = 1; pr_debug("%s iova %#lx, page_size %#lx\n", __func__, iova, page_size); BUG_ON(!is_power_of_2(page_size)); switch (page_size) { case (1UL << 33): level = 1; grn = PTE_GRN_8G; break; case (1UL << 29): level = 2; grn = PTE_GRN_512M; break; case (1UL << 23): level = 2; grn = PTE_GRN_8M; break; default: level = 3; break; } pte_base = sunway_domain->pt_root; iova_pfn = iova >> PAGE_SHIFT; offset = (iova_pfn >> 20) & 0x1ff; current_level = 1; while (current_level <= level) { pte = &pte_base[offset]; if (current_level == level) { if (grn == PTE_GRN_512M) { int i; for (i = 0; i < 64; i++) { *(pte + i) = 0; flush_pcache_by_addr(sunway_domain, (unsigned long)pte); } } else { *pte = 0; flush_pcache_by_addr(sunway_domain, (unsigned long)pte); } flush_ptlb_by_addr(sunway_domain, (iova >> PAGE_SHIFT)); break; } pte_base = (unsigned long *)((*pte & (~PTE_FLAGS_MASK)) | PAGE_OFFSET); offset = (iova_pfn >> (tmp--) * 10) & 0x3ff; current_level++; } return page_size; } int sunway_iommu_map_page(struct sunway_iommu_domain *sunway_domain, unsigned long bus_addr, unsigned long paddr, size_t page_size) { struct page *page; struct sunway_iommu *iommu; unsigned long iova_pfn, pte_val; unsigned long *pte_base, *pte; unsigned long offset, grn = 0; int level = 0, current_level; int tmp = 1; iommu = get_first_iommu_from_domain(sunway_domain); if (!iommu) return -1; iova_pfn = bus_addr >> PAGE_SHIFT; pte_base = sunway_domain->pt_root; switch (page_size) { case (1UL << 33): level = 1; grn = PTE_GRN_8G; break; case (1UL << 29): level = 2; grn = PTE_GRN_512M; break; case (1UL << 23): grn = PTE_GRN_8M; level = 2; break; default: level = 3; break; } offset = (iova_pfn >> 20) & 0x1ff; current_level = 1; while (current_level <= level) { pte = &pte_base[offset]; if (!(*pte) || (current_level == level)) { pte_val = PTE_VALID | PTE_RWE | grn; if (current_level == level) { *(volatile u64 *)(pte) = 0; pte_val |= ((paddr & PAGE_MASK) | LAST_STAGE); } else { page = alloc_pages_node(iommu->node, GFP_ATOMIC | __GFP_ZERO, 0); if (!page) { pr_err("Allocating level%d page table pages failed.\n", (level + 1)); return -ENOMEM; } pte_val |= (page_to_phys(page) & PAGE_MASK); } if ((grn == PTE_GRN_512M) && (current_level == 2)) { int i; for (i = 0; i < 64; i++) { cmpxchg64((volatile u64 *)(pte + i), 0UL, pte_val); flush_pcache_by_addr(sunway_domain, (unsigned long)(pte + i)); } } else { if (cmpxchg64((volatile u64 *)pte, 0UL, pte_val)) free_page((unsigned long)page_address(page)); else flush_pcache_by_addr(sunway_domain, (unsigned long)pte); } } pte_base = (unsigned long *)__va((*pte) & (~PTE_FLAGS_MASK)); offset = (iova_pfn >> (tmp--) * 10) & 0x3ff; current_level++; } return 0; } static unsigned long sunway_alloc_iova(struct dma_domain *dma_dom, unsigned long pages, struct pci_dev *pdev) { struct device *dev; unsigned long pfn = 0; pages = __roundup_pow_of_two(pages); dev = &(pdev->dev); if (min(dev->coherent_dma_mask, *dev->dma_mask) == DMA_BIT_MASK(32)) { pfn = alloc_iova_fast(&dma_dom->iovad, pages, IOVA_PFN(SW64_32BIT_DMA_LIMIT), true); } else { /* IOVA boundary should be 16M ~ 3.5G */ pfn = alloc_iova_fast(&dma_dom->iovad, pages, IOVA_PFN(SW64_64BIT_DMA_LIMIT), true); } return (pfn << PAGE_SHIFT); } static void sunway_free_iova(struct dma_domain *dma_dom, unsigned long address, unsigned long pages) { pages = __roundup_pow_of_two(pages); address >>= PAGE_SHIFT; free_iova_fast(&dma_dom->iovad, address, pages); } static dma_addr_t __sunway_map_single(struct dma_domain *dma_dom, struct pci_dev *pdev, phys_addr_t paddr, size_t size) { dma_addr_t ret, address, start; unsigned long npages, i; npages = iommu_num_pages(paddr, size, PAGE_SIZE); address = sunway_alloc_iova(dma_dom, npages, pdev); if (!address) return 0; start = address; for (i = 0; i < npages; ++i) { ret = sunway_iommu_map_page(&dma_dom->sdomain, start, paddr, PAGE_SIZE); if (ret) { pr_info("error when map page.\n"); goto out_unmap; } start += PAGE_SIZE; paddr += PAGE_SIZE; } address += paddr & ~PAGE_MASK; return address; out_unmap: for (--i; i >= 0; --i) { start -= PAGE_SIZE; sunway_iommu_unmap_page(&dma_dom->sdomain, start, PAGE_SIZE); } sunway_free_iova(dma_dom, address, npages); return 0; } static dma_addr_t pci_iommu_map_single(struct pci_dev *pdev, struct dma_domain *dma_dom, void *cpu_addr, size_t size) { struct pci_controller *hose = pci_bus_to_pci_controller(pdev->bus); unsigned long paddr; if (hose == NULL) { pr_err("%s: hose does not exist!\n", __func__); return 0; } paddr = __sunway_map_single(dma_dom, pdev, __pa(cpu_addr), size); pr_debug("pci_alloc_consistent: %zx -> [%px,%lx] from %ps\n", size, cpu_addr, paddr, __builtin_return_address(0)); return paddr; } static void *sunway_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_addr, gfp_t gfp, unsigned long attrs) { struct pci_dev *pdev = to_pci_dev(dev); struct pci_controller *hose; struct sunway_iommu_domain *sdomain; struct dma_domain *dma_dom; struct sunway_iommu_dev *sdev; struct page *page; void *cpu_addr; if (!pdev) return NULL; hose = pci_bus_to_pci_controller(pdev->bus); if (!hose) return NULL; gfp &= ~GFP_DMA; try_again: page = alloc_pages_node(dev_to_node(dev), gfp | __GFP_ZERO, get_order(size)); if (!page) { pr_err("Allocating pages failed.\n"); return NULL; } cpu_addr = page_address(page); if (!cpu_addr) { pr_info ("pci_alloc_consistent: get_free_pages failed from %ps\n", __builtin_return_address(0)); return NULL; } *dma_addr = __pa(cpu_addr); if (!(hose->iommu_enable)) return cpu_addr; sdev = dev_iommu_priv_get(dev); if (sdev->passthrough & DMA_MASK64) return cpu_addr; else if (sdev->passthrough) { if (min(dev->coherent_dma_mask, *dev->dma_mask) > DMA_BIT_MASK(32)) { sdev->passthrough |= DMA_MASK64; return cpu_addr; } __free_pages(page, get_order(size)); set_dma_ops(dev, get_arch_dma_ops(dev->bus)); return dev->dma_ops->alloc(dev, size, dma_addr, gfp, attrs); } sdomain = get_sunway_domain(dev); dma_dom = to_dma_domain(sdomain); *dma_addr = pci_iommu_map_single(pdev, dma_dom, cpu_addr, size); if (*dma_addr == 0) { free_pages((unsigned long)cpu_addr, get_order(size)); if (gfp & GFP_DMA) return NULL; gfp |= GFP_DMA; goto try_again; } return cpu_addr; } static void __sunway_unmap_single(struct dma_domain *dma_dom, dma_addr_t dma_addr, size_t size) { dma_addr_t start; unsigned long npages; int i; npages = iommu_num_pages(dma_addr, size, PAGE_SIZE); dma_addr &= PAGE_MASK; start = dma_addr; for (i = 0; i < npages; i++) { sunway_iommu_unmap_page(&dma_dom->sdomain, start, PAGE_SIZE); start += PAGE_SIZE; } sunway_free_iova(dma_dom, dma_addr, npages); pr_debug("pci_free_consistent: %zx -> [%llx] from %ps\n", size, dma_addr, __builtin_return_address(0)); } static void sunway_free_coherent(struct device *dev, size_t size, void *vaddr, dma_addr_t dma_addr, unsigned long attrs) { struct sunway_iommu_domain *sdomain; struct dma_domain *dma_dom; struct pci_dev *pdev = to_pci_dev(dev); struct pci_controller *hose; struct sunway_iommu_dev *sdev; if (!pdev) goto out_unmap; hose = pci_bus_to_pci_controller(pdev->bus); if (!hose || !(hose->iommu_enable)) goto out_unmap; sdev = dev_iommu_priv_get(dev); if (sdev->passthrough) goto out_unmap; sdomain = get_sunway_domain(dev); dma_dom = to_dma_domain(sdomain); __sunway_unmap_single(dma_dom, dma_addr, size); goto out_free; out_unmap: pci_unmap_single(pdev, dma_addr, size, PCI_DMA_BIDIRECTIONAL); out_free: pr_debug("sunway_free_consistent: [%llx,%zx] from %ps\n", dma_addr, size, __builtin_return_address(0)); free_pages((unsigned long)vaddr, get_order(size)); } static dma_addr_t sunway_map_page(struct device *dev, struct page *page, unsigned long offset, size_t size, enum dma_data_direction dir, unsigned long attrs) { struct pci_dev *pdev = to_pci_dev(dev); struct sunway_iommu_domain *sdomain; struct dma_domain *dma_dom; struct pci_controller *hose; struct sunway_iommu_dev *sdev; phys_addr_t paddr = page_to_phys(page) + offset; if (!pdev) return 0; hose = pci_bus_to_pci_controller(pdev->bus); if (!hose || !(hose->iommu_enable)) return paddr; sdev = dev_iommu_priv_get(dev); if (sdev->passthrough & DMA_MASK64) return paddr; else if (sdev->passthrough) { if (min(dev->coherent_dma_mask, *dev->dma_mask) > DMA_BIT_MASK(32)) { sdev->passthrough |= DMA_MASK64; return paddr; } set_dma_ops(dev, get_arch_dma_ops(dev->bus)); return dev->dma_ops->map_page(dev, page, offset, size, dir, attrs); } sdomain = get_sunway_domain(dev); dma_dom = to_dma_domain(sdomain); return pci_iommu_map_single(pdev, dma_dom, (char *)page_address(page) + offset, size); } static void sunway_unmap_page(struct device *dev, dma_addr_t dma_addr, size_t size, enum dma_data_direction dir, unsigned long attrs) { struct sunway_iommu_domain *sdomain; struct dma_domain *dma_dom; struct pci_dev *pdev; struct pci_controller *hose; struct sunway_iommu_dev *sdev; pdev = to_pci_dev(dev); if (!pdev) return; hose = pci_bus_to_pci_controller(pdev->bus); if (hose == NULL) return; if (!hose->iommu_enable) return; sdev = dev_iommu_priv_get(dev); if (sdev->passthrough) return; sdomain = get_sunway_domain(dev); dma_dom = to_dma_domain(sdomain); __sunway_unmap_single(dma_dom, dma_addr, size); } #define SG_ENT_VIRT_ADDRESS(SG) (sg_virt((SG))) static int sunway_map_sg(struct device *dev, struct scatterlist *sgl, int nents, enum dma_data_direction dir, unsigned long attrs) { struct sunway_iommu_domain *sdomain; struct dma_domain *dma_dom = NULL; struct scatterlist *sg; struct pci_dev *pdev = to_pci_dev(dev); struct pci_controller *hose; struct sunway_iommu_dev *sdev; int i, out_nents = 0; if (dir == PCI_DMA_NONE) BUG(); if (!pdev) return 0; hose = pci_bus_to_pci_controller(pdev->bus); if (!hose) return 0; sdomain = get_sunway_domain(dev); dma_dom = to_dma_domain(sdomain); for_each_sg(sgl, sg, nents, i) { BUG_ON(!sg_page(sg)); sg_dma_address(sg) = __pa(SG_ENT_VIRT_ADDRESS(sg)); if (!(hose->iommu_enable)) goto check; sdev = dev_iommu_priv_get(dev); if (sdev->passthrough & DMA_MASK64) goto check; else if (sdev->passthrough) { if (min(dev->coherent_dma_mask, *dev->dma_mask) > DMA_BIT_MASK(32)) { sdev->passthrough |= DMA_MASK64; goto check; } set_dma_ops(dev, get_arch_dma_ops(dev->bus)); return dev->dma_ops->map_sg(dev, sgl, nents, dir, attrs); } sg_dma_address(sg) = pci_iommu_map_single(pdev, dma_dom, SG_ENT_VIRT_ADDRESS(sg), sg->length); check: if (sg_dma_address(sg) == 0) goto error; sg_dma_len(sg) = sg->length; out_nents++; } return nents; error: pr_warn("pci_map_sg failed:"); pr_warn("could not allocate dma page tables\n"); if (out_nents) pci_unmap_sg(pdev, sgl, out_nents, dir); return 0; } static void sunway_unmap_sg(struct device *dev, struct scatterlist *sgl, int nents, enum dma_data_direction dir, unsigned long attrs) { struct sunway_iommu_domain *sdomain; struct dma_domain *dma_dom; struct scatterlist *sg; struct pci_dev *pdev; struct pci_controller *hose; struct sunway_iommu_dev *sdev; dma_addr_t dma_addr; long size; int j; pdev = to_pci_dev(dev); if (!pdev) return; hose = pci_bus_to_pci_controller(pdev->bus); if (!hose->iommu_enable) return; sdev = dev_iommu_priv_get(dev); if (sdev->passthrough) return; sdomain = get_sunway_domain(dev); dma_dom = to_dma_domain(sdomain); for_each_sg(sgl, sg, nents, j) { dma_addr = sg->dma_address; size = sg->dma_length; if (!size) break; __sunway_unmap_single(dma_dom, dma_addr, size); } } static const struct dma_map_ops sunway_dma_ops = { .alloc = sunway_alloc_coherent, .free = sunway_free_coherent, .map_sg = sunway_map_sg, .unmap_sg = sunway_unmap_sg, .map_page = sunway_map_page, .unmap_page = sunway_unmap_page, .dma_supported = dma_direct_supported, }; /********************************************************************** * * IOMMU OPS Functions * **********************************************************************/ static struct iommu_domain *sunway_iommu_domain_alloc(unsigned int type) { struct sunway_iommu_domain *sdomain; struct dma_domain *dma_dom; switch (type) { case IOMMU_DOMAIN_UNMANAGED: sdomain = sunway_domain_alloc(); if (!sdomain) { pr_err("Allocating sunway_domain failed!\n"); return NULL; } sdomain->domain.geometry.aperture_start = 0UL; sdomain->domain.geometry.aperture_end = ~0ULL; sdomain->domain.geometry.force_aperture = true; sdomain->type = IOMMU_DOMAIN_UNMANAGED; break; case IOMMU_DOMAIN_DMA: dma_dom = dma_domain_alloc(); if (!dma_dom) { pr_err("Failed to alloc dma domain!\n"); return NULL; } sdomain = &dma_dom->sdomain; break; case IOMMU_DOMAIN_IDENTITY: sdomain = sunway_domain_alloc(); if (!sdomain) return NULL; sdomain->type = IOMMU_DOMAIN_IDENTITY; break; default: return NULL; } return &sdomain->domain; } static void clean_domain(struct sunway_iommu_domain *sdomain) { struct sunway_iommu_dev *entry; unsigned long flags; spin_lock_irqsave(&sunway_iommu_device_table_lock, flags); while (!list_empty(&sdomain->dev_list)) { entry = list_first_entry(&sdomain->dev_list, struct sunway_iommu_dev, list); BUG_ON(!entry->domain); __detach_device(entry); } spin_unlock_irqrestore(&sunway_iommu_device_table_lock, flags); } static void sunway_iommu_domain_free(struct iommu_domain *dom) { struct sunway_iommu_domain *sdomain; struct dma_domain *dma_dom; sdomain = to_sunway_domain(dom); if (sdomain->dev_cnt > 0) clean_domain(sdomain); BUG_ON(sdomain->dev_cnt != 0); if (!dom) return; switch (dom->type) { case IOMMU_DOMAIN_DMA: dma_dom = to_dma_domain(sdomain); dma_domain_free(dma_dom); break; default: free_pagetable(sdomain); sunway_domain_free(sdomain); break; } } static int sunway_iommu_attach_device(struct iommu_domain *dom, struct device *dev) { struct sunway_iommu_domain *sdomain = to_sunway_domain(dom); struct sunway_iommu_dev *sdev; struct pci_dev *pdev; struct pci_controller *hose; int ret; pdev = to_pci_dev(dev); if (!pdev) return -EINVAL; hose = pci_bus_to_pci_controller(pdev->bus); if (!hose) return -EINVAL; if (!hose->iommu_enable) return -EINVAL; sdev = dev_iommu_priv_get(dev); if (!sdev) return -EINVAL; if (sdev->domain) detach_device(dev); ret = attach_device(dev, sdomain); return ret; } static void sunway_iommu_detach_device(struct iommu_domain *dom, struct device *dev) { struct sunway_iommu_dev *sdev; struct pci_dev *pdev = to_pci_dev(dev); if (!pdev) return; sdev = dev_iommu_priv_get(dev); if (sdev->domain != NULL) detach_device(dev); } static phys_addr_t sunway_iommu_iova_to_phys(struct iommu_domain *dom, dma_addr_t iova) { struct sunway_iommu_domain *sdomain = to_sunway_domain(dom); unsigned long paddr, grn; unsigned long is_last; if (iova > SW64_BAR_ADDRESS) return iova; paddr = fetch_pte(sdomain, iova, PTE_LEVEL1_VAL); if ((paddr & SW64_IOMMU_ENTRY_VALID) == 0) return 0; is_last = paddr & SW64_PTE_LAST_MASK; grn = paddr & SW64_PTE_GRN_MASK; if (is_last) { if (grn == PTE_GRN_8G) { paddr &= ~PTE_FLAGS_MASK; paddr += iova & PAGE_8G_OFFSET_MASK; return paddr; } return 0; } paddr = fetch_pte(sdomain, iova, PTE_LEVEL2_VAL); if ((paddr & SW64_IOMMU_ENTRY_VALID) == 0) return 0; is_last = paddr & SW64_PTE_LAST_MASK; grn = paddr & SW64_PTE_GRN_MASK; if (is_last) { if (grn == PTE_GRN_512M) { paddr &= ~PTE_FLAGS_MASK; paddr += iova & PAGE_512M_OFFSET_MASK; return paddr; } if (grn == PTE_GRN_8M) { paddr &= ~PTE_FLAGS_MASK; paddr += iova & PAGE_8M_OFFSET_MASK; return paddr; } return 0; } paddr = fetch_pte(sdomain, iova, PTE_LEVEL3_VAL); if ((paddr & SW64_IOMMU_ENTRY_VALID) == 0) return 0; grn = paddr & SW64_PTE_GRN_MASK; if (grn != 0) return 0; paddr &= ~PTE_FLAGS_MASK; paddr += iova & PAGE_MASK; return paddr; } static int sunway_iommu_map(struct iommu_domain *dom, unsigned long iova, phys_addr_t paddr, size_t page_size, int iommu_prot, gfp_t gfp) { struct sunway_iommu_domain *sdomain = to_sunway_domain(dom); int ret; /* * As VFIO cannot distinguish between normal DMA request * and pci device BAR, check should be introduced manually * to avoid VFIO trying to map pci config space. */ if (iova > SW64_BAR_ADDRESS) return 0; mutex_lock(&sdomain->api_lock); ret = sunway_iommu_map_page(sdomain, iova, paddr, page_size); mutex_unlock(&sdomain->api_lock); return ret; } static size_t sunway_iommu_unmap(struct iommu_domain *dom, unsigned long iova, size_t page_size, struct iommu_iotlb_gather *gather) { struct sunway_iommu_domain *sdomain = to_sunway_domain(dom); size_t unmap_size; if (iova > SW64_BAR_ADDRESS) return page_size; mutex_lock(&sdomain->api_lock); unmap_size = sunway_iommu_unmap_page(sdomain, iova, page_size); mutex_unlock(&sdomain->api_lock); return unmap_size; } static struct iommu_group *sunway_iommu_device_group(struct device *dev) { return pci_device_group(dev); } static void iommu_uninit_device(struct device *dev) { struct sunway_iommu_dev *sdev; sdev = dev_iommu_priv_get(dev); if (!sdev) return; if (sdev->domain) detach_device(dev); dev_iommu_priv_set(dev, NULL); } static void sunway_iommu_release_device(struct device *dev) { struct pci_dev *pdev; struct pci_controller *hose; pdev = to_pci_dev(dev); if (!pdev) return; hose = pci_bus_to_pci_controller(pdev->bus); if (!hose->iommu_enable) return; iommu_uninit_device(dev); } static int iommu_init_device(struct device *dev) { struct sunway_iommu_dev *sdev; struct sunway_iommu *iommu; struct pci_dev *pdev; struct pci_controller *hose; if (dev_iommu_priv_get(dev)) return 0; sdev = kzalloc(sizeof(struct sunway_iommu_dev), GFP_KERNEL); if (!sdev) return -ENOMEM; pdev = to_pci_dev(dev); hose = pci_bus_to_pci_controller(pdev->bus); iommu = hose->pci_iommu; llist_add(&sdev->dev_data_list, &dev_data_list); sdev->pdev = pdev; sdev->iommu = iommu; dev_iommu_priv_set(dev, sdev); return 0; } static struct iommu_device *sunway_iommu_probe_device(struct device *dev) { struct pci_dev *pdev; struct pci_controller *hose; struct sunway_iommu *iommu; int ret; pdev = to_pci_dev(dev); if (!pdev) return ERR_PTR(-ENODEV); if (pdev->hdr_type == PCI_HEADER_TYPE_BRIDGE) return ERR_PTR(-ENODEV); if (pci_pcie_type(pdev) == PCI_EXP_TYPE_ROOT_PORT) return ERR_PTR(-ENODEV); hose = pci_bus_to_pci_controller(pdev->bus); if (!hose) return ERR_PTR(-ENODEV); if (!hose->iommu_enable) return ERR_PTR(-ENODEV); if (dev_iommu_priv_get(dev)) { iommu = hose->pci_iommu; return &iommu->iommu; } ret = iommu_init_device(dev); if (ret) return ERR_PTR(ret); iommu = hose->pci_iommu; return &iommu->iommu; } static int sunway_iommu_def_domain_type(struct device *dev) { struct sunway_iommu_dev *sdev; sdev = dev_iommu_priv_get(dev); if (sdev->domain) return 0; return sdev->domain->type; } static bool sunway_iommu_capable(enum iommu_cap cap) { switch (cap) { case IOMMU_CAP_INTR_REMAP: return true; default: return false; } } static void sunway_iommu_probe_finalize(struct device *dev) { struct iommu_domain *domain; domain = iommu_get_domain_for_dev(dev); if (domain) set_dma_ops(dev, &sunway_dma_ops); } const struct iommu_ops sunway_iommu_ops = { .capable = sunway_iommu_capable, .domain_alloc = sunway_iommu_domain_alloc, .domain_free = sunway_iommu_domain_free, .attach_dev = sunway_iommu_attach_device, .detach_dev = sunway_iommu_detach_device, .probe_device = sunway_iommu_probe_device, .probe_finalize = sunway_iommu_probe_finalize, .release_device = sunway_iommu_release_device, .map = sunway_iommu_map, .unmap = sunway_iommu_unmap, .iova_to_phys = sunway_iommu_iova_to_phys, .device_group = sunway_iommu_device_group, .pgsize_bitmap = SW64_IOMMU_PGSIZES, .def_domain_type = sunway_iommu_def_domain_type, }; /***************************************************************************** * * Boot param handle * Each bit of iommu_enable bitmap represents an rc enable, and every 8 bits * represents one cpu node. For example, iommu_enable=0x0100 means enabling * rc0 for cpu node 1. * *****************************************************************************/ static int __init iommu_enable_setup(char *str) { int ret; unsigned long rc_bitmap = 0xffffffffUL; ret = kstrtoul(str, 16, &rc_bitmap); iommu_enable_cmd = rc_bitmap; return ret; } __setup("iommu_enable=", iommu_enable_setup);