diff options
Diffstat (limited to 'drivers/pci')
-rw-r--r-- | drivers/pci/access.c | 3 | ||||
-rw-r--r-- | drivers/pci/bus.c | 2 | ||||
-rw-r--r-- | drivers/pci/dmar.c | 82 | ||||
-rw-r--r-- | drivers/pci/hotplug/acpiphp_glue.c | 14 | ||||
-rw-r--r-- | drivers/pci/intel-iommu.c | 362 | ||||
-rw-r--r-- | drivers/pci/intr_remapping.c | 84 | ||||
-rw-r--r-- | drivers/pci/iov.c | 1 | ||||
-rw-r--r-- | drivers/pci/pci-acpi.c | 4 | ||||
-rw-r--r-- | drivers/pci/pci-driver.c | 3 | ||||
-rw-r--r-- | drivers/pci/pci-sysfs.c | 6 | ||||
-rw-r--r-- | drivers/pci/pci.c | 6 | ||||
-rw-r--r-- | drivers/pci/probe.c | 2 | ||||
-rw-r--r-- | drivers/pci/quirks.c | 62 | ||||
-rw-r--r-- | drivers/pci/setup-bus.c | 2 |
14 files changed, 542 insertions, 91 deletions
diff --git a/drivers/pci/access.c b/drivers/pci/access.c index 381444794778..64dd7df90e62 100644 --- a/drivers/pci/access.c +++ b/drivers/pci/access.c @@ -356,7 +356,8 @@ int pci_vpd_truncate(struct pci_dev *dev, size_t size) return -EINVAL; dev->vpd->len = size; - dev->vpd->attr->size = size; + if (dev->vpd->attr) + dev->vpd->attr->size = size; return 0; } diff --git a/drivers/pci/bus.c b/drivers/pci/bus.c index 68f91a252595..97a8194063b5 100644 --- a/drivers/pci/bus.c +++ b/drivers/pci/bus.c @@ -184,7 +184,7 @@ void pci_enable_bridges(struct pci_bus *bus) list_for_each_entry(dev, &bus->devices, bus_list) { if (dev->subordinate) { - if (atomic_read(&dev->enable_cnt) == 0) { + if (!pci_is_enabled(dev)) { retval = pci_enable_device(dev); pci_set_master(dev); } diff --git a/drivers/pci/dmar.c b/drivers/pci/dmar.c index d313039e2fdf..fa3a11365ec3 100644 --- a/drivers/pci/dmar.c +++ b/drivers/pci/dmar.c @@ -173,13 +173,23 @@ dmar_parse_one_drhd(struct acpi_dmar_header *header) struct dmar_drhd_unit *dmaru; int ret = 0; + drhd = (struct acpi_dmar_hardware_unit *)header; + if (!drhd->address) { + /* Promote an attitude of violence to a BIOS engineer today */ + WARN(1, "Your BIOS is broken; DMAR reported at address zero!\n" + "BIOS vendor: %s; Ver: %s; Product Version: %s\n", + dmi_get_system_info(DMI_BIOS_VENDOR), + dmi_get_system_info(DMI_BIOS_VERSION), + dmi_get_system_info(DMI_PRODUCT_VERSION)); + return -ENODEV; + } dmaru = kzalloc(sizeof(*dmaru), GFP_KERNEL); if (!dmaru) return -ENOMEM; dmaru->hdr = header; - drhd = (struct acpi_dmar_hardware_unit *)header; dmaru->reg_base_addr = drhd->address; + dmaru->segment = drhd->segment; dmaru->include_all = drhd->flags & 0x1; /* BIT0: INCLUDE_ALL */ ret = alloc_iommu(dmaru); @@ -790,14 +800,41 @@ end: } /* + * Enable queued invalidation. + */ +static void __dmar_enable_qi(struct intel_iommu *iommu) +{ + u32 cmd, sts; + unsigned long flags; + struct q_inval *qi = iommu->qi; + + qi->free_head = qi->free_tail = 0; + qi->free_cnt = QI_LENGTH; + + spin_lock_irqsave(&iommu->register_lock, flags); + + /* write zero to the tail reg */ + writel(0, iommu->reg + DMAR_IQT_REG); + + dmar_writeq(iommu->reg + DMAR_IQA_REG, virt_to_phys(qi->desc)); + + cmd = iommu->gcmd | DMA_GCMD_QIE; + iommu->gcmd |= DMA_GCMD_QIE; + writel(cmd, iommu->reg + DMAR_GCMD_REG); + + /* Make sure hardware complete it */ + IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl, (sts & DMA_GSTS_QIES), sts); + + spin_unlock_irqrestore(&iommu->register_lock, flags); +} + +/* * Enable Queued Invalidation interface. This is a must to support * interrupt-remapping. Also used by DMA-remapping, which replaces * register based IOTLB invalidation. */ int dmar_enable_qi(struct intel_iommu *iommu) { - u32 cmd, sts; - unsigned long flags; struct q_inval *qi; if (!ecap_qis(iommu->ecap)) @@ -835,19 +872,7 @@ int dmar_enable_qi(struct intel_iommu *iommu) spin_lock_init(&qi->q_lock); - spin_lock_irqsave(&iommu->register_lock, flags); - /* write zero to the tail reg */ - writel(0, iommu->reg + DMAR_IQT_REG); - - dmar_writeq(iommu->reg + DMAR_IQA_REG, virt_to_phys(qi->desc)); - - cmd = iommu->gcmd | DMA_GCMD_QIE; - iommu->gcmd |= DMA_GCMD_QIE; - writel(cmd, iommu->reg + DMAR_GCMD_REG); - - /* Make sure hardware complete it */ - IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl, (sts & DMA_GSTS_QIES), sts); - spin_unlock_irqrestore(&iommu->register_lock, flags); + __dmar_enable_qi(iommu); return 0; } @@ -1102,3 +1127,28 @@ int __init enable_drhd_fault_handling(void) return 0; } + +/* + * Re-enable Queued Invalidation interface. + */ +int dmar_reenable_qi(struct intel_iommu *iommu) +{ + if (!ecap_qis(iommu->ecap)) + return -ENOENT; + + if (!iommu->qi) + return -ENOENT; + + /* + * First disable queued invalidation. + */ + dmar_disable_qi(iommu); + /* + * Then enable queued invalidation again. Since there is no pending + * invalidation requests now, it's safe to re-enable queued + * invalidation. + */ + __dmar_enable_qi(iommu); + + return 0; +} diff --git a/drivers/pci/hotplug/acpiphp_glue.c b/drivers/pci/hotplug/acpiphp_glue.c index 803d9ddd6e75..a33794d9e0dc 100644 --- a/drivers/pci/hotplug/acpiphp_glue.c +++ b/drivers/pci/hotplug/acpiphp_glue.c @@ -38,6 +38,8 @@ * - The one in acpiphp_bridge has its refcount elevated by pci_get_slot() * when the bridge is scanned and it loses a refcount when the bridge * is removed. + * - When a P2P bridge is present, we elevate the refcount on the subordinate + * bus. It loses the refcount when the the driver unloads. */ #include <linux/init.h> @@ -440,6 +442,12 @@ static void add_p2p_bridge(acpi_handle *handle, struct pci_dev *pci_dev) goto err; } + /* + * Grab a ref to the subordinate PCI bus in case the bus is + * removed via PCI core logical hotplug. The ref pins the bus + * (which we access during module unload). + */ + get_device(&bridge->pci_bus->dev); spin_lock_init(&bridge->res_lock); init_bridge_misc(bridge); @@ -619,6 +627,12 @@ static void cleanup_bridge(struct acpiphp_bridge *bridge) slot = next; } + /* + * Only P2P bridges have a pci_dev + */ + if (bridge->pci_dev) + put_device(&bridge->pci_bus->dev); + pci_dev_put(bridge->pci_dev); list_del(&bridge->list); kfree(bridge); diff --git a/drivers/pci/intel-iommu.c b/drivers/pci/intel-iommu.c index 9dbd5066acaf..001b328adf80 100644 --- a/drivers/pci/intel-iommu.c +++ b/drivers/pci/intel-iommu.c @@ -36,6 +36,7 @@ #include <linux/iova.h> #include <linux/iommu.h> #include <linux/intel-iommu.h> +#include <linux/sysdev.h> #include <asm/cacheflush.h> #include <asm/iommu.h> #include "pci.h" @@ -55,8 +56,8 @@ #define DOMAIN_MAX_ADDR(gaw) ((((u64)1) << gaw) - 1) #define IOVA_PFN(addr) ((addr) >> PAGE_SHIFT) -#define DMA_32BIT_PFN IOVA_PFN(DMA_32BIT_MASK) -#define DMA_64BIT_PFN IOVA_PFN(DMA_64BIT_MASK) +#define DMA_32BIT_PFN IOVA_PFN(DMA_BIT_MASK(32)) +#define DMA_64BIT_PFN IOVA_PFN(DMA_BIT_MASK(64)) /* global iommu list, set NULL for ignored DMAR units */ static struct intel_iommu **g_iommus; @@ -164,7 +165,8 @@ static inline void context_clear_entry(struct context_entry *context) * 1: writable * 2-6: reserved * 7: super page - * 8-11: available + * 8-10: available + * 11: snoop behavior * 12-63: Host physcial address */ struct dma_pte { @@ -186,6 +188,11 @@ static inline void dma_set_pte_writable(struct dma_pte *pte) pte->val |= DMA_PTE_WRITE; } +static inline void dma_set_pte_snp(struct dma_pte *pte) +{ + pte->val |= DMA_PTE_SNP; +} + static inline void dma_set_pte_prot(struct dma_pte *pte, unsigned long prot) { pte->val = (pte->val & ~3) | (prot & 3); @@ -231,6 +238,7 @@ struct dmar_domain { int flags; /* flags to find out type of domain */ int iommu_coherency;/* indicate coherency of iommu access */ + int iommu_snooping; /* indicate snooping control feature*/ int iommu_count; /* reference count of iommu */ spinlock_t iommu_lock; /* protect iommu set in domain */ u64 max_addr; /* maximum mapped address */ @@ -240,7 +248,8 @@ struct dmar_domain { struct device_domain_info { struct list_head link; /* link to domain siblings */ struct list_head global; /* link to global list */ - u8 bus; /* PCI bus numer */ + int segment; /* PCI domain */ + u8 bus; /* PCI bus number */ u8 devfn; /* PCI devfn number */ struct pci_dev *dev; /* it's NULL for PCIE-to-PCI bridge */ struct dmar_domain *domain; /* pointer to domain */ @@ -421,7 +430,6 @@ static struct intel_iommu *domain_get_iommu(struct dmar_domain *domain) return g_iommus[iommu_id]; } -/* "Coherency" capability may be different across iommus */ static void domain_update_iommu_coherency(struct dmar_domain *domain) { int i; @@ -438,7 +446,30 @@ static void domain_update_iommu_coherency(struct dmar_domain *domain) } } -static struct intel_iommu *device_to_iommu(u8 bus, u8 devfn) +static void domain_update_iommu_snooping(struct dmar_domain *domain) +{ + int i; + + domain->iommu_snooping = 1; + + i = find_first_bit(&domain->iommu_bmp, g_num_of_iommus); + for (; i < g_num_of_iommus; ) { + if (!ecap_sc_support(g_iommus[i]->ecap)) { + domain->iommu_snooping = 0; + break; + } + i = find_next_bit(&domain->iommu_bmp, g_num_of_iommus, i+1); + } +} + +/* Some capabilities may be different across iommus */ +static void domain_update_iommu_cap(struct dmar_domain *domain) +{ + domain_update_iommu_coherency(domain); + domain_update_iommu_snooping(domain); +} + +static struct intel_iommu *device_to_iommu(int segment, u8 bus, u8 devfn) { struct dmar_drhd_unit *drhd = NULL; int i; @@ -446,12 +477,20 @@ static struct intel_iommu *device_to_iommu(u8 bus, u8 devfn) for_each_drhd_unit(drhd) { if (drhd->ignored) continue; + if (segment != drhd->segment) + continue; - for (i = 0; i < drhd->devices_cnt; i++) + for (i = 0; i < drhd->devices_cnt; i++) { if (drhd->devices[i] && drhd->devices[i]->bus->number == bus && drhd->devices[i]->devfn == devfn) return drhd->iommu; + if (drhd->devices[i] && + drhd->devices[i]->subordinate && + drhd->devices[i]->subordinate->number <= bus && + drhd->devices[i]->subordinate->subordinate >= bus) + return drhd->iommu; + } if (drhd->include_all) return drhd->iommu; @@ -689,15 +728,17 @@ static void dma_pte_clear_one(struct dmar_domain *domain, u64 addr) static void dma_pte_clear_range(struct dmar_domain *domain, u64 start, u64 end) { int addr_width = agaw_to_width(domain->agaw); + int npages; start &= (((u64)1) << addr_width) - 1; end &= (((u64)1) << addr_width) - 1; /* in case it's partial page */ - start = PAGE_ALIGN(start); - end &= PAGE_MASK; + start &= PAGE_MASK; + end = PAGE_ALIGN(end); + npages = (end - start) / VTD_PAGE_SIZE; /* we don't need lock here, nobody else touches the iova range */ - while (start < end) { + while (npages--) { dma_pte_clear_one(domain, start); start += VTD_PAGE_SIZE; } @@ -1241,6 +1282,11 @@ static int domain_init(struct dmar_domain *domain, int guest_width) else domain->iommu_coherency = 0; + if (ecap_sc_support(iommu->ecap)) + domain->iommu_snooping = 1; + else + domain->iommu_snooping = 0; + domain->iommu_count = 1; /* always allocate the top pgd */ @@ -1276,7 +1322,7 @@ static void domain_exit(struct dmar_domain *domain) } static int domain_context_mapping_one(struct dmar_domain *domain, - u8 bus, u8 devfn) + int segment, u8 bus, u8 devfn) { struct context_entry *context; unsigned long flags; @@ -1291,7 +1337,7 @@ static int domain_context_mapping_one(struct dmar_domain *domain, bus, PCI_SLOT(devfn), PCI_FUNC(devfn)); BUG_ON(!domain->pgd); - iommu = device_to_iommu(bus, devfn); + iommu = device_to_iommu(segment, bus, devfn); if (!iommu) return -ENODEV; @@ -1369,7 +1415,7 @@ static int domain_context_mapping_one(struct dmar_domain *domain, spin_lock_irqsave(&domain->iommu_lock, flags); if (!test_and_set_bit(iommu->seq_id, &domain->iommu_bmp)) { domain->iommu_count++; - domain_update_iommu_coherency(domain); + domain_update_iommu_cap(domain); } spin_unlock_irqrestore(&domain->iommu_lock, flags); return 0; @@ -1381,8 +1427,8 @@ domain_context_mapping(struct dmar_domain *domain, struct pci_dev *pdev) int ret; struct pci_dev *tmp, *parent; - ret = domain_context_mapping_one(domain, pdev->bus->number, - pdev->devfn); + ret = domain_context_mapping_one(domain, pci_domain_nr(pdev->bus), + pdev->bus->number, pdev->devfn); if (ret) return ret; @@ -1393,18 +1439,23 @@ domain_context_mapping(struct dmar_domain *domain, struct pci_dev *pdev) /* Secondary interface's bus number and devfn 0 */ parent = pdev->bus->self; while (parent != tmp) { - ret = domain_context_mapping_one(domain, parent->bus->number, - parent->devfn); + ret = domain_context_mapping_one(domain, + pci_domain_nr(parent->bus), + parent->bus->number, + parent->devfn); if (ret) return ret; parent = parent->bus->self; } if (tmp->is_pcie) /* this is a PCIE-to-PCI bridge */ return domain_context_mapping_one(domain, - tmp->subordinate->number, 0); + pci_domain_nr(tmp->subordinate), + tmp->subordinate->number, 0); else /* this is a legacy PCI bridge */ return domain_context_mapping_one(domain, - tmp->bus->number, tmp->devfn); + pci_domain_nr(tmp->bus), + tmp->bus->number, + tmp->devfn); } static int domain_context_mapped(struct pci_dev *pdev) @@ -1413,12 +1464,12 @@ static int domain_context_mapped(struct pci_dev *pdev) struct pci_dev *tmp, *parent; struct intel_iommu *iommu; - iommu = device_to_iommu(pdev->bus->number, pdev->devfn); + iommu = device_to_iommu(pci_domain_nr(pdev->bus), pdev->bus->number, + pdev->devfn); if (!iommu) return -ENODEV; - ret = device_context_mapped(iommu, - pdev->bus->number, pdev->devfn); + ret = device_context_mapped(iommu, pdev->bus->number, pdev->devfn); if (!ret) return ret; /* dependent device mapping */ @@ -1429,17 +1480,17 @@ static int domain_context_mapped(struct pci_dev *pdev) parent = pdev->bus->self; while (parent != tmp) { ret = device_context_mapped(iommu, parent->bus->number, - parent->devfn); + parent->devfn); if (!ret) return ret; parent = parent->bus->self; } if (tmp->is_pcie) - return device_context_mapped(iommu, - tmp->subordinate->number, 0); + return device_context_mapped(iommu, tmp->subordinate->number, + 0); else - return device_context_mapped(iommu, - tmp->bus->number, tmp->devfn); + return device_context_mapped(iommu, tmp->bus->number, + tmp->devfn); } static int @@ -1469,6 +1520,8 @@ domain_page_mapping(struct dmar_domain *domain, dma_addr_t iova, BUG_ON(dma_pte_addr(pte)); dma_set_pte_addr(pte, start_pfn << VTD_PAGE_SHIFT); dma_set_pte_prot(pte, prot); + if (prot & DMA_PTE_SNP) + dma_set_pte_snp(pte); domain_flush_cache(domain, pte, sizeof(*pte)); start_pfn++; index++; @@ -1504,7 +1557,7 @@ static void domain_remove_dev_info(struct dmar_domain *domain) info->dev->dev.archdata.iommu = NULL; spin_unlock_irqrestore(&device_domain_lock, flags); - iommu = device_to_iommu(info->bus, info->devfn); + iommu = device_to_iommu(info->segment, info->bus, info->devfn); iommu_detach_dev(iommu, info->bus, info->devfn); free_devinfo_mem(info); @@ -1539,11 +1592,14 @@ static struct dmar_domain *get_domain_for_dev(struct pci_dev *pdev, int gaw) struct pci_dev *dev_tmp; unsigned long flags; int bus = 0, devfn = 0; + int segment; domain = find_domain(pdev); if (domain) return domain; + segment = pci_domain_nr(pdev->bus); + dev_tmp = pci_find_upstream_pcie_bridge(pdev); if (dev_tmp) { if (dev_tmp->is_pcie) { @@ -1555,7 +1611,8 @@ static struct dmar_domain *get_domain_for_dev(struct pci_dev *pdev, int gaw) } spin_lock_irqsave(&device_domain_lock, flags); list_for_each_entry(info, &device_domain_list, global) { - if (info->bus == bus && info->devfn == devfn) { + if (info->segment == segment && + info->bus == bus && info->devfn == devfn) { found = info->domain; break; } @@ -1593,6 +1650,7 @@ static struct dmar_domain *get_domain_for_dev(struct pci_dev *pdev, int gaw) domain_exit(domain); goto error; } + info->segment = segment; info->bus = bus; info->devfn = devfn; info->dev = NULL; @@ -1604,7 +1662,8 @@ static struct dmar_domain *get_domain_for_dev(struct pci_dev *pdev, int gaw) found = NULL; spin_lock_irqsave(&device_domain_lock, flags); list_for_each_entry(tmp, &device_domain_list, global) { - if (tmp->bus == bus && tmp->devfn == devfn) { + if (tmp->segment == segment && + tmp->bus == bus && tmp->devfn == devfn) { found = tmp->domain; break; } @@ -1624,6 +1683,7 @@ found_domain: info = alloc_devinfo_mem(); if (!info) goto error; + info->segment = segment; info->bus = pdev->bus->number; info->devfn = pdev->devfn; info->dev = pdev; @@ -1908,6 +1968,15 @@ static int __init init_dmars(void) } } +#ifdef CONFIG_INTR_REMAP + if (!intr_remapping_enabled) { + ret = enable_intr_remapping(0); + if (ret) + printk(KERN_ERR + "IOMMU: enable interrupt remapping failed\n"); + } +#endif + /* * For each rmrr * for each dev attached to rmrr @@ -2011,15 +2080,15 @@ __intel_alloc_iova(struct device *dev, struct dmar_domain *domain, struct pci_dev *pdev = to_pci_dev(dev); struct iova *iova = NULL; - if (dma_mask <= DMA_32BIT_MASK || dmar_forcedac) + if (dma_mask <= DMA_BIT_MASK(32) || dmar_forcedac) iova = iommu_alloc_iova(domain, size, dma_mask); else { /* * First try to allocate an io virtual address in - * DMA_32BIT_MASK and if that fails then try allocating + * DMA_BIT_MASK(32) and if that fails then try allocating * from higher range */ - iova = iommu_alloc_iova(domain, size, DMA_32BIT_MASK); + iova = iommu_alloc_iova(domain, size, DMA_BIT_MASK(32)); if (!iova) iova = iommu_alloc_iova(domain, size, dma_mask); } @@ -2119,7 +2188,7 @@ static dma_addr_t __intel_map_single(struct device *hwdev, phys_addr_t paddr, error: if (iova) __free_iova(&domain->iovad, iova); - printk(KERN_ERR"Device %s request: %lx@%llx dir %d --- failed\n", + printk(KERN_ERR"Device %s request: %zx@%llx dir %d --- failed\n", pci_name(pdev), size, (unsigned long long)paddr, dir); return 0; } @@ -2218,7 +2287,7 @@ static void intel_unmap_page(struct device *dev, dma_addr_t dev_addr, start_addr = iova->pfn_lo << PAGE_SHIFT; size = aligned_size((u64)dev_addr, size); - pr_debug("Device %s unmapping: %lx@%llx\n", + pr_debug("Device %s unmapping: %zx@%llx\n", pci_name(pdev), size, (unsigned long long)start_addr); /* clear the whole page */ @@ -2282,8 +2351,6 @@ static void intel_free_coherent(struct device *hwdev, size_t size, void *vaddr, free_pages((unsigned long)vaddr, order); } -#define SG_ENT_VIRT_ADDRESS(sg) (sg_virt((sg))) - static void intel_unmap_sg(struct device *hwdev, struct scatterlist *sglist, int nelems, enum dma_data_direction dir, struct dma_attrs *attrs) @@ -2294,7 +2361,7 @@ static void intel_unmap_sg(struct device *hwdev, struct scatterlist *sglist, unsigned long start_addr; struct iova *iova; size_t size = 0; - void *addr; + phys_addr_t addr; struct scatterlist *sg; struct intel_iommu *iommu; @@ -2310,7 +2377,7 @@ static void intel_unmap_sg(struct device *hwdev, struct scatterlist *sglist, if (!iova) return; for_each_sg(sglist, sg, nelems, i) { - addr = SG_ENT_VIRT_ADDRESS(sg); + addr = page_to_phys(sg_page(sg)) + sg->offset; size += aligned_size((u64)addr, sg->length); } @@ -2337,7 +2404,7 @@ static int intel_nontranslate_map_sg(struct device *hddev, for_each_sg(sglist, sg, nelems, i) { BUG_ON(!sg_page(sg)); - sg->dma_address = virt_to_bus(SG_ENT_VIRT_ADDRESS(sg)); + sg->dma_address = page_to_phys(sg_page(sg)) + sg->offset; sg->dma_length = sg->length; } return nelems; @@ -2346,7 +2413,7 @@ static int intel_nontranslate_map_sg(struct device *hddev, static int intel_map_sg(struct device *hwdev, struct scatterlist *sglist, int nelems, enum dma_data_direction dir, struct dma_attrs *attrs) { - void *addr; + phys_addr_t addr; int i; struct pci_dev *pdev = to_pci_dev(hwdev); struct dmar_domain *domain; @@ -2370,8 +2437,7 @@ static int intel_map_sg(struct device *hwdev, struct scatterlist *sglist, int ne iommu = domain_get_iommu(domain); for_each_sg(sglist, sg, nelems, i) { - addr = SG_ENT_VIRT_ADDRESS(sg); - addr = (void *)virt_to_phys(addr); + addr = page_to_phys(sg_page(sg)) + sg->offset; size += aligned_size((u64)addr, sg->length); } @@ -2394,8 +2460,7 @@ static int intel_map_sg(struct device *hwdev, struct scatterlist *sglist, int ne start_addr = iova->pfn_lo << PAGE_SHIFT; offset = 0; for_each_sg(sglist, sg, nelems, i) { - addr = SG_ENT_VIRT_ADDRESS(sg); - addr = (void *)virt_to_phys(addr); + addr = page_to_phys(sg_page(sg)) + sg->offset; size = aligned_size((u64)addr, sg->length); ret = domain_page_mapping(domain, start_addr + offset, ((u64)addr) & PAGE_MASK, @@ -2563,6 +2628,150 @@ static void __init init_no_remapping_devices(void) } } +#ifdef CONFIG_SUSPEND +static int init_iommu_hw(void) +{ + struct dmar_drhd_unit *drhd; + struct intel_iommu *iommu = NULL; + + for_each_active_iommu(iommu, drhd) + if (iommu->qi) + dmar_reenable_qi(iommu); + + for_each_active_iommu(iommu, drhd) { + iommu_flush_write_buffer(iommu); + + iommu_set_root_entry(iommu); + + iommu->flush.flush_context(iommu, 0, 0, 0, + DMA_CCMD_GLOBAL_INVL, 0); + iommu->flush.flush_iotlb(iommu, 0, 0, 0, + DMA_TLB_GLOBAL_FLUSH, 0); + iommu_disable_protect_mem_regions(iommu); + iommu_enable_translation(iommu); + } + + return 0; +} + +static void iommu_flush_all(void) +{ + struct dmar_drhd_unit *drhd; + struct intel_iommu *iommu; + + for_each_active_iommu(iommu, drhd) { + iommu->flush.flush_context(iommu, 0, 0, 0, + DMA_CCMD_GLOBAL_INVL, 0); + iommu->flush.flush_iotlb(iommu, 0, 0, 0, + DMA_TLB_GLOBAL_FLUSH, 0); + } +} + +static int iommu_suspend(struct sys_device *dev, pm_message_t state) +{ + struct dmar_drhd_unit *drhd; + struct intel_iommu *iommu = NULL; + unsigned long flag; + + for_each_active_iommu(iommu, drhd) { + iommu->iommu_state = kzalloc(sizeof(u32) * MAX_SR_DMAR_REGS, + GFP_ATOMIC); + if (!iommu->iommu_state) + goto nomem; + } + + iommu_flush_all(); + + for_each_active_iommu(iommu, drhd) { + iommu_disable_translation(iommu); + + spin_lock_irqsave(&iommu->register_lock, flag); + + iommu->iommu_state[SR_DMAR_FECTL_REG] = + readl(iommu->reg + DMAR_FECTL_REG); + iommu->iommu_state[SR_DMAR_FEDATA_REG] = + readl(iommu->reg + DMAR_FEDATA_REG); + iommu->iommu_state[SR_DMAR_FEADDR_REG] = + readl(iommu->reg + DMAR_FEADDR_REG); + iommu->iommu_state[SR_DMAR_FEUADDR_REG] = + readl(iommu->reg + DMAR_FEUADDR_REG); + + spin_unlock_irqrestore(&iommu->register_lock, flag); + } + return 0; + +nomem: + for_each_active_iommu(iommu, drhd) + kfree(iommu->iommu_state); + + return -ENOMEM; +} + +static int iommu_resume(struct sys_device *dev) +{ + struct dmar_drhd_unit *drhd; + struct intel_iommu *iommu = NULL; + unsigned long flag; + + if (init_iommu_hw()) { + WARN(1, "IOMMU setup failed, DMAR can not resume!\n"); + return -EIO; + } + + for_each_active_iommu(iommu, drhd) { + + spin_lock_irqsave(&iommu->register_lock, flag); + + writel(iommu->iommu_state[SR_DMAR_FECTL_REG], + iommu->reg + DMAR_FECTL_REG); + writel(iommu->iommu_state[SR_DMAR_FEDATA_REG], + iommu->reg + DMAR_FEDATA_REG); + writel(iommu->iommu_state[SR_DMAR_FEADDR_REG], + iommu->reg + DMAR_FEADDR_REG); + writel(iommu->iommu_state[SR_DMAR_FEUADDR_REG], + iommu->reg + DMAR_FEUADDR_REG); + + spin_unlock_irqrestore(&iommu->register_lock, flag); + } + + for_each_active_iommu(iommu, drhd) + kfree(iommu->iommu_state); + + return 0; +} + +static struct sysdev_class iommu_sysclass = { + .name = "iommu", + .resume = iommu_resume, + .suspend = iommu_suspend, +}; + +static struct sys_device device_iommu = { + .cls = &iommu_sysclass, +}; + +static int __init init_iommu_sysfs(void) +{ + int error; + + error = sysdev_class_register(&iommu_sysclass); + if (error) + return error; + + error = sysdev_register(&device_iommu); + if (error) + sysdev_class_unregister(&iommu_sysclass); + + return error; +} + +#else +static int __init init_iommu_sysfs(void) +{ + return 0; +} +#endif /* CONFIG_PM */ + int __init intel_iommu_init(void) { int ret = 0; @@ -2598,6 +2807,7 @@ int __init intel_iommu_init(void) init_timer(&unmap_timer); force_iommu = 1; dma_ops = &intel_dma_ops; + init_iommu_sysfs(); register_iommu(&intel_iommu_ops); @@ -2614,6 +2824,7 @@ static int vm_domain_add_dev_info(struct dmar_domain *domain, if (!info) return -ENOMEM; + info->segment = pci_domain_nr(pdev->bus); info->bus = pdev->bus->number; info->devfn = pdev->devfn; info->dev = pdev; @@ -2628,6 +2839,33 @@ static int vm_domain_add_dev_info(struct dmar_domain *domain, return 0; } +static void iommu_detach_dependent_devices(struct intel_iommu *iommu, + struct pci_dev *pdev) +{ + struct pci_dev *tmp, *parent; + + if (!iommu || !pdev) + return; + + /* dependent device detach */ + tmp = pci_find_upstream_pcie_bridge(pdev); + /* Secondary interface's bus number and devfn 0 */ + if (tmp) { + parent = pdev->bus->self; + while (parent != tmp) { + iommu_detach_dev(iommu, parent->bus->number, + parent->devfn); + parent = parent->bus->self; + } + if (tmp->is_pcie) /* this is a PCIE-to-PCI bridge */ + iommu_detach_dev(iommu, + tmp->subordinate->number, 0); + else /* this is a legacy PCI bridge */ + iommu_detach_dev(iommu, tmp->bus->number, + tmp->devfn); + } +} + static void vm_domain_remove_one_dev_info(struct dmar_domain *domain, struct pci_dev *pdev) { @@ -2637,13 +2875,15 @@ static void vm_domain_remove_one_dev_info(struct dmar_domain *domain, int found = 0; struct list_head *entry, *tmp; - iommu = device_to_iommu(pdev->bus->number, pdev->devfn); + iommu = device_to_iommu(pci_domain_nr(pdev->bus), pdev->bus->number, + pdev->devfn); if (!iommu) return; spin_lock_irqsave(&device_domain_lock, flags); list_for_each_safe(entry, tmp, &domain->devices) { info = list_entry(entry, struct device_domain_info, link); + /* No need to compare PCI domain; it has to be the same */ if (info->bus == pdev->bus->number && info->devfn == pdev->devfn) { list_del(&info->link); @@ -2653,6 +2893,7 @@ static void vm_domain_remove_one_dev_info(struct dmar_domain *domain, spin_unlock_irqrestore(&device_domain_lock, flags); iommu_detach_dev(iommu, info->bus, info->devfn); + iommu_detach_dependent_devices(iommu, pdev); free_devinfo_mem(info); spin_lock_irqsave(&device_domain_lock, flags); @@ -2667,7 +2908,8 @@ static void vm_domain_remove_one_dev_info(struct dmar_domain *domain, * owned by this domain, clear this iommu in iommu_bmp * update iommu count and coherency */ - if (device_to_iommu(info->bus, info->devfn) == iommu) + if (iommu == device_to_iommu(info->segment, info->bus, + info->devfn)) found = 1; } @@ -2676,7 +2918,7 @@ static void vm_domain_remove_one_dev_info(struct dmar_domain *domain, spin_lock_irqsave(&domain->iommu_lock, tmp_flags); clear_bit(iommu->seq_id, &domain->iommu_bmp); domain->iommu_count--; - domain_update_iommu_coherency(domain); + domain_update_iommu_cap(domain); spin_unlock_irqrestore(&domain->iommu_lock, tmp_flags); } @@ -2700,17 +2942,18 @@ static void vm_domain_remove_all_dev_info(struct dmar_domain *domain) spin_unlock_irqrestore(&device_domain_lock, flags1); - iommu = device_to_iommu(info->bus, info->devfn); + iommu = device_to_iommu(info->segment, info->bus, info->devfn); iommu_detach_dev(iommu, info->bus, info->devfn); + iommu_detach_dependent_devices(iommu, info->dev); /* clear this iommu in iommu_bmp, update iommu count - * and coherency + * and capabilities */ spin_lock_irqsave(&domain->iommu_lock, flags2); if (test_and_clear_bit(iommu->seq_id, &domain->iommu_bmp)) { domain->iommu_count--; - domain_update_iommu_coherency(domain); + domain_update_iommu_cap(domain); } spin_unlock_irqrestore(&domain->iommu_lock, flags2); @@ -2887,7 +3130,8 @@ static int intel_iommu_attach_device(struct iommu_domain *domain, } } - iommu = device_to_iommu(pdev->bus->number, pdev->devfn); + iommu = device_to_iommu(pci_domain_nr(pdev->bus), pdev->bus->number, + pdev->devfn); if (!iommu) return -ENODEV; @@ -2933,6 +3177,8 @@ static int intel_iommu_map_range(struct iommu_domain *domain, prot |= DMA_PTE_READ; if (iommu_prot & IOMMU_WRITE) prot |= DMA_PTE_WRITE; + if ((iommu_prot & IOMMU_CACHE) && dmar_domain->iommu_snooping) + prot |= DMA_PTE_SNP; max_addr = (iova & VTD_PAGE_MASK) + VTD_PAGE_ALIGN(size); if (dmar_domain->max_addr < max_addr) { @@ -2986,6 +3232,17 @@ static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain, return phys; } +static int intel_iommu_domain_has_cap(struct iommu_domain *domain, + unsigned long cap) +{ + struct dmar_domain *dmar_domain = domain->priv; + + if (cap == IOMMU_CAP_CACHE_COHERENCY) + return dmar_domain->iommu_snooping; + + return 0; +} + static struct iommu_ops intel_iommu_ops = { .domain_init = intel_iommu_domain_init, .domain_destroy = intel_iommu_domain_destroy, @@ -2994,6 +3251,7 @@ static struct iommu_ops intel_iommu_ops = { .map = intel_iommu_map_range, .unmap = intel_iommu_unmap_range, .iova_to_phys = intel_iommu_iova_to_phys, + .domain_has_cap = intel_iommu_domain_has_cap, }; static void __devinit quirk_iommu_rwbf(struct pci_dev *dev) diff --git a/drivers/pci/intr_remapping.c b/drivers/pci/intr_remapping.c index b041a409f4a7..f5e0ea724a6f 100644 --- a/drivers/pci/intr_remapping.c +++ b/drivers/pci/intr_remapping.c @@ -9,6 +9,7 @@ #include <asm/cpu.h> #include <linux/intel-iommu.h> #include "intr_remapping.h" +#include <acpi/acpi.h> static struct ioapic_scope ir_ioapic[MAX_IO_APICS]; static int ir_ioapic_num; @@ -415,12 +416,27 @@ static void iommu_set_intr_remapping(struct intel_iommu *iommu, int mode) /* Set interrupt-remapping table pointer */ cmd = iommu->gcmd | DMA_GCMD_SIRTP; + iommu->gcmd |= DMA_GCMD_SIRTP; writel(cmd, iommu->reg + DMAR_GCMD_REG); IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl, (sts & DMA_GSTS_IRTPS), sts); spin_unlock_irqrestore(&iommu->register_lock, flags); + if (mode == 0) { + spin_lock_irqsave(&iommu->register_lock, flags); + + /* enable comaptiblity format interrupt pass through */ + cmd = iommu->gcmd | DMA_GCMD_CFI; + iommu->gcmd |= DMA_GCMD_CFI; + writel(cmd, iommu->reg + DMAR_GCMD_REG); + + IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, + readl, (sts & DMA_GSTS_CFIS), sts); + + spin_unlock_irqrestore(&iommu->register_lock, flags); + } + /* * global invalidation of interrupt entry cache before enabling * interrupt-remapping. @@ -470,7 +486,7 @@ static int setup_intr_remapping(struct intel_iommu *iommu, int mode) /* * Disable Interrupt Remapping. */ -static void disable_intr_remapping(struct intel_iommu *iommu) +static void iommu_disable_intr_remapping(struct intel_iommu *iommu) { unsigned long flags; u32 sts; @@ -478,6 +494,12 @@ static void disable_intr_remapping(struct intel_iommu *iommu) if (!ecap_ir_support(iommu->ecap)) return; + /* + * global invalidation of interrupt entry cache before disabling + * interrupt-remapping. + */ + qi_global_iec(iommu); + spin_lock_irqsave(&iommu->register_lock, flags); sts = dmar_readq(iommu->reg + DMAR_GSTS_REG); @@ -503,6 +525,13 @@ int __init enable_intr_remapping(int eim) struct intel_iommu *iommu = drhd->iommu; /* + * If the queued invalidation is already initialized, + * shouldn't disable it. + */ + if (iommu->qi) + continue; + + /* * Clear previous faults. */ dmar_fault(-1, iommu); @@ -511,7 +540,7 @@ int __init enable_intr_remapping(int eim) * Disable intr remapping and queued invalidation, if already * enabled prior to OS handover. */ - disable_intr_remapping(iommu); + iommu_disable_intr_remapping(iommu); dmar_disable_qi(iommu); } @@ -639,3 +668,54 @@ int __init parse_ioapics_under_ir(void) return ir_supported; } + +void disable_intr_remapping(void) +{ + struct dmar_drhd_unit *drhd; + struct intel_iommu *iommu = NULL; + + /* + * Disable Interrupt-remapping for all the DRHD's now. + */ + for_each_iommu(iommu, drhd) { + if (!ecap_ir_support(iommu->ecap)) + continue; + + iommu_disable_intr_remapping(iommu); + } +} + +int reenable_intr_remapping(int eim) +{ + struct dmar_drhd_unit *drhd; + int setup = 0; + struct intel_iommu *iommu = NULL; + + for_each_iommu(iommu, drhd) + if (iommu->qi) + dmar_reenable_qi(iommu); + + /* + * Setup Interrupt-remapping for all the DRHD's now. + */ + for_each_iommu(iommu, drhd) { + if (!ecap_ir_support(iommu->ecap)) + continue; + + /* Set up interrupt remapping for iommu.*/ + iommu_set_intr_remapping(iommu, eim); + setup = 1; + } + + if (!setup) + goto error; + + return 0; + +error: + /* + * handle error condition gracefully here! + */ + return -1; +} + diff --git a/drivers/pci/iov.c b/drivers/pci/iov.c index 7227efc760db..b497daab3d4a 100644 --- a/drivers/pci/iov.c +++ b/drivers/pci/iov.c @@ -631,6 +631,7 @@ int pci_iov_bus_range(struct pci_bus *bus) /** * pci_enable_sriov - enable the SR-IOV capability * @dev: the PCI device + * @nr_virtfn: number of virtual functions to enable * * Returns 0 on success, or negative on failure. */ diff --git a/drivers/pci/pci-acpi.c b/drivers/pci/pci-acpi.c index fac5eddcefd2..ea15b0537457 100644 --- a/drivers/pci/pci-acpi.c +++ b/drivers/pci/pci-acpi.c @@ -171,12 +171,12 @@ static int __init acpi_pci_init(void) { int ret; - if (acpi_gbl_FADT.boot_flags & BAF_MSI_NOT_SUPPORTED) { + if (acpi_gbl_FADT.boot_flags & ACPI_FADT_NO_MSI) { printk(KERN_INFO"ACPI FADT declares the system doesn't support MSI, so disable it\n"); pci_no_msi(); } - if (acpi_gbl_FADT.boot_flags & BAF_PCIE_ASPM_CONTROL) { + if (acpi_gbl_FADT.boot_flags & ACPI_FADT_NO_ASPM) { printk(KERN_INFO"ACPI FADT declares the system doesn't support PCIe ASPM, so disable it\n"); pcie_no_aspm(); } diff --git a/drivers/pci/pci-driver.c b/drivers/pci/pci-driver.c index c0cbbb5a245e..d76c4c85367e 100644 --- a/drivers/pci/pci-driver.c +++ b/drivers/pci/pci-driver.c @@ -277,10 +277,9 @@ static int pci_call_probe(struct pci_driver *drv, struct pci_dev *dev, node = dev_to_node(&dev->dev); if (node >= 0) { int cpu; - node_to_cpumask_ptr(nodecpumask, node); get_online_cpus(); - cpu = cpumask_any_and(nodecpumask, cpu_online_mask); + cpu = cpumask_any_and(cpumask_of_node(node), cpu_online_mask); if (cpu < nr_cpu_ids) error = work_on_cpu(cpu, local_pci_probe, &ddi); else diff --git a/drivers/pci/pci-sysfs.c b/drivers/pci/pci-sysfs.c index e9a8706a6401..a7eb1b46a5a8 100644 --- a/drivers/pci/pci-sysfs.c +++ b/drivers/pci/pci-sysfs.c @@ -148,7 +148,7 @@ static ssize_t is_enabled_store(struct device *dev, return -EPERM; if (!val) { - if (atomic_read(&pdev->enable_cnt) != 0) + if (pci_is_enabled(pdev)) pci_disable_device(pdev); else result = -EIO; @@ -277,14 +277,10 @@ remove_store(struct device *dev, struct device_attribute *dummy, { int ret = 0; unsigned long val; - struct pci_dev *pdev = to_pci_dev(dev); if (strict_strtoul(buf, 0, &val) < 0) return -EINVAL; - if (pci_is_root_bus(pdev->bus)) - return -EBUSY; - /* An attribute cannot be unregistered by one of its own methods, * so we have to use this roundabout approach. */ diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c index fe7ac2cea7c9..16fd0d4c3166 100644 --- a/drivers/pci/pci.c +++ b/drivers/pci/pci.c @@ -593,7 +593,7 @@ EXPORT_SYMBOL_GPL(__pci_complete_power_transition); * @dev: PCI device to handle. * @state: PCI power state (D0, D1, D2, D3hot) to put the device into. * - * Transition a device to a new power state, using the platform formware and/or + * Transition a device to a new power state, using the platform firmware and/or * the device's PCI PM registers. * * RETURN VALUE: @@ -844,7 +844,7 @@ static int do_pci_enable_device(struct pci_dev *dev, int bars) */ int pci_reenable_device(struct pci_dev *dev) { - if (atomic_read(&dev->enable_cnt)) + if (pci_is_enabled(dev)) return do_pci_enable_device(dev, (1 << PCI_NUM_RESOURCES) - 1); return 0; } @@ -1042,7 +1042,7 @@ static void do_pci_disable_device(struct pci_dev *dev) */ void pci_disable_enabled_device(struct pci_dev *dev) { - if (atomic_read(&dev->enable_cnt)) + if (pci_is_enabled(dev)) do_pci_disable_device(dev); } diff --git a/drivers/pci/probe.c b/drivers/pci/probe.c index e2f3dd098cfa..8eb50dffb78a 100644 --- a/drivers/pci/probe.c +++ b/drivers/pci/probe.c @@ -1220,7 +1220,7 @@ EXPORT_SYMBOL(pci_scan_bus_parented); * * Returns the max number of subordinate bus discovered. */ -unsigned int __devinit pci_rescan_bus(struct pci_bus *bus) +unsigned int __ref pci_rescan_bus(struct pci_bus *bus) { unsigned int max; struct pci_dev *dev; diff --git a/drivers/pci/quirks.c b/drivers/pci/quirks.c index 9b2f0d96900d..0254741bece0 100644 --- a/drivers/pci/quirks.c +++ b/drivers/pci/quirks.c @@ -36,17 +36,18 @@ EXPORT_SYMBOL(pcie_mch_quirk); #ifdef CONFIG_PCI_QUIRKS /* - * This quirk function disables the device and releases resources - * which is specified by kernel's boot parameter 'pci=resource_alignment='. + * This quirk function disables memory decoding and releases memory resources + * of the device specified by kernel's boot parameter 'pci=resource_alignment='. * It also rounds up size to specified alignment. * Later on, the kernel will assign page-aligned memory resource back - * to that device. + * to the device. */ static void __devinit quirk_resource_alignment(struct pci_dev *dev) { int i; struct resource *r; resource_size_t align, size; + u16 command; if (!pci_is_reassigndev(dev)) return; @@ -58,8 +59,11 @@ static void __devinit quirk_resource_alignment(struct pci_dev *dev) return; } - dev_info(&dev->dev, "Disabling device and release resources.\n"); - pci_disable_device(dev); + dev_info(&dev->dev, + "Disabling memory decoding and releasing memory resources.\n"); + pci_read_config_word(dev, PCI_COMMAND, &command); + command &= ~PCI_COMMAND_MEMORY; + pci_write_config_word(dev, PCI_COMMAND, command); align = pci_specified_resource_alignment(dev); for (i=0; i < PCI_BRIDGE_RESOURCES; i++) { @@ -2411,6 +2415,54 @@ DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_ATI, 0x4375, #endif /* CONFIG_PCI_MSI */ +#ifdef CONFIG_PCI_IOV + +/* + * For Intel 82576 SR-IOV NIC, if BIOS doesn't allocate resources for the + * SR-IOV BARs, zero the Flash BAR and program the SR-IOV BARs to use the + * old Flash Memory Space. + */ +static void __devinit quirk_i82576_sriov(struct pci_dev *dev) +{ + int pos, flags; + u32 bar, start, size; + + if (PAGE_SIZE > 0x10000) + return; + + flags = pci_resource_flags(dev, 0); + if ((flags & PCI_BASE_ADDRESS_SPACE) != + PCI_BASE_ADDRESS_SPACE_MEMORY || + (flags & PCI_BASE_ADDRESS_MEM_TYPE_MASK) != + PCI_BASE_ADDRESS_MEM_TYPE_32) + return; + + pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_SRIOV); + if (!pos) + return; + + pci_read_config_dword(dev, pos + PCI_SRIOV_BAR, &bar); + if (bar & PCI_BASE_ADDRESS_MEM_MASK) + return; + + start = pci_resource_start(dev, 1); + size = pci_resource_len(dev, 1); + if (!start || size != 0x400000 || start & (size - 1)) + return; + + pci_resource_flags(dev, 1) = 0; + pci_write_config_dword(dev, PCI_BASE_ADDRESS_1, 0); + pci_write_config_dword(dev, pos + PCI_SRIOV_BAR, start); + pci_write_config_dword(dev, pos + PCI_SRIOV_BAR + 12, start + size / 2); + + dev_info(&dev->dev, "use Flash Memory Space for SR-IOV BARs\n"); +} +DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x10c9, quirk_i82576_sriov); +DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x10e6, quirk_i82576_sriov); +DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x10e7, quirk_i82576_sriov); + +#endif /* CONFIG_PCI_IOV */ + static void pci_do_fixups(struct pci_dev *dev, struct pci_fixup *f, struct pci_fixup *end) { diff --git a/drivers/pci/setup-bus.c b/drivers/pci/setup-bus.c index 334285a8e237..8d9da9d30a61 100644 --- a/drivers/pci/setup-bus.c +++ b/drivers/pci/setup-bus.c @@ -144,7 +144,7 @@ static void pci_setup_bridge(struct pci_bus *bus) struct pci_bus_region region; u32 l, bu, lu, io_upper16; - if (!pci_is_root_bus(bus) && bus->is_added) + if (pci_is_enabled(bridge)) return; dev_info(&bridge->dev, "PCI bridge, secondary bus %04x:%02x\n", |