From a18037b27ebd23edf5edad8bc6ceb72e4bb5716d Mon Sep 17 00:00:00 2001 From: Mitchel Humpherys Date: Wed, 30 Jul 2014 18:58:13 +0100 Subject: iommu/arm-smmu: avoid calling request_irq in atomic context request_irq shouldn't be called from atomic context since it might sleep, but we're calling it with a spinlock held, resulting in: [ 9.172202] BUG: sleeping function called from invalid context at kernel/mm/slub.c:926 [ 9.182989] in_atomic(): 1, irqs_disabled(): 128, pid: 1, name: swapper/0 [ 9.189762] CPU: 1 PID: 1 Comm: swapper/0 Tainted: G W 3.10.40-gbc1b510b-38437-g55831d3bd9-dirty #97 [ 9.199757] [] (unwind_backtrace+0x0/0x11c) from [] (show_stack+0x10/0x14) [ 9.208346] [] (show_stack+0x10/0x14) from [] (kmem_cache_alloc_trace+0x3c/0x210) [ 9.217543] [] (kmem_cache_alloc_trace+0x3c/0x210) from [] (request_threaded_irq+0x88/0x11c) [ 9.227702] [] (request_threaded_irq+0x88/0x11c) from [] (arm_smmu_attach_dev+0x188/0x858) [ 9.237686] [] (arm_smmu_attach_dev+0x188/0x858) from [] (arm_iommu_attach_device+0x18/0xd0) [ 9.247837] [] (arm_iommu_attach_device+0x18/0xd0) from [] (arm_smmu_test_probe+0x68/0xd4) [ 9.257823] [] (arm_smmu_test_probe+0x68/0xd4) from [] (driver_probe_device+0x12c/0x330) [ 9.267629] [] (driver_probe_device+0x12c/0x330) from [] (__driver_attach+0x68/0x8c) [ 9.277090] [] (__driver_attach+0x68/0x8c) from [] (bus_for_each_dev+0x70/0x84) [ 9.286118] [] (bus_for_each_dev+0x70/0x84) from [] (bus_add_driver+0x100/0x244) [ 9.295233] [] (bus_add_driver+0x100/0x244) from [] (driver_register+0x9c/0x124) [ 9.304347] [] (driver_register+0x9c/0x124) from [] (arm_smmu_test_init+0x14/0x38) [ 9.313635] [] (arm_smmu_test_init+0x14/0x38) from [] (do_one_initcall+0xb8/0x160) [ 9.322926] [] (do_one_initcall+0xb8/0x160) from [] (kernel_init_freeable+0x108/0x1cc) [ 9.332564] [] (kernel_init_freeable+0x108/0x1cc) from [] (kernel_init+0xc/0xe4) [ 9.341675] [] (kernel_init+0xc/0xe4) from [] (ret_from_fork+0x14/0x3c) Fix this by moving the request_irq out of the critical section. This should be okay since smmu_domain->smmu is still being protected by the critical section. Also, we still don't program the Stream Match Register until after registering our interrupt handler so we shouldn't be missing any interrupts. Cc: Signed-off-by: Mitchel Humpherys [will: code cleanup and fixed request_irq token parameter] Signed-off-by: Will Deacon --- drivers/iommu/arm-smmu.c | 47 +++++++++++++++++++++++++---------------------- 1 file changed, 25 insertions(+), 22 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c index ca18d6d42a9b..98fcd87cbacb 100644 --- a/drivers/iommu/arm-smmu.c +++ b/drivers/iommu/arm-smmu.c @@ -868,10 +868,15 @@ static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain) static int arm_smmu_init_domain_context(struct iommu_domain *domain, struct arm_smmu_device *smmu) { - int irq, ret, start; + int irq, start, ret = 0; + unsigned long flags; struct arm_smmu_domain *smmu_domain = domain->priv; struct arm_smmu_cfg *cfg = &smmu_domain->cfg; + spin_lock_irqsave(&smmu_domain->lock, flags); + if (smmu_domain->smmu) + goto out_unlock; + if (smmu->features & ARM_SMMU_FEAT_TRANS_NESTED) { /* * We will likely want to change this if/when KVM gets @@ -890,7 +895,7 @@ static int arm_smmu_init_domain_context(struct iommu_domain *domain, ret = __arm_smmu_alloc_bitmap(smmu->context_map, start, smmu->num_context_banks); if (IS_ERR_VALUE(ret)) - return ret; + goto out_unlock; cfg->cbndx = ret; if (smmu->version == 1) { @@ -900,6 +905,10 @@ static int arm_smmu_init_domain_context(struct iommu_domain *domain, cfg->irptndx = cfg->cbndx; } + ACCESS_ONCE(smmu_domain->smmu) = smmu; + arm_smmu_init_context_bank(smmu_domain); + spin_unlock_irqrestore(&smmu_domain->lock, flags); + irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx]; ret = request_irq(irq, arm_smmu_context_fault, IRQF_SHARED, "arm-smmu-context-fault", domain); @@ -907,15 +916,12 @@ static int arm_smmu_init_domain_context(struct iommu_domain *domain, dev_err(smmu->dev, "failed to request context IRQ %d (%u)\n", cfg->irptndx, irq); cfg->irptndx = INVALID_IRPTNDX; - goto out_free_context; } - smmu_domain->smmu = smmu; - arm_smmu_init_context_bank(smmu_domain); return 0; -out_free_context: - __arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx); +out_unlock: + spin_unlock_irqrestore(&smmu_domain->lock, flags); return ret; } @@ -1172,11 +1178,10 @@ static void arm_smmu_domain_remove_master(struct arm_smmu_domain *smmu_domain, static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) { - int ret = -EINVAL; + int ret; struct arm_smmu_domain *smmu_domain = domain->priv; - struct arm_smmu_device *smmu; + struct arm_smmu_device *smmu, *dom_smmu; struct arm_smmu_master_cfg *cfg; - unsigned long flags; smmu = dev_get_master_dev(dev)->archdata.iommu; if (!smmu) { @@ -1188,20 +1193,22 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) * Sanity check the domain. We don't support domains across * different SMMUs. */ - spin_lock_irqsave(&smmu_domain->lock, flags); - if (!smmu_domain->smmu) { + dom_smmu = ACCESS_ONCE(smmu_domain->smmu); + if (!dom_smmu) { /* Now that we have a master, we can finalise the domain */ ret = arm_smmu_init_domain_context(domain, smmu); if (IS_ERR_VALUE(ret)) - goto err_unlock; - } else if (smmu_domain->smmu != smmu) { + return ret; + + dom_smmu = smmu_domain->smmu; + } + + if (dom_smmu != smmu) { dev_err(dev, "cannot attach to SMMU %s whilst already attached to domain on SMMU %s\n", - dev_name(smmu_domain->smmu->dev), - dev_name(smmu->dev)); - goto err_unlock; + dev_name(smmu_domain->smmu->dev), dev_name(smmu->dev)); + return -EINVAL; } - spin_unlock_irqrestore(&smmu_domain->lock, flags); /* Looks ok, so add the device to the domain */ cfg = find_smmu_master_cfg(smmu_domain->smmu, dev); @@ -1209,10 +1216,6 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) return -ENODEV; return arm_smmu_domain_add_master(smmu_domain, cfg); - -err_unlock: - spin_unlock_irqrestore(&smmu_domain->lock, flags); - return ret; } static void arm_smmu_detach_dev(struct iommu_domain *domain, struct device *dev) -- cgit v1.2.3 From 1fc870c7efa364862c3bc792cfbdb38afea26742 Mon Sep 17 00:00:00 2001 From: Olav Haugan Date: Mon, 4 Aug 2014 19:01:02 +0100 Subject: iommu/arm-smmu: fix programming of SMMU_CBn_TCR for stage 1 Stage-1 context banks do not have the SMMU_CBn_TCR[SL0] field since it is only applicable to stage-2 context banks. This patch ensures that we don't set the reserved TCR bits for stage-1 translations. Cc: Signed-off-by: Olav Haugan Signed-off-by: Will Deacon --- drivers/iommu/arm-smmu.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c index 98fcd87cbacb..1f10ac0d4135 100644 --- a/drivers/iommu/arm-smmu.c +++ b/drivers/iommu/arm-smmu.c @@ -843,8 +843,11 @@ static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain) reg |= TTBCR_EAE | (TTBCR_SH_IS << TTBCR_SH0_SHIFT) | (TTBCR_RGN_WBWA << TTBCR_ORGN0_SHIFT) | - (TTBCR_RGN_WBWA << TTBCR_IRGN0_SHIFT) | - (TTBCR_SL0_LVL_1 << TTBCR_SL0_SHIFT); + (TTBCR_RGN_WBWA << TTBCR_IRGN0_SHIFT); + + if (!stage1) + reg |= (TTBCR_SL0_LVL_1 << TTBCR_SL0_SHIFT); + writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBCR); /* MAIR0 (stage-1 only) */ -- cgit v1.2.3 From 93b14135cf24668b45eb57348312b627a8539bc4 Mon Sep 17 00:00:00 2001 From: Vladimir Murzin Date: Fri, 8 Aug 2014 10:31:47 +0100 Subject: iommu/arm-smmu: remove pgtable_page_{c,d}tor() If split page table lock for PTE tables is enabled (CONFIG_SPLIT_PTLOCK_CPUS <=NR_CPUS) pgtable_page_ctor() leads to non-atomic allocation for ptlock with a spinlock held, resulting in: ------------[ cut here ]------------ WARNING: CPU: 0 PID: 466 at kernel/locking/lockdep.c:2742 lockdep_trace_alloc+0xd8/0xf4() DEBUG_LOCKS_WARN_ON(irqs_disabled_flags(flags)) Modules linked in: CPU: 0 PID: 466 Comm: dma0chan0-copy0 Not tainted 3.16.0-3d47efb-clean-pl330-dma_test-ve-a15-a32-slr-m c-on-3+ #55 [<80014748>] (unwind_backtrace) from [<80011640>] (show_stack+0x10/0x14) [<80011640>] (show_stack) from [<802bf864>] (dump_stack+0x80/0xb4) [<802bf864>] (dump_stack) from [<8002385c>] (warn_slowpath_common+0x64/0x88) [<8002385c>] (warn_slowpath_common) from [<80023914>] (warn_slowpath_fmt+0x30/0x40) [<80023914>] (warn_slowpath_fmt) from [<8005d818>] (lockdep_trace_alloc+0xd8/0xf4) [<8005d818>] (lockdep_trace_alloc) from [<800d3d78>] (kmem_cache_alloc+0x24/0x144) [<800d3d78>] (kmem_cache_alloc) from [<800bfae4>] (ptlock_alloc+0x18/0x2c) [<800bfae4>] (ptlock_alloc) from [<802b1ec0>] (arm_smmu_handle_mapping+0x4c0/0x690) [<802b1ec0>] (arm_smmu_handle_mapping) from [<802b0cd8>] (iommu_map+0xe0/0x148) [<802b0cd8>] (iommu_map) from [<80019098>] (arm_coherent_iommu_map_page+0x160/0x278) [<80019098>] (arm_coherent_iommu_map_page) from [<801f4d78>] (dmatest_func+0x60c/0x1098) [<801f4d78>] (dmatest_func) from [<8003f8ac>] (kthread+0xcc/0xe8) [<8003f8ac>] (kthread) from [<8000e868>] (ret_from_fork+0x14/0x2c) ---[ end trace ce0d27e6f434acf8 ]-- Split page tables lock is not used in the driver. In fact, page tables are guarded with domain lock, so remove calls to pgtable_page_{c,d}tor(). Cc: Signed-off-by: Vladimir Murzin Signed-off-by: Will Deacon --- drivers/iommu/arm-smmu.c | 5 ----- 1 file changed, 5 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c index 1f10ac0d4135..398802ad9d2d 100644 --- a/drivers/iommu/arm-smmu.c +++ b/drivers/iommu/arm-smmu.c @@ -984,7 +984,6 @@ static void arm_smmu_free_ptes(pmd_t *pmd) { pgtable_t table = pmd_pgtable(*pmd); - pgtable_page_dtor(table); __free_page(table); } @@ -1253,10 +1252,6 @@ static int arm_smmu_alloc_init_pte(struct arm_smmu_device *smmu, pmd_t *pmd, return -ENOMEM; arm_smmu_flush_pgtable(smmu, page_address(table), PAGE_SIZE); - if (!pgtable_page_ctor(table)) { - __free_page(table); - return -ENOMEM; - } pmd_populate(NULL, pmd, table); arm_smmu_flush_pgtable(smmu, pmd, sizeof(*pmd)); } -- cgit v1.2.3 From 43b412bedb782e27a8771b61870dd45a24975266 Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Tue, 15 Jul 2014 11:22:24 +0100 Subject: iommu/arm-smmu: fix s2cr and smr teardown on device detach from domain When we attach a device to a domain, we configure the SMRs (if we have any) to match the Stream IDs for the corresponding SMMU master and program the s2crs accordingly. However, on detach we tear down the s2crs assuming stream-indexing (as opposed to stream-matching) and SMRs assuming they are present. This patch fixes the device detach code so that it operates as a converse of the attach code. Cc: Signed-off-by: Will Deacon --- drivers/iommu/arm-smmu.c | 27 ++++++++++++--------------- 1 file changed, 12 insertions(+), 15 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c index 398802ad9d2d..b7cd5d07d6fb 100644 --- a/drivers/iommu/arm-smmu.c +++ b/drivers/iommu/arm-smmu.c @@ -1116,6 +1116,9 @@ static void arm_smmu_master_free_smrs(struct arm_smmu_device *smmu, void __iomem *gr0_base = ARM_SMMU_GR0(smmu); struct arm_smmu_smr *smrs = cfg->smrs; + if (!smrs) + return; + /* Invalidate the SMRs before freeing back to the allocator */ for (i = 0; i < cfg->num_streamids; ++i) { u8 idx = smrs[i].idx; @@ -1128,20 +1131,6 @@ static void arm_smmu_master_free_smrs(struct arm_smmu_device *smmu, kfree(smrs); } -static void arm_smmu_bypass_stream_mapping(struct arm_smmu_device *smmu, - struct arm_smmu_master_cfg *cfg) -{ - int i; - void __iomem *gr0_base = ARM_SMMU_GR0(smmu); - - for (i = 0; i < cfg->num_streamids; ++i) { - u16 sid = cfg->streamids[i]; - - writel_relaxed(S2CR_TYPE_BYPASS, - gr0_base + ARM_SMMU_GR0_S2CR(sid)); - } -} - static int arm_smmu_domain_add_master(struct arm_smmu_domain *smmu_domain, struct arm_smmu_master_cfg *cfg) { @@ -1168,13 +1157,21 @@ static int arm_smmu_domain_add_master(struct arm_smmu_domain *smmu_domain, static void arm_smmu_domain_remove_master(struct arm_smmu_domain *smmu_domain, struct arm_smmu_master_cfg *cfg) { + int i; struct arm_smmu_device *smmu = smmu_domain->smmu; + void __iomem *gr0_base = ARM_SMMU_GR0(smmu); /* * We *must* clear the S2CR first, because freeing the SMR means * that it can be re-allocated immediately. */ - arm_smmu_bypass_stream_mapping(smmu, cfg); + for (i = 0; i < cfg->num_streamids; ++i) { + u32 idx = cfg->smrs ? cfg->smrs[i].idx : cfg->streamids[i]; + + writel_relaxed(S2CR_TYPE_BYPASS, + gr0_base + ARM_SMMU_GR0_S2CR(idx)); + } + arm_smmu_master_free_smrs(smmu, cfg); } -- cgit v1.2.3 From 3c8766d0ca9618a406b0a1ecc29147820e93d984 Mon Sep 17 00:00:00 2001 From: Olav Haugan Date: Fri, 22 Aug 2014 17:12:32 -0700 Subject: iommu/arm-smmu: Do not access non-existing S2CR registers The number of S2CR registers is not properly set when stream matching is not supported. Fix this and add check that we do not try to access outside of the number of S2CR regisrers. Signed-off-by: Olav Haugan [will: added missing NUMSIDB_* definitions] Signed-off-by: Will Deacon --- drivers/iommu/arm-smmu.c | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c index b7cd5d07d6fb..ba9d7e52ed1a 100644 --- a/drivers/iommu/arm-smmu.c +++ b/drivers/iommu/arm-smmu.c @@ -146,6 +146,8 @@ #define ID0_CTTW (1 << 14) #define ID0_NUMIRPT_SHIFT 16 #define ID0_NUMIRPT_MASK 0xff +#define ID0_NUMSIDB_SHIFT 9 +#define ID0_NUMSIDB_MASK 0xf #define ID0_NUMSMRG_SHIFT 0 #define ID0_NUMSMRG_MASK 0xff @@ -524,9 +526,18 @@ static int register_smmu_master(struct arm_smmu_device *smmu, master->of_node = masterspec->np; master->cfg.num_streamids = masterspec->args_count; - for (i = 0; i < master->cfg.num_streamids; ++i) - master->cfg.streamids[i] = masterspec->args[i]; + for (i = 0; i < master->cfg.num_streamids; ++i) { + u16 streamid = masterspec->args[i]; + if (!(smmu->features & ARM_SMMU_FEAT_STREAM_MATCH) && + (streamid >= smmu->num_mapping_groups)) { + dev_err(dev, + "stream ID for master device %s greater than maximum allowed (%d)\n", + masterspec->np->name, smmu->num_mapping_groups); + return -ERANGE; + } + master->cfg.streamids[i] = streamid; + } return insert_smmu_master(smmu, master); } @@ -1624,7 +1635,7 @@ static void arm_smmu_device_reset(struct arm_smmu_device *smmu) /* Mark all SMRn as invalid and all S2CRn as bypass */ for (i = 0; i < smmu->num_mapping_groups; ++i) { - writel_relaxed(~SMR_VALID, gr0_base + ARM_SMMU_GR0_SMR(i)); + writel_relaxed(0, gr0_base + ARM_SMMU_GR0_SMR(i)); writel_relaxed(S2CR_TYPE_BYPASS, gr0_base + ARM_SMMU_GR0_S2CR(i)); } @@ -1759,6 +1770,9 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu) dev_notice(smmu->dev, "\tstream matching with %u register groups, mask 0x%x", smmu->num_mapping_groups, mask); + } else { + smmu->num_mapping_groups = (id >> ID0_NUMSIDB_SHIFT) & + ID0_NUMSIDB_MASK; } /* ID1 */ @@ -1887,6 +1901,10 @@ static int arm_smmu_device_dt_probe(struct platform_device *pdev) smmu->irqs[i] = irq; } + err = arm_smmu_device_cfg_probe(smmu); + if (err) + return err; + i = 0; smmu->masters = RB_ROOT; while (!of_parse_phandle_with_args(dev->of_node, "mmu-masters", @@ -1903,10 +1921,6 @@ static int arm_smmu_device_dt_probe(struct platform_device *pdev) } dev_notice(dev, "registered %d master devices\n", i); - err = arm_smmu_device_cfg_probe(smmu); - if (err) - goto out_put_masters; - parse_driver_options(smmu); if (smmu->version > 1 && -- cgit v1.2.3 From 70c9a7db93a152e975c4158b944c6b50579415c2 Mon Sep 17 00:00:00 2001 From: Hans Wennborg Date: Wed, 6 Aug 2014 05:42:01 +0100 Subject: iommu/arm-smmu: fix decimal printf format specifiers prefixed with 0x The prefix suggests the number should be printed in hex, so use the %x specifier to do that. Found by using regex suggested by Joe Perches. Signed-off-by: Hans Wennborg Signed-off-by: Will Deacon --- drivers/iommu/arm-smmu.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c index ba9d7e52ed1a..1d69f147c924 100644 --- a/drivers/iommu/arm-smmu.c +++ b/drivers/iommu/arm-smmu.c @@ -634,7 +634,7 @@ static irqreturn_t arm_smmu_context_fault(int irq, void *dev) if (fsr & FSR_IGN) dev_err_ratelimited(smmu->dev, - "Unexpected context fault (fsr 0x%u)\n", + "Unexpected context fault (fsr 0x%x)\n", fsr); fsynr = readl_relaxed(cb_base + ARM_SMMU_CB_FSYNR0); -- cgit v1.2.3 From 4d09d99d543ec28b2280a1f62382697f47ea6366 Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Wed, 23 Jul 2014 13:20:43 +0100 Subject: iommu/arm-smmu: fix corner cases in address size calculations Working out the usable address sizes for the SMMU is surprisingly tricky. We must take into account both the limitations of the hardware for VA, IPA and PA sizes but also any restrictions imposed by the Linux page table code, particularly when dealing with nested translation (where the IPA size is limited by the input address size at stage-2). This patch fixes a few corner cases in our address size handling so that we correctly deal with 40-bit addresses in TTBCR2 and restrict the IPA size differently depending on whether or not we have support for nested translation. Signed-off-by: Will Deacon --- drivers/iommu/arm-smmu.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c index 1d69f147c924..a83cc2a2a2ca 100644 --- a/drivers/iommu/arm-smmu.c +++ b/drivers/iommu/arm-smmu.c @@ -763,6 +763,7 @@ static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain) reg = (TTBCR2_ADDR_36 << TTBCR2_SEP_SHIFT); break; case 39: + case 40: reg = (TTBCR2_ADDR_40 << TTBCR2_SEP_SHIFT); break; case 42: @@ -784,6 +785,7 @@ static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain) reg |= (TTBCR2_ADDR_36 << TTBCR2_PASIZE_SHIFT); break; case 39: + case 40: reg |= (TTBCR2_ADDR_40 << TTBCR2_PASIZE_SHIFT); break; case 42: @@ -1806,11 +1808,16 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu) * Stage-1 output limited by stage-2 input size due to pgd * allocation (PTRS_PER_PGD). */ + if (smmu->features & ARM_SMMU_FEAT_TRANS_NESTED) { #ifdef CONFIG_64BIT - smmu->s1_output_size = min_t(unsigned long, VA_BITS, size); + smmu->s1_output_size = min_t(unsigned long, VA_BITS, size); #else - smmu->s1_output_size = min(32UL, size); + smmu->s1_output_size = min(32UL, size); #endif + } else { + smmu->s1_output_size = min_t(unsigned long, PHYS_MASK_SHIFT, + size); + } /* The stage-2 output mask is also applied for bypass */ size = arm_smmu_id_size_to_bits((id >> ID2_OAS_SHIFT) & ID2_OAS_MASK); -- cgit v1.2.3