Merge branch '2023-01-27-apple-soc-updates'

First, to quote the author:
This series adds support for the PCIe controller found on Apple M1 and
M2 machines and enables support for PCIe XHCI controllers.  This makes
the type-A USB ports on the M1 Mac mini work.  Since the use of Apples
DART IOMMU is mandatory (these PCIe DARTs don't support bypass mode),
this adds DMA mapping operations to the IOMMU uclass and implements
them for the Apple DART.  It modifies the XHCI driver code to go map
DMA buffers through the IOMMU if there is one.  Since the M1 Mac mini
now has two types of XHCI controllers with different number of ports
(2 for the DWC3 controllers, 8 for the Fresco Logic PCIe controller)
this uncovered an issue in with the way the hub descriptor is
implemented in the XHCI driver.

Second, Mark also fixes some Apple-specific instances of
CONFIG_IS_ENABLED that should be IS_ENABLED.

1 files changed, 277 insertions, 34 deletions

diff --git a/drivers/iommu/apple_dart.c b/drivers/iommu/apple_dart.c
index 2faacb8f3b6..6ecd84303bc 100644
--- a/drivers/iommu/apple_dart.c
+++ b/drivers/iommu/apple_dart.c
@@ -6,57 +6,296 @@
 #include <common.h>
 #include <cpu_func.h>
 #include <dm.h>
+#include <iommu.h>
+#include <lmb.h>
+#include <memalign.h>
 #include <asm/io.h>
 
 #define DART_PARAMS2		0x0004
 #define  DART_PARAMS2_BYPASS_SUPPORT	BIT(0)
-#define DART_TLB_OP		0x0020
-#define  DART_TLB_OP_OPMASK	(0xfff << 20)
-#define  DART_TLB_OP_FLUSH	(0x001 << 20)
-#define  DART_TLB_OP_BUSY	BIT(2)
-#define DART_TLB_OP_SIDMASK	0x0034
-#define DART_ERROR_STATUS	0x0040
-#define DART_TCR(sid)		(0x0100 + 4 * (sid))
-#define  DART_TCR_TRANSLATE_ENABLE	BIT(7)
-#define  DART_TCR_BYPASS_DART		BIT(8)
-#define  DART_TCR_BYPASS_DAPF		BIT(12)
-#define DART_TTBR(sid, idx)	(0x0200 + 16 * (sid) + 4 * (idx))
-#define  DART_TTBR_VALID	BIT(31)
-#define  DART_TTBR_SHIFT	12
 
-#define DART_T8110_TCR(sid)	(0x1000 + 4 * (sid))
+#define DART_T8020_TLB_CMD		0x0020
+#define  DART_T8020_TLB_CMD_FLUSH		BIT(20)
+#define  DART_T8020_TLB_CMD_BUSY		BIT(2)
+#define DART_T8020_TLB_SIDMASK		0x0034
+#define DART_T8020_ERROR		0x0040
+#define DART_T8020_ERROR_ADDR_LO	0x0050
+#define DART_T8020_ERROR_ADDR_HI	0x0054
+#define DART_T8020_CONFIG		0x0060
+#define  DART_T8020_CONFIG_LOCK			BIT(15)
+#define DART_T8020_SID_ENABLE		0x00fc
+#define DART_T8020_TCR_BASE		0x0100
+#define  DART_T8020_TCR_TRANSLATE_ENABLE	BIT(7)
+#define  DART_T8020_TCR_BYPASS_DART		BIT(8)
+#define  DART_T8020_TCR_BYPASS_DAPF		BIT(12)
+#define DART_T8020_TTBR_BASE		0x0200
+#define  DART_T8020_TTBR_VALID			BIT(31)
+
+#define DART_T8110_PARAMS4		0x000c
+#define  DART_T8110_PARAMS4_NSID_MASK		(0x1ff << 0)
+#define DART_T8110_TLB_CMD		0x0080
+#define  DART_T8110_TLB_CMD_BUSY		BIT(31)
+#define  DART_T8110_TLB_CMD_FLUSH_ALL		BIT(8)
+#define DART_T8110_ERROR		0x0100
+#define DART_T8110_ERROR_MASK		0x0104
+#define DART_T8110_ERROR_ADDR_LO	0x0170
+#define DART_T8110_ERROR_ADDR_HI	0x0174
+#define DART_T8110_PROTECT		0x0200
+#define  DART_T8110_PROTECT_TTBR_TCR		BIT(0)
+#define DART_T8110_SID_ENABLE_BASE	0x0c00
+#define DART_T8110_TCR_BASE		0x1000
 #define  DART_T8110_TCR_BYPASS_DAPF		BIT(2)
 #define  DART_T8110_TCR_BYPASS_DART		BIT(1)
 #define  DART_T8110_TCR_TRANSLATE_ENABLE	BIT(0)
-#define DART_T8110_TTBR(sid)	(0x1400 + 4 * (sid))
+#define DART_T8110_TTBR_BASE		0x1400
+#define  DART_T8110_TTBR_VALID			BIT(0)
+
+#define DART_SID_ENABLE(priv, idx) \
+	((priv)->sid_enable_base + 4 * (idx))
+#define DART_TCR(priv, sid)	((priv)->tcr_base + 4 * (sid))
+#define DART_TTBR(priv, sid, idx)	\
+	((priv)->ttbr_base + 4 * (priv)->nttbr * (sid) + 4 * (idx))
+#define  DART_TTBR_SHIFT	12
+
+#define DART_ALL_STREAMS(priv)	((1U << (priv)->nsid) - 1)
+
+#define DART_PAGE_SIZE		SZ_16K
+#define DART_PAGE_MASK		(DART_PAGE_SIZE - 1)
+
+#define DART_L1_TABLE		0x3
+#define DART_L2_INVAL		0
+#define DART_L2_VALID		BIT(0)
+#define DART_L2_FULL_PAGE	BIT(1)
+#define DART_L2_START(addr)	((((addr) & DART_PAGE_MASK) >> 2) << 52)
+#define DART_L2_END(addr)	((((addr) & DART_PAGE_MASK) >> 2) << 40)
+
+struct apple_dart_priv {
+	void *base;
+	struct lmb lmb;
+	u64 *l1, *l2;
+	int bypass, shift;
+
+	dma_addr_t dvabase;
+	dma_addr_t dvaend;
+
+	int nsid;
+	int nttbr;
+	int sid_enable_base;
+	int tcr_base;
+	u32 tcr_translate_enable;
+	u32 tcr_bypass;
+	int ttbr_base;
+	u32 ttbr_valid;
+	void (*flush_tlb)(struct apple_dart_priv *priv);
+};
+
+static void apple_dart_t8020_flush_tlb(struct apple_dart_priv *priv)
+{
+	dsb();
+
+	writel(DART_ALL_STREAMS(priv), priv->base + DART_T8020_TLB_SIDMASK);
+	writel(DART_T8020_TLB_CMD_FLUSH, priv->base + DART_T8020_TLB_CMD);
+	while (readl(priv->base + DART_T8020_TLB_CMD) &
+	       DART_T8020_TLB_CMD_BUSY)
+		continue;
+}
+
+static void apple_dart_t8110_flush_tlb(struct apple_dart_priv *priv)
+{
+	dsb();
+
+	writel(DART_T8110_TLB_CMD_FLUSH_ALL,
+	       priv->base + DART_T8110_TLB_CMD_FLUSH_ALL);
+	while (readl(priv->base + DART_T8110_TLB_CMD) &
+	       DART_T8110_TLB_CMD_BUSY)
+		continue;
+}
+
+static dma_addr_t apple_dart_map(struct udevice *dev, void *addr, size_t size)
+{
+	struct apple_dart_priv *priv = dev_get_priv(dev);
+	phys_addr_t paddr, dva;
+	phys_size_t psize, off;
+	int i, idx;
+
+	if (priv->bypass)
+		return (phys_addr_t)addr;
+
+	paddr = ALIGN_DOWN((phys_addr_t)addr, DART_PAGE_SIZE);
+	off = (phys_addr_t)addr - paddr;
+	psize = ALIGN(size + off, DART_PAGE_SIZE);
+
+	dva = lmb_alloc(&priv->lmb, psize, DART_PAGE_SIZE);
+
+	idx = dva / DART_PAGE_SIZE;
+	for (i = 0; i < psize / DART_PAGE_SIZE; i++) {
+		priv->l2[idx + i] = (paddr  >> priv->shift) | DART_L2_VALID |
+			DART_L2_START(0LL) | DART_L2_END(~0LL);
+		paddr += DART_PAGE_SIZE;
+	}
+	flush_dcache_range((unsigned long)&priv->l2[idx],
+			   (unsigned long)&priv->l2[idx + i]);
+	priv->flush_tlb(priv);
+
+	return dva + off;
+}
+
+static void apple_dart_unmap(struct udevice *dev, dma_addr_t addr, size_t size)
+{
+	struct apple_dart_priv *priv = dev_get_priv(dev);
+	phys_addr_t dva;
+	phys_size_t psize;
+	int i, idx;
+
+	if (priv->bypass)
+		return;
+
+	dva = ALIGN_DOWN(addr, DART_PAGE_SIZE);
+	psize = size + (addr - dva);
+	psize = ALIGN(psize, DART_PAGE_SIZE);
+
+	idx = dva / DART_PAGE_SIZE;
+	for (i = 0; i < psize / DART_PAGE_SIZE; i++)
+		priv->l2[idx + i] = DART_L2_INVAL;
+	flush_dcache_range((unsigned long)&priv->l2[idx],
+			   (unsigned long)&priv->l2[idx + i]);
+	priv->flush_tlb(priv);
+
+	lmb_free(&priv->lmb, dva, psize);
+}
+
+static struct iommu_ops apple_dart_ops = {
+	.map = apple_dart_map,
+	.unmap = apple_dart_unmap,
+};
 
 static int apple_dart_probe(struct udevice *dev)
 {
-	void *base;
+	struct apple_dart_priv *priv = dev_get_priv(dev);
+	dma_addr_t addr;
+	phys_addr_t l2;
+	int ntte, nl1, nl2;
 	int sid, i;
+	u32 params2, params4;
 
-	base = dev_read_addr_ptr(dev);
-	if (!base)
+	priv->base = dev_read_addr_ptr(dev);
+	if (!priv->base)
 		return -EINVAL;
 
-	u32 params2 = readl(base + DART_PARAMS2);
-	if (!(params2 & DART_PARAMS2_BYPASS_SUPPORT))
-		return 0;
+	if (device_is_compatible(dev, "apple,t8110-dart")) {
+		params4 = readl(priv->base + DART_T8110_PARAMS4);
+		priv->nsid = params4 & DART_T8110_PARAMS4_NSID_MASK;
+		priv->nttbr = 1;
+		priv->sid_enable_base = DART_T8110_SID_ENABLE_BASE;
+		priv->tcr_base = DART_T8110_TCR_BASE;
+		priv->tcr_translate_enable = DART_T8110_TCR_TRANSLATE_ENABLE;
+		priv->tcr_bypass =
+		    DART_T8110_TCR_BYPASS_DAPF | DART_T8110_TCR_BYPASS_DART;
+		priv->ttbr_base = DART_T8110_TTBR_BASE;
+		priv->ttbr_valid = DART_T8110_TTBR_VALID;
+		priv->flush_tlb = apple_dart_t8110_flush_tlb;
+	} else {
+		priv->nsid = 16;
+		priv->nttbr = 4;
+		priv->sid_enable_base = DART_T8020_SID_ENABLE;
+		priv->tcr_base = DART_T8020_TCR_BASE;
+		priv->tcr_translate_enable = DART_T8020_TCR_TRANSLATE_ENABLE;
+		priv->tcr_bypass =
+		    DART_T8020_TCR_BYPASS_DAPF | DART_T8020_TCR_BYPASS_DART;
+		priv->ttbr_base = DART_T8020_TTBR_BASE;
+		priv->ttbr_valid = DART_T8020_TTBR_VALID;
+		priv->flush_tlb = apple_dart_t8020_flush_tlb;
+	}
+
+	if (device_is_compatible(dev, "apple,t6000-dart") ||
+	    device_is_compatible(dev, "apple,t8110-dart"))
+		priv->shift = 4;
 
-	if (device_is_compatible(dev, "apple,t8112-dart")) {
-		for (sid = 0; sid < 256; sid++) {
-			writel(DART_T8110_TCR_BYPASS_DART | DART_T8110_TCR_BYPASS_DAPF,
-			       base + DART_T8110_TCR(sid));
-			writel(0, base + DART_T8110_TTBR(sid));
+	priv->dvabase = DART_PAGE_SIZE;
+	priv->dvaend = SZ_4G - DART_PAGE_SIZE;
+
+	lmb_init(&priv->lmb);
+	lmb_add(&priv->lmb, priv->dvabase, priv->dvaend - priv->dvabase);
+
+	/* Disable translations. */
+	for (sid = 0; sid < priv->nsid; sid++)
+		writel(0, priv->base + DART_TCR(priv, sid));
+
+	/* Remove page tables. */
+	for (sid = 0; sid < priv->nsid; sid++) {
+		for (i = 0; i < priv->nttbr; i++)
+			writel(0, priv->base + DART_TTBR(priv, sid, i));
+	}
+	priv->flush_tlb(priv);
+
+	params2 = readl(priv->base + DART_PARAMS2);
+	if (params2 & DART_PARAMS2_BYPASS_SUPPORT) {
+		for (sid = 0; sid < priv->nsid; sid++) {
+			writel(priv->tcr_bypass,
+			       priv->base + DART_TCR(priv, sid));
 		}
-	} else {
-		for (sid = 0; sid < 16; sid++) {
-			writel(DART_TCR_BYPASS_DART | DART_TCR_BYPASS_DAPF,
-			       base + DART_TCR(sid));
-			for (i = 0; i < 4; i++)
-				writel(0, base + DART_TTBR(sid, i));
+		priv->bypass = 1;
+		return 0;
+	}
+
+	ntte = DIV_ROUND_UP(priv->dvaend, DART_PAGE_SIZE);
+	nl2 = DIV_ROUND_UP(ntte, DART_PAGE_SIZE / sizeof(u64));
+	nl1 = DIV_ROUND_UP(nl2, DART_PAGE_SIZE / sizeof(u64));
+
+	priv->l2 = memalign(DART_PAGE_SIZE, nl2 * DART_PAGE_SIZE);
+	memset(priv->l2, 0, nl2 * DART_PAGE_SIZE);
+	flush_dcache_range((unsigned long)priv->l2,
+			   (unsigned long)priv->l2 + nl2 * DART_PAGE_SIZE);
+
+	priv->l1 = memalign(DART_PAGE_SIZE, nl1 * DART_PAGE_SIZE);
+	memset(priv->l1, 0, nl1 * DART_PAGE_SIZE);
+	l2 = (phys_addr_t)priv->l2;
+	for (i = 0; i < nl2; i++) {
+		priv->l1[i] = (l2 >> priv->shift) | DART_L1_TABLE;
+		l2 += DART_PAGE_SIZE;
+	}
+	flush_dcache_range((unsigned long)priv->l1,
+			   (unsigned long)priv->l1 + nl1 * DART_PAGE_SIZE);
+
+	/* Install page tables. */
+	for (sid = 0; sid < priv->nsid; sid++) {
+		addr = (phys_addr_t)priv->l1;
+		for (i = 0; i < nl1; i++) {
+			writel(addr >> DART_TTBR_SHIFT | priv->ttbr_valid,
+			       priv->base + DART_TTBR(priv, sid, i));
+			addr += DART_PAGE_SIZE;
 		}
 	}
+	priv->flush_tlb(priv);
+
+	/* Enable all streams. */
+	for (i = 0; i < priv->nsid / 32; i++)
+		writel(~0, priv->base + DART_SID_ENABLE(priv, i));
+
+	/* Enable translations. */
+	for (sid = 0; sid < priv->nsid; sid++) {
+		writel(priv->tcr_translate_enable,
+		       priv->base + DART_TCR(priv, sid));
+	}
+
+	return 0;
+}
+
+static int apple_dart_remove(struct udevice *dev)
+{
+	struct apple_dart_priv *priv = dev_get_priv(dev);
+	int sid, i;
+
+	/* Disable translations. */
+	for (sid = 0; sid < priv->nsid; sid++)
+		writel(0, priv->base + DART_TCR(priv, sid));
+
+	/* Remove page tables. */
+	for (sid = 0; sid < priv->nsid; sid++) {
+		for (i = 0; i < priv->nttbr; i++)
+			writel(0, priv->base + DART_TTBR(priv, sid, i));
+	}
+	priv->flush_tlb(priv);
 
 	return 0;
 }
@@ -64,7 +303,7 @@ static int apple_dart_probe(struct udevice *dev)
 static const struct udevice_id apple_dart_ids[] = {
 	{ .compatible = "apple,t8103-dart" },
 	{ .compatible = "apple,t6000-dart" },
-	{ .compatible = "apple,t8112-dart" },
+	{ .compatible = "apple,t8110-dart" },
 	{ /* sentinel */ }
 };
 
@@ -72,5 +311,9 @@ U_BOOT_DRIVER(apple_dart) = {
 	.name = "apple_dart",
 	.id = UCLASS_IOMMU,
 	.of_match = apple_dart_ids,
-	.probe = apple_dart_probe
+	.priv_auto = sizeof(struct apple_dart_priv),
+	.ops = &apple_dart_ops,
+	.probe = apple_dart_probe,
+	.remove = apple_dart_remove,
+	.flags	= DM_FLAG_OS_PREPARE
 };