diff options
Diffstat (limited to 'drivers')
| -rw-r--r-- | drivers/net/Kconfig | 4 | ||||
| -rw-r--r-- | drivers/net/ntb_netdev.c | 58 | ||||
| -rw-r--r-- | drivers/ntb/Kconfig | 39 | ||||
| -rw-r--r-- | drivers/ntb/Makefile | 5 | ||||
| -rw-r--r-- | drivers/ntb/hw/Kconfig | 1 | ||||
| -rw-r--r-- | drivers/ntb/hw/Makefile | 1 | ||||
| -rw-r--r-- | drivers/ntb/hw/intel/Kconfig | 7 | ||||
| -rw-r--r-- | drivers/ntb/hw/intel/Makefile | 1 | ||||
| -rw-r--r-- | drivers/ntb/hw/intel/ntb_hw_intel.c | 2274 | ||||
| -rw-r--r-- | drivers/ntb/hw/intel/ntb_hw_intel.h | 342 | ||||
| -rw-r--r-- | drivers/ntb/ntb.c | 251 | ||||
| -rw-r--r-- | drivers/ntb/ntb_hw.c | 1895 | ||||
| -rw-r--r-- | drivers/ntb/ntb_hw.h | 256 | ||||
| -rw-r--r-- | drivers/ntb/ntb_regs.h | 177 | ||||
| -rw-r--r-- | drivers/ntb/ntb_transport.c | 1029 | ||||
| -rw-r--r-- | drivers/ntb/test/Kconfig | 19 | ||||
| -rw-r--r-- | drivers/ntb/test/Makefile | 2 | ||||
| -rw-r--r-- | drivers/ntb/test/ntb_pingpong.c | 250 | ||||
| -rw-r--r-- | drivers/ntb/test/ntb_tool.c | 556 |
19 files changed, 4389 insertions, 2778 deletions
diff --git a/drivers/net/Kconfig b/drivers/net/Kconfig index b7f31150753d..c18f9e62a9fa 100644 --- a/drivers/net/Kconfig +++ b/drivers/net/Kconfig @@ -217,8 +217,8 @@ config NET_POLL_CONTROLLER def_bool NETPOLL config NTB_NETDEV - tristate "Virtual Ethernet over NTB" - depends on NTB + tristate "Virtual Ethernet over NTB Transport" + depends on NTB_TRANSPORT config RIONET tristate "RapidIO Ethernet over messaging driver support" diff --git a/drivers/net/ntb_netdev.c b/drivers/net/ntb_netdev.c index 5a7e6397440a..3cc316cb7e6b 100644 --- a/drivers/net/ntb_netdev.c +++ b/drivers/net/ntb_netdev.c @@ -5,6 +5,7 @@ * GPL LICENSE SUMMARY * * Copyright(c) 2012 Intel Corporation. All rights reserved. + * Copyright (C) 2015 EMC Corporation. All Rights Reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of version 2 of the GNU General Public License as @@ -13,6 +14,7 @@ * BSD LICENSE * * Copyright(c) 2012 Intel Corporation. All rights reserved. + * Copyright (C) 2015 EMC Corporation. All Rights Reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -40,7 +42,7 @@ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * - * Intel PCIe NTB Network Linux driver + * PCIe NTB Network Linux driver * * Contact Information: * Jon Mason <jon.mason@intel.com> @@ -50,6 +52,7 @@ #include <linux/module.h> #include <linux/pci.h> #include <linux/ntb.h> +#include <linux/ntb_transport.h> #define NTB_NETDEV_VER "0.7" @@ -70,26 +73,19 @@ struct ntb_netdev { static LIST_HEAD(dev_list); -static void ntb_netdev_event_handler(void *data, int status) +static void ntb_netdev_event_handler(void *data, int link_is_up) { struct net_device *ndev = data; struct ntb_netdev *dev = netdev_priv(ndev); - netdev_dbg(ndev, "Event %x, Link %x\n", status, + netdev_dbg(ndev, "Event %x, Link %x\n", link_is_up, ntb_transport_link_query(dev->qp)); - switch (status) { - case NTB_LINK_DOWN: + if (link_is_up) { + if (ntb_transport_link_query(dev->qp)) + netif_carrier_on(ndev); + } else { netif_carrier_off(ndev); - break; - case NTB_LINK_UP: - if (!ntb_transport_link_query(dev->qp)) - return; - - netif_carrier_on(ndev); - break; - default: - netdev_warn(ndev, "Unsupported event type %d\n", status); } } @@ -160,8 +156,6 @@ static netdev_tx_t ntb_netdev_start_xmit(struct sk_buff *skb, struct ntb_netdev *dev = netdev_priv(ndev); int rc; - netdev_dbg(ndev, "%s: skb len %d\n", __func__, skb->len); - rc = ntb_transport_tx_enqueue(dev->qp, skb, skb->data, skb->len); if (rc) goto err; @@ -322,20 +316,26 @@ static const struct ntb_queue_handlers ntb_netdev_handlers = { .event_handler = ntb_netdev_event_handler, }; -static int ntb_netdev_probe(struct pci_dev *pdev) +static int ntb_netdev_probe(struct device *client_dev) { + struct ntb_dev *ntb; struct net_device *ndev; + struct pci_dev *pdev; struct ntb_netdev *dev; int rc; - ndev = alloc_etherdev(sizeof(struct ntb_netdev)); + ntb = dev_ntb(client_dev->parent); + pdev = ntb->pdev; + if (!pdev) + return -ENODEV; + + ndev = alloc_etherdev(sizeof(*dev)); if (!ndev) return -ENOMEM; dev = netdev_priv(ndev); dev->ndev = ndev; dev->pdev = pdev; - BUG_ON(!dev->pdev); ndev->features = NETIF_F_HIGHDMA; ndev->priv_flags |= IFF_LIVE_ADDR_CHANGE; @@ -349,7 +349,8 @@ static int ntb_netdev_probe(struct pci_dev *pdev) ndev->netdev_ops = &ntb_netdev_ops; ndev->ethtool_ops = &ntb_ethtool_ops; - dev->qp = ntb_transport_create_queue(ndev, pdev, &ntb_netdev_handlers); + dev->qp = ntb_transport_create_queue(ndev, client_dev, + &ntb_netdev_handlers); if (!dev->qp) { rc = -EIO; goto err; @@ -372,12 +373,17 @@ err: return rc; } -static void ntb_netdev_remove(struct pci_dev *pdev) +static void ntb_netdev_remove(struct device *client_dev) { + struct ntb_dev *ntb; struct net_device *ndev; + struct pci_dev *pdev; struct ntb_netdev *dev; bool found = false; + ntb = dev_ntb(client_dev->parent); + pdev = ntb->pdev; + list_for_each_entry(dev, &dev_list, list) { if (dev->pdev == pdev) { found = true; @@ -396,7 +402,7 @@ static void ntb_netdev_remove(struct pci_dev *pdev) free_netdev(ndev); } -static struct ntb_client ntb_netdev_client = { +static struct ntb_transport_client ntb_netdev_client = { .driver.name = KBUILD_MODNAME, .driver.owner = THIS_MODULE, .probe = ntb_netdev_probe, @@ -407,16 +413,16 @@ static int __init ntb_netdev_init_module(void) { int rc; - rc = ntb_register_client_dev(KBUILD_MODNAME); + rc = ntb_transport_register_client_dev(KBUILD_MODNAME); if (rc) return rc; - return ntb_register_client(&ntb_netdev_client); + return ntb_transport_register_client(&ntb_netdev_client); } module_init(ntb_netdev_init_module); static void __exit ntb_netdev_exit_module(void) { - ntb_unregister_client(&ntb_netdev_client); - ntb_unregister_client_dev(KBUILD_MODNAME); + ntb_transport_unregister_client(&ntb_netdev_client); + ntb_transport_unregister_client_dev(KBUILD_MODNAME); } module_exit(ntb_netdev_exit_module); diff --git a/drivers/ntb/Kconfig b/drivers/ntb/Kconfig index f69df793dbe2..95944e52fa36 100644 --- a/drivers/ntb/Kconfig +++ b/drivers/ntb/Kconfig @@ -1,13 +1,28 @@ -config NTB - tristate "Intel Non-Transparent Bridge support" - depends on PCI - depends on X86 - help - The PCI-E Non-transparent bridge hardware is a point-to-point PCI-E bus - connecting 2 systems. When configured, writes to the device's PCI - mapped memory will be mirrored to a buffer on the remote system. The - ntb Linux driver uses this point-to-point communication as a method to - transfer data from one system to the other. - - If unsure, say N. +menuconfig NTB + tristate "Non-Transparent Bridge support" + depends on PCI + help + The PCI-E Non-transparent bridge hardware is a point-to-point PCI-E bus + connecting 2 systems. When configured, writes to the device's PCI + mapped memory will be mirrored to a buffer on the remote system. The + ntb Linux driver uses this point-to-point communication as a method to + transfer data from one system to the other. + If unsure, say N. + +if NTB + +source "drivers/ntb/hw/Kconfig" + +source "drivers/ntb/test/Kconfig" + +config NTB_TRANSPORT + tristate "NTB Transport Client" + help + This is a transport driver that enables connected systems to exchange + messages over the ntb hardware. The transport exposes a queue pair api + to client drivers. + + If unsure, say N. + +endif # NTB diff --git a/drivers/ntb/Makefile b/drivers/ntb/Makefile index 15cb59fd354e..1921dec1949d 100644 --- a/drivers/ntb/Makefile +++ b/drivers/ntb/Makefile @@ -1,3 +1,2 @@ -obj-$(CONFIG_NTB) += ntb.o - -ntb-objs := ntb_hw.o ntb_transport.o +obj-$(CONFIG_NTB) += ntb.o hw/ test/ +obj-$(CONFIG_NTB_TRANSPORT) += ntb_transport.o diff --git a/drivers/ntb/hw/Kconfig b/drivers/ntb/hw/Kconfig new file mode 100644 index 000000000000..4d5535c4cddf --- /dev/null +++ b/drivers/ntb/hw/Kconfig @@ -0,0 +1 @@ +source "drivers/ntb/hw/intel/Kconfig" diff --git a/drivers/ntb/hw/Makefile b/drivers/ntb/hw/Makefile new file mode 100644 index 000000000000..175d7c92a569 --- /dev/null +++ b/drivers/ntb/hw/Makefile @@ -0,0 +1 @@ +obj-$(CONFIG_NTB_INTEL) += intel/ diff --git a/drivers/ntb/hw/intel/Kconfig b/drivers/ntb/hw/intel/Kconfig new file mode 100644 index 000000000000..91f995e33ac6 --- /dev/null +++ b/drivers/ntb/hw/intel/Kconfig @@ -0,0 +1,7 @@ +config NTB_INTEL + tristate "Intel Non-Transparent Bridge support" + depends on X86_64 + help + This driver supports Intel NTB on capable Xeon and Atom hardware. + + If unsure, say N. diff --git a/drivers/ntb/hw/intel/Makefile b/drivers/ntb/hw/intel/Makefile new file mode 100644 index 000000000000..1b434568d2ad --- /dev/null +++ b/drivers/ntb/hw/intel/Makefile @@ -0,0 +1 @@ +obj-$(CONFIG_NTB_INTEL) += ntb_hw_intel.o diff --git a/drivers/ntb/hw/intel/ntb_hw_intel.c b/drivers/ntb/hw/intel/ntb_hw_intel.c new file mode 100644 index 000000000000..87751cfd6f4f --- /dev/null +++ b/drivers/ntb/hw/intel/ntb_hw_intel.c @@ -0,0 +1,2274 @@ +/* + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * GPL LICENSE SUMMARY + * + * Copyright(c) 2012 Intel Corporation. All rights reserved. + * Copyright (C) 2015 EMC Corporation. All Rights Reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of version 2 of the GNU General Public License as + * published by the Free Software Foundation. + * + * BSD LICENSE + * + * Copyright(c) 2012 Intel Corporation. All rights reserved. + * Copyright (C) 2015 EMC Corporation. All Rights Reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copy + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * Intel PCIe NTB Linux driver + * + * Contact Information: + * Jon Mason <jon.mason@intel.com> + */ + +#include <linux/debugfs.h> +#include <linux/delay.h> +#include <linux/init.h> +#include <linux/interrupt.h> +#include <linux/module.h> +#include <linux/pci.h> +#include <linux/random.h> +#include <linux/slab.h> +#include <linux/ntb.h> + +#include "ntb_hw_intel.h" + +#define NTB_NAME "ntb_hw_intel" +#define NTB_DESC "Intel(R) PCI-E Non-Transparent Bridge Driver" +#define NTB_VER "2.0" + +MODULE_DESCRIPTION(NTB_DESC); +MODULE_VERSION(NTB_VER); +MODULE_LICENSE("Dual BSD/GPL"); +MODULE_AUTHOR("Intel Corporation"); + +#define bar0_off(base, bar) ((base) + ((bar) << 2)) +#define bar2_off(base, bar) bar0_off(base, (bar) - 2) + +static const struct intel_ntb_reg atom_reg; +static const struct intel_ntb_alt_reg atom_pri_reg; +static const struct intel_ntb_alt_reg atom_sec_reg; +static const struct intel_ntb_alt_reg atom_b2b_reg; +static const struct intel_ntb_xlat_reg atom_pri_xlat; +static const struct intel_ntb_xlat_reg atom_sec_xlat; +static const struct intel_ntb_reg xeon_reg; +static const struct intel_ntb_alt_reg xeon_pri_reg; +static const struct intel_ntb_alt_reg xeon_sec_reg; +static const struct intel_ntb_alt_reg xeon_b2b_reg; +static const struct intel_ntb_xlat_reg xeon_pri_xlat; +static const struct intel_ntb_xlat_reg xeon_sec_xlat; +static struct intel_b2b_addr xeon_b2b_usd_addr; +static struct intel_b2b_addr xeon_b2b_dsd_addr; +static const struct ntb_dev_ops intel_ntb_ops; + +static const struct file_operations intel_ntb_debugfs_info; +static struct dentry *debugfs_dir; + +static int b2b_mw_idx = -1; +module_param(b2b_mw_idx, int, 0644); +MODULE_PARM_DESC(b2b_mw_idx, "Use this mw idx to access the peer ntb. A " + "value of zero or positive starts from first mw idx, and a " + "negative value starts from last mw idx. Both sides MUST " + "set the same value here!"); + +static unsigned int b2b_mw_share; +module_param(b2b_mw_share, uint, 0644); +MODULE_PARM_DESC(b2b_mw_share, "If the b2b mw is large enough, configure the " + "ntb so that the peer ntb only occupies the first half of " + "the mw, so the second half can still be used as a mw. Both " + "sides MUST set the same value here!"); + +module_param_named(xeon_b2b_usd_bar2_addr64, + xeon_b2b_usd_addr.bar2_addr64, ullong, 0644); +MODULE_PARM_DESC(xeon_b2b_usd_bar2_addr64, + "XEON B2B USD BAR 2 64-bit address"); + +module_param_named(xeon_b2b_usd_bar4_addr64, + xeon_b2b_usd_addr.bar4_addr64, ullong, 0644); +MODULE_PARM_DESC(xeon_b2b_usd_bar2_addr64, + "XEON B2B USD BAR 4 64-bit address"); + +module_param_named(xeon_b2b_usd_bar4_addr32, + xeon_b2b_usd_addr.bar4_addr32, ullong, 0644); +MODULE_PARM_DESC(xeon_b2b_usd_bar2_addr64, + "XEON B2B USD split-BAR 4 32-bit address"); + +module_param_named(xeon_b2b_usd_bar5_addr32, + xeon_b2b_usd_addr.bar5_addr32, ullong, 0644); +MODULE_PARM_DESC(xeon_b2b_usd_bar2_addr64, + "XEON B2B USD split-BAR 5 32-bit address"); + +module_param_named(xeon_b2b_dsd_bar2_addr64, + xeon_b2b_dsd_addr.bar2_addr64, ullong, 0644); +MODULE_PARM_DESC(xeon_b2b_dsd_bar2_addr64, + "XEON B2B DSD BAR 2 64-bit address"); + +module_param_named(xeon_b2b_dsd_bar4_addr64, + xeon_b2b_dsd_addr.bar4_addr64, ullong, 0644); +MODULE_PARM_DESC(xeon_b2b_dsd_bar2_addr64, + "XEON B2B DSD BAR 4 64-bit address"); + +module_param_named(xeon_b2b_dsd_bar4_addr32, + xeon_b2b_dsd_addr.bar4_addr32, ullong, 0644); +MODULE_PARM_DESC(xeon_b2b_dsd_bar2_addr64, + "XEON B2B DSD split-BAR 4 32-bit address"); + +module_param_named(xeon_b2b_dsd_bar5_addr32, + xeon_b2b_dsd_addr.bar5_addr32, ullong, 0644); +MODULE_PARM_DESC(xeon_b2b_dsd_bar2_addr64, + "XEON B2B DSD split-BAR 5 32-bit address"); + +#ifndef ioread64 +#ifdef readq +#define ioread64 readq +#else +#define ioread64 _ioread64 +static inline u64 _ioread64(void __iomem *mmio) +{ + u64 low, high; + + low = ioread32(mmio); + high = ioread32(mmio + sizeof(u32)); + return low | (high << 32); +} +#endif +#endif + +#ifndef iowrite64 +#ifdef writeq +#define iowrite64 writeq +#else +#define iowrite64 _iowrite64 +static inline void _iowrite64(u64 val, void __iomem *mmio) +{ + iowrite32(val, mmio); + iowrite32(val >> 32, mmio + sizeof(u32)); +} +#endif +#endif + +static inline int pdev_is_atom(struct pci_dev *pdev) +{ + switch (pdev->device) { + case PCI_DEVICE_ID_INTEL_NTB_B2B_BWD: + return 1; + } + return 0; +} + +static inline int pdev_is_xeon(struct pci_dev *pdev) +{ + switch (pdev->device) { + case PCI_DEVICE_ID_INTEL_NTB_SS_JSF: + case PCI_DEVICE_ID_INTEL_NTB_SS_SNB: + case PCI_DEVICE_ID_INTEL_NTB_SS_IVT: + case PCI_DEVICE_ID_INTEL_NTB_SS_HSX: + case PCI_DEVICE_ID_INTEL_NTB_PS_JSF: + case PCI_DEVICE_ID_INTEL_NTB_PS_SNB: + case PCI_DEVICE_ID_INTEL_NTB_PS_IVT: + case PCI_DEVICE_ID_INTEL_NTB_PS_HSX: + case PCI_DEVICE_ID_INTEL_NTB_B2B_JSF: + case PCI_DEVICE_ID_INTEL_NTB_B2B_SNB: + case PCI_DEVICE_ID_INTEL_NTB_B2B_IVT: + case PCI_DEVICE_ID_INTEL_NTB_B2B_HSX: + return 1; + } + return 0; +} + +static inline void ndev_reset_unsafe_flags(struct intel_ntb_dev *ndev) +{ + ndev->unsafe_flags = 0; + ndev->unsafe_flags_ignore = 0; + + /* Only B2B has a workaround to avoid SDOORBELL */ + if (ndev->hwerr_flags & NTB_HWERR_SDOORBELL_LOCKUP) + if (!ntb_topo_is_b2b(ndev->ntb.topo)) + ndev->unsafe_flags |= NTB_UNSAFE_DB; + + /* No low level workaround to avoid SB01BASE */ + if (ndev->hwerr_flags & NTB_HWERR_SB01BASE_LOCKUP) { + ndev->unsafe_flags |= NTB_UNSAFE_DB; + ndev->unsafe_flags |= NTB_UNSAFE_SPAD; + } +} + +static inline int ndev_is_unsafe(struct intel_ntb_dev *ndev, + unsigned long flag) +{ + return !!(flag & ndev->unsafe_flags & ~ndev->unsafe_flags_ignore); +} + +static inline int ndev_ignore_unsafe(struct intel_ntb_dev *ndev, + unsigned long flag) +{ + flag &= ndev->unsafe_flags; + ndev->unsafe_flags_ignore |= flag; + + return !!flag; +} + +static int ndev_mw_to_bar(struct intel_ntb_dev *ndev, int idx) +{ + if (idx < 0 || idx > ndev->mw_count) + return -EINVAL; + return ndev->reg->mw_bar[idx]; +} + +static inline int ndev_db_addr(struct intel_ntb_dev *ndev, + phys_addr_t *db_addr, resource_size_t *db_size, + phys_addr_t reg_addr, unsigned long reg) +{ + if (ndev_is_unsafe(ndev, NTB_UNSAFE_DB)) + pr_warn_once("%s: NTB unsafe doorbell access", __func__); + + if (db_addr) { + *db_addr = reg_addr + reg; + dev_dbg(ndev_dev(ndev), "Peer db addr %llx\n", *db_addr); + } + + if (db_size) { + *db_size = ndev->reg->db_size; + dev_dbg(ndev_dev(ndev), "Peer db size %llx\n", *db_size); + } + + return 0; +} + +static inline u64 ndev_db_read(struct intel_ntb_dev *ndev, + void __iomem *mmio) +{ + if (ndev_is_unsafe(ndev, NTB_UNSAFE_DB)) + pr_warn_once("%s: NTB unsafe doorbell access", __func__); + + return ndev->reg->db_ioread(mmio); +} + +static inline int ndev_db_write(struct intel_ntb_dev *ndev, u64 db_bits, + void __iomem *mmio) +{ + if (ndev_is_unsafe(ndev, NTB_UNSAFE_DB)) + pr_warn_once("%s: NTB unsafe doorbell access", __func__); + + if (db_bits & ~ndev->db_valid_mask) + return -EINVAL; + + ndev->reg->db_iowrite(db_bits, mmio); + + return 0; +} + +static inline int ndev_db_set_mask(struct intel_ntb_dev *ndev, u64 db_bits, + void __iomem *mmio) +{ + unsigned long irqflags; + + if (ndev_is_unsafe(ndev, NTB_UNSAFE_DB)) + pr_warn_once("%s: NTB unsafe doorbell access", __func__); + + if (db_bits & ~ndev->db_valid_mask) + return -EINVAL; + + spin_lock_irqsave(&ndev->db_mask_lock, irqflags); + { + ndev->db_mask |= db_bits; + ndev->reg->db_iowrite(ndev->db_mask, mmio); + } + spin_unlock_irqrestore(&ndev->db_mask_lock, irqflags); + + return 0; +} + +static inline int ndev_db_clear_mask(struct intel_ntb_dev *ndev, u64 db_bits, + void __iomem *mmio) +{ + unsigned long irqflags; + + if (ndev_is_unsafe(ndev, NTB_UNSAFE_DB)) + pr_warn_once("%s: NTB unsafe doorbell access", __func__); + + if (db_bits & ~ndev->db_valid_mask) + return -EINVAL; + + spin_lock_irqsave(&ndev->db_mask_lock, irqflags); + { + ndev->db_mask &= ~db_bits; + ndev->reg->db_iowrite(ndev->db_mask, mmio); + } + spin_unlock_irqrestore(&ndev->db_mask_lock, irqflags); + + return 0; +} + +static inline int ndev_vec_mask(struct intel_ntb_dev *ndev, int db_vector) +{ + u64 shift, mask; + + shift = ndev->db_vec_shift; + mask = BIT_ULL(shift) - 1; + + return mask << (shift * db_vector); +} + +static inline int ndev_spad_addr(struct intel_ntb_dev *ndev, int idx, + phys_addr_t *spad_addr, phys_addr_t reg_addr, + unsigned long reg) +{ + if (ndev_is_unsafe(ndev, NTB_UNSAFE_SPAD)) + pr_warn_once("%s: NTB unsafe scratchpad access", __func__); + + if (idx < 0 || idx >= ndev->spad_count) + return -EINVAL; + + if (spad_addr) { + *spad_addr = reg_addr + reg + (idx << 2); + dev_dbg(ndev_dev(ndev), "Peer spad addr %llx\n", *spad_addr); + } + + return 0; +} + +static inline u32 ndev_spad_read(struct intel_ntb_dev *ndev, int idx, + void __iomem *mmio) +{ + if (ndev_is_unsafe(ndev, NTB_UNSAFE_SPAD)) + pr_warn_once("%s: NTB unsafe scratchpad access", __func__); + + if (idx < 0 || idx >= ndev->spad_count) + return 0; + + return ioread32(mmio + (idx << 2)); +} + +static inline int ndev_spad_write(struct intel_ntb_dev *ndev, int idx, u32 val, + void __iomem *mmio) +{ + if (ndev_is_unsafe(ndev, NTB_UNSAFE_SPAD)) + pr_warn_once("%s: NTB unsafe scratchpad access", __func__); + + if (idx < 0 || idx >= ndev->spad_count) + return -EINVAL; + + iowrite32(val, mmio + (idx << 2)); + + return 0; +} + +static irqreturn_t ndev_interrupt(struct intel_ntb_dev *ndev, int vec) +{ + u64 vec_mask; + + vec_mask = ndev_vec_mask(ndev, vec); + + dev_dbg(ndev_dev(ndev), "vec %d vec_mask %llx\n", vec, vec_mask); + + ndev->last_ts = jiffies; + + if (vec_mask & ndev->db_link_mask) { + if (ndev->reg->poll_link(ndev)) + ntb_link_event(&ndev->ntb); + } + + if (vec_mask & ndev->db_valid_mask) + ntb_db_event(&ndev->ntb, vec); + + return IRQ_HANDLED; +} + +static irqreturn_t ndev_vec_isr(int irq, void *dev) +{ + struct intel_ntb_vec *nvec = dev; + + return ndev_interrupt(nvec->ndev, nvec->num); +} + +static irqreturn_t ndev_irq_isr(int irq, void *dev) +{ + struct intel_ntb_dev *ndev = dev; + + return ndev_interrupt(ndev, irq - ndev_pdev(ndev)->irq); +} + +static int ndev_init_isr(struct intel_ntb_dev *ndev, + int msix_min, int msix_max, + int msix_shift, int total_shift) +{ + struct pci_dev *pdev; + int rc, i, msix_count, node; + + pdev = ndev_pdev(ndev); + + node = dev_to_node(&pdev->dev); + + /* Mask all doorbell interrupts */ + ndev->db_mask = ndev->db_valid_mask; + ndev->reg->db_iowrite(ndev->db_mask, + ndev->self_mmio + + ndev->self_reg->db_mask); + + /* Try to set up msix irq */ + + ndev->vec = kzalloc_node(msix_max * sizeof(*ndev->vec), + GFP_KERNEL, node); + if (!ndev->vec) + goto err_msix_vec_alloc; + + ndev->msix = kzalloc_node(msix_max * sizeof(*ndev->msix), + GFP_KERNEL, node); + if (!ndev->msix) + goto err_msix_alloc; + + for (i = 0; i < msix_max; ++i) + ndev->msix[i].entry = i; + + msix_count = pci_enable_msix_range(pdev, ndev->msix, + msix_min, msix_max); + if (msix_count < 0) + goto err_msix_enable; + + for (i = 0; i < msix_count; ++i) { + ndev->vec[i].ndev = ndev; + ndev->vec[i].num = i; + rc = request_irq(ndev->msix[i].vector, ndev_vec_isr, 0, + "ndev_vec_isr", &ndev->vec[i]); + if (rc) + goto err_msix_request; + } + + dev_dbg(ndev_dev(ndev), "Using msix interrupts\n"); + ndev->db_vec_count = msix_count; + ndev->db_vec_shift = msix_shift; + return 0; + +err_msix_request: + while (i-- > 0) + free_irq(ndev->msix[i].vector, ndev); + pci_disable_msix(pdev); +err_msix_enable: + kfree(ndev->msix); +err_msix_alloc: + kfree(ndev->vec); +err_msix_vec_alloc: + ndev->msix = NULL; + ndev->vec = NULL; + + /* Try to set up msi irq */ + + rc = pci_enable_msi(pdev); + if (rc) + goto err_msi_enable; + + rc = request_irq(pdev->irq, ndev_irq_isr, 0, + "ndev_irq_isr", ndev); + if (rc) + goto err_msi_request; + + dev_dbg(ndev_dev(ndev), "Using msi interrupts\n"); + ndev->db_vec_count = 1; + ndev->db_vec_shift = total_shift; + return 0; + +err_msi_request: + pci_disable_msi(pdev); +err_msi_enable: + + /* Try to set up intx irq */ + + pci_intx(pdev, 1); + + rc = request_irq(pdev->irq, ndev_irq_isr, IRQF_SHARED, + "ndev_irq_isr", ndev); + if (rc) + goto err_intx_request; + + dev_dbg(ndev_dev(ndev), "Using intx interrupts\n"); + ndev->db_vec_count = 1; + ndev->db_vec_shift = total_shift; + return 0; + +err_intx_request: + return rc; +} + +static void ndev_deinit_isr(struct intel_ntb_dev *ndev) +{ + struct pci_dev *pdev; + int i; + + pdev = ndev_pdev(ndev); + + /* Mask all doorbell interrupts */ + ndev->db_mask = ndev->db_valid_mask; + ndev->reg->db_iowrite(ndev->db_mask, + ndev->self_mmio + + ndev->self_reg->db_mask); + + if (ndev->msix) { + i = ndev->db_vec_count; + while (i--) + free_irq(ndev->msix[i].vector, &ndev->vec[i]); + pci_disable_msix(pdev); + kfree(ndev->msix); + kfree(ndev->vec); + } else { + free_irq(pdev->irq, ndev); + if (pci_dev_msi_enabled(pdev)) + pci_disable_msi(pdev); + } +} + +static ssize_t ndev_debugfs_read(struct file *filp, char __user *ubuf, + size_t count, loff_t *offp) +{ + struct intel_ntb_dev *ndev; + void __iomem *mmio; + char *buf; + size_t buf_size; + ssize_t ret, off; + union { u64 v64; u32 v32; u16 v16; } u; + + ndev = filp->private_data; + mmio = ndev->self_mmio; + + buf_size = min(count, 0x800ul); + + buf = kmalloc(buf_size, GFP_KERNEL); + if (!buf) + return -ENOMEM; + + off = 0; + + off += scnprintf(buf + off, buf_size - off, + "NTB Device Information:\n"); + + off += scnprintf(buf + off, buf_size - off, + "Connection Topology -\t%s\n", + ntb_topo_string(ndev->ntb.topo)); + + off += scnprintf(buf + off, buf_size - off, + "B2B Offset -\t\t%#lx\n", ndev->b2b_off); + off += scnprintf(buf + off, buf_size - off, + "B2B MW Idx -\t\t%d\n", ndev->b2b_idx); + off += scnprintf(buf + off, buf_size - off, + "BAR4 Split -\t\t%s\n", + ndev->bar4_split ? "yes" : "no"); + + off += scnprintf(buf + off, buf_size - off, + "NTB CTL -\t\t%#06x\n", ndev->ntb_ctl); + off += scnprintf(buf + off, buf_size - off, + "LNK STA -\t\t%#06x\n", ndev->lnk_sta); + + if (!ndev->reg->link_is_up(ndev)) { + off += scnprintf(buf + off, buf_size - off, + "Link Status -\t\tDown\n"); + } else { + off += scnprintf(buf + off, buf_size - off, + "Link Status -\t\tUp\n"); + off += scnprintf(buf + off, buf_size - off, + "Link Speed -\t\tPCI-E Gen %u\n", + NTB_LNK_STA_SPEED(ndev->lnk_sta)); + off += scnprintf(buf + off, buf_size - off, + "Link Width -\t\tx%u\n", + NTB_LNK_STA_WIDTH(ndev->lnk_sta)); + } + + off += scnprintf(buf + off, buf_size - off, + "Memory Window Count -\t%u\n", ndev->mw_count); + off += scnprintf(buf + off, buf_size - off, + "Scratchpad Count -\t%u\n", ndev->spad_count); + off += scnprintf(buf + off, buf_size - off, + "Doorbell Count -\t%u\n", ndev->db_count); + off += scnprintf(buf + off, buf_size - off, + "Doorbell Vector Count -\t%u\n", ndev->db_vec_count); + off += scnprintf(buf + off, buf_size - off, + "Doorbell Vector Shift -\t%u\n", ndev->db_vec_shift); + + off += scnprintf(buf + off, buf_size - off, + "Doorbell Valid Mask -\t%#llx\n", ndev->db_valid_mask); + off += scnprintf(buf + off, buf_size - off, + "Doorbell Link Mask -\t%#llx\n", ndev->db_link_mask); + off += scnprintf(buf + off, buf_size - off, + "Doorbell Mask Cached -\t%#llx\n", ndev->db_mask); + + u.v64 = ndev_db_read(ndev, mmio + ndev->self_reg->db_mask); + off += scnprintf(buf + off, buf_size - off, + "Doorbell Mask -\t\t%#llx\n", u.v64); + + u.v64 = ndev_db_read(ndev, mmio + ndev->self_reg->db_bell); + off += scnprintf(buf + off, buf_size - off, + "Doorbell Bell -\t\t%#llx\n", u.v64); + + off += scnprintf(buf + off, buf_size - off, + "\nNTB Incoming XLAT:\n"); + + u.v64 = ioread64(mmio + bar2_off(ndev->xlat_reg->bar2_xlat, 2)); + off += scnprintf(buf + off, buf_size - off, + "XLAT23 -\t\t%#018llx\n", u.v64); + + if (ndev->bar4_split) { + u.v32 = ioread32(mmio + bar2_off(ndev->xlat_reg->bar2_xlat, 4)); + off += scnprintf(buf + off, buf_size - off, + "XLAT4 -\t\t\t%#06x\n", u.v32); + + u.v32 = ioread32(mmio + bar2_off(ndev->xlat_reg->bar2_xlat, 5)); + off += scnprintf(buf + off, buf_size - off, + "XLAT5 -\t\t\t%#06x\n", u.v32); + } else { + u.v64 = ioread64(mmio + bar2_off(ndev->xlat_reg->bar2_xlat, 4)); + off += scnprintf(buf + off, buf_size - off, + "XLAT45 -\t\t%#018llx\n", u.v64); + } + + u.v64 = ioread64(mmio + bar2_off(ndev->xlat_reg->bar2_limit, 2)); + off += scnprintf(buf + off, buf_size - off, + "LMT23 -\t\t\t%#018llx\n", u.v64); + + if (ndev->bar4_split) { + u.v32 = ioread32(mmio + bar2_off(ndev->xlat_reg->bar2_limit, 4)); + off += scnprintf(buf + off, buf_size - off, + "LMT4 -\t\t\t%#06x\n", u.v32); + u.v32 = ioread32(mmio + bar2_off(ndev->xlat_reg->bar2_limit, 5)); + off += scnprintf(buf + off, buf_size - off, + "LMT5 -\t\t\t%#06x\n", u.v32); + } else { + u.v64 = ioread64(mmio + bar2_off(ndev->xlat_reg->bar2_limit, 4)); + off += scnprintf(buf + off, buf_size - off, + "LMT45 -\t\t\t%#018llx\n", u.v64); + } + + if (pdev_is_xeon(ndev->ntb.pdev)) { + if (ntb_topo_is_b2b(ndev->ntb.topo)) { + off += scnprintf(buf + off, buf_size - off, + "\nNTB Outgoing B2B XLAT:\n"); + + u.v64 = ioread64(mmio + XEON_PBAR23XLAT_OFFSET); + off += scnprintf(buf + off, buf_size - off, + "B2B XLAT23 -\t\t%#018llx\n", u.v64); + + if (ndev->bar4_split) { + u.v32 = ioread32(mmio + XEON_PBAR4XLAT_OFFSET); + off += scnprintf(buf + off, buf_size - off, + "B2B XLAT4 -\t\t%#06x\n", + u.v32); + u.v32 = ioread32(mmio + XEON_PBAR5XLAT_OFFSET); + off += scnprintf(buf + off, buf_size - off, + "B2B XLAT5 -\t\t%#06x\n", + u.v32); + } else { + u.v64 = ioread64(mmio + XEON_PBAR45XLAT_OFFSET); + off += scnprintf(buf + off, buf_size - off, + "B2B XLAT45 -\t\t%#018llx\n", + u.v64); + } + + u.v64 = ioread64(mmio + XEON_PBAR23LMT_OFFSET); + off += scnprintf(buf + off, buf_size - off, + "B2B LMT23 -\t\t%#018llx\n", u.v64); + + if (ndev->bar4_split) { + u.v32 = ioread32(mmio + XEON_PBAR4LMT_OFFSET); + off += scnprintf(buf + off, buf_size - off, + "B2B LMT4 -\t\t%#06x\n", + u.v32); + u.v32 = ioread32(mmio + XEON_PBAR5LMT_OFFSET); + off += scnprintf(buf + off, buf_size - off, + "B2B LMT5 -\t\t%#06x\n", + u.v32); + } else { + u.v64 = ioread64(mmio + XEON_PBAR45LMT_OFFSET); + off += scnprintf(buf + off, buf_size - off, + "B2B LMT45 -\t\t%#018llx\n", + u.v64); + } + + off += scnprintf(buf + off, buf_size - off, + "\nNTB Secondary BAR:\n"); + + u.v64 = ioread64(mmio + XEON_SBAR0BASE_OFFSET); + off += scnprintf(buf + off, buf_size - off, + "SBAR01 -\t\t%#018llx\n", u.v64); + + u.v64 = ioread64(mmio + XEON_SBAR23BASE_OFFSET); + off += scnprintf(buf + off, buf_size - off, + "SBAR23 -\t\t%#018llx\n", u.v64); + + if (ndev->bar4_split) { + u.v32 = ioread32(mmio + XEON_SBAR4BASE_OFFSET); + off += scnprintf(buf + off, buf_size - off, + "SBAR4 -\t\t\t%#06x\n", u.v32); + u.v32 = ioread32(mmio + XEON_SBAR5BASE_OFFSET); + off += scnprintf(buf + off, buf_size - off, + "SBAR5 -\t\t\t%#06x\n", u.v32); + } else { + u.v64 = ioread64(mmio + XEON_SBAR45BASE_OFFSET); + off += scnprintf(buf + off, buf_size - off, + "SBAR45 -\t\t%#018llx\n", + u.v64); + } + } + + off += scnprintf(buf + off, buf_size - off, + "\nXEON NTB Statistics:\n"); + + u.v16 = ioread16(mmio + XEON_USMEMMISS_OFFSET); + off += scnprintf(buf + off, buf_size - off, + "Upstream Memory Miss -\t%u\n", u.v16); + + off += scnprintf(buf + off, buf_size - off, + "\nXEON NTB Hardware Errors:\n"); + + if (!pci_read_config_word(ndev->ntb.pdev, + XEON_DEVSTS_OFFSET, &u.v16)) + off += scnprintf(buf + off, buf_size - off, + "DEVSTS -\t\t%#06x\n", u.v16); + + if (!pci_read_config_word(ndev->ntb.pdev, + XEON_LINK_STATUS_OFFSET, &u.v16)) + off += scnprintf(buf + off, buf_size - off, + "LNKSTS -\t\t%#06x\n", u.v16); + + if (!pci_read_config_dword(ndev->ntb.pdev, + XEON_UNCERRSTS_OFFSET, &u.v32)) + off += scnprintf(buf + off, buf_size - off, + "UNCERRSTS -\t\t%#06x\n", u.v32); + + if (!pci_read_config_dword(ndev->ntb.pdev, + XEON_CORERRSTS_OFFSET, &u.v32)) + off += scnprintf(buf + off, buf_size - off, + "CORERRSTS -\t\t%#06x\n", u.v32); + } + + ret = simple_read_from_buffer(ubuf, count, offp, buf, off); + kfree(buf); + return ret; +} + +static void ndev_init_debugfs(struct intel_ntb_dev *ndev) +{ + if (!debugfs_dir) { + ndev->debugfs_dir = NULL; + ndev->debugfs_info = NULL; + } else { + ndev->debugfs_dir = + debugfs_create_dir(ndev_name(ndev), debugfs_dir); + if (!ndev->debugfs_dir) + ndev->debugfs_info = NULL; + else + ndev->debugfs_info = + debugfs_create_file("info", S_IRUSR, + ndev->debugfs_dir, ndev, + &intel_ntb_debugfs_info); + } +} + +static void ndev_deinit_debugfs(struct intel_ntb_dev *ndev) +{ + debugfs_remove_recursive(ndev->debugfs_dir); +} + +static int intel_ntb_mw_count(struct ntb_dev *ntb) +{ + return ntb_ndev(ntb)->mw_count; +} + +static int intel_ntb_mw_get_range(struct ntb_dev *ntb, int idx, + phys_addr_t *base, + resource_size_t *size, + resource_size_t *align, + resource_size_t *align_size) +{ + struct intel_ntb_dev *ndev = ntb_ndev(ntb); + int bar; + + if (idx >= ndev->b2b_idx && !ndev->b2b_off) + idx += 1; + + bar = ndev_mw_to_bar(ndev, idx); + if (bar < 0) + return bar; + + if (base) + *base = pci_resource_start(ndev->ntb.pdev, bar) + + (idx == ndev->b2b_idx ? ndev->b2b_off : 0); + + if (size) + *size = pci_resource_len(ndev->ntb.pdev, bar) - + (idx == ndev->b2b_idx ? ndev->b2b_off : 0); + + if (align) + *align = pci_resource_len(ndev->ntb.pdev, bar); + + if (align_size) + *align_size = 1; + + return 0; +} + +static int intel_ntb_mw_set_trans(struct ntb_dev *ntb, int idx, + dma_addr_t addr, resource_size_t size) +{ + struct intel_ntb_dev *ndev = ntb_ndev(ntb); + unsigned long base_reg, xlat_reg, limit_reg; + resource_size_t bar_size, mw_size; + void __iomem *mmio; + u64 base, limit, reg_val; + int bar; + + if (idx >= ndev->b2b_idx && !ndev->b2b_off) + idx += 1; + + bar = ndev_mw_to_bar(ndev, idx); + if (bar < 0) + return bar; + + bar_size = pci_resource_len(ndev->ntb.pdev, bar); + + if (idx == ndev->b2b_idx) + mw_size = bar_size - ndev->b2b_off; + else + mw_size = bar_size; + + /* hardware requires that addr is aligned to bar size */ + if (addr & (bar_size - 1)) + return -EINVAL; + + /* make sure the range fits in the usable mw size */ + if (size > mw_size) + return -EINVAL; + + mmio = ndev->self_mmio; + base_reg = bar0_off(ndev->xlat_reg->bar0_base, bar); + xlat_reg = bar2_off(ndev->xlat_reg->bar2_xlat, bar); + limit_reg = bar2_off(ndev->xlat_reg->bar2_limit, bar); + + if (bar < 4 || !ndev->bar4_split) { + base = ioread64(mmio + base_reg); + + /* Set the limit if supported, if size is not mw_size */ + if (limit_reg && size != mw_size) + limit = base + size; + else + limit = 0; + + /* set and verify setting the translation address */ + iowrite64(addr, mmio + xlat_reg); + reg_val = ioread64(mmio + xlat_reg); + if (reg_val != addr) { + iowrite64(0, mmio + xlat_reg); + return -EIO; + } + + /* set and verify setting the limit */ + iowrite64(limit, mmio + limit_reg); + reg_val = ioread64(mmio + limit_reg); + if (reg_val != limit) { + iowrite64(base, mmio + limit_reg); + iowrite64(0, mmio + xlat_reg); + return -EIO; + } + } else { + /* split bar addr range must all be 32 bit */ + if (addr & (~0ull << 32)) + return -EINVAL; + if ((addr + size) & (~0ull << 32)) + return -EINVAL; + + base = ioread32(mmio + base_reg); + + /* Set the limit if supported, if size is not mw_size */ + if (limit_reg && size != mw_size) + limit = base + size; + else + limit = 0; + + /* set and verify setting the translation address */ + iowrite32(addr, mmio + xlat_reg); + reg_val = ioread32(mmio + xlat_reg); + if (reg_val != addr) { + iowrite32(0, mmio + xlat_reg); + return -EIO; + } + + /* set and verify setting the limit */ + iowrite32(limit, mmio + limit_reg); + reg_val = ioread32(mmio + limit_reg); + if (reg_val != limit) { + iowrite32(base, mmio + limit_reg); + iowrite32(0, mmio + xlat_reg); + return -EIO; + } + } + + return 0; +} + +static int intel_ntb_link_is_up(struct ntb_dev *ntb, + enum ntb_speed *speed, + enum ntb_width *width) +{ + struct intel_ntb_dev *ndev = ntb_ndev(ntb); + + if (ndev->reg->link_is_up(ndev)) { + if (speed) + *speed = NTB_LNK_STA_SPEED(ndev->lnk_sta); + if (width) + *width = NTB_LNK_STA_WIDTH(ndev->lnk_sta); + return 1; + } else { + /* TODO MAYBE: is it possible to observe the link speed and + * width while link is training? */ + if (speed) + *speed = NTB_SPEED_NONE; + if (width) + *width = NTB_WIDTH_NONE; + return 0; + } +} + +static int intel_ntb_link_enable(struct ntb_dev *ntb, + enum ntb_speed max_speed, + enum ntb_width max_width) +{ + struct intel_ntb_dev *ndev; + u32 ntb_ctl; + + ndev = container_of(ntb, struct intel_ntb_dev, ntb); + + if (ndev->ntb.topo == NTB_TOPO_SEC) + return -EINVAL; + + dev_dbg(ndev_dev(ndev), + "Enabling link with max_speed %d max_width %d\n", + max_speed, max_width); + if (max_speed != NTB_SPEED_AUTO) + dev_dbg(ndev_dev(ndev), "ignoring max_speed %d\n", max_speed); + if (max_width != NTB_WIDTH_AUTO) + dev_dbg(ndev_dev(ndev), "ignoring max_width %d\n", max_width); + + ntb_ctl = ioread32(ndev->self_mmio + ndev->reg->ntb_ctl); + ntb_ctl &= ~(NTB_CTL_DISABLE | NTB_CTL_CFG_LOCK); + ntb_ctl |= NTB_CTL_P2S_BAR2_SNOOP | NTB_CTL_S2P_BAR2_SNOOP; + ntb_ctl |= NTB_CTL_P2S_BAR4_SNOOP | NTB_CTL_S2P_BAR4_SNOOP; + if (ndev->bar4_split) + ntb_ctl |= NTB_CTL_P2S_BAR5_SNOOP | NTB_CTL_S2P_BAR5_SNOOP; + iowrite32(ntb_ctl, ndev->self_mmio + ndev->reg->ntb_ctl); + + return 0; +} + +static int intel_ntb_link_disable(struct ntb_dev *ntb) +{ + struct intel_ntb_dev *ndev; + u32 ntb_cntl; + + ndev = container_of(ntb, struct intel_ntb_dev, ntb); + + if (ndev->ntb.topo == NTB_TOPO_SEC) + return -EINVAL; + + dev_dbg(ndev_dev(ndev), "Disabling link\n"); + + /* Bring NTB link down */ + ntb_cntl = ioread32(ndev->self_mmio + ndev->reg->ntb_ctl); + ntb_cntl &= ~(NTB_CTL_P2S_BAR2_SNOOP | NTB_CTL_S2P_BAR2_SNOOP); + ntb_cntl &= ~(NTB_CTL_P2S_BAR4_SNOOP | NTB_CTL_S2P_BAR4_SNOOP); + if (ndev->bar4_split) + ntb_cntl &= ~(NTB_CTL_P2S_BAR5_SNOOP | NTB_CTL_S2P_BAR5_SNOOP); + ntb_cntl |= NTB_CTL_DISABLE | NTB_CTL_CFG_LOCK; + iowrite32(ntb_cntl, ndev->self_mmio + ndev->reg->ntb_ctl); + + return 0; +} + +static int intel_ntb_db_is_unsafe(struct ntb_dev *ntb) +{ + return ndev_ignore_unsafe(ntb_ndev(ntb), NTB_UNSAFE_DB); +} + +static u64 intel_ntb_db_valid_mask(struct ntb_dev *ntb) +{ + return ntb_ndev(ntb)->db_valid_mask; +} + +static int intel_ntb_db_vector_count(struct ntb_dev *ntb) +{ + struct intel_ntb_dev *ndev; + + ndev = container_of(ntb, struct intel_ntb_dev, ntb); + + return ndev->db_vec_count; +} + +static u64 intel_ntb_db_vector_mask(struct ntb_dev *ntb, int db_vector) +{ + struct intel_ntb_dev *ndev = ntb_ndev(ntb); + + if (db_vector < 0 || db_vector > ndev->db_vec_count) + return 0; + + return ndev->db_valid_mask & ndev_vec_mask(ndev, db_vector); +} + +static u64 intel_ntb_db_read(struct ntb_dev *ntb) +{ + struct intel_ntb_dev *ndev = ntb_ndev(ntb); + + return ndev_db_read(ndev, + ndev->self_mmio + + ndev->self_reg->db_bell); +} + +static int intel_ntb_db_clear(struct ntb_dev *ntb, u64 db_bits) +{ + struct intel_ntb_dev *ndev = ntb_ndev(ntb); + + return ndev_db_write(ndev, db_bits, + ndev->self_mmio + + ndev->self_reg->db_bell); +} + +static int intel_ntb_db_set_mask(struct ntb_dev *ntb, u64 db_bits) +{ + struct intel_ntb_dev *ndev = ntb_ndev(ntb); + + return ndev_db_set_mask(ndev, db_bits, + ndev->self_mmio + + ndev->self_reg->db_mask); +} + +static int intel_ntb_db_clear_mask(struct ntb_dev *ntb, u64 db_bits) +{ + struct intel_ntb_dev *ndev = ntb_ndev(ntb); + + return ndev_db_clear_mask(ndev, db_bits, + ndev->self_mmio + + ndev->self_reg->db_mask); +} + +static int intel_ntb_peer_db_addr(struct ntb_dev *ntb, + phys_addr_t *db_addr, + resource_size_t *db_size) +{ + struct intel_ntb_dev *ndev = ntb_ndev(ntb); + + return ndev_db_addr(ndev, db_addr, db_size, ndev->peer_addr, + ndev->peer_reg->db_bell); +} + +static int intel_ntb_peer_db_set(struct ntb_dev *ntb, u64 db_bits) +{ + struct intel_ntb_dev *ndev = ntb_ndev(ntb); + + return ndev_db_write(ndev, db_bits, + ndev->peer_mmio + + ndev->peer_reg->db_bell); +} + +static int intel_ntb_spad_is_unsafe(struct ntb_dev *ntb) +{ + return ndev_ignore_unsafe(ntb_ndev(ntb), NTB_UNSAFE_SPAD); +} + +static int intel_ntb_spad_count(struct ntb_dev *ntb) +{ + struct intel_ntb_dev *ndev; + + ndev = container_of(ntb, struct intel_ntb_dev, ntb); + + return ndev->spad_count; +} + +static u32 intel_ntb_spad_read(struct ntb_dev *ntb, int idx) +{ + struct intel_ntb_dev *ndev = ntb_ndev(ntb); + + return ndev_spad_read(ndev, idx, + ndev->self_mmio + + ndev->self_reg->spad); +} + +static int intel_ntb_spad_write(struct ntb_dev *ntb, + int idx, u32 val) +{ + struct intel_ntb_dev *ndev = ntb_ndev(ntb); + + return ndev_spad_write(ndev, idx, val, + ndev->self_mmio + + ndev->self_reg->spad); +} + +static int intel_ntb_peer_spad_addr(struct ntb_dev *ntb, int idx, + phys_addr_t *spad_addr) +{ + struct intel_ntb_dev *ndev = ntb_ndev(ntb); + + return ndev_spad_addr(ndev, idx, spad_addr, ndev->peer_addr, + ndev->peer_reg->spad); +} + +static u32 intel_ntb_peer_spad_read(struct ntb_dev *ntb, int idx) +{ + struct intel_ntb_dev *ndev = ntb_ndev(ntb); + + return ndev_spad_read(ndev, idx, + ndev->peer_mmio + + ndev->peer_reg->spad); +} + +static int intel_ntb_peer_spad_write(struct ntb_dev *ntb, + int idx, u32 val) +{ + struct intel_ntb_dev *ndev = ntb_ndev(ntb); + + return ndev_spad_write(ndev, idx, val, + ndev->peer_mmio + + ndev->peer_reg->spad); +} + +/* ATOM */ + +static u64 atom_db_ioread(void __iomem *mmio) +{ + return ioread64(mmio); +} + +static void atom_db_iowrite(u64 bits, void __iomem *mmio) +{ + iowrite64(bits, mmio); +} + +static int atom_poll_link(struct intel_ntb_dev *ndev) +{ + u32 ntb_ctl; + + ntb_ctl = ioread32(ndev->self_mmio + ATOM_NTBCNTL_OFFSET); + + if (ntb_ctl == ndev->ntb_ctl) + return 0; + + ndev->ntb_ctl = ntb_ctl; + + ndev->lnk_sta = ioread32(ndev->self_mmio + ATOM_LINK_STATUS_OFFSET); + + return 1; +} + +static int atom_link_is_up(struct intel_ntb_dev *ndev) +{ + return ATOM_NTB_CTL_ACTIVE(ndev->ntb_ctl); +} + +static int atom_link_is_err(struct intel_ntb_dev *ndev) +{ + if (ioread32(ndev->self_mmio + ATOM_LTSSMSTATEJMP_OFFSET) + & ATOM_LTSSMSTATEJMP_FORCEDETECT) + return 1; + + if (ioread32(ndev->self_mmio + ATOM_IBSTERRRCRVSTS0_OFFSET) + & ATOM_IBIST_ERR_OFLOW) + return 1; + + return 0; +} + +static inline enum ntb_topo atom_ppd_topo(struct intel_ntb_dev *ndev, u32 ppd) +{ + switch (ppd & ATOM_PPD_TOPO_MASK) { + case ATOM_PPD_TOPO_B2B_USD: + dev_dbg(ndev_dev(ndev), "PPD %d B2B USD\n", ppd); + return NTB_TOPO_B2B_USD; + + case ATOM_PPD_TOPO_B2B_DSD: + dev_dbg(ndev_dev(ndev), "PPD %d B2B DSD\n", ppd); + return NTB_TOPO_B2B_DSD; + + case ATOM_PPD_TOPO_PRI_USD: + case ATOM_PPD_TOPO_PRI_DSD: /* accept bogus PRI_DSD */ + case ATOM_PPD_TOPO_SEC_USD: + case ATOM_PPD_TOPO_SEC_DSD: /* accept bogus SEC_DSD */ + dev_dbg(ndev_dev(ndev), "PPD %d non B2B disabled\n", ppd); + return NTB_TOPO_NONE; + } + + dev_dbg(ndev_dev(ndev), "PPD %d invalid\n", ppd); + return NTB_TOPO_NONE; +} + +static void atom_link_hb(struct work_struct *work) +{ + struct intel_ntb_dev *ndev = hb_ndev(work); + unsigned long poll_ts; + void __iomem *mmio; + u32 status32; + + poll_ts = ndev->last_ts + ATOM_LINK_HB_TIMEOUT; + + /* Delay polling the link status if an interrupt was received, + * unless the cached link status says the link is down. + */ + if (time_after(poll_ts, jiffies) && atom_link_is_up(ndev)) { + schedule_delayed_work(&ndev->hb_timer, poll_ts - jiffies); + return; + } + + if (atom_poll_link(ndev)) + ntb_link_event(&ndev->ntb); + + if (atom_link_is_up(ndev) || !atom_link_is_err(ndev)) { + schedule_delayed_work(&ndev->hb_timer, ATOM_LINK_HB_TIMEOUT); + return; + } + + /* Link is down with error: recover the link! */ + + mmio = ndev->self_mmio; + + /* Driver resets the NTB ModPhy lanes - magic! */ + iowrite8(0xe0, mmio + ATOM_MODPHY_PCSREG6); + iowrite8(0x40, mmio + ATOM_MODPHY_PCSREG4); + iowrite8(0x60, mmio + ATOM_MODPHY_PCSREG4); + iowrite8(0x60, mmio + ATOM_MODPHY_PCSREG6); + + /* Driver waits 100ms to allow the NTB ModPhy to settle */ + msleep(100); + + /* Clear AER Errors, write to clear */ + status32 = ioread32(mmio + ATOM_ERRCORSTS_OFFSET); + dev_dbg(ndev_dev(ndev), "ERRCORSTS = %x\n", status32); + status32 &= PCI_ERR_COR_REP_ROLL; + iowrite32(status32, mmio + ATOM_ERRCORSTS_OFFSET); + + /* Clear unexpected electrical idle event in LTSSM, write to clear */ + status32 = ioread32(mmio + ATOM_LTSSMERRSTS0_OFFSET); + dev_dbg(ndev_dev(ndev), "LTSSMERRSTS0 = %x\n", status32); + status32 |= ATOM_LTSSMERRSTS0_UNEXPECTEDEI; + iowrite32(status32, mmio + ATOM_LTSSMERRSTS0_OFFSET); + + /* Clear DeSkew Buffer error, write to clear */ + status32 = ioread32(mmio + ATOM_DESKEWSTS_OFFSET); + dev_dbg(ndev_dev(ndev), "DESKEWSTS = %x\n", status32); + status32 |= ATOM_DESKEWSTS_DBERR; + iowrite32(status32, mmio + ATOM_DESKEWSTS_OFFSET); + + status32 = ioread32(mmio + ATOM_IBSTERRRCRVSTS0_OFFSET); + dev_dbg(ndev_dev(ndev), "IBSTERRRCRVSTS0 = %x\n", status32); + status32 &= ATOM_IBIST_ERR_OFLOW; + iowrite32(status32, mmio + ATOM_IBSTERRRCRVSTS0_OFFSET); + + /* Releases the NTB state machine to allow the link to retrain */ + status32 = ioread32(mmio + ATOM_LTSSMSTATEJMP_OFFSET); + dev_dbg(ndev_dev(ndev), "LTSSMSTATEJMP = %x\n", status32); + status32 &= ~ATOM_LTSSMSTATEJMP_FORCEDETECT; + iowrite32(status32, mmio + ATOM_LTSSMSTATEJMP_OFFSET); + + /* There is a potential race between the 2 NTB devices recovering at the + * same time. If the times are the same, the link will not recover and + * the driver will be stuck in this loop forever. Add a random interval + * to the recovery time to prevent this race. + */ + schedule_delayed_work(&ndev->hb_timer, ATOM_LINK_RECOVERY_TIME + + prandom_u32() % ATOM_LINK_RECOVERY_TIME); +} + +static int atom_init_isr(struct intel_ntb_dev *ndev) +{ + int rc; + + rc = ndev_init_isr(ndev, 1, ATOM_DB_MSIX_VECTOR_COUNT, + ATOM_DB_MSIX_VECTOR_SHIFT, ATOM_DB_TOTAL_SHIFT); + if (rc) + return rc; + + /* ATOM doesn't have link status interrupt, poll on that platform */ + ndev->last_ts = jiffies; + INIT_DELAYED_WORK(&ndev->hb_timer, atom_link_hb); + schedule_delayed_work(&ndev->hb_timer, ATOM_LINK_HB_TIMEOUT); + + return 0; +} + +static void atom_deinit_isr(struct intel_ntb_dev *ndev) +{ + cancel_delayed_work_sync(&ndev->hb_timer); + ndev_deinit_isr(ndev); +} + +static int atom_init_ntb(struct intel_ntb_dev *ndev) +{ + ndev->mw_count = ATOM_MW_COUNT; + ndev->spad_count = ATOM_SPAD_COUNT; + ndev->db_count = ATOM_DB_COUNT; + + switch (ndev->ntb.topo) { + case NTB_TOPO_B2B_USD: + case NTB_TOPO_B2B_DSD: + ndev->self_reg = &atom_pri_reg; + ndev->peer_reg = &atom_b2b_reg; + ndev->xlat_reg = &atom_sec_xlat; + + /* Enable Bus Master and Memory Space on the secondary side */ + iowrite16(PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER, + ndev->self_mmio + ATOM_SPCICMD_OFFSET); + + break; + + default: + return -EINVAL; + } + + ndev->db_valid_mask = BIT_ULL(ndev->db_count) - 1; + + return 0; +} + +static int atom_init_dev(struct intel_ntb_dev *ndev) +{ + u32 ppd; + int rc; + + rc = pci_read_config_dword(ndev->ntb.pdev, ATOM_PPD_OFFSET, &ppd); + if (rc) + return -EIO; + + ndev->ntb.topo = atom_ppd_topo(ndev, ppd); + if (ndev->ntb.topo == NTB_TOPO_NONE) + return -EINVAL; + + rc = atom_init_ntb(ndev); + if (rc) + return rc; + + rc = atom_init_isr(ndev); + if (rc) + return rc; + + if (ndev->ntb.topo != NTB_TOPO_SEC) { + /* Initiate PCI-E link training */ + rc = pci_write_config_dword(ndev->ntb.pdev, ATOM_PPD_OFFSET, + ppd | ATOM_PPD_INIT_LINK); + if (rc) + return rc; + } + + return 0; +} + +static void atom_deinit_dev(struct intel_ntb_dev *ndev) +{ + atom_deinit_isr(ndev); +} + +/* XEON */ + +static u64 xeon_db_ioread(void __iomem *mmio) +{ + return (u64)ioread16(mmio); +} + +static void xeon_db_iowrite(u64 bits, void __iomem *mmio) +{ + iowrite16((u16)bits, mmio); +} + +static int xeon_poll_link(struct intel_ntb_dev *ndev) +{ + u16 reg_val; + int rc; + + ndev->reg->db_iowrite(ndev->db_link_mask, + ndev->self_mmio + + ndev->self_reg->db_bell); + + rc = pci_read_config_word(ndev->ntb.pdev, + XEON_LINK_STATUS_OFFSET, ®_val); + if (rc) + return 0; + + if (reg_val == ndev->lnk_sta) + return 0; + + ndev->lnk_sta = reg_val; + + return 1; +} + +static int xeon_link_is_up(struct intel_ntb_dev *ndev) +{ + if (ndev->ntb.topo == NTB_TOPO_SEC) + return 1; + + return NTB_LNK_STA_ACTIVE(ndev->lnk_sta); +} + +static inline enum ntb_topo xeon_ppd_topo(struct intel_ntb_dev *ndev, u8 ppd) +{ + switch (ppd & XEON_PPD_TOPO_MASK) { + case XEON_PPD_TOPO_B2B_USD: + return NTB_TOPO_B2B_USD; + + case XEON_PPD_TOPO_B2B_DSD: + return NTB_TOPO_B2B_DSD; + + case XEON_PPD_TOPO_PRI_USD: + case XEON_PPD_TOPO_PRI_DSD: /* accept bogus PRI_DSD */ + return NTB_TOPO_PRI; + + case XEON_PPD_TOPO_SEC_USD: + case XEON_PPD_TOPO_SEC_DSD: /* accept bogus SEC_DSD */ + return NTB_TOPO_SEC; + } + + return NTB_TOPO_NONE; +} + +static inline int xeon_ppd_bar4_split(struct intel_ntb_dev *ndev, u8 ppd) +{ + if (ppd & XEON_PPD_SPLIT_BAR_MASK) { + dev_dbg(ndev_dev(ndev), "PPD %d split bar\n", ppd); + return 1; + } + return 0; +} + +static int xeon_init_isr(struct intel_ntb_dev *ndev) +{ + return ndev_init_isr(ndev, XEON_DB_MSIX_VECTOR_COUNT, + XEON_DB_MSIX_VECTOR_COUNT, + XEON_DB_MSIX_VECTOR_SHIFT, + XEON_DB_TOTAL_SHIFT); +} + +static void xeon_deinit_isr(struct intel_ntb_dev *ndev) +{ + ndev_deinit_isr(ndev); +} + +static int xeon_setup_b2b_mw(struct intel_ntb_dev *ndev, + const struct intel_b2b_addr *addr, + const struct intel_b2b_addr *peer_addr) +{ + struct pci_dev *pdev; + void __iomem *mmio; + resource_size_t bar_size; + phys_addr_t bar_addr; + int b2b_bar; + u8 bar_sz; + + pdev = ndev_pdev(ndev); + mmio = ndev->self_mmio; + + if (ndev->b2b_idx >= ndev->mw_count) { + dev_dbg(ndev_dev(ndev), "not using b2b mw\n"); + b2b_bar = 0; + ndev->b2b_off = 0; + } else { + b2b_bar = ndev_mw_to_bar(ndev, ndev->b2b_idx); + if (b2b_bar < 0) + return -EIO; + + dev_dbg(ndev_dev(ndev), "using b2b mw bar %d\n", b2b_bar); + + bar_size = pci_resource_len(ndev->ntb.pdev, b2b_bar); + + dev_dbg(ndev_dev(ndev), "b2b bar size %#llx\n", bar_size); + + if (b2b_mw_share && XEON_B2B_MIN_SIZE <= bar_size >> 1) { + dev_dbg(ndev_dev(ndev), + "b2b using first half of bar\n"); + ndev->b2b_off = bar_size >> 1; + } else if (XEON_B2B_MIN_SIZE <= bar_size) { + dev_dbg(ndev_dev(ndev), + "b2b using whole bar\n"); + ndev->b2b_off = 0; + --ndev->mw_count; + } else { + dev_dbg(ndev_dev(ndev), + "b2b bar size is too small\n"); + return -EIO; + } + } + + /* Reset the secondary bar sizes to match the primary bar sizes, + * except disable or halve the size of the b2b secondary bar. + * + * Note: code for each specific bar size register, because the register + * offsets are not in a consistent order (bar5sz comes after ppd, odd). + */ + pci_read_config_byte(pdev, XEON_PBAR23SZ_OFFSET, &bar_sz); + dev_dbg(ndev_dev(ndev), "PBAR23SZ %#x\n", bar_sz); + if (b2b_bar == 2) { + if (ndev->b2b_off) + bar_sz -= 1; + else + bar_sz = 0; + } + pci_write_config_byte(pdev, XEON_SBAR23SZ_OFFSET, bar_sz); + pci_read_config_byte(pdev, XEON_SBAR23SZ_OFFSET, &bar_sz); + dev_dbg(ndev_dev(ndev), "SBAR23SZ %#x\n", bar_sz); + + if (!ndev->bar4_split) { + pci_read_config_byte(pdev, XEON_PBAR45SZ_OFFSET, &bar_sz); + dev_dbg(ndev_dev(ndev), "PBAR45SZ %#x\n", bar_sz); + if (b2b_bar == 4) { + if (ndev->b2b_off) + bar_sz -= 1; + else + bar_sz = 0; + } + pci_write_config_byte(pdev, XEON_SBAR45SZ_OFFSET, bar_sz); + pci_read_config_byte(pdev, XEON_SBAR45SZ_OFFSET, &bar_sz); + dev_dbg(ndev_dev(ndev), "SBAR45SZ %#x\n", bar_sz); + } else { + pci_read_config_byte(pdev, XEON_PBAR4SZ_OFFSET, &bar_sz); + dev_dbg(ndev_dev(ndev), "PBAR4SZ %#x\n", bar_sz); + if (b2b_bar == 4) { + if (ndev->b2b_off) + bar_sz -= 1; + else + bar_sz = 0; + } + pci_write_config_byte(pdev, XEON_SBAR4SZ_OFFSET, bar_sz); + pci_read_config_byte(pdev, XEON_SBAR4SZ_OFFSET, &bar_sz); + dev_dbg(ndev_dev(ndev), "SBAR4SZ %#x\n", bar_sz); + + pci_read_config_byte(pdev, XEON_PBAR5SZ_OFFSET, &bar_sz); + dev_dbg(ndev_dev(ndev), "PBAR5SZ %#x\n", bar_sz); + if (b2b_bar == 5) { + if (ndev->b2b_off) + bar_sz -= 1; + else + bar_sz = 0; + } + pci_write_config_byte(pdev, XEON_SBAR5SZ_OFFSET, bar_sz); + pci_read_config_byte(pdev, XEON_SBAR5SZ_OFFSET, &bar_sz); + dev_dbg(ndev_dev(ndev), "SBAR5SZ %#x\n", bar_sz); + } + + /* SBAR01 hit by first part of the b2b bar */ + if (b2b_bar == 0) + bar_addr = addr->bar0_addr; + else if (b2b_bar == 2) + bar_addr = addr->bar2_addr64; + else if (b2b_bar == 4 && !ndev->bar4_split) + bar_addr = addr->bar4_addr64; + else if (b2b_bar == 4) + bar_addr = addr->bar4_addr32; + else if (b2b_bar == 5) + bar_addr = addr->bar5_addr32; + else + return -EIO; + + dev_dbg(ndev_dev(ndev), "SBAR01 %#018llx\n", bar_addr); + iowrite64(bar_addr, mmio + XEON_SBAR0BASE_OFFSET); + + /* Other SBAR are normally hit by the PBAR xlat, except for b2b bar. + * The b2b bar is either disabled above, or configured half-size, and + * it starts at the PBAR xlat + offset. + */ + + bar_addr = addr->bar2_addr64 + (b2b_bar == 2 ? ndev->b2b_off : 0); + iowrite64(bar_addr, mmio + XEON_SBAR23BASE_OFFSET); + bar_addr = ioread64(mmio + XEON_SBAR23BASE_OFFSET); + dev_dbg(ndev_dev(ndev), "SBAR23 %#018llx\n", bar_addr); + + if (!ndev->bar4_split) { + bar_addr = addr->bar4_addr64 + + (b2b_bar == 4 ? ndev->b2b_off : 0); + iowrite64(bar_addr, mmio + XEON_SBAR45BASE_OFFSET); + bar_addr = ioread64(mmio + XEON_SBAR45BASE_OFFSET); + dev_dbg(ndev_dev(ndev), "SBAR45 %#018llx\n", bar_addr); + } else { + bar_addr = addr->bar4_addr32 + + (b2b_bar == 4 ? ndev->b2b_off : 0); + iowrite32(bar_addr, mmio + XEON_SBAR4BASE_OFFSET); + bar_addr = ioread32(mmio + XEON_SBAR4BASE_OFFSET); + dev_dbg(ndev_dev(ndev), "SBAR4 %#010llx\n", bar_addr); + + bar_addr = addr->bar5_addr32 + + (b2b_bar == 5 ? ndev->b2b_off : 0); + iowrite32(bar_addr, mmio + XEON_SBAR5BASE_OFFSET); + bar_addr = ioread32(mmio + XEON_SBAR5BASE_OFFSET); + dev_dbg(ndev_dev(ndev), "SBAR5 %#010llx\n", bar_addr); + } + + /* setup incoming bar limits == base addrs (zero length windows) */ + + bar_addr = addr->bar2_addr64 + (b2b_bar == 2 ? ndev->b2b_off : 0); + iowrite64(bar_addr, mmio + XEON_SBAR23LMT_OFFSET); + bar_addr = ioread64(mmio + XEON_SBAR23LMT_OFFSET); + dev_dbg(ndev_dev(ndev), "SBAR23LMT %#018llx\n", bar_addr); + + if (!ndev->bar4_split) { + bar_addr = addr->bar4_addr64 + + (b2b_bar == 4 ? ndev->b2b_off : 0); + iowrite64(bar_addr, mmio + XEON_SBAR45LMT_OFFSET); + bar_addr = ioread64(mmio + XEON_SBAR45LMT_OFFSET); + dev_dbg(ndev_dev(ndev), "SBAR45LMT %#018llx\n", bar_addr); + } else { + bar_addr = addr->bar4_addr32 + + (b2b_bar == 4 ? ndev->b2b_off : 0); + iowrite32(bar_addr, mmio + XEON_SBAR4LMT_OFFSET); + bar_addr = ioread32(mmio + XEON_SBAR4LMT_OFFSET); + dev_dbg(ndev_dev(ndev), "SBAR4LMT %#010llx\n", bar_addr); + + bar_addr = addr->bar5_addr32 + + (b2b_bar == 5 ? ndev->b2b_off : 0); + iowrite32(bar_addr, mmio + XEON_SBAR5LMT_OFFSET); + bar_addr = ioread32(mmio + XEON_SBAR5LMT_OFFSET); + dev_dbg(ndev_dev(ndev), "SBAR5LMT %#05llx\n", bar_addr); + } + + /* zero incoming translation addrs */ + iowrite64(0, mmio + XEON_SBAR23XLAT_OFFSET); + + if (!ndev->bar4_split) { + iowrite64(0, mmio + XEON_SBAR45XLAT_OFFSET); + } else { + iowrite32(0, mmio + XEON_SBAR4XLAT_OFFSET); + iowrite32(0, mmio + XEON_SBAR5XLAT_OFFSET); + } + + /* zero outgoing translation limits (whole bar size windows) */ + iowrite64(0, mmio + XEON_PBAR23LMT_OFFSET); + if (!ndev->bar4_split) { + iowrite64(0, mmio + XEON_PBAR45LMT_OFFSET); + } else { + iowrite32(0, mmio + XEON_PBAR4LMT_OFFSET); + iowrite32(0, mmio + XEON_PBAR5LMT_OFFSET); + } + + /* set outgoing translation offsets */ + bar_addr = peer_addr->bar2_addr64; + iowrite64(bar_addr, mmio + XEON_PBAR23XLAT_OFFSET); + bar_addr = ioread64(mmio + XEON_PBAR23XLAT_OFFSET); + dev_dbg(ndev_dev(ndev), "PBAR23XLAT %#018llx\n", bar_addr); + + if (!ndev->bar4_split) { + bar_addr = peer_addr->bar4_addr64; + iowrite64(bar_addr, mmio + XEON_PBAR45XLAT_OFFSET); + bar_addr = ioread64(mmio + XEON_PBAR45XLAT_OFFSET); + dev_dbg(ndev_dev(ndev), "PBAR45XLAT %#018llx\n", bar_addr); + } else { + bar_addr = peer_addr->bar4_addr32; + iowrite32(bar_addr, mmio + XEON_PBAR4XLAT_OFFSET); + bar_addr = ioread32(mmio + XEON_PBAR4XLAT_OFFSET); + dev_dbg(ndev_dev(ndev), "PBAR4XLAT %#010llx\n", bar_addr); + + bar_addr = peer_addr->bar5_addr32; + iowrite32(bar_addr, mmio + XEON_PBAR5XLAT_OFFSET); + bar_addr = ioread32(mmio + XEON_PBAR5XLAT_OFFSET); + dev_dbg(ndev_dev(ndev), "PBAR5XLAT %#010llx\n", bar_addr); + } + + /* set the translation offset for b2b registers */ + if (b2b_bar == 0) + bar_addr = peer_addr->bar0_addr; + else if (b2b_bar == 2) + bar_addr = peer_addr->bar2_addr64; + else if (b2b_bar == 4 && !ndev->bar4_split) + bar_addr = peer_addr->bar4_addr64; + else if (b2b_bar == 4) + bar_addr = peer_addr->bar4_addr32; + else if (b2b_bar == 5) + bar_addr = peer_addr->bar5_addr32; + else + return -EIO; + + /* B2B_XLAT_OFFSET is 64bit, but can only take 32bit writes */ + dev_dbg(ndev_dev(ndev), "B2BXLAT %#018llx\n", bar_addr); + iowrite32(bar_addr, mmio + XEON_B2B_XLAT_OFFSETL); + iowrite32(bar_addr >> 32, mmio + XEON_B2B_XLAT_OFFSETU); + + if (b2b_bar) { + /* map peer ntb mmio config space registers */ + ndev->peer_mmio = pci_iomap(pdev, b2b_bar, + XEON_B2B_MIN_SIZE); + if (!ndev->peer_mmio) + return -EIO; + } + + return 0; +} + +static int xeon_init_ntb(struct intel_ntb_dev *ndev) +{ + int rc; + u32 ntb_ctl; + + if (ndev->bar4_split) + ndev->mw_count = HSX_SPLIT_BAR_MW_COUNT; + else + ndev->mw_count = XEON_MW_COUNT; + + ndev->spad_count = XEON_SPAD_COUNT; + ndev->db_count = XEON_DB_COUNT; + ndev->db_link_mask = XEON_DB_LINK_BIT; + + switch (ndev->ntb.topo) { + case NTB_TOPO_PRI: + if (ndev->hwerr_flags & NTB_HWERR_SDOORBELL_LOCKUP) { + dev_err(ndev_dev(ndev), "NTB Primary config disabled\n"); + return -EINVAL; + } + + /* enable link to allow secondary side device to appear */ + ntb_ctl = ioread32(ndev->self_mmio + ndev->reg->ntb_ctl); + ntb_ctl &= ~NTB_CTL_DISABLE; + iowrite32(ntb_ctl, ndev->self_mmio + ndev->reg->ntb_ctl); + + /* use half the spads for the peer */ + ndev->spad_count >>= 1; + ndev->self_reg = &xeon_pri_reg; + ndev->peer_reg = &xeon_sec_reg; + ndev->xlat_reg = &xeon_sec_xlat; + break; + + case NTB_TOPO_SEC: + if (ndev->hwerr_flags & NTB_HWERR_SDOORBELL_LOCKUP) { + dev_err(ndev_dev(ndev), "NTB Secondary config disabled\n"); + return -EINVAL; + } + /* use half the spads for the peer */ + ndev->spad_count >>= 1; + ndev->self_reg = &xeon_sec_reg; + ndev->peer_reg = &xeon_pri_reg; + ndev->xlat_reg = &xeon_pri_xlat; + break; + + case NTB_TOPO_B2B_USD: + case NTB_TOPO_B2B_DSD: + ndev->self_reg = &xeon_pri_reg; + ndev->peer_reg = &xeon_b2b_reg; + ndev->xlat_reg = &xeon_sec_xlat; + + if (ndev->hwerr_flags & NTB_HWERR_SDOORBELL_LOCKUP) { + ndev->peer_reg = &xeon_pri_reg; + + if (b2b_mw_idx < 0) + ndev->b2b_idx = b2b_mw_idx + ndev->mw_count; + else + ndev->b2b_idx = b2b_mw_idx; + + dev_dbg(ndev_dev(ndev), + "setting up b2b mw idx %d means %d\n", + b2b_mw_idx, ndev->b2b_idx); + + } else if (ndev->hwerr_flags & NTB_HWERR_B2BDOORBELL_BIT14) { + dev_warn(ndev_dev(ndev), "Reduce doorbell count by 1\n"); + ndev->db_count -= 1; + } + + if (ndev->ntb.topo == NTB_TOPO_B2B_USD) { + rc = xeon_setup_b2b_mw(ndev, + &xeon_b2b_dsd_addr, + &xeon_b2b_usd_addr); + } else { + rc = xeon_setup_b2b_mw(ndev, + &xeon_b2b_usd_addr, + &xeon_b2b_dsd_addr); + } + if (rc) + return rc; + + /* Enable Bus Master and Memory Space on the secondary side */ + iowrite16(PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER, + ndev->self_mmio + XEON_SPCICMD_OFFSET); + + break; + + default: + return -EINVAL; + } + + ndev->db_valid_mask = BIT_ULL(ndev->db_count) - 1; + + ndev->reg->db_iowrite(ndev->db_valid_mask, + ndev->self_mmio + + ndev->self_reg->db_mask); + + return 0; +} + +static int xeon_init_dev(struct intel_ntb_dev *ndev) +{ + struct pci_dev *pdev; + u8 ppd; + int rc, mem; + + pdev = ndev_pdev(ndev); + + switch (pdev->device) { + /* There is a Xeon hardware errata related to writes to SDOORBELL or + * B2BDOORBELL in conjunction with inbound access to NTB MMIO Space, + * which may hang the system. To workaround this use the second memory + * window to access the interrupt and scratch pad registers on the + * remote system. + */ + case PCI_DEVICE_ID_INTEL_NTB_SS_JSF: + case PCI_DEVICE_ID_INTEL_NTB_PS_JSF: + case PCI_DEVICE_ID_INTEL_NTB_B2B_JSF: + case PCI_DEVICE_ID_INTEL_NTB_SS_SNB: + case PCI_DEVICE_ID_INTEL_NTB_PS_SNB: + case PCI_DEVICE_ID_INTEL_NTB_B2B_SNB: + case PCI_DEVICE_ID_INTEL_NTB_SS_IVT: + case PCI_DEVICE_ID_INTEL_NTB_PS_IVT: + case PCI_DEVICE_ID_INTEL_NTB_B2B_IVT: + case PCI_DEVICE_ID_INTEL_NTB_SS_HSX: + case PCI_DEVICE_ID_INTEL_NTB_PS_HSX: + case PCI_DEVICE_ID_INTEL_NTB_B2B_HSX: + ndev->hwerr_flags |= NTB_HWERR_SDOORBELL_LOCKUP; + break; + } + + switch (pdev->device) { + /* There is a hardware errata related to accessing any register in + * SB01BASE in the presence of bidirectional traffic crossing the NTB. + */ + case PCI_DEVICE_ID_INTEL_NTB_SS_IVT: + case PCI_DEVICE_ID_INTEL_NTB_PS_IVT: + case PCI_DEVICE_ID_INTEL_NTB_B2B_IVT: + case PCI_DEVICE_ID_INTEL_NTB_SS_HSX: + case PCI_DEVICE_ID_INTEL_NTB_PS_HSX: + case PCI_DEVICE_ID_INTEL_NTB_B2B_HSX: + ndev->hwerr_flags |= NTB_HWERR_SB01BASE_LOCKUP; + break; + } + + switch (pdev->device) { + /* HW Errata on bit 14 of b2bdoorbell register. Writes will not be + * mirrored to the remote system. Shrink the number of bits by one, + * since bit 14 is the last bit. + */ + case PCI_DEVICE_ID_INTEL_NTB_SS_JSF: + case PCI_DEVICE_ID_INTEL_NTB_PS_JSF: + case PCI_DEVICE_ID_INTEL_NTB_B2B_JSF: + case PCI_DEVICE_ID_INTEL_NTB_SS_SNB: + case PCI_DEVICE_ID_INTEL_NTB_PS_SNB: + case PCI_DEVICE_ID_INTEL_NTB_B2B_SNB: + case PCI_DEVICE_ID_INTEL_NTB_SS_IVT: + case PCI_DEVICE_ID_INTEL_NTB_PS_IVT: + case PCI_DEVICE_ID_INTEL_NTB_B2B_IVT: + case PCI_DEVICE_ID_INTEL_NTB_SS_HSX: + case PCI_DEVICE_ID_INTEL_NTB_PS_HSX: + case PCI_DEVICE_ID_INTEL_NTB_B2B_HSX: + ndev->hwerr_flags |= NTB_HWERR_B2BDOORBELL_BIT14; + break; + } + + ndev->reg = &xeon_reg; + + rc = pci_read_config_byte(pdev, XEON_PPD_OFFSET, &ppd); + if (rc) + return -EIO; + + ndev->ntb.topo = xeon_ppd_topo(ndev, ppd); + dev_dbg(ndev_dev(ndev), "ppd %#x topo %s\n", ppd, + ntb_topo_string(ndev->ntb.topo)); + if (ndev->ntb.topo == NTB_TOPO_NONE) + return -EINVAL; + + if (ndev->ntb.topo != NTB_TOPO_SEC) { + ndev->bar4_split = xeon_ppd_bar4_split(ndev, ppd); + dev_dbg(ndev_dev(ndev), "ppd %#x bar4_split %d\n", + ppd, ndev->bar4_split); + } else { + /* This is a way for transparent BAR to figure out if we are + * doing split BAR or not. There is no way for the hw on the + * transparent side to know and set the PPD. + */ + mem = pci_select_bars(pdev, IORESOURCE_MEM); + ndev->bar4_split = hweight32(mem) == + HSX_SPLIT_BAR_MW_COUNT + 1; + dev_dbg(ndev_dev(ndev), "mem %#x bar4_split %d\n", + mem, ndev->bar4_split); + } + + rc = xeon_init_ntb(ndev); + if (rc) + return rc; + + return xeon_init_isr(ndev); +} + +static void xeon_deinit_dev(struct intel_ntb_dev *ndev) +{ + xeon_deinit_isr(ndev); +} + +static int intel_ntb_init_pci(struct intel_ntb_dev *ndev, struct pci_dev *pdev) +{ + int rc; + + pci_set_drvdata(pdev, ndev); + + rc = pci_enable_device(pdev); + if (rc) + goto err_pci_enable; + + rc = pci_request_regions(pdev, NTB_NAME); + if (rc) + goto err_pci_regions; + + pci_set_master(pdev); + + rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(64)); + if (rc) { + rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(32)); + if (rc) + goto err_dma_mask; + dev_warn(ndev_dev(ndev), "Cannot DMA highmem\n"); + } + + rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64)); + if (rc) { + rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32)); + if (rc) + goto err_dma_mask; + dev_warn(ndev_dev(ndev), "Cannot DMA consistent highmem\n"); + } + + ndev->self_mmio = pci_iomap(pdev, 0, 0); + if (!ndev->self_mmio) { + rc = -EIO; + goto err_mmio; + } + ndev->peer_mmio = ndev->self_mmio; + + return 0; + +err_mmio: +err_dma_mask: + pci_clear_master(pdev); + pci_release_regions(pdev); +err_pci_regions: + pci_disable_device(pdev); +err_pci_enable: + pci_set_drvdata(pdev, NULL); + return rc; +} + +static void intel_ntb_deinit_pci(struct intel_ntb_dev *ndev) +{ + struct pci_dev *pdev = ndev_pdev(ndev); + + if (ndev->peer_mmio && ndev->peer_mmio != ndev->self_mmio) + pci_iounmap(pdev, ndev->peer_mmio); + pci_iounmap(pdev, ndev->self_mmio); + + pci_clear_master(pdev); + pci_release_regions(pdev); + pci_disable_device(pdev); + pci_set_drvdata(pdev, NULL); +} + +static inline void ndev_init_struct(struct intel_ntb_dev *ndev, + struct pci_dev *pdev) +{ + ndev->ntb.pdev = pdev; + ndev->ntb.topo = NTB_TOPO_NONE; + ndev->ntb.ops = &intel_ntb_ops; + + ndev->b2b_off = 0; + ndev->b2b_idx = INT_MAX; + + ndev->bar4_split = 0; + + ndev->mw_count = 0; + ndev->spad_count = 0; + ndev->db_count = 0; + ndev->db_vec_count = 0; + ndev->db_vec_shift = 0; + + ndev->ntb_ctl = 0; + ndev->lnk_sta = 0; + + ndev->db_valid_mask = 0; + ndev->db_link_mask = 0; + ndev->db_mask = 0; + + spin_lock_init(&ndev->db_mask_lock); +} + +static int intel_ntb_pci_probe(struct pci_dev *pdev, + const struct pci_device_id *id) +{ + struct intel_ntb_dev *ndev; + int rc, node; + + node = dev_to_node(&pdev->dev); + + if (pdev_is_atom(pdev)) { + ndev = kzalloc_node(sizeof(*ndev), GFP_KERNEL, node); + if (!ndev) { + rc = -ENOMEM; + goto err_ndev; + } + + ndev_init_struct(ndev, pdev); + + rc = intel_ntb_init_pci(ndev, pdev); + if (rc) + goto err_init_pci; + + rc = atom_init_dev(ndev); + if (rc) + goto err_init_dev; + + } else if (pdev_is_xeon(pdev)) { + ndev = kzalloc_node(sizeof(*ndev), GFP_KERNEL, node); + if (!ndev) { + rc = -ENOMEM; + goto err_ndev; + } + + ndev_init_struct(ndev, pdev); + + rc = intel_ntb_init_pci(ndev, pdev); + if (rc) + goto err_init_pci; + + rc = xeon_init_dev(ndev); + if (rc) + goto err_init_dev; + + } else { + rc = -EINVAL; + goto err_ndev; + } + + ndev_reset_unsafe_flags(ndev); + + ndev->reg->poll_link(ndev); + + ndev_init_debugfs(ndev); + + rc = ntb_register_device(&ndev->ntb); + if (rc) + goto err_register; + + dev_info(&pdev->dev, "NTB device registered.\n"); + + return 0; + +err_register: + ndev_deinit_debugfs(ndev); + if (pdev_is_atom(pdev)) + atom_deinit_dev(ndev); + else if (pdev_is_xeon(pdev)) + xeon_deinit_dev(ndev); +err_init_dev: + intel_ntb_deinit_pci(ndev); +err_init_pci: + kfree(ndev); +err_ndev: + return rc; +} + +static void intel_ntb_pci_remove(struct pci_dev *pdev) +{ + struct intel_ntb_dev *ndev = pci_get_drvdata(pdev); + + ntb_unregister_device(&ndev->ntb); + ndev_deinit_debugfs(ndev); + if (pdev_is_atom(pdev)) + atom_deinit_dev(ndev); + else if (pdev_is_xeon(pdev)) + xeon_deinit_dev(ndev); + intel_ntb_deinit_pci(ndev); + kfree(ndev); +} + +static const struct intel_ntb_reg atom_reg = { + .poll_link = atom_poll_link, + .link_is_up = atom_link_is_up, + .db_ioread = atom_db_ioread, + .db_iowrite = atom_db_iowrite, + .db_size = sizeof(u64), + .ntb_ctl = ATOM_NTBCNTL_OFFSET, + .mw_bar = {2, 4}, +}; + +static const struct intel_ntb_alt_reg atom_pri_reg = { + .db_bell = ATOM_PDOORBELL_OFFSET, + .db_mask = ATOM_PDBMSK_OFFSET, + .spad = ATOM_SPAD_OFFSET, +}; + +static const struct intel_ntb_alt_reg atom_b2b_reg = { + .db_bell = ATOM_B2B_DOORBELL_OFFSET, + .spad = ATOM_B2B_SPAD_OFFSET, +}; + +static const struct intel_ntb_xlat_reg atom_sec_xlat = { + /* FIXME : .bar0_base = ATOM_SBAR0BASE_OFFSET, */ + /* FIXME : .bar2_limit = ATOM_SBAR2LMT_OFFSET, */ + .bar2_xlat = ATOM_SBAR2XLAT_OFFSET, +}; + +static const struct intel_ntb_reg xeon_reg = { + .poll_link = xeon_poll_link, + .link_is_up = xeon_link_is_up, + .db_ioread = xeon_db_ioread, + .db_iowrite = xeon_db_iowrite, + .db_size = sizeof(u32), + .ntb_ctl = XEON_NTBCNTL_OFFSET, + .mw_bar = {2, 4, 5}, +}; + +static const struct intel_ntb_alt_reg xeon_pri_reg = { + .db_bell = XEON_PDOORBELL_OFFSET, + .db_mask = XEON_PDBMSK_OFFSET, + .spad = XEON_SPAD_OFFSET, +}; + +static const struct intel_ntb_alt_reg xeon_sec_reg = { + .db_bell = XEON_SDOORBELL_OFFSET, + .db_mask = XEON_SDBMSK_OFFSET, + /* second half of the scratchpads */ + .spad = XEON_SPAD_OFFSET + (XEON_SPAD_COUNT << 1), +}; + +static const struct intel_ntb_alt_reg xeon_b2b_reg = { + .db_bell = XEON_B2B_DOORBELL_OFFSET, + .spad = XEON_B2B_SPAD_OFFSET, +}; + +static const struct intel_ntb_xlat_reg xeon_pri_xlat = { + /* Note: no primary .bar0_base visible to the secondary side. + * + * The secondary side cannot get the base address stored in primary + * bars. The base address is necessary to set the limit register to + * any value other than zero, or unlimited. + * + * WITHOUT THE BASE ADDRESS, THE SECONDARY SIDE CANNOT DISABLE the + * window by setting the limit equal to base, nor can it limit the size + * of the memory window by setting the limit to base + size. + */ + .bar2_limit = XEON_PBAR23LMT_OFFSET, + .bar2_xlat = XEON_PBAR23XLAT_OFFSET, +}; + +static const struct intel_ntb_xlat_reg xeon_sec_xlat = { + .bar0_base = XEON_SBAR0BASE_OFFSET, + .bar2_limit = XEON_SBAR23LMT_OFFSET, + .bar2_xlat = XEON_SBAR23XLAT_OFFSET, +}; + +static struct intel_b2b_addr xeon_b2b_usd_addr = { + .bar2_addr64 = XEON_B2B_BAR2_USD_ADDR64, + .bar4_addr64 = XEON_B2B_BAR4_USD_ADDR64, + .bar4_addr32 = XEON_B2B_BAR4_USD_ADDR32, + .bar5_addr32 = XEON_B2B_BAR5_USD_ADDR32, +}; + +static struct intel_b2b_addr xeon_b2b_dsd_addr = { + .bar2_addr64 = XEON_B2B_BAR2_DSD_ADDR64, + .bar4_addr64 = XEON_B2B_BAR4_DSD_ADDR64, + .bar4_addr32 = XEON_B2B_BAR4_DSD_ADDR32, + .bar5_addr32 = XEON_B2B_BAR5_DSD_ADDR32, +}; + +/* operations for primary side of local ntb */ +static const struct ntb_dev_ops intel_ntb_ops = { + .mw_count = intel_ntb_mw_count, + .mw_get_range = intel_ntb_mw_get_range, + .mw_set_trans = intel_ntb_mw_set_trans, + .link_is_up = intel_ntb_link_is_up, + .link_enable = intel_ntb_link_enable, + .link_disable = intel_ntb_link_disable, + .db_is_unsafe = intel_ntb_db_is_unsafe, + .db_valid_mask = intel_ntb_db_valid_mask, + .db_vector_count = intel_ntb_db_vector_count, + .db_vector_mask = intel_ntb_db_vector_mask, + .db_read = intel_ntb_db_read, + .db_clear = intel_ntb_db_clear, + .db_set_mask = intel_ntb_db_set_mask, + .db_clear_mask = intel_ntb_db_clear_mask, + .peer_db_addr = intel_ntb_peer_db_addr, + .peer_db_set = intel_ntb_peer_db_set, + .spad_is_unsafe = intel_ntb_spad_is_unsafe, + .spad_count = intel_ntb_spad_count, + .spad_read = intel_ntb_spad_read, + .spad_write = intel_ntb_spad_write, + .peer_spad_addr = intel_ntb_peer_spad_addr, + .peer_spad_read = intel_ntb_peer_spad_read, + .peer_spad_write = intel_ntb_peer_spad_write, +}; + +static const struct file_operations intel_ntb_debugfs_info = { + .owner = THIS_MODULE, + .open = simple_open, + .read = ndev_debugfs_read, +}; + +static const struct pci_device_id intel_ntb_pci_tbl[] = { + {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_BWD)}, + {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_JSF)}, + {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_SNB)}, + {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_IVT)}, + {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_HSX)}, + {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_JSF)}, + {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_SNB)}, + {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_IVT)}, + {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_HSX)}, + {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_JSF)}, + {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_SNB)}, + {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_IVT)}, + {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_HSX)}, + {0} +}; +MODULE_DEVICE_TABLE(pci, intel_ntb_pci_tbl); + +static struct pci_driver intel_ntb_pci_driver = { + .name = KBUILD_MODNAME, + .id_table = intel_ntb_pci_tbl, + .probe = intel_ntb_pci_probe, + .remove = intel_ntb_pci_remove, +}; + +static int __init intel_ntb_pci_driver_init(void) +{ + pr_info("%s %s\n", NTB_DESC, NTB_VER); + + if (debugfs_initialized()) + debugfs_dir = debugfs_create_dir(KBUILD_MODNAME, NULL); + + return pci_register_driver(&intel_ntb_pci_driver); +} +module_init(intel_ntb_pci_driver_init); + +static void __exit intel_ntb_pci_driver_exit(void) +{ + pci_unregister_driver(&intel_ntb_pci_driver); + + debugfs_remove_recursive(debugfs_dir); +} +module_exit(intel_ntb_pci_driver_exit); + diff --git a/drivers/ntb/hw/intel/ntb_hw_intel.h b/drivers/ntb/hw/intel/ntb_hw_intel.h new file mode 100644 index 000000000000..7ddaf387b679 --- /dev/null +++ b/drivers/ntb/hw/intel/ntb_hw_intel.h @@ -0,0 +1,342 @@ +/* + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * GPL LICENSE SUMMARY + * + * Copyright(c) 2012 Intel Corporation. All rights reserved. + * Copyright (C) 2015 EMC Corporation. All Rights Reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of version 2 of the GNU General Public License as + * published by the Free Software Foundation. + * + * BSD LICENSE + * + * Copyright(c) 2012 Intel Corporation. All rights reserved. + * Copyright (C) 2015 EMC Corporation. All Rights Reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copy + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * Intel PCIe NTB Linux driver + * + * Contact Information: + * Jon Mason <jon.mason@intel.com> + */ + +#ifndef NTB_HW_INTEL_H +#define NTB_HW_INTEL_H + +#include <linux/ntb.h> +#include <linux/pci.h> + +#define PCI_DEVICE_ID_INTEL_NTB_B2B_JSF 0x3725 +#define PCI_DEVICE_ID_INTEL_NTB_PS_JSF 0x3726 +#define PCI_DEVICE_ID_INTEL_NTB_SS_JSF 0x3727 +#define PCI_DEVICE_ID_INTEL_NTB_B2B_SNB 0x3C0D +#define PCI_DEVICE_ID_INTEL_NTB_PS_SNB 0x3C0E +#define PCI_DEVICE_ID_INTEL_NTB_SS_SNB 0x3C0F +#define PCI_DEVICE_ID_INTEL_NTB_B2B_IVT 0x0E0D +#define PCI_DEVICE_ID_INTEL_NTB_PS_IVT 0x0E0E +#define PCI_DEVICE_ID_INTEL_NTB_SS_IVT 0x0E0F +#define PCI_DEVICE_ID_INTEL_NTB_B2B_HSX 0x2F0D +#define PCI_DEVICE_ID_INTEL_NTB_PS_HSX 0x2F0E +#define PCI_DEVICE_ID_INTEL_NTB_SS_HSX 0x2F0F +#define PCI_DEVICE_ID_INTEL_NTB_B2B_BWD 0x0C4E + +/* Intel Xeon hardware */ + +#define XEON_PBAR23LMT_OFFSET 0x0000 +#define XEON_PBAR45LMT_OFFSET 0x0008 +#define XEON_PBAR4LMT_OFFSET 0x0008 +#define XEON_PBAR5LMT_OFFSET 0x000c +#define XEON_PBAR23XLAT_OFFSET 0x0010 +#define XEON_PBAR45XLAT_OFFSET 0x0018 +#define XEON_PBAR4XLAT_OFFSET 0x0018 +#define XEON_PBAR5XLAT_OFFSET 0x001c +#define XEON_SBAR23LMT_OFFSET 0x0020 +#define XEON_SBAR45LMT_OFFSET 0x0028 +#define XEON_SBAR4LMT_OFFSET 0x0028 +#define XEON_SBAR5LMT_OFFSET 0x002c +#define XEON_SBAR23XLAT_OFFSET 0x0030 +#define XEON_SBAR45XLAT_OFFSET 0x0038 +#define XEON_SBAR4XLAT_OFFSET 0x0038 +#define XEON_SBAR5XLAT_OFFSET 0x003c +#define XEON_SBAR0BASE_OFFSET 0x0040 +#define XEON_SBAR23BASE_OFFSET 0x0048 +#define XEON_SBAR45BASE_OFFSET 0x0050 +#define XEON_SBAR4BASE_OFFSET 0x0050 +#define XEON_SBAR5BASE_OFFSET 0x0054 +#define XEON_SBDF_OFFSET 0x005c +#define XEON_NTBCNTL_OFFSET 0x0058 +#define XEON_PDOORBELL_OFFSET 0x0060 +#define XEON_PDBMSK_OFFSET 0x0062 +#define XEON_SDOORBELL_OFFSET 0x0064 +#define XEON_SDBMSK_OFFSET 0x0066 +#define XEON_USMEMMISS_OFFSET 0x0070 +#define XEON_SPAD_OFFSET 0x0080 +#define XEON_PBAR23SZ_OFFSET 0x00d0 +#define XEON_PBAR45SZ_OFFSET 0x00d1 +#define XEON_PBAR4SZ_OFFSET 0x00d1 +#define XEON_SBAR23SZ_OFFSET 0x00d2 +#define XEON_SBAR45SZ_OFFSET 0x00d3 +#define XEON_SBAR4SZ_OFFSET 0x00d3 +#define XEON_PPD_OFFSET 0x00d4 +#define XEON_PBAR5SZ_OFFSET 0x00d5 +#define XEON_SBAR5SZ_OFFSET 0x00d6 +#define XEON_WCCNTRL_OFFSET 0x00e0 +#define XEON_UNCERRSTS_OFFSET 0x014c +#define XEON_CORERRSTS_OFFSET 0x0158 +#define XEON_LINK_STATUS_OFFSET 0x01a2 +#define XEON_SPCICMD_OFFSET 0x0504 +#define XEON_DEVCTRL_OFFSET 0x0598 +#define XEON_DEVSTS_OFFSET 0x059a +#define XEON_SLINK_STATUS_OFFSET 0x05a2 +#define XEON_B2B_SPAD_OFFSET 0x0100 +#define XEON_B2B_DOORBELL_OFFSET 0x0140 +#define XEON_B2B_XLAT_OFFSETL 0x0144 +#define XEON_B2B_XLAT_OFFSETU 0x0148 +#define XEON_PPD_CONN_MASK 0x03 +#define XEON_PPD_CONN_TRANSPARENT 0x00 +#define XEON_PPD_CONN_B2B 0x01 +#define XEON_PPD_CONN_RP 0x02 +#define XEON_PPD_DEV_MASK 0x10 +#define XEON_PPD_DEV_USD 0x00 +#define XEON_PPD_DEV_DSD 0x10 +#define XEON_PPD_SPLIT_BAR_MASK 0x40 + +#define XEON_PPD_TOPO_MASK (XEON_PPD_CONN_MASK | XEON_PPD_DEV_MASK) +#define XEON_PPD_TOPO_PRI_USD (XEON_PPD_CONN_RP | XEON_PPD_DEV_USD) +#define XEON_PPD_TOPO_PRI_DSD (XEON_PPD_CONN_RP | XEON_PPD_DEV_DSD) +#define XEON_PPD_TOPO_SEC_USD (XEON_PPD_CONN_TRANSPARENT | XEON_PPD_DEV_USD) +#define XEON_PPD_TOPO_SEC_DSD (XEON_PPD_CONN_TRANSPARENT | XEON_PPD_DEV_DSD) +#define XEON_PPD_TOPO_B2B_USD (XEON_PPD_CONN_B2B | XEON_PPD_DEV_USD) +#define XEON_PPD_TOPO_B2B_DSD (XEON_PPD_CONN_B2B | XEON_PPD_DEV_DSD) + +#define XEON_MW_COUNT 2 +#define HSX_SPLIT_BAR_MW_COUNT 3 +#define XEON_DB_COUNT 15 +#define XEON_DB_LINK 15 +#define XEON_DB_LINK_BIT BIT_ULL(XEON_DB_LINK) +#define XEON_DB_MSIX_VECTOR_COUNT 4 +#define XEON_DB_MSIX_VECTOR_SHIFT 5 +#define XEON_DB_TOTAL_SHIFT 16 +#define XEON_SPAD_COUNT 16 + +/* Intel Atom hardware */ + +#define ATOM_SBAR2XLAT_OFFSET 0x0008 +#define ATOM_PDOORBELL_OFFSET 0x0020 +#define ATOM_PDBMSK_OFFSET 0x0028 +#define ATOM_NTBCNTL_OFFSET 0x0060 +#define ATOM_SPAD_OFFSET 0x0080 +#define ATOM_PPD_OFFSET 0x00d4 +#define ATOM_PBAR2XLAT_OFFSET 0x8008 +#define ATOM_B2B_DOORBELL_OFFSET 0x8020 +#define ATOM_B2B_SPAD_OFFSET 0x8080 +#define ATOM_SPCICMD_OFFSET 0xb004 +#define ATOM_LINK_STATUS_OFFSET 0xb052 +#define ATOM_ERRCORSTS_OFFSET 0xb110 +#define ATOM_IP_BASE 0xc000 +#define ATOM_DESKEWSTS_OFFSET (ATOM_IP_BASE + 0x3024) +#define ATOM_LTSSMERRSTS0_OFFSET (ATOM_IP_BASE + 0x3180) +#define ATOM_LTSSMSTATEJMP_OFFSET (ATOM_IP_BASE + 0x3040) +#define ATOM_IBSTERRRCRVSTS0_OFFSET (ATOM_IP_BASE + 0x3324) +#define ATOM_MODPHY_PCSREG4 0x1c004 +#define ATOM_MODPHY_PCSREG6 0x1c006 + +#define ATOM_PPD_INIT_LINK 0x0008 +#define ATOM_PPD_CONN_MASK 0x0300 +#define ATOM_PPD_CONN_TRANSPARENT 0x0000 +#define ATOM_PPD_CONN_B2B 0x0100 +#define ATOM_PPD_CONN_RP 0x0200 +#define ATOM_PPD_DEV_MASK 0x1000 +#define ATOM_PPD_DEV_USD 0x0000 +#define ATOM_PPD_DEV_DSD 0x1000 +#define ATOM_PPD_TOPO_MASK (ATOM_PPD_CONN_MASK | ATOM_PPD_DEV_MASK) +#define ATOM_PPD_TOPO_PRI_USD (ATOM_PPD_CONN_TRANSPARENT | ATOM_PPD_DEV_USD) +#define ATOM_PPD_TOPO_PRI_DSD (ATOM_PPD_CONN_TRANSPARENT | ATOM_PPD_DEV_DSD) +#define ATOM_PPD_TOPO_SEC_USD (ATOM_PPD_CONN_RP | ATOM_PPD_DEV_USD) +#define ATOM_PPD_TOPO_SEC_DSD (ATOM_PPD_CONN_RP | ATOM_PPD_DEV_DSD) +#define ATOM_PPD_TOPO_B2B_USD (ATOM_PPD_CONN_B2B | ATOM_PPD_DEV_USD) +#define ATOM_PPD_TOPO_B2B_DSD (ATOM_PPD_CONN_B2B | ATOM_PPD_DEV_DSD) + +#define ATOM_MW_COUNT 2 +#define ATOM_DB_COUNT 34 +#define ATOM_DB_VALID_MASK (BIT_ULL(ATOM_DB_COUNT) - 1) +#define ATOM_DB_MSIX_VECTOR_COUNT 34 +#define ATOM_DB_MSIX_VECTOR_SHIFT 1 +#define ATOM_DB_TOTAL_SHIFT 34 +#define ATOM_SPAD_COUNT 16 + +#define ATOM_NTB_CTL_DOWN_BIT BIT(16) +#define ATOM_NTB_CTL_ACTIVE(x) !(x & ATOM_NTB_CTL_DOWN_BIT) + +#define ATOM_DESKEWSTS_DBERR BIT(15) +#define ATOM_LTSSMERRSTS0_UNEXPECTEDEI BIT(20) +#define ATOM_LTSSMSTATEJMP_FORCEDETECT BIT(2) +#define ATOM_IBIST_ERR_OFLOW 0x7FFF7FFF + +#define ATOM_LINK_HB_TIMEOUT msecs_to_jiffies(1000) +#define ATOM_LINK_RECOVERY_TIME msecs_to_jiffies(500) + +/* Ntb control and link status */ + +#define NTB_CTL_CFG_LOCK BIT(0) +#define NTB_CTL_DISABLE BIT(1) +#define NTB_CTL_S2P_BAR2_SNOOP BIT(2) +#define NTB_CTL_P2S_BAR2_SNOOP BIT(4) +#define NTB_CTL_S2P_BAR4_SNOOP BIT(6) +#define NTB_CTL_P2S_BAR4_SNOOP BIT(8) +#define NTB_CTL_S2P_BAR5_SNOOP BIT(12) +#define NTB_CTL_P2S_BAR5_SNOOP BIT(14) + +#define NTB_LNK_STA_ACTIVE_BIT 0x2000 +#define NTB_LNK_STA_SPEED_MASK 0x000f +#define NTB_LNK_STA_WIDTH_MASK 0x03f0 +#define NTB_LNK_STA_ACTIVE(x) (!!((x) & NTB_LNK_STA_ACTIVE_BIT)) +#define NTB_LNK_STA_SPEED(x) ((x) & NTB_LNK_STA_SPEED_MASK) +#define NTB_LNK_STA_WIDTH(x) (((x) & NTB_LNK_STA_WIDTH_MASK) >> 4) + +/* Use the following addresses for translation between b2b ntb devices in case + * the hardware default values are not reliable. */ +#define XEON_B2B_BAR0_USD_ADDR 0x1000000000000000ull +#define XEON_B2B_BAR2_USD_ADDR64 0x2000000000000000ull +#define XEON_B2B_BAR4_USD_ADDR64 0x4000000000000000ull +#define XEON_B2B_BAR4_USD_ADDR32 0x20000000u +#define XEON_B2B_BAR5_USD_ADDR32 0x40000000u +#define XEON_B2B_BAR0_DSD_ADDR 0x9000000000000000ull +#define XEON_B2B_BAR2_DSD_ADDR64 0xa000000000000000ull +#define XEON_B2B_BAR4_DSD_ADDR64 0xc000000000000000ull +#define XEON_B2B_BAR4_DSD_ADDR32 0xa0000000u +#define XEON_B2B_BAR5_DSD_ADDR32 0xc0000000u + +/* The peer ntb secondary config space is 32KB fixed size */ +#define XEON_B2B_MIN_SIZE 0x8000 + +/* flags to indicate hardware errata */ +#define NTB_HWERR_SDOORBELL_LOCKUP BIT_ULL(0) +#define NTB_HWERR_SB01BASE_LOCKUP BIT_ULL(1) +#define NTB_HWERR_B2BDOORBELL_BIT14 BIT_ULL(2) + +/* flags to indicate unsafe api */ +#define NTB_UNSAFE_DB BIT_ULL(0) +#define NTB_UNSAFE_SPAD BIT_ULL(1) + +struct intel_ntb_dev; + +struct intel_ntb_reg { + int (*poll_link)(struct intel_ntb_dev *ndev); + int (*link_is_up)(struct intel_ntb_dev *ndev); + u64 (*db_ioread)(void __iomem *mmio); + void (*db_iowrite)(u64 db_bits, void __iomem *mmio); + unsigned long ntb_ctl; + resource_size_t db_size; + int mw_bar[]; +}; + +struct intel_ntb_alt_reg { + unsigned long db_bell; + unsigned long db_mask; + unsigned long spad; +}; + +struct intel_ntb_xlat_reg { + unsigned long bar0_base; + unsigned long bar2_xlat; + unsigned long bar2_limit; +}; + +struct intel_b2b_addr { + phys_addr_t bar0_addr; + phys_addr_t bar2_addr64; + phys_addr_t bar4_addr64; + phys_addr_t bar4_addr32; + phys_addr_t bar5_addr32; +}; + +struct intel_ntb_vec { + struct intel_ntb_dev *ndev; + int num; +}; + +struct intel_ntb_dev { + struct ntb_dev ntb; + + /* offset of peer bar0 in b2b bar */ + unsigned long b2b_off; + /* mw idx used to access peer bar0 */ + unsigned int b2b_idx; + + /* BAR45 is split into BAR4 and BAR5 */ + bool bar4_split; + + u32 ntb_ctl; + u32 lnk_sta; + + unsigned char mw_count; + unsigned char spad_count; + unsigned char db_count; + unsigned char db_vec_count; + unsigned char db_vec_shift; + + u64 db_valid_mask; + u64 db_link_mask; + u64 db_mask; + + /* synchronize rmw access of db_mask and hw reg */ + spinlock_t db_mask_lock; + + struct msix_entry *msix; + struct intel_ntb_vec *vec; + + const struct intel_ntb_reg *reg; + const struct intel_ntb_alt_reg *self_reg; + const struct intel_ntb_alt_reg *peer_reg; + const struct intel_ntb_xlat_reg *xlat_reg; + void __iomem *self_mmio; + void __iomem *peer_mmio; + phys_addr_t peer_addr; + + unsigned long last_ts; + struct delayed_work hb_timer; + + unsigned long hwerr_flags; + unsigned long unsafe_flags; + unsigned long unsafe_flags_ignore; + + struct dentry *debugfs_dir; + struct dentry *debugfs_info; +}; + +#define ndev_pdev(ndev) ((ndev)->ntb.pdev) +#define ndev_name(ndev) pci_name(ndev_pdev(ndev)) +#define ndev_dev(ndev) (&ndev_pdev(ndev)->dev) +#define ntb_ndev(ntb) container_of(ntb, struct intel_ntb_dev, ntb) +#define hb_ndev(work) container_of(work, struct intel_ntb_dev, hb_timer.work) + +#endif diff --git a/drivers/ntb/ntb.c b/drivers/ntb/ntb.c new file mode 100644 index 000000000000..23435f2a5486 --- /dev/null +++ b/drivers/ntb/ntb.c @@ -0,0 +1,251 @@ +/* + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * GPL LICENSE SUMMARY + * + * Copyright (C) 2015 EMC Corporation. All Rights Reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of version 2 of the GNU General Public License as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * BSD LICENSE + * + * Copyright (C) 2015 EMC Corporation. All Rights Reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copy + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * PCIe NTB Linux driver + * + * Contact Information: + * Allen Hubbe <Allen.Hubbe@emc.com> + */ + +#include <linux/device.h> +#include <linux/kernel.h> +#include <linux/module.h> + +#include <linux/ntb.h> +#include <linux/pci.h> + +#define DRIVER_NAME "ntb" +#define DRIVER_DESCRIPTION "PCIe NTB Driver Framework" + +#define DRIVER_LICENSE "Dual BSD/GPL" +#define DRIVER_VERSION "1.0" +#define DRIVER_RELDATE "24 March 2015" +#define DRIVER_AUTHOR "Allen Hubbe <Allen.Hubbe@emc.com>" + +MODULE_LICENSE(DRIVER_LICENSE); +MODULE_VERSION(DRIVER_VERSION); +MODULE_AUTHOR(DRIVER_AUTHOR); +MODULE_DESCRIPTION(DRIVER_DESCRIPTION); + +static struct bus_type ntb_bus; +static void ntb_dev_release(struct device *dev); + +int __ntb_register_client(struct ntb_client *client, struct module *mod, + const char *mod_name) +{ + if (!client) + return -EINVAL; + if (!ntb_client_ops_is_valid(&client->ops)) + return -EINVAL; + + memset(&client->drv, 0, sizeof(client->drv)); + client->drv.bus = &ntb_bus; + client->drv.name = mod_name; + client->drv.owner = mod; + + return driver_register(&client->drv); +} +EXPORT_SYMBOL(__ntb_register_client); + +void ntb_unregister_client(struct ntb_client *client) +{ + driver_unregister(&client->drv); +} +EXPORT_SYMBOL(ntb_unregister_client); + +int ntb_register_device(struct ntb_dev *ntb) +{ + if (!ntb) + return -EINVAL; + if (!ntb->pdev) + return -EINVAL; + if (!ntb->ops) + return -EINVAL; + if (!ntb_dev_ops_is_valid(ntb->ops)) + return -EINVAL; + + init_completion(&ntb->released); + + memset(&ntb->dev, 0, sizeof(ntb->dev)); + ntb->dev.bus = &ntb_bus; + ntb->dev.parent = &ntb->pdev->dev; + ntb->dev.release = ntb_dev_release; + dev_set_name(&ntb->dev, pci_name(ntb->pdev)); + + ntb->ctx = NULL; + ntb->ctx_ops = NULL; + spin_lock_init(&ntb->ctx_lock); + + return device_register(&ntb->dev); +} +EXPORT_SYMBOL(ntb_register_device); + +void ntb_unregister_device(struct ntb_dev *ntb) +{ + device_unregister(&ntb->dev); + wait_for_completion(&ntb->released); +} +EXPORT_SYMBOL(ntb_unregister_device); + +int ntb_set_ctx(struct ntb_dev *ntb, void *ctx, + const struct ntb_ctx_ops *ctx_ops) +{ + unsigned long irqflags; + + if (!ntb_ctx_ops_is_valid(ctx_ops)) + return -EINVAL; + if (ntb->ctx_ops) + return -EINVAL; + + spin_lock_irqsave(&ntb->ctx_lock, irqflags); + { + ntb->ctx = ctx; + ntb->ctx_ops = ctx_ops; + } + spin_unlock_irqrestore(&ntb->ctx_lock, irqflags); + + return 0; +} +EXPORT_SYMBOL(ntb_set_ctx); + +void ntb_clear_ctx(struct ntb_dev *ntb) +{ + unsigned long irqflags; + + spin_lock_irqsave(&ntb->ctx_lock, irqflags); + { + ntb->ctx_ops = NULL; + ntb->ctx = NULL; + } + spin_unlock_irqrestore(&ntb->ctx_lock, irqflags); +} +EXPORT_SYMBOL(ntb_clear_ctx); + +void ntb_link_event(struct ntb_dev *ntb) +{ + unsigned long irqflags; + + spin_lock_irqsave(&ntb->ctx_lock, irqflags); + { + if (ntb->ctx_ops && ntb->ctx_ops->link_event) + ntb->ctx_ops->link_event(ntb->ctx); + } + spin_unlock_irqrestore(&ntb->ctx_lock, irqflags); +} +EXPORT_SYMBOL(ntb_link_event); + +void ntb_db_event(struct ntb_dev *ntb, int vector) +{ + unsigned long irqflags; + + spin_lock_irqsave(&ntb->ctx_lock, irqflags); + { + if (ntb->ctx_ops && ntb->ctx_ops->db_event) + ntb->ctx_ops->db_event(ntb->ctx, vector); + } + spin_unlock_irqrestore(&ntb->ctx_lock, irqflags); +} +EXPORT_SYMBOL(ntb_db_event); + +static int ntb_probe(struct device *dev) +{ + struct ntb_dev *ntb; + struct ntb_client *client; + int rc; + + get_device(dev); + ntb = dev_ntb(dev); + client = drv_ntb_client(dev->driver); + + rc = client->ops.probe(client, ntb); + if (rc) + put_device(dev); + + return rc; +} + +static int ntb_remove(struct device *dev) +{ + struct ntb_dev *ntb; + struct ntb_client *client; + + if (dev->driver) { + ntb = dev_ntb(dev); + client = drv_ntb_client(dev->driver); + + client->ops.remove(client, ntb); + put_device(dev); + } + + return 0; +} + +static void ntb_dev_release(struct device *dev) +{ + struct ntb_dev *ntb = dev_ntb(dev); + + complete(&ntb->released); +} + +static struct bus_type ntb_bus = { + .name = "ntb", + .probe = ntb_probe, + .remove = ntb_remove, +}; + +static int __init ntb_driver_init(void) +{ + return bus_register(&ntb_bus); +} +module_init(ntb_driver_init); + +static void __exit ntb_driver_exit(void) +{ + bus_unregister(&ntb_bus); +} +module_exit(ntb_driver_exit); + diff --git a/drivers/ntb/ntb_hw.c b/drivers/ntb/ntb_hw.c deleted file mode 100644 index 3f6738612f45..000000000000 --- a/drivers/ntb/ntb_hw.c +++ /dev/null @@ -1,1895 +0,0 @@ -/* - * This file is provided under a dual BSD/GPLv2 license. When using or - * redistributing this file, you may do so under either license. - * - * GPL LICENSE SUMMARY - * - * Copyright(c) 2012 Intel Corporation. All rights reserved. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of version 2 of the GNU General Public License as - * published by the Free Software Foundation. - * - * BSD LICENSE - * - * Copyright(c) 2012 Intel Corporation. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copy - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * Intel PCIe NTB Linux driver - * - * Contact Information: - * Jon Mason <jon.mason@intel.com> - */ -#include <linux/debugfs.h> -#include <linux/delay.h> -#include <linux/init.h> -#include <linux/interrupt.h> -#include <linux/module.h> -#include <linux/pci.h> -#include <linux/random.h> -#include <linux/slab.h> -#include "ntb_hw.h" -#include "ntb_regs.h" - -#define NTB_NAME "Intel(R) PCI-E Non-Transparent Bridge Driver" -#define NTB_VER "1.0" - -MODULE_DESCRIPTION(NTB_NAME); -MODULE_VERSION(NTB_VER); -MODULE_LICENSE("Dual BSD/GPL"); -MODULE_AUTHOR("Intel Corporation"); - -enum { - NTB_CONN_TRANSPARENT = 0, - NTB_CONN_B2B, - NTB_CONN_RP, -}; - -enum { - NTB_DEV_USD = 0, - NTB_DEV_DSD, -}; - -enum { - SNB_HW = 0, - BWD_HW, -}; - -static struct dentry *debugfs_dir; - -#define BWD_LINK_RECOVERY_TIME 500 - -/* Translate memory window 0,1,2 to BAR 2,4,5 */ -#define MW_TO_BAR(mw) (mw == 0 ? 2 : (mw == 1 ? 4 : 5)) - -static const struct pci_device_id ntb_pci_tbl[] = { - {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_BWD)}, - {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_JSF)}, - {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_SNB)}, - {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_IVT)}, - {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_HSX)}, - {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_JSF)}, - {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_SNB)}, - {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_IVT)}, - {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_HSX)}, - {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_JSF)}, - {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_SNB)}, - {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_IVT)}, - {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_HSX)}, - {0} -}; -MODULE_DEVICE_TABLE(pci, ntb_pci_tbl); - -static int is_ntb_xeon(struct ntb_device *ndev) -{ - switch (ndev->pdev->device) { - case PCI_DEVICE_ID_INTEL_NTB_SS_JSF: - case PCI_DEVICE_ID_INTEL_NTB_SS_SNB: - case PCI_DEVICE_ID_INTEL_NTB_SS_IVT: - case PCI_DEVICE_ID_INTEL_NTB_SS_HSX: - case PCI_DEVICE_ID_INTEL_NTB_PS_JSF: - case PCI_DEVICE_ID_INTEL_NTB_PS_SNB: - case PCI_DEVICE_ID_INTEL_NTB_PS_IVT: - case PCI_DEVICE_ID_INTEL_NTB_PS_HSX: - case PCI_DEVICE_ID_INTEL_NTB_B2B_JSF: - case PCI_DEVICE_ID_INTEL_NTB_B2B_SNB: - case PCI_DEVICE_ID_INTEL_NTB_B2B_IVT: - case PCI_DEVICE_ID_INTEL_NTB_B2B_HSX: - return 1; - default: - return 0; - } - - return 0; -} - -static int is_ntb_atom(struct ntb_device *ndev) -{ - switch (ndev->pdev->device) { - case PCI_DEVICE_ID_INTEL_NTB_B2B_BWD: - return 1; - default: - return 0; - } - - return 0; -} - -static void ntb_set_errata_flags(struct ntb_device *ndev) -{ - switch (ndev->pdev->device) { - /* - * this workaround applies to all platform up to IvyBridge - * Haswell has splitbar support and use a different workaround - */ - case PCI_DEVICE_ID_INTEL_NTB_SS_JSF: - case PCI_DEVICE_ID_INTEL_NTB_SS_SNB: - case PCI_DEVICE_ID_INTEL_NTB_SS_IVT: - case PCI_DEVICE_ID_INTEL_NTB_SS_HSX: - case PCI_DEVICE_ID_INTEL_NTB_PS_JSF: - case PCI_DEVICE_ID_INTEL_NTB_PS_SNB: - case PCI_DEVICE_ID_INTEL_NTB_PS_IVT: - case PCI_DEVICE_ID_INTEL_NTB_PS_HSX: - case PCI_DEVICE_ID_INTEL_NTB_B2B_JSF: - case PCI_DEVICE_ID_INTEL_NTB_B2B_SNB: - case PCI_DEVICE_ID_INTEL_NTB_B2B_IVT: - case PCI_DEVICE_ID_INTEL_NTB_B2B_HSX: - ndev->wa_flags |= WA_SNB_ERR; - break; - } -} - -/** - * ntb_register_event_callback() - register event callback - * @ndev: pointer to ntb_device instance - * @func: callback function to register - * - * This function registers a callback for any HW driver events such as link - * up/down, power management notices and etc. - * - * RETURNS: An appropriate -ERRNO error value on error, or zero for success. - */ -int ntb_register_event_callback(struct ntb_device *ndev, - void (*func)(void *handle, - enum ntb_hw_event event)) -{ - if (ndev->event_cb) - return -EINVAL; - - ndev->event_cb = func; - - return 0; -} - -/** - * ntb_unregister_event_callback() - unregisters the event callback - * @ndev: pointer to ntb_device instance - * - * This function unregisters the existing callback from transport - */ -void ntb_unregister_event_callback(struct ntb_device *ndev) -{ - ndev->event_cb = NULL; -} - -static void ntb_irq_work(unsigned long data) -{ - struct ntb_db_cb *db_cb = (struct ntb_db_cb *)data; - int rc; - - rc = db_cb->callback(db_cb->data, db_cb->db_num); - if (rc) - tasklet_schedule(&db_cb->irq_work); - else { - struct ntb_device *ndev = db_cb->ndev; - unsigned long mask; - - mask = readw(ndev->reg_ofs.ldb_mask); - clear_bit(db_cb->db_num * ndev->bits_per_vector, &mask); - writew(mask, ndev->reg_ofs.ldb_mask); - } -} - -/** - * ntb_register_db_callback() - register a callback for doorbell interrupt - * @ndev: pointer to ntb_device instance - * @idx: doorbell index to register callback, zero based - * @data: pointer to be returned to caller with every callback - * @func: callback function to register - * - * This function registers a callback function for the doorbell interrupt - * on the primary side. The function will unmask the doorbell as well to - * allow interrupt. - * - * RETURNS: An appropriate -ERRNO error value on error, or zero for success. - */ -int ntb_register_db_callback(struct ntb_device *ndev, unsigned int idx, - void *data, int (*func)(void *data, int db_num)) -{ - unsigned long mask; - - if (idx >= ndev->max_cbs || ndev->db_cb[idx].callback) { - dev_warn(&ndev->pdev->dev, "Invalid Index.\n"); - return -EINVAL; - } - - ndev->db_cb[idx].callback = func; - ndev->db_cb[idx].data = data; - ndev->db_cb[idx].ndev = ndev; - - tasklet_init(&ndev->db_cb[idx].irq_work, ntb_irq_work, - (unsigned long) &ndev->db_cb[idx]); - - /* unmask interrupt */ - mask = readw(ndev->reg_ofs.ldb_mask); - clear_bit(idx * ndev->bits_per_vector, &mask); - writew(mask, ndev->reg_ofs.ldb_mask); - - return 0; -} - -/** - * ntb_unregister_db_callback() - unregister a callback for doorbell interrupt - * @ndev: pointer to ntb_device instance - * @idx: doorbell index to register callback, zero based - * - * This function unregisters a callback function for the doorbell interrupt - * on the primary side. The function will also mask the said doorbell. - */ -void ntb_unregister_db_callback(struct ntb_device *ndev, unsigned int idx) -{ - unsigned long mask; - - if (idx >= ndev->max_cbs || !ndev->db_cb[idx].callback) - return; - - mask = readw(ndev->reg_ofs.ldb_mask); - set_bit(idx * ndev->bits_per_vector, &mask); - writew(mask, ndev->reg_ofs.ldb_mask); - - tasklet_disable(&ndev->db_cb[idx].irq_work); - - ndev->db_cb[idx].callback = NULL; -} - -/** - * ntb_find_transport() - find the transport pointer - * @transport: pointer to pci device - * - * Given the pci device pointer, return the transport pointer passed in when - * the transport attached when it was inited. - * - * RETURNS: pointer to transport. - */ -void *ntb_find_transport(struct pci_dev *pdev) -{ - struct ntb_device *ndev = pci_get_drvdata(pdev); - return ndev->ntb_transport; -} - -/** - * ntb_register_transport() - Register NTB transport with NTB HW driver - * @transport: transport identifier - * - * This function allows a transport to reserve the hardware driver for - * NTB usage. - * - * RETURNS: pointer to ntb_device, NULL on error. - */ -struct ntb_device *ntb_register_transport(struct pci_dev *pdev, void *transport) -{ - struct ntb_device *ndev = pci_get_drvdata(pdev); - - if (ndev->ntb_transport) - return NULL; - - ndev->ntb_transport = transport; - return ndev; -} - -/** - * ntb_unregister_transport() - Unregister the transport with the NTB HW driver - * @ndev - ntb_device of the transport to be freed - * - * This function unregisters the transport from the HW driver and performs any - * necessary cleanups. - */ -void ntb_unregister_transport(struct ntb_device *ndev) -{ - int i; - - if (!ndev->ntb_transport) - return; - - for (i = 0; i < ndev->max_cbs; i++) - ntb_unregister_db_callback(ndev, i); - - ntb_unregister_event_callback(ndev); - ndev->ntb_transport = NULL; -} - -/** - * ntb_write_local_spad() - write to the secondary scratchpad register - * @ndev: pointer to ntb_device instance - * @idx: index to the scratchpad register, 0 based - * @val: the data value to put into the register - * - * This function allows writing of a 32bit value to the indexed scratchpad - * register. This writes over the data mirrored to the local scratchpad register - * by the remote system. - * - * RETURNS: An appropriate -ERRNO error value on error, or zero for success. - */ -int ntb_write_local_spad(struct ntb_device *ndev, unsigned int idx, u32 val) -{ - if (idx >= ndev->limits.max_spads) - return -EINVAL; - - dev_dbg(&ndev->pdev->dev, "Writing %x to local scratch pad index %d\n", - val, idx); - writel(val, ndev->reg_ofs.spad_read + idx * 4); - - return 0; -} - -/** - * ntb_read_local_spad() - read from the primary scratchpad register - * @ndev: pointer to ntb_device instance - * @idx: index to scratchpad register, 0 based - * @val: pointer to 32bit integer for storing the register value - * - * This function allows reading of the 32bit scratchpad register on - * the primary (internal) side. This allows the local system to read data - * written and mirrored to the scratchpad register by the remote system. - * - * RETURNS: An appropriate -ERRNO error value on error, or zero for success. - */ -int ntb_read_local_spad(struct ntb_device *ndev, unsigned int idx, u32 *val) -{ - if (idx >= ndev->limits.max_spads) - return -EINVAL; - - *val = readl(ndev->reg_ofs.spad_write + idx * 4); - dev_dbg(&ndev->pdev->dev, - "Reading %x from local scratch pad index %d\n", *val, idx); - - return 0; -} - -/** - * ntb_write_remote_spad() - write to the secondary scratchpad register - * @ndev: pointer to ntb_device instance - * @idx: index to the scratchpad register, 0 based - * @val: the data value to put into the register - * - * This function allows writing of a 32bit value to the indexed scratchpad - * register. The register resides on the secondary (external) side. This allows - * the local system to write data to be mirrored to the remote systems - * scratchpad register. - * - * RETURNS: An appropriate -ERRNO error value on error, or zero for success. - */ -int ntb_write_remote_spad(struct ntb_device *ndev, unsigned int idx, u32 val) -{ - if (idx >= ndev->limits.max_spads) - return -EINVAL; - - dev_dbg(&ndev->pdev->dev, "Writing %x to remote scratch pad index %d\n", - val, idx); - writel(val, ndev->reg_ofs.spad_write + idx * 4); - - return 0; -} - -/** - * ntb_read_remote_spad() - read from the primary scratchpad register - * @ndev: pointer to ntb_device instance - * @idx: index to scratchpad register, 0 based - * @val: pointer to 32bit integer for storing the register value - * - * This function allows reading of the 32bit scratchpad register on - * the primary (internal) side. This alloows the local system to read the data - * it wrote to be mirrored on the remote system. - * - * RETURNS: An appropriate -ERRNO error value on error, or zero for success. - */ -int ntb_read_remote_spad(struct ntb_device *ndev, unsigned int idx, u32 *val) -{ - if (idx >= ndev->limits.max_spads) - return -EINVAL; - - *val = readl(ndev->reg_ofs.spad_read + idx * 4); - dev_dbg(&ndev->pdev->dev, - "Reading %x from remote scratch pad index %d\n", *val, idx); - - return 0; -} - -/** - * ntb_get_mw_base() - get addr for the NTB memory window - * @ndev: pointer to ntb_device instance - * @mw: memory window number - * - * This function provides the base address of the memory window specified. - * - * RETURNS: address, or NULL on error. - */ -resource_size_t ntb_get_mw_base(struct ntb_device *ndev, unsigned int mw) -{ - if (mw >= ntb_max_mw(ndev)) - return 0; - - return pci_resource_start(ndev->pdev, MW_TO_BAR(mw)); -} - -/** - * ntb_get_mw_vbase() - get virtual addr for the NTB memory window - * @ndev: pointer to ntb_device instance - * @mw: memory window number - * - * This function provides the base virtual address of the memory window - * specified. - * - * RETURNS: pointer to virtual address, or NULL on error. - */ -void __iomem *ntb_get_mw_vbase(struct ntb_device *ndev, unsigned int mw) -{ - if (mw >= ntb_max_mw(ndev)) - return NULL; - - return ndev->mw[mw].vbase; -} - -/** - * ntb_get_mw_size() - return size of NTB memory window - * @ndev: pointer to ntb_device instance - * @mw: memory window number - * - * This function provides the physical size of the memory window specified - * - * RETURNS: the size of the memory window or zero on error - */ -u64 ntb_get_mw_size(struct ntb_device *ndev, unsigned int mw) -{ - if (mw >= ntb_max_mw(ndev)) - return 0; - - return ndev->mw[mw].bar_sz; -} - -/** - * ntb_set_mw_addr - set the memory window address - * @ndev: pointer to ntb_device instance - * @mw: memory window number - * @addr: base address for data - * - * This function sets the base physical address of the memory window. This - * memory address is where data from the remote system will be transfered into - * or out of depending on how the transport is configured. - */ -void ntb_set_mw_addr(struct ntb_device *ndev, unsigned int mw, u64 addr) -{ - if (mw >= ntb_max_mw(ndev)) - return; - - dev_dbg(&ndev->pdev->dev, "Writing addr %Lx to BAR %d\n", addr, - MW_TO_BAR(mw)); - - ndev->mw[mw].phys_addr = addr; - - switch (MW_TO_BAR(mw)) { - case NTB_BAR_23: - writeq(addr, ndev->reg_ofs.bar2_xlat); - break; - case NTB_BAR_4: - if (ndev->split_bar) - writel(addr, ndev->reg_ofs.bar4_xlat); - else - writeq(addr, ndev->reg_ofs.bar4_xlat); - break; - case NTB_BAR_5: - writel(addr, ndev->reg_ofs.bar5_xlat); - break; - } -} - -/** - * ntb_ring_doorbell() - Set the doorbell on the secondary/external side - * @ndev: pointer to ntb_device instance - * @db: doorbell to ring - * - * This function allows triggering of a doorbell on the secondary/external - * side that will initiate an interrupt on the remote host - * - * RETURNS: An appropriate -ERRNO error value on error, or zero for success. - */ -void ntb_ring_doorbell(struct ntb_device *ndev, unsigned int db) -{ - dev_dbg(&ndev->pdev->dev, "%s: ringing doorbell %d\n", __func__, db); - - if (ndev->hw_type == BWD_HW) - writeq((u64) 1 << db, ndev->reg_ofs.rdb); - else - writew(((1 << ndev->bits_per_vector) - 1) << - (db * ndev->bits_per_vector), ndev->reg_ofs.rdb); -} - -static void bwd_recover_link(struct ntb_device *ndev) -{ - u32 status; - - /* Driver resets the NTB ModPhy lanes - magic! */ - writeb(0xe0, ndev->reg_base + BWD_MODPHY_PCSREG6); - writeb(0x40, ndev->reg_base + BWD_MODPHY_PCSREG4); - writeb(0x60, ndev->reg_base + BWD_MODPHY_PCSREG4); - writeb(0x60, ndev->reg_base + BWD_MODPHY_PCSREG6); - - /* Driver waits 100ms to allow the NTB ModPhy to settle */ - msleep(100); - - /* Clear AER Errors, write to clear */ - status = readl(ndev->reg_base + BWD_ERRCORSTS_OFFSET); - dev_dbg(&ndev->pdev->dev, "ERRCORSTS = %x\n", status); - status &= PCI_ERR_COR_REP_ROLL; - writel(status, ndev->reg_base + BWD_ERRCORSTS_OFFSET); - - /* Clear unexpected electrical idle event in LTSSM, write to clear */ - status = readl(ndev->reg_base + BWD_LTSSMERRSTS0_OFFSET); - dev_dbg(&ndev->pdev->dev, "LTSSMERRSTS0 = %x\n", status); - status |= BWD_LTSSMERRSTS0_UNEXPECTEDEI; - writel(status, ndev->reg_base + BWD_LTSSMERRSTS0_OFFSET); - - /* Clear DeSkew Buffer error, write to clear */ - status = readl(ndev->reg_base + BWD_DESKEWSTS_OFFSET); - dev_dbg(&ndev->pdev->dev, "DESKEWSTS = %x\n", status); - status |= BWD_DESKEWSTS_DBERR; - writel(status, ndev->reg_base + BWD_DESKEWSTS_OFFSET); - - status = readl(ndev->reg_base + BWD_IBSTERRRCRVSTS0_OFFSET); - dev_dbg(&ndev->pdev->dev, "IBSTERRRCRVSTS0 = %x\n", status); - status &= BWD_IBIST_ERR_OFLOW; - writel(status, ndev->reg_base + BWD_IBSTERRRCRVSTS0_OFFSET); - - /* Releases the NTB state machine to allow the link to retrain */ - status = readl(ndev->reg_base + BWD_LTSSMSTATEJMP_OFFSET); - dev_dbg(&ndev->pdev->dev, "LTSSMSTATEJMP = %x\n", status); - status &= ~BWD_LTSSMSTATEJMP_FORCEDETECT; - writel(status, ndev->reg_base + BWD_LTSSMSTATEJMP_OFFSET); -} - -static void ntb_link_event(struct ntb_device *ndev, int link_state) -{ - unsigned int event; - - if (ndev->link_status == link_state) - return; - - if (link_state == NTB_LINK_UP) { - u16 status; - - dev_info(&ndev->pdev->dev, "Link Up\n"); - ndev->link_status = NTB_LINK_UP; - event = NTB_EVENT_HW_LINK_UP; - - if (is_ntb_atom(ndev) || - ndev->conn_type == NTB_CONN_TRANSPARENT) - status = readw(ndev->reg_ofs.lnk_stat); - else { - int rc = pci_read_config_word(ndev->pdev, - SNB_LINK_STATUS_OFFSET, - &status); - if (rc) - return; - } - - ndev->link_width = (status & NTB_LINK_WIDTH_MASK) >> 4; - ndev->link_speed = (status & NTB_LINK_SPEED_MASK); - dev_info(&ndev->pdev->dev, "Link Width %d, Link Speed %d\n", - ndev->link_width, ndev->link_speed); - } else { - dev_info(&ndev->pdev->dev, "Link Down\n"); - ndev->link_status = NTB_LINK_DOWN; - event = NTB_EVENT_HW_LINK_DOWN; - /* Don't modify link width/speed, we need it in link recovery */ - } - - /* notify the upper layer if we have an event change */ - if (ndev->event_cb) - ndev->event_cb(ndev->ntb_transport, event); -} - -static int ntb_link_status(struct ntb_device *ndev) -{ - int link_state; - - if (is_ntb_atom(ndev)) { - u32 ntb_cntl; - - ntb_cntl = readl(ndev->reg_ofs.lnk_cntl); - if (ntb_cntl & BWD_CNTL_LINK_DOWN) - link_state = NTB_LINK_DOWN; - else - link_state = NTB_LINK_UP; - } else { - u16 status; - int rc; - - rc = pci_read_config_word(ndev->pdev, SNB_LINK_STATUS_OFFSET, - &status); - if (rc) - return rc; - - if (status & NTB_LINK_STATUS_ACTIVE) - link_state = NTB_LINK_UP; - else - link_state = NTB_LINK_DOWN; - } - - ntb_link_event(ndev, link_state); - - return 0; -} - -static void bwd_link_recovery(struct work_struct *work) -{ - struct ntb_device *ndev = container_of(work, struct ntb_device, - lr_timer.work); - u32 status32; - - bwd_recover_link(ndev); - /* There is a potential race between the 2 NTB devices recovering at the - * same time. If the times are the same, the link will not recover and - * the driver will be stuck in this loop forever. Add a random interval - * to the recovery time to prevent this race. - */ - msleep(BWD_LINK_RECOVERY_TIME + prandom_u32() % BWD_LINK_RECOVERY_TIME); - - status32 = readl(ndev->reg_base + BWD_LTSSMSTATEJMP_OFFSET); - if (status32 & BWD_LTSSMSTATEJMP_FORCEDETECT) - goto retry; - - status32 = readl(ndev->reg_base + BWD_IBSTERRRCRVSTS0_OFFSET); - if (status32 & BWD_IBIST_ERR_OFLOW) - goto retry; - - status32 = readl(ndev->reg_ofs.lnk_cntl); - if (!(status32 & BWD_CNTL_LINK_DOWN)) { - unsigned char speed, width; - u16 status16; - - status16 = readw(ndev->reg_ofs.lnk_stat); - width = (status16 & NTB_LINK_WIDTH_MASK) >> 4; - speed = (status16 & NTB_LINK_SPEED_MASK); - if (ndev->link_width != width || ndev->link_speed != speed) - goto retry; - } - - schedule_delayed_work(&ndev->hb_timer, NTB_HB_TIMEOUT); - return; - -retry: - schedule_delayed_work(&ndev->lr_timer, NTB_HB_TIMEOUT); -} - -/* BWD doesn't have link status interrupt, poll on that platform */ -static void bwd_link_poll(struct work_struct *work) -{ - struct ntb_device *ndev = container_of(work, struct ntb_device, - hb_timer.work); - unsigned long ts = jiffies; - - /* If we haven't gotten an interrupt in a while, check the BWD link - * status bit - */ - if (ts > ndev->last_ts + NTB_HB_TIMEOUT) { - int rc = ntb_link_status(ndev); - if (rc) - dev_err(&ndev->pdev->dev, - "Error determining link status\n"); - - /* Check to see if a link error is the cause of the link down */ - if (ndev->link_status == NTB_LINK_DOWN) { - u32 status32 = readl(ndev->reg_base + - BWD_LTSSMSTATEJMP_OFFSET); - if (status32 & BWD_LTSSMSTATEJMP_FORCEDETECT) { - schedule_delayed_work(&ndev->lr_timer, 0); - return; - } - } - } - - schedule_delayed_work(&ndev->hb_timer, NTB_HB_TIMEOUT); -} - -static int ntb_xeon_setup(struct ntb_device *ndev) -{ - switch (ndev->conn_type) { - case NTB_CONN_B2B: - ndev->reg_ofs.ldb = ndev->reg_base + SNB_PDOORBELL_OFFSET; - ndev->reg_ofs.ldb_mask = ndev->reg_base + SNB_PDBMSK_OFFSET; - ndev->reg_ofs.spad_read = ndev->reg_base + SNB_SPAD_OFFSET; - ndev->reg_ofs.bar2_xlat = ndev->reg_base + SNB_SBAR2XLAT_OFFSET; - ndev->reg_ofs.bar4_xlat = ndev->reg_base + SNB_SBAR4XLAT_OFFSET; - if (ndev->split_bar) - ndev->reg_ofs.bar5_xlat = - ndev->reg_base + SNB_SBAR5XLAT_OFFSET; - ndev->limits.max_spads = SNB_MAX_B2B_SPADS; - - /* There is a Xeon hardware errata related to writes to - * SDOORBELL or B2BDOORBELL in conjunction with inbound access - * to NTB MMIO Space, which may hang the system. To workaround - * this use the second memory window to access the interrupt and - * scratch pad registers on the remote system. - */ - if (ndev->wa_flags & WA_SNB_ERR) { - if (!ndev->mw[ndev->limits.max_mw - 1].bar_sz) - return -EINVAL; - - ndev->limits.max_db_bits = SNB_MAX_DB_BITS; - ndev->reg_ofs.spad_write = - ndev->mw[ndev->limits.max_mw - 1].vbase + - SNB_SPAD_OFFSET; - ndev->reg_ofs.rdb = - ndev->mw[ndev->limits.max_mw - 1].vbase + - SNB_PDOORBELL_OFFSET; - - /* Set the Limit register to 4k, the minimum size, to - * prevent an illegal access - */ - writeq(ndev->mw[1].bar_sz + 0x1000, ndev->reg_base + - SNB_PBAR4LMT_OFFSET); - /* HW errata on the Limit registers. They can only be - * written when the base register is 4GB aligned and - * < 32bit. This should already be the case based on - * the driver defaults, but write the Limit registers - * first just in case. - */ - - ndev->limits.max_mw = SNB_ERRATA_MAX_MW; - } else { - /* HW Errata on bit 14 of b2bdoorbell register. Writes - * will not be mirrored to the remote system. Shrink - * the number of bits by one, since bit 14 is the last - * bit. - */ - ndev->limits.max_db_bits = SNB_MAX_DB_BITS - 1; - ndev->reg_ofs.spad_write = ndev->reg_base + - SNB_B2B_SPAD_OFFSET; - ndev->reg_ofs.rdb = ndev->reg_base + - SNB_B2B_DOORBELL_OFFSET; - - /* Disable the Limit register, just incase it is set to - * something silly. A 64bit write should handle it - * regardless of whether it has a split BAR or not. - */ - writeq(0, ndev->reg_base + SNB_PBAR4LMT_OFFSET); - /* HW errata on the Limit registers. They can only be - * written when the base register is 4GB aligned and - * < 32bit. This should already be the case based on - * the driver defaults, but write the Limit registers - * first just in case. - */ - if (ndev->split_bar) - ndev->limits.max_mw = HSX_SPLITBAR_MAX_MW; - else - ndev->limits.max_mw = SNB_MAX_MW; - } - - /* The Xeon errata workaround requires setting SBAR Base - * addresses to known values, so that the PBAR XLAT can be - * pointed at SBAR0 of the remote system. - */ - if (ndev->dev_type == NTB_DEV_USD) { - writeq(SNB_MBAR23_DSD_ADDR, ndev->reg_base + - SNB_PBAR2XLAT_OFFSET); - if (ndev->wa_flags & WA_SNB_ERR) - writeq(SNB_MBAR01_DSD_ADDR, ndev->reg_base + - SNB_PBAR4XLAT_OFFSET); - else { - if (ndev->split_bar) { - writel(SNB_MBAR4_DSD_ADDR, - ndev->reg_base + - SNB_PBAR4XLAT_OFFSET); - writel(SNB_MBAR5_DSD_ADDR, - ndev->reg_base + - SNB_PBAR5XLAT_OFFSET); - } else - writeq(SNB_MBAR4_DSD_ADDR, - ndev->reg_base + - SNB_PBAR4XLAT_OFFSET); - - /* B2B_XLAT_OFFSET is a 64bit register, but can - * only take 32bit writes - */ - writel(SNB_MBAR01_DSD_ADDR & 0xffffffff, - ndev->reg_base + SNB_B2B_XLAT_OFFSETL); - writel(SNB_MBAR01_DSD_ADDR >> 32, - ndev->reg_base + SNB_B2B_XLAT_OFFSETU); - } - - writeq(SNB_MBAR01_USD_ADDR, ndev->reg_base + - SNB_SBAR0BASE_OFFSET); - writeq(SNB_MBAR23_USD_ADDR, ndev->reg_base + - SNB_SBAR2BASE_OFFSET); - if (ndev->split_bar) { - writel(SNB_MBAR4_USD_ADDR, ndev->reg_base + - SNB_SBAR4BASE_OFFSET); - writel(SNB_MBAR5_USD_ADDR, ndev->reg_base + - SNB_SBAR5BASE_OFFSET); - } else - writeq(SNB_MBAR4_USD_ADDR, ndev->reg_base + - SNB_SBAR4BASE_OFFSET); - } else { - writeq(SNB_MBAR23_USD_ADDR, ndev->reg_base + - SNB_PBAR2XLAT_OFFSET); - if (ndev->wa_flags & WA_SNB_ERR) - writeq(SNB_MBAR01_USD_ADDR, ndev->reg_base + - SNB_PBAR4XLAT_OFFSET); - else { - if (ndev->split_bar) { - writel(SNB_MBAR4_USD_ADDR, - ndev->reg_base + - SNB_PBAR4XLAT_OFFSET); - writel(SNB_MBAR5_USD_ADDR, - ndev->reg_base + - SNB_PBAR5XLAT_OFFSET); - } else - writeq(SNB_MBAR4_USD_ADDR, - ndev->reg_base + - SNB_PBAR4XLAT_OFFSET); - - /* - * B2B_XLAT_OFFSET is a 64bit register, but can - * only take 32bit writes - */ - writel(SNB_MBAR01_USD_ADDR & 0xffffffff, - ndev->reg_base + SNB_B2B_XLAT_OFFSETL); - writel(SNB_MBAR01_USD_ADDR >> 32, - ndev->reg_base + SNB_B2B_XLAT_OFFSETU); - } - writeq(SNB_MBAR01_DSD_ADDR, ndev->reg_base + - SNB_SBAR0BASE_OFFSET); - writeq(SNB_MBAR23_DSD_ADDR, ndev->reg_base + - SNB_SBAR2BASE_OFFSET); - if (ndev->split_bar) { - writel(SNB_MBAR4_DSD_ADDR, ndev->reg_base + - SNB_SBAR4BASE_OFFSET); - writel(SNB_MBAR5_DSD_ADDR, ndev->reg_base + - SNB_SBAR5BASE_OFFSET); - } else - writeq(SNB_MBAR4_DSD_ADDR, ndev->reg_base + - SNB_SBAR4BASE_OFFSET); - - } - break; - case NTB_CONN_RP: - if (ndev->wa_flags & WA_SNB_ERR) { - dev_err(&ndev->pdev->dev, - "NTB-RP disabled due to hardware errata.\n"); - return -EINVAL; - } - - /* Scratch pads need to have exclusive access from the primary - * or secondary side. Halve the num spads so that each side can - * have an equal amount. - */ - ndev->limits.max_spads = SNB_MAX_COMPAT_SPADS / 2; - ndev->limits.max_db_bits = SNB_MAX_DB_BITS; - /* Note: The SDOORBELL is the cause of the errata. You REALLY - * don't want to touch it. - */ - ndev->reg_ofs.rdb = ndev->reg_base + SNB_SDOORBELL_OFFSET; - ndev->reg_ofs.ldb = ndev->reg_base + SNB_PDOORBELL_OFFSET; - ndev->reg_ofs.ldb_mask = ndev->reg_base + SNB_PDBMSK_OFFSET; - /* Offset the start of the spads to correspond to whether it is - * primary or secondary - */ - ndev->reg_ofs.spad_write = ndev->reg_base + SNB_SPAD_OFFSET + - ndev->limits.max_spads * 4; - ndev->reg_ofs.spad_read = ndev->reg_base + SNB_SPAD_OFFSET; - ndev->reg_ofs.bar2_xlat = ndev->reg_base + SNB_SBAR2XLAT_OFFSET; - ndev->reg_ofs.bar4_xlat = ndev->reg_base + SNB_SBAR4XLAT_OFFSET; - if (ndev->split_bar) { - ndev->reg_ofs.bar5_xlat = - ndev->reg_base + SNB_SBAR5XLAT_OFFSET; - ndev->limits.max_mw = HSX_SPLITBAR_MAX_MW; - } else - ndev->limits.max_mw = SNB_MAX_MW; - break; - case NTB_CONN_TRANSPARENT: - if (ndev->wa_flags & WA_SNB_ERR) { - dev_err(&ndev->pdev->dev, - "NTB-TRANSPARENT disabled due to hardware errata.\n"); - return -EINVAL; - } - - /* Scratch pads need to have exclusive access from the primary - * or secondary side. Halve the num spads so that each side can - * have an equal amount. - */ - ndev->limits.max_spads = SNB_MAX_COMPAT_SPADS / 2; - ndev->limits.max_db_bits = SNB_MAX_DB_BITS; - ndev->reg_ofs.rdb = ndev->reg_base + SNB_PDOORBELL_OFFSET; - ndev->reg_ofs.ldb = ndev->reg_base + SNB_SDOORBELL_OFFSET; - ndev->reg_ofs.ldb_mask = ndev->reg_base + SNB_SDBMSK_OFFSET; - ndev->reg_ofs.spad_write = ndev->reg_base + SNB_SPAD_OFFSET; - /* Offset the start of the spads to correspond to whether it is - * primary or secondary - */ - ndev->reg_ofs.spad_read = ndev->reg_base + SNB_SPAD_OFFSET + - ndev->limits.max_spads * 4; - ndev->reg_ofs.bar2_xlat = ndev->reg_base + SNB_PBAR2XLAT_OFFSET; - ndev->reg_ofs.bar4_xlat = ndev->reg_base + SNB_PBAR4XLAT_OFFSET; - - if (ndev->split_bar) { - ndev->reg_ofs.bar5_xlat = - ndev->reg_base + SNB_PBAR5XLAT_OFFSET; - ndev->limits.max_mw = HSX_SPLITBAR_MAX_MW; - } else - ndev->limits.max_mw = SNB_MAX_MW; - break; - default: - /* - * we should never hit this. the detect function should've - * take cared of everything. - */ - return -EINVAL; - } - - ndev->reg_ofs.lnk_cntl = ndev->reg_base + SNB_NTBCNTL_OFFSET; - ndev->reg_ofs.lnk_stat = ndev->reg_base + SNB_SLINK_STATUS_OFFSET; - ndev->reg_ofs.spci_cmd = ndev->reg_base + SNB_PCICMD_OFFSET; - - ndev->limits.msix_cnt = SNB_MSIX_CNT; - ndev->bits_per_vector = SNB_DB_BITS_PER_VEC; - - return 0; -} - -static int ntb_bwd_setup(struct ntb_device *ndev) -{ - int rc; - u32 val; - - ndev->hw_type = BWD_HW; - - rc = pci_read_config_dword(ndev->pdev, NTB_PPD_OFFSET, &val); - if (rc) - return rc; - - switch ((val & BWD_PPD_CONN_TYPE) >> 8) { - case NTB_CONN_B2B: - ndev->conn_type = NTB_CONN_B2B; - break; - case NTB_CONN_RP: - default: - dev_err(&ndev->pdev->dev, "Unsupported NTB configuration\n"); - return -EINVAL; - } - - if (val & BWD_PPD_DEV_TYPE) - ndev->dev_type = NTB_DEV_DSD; - else - ndev->dev_type = NTB_DEV_USD; - - /* Initiate PCI-E link training */ - rc = pci_write_config_dword(ndev->pdev, NTB_PPD_OFFSET, - val | BWD_PPD_INIT_LINK); - if (rc) - return rc; - - ndev->reg_ofs.ldb = ndev->reg_base + BWD_PDOORBELL_OFFSET; - ndev->reg_ofs.ldb_mask = ndev->reg_base + BWD_PDBMSK_OFFSET; - ndev->reg_ofs.rdb = ndev->reg_base + BWD_B2B_DOORBELL_OFFSET; - ndev->reg_ofs.bar2_xlat = ndev->reg_base + BWD_SBAR2XLAT_OFFSET; - ndev->reg_ofs.bar4_xlat = ndev->reg_base + BWD_SBAR4XLAT_OFFSET; - ndev->reg_ofs.lnk_cntl = ndev->reg_base + BWD_NTBCNTL_OFFSET; - ndev->reg_ofs.lnk_stat = ndev->reg_base + BWD_LINK_STATUS_OFFSET; - ndev->reg_ofs.spad_read = ndev->reg_base + BWD_SPAD_OFFSET; - ndev->reg_ofs.spad_write = ndev->reg_base + BWD_B2B_SPAD_OFFSET; - ndev->reg_ofs.spci_cmd = ndev->reg_base + BWD_PCICMD_OFFSET; - ndev->limits.max_mw = BWD_MAX_MW; - ndev->limits.max_spads = BWD_MAX_SPADS; - ndev->limits.max_db_bits = BWD_MAX_DB_BITS; - ndev->limits.msix_cnt = BWD_MSIX_CNT; - ndev->bits_per_vector = BWD_DB_BITS_PER_VEC; - - /* Since bwd doesn't have a link interrupt, setup a poll timer */ - INIT_DELAYED_WORK(&ndev->hb_timer, bwd_link_poll); - INIT_DELAYED_WORK(&ndev->lr_timer, bwd_link_recovery); - schedule_delayed_work(&ndev->hb_timer, NTB_HB_TIMEOUT); - - return 0; -} - -static int ntb_device_setup(struct ntb_device *ndev) -{ - int rc; - - if (is_ntb_xeon(ndev)) - rc = ntb_xeon_setup(ndev); - else if (is_ntb_atom(ndev)) - rc = ntb_bwd_setup(ndev); - else - rc = -ENODEV; - - if (rc) - return rc; - - if (ndev->conn_type == NTB_CONN_B2B) - /* Enable Bus Master and Memory Space on the secondary side */ - writew(PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER, - ndev->reg_ofs.spci_cmd); - - return 0; -} - -static void ntb_device_free(struct ntb_device *ndev) -{ - if (is_ntb_atom(ndev)) { - cancel_delayed_work_sync(&ndev->hb_timer); - cancel_delayed_work_sync(&ndev->lr_timer); - } -} - -static irqreturn_t bwd_callback_msix_irq(int irq, void *data) -{ - struct ntb_db_cb *db_cb = data; - struct ntb_device *ndev = db_cb->ndev; - unsigned long mask; - - dev_dbg(&ndev->pdev->dev, "MSI-X irq %d received for DB %d\n", irq, - db_cb->db_num); - - mask = readw(ndev->reg_ofs.ldb_mask); - set_bit(db_cb->db_num * ndev->bits_per_vector, &mask); - writew(mask, ndev->reg_ofs.ldb_mask); - - tasklet_schedule(&db_cb->irq_work); - - /* No need to check for the specific HB irq, any interrupt means - * we're connected. - */ - ndev->last_ts = jiffies; - - writeq((u64) 1 << db_cb->db_num, ndev->reg_ofs.ldb); - - return IRQ_HANDLED; -} - -static irqreturn_t xeon_callback_msix_irq(int irq, void *data) -{ - struct ntb_db_cb *db_cb = data; - struct ntb_device *ndev = db_cb->ndev; - unsigned long mask; - - dev_dbg(&ndev->pdev->dev, "MSI-X irq %d received for DB %d\n", irq, - db_cb->db_num); - - mask = readw(ndev->reg_ofs.ldb_mask); - set_bit(db_cb->db_num * ndev->bits_per_vector, &mask); - writew(mask, ndev->reg_ofs.ldb_mask); - - tasklet_schedule(&db_cb->irq_work); - - /* On Sandybridge, there are 16 bits in the interrupt register - * but only 4 vectors. So, 5 bits are assigned to the first 3 - * vectors, with the 4th having a single bit for link - * interrupts. - */ - writew(((1 << ndev->bits_per_vector) - 1) << - (db_cb->db_num * ndev->bits_per_vector), ndev->reg_ofs.ldb); - - return IRQ_HANDLED; -} - -/* Since we do not have a HW doorbell in BWD, this is only used in JF/JT */ -static irqreturn_t xeon_event_msix_irq(int irq, void *dev) -{ - struct ntb_device *ndev = dev; - int rc; - - dev_dbg(&ndev->pdev->dev, "MSI-X irq %d received for Events\n", irq); - - rc = ntb_link_status(ndev); - if (rc) - dev_err(&ndev->pdev->dev, "Error determining link status\n"); - - /* bit 15 is always the link bit */ - writew(1 << SNB_LINK_DB, ndev->reg_ofs.ldb); - - return IRQ_HANDLED; -} - -static irqreturn_t ntb_interrupt(int irq, void *dev) -{ - struct ntb_device *ndev = dev; - unsigned int i = 0; - - if (is_ntb_atom(ndev)) { - u64 ldb = readq(ndev->reg_ofs.ldb); - - dev_dbg(&ndev->pdev->dev, "irq %d - ldb = %Lx\n", irq, ldb); - - while (ldb) { - i = __ffs(ldb); - ldb &= ldb - 1; - bwd_callback_msix_irq(irq, &ndev->db_cb[i]); - } - } else { - u16 ldb = readw(ndev->reg_ofs.ldb); - - dev_dbg(&ndev->pdev->dev, "irq %d - ldb = %x\n", irq, ldb); - - if (ldb & SNB_DB_HW_LINK) { - xeon_event_msix_irq(irq, dev); - ldb &= ~SNB_DB_HW_LINK; - } - - while (ldb) { - i = __ffs(ldb); - ldb &= ldb - 1; - xeon_callback_msix_irq(irq, &ndev->db_cb[i]); - } - } - - return IRQ_HANDLED; -} - -static int ntb_setup_snb_msix(struct ntb_device *ndev, int msix_entries) -{ - struct pci_dev *pdev = ndev->pdev; - struct msix_entry *msix; - int rc, i; - - if (msix_entries < ndev->limits.msix_cnt) - return -ENOSPC; - - rc = pci_enable_msix_exact(pdev, ndev->msix_entries, msix_entries); - if (rc < 0) - return rc; - - for (i = 0; i < msix_entries; i++) { - msix = &ndev->msix_entries[i]; - WARN_ON(!msix->vector); - - if (i == msix_entries - 1) { - rc = request_irq(msix->vector, - xeon_event_msix_irq, 0, - "ntb-event-msix", ndev); - if (rc) - goto err; - } else { - rc = request_irq(msix->vector, - xeon_callback_msix_irq, 0, - "ntb-callback-msix", - &ndev->db_cb[i]); - if (rc) - goto err; - } - } - - ndev->num_msix = msix_entries; - ndev->max_cbs = msix_entries - 1; - - return 0; - -err: - while (--i >= 0) { - /* Code never reaches here for entry nr 'ndev->num_msix - 1' */ - msix = &ndev->msix_entries[i]; - free_irq(msix->vector, &ndev->db_cb[i]); - } - - pci_disable_msix(pdev); - ndev->num_msix = 0; - - return rc; -} - -static int ntb_setup_bwd_msix(struct ntb_device *ndev, int msix_entries) -{ - struct pci_dev *pdev = ndev->pdev; - struct msix_entry *msix; - int rc, i; - - msix_entries = pci_enable_msix_range(pdev, ndev->msix_entries, - 1, msix_entries); - if (msix_entries < 0) - return msix_entries; - - for (i = 0; i < msix_entries; i++) { - msix = &ndev->msix_entries[i]; - WARN_ON(!msix->vector); - - rc = request_irq(msix->vector, bwd_callback_msix_irq, 0, - "ntb-callback-msix", &ndev->db_cb[i]); - if (rc) - goto err; - } - - ndev->num_msix = msix_entries; - ndev->max_cbs = msix_entries; - - return 0; - -err: - while (--i >= 0) - free_irq(msix->vector, &ndev->db_cb[i]); - - pci_disable_msix(pdev); - ndev->num_msix = 0; - - return rc; -} - -static int ntb_setup_msix(struct ntb_device *ndev) -{ - struct pci_dev *pdev = ndev->pdev; - int msix_entries; - int rc, i; - - msix_entries = pci_msix_vec_count(pdev); - if (msix_entries < 0) { - rc = msix_entries; - goto err; - } else if (msix_entries > ndev->limits.msix_cnt) { - rc = -EINVAL; - goto err; - } - - ndev->msix_entries = kmalloc(sizeof(struct msix_entry) * msix_entries, - GFP_KERNEL); - if (!ndev->msix_entries) { - rc = -ENOMEM; - goto err; - } - - for (i = 0; i < msix_entries; i++) - ndev->msix_entries[i].entry = i; - - if (is_ntb_atom(ndev)) - rc = ntb_setup_bwd_msix(ndev, msix_entries); - else - rc = ntb_setup_snb_msix(ndev, msix_entries); - if (rc) - goto err1; - - return 0; - -err1: - kfree(ndev->msix_entries); -err: - dev_err(&pdev->dev, "Error allocating MSI-X interrupt\n"); - return rc; -} - -static int ntb_setup_msi(struct ntb_device *ndev) -{ - struct pci_dev *pdev = ndev->pdev; - int rc; - - rc = pci_enable_msi(pdev); - if (rc) - return rc; - - rc = request_irq(pdev->irq, ntb_interrupt, 0, "ntb-msi", ndev); - if (rc) { - pci_disable_msi(pdev); - dev_err(&pdev->dev, "Error allocating MSI interrupt\n"); - return rc; - } - - return 0; -} - -static int ntb_setup_intx(struct ntb_device *ndev) -{ - struct pci_dev *pdev = ndev->pdev; - int rc; - - /* Verify intx is enabled */ - pci_intx(pdev, 1); - - rc = request_irq(pdev->irq, ntb_interrupt, IRQF_SHARED, "ntb-intx", - ndev); - if (rc) - return rc; - - return 0; -} - -static int ntb_setup_interrupts(struct ntb_device *ndev) -{ - int rc; - - /* On BWD, disable all interrupts. On SNB, disable all but Link - * Interrupt. The rest will be unmasked as callbacks are registered. - */ - if (is_ntb_atom(ndev)) - writeq(~0, ndev->reg_ofs.ldb_mask); - else { - u16 var = 1 << SNB_LINK_DB; - writew(~var, ndev->reg_ofs.ldb_mask); - } - - rc = ntb_setup_msix(ndev); - if (!rc) - goto done; - - ndev->bits_per_vector = 1; - ndev->max_cbs = ndev->limits.max_db_bits; - - rc = ntb_setup_msi(ndev); - if (!rc) - goto done; - - rc = ntb_setup_intx(ndev); - if (rc) { - dev_err(&ndev->pdev->dev, "no usable interrupts\n"); - return rc; - } - -done: - return 0; -} - -static void ntb_free_interrupts(struct ntb_device *ndev) -{ - struct pci_dev *pdev = ndev->pdev; - - /* mask interrupts */ - if (is_ntb_atom(ndev)) - writeq(~0, ndev->reg_ofs.ldb_mask); - else - writew(~0, ndev->reg_ofs.ldb_mask); - - if (ndev->num_msix) { - struct msix_entry *msix; - u32 i; - - for (i = 0; i < ndev->num_msix; i++) { - msix = &ndev->msix_entries[i]; - if (is_ntb_xeon(ndev) && i == ndev->num_msix - 1) - free_irq(msix->vector, ndev); - else - free_irq(msix->vector, &ndev->db_cb[i]); - } - pci_disable_msix(pdev); - kfree(ndev->msix_entries); - } else { - free_irq(pdev->irq, ndev); - - if (pci_dev_msi_enabled(pdev)) - pci_disable_msi(pdev); - } -} - -static int ntb_create_callbacks(struct ntb_device *ndev) -{ - int i; - - /* Chicken-egg issue. We won't know how many callbacks are necessary - * until we see how many MSI-X vectors we get, but these pointers need - * to be passed into the MSI-X register function. So, we allocate the - * max, knowing that they might not all be used, to work around this. - */ - ndev->db_cb = kcalloc(ndev->limits.max_db_bits, - sizeof(struct ntb_db_cb), - GFP_KERNEL); - if (!ndev->db_cb) - return -ENOMEM; - - for (i = 0; i < ndev->limits.max_db_bits; i++) { - ndev->db_cb[i].db_num = i; - ndev->db_cb[i].ndev = ndev; - } - - return 0; -} - -static void ntb_free_callbacks(struct ntb_device *ndev) -{ - int i; - - for (i = 0; i < ndev->limits.max_db_bits; i++) - ntb_unregister_db_callback(ndev, i); - - kfree(ndev->db_cb); -} - -static ssize_t ntb_debugfs_read(struct file *filp, char __user *ubuf, - size_t count, loff_t *offp) -{ - struct ntb_device *ndev; - char *buf; - ssize_t ret, offset, out_count; - - out_count = 500; - - buf = kmalloc(out_count, GFP_KERNEL); - if (!buf) - return -ENOMEM; - - ndev = filp->private_data; - offset = 0; - offset += snprintf(buf + offset, out_count - offset, - "NTB Device Information:\n"); - offset += snprintf(buf + offset, out_count - offset, - "Connection Type - \t\t%s\n", - ndev->conn_type == NTB_CONN_TRANSPARENT ? - "Transparent" : (ndev->conn_type == NTB_CONN_B2B) ? - "Back to back" : "Root Port"); - offset += snprintf(buf + offset, out_count - offset, - "Device Type - \t\t\t%s\n", - ndev->dev_type == NTB_DEV_USD ? - "DSD/USP" : "USD/DSP"); - offset += snprintf(buf + offset, out_count - offset, - "Max Number of Callbacks - \t%u\n", - ntb_max_cbs(ndev)); - offset += snprintf(buf + offset, out_count - offset, - "Link Status - \t\t\t%s\n", - ntb_hw_link_status(ndev) ? "Up" : "Down"); - if (ntb_hw_link_status(ndev)) { - offset += snprintf(buf + offset, out_count - offset, - "Link Speed - \t\t\tPCI-E Gen %u\n", - ndev->link_speed); - offset += snprintf(buf + offset, out_count - offset, - "Link Width - \t\t\tx%u\n", - ndev->link_width); - } - - if (is_ntb_xeon(ndev)) { - u32 status32; - u16 status16; - int rc; - - offset += snprintf(buf + offset, out_count - offset, - "\nNTB Device Statistics:\n"); - offset += snprintf(buf + offset, out_count - offset, - "Upstream Memory Miss - \t%u\n", - readw(ndev->reg_base + - SNB_USMEMMISS_OFFSET)); - - offset += snprintf(buf + offset, out_count - offset, - "\nNTB Hardware Errors:\n"); - - rc = pci_read_config_word(ndev->pdev, SNB_DEVSTS_OFFSET, - &status16); - if (!rc) - offset += snprintf(buf + offset, out_count - offset, - "DEVSTS - \t%#06x\n", status16); - - rc = pci_read_config_word(ndev->pdev, SNB_LINK_STATUS_OFFSET, - &status16); - if (!rc) - offset += snprintf(buf + offset, out_count - offset, - "LNKSTS - \t%#06x\n", status16); - - rc = pci_read_config_dword(ndev->pdev, SNB_UNCERRSTS_OFFSET, - &status32); - if (!rc) - offset += snprintf(buf + offset, out_count - offset, - "UNCERRSTS - \t%#010x\n", status32); - - rc = pci_read_config_dword(ndev->pdev, SNB_CORERRSTS_OFFSET, - &status32); - if (!rc) - offset += snprintf(buf + offset, out_count - offset, - "CORERRSTS - \t%#010x\n", status32); - } - - if (offset > out_count) - offset = out_count; - - ret = simple_read_from_buffer(ubuf, count, offp, buf, offset); - kfree(buf); - return ret; -} - -static const struct file_operations ntb_debugfs_info = { - .owner = THIS_MODULE, - .open = simple_open, - .read = ntb_debugfs_read, -}; - -static void ntb_setup_debugfs(struct ntb_device *ndev) -{ - if (!debugfs_initialized()) - return; - - if (!debugfs_dir) - debugfs_dir = debugfs_create_dir(KBUILD_MODNAME, NULL); - - ndev->debugfs_dir = debugfs_create_dir(pci_name(ndev->pdev), - debugfs_dir); - if (ndev->debugfs_dir) - ndev->debugfs_info = debugfs_create_file("info", S_IRUSR, - ndev->debugfs_dir, - ndev, - &ntb_debugfs_info); -} - -static void ntb_free_debugfs(struct ntb_device *ndev) -{ - debugfs_remove_recursive(ndev->debugfs_dir); - - if (debugfs_dir && simple_empty(debugfs_dir)) { - debugfs_remove_recursive(debugfs_dir); - debugfs_dir = NULL; - } -} - -static void ntb_hw_link_up(struct ntb_device *ndev) -{ - if (ndev->conn_type == NTB_CONN_TRANSPARENT) - ntb_link_event(ndev, NTB_LINK_UP); - else { - u32 ntb_cntl; - - /* Let's bring the NTB link up */ - ntb_cntl = readl(ndev->reg_ofs.lnk_cntl); - ntb_cntl &= ~(NTB_CNTL_LINK_DISABLE | NTB_CNTL_CFG_LOCK); - ntb_cntl |= NTB_CNTL_P2S_BAR23_SNOOP | NTB_CNTL_S2P_BAR23_SNOOP; - ntb_cntl |= NTB_CNTL_P2S_BAR4_SNOOP | NTB_CNTL_S2P_BAR4_SNOOP; - if (ndev->split_bar) - ntb_cntl |= NTB_CNTL_P2S_BAR5_SNOOP | - NTB_CNTL_S2P_BAR5_SNOOP; - - writel(ntb_cntl, ndev->reg_ofs.lnk_cntl); - } -} - -static void ntb_hw_link_down(struct ntb_device *ndev) -{ - u32 ntb_cntl; - - if (ndev->conn_type == NTB_CONN_TRANSPARENT) { - ntb_link_event(ndev, NTB_LINK_DOWN); - return; - } - - /* Bring NTB link down */ - ntb_cntl = readl(ndev->reg_ofs.lnk_cntl); - ntb_cntl &= ~(NTB_CNTL_P2S_BAR23_SNOOP | NTB_CNTL_S2P_BAR23_SNOOP); - ntb_cntl &= ~(NTB_CNTL_P2S_BAR4_SNOOP | NTB_CNTL_S2P_BAR4_SNOOP); - if (ndev->split_bar) - ntb_cntl &= ~(NTB_CNTL_P2S_BAR5_SNOOP | - NTB_CNTL_S2P_BAR5_SNOOP); - ntb_cntl |= NTB_CNTL_LINK_DISABLE | NTB_CNTL_CFG_LOCK; - writel(ntb_cntl, ndev->reg_ofs.lnk_cntl); -} - -static void ntb_max_mw_detect(struct ntb_device *ndev) -{ - if (ndev->split_bar) - ndev->limits.max_mw = HSX_SPLITBAR_MAX_MW; - else - ndev->limits.max_mw = SNB_MAX_MW; -} - -static int ntb_xeon_detect(struct ntb_device *ndev) -{ - int rc, bars_mask; - u32 bars; - u8 ppd; - - ndev->hw_type = SNB_HW; - - rc = pci_read_config_byte(ndev->pdev, NTB_PPD_OFFSET, &ppd); - if (rc) - return -EIO; - - if (ppd & SNB_PPD_DEV_TYPE) - ndev->dev_type = NTB_DEV_USD; - else - ndev->dev_type = NTB_DEV_DSD; - - ndev->split_bar = (ppd & SNB_PPD_SPLIT_BAR) ? 1 : 0; - - switch (ppd & SNB_PPD_CONN_TYPE) { - case NTB_CONN_B2B: - dev_info(&ndev->pdev->dev, "Conn Type = B2B\n"); - ndev->conn_type = NTB_CONN_B2B; - break; - case NTB_CONN_RP: - dev_info(&ndev->pdev->dev, "Conn Type = RP\n"); - ndev->conn_type = NTB_CONN_RP; - break; - case NTB_CONN_TRANSPARENT: - dev_info(&ndev->pdev->dev, "Conn Type = TRANSPARENT\n"); - ndev->conn_type = NTB_CONN_TRANSPARENT; - /* - * This mode is default to USD/DSP. HW does not report - * properly in transparent mode as it has no knowledge of - * NTB. We will just force correct here. - */ - ndev->dev_type = NTB_DEV_USD; - - /* - * This is a way for transparent BAR to figure out if we - * are doing split BAR or not. There is no way for the hw - * on the transparent side to know and set the PPD. - */ - bars_mask = pci_select_bars(ndev->pdev, IORESOURCE_MEM); - bars = hweight32(bars_mask); - if (bars == (HSX_SPLITBAR_MAX_MW + 1)) - ndev->split_bar = 1; - - break; - default: - dev_err(&ndev->pdev->dev, "Unknown PPD %x\n", ppd); - return -ENODEV; - } - - ntb_max_mw_detect(ndev); - - return 0; -} - -static int ntb_atom_detect(struct ntb_device *ndev) -{ - int rc; - u32 ppd; - - ndev->hw_type = BWD_HW; - ndev->limits.max_mw = BWD_MAX_MW; - - rc = pci_read_config_dword(ndev->pdev, NTB_PPD_OFFSET, &ppd); - if (rc) - return rc; - - switch ((ppd & BWD_PPD_CONN_TYPE) >> 8) { - case NTB_CONN_B2B: - dev_info(&ndev->pdev->dev, "Conn Type = B2B\n"); - ndev->conn_type = NTB_CONN_B2B; - break; - case NTB_CONN_RP: - default: - dev_err(&ndev->pdev->dev, "Unsupported NTB configuration\n"); - return -EINVAL; - } - - if (ppd & BWD_PPD_DEV_TYPE) - ndev->dev_type = NTB_DEV_DSD; - else - ndev->dev_type = NTB_DEV_USD; - - return 0; -} - -static int ntb_device_detect(struct ntb_device *ndev) -{ - int rc; - - if (is_ntb_xeon(ndev)) - rc = ntb_xeon_detect(ndev); - else if (is_ntb_atom(ndev)) - rc = ntb_atom_detect(ndev); - else - rc = -ENODEV; - - dev_info(&ndev->pdev->dev, "Device Type = %s\n", - ndev->dev_type == NTB_DEV_USD ? "USD/DSP" : "DSD/USP"); - - return 0; -} - -static int ntb_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) -{ - struct ntb_device *ndev; - int rc, i; - - ndev = kzalloc(sizeof(struct ntb_device), GFP_KERNEL); - if (!ndev) - return -ENOMEM; - - ndev->pdev = pdev; - - ntb_set_errata_flags(ndev); - - ndev->link_status = NTB_LINK_DOWN; - pci_set_drvdata(pdev, ndev); - ntb_setup_debugfs(ndev); - - rc = pci_enable_device(pdev); - if (rc) - goto err; - - pci_set_master(ndev->pdev); - - rc = ntb_device_detect(ndev); - if (rc) - goto err; - - ndev->mw = kcalloc(ndev->limits.max_mw, sizeof(struct ntb_mw), - GFP_KERNEL); - if (!ndev->mw) { - rc = -ENOMEM; - goto err1; - } - - if (ndev->split_bar) - rc = pci_request_selected_regions(pdev, NTB_SPLITBAR_MASK, - KBUILD_MODNAME); - else - rc = pci_request_selected_regions(pdev, NTB_BAR_MASK, - KBUILD_MODNAME); - - if (rc) - goto err2; - - ndev->reg_base = pci_ioremap_bar(pdev, NTB_BAR_MMIO); - if (!ndev->reg_base) { - dev_warn(&pdev->dev, "Cannot remap BAR 0\n"); - rc = -EIO; - goto err3; - } - - for (i = 0; i < ndev->limits.max_mw; i++) { - ndev->mw[i].bar_sz = pci_resource_len(pdev, MW_TO_BAR(i)); - - /* - * with the errata we need to steal last of the memory - * windows for workarounds and they point to MMIO registers. - */ - if ((ndev->wa_flags & WA_SNB_ERR) && - (i == (ndev->limits.max_mw - 1))) { - ndev->mw[i].vbase = - ioremap_nocache(pci_resource_start(pdev, - MW_TO_BAR(i)), - ndev->mw[i].bar_sz); - } else { - ndev->mw[i].vbase = - ioremap_wc(pci_resource_start(pdev, - MW_TO_BAR(i)), - ndev->mw[i].bar_sz); - } - - dev_info(&pdev->dev, "MW %d size %llu\n", i, - (unsigned long long) ndev->mw[i].bar_sz); - if (!ndev->mw[i].vbase) { - dev_warn(&pdev->dev, "Cannot remap BAR %d\n", - MW_TO_BAR(i)); - rc = -EIO; - goto err4; - } - } - - rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(64)); - if (rc) { - rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(32)); - if (rc) - goto err4; - - dev_warn(&pdev->dev, "Cannot DMA highmem\n"); - } - - rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64)); - if (rc) { - rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32)); - if (rc) - goto err4; - - dev_warn(&pdev->dev, "Cannot DMA consistent highmem\n"); - } - - rc = ntb_device_setup(ndev); - if (rc) - goto err4; - - rc = ntb_create_callbacks(ndev); - if (rc) - goto err5; - - rc = ntb_setup_interrupts(ndev); - if (rc) - goto err6; - - /* The scratchpad registers keep the values between rmmod/insmod, - * blast them now - */ - for (i = 0; i < ndev->limits.max_spads; i++) { - ntb_write_local_spad(ndev, i, 0); - ntb_write_remote_spad(ndev, i, 0); - } - - rc = ntb_transport_init(pdev); - if (rc) - goto err7; - - ntb_hw_link_up(ndev); - - return 0; - -err7: - ntb_free_interrupts(ndev); -err6: - ntb_free_callbacks(ndev); -err5: - ntb_device_free(ndev); -err4: - for (i--; i >= 0; i--) - iounmap(ndev->mw[i].vbase); - iounmap(ndev->reg_base); -err3: - if (ndev->split_bar) - pci_release_selected_regions(pdev, NTB_SPLITBAR_MASK); - else - pci_release_selected_regions(pdev, NTB_BAR_MASK); -err2: - kfree(ndev->mw); -err1: - pci_disable_device(pdev); -err: - ntb_free_debugfs(ndev); - kfree(ndev); - - dev_err(&pdev->dev, "Error loading %s module\n", KBUILD_MODNAME); - return rc; -} - -static void ntb_pci_remove(struct pci_dev *pdev) -{ - struct ntb_device *ndev = pci_get_drvdata(pdev); - int i; - - ntb_hw_link_down(ndev); - - ntb_transport_free(ndev->ntb_transport); - - ntb_free_interrupts(ndev); - ntb_free_callbacks(ndev); - ntb_device_free(ndev); - - /* need to reset max_mw limits so we can unmap properly */ - if (ndev->hw_type == SNB_HW) - ntb_max_mw_detect(ndev); - - for (i = 0; i < ndev->limits.max_mw; i++) - iounmap(ndev->mw[i].vbase); - - kfree(ndev->mw); - iounmap(ndev->reg_base); - if (ndev->split_bar) - pci_release_selected_regions(pdev, NTB_SPLITBAR_MASK); - else - pci_release_selected_regions(pdev, NTB_BAR_MASK); - pci_disable_device(pdev); - ntb_free_debugfs(ndev); - kfree(ndev); -} - -static struct pci_driver ntb_pci_driver = { - .name = KBUILD_MODNAME, - .id_table = ntb_pci_tbl, - .probe = ntb_pci_probe, - .remove = ntb_pci_remove, -}; - -module_pci_driver(ntb_pci_driver); diff --git a/drivers/ntb/ntb_hw.h b/drivers/ntb/ntb_hw.h deleted file mode 100644 index 96de5fc95f90..000000000000 --- a/drivers/ntb/ntb_hw.h +++ /dev/null @@ -1,256 +0,0 @@ -/* - * This file is provided under a dual BSD/GPLv2 license. When using or - * redistributing this file, you may do so under either license. - * - * GPL LICENSE SUMMARY - * - * Copyright(c) 2012 Intel Corporation. All rights reserved. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of version 2 of the GNU General Public License as - * published by the Free Software Foundation. - * - * BSD LICENSE - * - * Copyright(c) 2012 Intel Corporation. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copy - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * Intel PCIe NTB Linux driver - * - * Contact Information: - * Jon Mason <jon.mason@intel.com> - */ -#include <linux/ntb.h> - -#define PCI_DEVICE_ID_INTEL_NTB_B2B_JSF 0x3725 -#define PCI_DEVICE_ID_INTEL_NTB_PS_JSF 0x3726 -#define PCI_DEVICE_ID_INTEL_NTB_SS_JSF 0x3727 -#define PCI_DEVICE_ID_INTEL_NTB_B2B_SNB 0x3C0D -#define PCI_DEVICE_ID_INTEL_NTB_PS_SNB 0x3C0E -#define PCI_DEVICE_ID_INTEL_NTB_SS_SNB 0x3C0F -#define PCI_DEVICE_ID_INTEL_NTB_B2B_IVT 0x0E0D -#define PCI_DEVICE_ID_INTEL_NTB_PS_IVT 0x0E0E -#define PCI_DEVICE_ID_INTEL_NTB_SS_IVT 0x0E0F -#define PCI_DEVICE_ID_INTEL_NTB_B2B_HSX 0x2F0D -#define PCI_DEVICE_ID_INTEL_NTB_PS_HSX 0x2F0E -#define PCI_DEVICE_ID_INTEL_NTB_SS_HSX 0x2F0F -#define PCI_DEVICE_ID_INTEL_NTB_B2B_BWD 0x0C4E - -#ifndef readq -static inline u64 readq(void __iomem *addr) -{ - return readl(addr) | (((u64) readl(addr + 4)) << 32LL); -} -#endif - -#ifndef writeq -static inline void writeq(u64 val, void __iomem *addr) -{ - writel(val & 0xffffffff, addr); - writel(val >> 32, addr + 4); -} -#endif - -#define NTB_BAR_MMIO 0 -#define NTB_BAR_23 2 -#define NTB_BAR_4 4 -#define NTB_BAR_5 5 - -#define NTB_BAR_MASK ((1 << NTB_BAR_MMIO) | (1 << NTB_BAR_23) |\ - (1 << NTB_BAR_4)) -#define NTB_SPLITBAR_MASK ((1 << NTB_BAR_MMIO) | (1 << NTB_BAR_23) |\ - (1 << NTB_BAR_4) | (1 << NTB_BAR_5)) - -#define NTB_HB_TIMEOUT msecs_to_jiffies(1000) - -enum ntb_hw_event { - NTB_EVENT_SW_EVENT0 = 0, - NTB_EVENT_SW_EVENT1, - NTB_EVENT_SW_EVENT2, - NTB_EVENT_HW_ERROR, - NTB_EVENT_HW_LINK_UP, - NTB_EVENT_HW_LINK_DOWN, -}; - -struct ntb_mw { - dma_addr_t phys_addr; - void __iomem *vbase; - resource_size_t bar_sz; -}; - -struct ntb_db_cb { - int (*callback)(void *data, int db_num); - unsigned int db_num; - void *data; - struct ntb_device *ndev; - struct tasklet_struct irq_work; -}; - -#define WA_SNB_ERR 0x00000001 - -struct ntb_device { - struct pci_dev *pdev; - struct msix_entry *msix_entries; - void __iomem *reg_base; - struct ntb_mw *mw; - struct { - unsigned char max_mw; - unsigned char max_spads; - unsigned char max_db_bits; - unsigned char msix_cnt; - } limits; - struct { - void __iomem *ldb; - void __iomem *ldb_mask; - void __iomem *rdb; - void __iomem *bar2_xlat; - void __iomem *bar4_xlat; - void __iomem *bar5_xlat; - void __iomem *spad_write; - void __iomem *spad_read; - void __iomem *lnk_cntl; - void __iomem *lnk_stat; - void __iomem *spci_cmd; - } reg_ofs; - struct ntb_transport *ntb_transport; - void (*event_cb)(void *handle, enum ntb_hw_event event); - - struct ntb_db_cb *db_cb; - unsigned char hw_type; - unsigned char conn_type; - unsigned char dev_type; - unsigned char num_msix; - unsigned char bits_per_vector; - unsigned char max_cbs; - unsigned char link_width; - unsigned char link_speed; - unsigned char link_status; - unsigned char split_bar; - - struct delayed_work hb_timer; - unsigned long last_ts; - - struct delayed_work lr_timer; - - struct dentry *debugfs_dir; - struct dentry *debugfs_info; - - unsigned int wa_flags; -}; - -/** - * ntb_max_cbs() - return the max callbacks - * @ndev: pointer to ntb_device instance - * - * Given the ntb pointer, return the maximum number of callbacks - * - * RETURNS: the maximum number of callbacks - */ -static inline unsigned char ntb_max_cbs(struct ntb_device *ndev) -{ - return ndev->max_cbs; -} - -/** - * ntb_max_mw() - return the max number of memory windows - * @ndev: pointer to ntb_device instance - * - * Given the ntb pointer, return the maximum number of memory windows - * - * RETURNS: the maximum number of memory windows - */ -static inline unsigned char ntb_max_mw(struct ntb_device *ndev) -{ - return ndev->limits.max_mw; -} - -/** - * ntb_hw_link_status() - return the hardware link status - * @ndev: pointer to ntb_device instance - * - * Returns true if the hardware is connected to the remote system - * - * RETURNS: true or false based on the hardware link state - */ -static inline bool ntb_hw_link_status(struct ntb_device *ndev) -{ - return ndev->link_status == NTB_LINK_UP; -} - -/** - * ntb_query_pdev() - return the pci_dev pointer - * @ndev: pointer to ntb_device instance - * - * Given the ntb pointer, return the pci_dev pointer for the NTB hardware device - * - * RETURNS: a pointer to the ntb pci_dev - */ -static inline struct pci_dev *ntb_query_pdev(struct ntb_device *ndev) -{ - return ndev->pdev; -} - -/** - * ntb_query_debugfs() - return the debugfs pointer - * @ndev: pointer to ntb_device instance - * - * Given the ntb pointer, return the debugfs directory pointer for the NTB - * hardware device - * - * RETURNS: a pointer to the debugfs directory - */ -static inline struct dentry *ntb_query_debugfs(struct ntb_device *ndev) -{ - return ndev->debugfs_dir; -} - -struct ntb_device *ntb_register_transport(struct pci_dev *pdev, - void *transport); -void ntb_unregister_transport(struct ntb_device *ndev); -void ntb_set_mw_addr(struct ntb_device *ndev, unsigned int mw, u64 addr); -int ntb_register_db_callback(struct ntb_device *ndev, unsigned int idx, - void *data, int (*db_cb_func)(void *data, - int db_num)); -void ntb_unregister_db_callback(struct ntb_device *ndev, unsigned int idx); -int ntb_register_event_callback(struct ntb_device *ndev, - void (*event_cb_func)(void *handle, - enum ntb_hw_event event)); -void ntb_unregister_event_callback(struct ntb_device *ndev); -int ntb_get_max_spads(struct ntb_device *ndev); -int ntb_write_local_spad(struct ntb_device *ndev, unsigned int idx, u32 val); -int ntb_read_local_spad(struct ntb_device *ndev, unsigned int idx, u32 *val); -int ntb_write_remote_spad(struct ntb_device *ndev, unsigned int idx, u32 val); -int ntb_read_remote_spad(struct ntb_device *ndev, unsigned int idx, u32 *val); -resource_size_t ntb_get_mw_base(struct ntb_device *ndev, unsigned int mw); -void __iomem *ntb_get_mw_vbase(struct ntb_device *ndev, unsigned int mw); -u64 ntb_get_mw_size(struct ntb_device *ndev, unsigned int mw); -void ntb_ring_doorbell(struct ntb_device *ndev, unsigned int idx); -void *ntb_find_transport(struct pci_dev *pdev); - -int ntb_transport_init(struct pci_dev *pdev); -void ntb_transport_free(void *transport); diff --git a/drivers/ntb/ntb_regs.h b/drivers/ntb/ntb_regs.h deleted file mode 100644 index f028ff81fd77..000000000000 --- a/drivers/ntb/ntb_regs.h +++ /dev/null @@ -1,177 +0,0 @@ -/* - * This file is provided under a dual BSD/GPLv2 license. When using or - * redistributing this file, you may do so under either license. - * - * GPL LICENSE SUMMARY - * - * Copyright(c) 2012 Intel Corporation. All rights reserved. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of version 2 of the GNU General Public License as - * published by the Free Software Foundation. - * - * BSD LICENSE - * - * Copyright(c) 2012 Intel Corporation. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copy - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * Intel PCIe NTB Linux driver - * - * Contact Information: - * Jon Mason <jon.mason@intel.com> - */ - -#define NTB_LINK_STATUS_ACTIVE 0x2000 -#define NTB_LINK_SPEED_MASK 0x000f -#define NTB_LINK_WIDTH_MASK 0x03f0 - -#define SNB_MSIX_CNT 4 -#define SNB_MAX_B2B_SPADS 16 -#define SNB_MAX_COMPAT_SPADS 16 -/* Reserve the uppermost bit for link interrupt */ -#define SNB_MAX_DB_BITS 15 -#define SNB_LINK_DB 15 -#define SNB_DB_BITS_PER_VEC 5 -#define HSX_SPLITBAR_MAX_MW 3 -#define SNB_MAX_MW 2 -#define SNB_ERRATA_MAX_MW 1 - -#define SNB_DB_HW_LINK 0x8000 - -#define SNB_UNCERRSTS_OFFSET 0x014C -#define SNB_CORERRSTS_OFFSET 0x0158 -#define SNB_LINK_STATUS_OFFSET 0x01A2 -#define SNB_PCICMD_OFFSET 0x0504 -#define SNB_DEVCTRL_OFFSET 0x0598 -#define SNB_DEVSTS_OFFSET 0x059A -#define SNB_SLINK_STATUS_OFFSET 0x05A2 - -#define SNB_PBAR2LMT_OFFSET 0x0000 -#define SNB_PBAR4LMT_OFFSET 0x0008 -#define SNB_PBAR5LMT_OFFSET 0x000C -#define SNB_PBAR2XLAT_OFFSET 0x0010 -#define SNB_PBAR4XLAT_OFFSET 0x0018 -#define SNB_PBAR5XLAT_OFFSET 0x001C -#define SNB_SBAR2LMT_OFFSET 0x0020 -#define SNB_SBAR4LMT_OFFSET 0x0028 -#define SNB_SBAR5LMT_OFFSET 0x002C -#define SNB_SBAR2XLAT_OFFSET 0x0030 -#define SNB_SBAR4XLAT_OFFSET 0x0038 -#define SNB_SBAR5XLAT_OFFSET 0x003C -#define SNB_SBAR0BASE_OFFSET 0x0040 -#define SNB_SBAR2BASE_OFFSET 0x0048 -#define SNB_SBAR4BASE_OFFSET 0x0050 -#define SNB_SBAR5BASE_OFFSET 0x0054 -#define SNB_NTBCNTL_OFFSET 0x0058 -#define SNB_SBDF_OFFSET 0x005C -#define SNB_PDOORBELL_OFFSET 0x0060 -#define SNB_PDBMSK_OFFSET 0x0062 -#define SNB_SDOORBELL_OFFSET 0x0064 -#define SNB_SDBMSK_OFFSET 0x0066 -#define SNB_USMEMMISS_OFFSET 0x0070 -#define SNB_SPAD_OFFSET 0x0080 -#define SNB_SPADSEMA4_OFFSET 0x00c0 -#define SNB_WCCNTRL_OFFSET 0x00e0 -#define SNB_B2B_SPAD_OFFSET 0x0100 -#define SNB_B2B_DOORBELL_OFFSET 0x0140 -#define SNB_B2B_XLAT_OFFSETL 0x0144 -#define SNB_B2B_XLAT_OFFSETU 0x0148 - -/* - * The addresses are setup so the 32bit BARs can function. Thus - * the addresses are all in 32bit space - */ -#define SNB_MBAR01_USD_ADDR 0x000000002100000CULL -#define SNB_MBAR23_USD_ADDR 0x000000004100000CULL -#define SNB_MBAR4_USD_ADDR 0x000000008100000CULL -#define SNB_MBAR5_USD_ADDR 0x00000000A100000CULL -#define SNB_MBAR01_DSD_ADDR 0x000000002000000CULL -#define SNB_MBAR23_DSD_ADDR 0x000000004000000CULL -#define SNB_MBAR4_DSD_ADDR 0x000000008000000CULL -#define SNB_MBAR5_DSD_ADDR 0x00000000A000000CULL - -#define BWD_MSIX_CNT 34 -#define BWD_MAX_SPADS 16 -#define BWD_MAX_DB_BITS 34 -#define BWD_DB_BITS_PER_VEC 1 -#define BWD_MAX_MW 2 - -#define BWD_PCICMD_OFFSET 0xb004 -#define BWD_MBAR23_OFFSET 0xb018 -#define BWD_MBAR45_OFFSET 0xb020 -#define BWD_DEVCTRL_OFFSET 0xb048 -#define BWD_LINK_STATUS_OFFSET 0xb052 -#define BWD_ERRCORSTS_OFFSET 0xb110 - -#define BWD_SBAR2XLAT_OFFSET 0x0008 -#define BWD_SBAR4XLAT_OFFSET 0x0010 -#define BWD_PDOORBELL_OFFSET 0x0020 -#define BWD_PDBMSK_OFFSET 0x0028 -#define BWD_NTBCNTL_OFFSET 0x0060 -#define BWD_EBDF_OFFSET 0x0064 -#define BWD_SPAD_OFFSET 0x0080 -#define BWD_SPADSEMA_OFFSET 0x00c0 -#define BWD_STKYSPAD_OFFSET 0x00c4 -#define BWD_PBAR2XLAT_OFFSET 0x8008 -#define BWD_PBAR4XLAT_OFFSET 0x8010 -#define BWD_B2B_DOORBELL_OFFSET 0x8020 -#define BWD_B2B_SPAD_OFFSET 0x8080 -#define BWD_B2B_SPADSEMA_OFFSET 0x80c0 -#define BWD_B2B_STKYSPAD_OFFSET 0x80c4 - -#define BWD_MODPHY_PCSREG4 0x1c004 -#define BWD_MODPHY_PCSREG6 0x1c006 - -#define BWD_IP_BASE 0xC000 -#define BWD_DESKEWSTS_OFFSET (BWD_IP_BASE + 0x3024) -#define BWD_LTSSMERRSTS0_OFFSET (BWD_IP_BASE + 0x3180) -#define BWD_LTSSMSTATEJMP_OFFSET (BWD_IP_BASE + 0x3040) -#define BWD_IBSTERRRCRVSTS0_OFFSET (BWD_IP_BASE + 0x3324) - -#define BWD_DESKEWSTS_DBERR (1 << 15) -#define BWD_LTSSMERRSTS0_UNEXPECTEDEI (1 << 20) -#define BWD_LTSSMSTATEJMP_FORCEDETECT (1 << 2) -#define BWD_IBIST_ERR_OFLOW 0x7FFF7FFF - -#define NTB_CNTL_CFG_LOCK (1 << 0) -#define NTB_CNTL_LINK_DISABLE (1 << 1) -#define NTB_CNTL_S2P_BAR23_SNOOP (1 << 2) -#define NTB_CNTL_P2S_BAR23_SNOOP (1 << 4) -#define NTB_CNTL_S2P_BAR4_SNOOP (1 << 6) -#define NTB_CNTL_P2S_BAR4_SNOOP (1 << 8) -#define NTB_CNTL_S2P_BAR5_SNOOP (1 << 12) -#define NTB_CNTL_P2S_BAR5_SNOOP (1 << 14) -#define BWD_CNTL_LINK_DOWN (1 << 16) - -#define NTB_PPD_OFFSET 0x00D4 -#define SNB_PPD_CONN_TYPE 0x0003 -#define SNB_PPD_DEV_TYPE 0x0010 -#define SNB_PPD_SPLIT_BAR (1 << 6) -#define BWD_PPD_INIT_LINK 0x0008 -#define BWD_PPD_CONN_TYPE 0x0300 -#define BWD_PPD_DEV_TYPE 0x1000 diff --git a/drivers/ntb/ntb_transport.c b/drivers/ntb/ntb_transport.c index e9bf2f47b61a..efe3ad4122f2 100644 --- a/drivers/ntb/ntb_transport.c +++ b/drivers/ntb/ntb_transport.c @@ -5,6 +5,7 @@ * GPL LICENSE SUMMARY * * Copyright(c) 2012 Intel Corporation. All rights reserved. + * Copyright (C) 2015 EMC Corporation. All Rights Reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of version 2 of the GNU General Public License as @@ -13,6 +14,7 @@ * BSD LICENSE * * Copyright(c) 2012 Intel Corporation. All rights reserved. + * Copyright (C) 2015 EMC Corporation. All Rights Reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -40,7 +42,7 @@ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * - * Intel PCIe NTB Linux driver + * PCIe NTB Transport Linux driver * * Contact Information: * Jon Mason <jon.mason@intel.com> @@ -56,11 +58,25 @@ #include <linux/pci.h> #include <linux/slab.h> #include <linux/types.h> -#include "ntb_hw.h" +#include <linux/uaccess.h> +#include "linux/ntb.h" +#include "linux/ntb_transport.h" -#define NTB_TRANSPORT_VERSION 3 +#define NTB_TRANSPORT_VERSION 4 +#define NTB_TRANSPORT_VER "4" +#define NTB_TRANSPORT_NAME "ntb_transport" +#define NTB_TRANSPORT_DESC "Software Queue-Pair Transport over NTB" -static unsigned int transport_mtu = 0x401E; +MODULE_DESCRIPTION(NTB_TRANSPORT_DESC); +MODULE_VERSION(NTB_TRANSPORT_VER); +MODULE_LICENSE("Dual BSD/GPL"); +MODULE_AUTHOR("Intel Corporation"); + +static unsigned long max_mw_size; +module_param(max_mw_size, ulong, 0644); +MODULE_PARM_DESC(max_mw_size, "Limit size of large memory windows"); + +static unsigned int transport_mtu = 0x10000; module_param(transport_mtu, uint, 0644); MODULE_PARM_DESC(transport_mtu, "Maximum size of NTB transport packets"); @@ -72,10 +88,16 @@ static unsigned int copy_bytes = 1024; module_param(copy_bytes, uint, 0644); MODULE_PARM_DESC(copy_bytes, "Threshold under which NTB will use the CPU to copy instead of DMA"); +static bool use_dma; +module_param(use_dma, bool, 0644); +MODULE_PARM_DESC(use_dma, "Use DMA engine to perform large data copy"); + +static struct dentry *nt_debugfs_dir; + struct ntb_queue_entry { /* ntb_queue list reference */ struct list_head entry; - /* pointers to data to be transfered */ + /* pointers to data to be transferred */ void *cb_data; void *buf; unsigned int len; @@ -94,14 +116,16 @@ struct ntb_rx_info { }; struct ntb_transport_qp { - struct ntb_transport *transport; - struct ntb_device *ndev; + struct ntb_transport_ctx *transport; + struct ntb_dev *ndev; void *cb_data; struct dma_chan *dma_chan; bool client_ready; - bool qp_link; + bool link_is_up; + u8 qp_num; /* Only 64 QP's are allowed. 0-63 */ + u64 qp_bit; struct ntb_rx_info __iomem *rx_info; struct ntb_rx_info *remote_rx_info; @@ -127,6 +151,7 @@ struct ntb_transport_qp { unsigned int rx_max_entry; unsigned int rx_max_frame; dma_cookie_t last_cookie; + struct tasklet_struct rxc_db_work; void (*event_handler)(void *data, int status); struct delayed_work link_work; @@ -153,33 +178,44 @@ struct ntb_transport_qp { }; struct ntb_transport_mw { - size_t size; + phys_addr_t phys_addr; + resource_size_t phys_size; + resource_size_t xlat_align; + resource_size_t xlat_align_size; + void __iomem *vbase; + size_t xlat_size; + size_t buff_size; void *virt_addr; dma_addr_t dma_addr; }; struct ntb_transport_client_dev { struct list_head entry; + struct ntb_transport_ctx *nt; struct device dev; }; -struct ntb_transport { +struct ntb_transport_ctx { struct list_head entry; struct list_head client_devs; - struct ntb_device *ndev; - struct ntb_transport_mw *mw; - struct ntb_transport_qp *qps; - unsigned int max_qps; - unsigned long qp_bitmap; - bool transport_link; + struct ntb_dev *ndev; + + struct ntb_transport_mw *mw_vec; + struct ntb_transport_qp *qp_vec; + unsigned int mw_count; + unsigned int qp_count; + u64 qp_bitmap; + u64 qp_bitmap_free; + + bool link_is_up; struct delayed_work link_work; struct work_struct link_cleanup; }; enum { - DESC_DONE_FLAG = 1 << 0, - LINK_DOWN_FLAG = 1 << 1, + DESC_DONE_FLAG = BIT(0), + LINK_DOWN_FLAG = BIT(1), }; struct ntb_payload_header { @@ -200,68 +236,69 @@ enum { MAX_SPAD, }; -#define QP_TO_MW(ndev, qp) ((qp) % ntb_max_mw(ndev)) +#define dev_client_dev(__dev) \ + container_of((__dev), struct ntb_transport_client_dev, dev) + +#define drv_client(__drv) \ + container_of((__drv), struct ntb_transport_client, driver) + +#define QP_TO_MW(nt, qp) ((qp) % nt->mw_count) #define NTB_QP_DEF_NUM_ENTRIES 100 #define NTB_LINK_DOWN_TIMEOUT 10 -static int ntb_match_bus(struct device *dev, struct device_driver *drv) +static void ntb_transport_rxc_db(unsigned long data); +static const struct ntb_ctx_ops ntb_transport_ops; +static struct ntb_client ntb_transport_client; + +static int ntb_transport_bus_match(struct device *dev, + struct device_driver *drv) { return !strncmp(dev_name(dev), drv->name, strlen(drv->name)); } -static int ntb_client_probe(struct device *dev) +static int ntb_transport_bus_probe(struct device *dev) { - const struct ntb_client *drv = container_of(dev->driver, - struct ntb_client, driver); - struct pci_dev *pdev = container_of(dev->parent, struct pci_dev, dev); + const struct ntb_transport_client *client; int rc = -EINVAL; get_device(dev); - if (drv && drv->probe) - rc = drv->probe(pdev); + + client = drv_client(dev->driver); + rc = client->probe(dev); if (rc) put_device(dev); return rc; } -static int ntb_client_remove(struct device *dev) +static int ntb_transport_bus_remove(struct device *dev) { - const struct ntb_client *drv = container_of(dev->driver, - struct ntb_client, driver); - struct pci_dev *pdev = container_of(dev->parent, struct pci_dev, dev); + const struct ntb_transport_client *client; - if (drv && drv->remove) - drv->remove(pdev); + client = drv_client(dev->driver); + client->remove(dev); put_device(dev); return 0; } -static struct bus_type ntb_bus_type = { - .name = "ntb_bus", - .match = ntb_match_bus, - .probe = ntb_client_probe, - .remove = ntb_client_remove, +static struct bus_type ntb_transport_bus = { + .name = "ntb_transport", + .match = ntb_transport_bus_match, + .probe = ntb_transport_bus_probe, + .remove = ntb_transport_bus_remove, }; static LIST_HEAD(ntb_transport_list); -static int ntb_bus_init(struct ntb_transport *nt) +static int ntb_bus_init(struct ntb_transport_ctx *nt) { - if (list_empty(&ntb_transport_list)) { - int rc = bus_register(&ntb_bus_type); - if (rc) - return rc; - } - list_add(&nt->entry, &ntb_transport_list); - return 0; } -static void ntb_bus_remove(struct ntb_transport *nt) +static void ntb_bus_remove(struct ntb_transport_ctx *nt) { struct ntb_transport_client_dev *client_dev, *cd; @@ -273,29 +310,26 @@ static void ntb_bus_remove(struct ntb_transport *nt) } list_del(&nt->entry); - - if (list_empty(&ntb_transport_list)) - bus_unregister(&ntb_bus_type); } -static void ntb_client_release(struct device *dev) +static void ntb_transport_client_release(struct device *dev) { struct ntb_transport_client_dev *client_dev; - client_dev = container_of(dev, struct ntb_transport_client_dev, dev); + client_dev = dev_client_dev(dev); kfree(client_dev); } /** - * ntb_unregister_client_dev - Unregister NTB client device + * ntb_transport_unregister_client_dev - Unregister NTB client device * @device_name: Name of NTB client device * * Unregister an NTB client device with the NTB transport layer */ -void ntb_unregister_client_dev(char *device_name) +void ntb_transport_unregister_client_dev(char *device_name) { struct ntb_transport_client_dev *client, *cd; - struct ntb_transport *nt; + struct ntb_transport_ctx *nt; list_for_each_entry(nt, &ntb_transport_list, entry) list_for_each_entry_safe(client, cd, &nt->client_devs, entry) @@ -305,18 +339,19 @@ void ntb_unregister_client_dev(char *device_name) device_unregister(&client->dev); } } -EXPORT_SYMBOL_GPL(ntb_unregister_client_dev); +EXPORT_SYMBOL_GPL(ntb_transport_unregister_client_dev); /** - * ntb_register_client_dev - Register NTB client device + * ntb_transport_register_client_dev - Register NTB client device * @device_name: Name of NTB client device * * Register an NTB client device with the NTB transport layer */ -int ntb_register_client_dev(char *device_name) +int ntb_transport_register_client_dev(char *device_name) { struct ntb_transport_client_dev *client_dev; - struct ntb_transport *nt; + struct ntb_transport_ctx *nt; + int node; int rc, i = 0; if (list_empty(&ntb_transport_list)) @@ -325,8 +360,10 @@ int ntb_register_client_dev(char *device_name) list_for_each_entry(nt, &ntb_transport_list, entry) { struct device *dev; - client_dev = kzalloc(sizeof(struct ntb_transport_client_dev), - GFP_KERNEL); + node = dev_to_node(&nt->ndev->dev); + + client_dev = kzalloc_node(sizeof(*client_dev), + GFP_KERNEL, node); if (!client_dev) { rc = -ENOMEM; goto err; @@ -336,9 +373,9 @@ int ntb_register_client_dev(char *device_name) /* setup and register client devices */ dev_set_name(dev, "%s%d", device_name, i); - dev->bus = &ntb_bus_type; - dev->release = ntb_client_release; - dev->parent = &ntb_query_pdev(nt->ndev)->dev; + dev->bus = &ntb_transport_bus; + dev->release = ntb_transport_client_release; + dev->parent = &nt->ndev->dev; rc = device_register(dev); if (rc) { @@ -353,44 +390,44 @@ int ntb_register_client_dev(char *device_name) return 0; err: - ntb_unregister_client_dev(device_name); + ntb_transport_unregister_client_dev(device_name); return rc; } -EXPORT_SYMBOL_GPL(ntb_register_client_dev); +EXPORT_SYMBOL_GPL(ntb_transport_register_client_dev); /** - * ntb_register_client - Register NTB client driver + * ntb_transport_register_client - Register NTB client driver * @drv: NTB client driver to be registered * * Register an NTB client driver with the NTB transport layer * * RETURNS: An appropriate -ERRNO error value on error, or zero for success. */ -int ntb_register_client(struct ntb_client *drv) +int ntb_transport_register_client(struct ntb_transport_client *drv) { - drv->driver.bus = &ntb_bus_type; + drv->driver.bus = &ntb_transport_bus; if (list_empty(&ntb_transport_list)) return -ENODEV; return driver_register(&drv->driver); } -EXPORT_SYMBOL_GPL(ntb_register_client); +EXPORT_SYMBOL_GPL(ntb_transport_register_client); /** - * ntb_unregister_client - Unregister NTB client driver + * ntb_transport_unregister_client - Unregister NTB client driver * @drv: NTB client driver to be unregistered * * Unregister an NTB client driver with the NTB transport layer * * RETURNS: An appropriate -ERRNO error value on error, or zero for success. */ -void ntb_unregister_client(struct ntb_client *drv) +void ntb_transport_unregister_client(struct ntb_transport_client *drv) { driver_unregister(&drv->driver); } -EXPORT_SYMBOL_GPL(ntb_unregister_client); +EXPORT_SYMBOL_GPL(ntb_transport_unregister_client); static ssize_t debugfs_read(struct file *filp, char __user *ubuf, size_t count, loff_t *offp) @@ -452,8 +489,8 @@ static ssize_t debugfs_read(struct file *filp, char __user *ubuf, size_t count, "tx_max_entry - \t%u\n", qp->tx_max_entry); out_offset += snprintf(buf + out_offset, out_count - out_offset, - "\nQP Link %s\n", (qp->qp_link == NTB_LINK_UP) ? - "Up" : "Down"); + "\nQP Link %s\n", + qp->link_is_up ? "Up" : "Down"); if (out_offset > out_count) out_offset = out_count; @@ -497,26 +534,31 @@ out: return entry; } -static void ntb_transport_setup_qp_mw(struct ntb_transport *nt, - unsigned int qp_num) +static int ntb_transport_setup_qp_mw(struct ntb_transport_ctx *nt, + unsigned int qp_num) { - struct ntb_transport_qp *qp = &nt->qps[qp_num]; + struct ntb_transport_qp *qp = &nt->qp_vec[qp_num]; + struct ntb_transport_mw *mw; unsigned int rx_size, num_qps_mw; - u8 mw_num, mw_max; + unsigned int mw_num, mw_count, qp_count; unsigned int i; - mw_max = ntb_max_mw(nt->ndev); - mw_num = QP_TO_MW(nt->ndev, qp_num); + mw_count = nt->mw_count; + qp_count = nt->qp_count; + + mw_num = QP_TO_MW(nt, qp_num); + mw = &nt->mw_vec[mw_num]; - WARN_ON(nt->mw[mw_num].virt_addr == NULL); + if (!mw->virt_addr) + return -ENOMEM; - if (nt->max_qps % mw_max && mw_num + 1 < nt->max_qps / mw_max) - num_qps_mw = nt->max_qps / mw_max + 1; + if (qp_count % mw_count && mw_num + 1 < qp_count / mw_count) + num_qps_mw = qp_count / mw_count + 1; else - num_qps_mw = nt->max_qps / mw_max; + num_qps_mw = qp_count / mw_count; - rx_size = (unsigned int) nt->mw[mw_num].size / num_qps_mw; - qp->rx_buff = nt->mw[mw_num].virt_addr + qp_num / mw_max * rx_size; + rx_size = (unsigned int)mw->xlat_size / num_qps_mw; + qp->rx_buff = mw->virt_addr + rx_size * qp_num / mw_count; rx_size -= sizeof(struct ntb_rx_info); qp->remote_rx_info = qp->rx_buff + rx_size; @@ -530,49 +572,63 @@ static void ntb_transport_setup_qp_mw(struct ntb_transport *nt, /* setup the hdr offsets with 0's */ for (i = 0; i < qp->rx_max_entry; i++) { - void *offset = qp->rx_buff + qp->rx_max_frame * (i + 1) - - sizeof(struct ntb_payload_header); + void *offset = (qp->rx_buff + qp->rx_max_frame * (i + 1) - + sizeof(struct ntb_payload_header)); memset(offset, 0, sizeof(struct ntb_payload_header)); } qp->rx_pkts = 0; qp->tx_pkts = 0; qp->tx_index = 0; + + return 0; } -static void ntb_free_mw(struct ntb_transport *nt, int num_mw) +static void ntb_free_mw(struct ntb_transport_ctx *nt, int num_mw) { - struct ntb_transport_mw *mw = &nt->mw[num_mw]; - struct pci_dev *pdev = ntb_query_pdev(nt->ndev); + struct ntb_transport_mw *mw = &nt->mw_vec[num_mw]; + struct pci_dev *pdev = nt->ndev->pdev; if (!mw->virt_addr) return; - dma_free_coherent(&pdev->dev, mw->size, mw->virt_addr, mw->dma_addr); + ntb_mw_clear_trans(nt->ndev, num_mw); + dma_free_coherent(&pdev->dev, mw->buff_size, + mw->virt_addr, mw->dma_addr); + mw->xlat_size = 0; + mw->buff_size = 0; mw->virt_addr = NULL; } -static int ntb_set_mw(struct ntb_transport *nt, int num_mw, unsigned int size) +static int ntb_set_mw(struct ntb_transport_ctx *nt, int num_mw, + unsigned int size) { - struct ntb_transport_mw *mw = &nt->mw[num_mw]; - struct pci_dev *pdev = ntb_query_pdev(nt->ndev); + struct ntb_transport_mw *mw = &nt->mw_vec[num_mw]; + struct pci_dev *pdev = nt->ndev->pdev; + unsigned int xlat_size, buff_size; + int rc; + + xlat_size = round_up(size, mw->xlat_align_size); + buff_size = round_up(size, mw->xlat_align); /* No need to re-setup */ - if (mw->size == ALIGN(size, 4096)) + if (mw->xlat_size == xlat_size) return 0; - if (mw->size != 0) + if (mw->buff_size) ntb_free_mw(nt, num_mw); - /* Alloc memory for receiving data. Must be 4k aligned */ - mw->size = ALIGN(size, 4096); + /* Alloc memory for receiving data. Must be aligned */ + mw->xlat_size = xlat_size; + mw->buff_size = buff_size; - mw->virt_addr = dma_alloc_coherent(&pdev->dev, mw->size, &mw->dma_addr, - GFP_KERNEL); + mw->virt_addr = dma_alloc_coherent(&pdev->dev, buff_size, + &mw->dma_addr, GFP_KERNEL); if (!mw->virt_addr) { - mw->size = 0; - dev_err(&pdev->dev, "Unable to allocate MW buffer of size %d\n", - (int) mw->size); + mw->xlat_size = 0; + mw->buff_size = 0; + dev_err(&pdev->dev, "Unable to alloc MW buff of size %d\n", + buff_size); return -ENOMEM; } @@ -582,34 +638,58 @@ static int ntb_set_mw(struct ntb_transport *nt, int num_mw, unsigned int size) * is a requirement of the hardware. It is recommended to setup CMA * for BAR sizes equal or greater than 4MB. */ - if (!IS_ALIGNED(mw->dma_addr, mw->size)) { - dev_err(&pdev->dev, "DMA memory %pad not aligned to BAR size\n", + if (!IS_ALIGNED(mw->dma_addr, mw->xlat_align)) { + dev_err(&pdev->dev, "DMA memory %pad is not aligned\n", &mw->dma_addr); ntb_free_mw(nt, num_mw); return -ENOMEM; } /* Notify HW the memory location of the receive buffer */ - ntb_set_mw_addr(nt->ndev, num_mw, mw->dma_addr); + rc = ntb_mw_set_trans(nt->ndev, num_mw, mw->dma_addr, mw->xlat_size); + if (rc) { + dev_err(&pdev->dev, "Unable to set mw%d translation", num_mw); + ntb_free_mw(nt, num_mw); + return -EIO; + } return 0; } +static void ntb_qp_link_down_reset(struct ntb_transport_qp *qp) +{ + qp->link_is_up = false; + + qp->tx_index = 0; + qp->rx_index = 0; + qp->rx_bytes = 0; + qp->rx_pkts = 0; + qp->rx_ring_empty = 0; + qp->rx_err_no_buf = 0; + qp->rx_err_oflow = 0; + qp->rx_err_ver = 0; + qp->rx_memcpy = 0; + qp->rx_async = 0; + qp->tx_bytes = 0; + qp->tx_pkts = 0; + qp->tx_ring_full = 0; + qp->tx_err_no_buf = 0; + qp->tx_memcpy = 0; + qp->tx_async = 0; +} + static void ntb_qp_link_cleanup(struct ntb_transport_qp *qp) { - struct ntb_transport *nt = qp->transport; - struct pci_dev *pdev = ntb_query_pdev(nt->ndev); + struct ntb_transport_ctx *nt = qp->transport; + struct pci_dev *pdev = nt->ndev->pdev; - if (qp->qp_link == NTB_LINK_DOWN) { - cancel_delayed_work_sync(&qp->link_work); - return; - } + dev_info(&pdev->dev, "qp %d: Link Cleanup\n", qp->qp_num); - if (qp->event_handler) - qp->event_handler(qp->cb_data, NTB_LINK_DOWN); + cancel_delayed_work_sync(&qp->link_work); + ntb_qp_link_down_reset(qp); - dev_info(&pdev->dev, "qp %d: Link Down\n", qp->qp_num); - qp->qp_link = NTB_LINK_DOWN; + if (qp->event_handler) + qp->event_handler(qp->cb_data, qp->link_is_up); } static void ntb_qp_link_cleanup_work(struct work_struct *work) @@ -617,11 +697,11 @@ static void ntb_qp_link_cleanup_work(struct work_struct *work) struct ntb_transport_qp *qp = container_of(work, struct ntb_transport_qp, link_cleanup); - struct ntb_transport *nt = qp->transport; + struct ntb_transport_ctx *nt = qp->transport; ntb_qp_link_cleanup(qp); - if (nt->transport_link == NTB_LINK_UP) + if (nt->link_is_up) schedule_delayed_work(&qp->link_work, msecs_to_jiffies(NTB_LINK_DOWN_TIMEOUT)); } @@ -631,180 +711,132 @@ static void ntb_qp_link_down(struct ntb_transport_qp *qp) schedule_work(&qp->link_cleanup); } -static void ntb_transport_link_cleanup(struct ntb_transport *nt) +static void ntb_transport_link_cleanup(struct ntb_transport_ctx *nt) { + struct ntb_transport_qp *qp; + u64 qp_bitmap_alloc; int i; + qp_bitmap_alloc = nt->qp_bitmap & ~nt->qp_bitmap_free; + /* Pass along the info to any clients */ - for (i = 0; i < nt->max_qps; i++) - if (!test_bit(i, &nt->qp_bitmap)) - ntb_qp_link_cleanup(&nt->qps[i]); + for (i = 0; i < nt->qp_count; i++) + if (qp_bitmap_alloc & BIT_ULL(i)) { + qp = &nt->qp_vec[i]; + ntb_qp_link_cleanup(qp); + cancel_work_sync(&qp->link_cleanup); + cancel_delayed_work_sync(&qp->link_work); + } - if (nt->transport_link == NTB_LINK_DOWN) + if (!nt->link_is_up) cancel_delayed_work_sync(&nt->link_work); - else - nt->transport_link = NTB_LINK_DOWN; /* The scratchpad registers keep the values if the remote side * goes down, blast them now to give them a sane value the next * time they are accessed */ for (i = 0; i < MAX_SPAD; i++) - ntb_write_local_spad(nt->ndev, i, 0); + ntb_spad_write(nt->ndev, i, 0); } static void ntb_transport_link_cleanup_work(struct work_struct *work) { - struct ntb_transport *nt = container_of(work, struct ntb_transport, - link_cleanup); + struct ntb_transport_ctx *nt = + container_of(work, struct ntb_transport_ctx, link_cleanup); ntb_transport_link_cleanup(nt); } -static void ntb_transport_event_callback(void *data, enum ntb_hw_event event) +static void ntb_transport_event_callback(void *data) { - struct ntb_transport *nt = data; + struct ntb_transport_ctx *nt = data; - switch (event) { - case NTB_EVENT_HW_LINK_UP: + if (ntb_link_is_up(nt->ndev, NULL, NULL) == 1) schedule_delayed_work(&nt->link_work, 0); - break; - case NTB_EVENT_HW_LINK_DOWN: + else schedule_work(&nt->link_cleanup); - break; - default: - BUG(); - } } static void ntb_transport_link_work(struct work_struct *work) { - struct ntb_transport *nt = container_of(work, struct ntb_transport, - link_work.work); - struct ntb_device *ndev = nt->ndev; - struct pci_dev *pdev = ntb_query_pdev(ndev); + struct ntb_transport_ctx *nt = + container_of(work, struct ntb_transport_ctx, link_work.work); + struct ntb_dev *ndev = nt->ndev; + struct pci_dev *pdev = ndev->pdev; + resource_size_t size; u32 val; - int rc, i; + int rc, i, spad; /* send the local info, in the opposite order of the way we read it */ - for (i = 0; i < ntb_max_mw(ndev); i++) { - rc = ntb_write_remote_spad(ndev, MW0_SZ_HIGH + (i * 2), - ntb_get_mw_size(ndev, i) >> 32); - if (rc) { - dev_err(&pdev->dev, "Error writing %u to remote spad %d\n", - (u32)(ntb_get_mw_size(ndev, i) >> 32), - MW0_SZ_HIGH + (i * 2)); - goto out; - } + for (i = 0; i < nt->mw_count; i++) { + size = nt->mw_vec[i].phys_size; - rc = ntb_write_remote_spad(ndev, MW0_SZ_LOW + (i * 2), - (u32) ntb_get_mw_size(ndev, i)); - if (rc) { - dev_err(&pdev->dev, "Error writing %u to remote spad %d\n", - (u32) ntb_get_mw_size(ndev, i), - MW0_SZ_LOW + (i * 2)); - goto out; - } - } + if (max_mw_size && size > max_mw_size) + size = max_mw_size; - rc = ntb_write_remote_spad(ndev, NUM_MWS, ntb_max_mw(ndev)); - if (rc) { - dev_err(&pdev->dev, "Error writing %x to remote spad %d\n", - ntb_max_mw(ndev), NUM_MWS); - goto out; - } + spad = MW0_SZ_HIGH + (i * 2); + ntb_peer_spad_write(ndev, spad, (u32)(size >> 32)); - rc = ntb_write_remote_spad(ndev, NUM_QPS, nt->max_qps); - if (rc) { - dev_err(&pdev->dev, "Error writing %x to remote spad %d\n", - nt->max_qps, NUM_QPS); - goto out; + spad = MW0_SZ_LOW + (i * 2); + ntb_peer_spad_write(ndev, spad, (u32)size); } - rc = ntb_write_remote_spad(ndev, VERSION, NTB_TRANSPORT_VERSION); - if (rc) { - dev_err(&pdev->dev, "Error writing %x to remote spad %d\n", - NTB_TRANSPORT_VERSION, VERSION); - goto out; - } + ntb_peer_spad_write(ndev, NUM_MWS, nt->mw_count); - /* Query the remote side for its info */ - rc = ntb_read_remote_spad(ndev, VERSION, &val); - if (rc) { - dev_err(&pdev->dev, "Error reading remote spad %d\n", VERSION); - goto out; - } + ntb_peer_spad_write(ndev, NUM_QPS, nt->qp_count); - if (val != NTB_TRANSPORT_VERSION) - goto out; - dev_dbg(&pdev->dev, "Remote version = %d\n", val); + ntb_peer_spad_write(ndev, VERSION, NTB_TRANSPORT_VERSION); - rc = ntb_read_remote_spad(ndev, NUM_QPS, &val); - if (rc) { - dev_err(&pdev->dev, "Error reading remote spad %d\n", NUM_QPS); + /* Query the remote side for its info */ + val = ntb_spad_read(ndev, VERSION); + dev_dbg(&pdev->dev, "Remote version = %d\n", val); + if (val != NTB_TRANSPORT_VERSION) goto out; - } - if (val != nt->max_qps) - goto out; + val = ntb_spad_read(ndev, NUM_QPS); dev_dbg(&pdev->dev, "Remote max number of qps = %d\n", val); - - rc = ntb_read_remote_spad(ndev, NUM_MWS, &val); - if (rc) { - dev_err(&pdev->dev, "Error reading remote spad %d\n", NUM_MWS); + if (val != nt->qp_count) goto out; - } - if (val != ntb_max_mw(ndev)) - goto out; + val = ntb_spad_read(ndev, NUM_MWS); dev_dbg(&pdev->dev, "Remote number of mws = %d\n", val); + if (val != nt->mw_count) + goto out; - for (i = 0; i < ntb_max_mw(ndev); i++) { + for (i = 0; i < nt->mw_count; i++) { u64 val64; - rc = ntb_read_remote_spad(ndev, MW0_SZ_HIGH + (i * 2), &val); - if (rc) { - dev_err(&pdev->dev, "Error reading remote spad %d\n", - MW0_SZ_HIGH + (i * 2)); - goto out1; - } - - val64 = (u64) val << 32; - - rc = ntb_read_remote_spad(ndev, MW0_SZ_LOW + (i * 2), &val); - if (rc) { - dev_err(&pdev->dev, "Error reading remote spad %d\n", - MW0_SZ_LOW + (i * 2)); - goto out1; - } + val = ntb_spad_read(ndev, MW0_SZ_HIGH + (i * 2)); + val64 = (u64)val << 32; + val = ntb_spad_read(ndev, MW0_SZ_LOW + (i * 2)); val64 |= val; - dev_dbg(&pdev->dev, "Remote MW%d size = %llu\n", i, val64); + dev_dbg(&pdev->dev, "Remote MW%d size = %#llx\n", i, val64); rc = ntb_set_mw(nt, i, val64); if (rc) goto out1; } - nt->transport_link = NTB_LINK_UP; + nt->link_is_up = true; - for (i = 0; i < nt->max_qps; i++) { - struct ntb_transport_qp *qp = &nt->qps[i]; + for (i = 0; i < nt->qp_count; i++) { + struct ntb_transport_qp *qp = &nt->qp_vec[i]; ntb_transport_setup_qp_mw(nt, i); - if (qp->client_ready == NTB_LINK_UP) + if (qp->client_ready) schedule_delayed_work(&qp->link_work, 0); } return; out1: - for (i = 0; i < ntb_max_mw(ndev); i++) + for (i = 0; i < nt->mw_count; i++) ntb_free_mw(nt, i); out: - if (ntb_hw_link_status(ndev)) + if (ntb_link_is_up(ndev, NULL, NULL) == 1) schedule_delayed_work(&nt->link_work, msecs_to_jiffies(NTB_LINK_DOWN_TIMEOUT)); } @@ -814,73 +846,73 @@ static void ntb_qp_link_work(struct work_struct *work) struct ntb_transport_qp *qp = container_of(work, struct ntb_transport_qp, link_work.work); - struct pci_dev *pdev = ntb_query_pdev(qp->ndev); - struct ntb_transport *nt = qp->transport; - int rc, val; + struct pci_dev *pdev = qp->ndev->pdev; + struct ntb_transport_ctx *nt = qp->transport; + int val; - WARN_ON(nt->transport_link != NTB_LINK_UP); + WARN_ON(!nt->link_is_up); - rc = ntb_read_local_spad(nt->ndev, QP_LINKS, &val); - if (rc) { - dev_err(&pdev->dev, "Error reading spad %d\n", QP_LINKS); - return; - } + val = ntb_spad_read(nt->ndev, QP_LINKS); - rc = ntb_write_remote_spad(nt->ndev, QP_LINKS, val | 1 << qp->qp_num); - if (rc) - dev_err(&pdev->dev, "Error writing %x to remote spad %d\n", - val | 1 << qp->qp_num, QP_LINKS); + ntb_peer_spad_write(nt->ndev, QP_LINKS, val | BIT(qp->qp_num)); /* query remote spad for qp ready bits */ - rc = ntb_read_remote_spad(nt->ndev, QP_LINKS, &val); - if (rc) - dev_err(&pdev->dev, "Error reading remote spad %d\n", QP_LINKS); - - dev_dbg(&pdev->dev, "Remote QP link status = %x\n", val); + ntb_peer_spad_read(nt->ndev, QP_LINKS); + dev_dbg_ratelimited(&pdev->dev, "Remote QP link status = %x\n", val); /* See if the remote side is up */ - if (1 << qp->qp_num & val) { - qp->qp_link = NTB_LINK_UP; - + if (val & BIT(qp->qp_num)) { dev_info(&pdev->dev, "qp %d: Link Up\n", qp->qp_num); + qp->link_is_up = true; + if (qp->event_handler) - qp->event_handler(qp->cb_data, NTB_LINK_UP); - } else if (nt->transport_link == NTB_LINK_UP) + qp->event_handler(qp->cb_data, qp->link_is_up); + } else if (nt->link_is_up) schedule_delayed_work(&qp->link_work, msecs_to_jiffies(NTB_LINK_DOWN_TIMEOUT)); } -static int ntb_transport_init_queue(struct ntb_transport *nt, +static int ntb_transport_init_queue(struct ntb_transport_ctx *nt, unsigned int qp_num) { struct ntb_transport_qp *qp; + struct ntb_transport_mw *mw; + phys_addr_t mw_base; + resource_size_t mw_size; unsigned int num_qps_mw, tx_size; - u8 mw_num, mw_max; + unsigned int mw_num, mw_count, qp_count; u64 qp_offset; - mw_max = ntb_max_mw(nt->ndev); - mw_num = QP_TO_MW(nt->ndev, qp_num); + mw_count = nt->mw_count; + qp_count = nt->qp_count; - qp = &nt->qps[qp_num]; + mw_num = QP_TO_MW(nt, qp_num); + mw = &nt->mw_vec[mw_num]; + + qp = &nt->qp_vec[qp_num]; qp->qp_num = qp_num; qp->transport = nt; qp->ndev = nt->ndev; - qp->qp_link = NTB_LINK_DOWN; - qp->client_ready = NTB_LINK_DOWN; + qp->client_ready = false; qp->event_handler = NULL; + ntb_qp_link_down_reset(qp); - if (nt->max_qps % mw_max && mw_num + 1 < nt->max_qps / mw_max) - num_qps_mw = nt->max_qps / mw_max + 1; + if (qp_count % mw_count && mw_num + 1 < qp_count / mw_count) + num_qps_mw = qp_count / mw_count + 1; else - num_qps_mw = nt->max_qps / mw_max; + num_qps_mw = qp_count / mw_count; + + mw_base = nt->mw_vec[mw_num].phys_addr; + mw_size = nt->mw_vec[mw_num].phys_size; - tx_size = (unsigned int) ntb_get_mw_size(qp->ndev, mw_num) / num_qps_mw; - qp_offset = qp_num / mw_max * tx_size; - qp->tx_mw = ntb_get_mw_vbase(nt->ndev, mw_num) + qp_offset; + tx_size = (unsigned int)mw_size / num_qps_mw; + qp_offset = tx_size * qp_num / mw_count; + + qp->tx_mw = nt->mw_vec[mw_num].vbase + qp_offset; if (!qp->tx_mw) return -EINVAL; - qp->tx_mw_phys = ntb_get_mw_base(qp->ndev, mw_num) + qp_offset; + qp->tx_mw_phys = mw_base + qp_offset; if (!qp->tx_mw_phys) return -EINVAL; @@ -891,16 +923,19 @@ static int ntb_transport_init_queue(struct ntb_transport *nt, qp->tx_max_frame = min(transport_mtu, tx_size / 2); qp->tx_max_entry = tx_size / qp->tx_max_frame; - if (ntb_query_debugfs(nt->ndev)) { + if (nt_debugfs_dir) { char debugfs_name[4]; snprintf(debugfs_name, 4, "qp%d", qp_num); qp->debugfs_dir = debugfs_create_dir(debugfs_name, - ntb_query_debugfs(nt->ndev)); + nt_debugfs_dir); qp->debugfs_stats = debugfs_create_file("stats", S_IRUSR, qp->debugfs_dir, qp, &ntb_qp_debugfs_stats); + } else { + qp->debugfs_dir = NULL; + qp->debugfs_stats = NULL; } INIT_DELAYED_WORK(&qp->link_work, ntb_qp_link_work); @@ -914,46 +949,89 @@ static int ntb_transport_init_queue(struct ntb_transport *nt, INIT_LIST_HEAD(&qp->rx_free_q); INIT_LIST_HEAD(&qp->tx_free_q); + tasklet_init(&qp->rxc_db_work, ntb_transport_rxc_db, + (unsigned long)qp); + return 0; } -int ntb_transport_init(struct pci_dev *pdev) +static int ntb_transport_probe(struct ntb_client *self, struct ntb_dev *ndev) { - struct ntb_transport *nt; + struct ntb_transport_ctx *nt; + struct ntb_transport_mw *mw; + unsigned int mw_count, qp_count; + u64 qp_bitmap; + int node; int rc, i; - nt = kzalloc(sizeof(struct ntb_transport), GFP_KERNEL); + if (ntb_db_is_unsafe(ndev)) + dev_dbg(&ndev->dev, + "doorbell is unsafe, proceed anyway...\n"); + if (ntb_spad_is_unsafe(ndev)) + dev_dbg(&ndev->dev, + "scratchpad is unsafe, proceed anyway...\n"); + + node = dev_to_node(&ndev->dev); + + nt = kzalloc_node(sizeof(*nt), GFP_KERNEL, node); if (!nt) return -ENOMEM; - nt->ndev = ntb_register_transport(pdev, nt); - if (!nt->ndev) { - rc = -EIO; + nt->ndev = ndev; + + mw_count = ntb_mw_count(ndev); + + nt->mw_count = mw_count; + + nt->mw_vec = kzalloc_node(mw_count * sizeof(*nt->mw_vec), + GFP_KERNEL, node); + if (!nt->mw_vec) { + rc = -ENOMEM; goto err; } - nt->mw = kcalloc(ntb_max_mw(nt->ndev), sizeof(struct ntb_transport_mw), - GFP_KERNEL); - if (!nt->mw) { - rc = -ENOMEM; - goto err1; + for (i = 0; i < mw_count; i++) { + mw = &nt->mw_vec[i]; + + rc = ntb_mw_get_range(ndev, i, &mw->phys_addr, &mw->phys_size, + &mw->xlat_align, &mw->xlat_align_size); + if (rc) + goto err1; + + mw->vbase = ioremap_wc(mw->phys_addr, mw->phys_size); + if (!mw->vbase) { + rc = -ENOMEM; + goto err1; + } + + mw->buff_size = 0; + mw->xlat_size = 0; + mw->virt_addr = NULL; + mw->dma_addr = 0; } - if (max_num_clients) - nt->max_qps = min(ntb_max_cbs(nt->ndev), max_num_clients); - else - nt->max_qps = min(ntb_max_cbs(nt->ndev), ntb_max_mw(nt->ndev)); + qp_bitmap = ntb_db_valid_mask(ndev); + + qp_count = ilog2(qp_bitmap); + if (max_num_clients && max_num_clients < qp_count) + qp_count = max_num_clients; + else if (mw_count < qp_count) + qp_count = mw_count; + + qp_bitmap &= BIT_ULL(qp_count) - 1; - nt->qps = kcalloc(nt->max_qps, sizeof(struct ntb_transport_qp), - GFP_KERNEL); - if (!nt->qps) { + nt->qp_count = qp_count; + nt->qp_bitmap = qp_bitmap; + nt->qp_bitmap_free = qp_bitmap; + + nt->qp_vec = kzalloc_node(qp_count * sizeof(*nt->qp_vec), + GFP_KERNEL, node); + if (!nt->qp_vec) { rc = -ENOMEM; goto err2; } - nt->qp_bitmap = ((u64) 1 << nt->max_qps) - 1; - - for (i = 0; i < nt->max_qps; i++) { + for (i = 0; i < qp_count; i++) { rc = ntb_transport_init_queue(nt, i); if (rc) goto err3; @@ -962,8 +1040,7 @@ int ntb_transport_init(struct pci_dev *pdev) INIT_DELAYED_WORK(&nt->link_work, ntb_transport_link_work); INIT_WORK(&nt->link_cleanup, ntb_transport_link_cleanup_work); - rc = ntb_register_event_callback(nt->ndev, - ntb_transport_event_callback); + rc = ntb_set_ctx(ndev, nt, &ntb_transport_ops); if (rc) goto err3; @@ -972,51 +1049,61 @@ int ntb_transport_init(struct pci_dev *pdev) if (rc) goto err4; - if (ntb_hw_link_status(nt->ndev)) - schedule_delayed_work(&nt->link_work, 0); + nt->link_is_up = false; + ntb_link_enable(ndev, NTB_SPEED_AUTO, NTB_WIDTH_AUTO); + ntb_link_event(ndev); return 0; err4: - ntb_unregister_event_callback(nt->ndev); + ntb_clear_ctx(ndev); err3: - kfree(nt->qps); + kfree(nt->qp_vec); err2: - kfree(nt->mw); + kfree(nt->mw_vec); err1: - ntb_unregister_transport(nt->ndev); + while (i--) { + mw = &nt->mw_vec[i]; + iounmap(mw->vbase); + } err: kfree(nt); return rc; } -void ntb_transport_free(void *transport) +static void ntb_transport_free(struct ntb_client *self, struct ntb_dev *ndev) { - struct ntb_transport *nt = transport; - struct ntb_device *ndev = nt->ndev; + struct ntb_transport_ctx *nt = ndev->ctx; + struct ntb_transport_qp *qp; + u64 qp_bitmap_alloc; int i; ntb_transport_link_cleanup(nt); + cancel_work_sync(&nt->link_cleanup); + cancel_delayed_work_sync(&nt->link_work); + + qp_bitmap_alloc = nt->qp_bitmap & ~nt->qp_bitmap_free; /* verify that all the qp's are freed */ - for (i = 0; i < nt->max_qps; i++) { - if (!test_bit(i, &nt->qp_bitmap)) - ntb_transport_free_queue(&nt->qps[i]); - debugfs_remove_recursive(nt->qps[i].debugfs_dir); + for (i = 0; i < nt->qp_count; i++) { + qp = &nt->qp_vec[i]; + if (qp_bitmap_alloc & BIT_ULL(i)) + ntb_transport_free_queue(qp); + debugfs_remove_recursive(qp->debugfs_dir); } - ntb_bus_remove(nt); + ntb_link_disable(ndev); + ntb_clear_ctx(ndev); - cancel_delayed_work_sync(&nt->link_work); - - ntb_unregister_event_callback(ndev); + ntb_bus_remove(nt); - for (i = 0; i < ntb_max_mw(ndev); i++) + for (i = nt->mw_count; i--; ) { ntb_free_mw(nt, i); + iounmap(nt->mw_vec[i].vbase); + } - kfree(nt->qps); - kfree(nt->mw); - ntb_unregister_transport(ndev); + kfree(nt->qp_vec); + kfree(nt->mw_vec); kfree(nt); } @@ -1028,15 +1115,13 @@ static void ntb_rx_copy_callback(void *data) unsigned int len = entry->len; struct ntb_payload_header *hdr = entry->rx_hdr; - /* Ensure that the data is fully copied out before clearing the flag */ - wmb(); hdr->flags = 0; iowrite32(entry->index, &qp->rx_info->entry); ntb_list_add(&qp->ntb_rx_free_q_lock, &entry->entry, &qp->rx_free_q); - if (qp->rx_handler && qp->client_ready == NTB_LINK_UP) + if (qp->rx_handler && qp->client_ready) qp->rx_handler(qp, qp->cb_data, cb_data, len); } @@ -1047,6 +1132,9 @@ static void ntb_memcpy_rx(struct ntb_queue_entry *entry, void *offset) memcpy(buf, offset, len); + /* Ensure that the data is fully copied out before clearing the flag */ + wmb(); + ntb_rx_copy_callback(entry); } @@ -1071,8 +1159,8 @@ static void ntb_async_rx(struct ntb_queue_entry *entry, void *offset, goto err_wait; device = chan->device; - pay_off = (size_t) offset & ~PAGE_MASK; - buff_off = (size_t) buf & ~PAGE_MASK; + pay_off = (size_t)offset & ~PAGE_MASK; + buff_off = (size_t)buf & ~PAGE_MASK; if (!is_dma_copy_aligned(device, pay_off, buff_off, len)) goto err_wait; @@ -1138,86 +1226,103 @@ static int ntb_process_rxc(struct ntb_transport_qp *qp) struct ntb_payload_header *hdr; struct ntb_queue_entry *entry; void *offset; + int rc; offset = qp->rx_buff + qp->rx_max_frame * qp->rx_index; hdr = offset + qp->rx_max_frame - sizeof(struct ntb_payload_header); - entry = ntb_list_rm(&qp->ntb_rx_pend_q_lock, &qp->rx_pend_q); - if (!entry) { - dev_dbg(&ntb_query_pdev(qp->ndev)->dev, - "no buffer - HDR ver %u, len %d, flags %x\n", - hdr->ver, hdr->len, hdr->flags); - qp->rx_err_no_buf++; - return -ENOMEM; - } + dev_dbg(&qp->ndev->pdev->dev, "qp %d: RX ver %u len %d flags %x\n", + qp->qp_num, hdr->ver, hdr->len, hdr->flags); if (!(hdr->flags & DESC_DONE_FLAG)) { - ntb_list_add(&qp->ntb_rx_pend_q_lock, &entry->entry, - &qp->rx_pend_q); + dev_dbg(&qp->ndev->pdev->dev, "done flag not set\n"); qp->rx_ring_empty++; return -EAGAIN; } - if (hdr->ver != (u32) qp->rx_pkts) { - dev_dbg(&ntb_query_pdev(qp->ndev)->dev, - "qp %d: version mismatch, expected %llu - got %u\n", - qp->qp_num, qp->rx_pkts, hdr->ver); - ntb_list_add(&qp->ntb_rx_pend_q_lock, &entry->entry, - &qp->rx_pend_q); + if (hdr->flags & LINK_DOWN_FLAG) { + dev_dbg(&qp->ndev->pdev->dev, "link down flag set\n"); + ntb_qp_link_down(qp); + hdr->flags = 0; + return -EAGAIN; + } + + if (hdr->ver != (u32)qp->rx_pkts) { + dev_dbg(&qp->ndev->pdev->dev, + "version mismatch, expected %llu - got %u\n", + qp->rx_pkts, hdr->ver); qp->rx_err_ver++; return -EIO; } - if (hdr->flags & LINK_DOWN_FLAG) { - ntb_qp_link_down(qp); + entry = ntb_list_rm(&qp->ntb_rx_pend_q_lock, &qp->rx_pend_q); + if (!entry) { + dev_dbg(&qp->ndev->pdev->dev, "no receive buffer\n"); + qp->rx_err_no_buf++; + rc = -ENOMEM; goto err; } - dev_dbg(&ntb_query_pdev(qp->ndev)->dev, - "rx offset %u, ver %u - %d payload received, buf size %d\n", - qp->rx_index, hdr->ver, hdr->len, entry->len); - - qp->rx_bytes += hdr->len; - qp->rx_pkts++; - if (hdr->len > entry->len) { - qp->rx_err_oflow++; - dev_dbg(&ntb_query_pdev(qp->ndev)->dev, - "RX overflow! Wanted %d got %d\n", + dev_dbg(&qp->ndev->pdev->dev, + "receive buffer overflow! Wanted %d got %d\n", hdr->len, entry->len); + qp->rx_err_oflow++; + rc = -EIO; goto err; } + dev_dbg(&qp->ndev->pdev->dev, + "RX OK index %u ver %u size %d into buf size %d\n", + qp->rx_index, hdr->ver, hdr->len, entry->len); + + qp->rx_bytes += hdr->len; + qp->rx_pkts++; + entry->index = qp->rx_index; entry->rx_hdr = hdr; ntb_async_rx(entry, offset, hdr->len); -out: qp->rx_index++; qp->rx_index %= qp->rx_max_entry; return 0; err: - ntb_list_add(&qp->ntb_rx_pend_q_lock, &entry->entry, &qp->rx_pend_q); - /* Ensure that the data is fully copied out before clearing the flag */ - wmb(); + /* FIXME: if this syncrhonous update of the rx_index gets ahead of + * asyncrhonous ntb_rx_copy_callback of previous entry, there are three + * scenarios: + * + * 1) The peer might miss this update, but observe the update + * from the memcpy completion callback. In this case, the buffer will + * not be freed on the peer to be reused for a different packet. The + * successful rx of a later packet would clear the condition, but the + * condition could persist if several rx fail in a row. + * + * 2) The peer may observe this update before the asyncrhonous copy of + * prior packets is completed. The peer may overwrite the buffers of + * the prior packets before they are copied. + * + * 3) Both: the peer may observe the update, and then observe the index + * decrement by the asynchronous completion callback. Who knows what + * badness that will cause. + */ hdr->flags = 0; iowrite32(qp->rx_index, &qp->rx_info->entry); - goto out; + return rc; } -static int ntb_transport_rxc_db(void *data, int db_num) +static void ntb_transport_rxc_db(unsigned long data) { - struct ntb_transport_qp *qp = data; + struct ntb_transport_qp *qp = (void *)data; int rc, i; - dev_dbg(&ntb_query_pdev(qp->ndev)->dev, "%s: doorbell %d received\n", - __func__, db_num); + dev_dbg(&qp->ndev->pdev->dev, "%s: doorbell %d received\n", + __func__, qp->qp_num); /* Limit the number of packets processed in a single interrupt to * provide fairness to others @@ -1231,7 +1336,21 @@ static int ntb_transport_rxc_db(void *data, int db_num) if (qp->dma_chan) dma_async_issue_pending(qp->dma_chan); - return i; + if (i == qp->rx_max_entry) { + /* there is more work to do */ + tasklet_schedule(&qp->rxc_db_work); + } else if (ntb_db_read(qp->ndev) & BIT_ULL(qp->qp_num)) { + /* the doorbell bit is set: clear it */ + ntb_db_clear(qp->ndev, BIT_ULL(qp->qp_num)); + /* ntb_db_read ensures ntb_db_clear write is committed */ + ntb_db_read(qp->ndev); + + /* an interrupt may have arrived between finishing + * ntb_process_rxc and clearing the doorbell bit: + * there might be some more work to do. + */ + tasklet_schedule(&qp->rxc_db_work); + } } static void ntb_tx_copy_callback(void *data) @@ -1240,11 +1359,9 @@ static void ntb_tx_copy_callback(void *data) struct ntb_transport_qp *qp = entry->qp; struct ntb_payload_header __iomem *hdr = entry->tx_hdr; - /* Ensure that the data is fully copied out before setting the flags */ - wmb(); iowrite32(entry->flags | DESC_DONE_FLAG, &hdr->flags); - ntb_ring_doorbell(qp->ndev, qp->qp_num); + ntb_peer_db_set(qp->ndev, BIT_ULL(qp->qp_num)); /* The entry length can only be zero if the packet is intended to be a * "link down" or similar. Since no payload is being sent in these @@ -1263,7 +1380,18 @@ static void ntb_tx_copy_callback(void *data) static void ntb_memcpy_tx(struct ntb_queue_entry *entry, void __iomem *offset) { +#ifdef ARCH_HAS_NOCACHE_UACCESS + /* + * Using non-temporal mov to improve performance on non-cached + * writes, even though we aren't actually copying from user space. + */ + __copy_from_user_inatomic_nocache(offset, entry->buf, entry->len); +#else memcpy_toio(offset, entry->buf, entry->len); +#endif + + /* Ensure that the data is fully copied out before setting the flags */ + wmb(); ntb_tx_copy_callback(entry); } @@ -1288,7 +1416,7 @@ static void ntb_async_tx(struct ntb_transport_qp *qp, entry->tx_hdr = hdr; iowrite32(entry->len, &hdr->len); - iowrite32((u32) qp->tx_pkts, &hdr->ver); + iowrite32((u32)qp->tx_pkts, &hdr->ver); if (!chan) goto err; @@ -1298,8 +1426,8 @@ static void ntb_async_tx(struct ntb_transport_qp *qp, device = chan->device; dest = qp->tx_mw_phys + qp->tx_max_frame * qp->tx_index; - buff_off = (size_t) buf & ~PAGE_MASK; - dest_off = (size_t) dest & ~PAGE_MASK; + buff_off = (size_t)buf & ~PAGE_MASK; + dest_off = (size_t)dest & ~PAGE_MASK; if (!is_dma_copy_aligned(device, buff_off, dest_off, len)) goto err; @@ -1347,9 +1475,6 @@ err: static int ntb_process_tx(struct ntb_transport_qp *qp, struct ntb_queue_entry *entry) { - dev_dbg(&ntb_query_pdev(qp->ndev)->dev, "%lld - tx %u, entry len %d flags %x buff %p\n", - qp->tx_pkts, qp->tx_index, entry->len, entry->flags, - entry->buf); if (qp->tx_index == qp->remote_rx_info->entry) { qp->tx_ring_full++; return -EAGAIN; @@ -1376,15 +1501,14 @@ static int ntb_process_tx(struct ntb_transport_qp *qp, static void ntb_send_link_down(struct ntb_transport_qp *qp) { - struct pci_dev *pdev = ntb_query_pdev(qp->ndev); + struct pci_dev *pdev = qp->ndev->pdev; struct ntb_queue_entry *entry; int i, rc; - if (qp->qp_link == NTB_LINK_DOWN) + if (!qp->link_is_up) return; - qp->qp_link = NTB_LINK_DOWN; - dev_info(&pdev->dev, "qp %d: Link Down\n", qp->qp_num); + dev_info(&pdev->dev, "qp %d: Send Link Down\n", qp->qp_num); for (i = 0; i < NTB_LINK_DOWN_TIMEOUT; i++) { entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q); @@ -1405,6 +1529,13 @@ static void ntb_send_link_down(struct ntb_transport_qp *qp) if (rc) dev_err(&pdev->dev, "ntb: QP%d unable to send linkdown msg\n", qp->qp_num); + + ntb_qp_link_down_reset(qp); +} + +static bool ntb_dma_filter_fn(struct dma_chan *chan, void *node) +{ + return dev_to_node(&chan->dev->device) == (int)(unsigned long)node; } /** @@ -1422,18 +1553,25 @@ static void ntb_send_link_down(struct ntb_transport_qp *qp) * RETURNS: pointer to newly created ntb_queue, NULL on error. */ struct ntb_transport_qp * -ntb_transport_create_queue(void *data, struct pci_dev *pdev, +ntb_transport_create_queue(void *data, struct device *client_dev, const struct ntb_queue_handlers *handlers) { + struct ntb_dev *ndev; + struct pci_dev *pdev; + struct ntb_transport_ctx *nt; struct ntb_queue_entry *entry; struct ntb_transport_qp *qp; - struct ntb_transport *nt; + u64 qp_bit; unsigned int free_queue; - int rc, i; + dma_cap_mask_t dma_mask; + int node; + int i; - nt = ntb_find_transport(pdev); - if (!nt) - goto err; + ndev = dev_ntb(client_dev->parent); + pdev = ndev->pdev; + nt = ndev->ctx; + + node = dev_to_node(&ndev->dev); free_queue = ffs(nt->qp_bitmap); if (!free_queue) @@ -1442,23 +1580,31 @@ ntb_transport_create_queue(void *data, struct pci_dev *pdev, /* decrement free_queue to make it zero based */ free_queue--; - clear_bit(free_queue, &nt->qp_bitmap); + qp = &nt->qp_vec[free_queue]; + qp_bit = BIT_ULL(qp->qp_num); + + nt->qp_bitmap_free &= ~qp_bit; - qp = &nt->qps[free_queue]; qp->cb_data = data; qp->rx_handler = handlers->rx_handler; qp->tx_handler = handlers->tx_handler; qp->event_handler = handlers->event_handler; - dmaengine_get(); - qp->dma_chan = dma_find_channel(DMA_MEMCPY); - if (!qp->dma_chan) { - dmaengine_put(); - dev_info(&pdev->dev, "Unable to allocate DMA channel, using CPU instead\n"); + dma_cap_zero(dma_mask); + dma_cap_set(DMA_MEMCPY, dma_mask); + + if (use_dma) { + qp->dma_chan = dma_request_channel(dma_mask, ntb_dma_filter_fn, + (void *)(unsigned long)node); + if (!qp->dma_chan) + dev_info(&pdev->dev, "Unable to allocate DMA channel\n"); + } else { + qp->dma_chan = NULL; } + dev_dbg(&pdev->dev, "Using %s memcpy\n", qp->dma_chan ? "DMA" : "CPU"); for (i = 0; i < NTB_QP_DEF_NUM_ENTRIES; i++) { - entry = kzalloc(sizeof(struct ntb_queue_entry), GFP_ATOMIC); + entry = kzalloc_node(sizeof(*entry), GFP_ATOMIC, node); if (!entry) goto err1; @@ -1468,7 +1614,7 @@ ntb_transport_create_queue(void *data, struct pci_dev *pdev, } for (i = 0; i < NTB_QP_DEF_NUM_ENTRIES; i++) { - entry = kzalloc(sizeof(struct ntb_queue_entry), GFP_ATOMIC); + entry = kzalloc_node(sizeof(*entry), GFP_ATOMIC, node); if (!entry) goto err2; @@ -1477,10 +1623,8 @@ ntb_transport_create_queue(void *data, struct pci_dev *pdev, &qp->tx_free_q); } - rc = ntb_register_db_callback(qp->ndev, free_queue, qp, - ntb_transport_rxc_db); - if (rc) - goto err2; + ntb_db_clear(qp->ndev, qp_bit); + ntb_db_clear_mask(qp->ndev, qp_bit); dev_info(&pdev->dev, "NTB Transport QP %d created\n", qp->qp_num); @@ -1493,8 +1637,8 @@ err1: while ((entry = ntb_list_rm(&qp->ntb_rx_free_q_lock, &qp->rx_free_q))) kfree(entry); if (qp->dma_chan) - dmaengine_put(); - set_bit(free_queue, &nt->qp_bitmap); + dma_release_channel(qp->dma_chan); + nt->qp_bitmap_free |= qp_bit; err: return NULL; } @@ -1508,13 +1652,15 @@ EXPORT_SYMBOL_GPL(ntb_transport_create_queue); */ void ntb_transport_free_queue(struct ntb_transport_qp *qp) { + struct ntb_transport_ctx *nt = qp->transport; struct pci_dev *pdev; struct ntb_queue_entry *entry; + u64 qp_bit; if (!qp) return; - pdev = ntb_query_pdev(qp->ndev); + pdev = qp->ndev->pdev; if (qp->dma_chan) { struct dma_chan *chan = qp->dma_chan; @@ -1528,13 +1674,21 @@ void ntb_transport_free_queue(struct ntb_transport_qp *qp) */ dma_sync_wait(chan, qp->last_cookie); dmaengine_terminate_all(chan); - dmaengine_put(); + dma_release_channel(chan); } - ntb_unregister_db_callback(qp->ndev, qp->qp_num); + qp_bit = BIT_ULL(qp->qp_num); + + ntb_db_set_mask(qp->ndev, qp_bit); + tasklet_disable(&qp->rxc_db_work); cancel_delayed_work_sync(&qp->link_work); + qp->cb_data = NULL; + qp->rx_handler = NULL; + qp->tx_handler = NULL; + qp->event_handler = NULL; + while ((entry = ntb_list_rm(&qp->ntb_rx_free_q_lock, &qp->rx_free_q))) kfree(entry); @@ -1546,7 +1700,7 @@ void ntb_transport_free_queue(struct ntb_transport_qp *qp) while ((entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q))) kfree(entry); - set_bit(qp->qp_num, &qp->transport->qp_bitmap); + nt->qp_bitmap_free |= qp_bit; dev_info(&pdev->dev, "NTB Transport QP %d freed\n", qp->qp_num); } @@ -1567,7 +1721,7 @@ void *ntb_transport_rx_remove(struct ntb_transport_qp *qp, unsigned int *len) struct ntb_queue_entry *entry; void *buf; - if (!qp || qp->client_ready == NTB_LINK_UP) + if (!qp || qp->client_ready) return NULL; entry = ntb_list_rm(&qp->ntb_rx_pend_q_lock, &qp->rx_pend_q); @@ -1636,7 +1790,7 @@ int ntb_transport_tx_enqueue(struct ntb_transport_qp *qp, void *cb, void *data, struct ntb_queue_entry *entry; int rc; - if (!qp || qp->qp_link != NTB_LINK_UP || !len) + if (!qp || !qp->link_is_up || !len) return -EINVAL; entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q); @@ -1670,9 +1824,9 @@ void ntb_transport_link_up(struct ntb_transport_qp *qp) if (!qp) return; - qp->client_ready = NTB_LINK_UP; + qp->client_ready = true; - if (qp->transport->transport_link == NTB_LINK_UP) + if (qp->transport->link_is_up) schedule_delayed_work(&qp->link_work, 0); } EXPORT_SYMBOL_GPL(ntb_transport_link_up); @@ -1688,27 +1842,20 @@ EXPORT_SYMBOL_GPL(ntb_transport_link_up); void ntb_transport_link_down(struct ntb_transport_qp *qp) { struct pci_dev *pdev; - int rc, val; + int val; if (!qp) return; - pdev = ntb_query_pdev(qp->ndev); - qp->client_ready = NTB_LINK_DOWN; + pdev = qp->ndev->pdev; + qp->client_ready = false; - rc = ntb_read_local_spad(qp->ndev, QP_LINKS, &val); - if (rc) { - dev_err(&pdev->dev, "Error reading spad %d\n", QP_LINKS); - return; - } + val = ntb_spad_read(qp->ndev, QP_LINKS); - rc = ntb_write_remote_spad(qp->ndev, QP_LINKS, - val & ~(1 << qp->qp_num)); - if (rc) - dev_err(&pdev->dev, "Error writing %x to remote spad %d\n", - val & ~(1 << qp->qp_num), QP_LINKS); + ntb_peer_spad_write(qp->ndev, QP_LINKS, + val & ~BIT(qp->qp_num)); - if (qp->qp_link == NTB_LINK_UP) + if (qp->link_is_up) ntb_send_link_down(qp); else cancel_delayed_work_sync(&qp->link_work); @@ -1728,7 +1875,7 @@ bool ntb_transport_link_query(struct ntb_transport_qp *qp) if (!qp) return false; - return qp->qp_link == NTB_LINK_UP; + return qp->link_is_up; } EXPORT_SYMBOL_GPL(ntb_transport_link_query); @@ -1774,3 +1921,71 @@ unsigned int ntb_transport_max_size(struct ntb_transport_qp *qp) return max; } EXPORT_SYMBOL_GPL(ntb_transport_max_size); + +static void ntb_transport_doorbell_callback(void *data, int vector) +{ + struct ntb_transport_ctx *nt = data; + struct ntb_transport_qp *qp; + u64 db_bits; + unsigned int qp_num; + + db_bits = (nt->qp_bitmap & ~nt->qp_bitmap_free & + ntb_db_vector_mask(nt->ndev, vector)); + + while (db_bits) { + qp_num = __ffs(db_bits); + qp = &nt->qp_vec[qp_num]; + + tasklet_schedule(&qp->rxc_db_work); + + db_bits &= ~BIT_ULL(qp_num); + } +} + +static const struct ntb_ctx_ops ntb_transport_ops = { + .link_event = ntb_transport_event_callback, + .db_event = ntb_transport_doorbell_callback, +}; + +static struct ntb_client ntb_transport_client = { + .ops = { + .probe = ntb_transport_probe, + .remove = ntb_transport_free, + }, +}; + +static int __init ntb_transport_init(void) +{ + int rc; + + pr_info("%s, version %s\n", NTB_TRANSPORT_DESC, NTB_TRANSPORT_VER); + + if (debugfs_initialized()) + nt_debugfs_dir = debugfs_create_dir(KBUILD_MODNAME, NULL); + + rc = bus_register(&ntb_transport_bus); + if (rc) + goto err_bus; + + rc = ntb_register_client(&ntb_transport_client); + if (rc) + goto err_client; + + return 0; + +err_client: + bus_unregister(&ntb_transport_bus); +err_bus: + debugfs_remove_recursive(nt_debugfs_dir); + return rc; +} +module_init(ntb_transport_init); + +static void __exit ntb_transport_exit(void) +{ + debugfs_remove_recursive(nt_debugfs_dir); + + ntb_unregister_client(&ntb_transport_client); + bus_unregister(&ntb_transport_bus); +} +module_exit(ntb_transport_exit); diff --git a/drivers/ntb/test/Kconfig b/drivers/ntb/test/Kconfig new file mode 100644 index 000000000000..01852f98a843 --- /dev/null +++ b/drivers/ntb/test/Kconfig @@ -0,0 +1,19 @@ +config NTB_PINGPONG + tristate "NTB Ping Pong Test Client" + help + This is a simple ping pong driver that exercises the scratchpads and + doorbells of the ntb hardware. This driver may be used to test that + your ntb hardware and drivers are functioning at a basic level. + + If unsure, say N. + +config NTB_TOOL + tristate "NTB Debugging Tool Test Client" + help + This is a simple debugging driver that enables the doorbell and + scratchpad registers to be read and written from the debugfs. This + enables more complicated debugging to be scripted from user space. + This driver may be used to test that your ntb hardware and drivers are + functioning at a basic level. + + If unsure, say N. diff --git a/drivers/ntb/test/Makefile b/drivers/ntb/test/Makefile new file mode 100644 index 000000000000..0ea32a324b6c --- /dev/null +++ b/drivers/ntb/test/Makefile @@ -0,0 +1,2 @@ +obj-$(CONFIG_NTB_PINGPONG) += ntb_pingpong.o +obj-$(CONFIG_NTB_TOOL) += ntb_tool.o diff --git a/drivers/ntb/test/ntb_pingpong.c b/drivers/ntb/test/ntb_pingpong.c new file mode 100644 index 000000000000..fe1600566981 --- /dev/null +++ b/drivers/ntb/test/ntb_pingpong.c @@ -0,0 +1,250 @@ +/* + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * GPL LICENSE SUMMARY + * + * Copyright (C) 2015 EMC Corporation. All Rights Reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of version 2 of the GNU General Public License as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * BSD LICENSE + * + * Copyright (C) 2015 EMC Corporation. All Rights Reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copy + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * PCIe NTB Pingpong Linux driver + * + * Contact Information: + * Allen Hubbe <Allen.Hubbe@emc.com> + */ + +/* Note: load this module with option 'dyndbg=+p' */ + +#include <linux/init.h> +#include <linux/kernel.h> +#include <linux/module.h> + +#include <linux/dma-mapping.h> +#include <linux/pci.h> +#include <linux/slab.h> +#include <linux/spinlock.h> + +#include <linux/ntb.h> + +#define DRIVER_NAME "ntb_pingpong" +#define DRIVER_DESCRIPTION "PCIe NTB Simple Pingpong Client" + +#define DRIVER_LICENSE "Dual BSD/GPL" +#define DRIVER_VERSION "1.0" +#define DRIVER_RELDATE "24 March 2015" +#define DRIVER_AUTHOR "Allen Hubbe <Allen.Hubbe@emc.com>" + +MODULE_LICENSE(DRIVER_LICENSE); +MODULE_VERSION(DRIVER_VERSION); +MODULE_AUTHOR(DRIVER_AUTHOR); +MODULE_DESCRIPTION(DRIVER_DESCRIPTION); + +static unsigned int unsafe; +module_param(unsafe, uint, 0644); +MODULE_PARM_DESC(unsafe, "Run even though ntb operations may be unsafe"); + +static unsigned int delay_ms = 1000; +module_param(delay_ms, uint, 0644); +MODULE_PARM_DESC(delay_ms, "Milliseconds to delay the response to peer"); + +static unsigned long db_init = 0x7; +module_param(db_init, ulong, 0644); +MODULE_PARM_DESC(delay_ms, "Initial doorbell bits to ring on the peer"); + +struct pp_ctx { + struct ntb_dev *ntb; + u64 db_bits; + /* synchronize access to db_bits by ping and pong */ + spinlock_t db_lock; + struct timer_list db_timer; + unsigned long db_delay; +}; + +static void pp_ping(unsigned long ctx) +{ + struct pp_ctx *pp = (void *)ctx; + unsigned long irqflags; + u64 db_bits, db_mask; + u32 spad_rd, spad_wr; + + spin_lock_irqsave(&pp->db_lock, irqflags); + { + db_mask = ntb_db_valid_mask(pp->ntb); + db_bits = ntb_db_read(pp->ntb); + + if (db_bits) { + dev_dbg(&pp->ntb->dev, + "Masked pongs %#llx\n", + db_bits); + ntb_db_clear(pp->ntb, db_bits); + } + + db_bits = ((pp->db_bits | db_bits) << 1) & db_mask; + + if (!db_bits) + db_bits = db_init; + + spad_rd = ntb_spad_read(pp->ntb, 0); + spad_wr = spad_rd + 1; + + dev_dbg(&pp->ntb->dev, + "Ping bits %#llx read %#x write %#x\n", + db_bits, spad_rd, spad_wr); + + ntb_peer_spad_write(pp->ntb, 0, spad_wr); + ntb_peer_db_set(pp->ntb, db_bits); + ntb_db_clear_mask(pp->ntb, db_mask); + + pp->db_bits = 0; + } + spin_unlock_irqrestore(&pp->db_lock, irqflags); +} + +static void pp_link_event(void *ctx) +{ + struct pp_ctx *pp = ctx; + + if (ntb_link_is_up(pp->ntb, NULL, NULL) == 1) { + dev_dbg(&pp->ntb->dev, "link is up\n"); + pp_ping((unsigned long)pp); + } else { + dev_dbg(&pp->ntb->dev, "link is down\n"); + del_timer(&pp->db_timer); + } +} + +static void pp_db_event(void *ctx, int vec) +{ + struct pp_ctx *pp = ctx; + u64 db_bits, db_mask; + unsigned long irqflags; + + spin_lock_irqsave(&pp->db_lock, irqflags); + { + db_mask = ntb_db_vector_mask(pp->ntb, vec); + db_bits = db_mask & ntb_db_read(pp->ntb); + ntb_db_set_mask(pp->ntb, db_mask); + ntb_db_clear(pp->ntb, db_bits); + + pp->db_bits |= db_bits; + + mod_timer(&pp->db_timer, jiffies + pp->db_delay); + + dev_dbg(&pp->ntb->dev, + "Pong vec %d bits %#llx\n", + vec, db_bits); + } + spin_unlock_irqrestore(&pp->db_lock, irqflags); +} + +static const struct ntb_ctx_ops pp_ops = { + .link_event = pp_link_event, + .db_event = pp_db_event, +}; + +static int pp_probe(struct ntb_client *client, + struct ntb_dev *ntb) +{ + struct pp_ctx *pp; + int rc; + + if (ntb_db_is_unsafe(ntb)) { + dev_dbg(&ntb->dev, "doorbell is unsafe\n"); + if (!unsafe) { + rc = -EINVAL; + goto err_pp; + } + } + + if (ntb_spad_is_unsafe(ntb)) { + dev_dbg(&ntb->dev, "scratchpad is unsafe\n"); + if (!unsafe) { + rc = -EINVAL; + goto err_pp; + } + } + + pp = kmalloc(sizeof(*pp), GFP_KERNEL); + if (!pp) { + rc = -ENOMEM; + goto err_pp; + } + + pp->ntb = ntb; + pp->db_bits = 0; + spin_lock_init(&pp->db_lock); + setup_timer(&pp->db_timer, pp_ping, (unsigned long)pp); + pp->db_delay = msecs_to_jiffies(delay_ms); + + rc = ntb_set_ctx(ntb, pp, &pp_ops); + if (rc) + goto err_ctx; + + ntb_link_enable(ntb, NTB_SPEED_AUTO, NTB_WIDTH_AUTO); + ntb_link_event(ntb); + + return 0; + +err_ctx: + kfree(pp); +err_pp: + return rc; +} + +static void pp_remove(struct ntb_client *client, + struct ntb_dev *ntb) +{ + struct pp_ctx *pp = ntb->ctx; + + ntb_clear_ctx(ntb); + del_timer_sync(&pp->db_timer); + ntb_link_disable(ntb); + + kfree(pp); +} + +static struct ntb_client pp_client = { + .ops = { + .probe = pp_probe, + .remove = pp_remove, + }, +}; +module_ntb_client(pp_client); diff --git a/drivers/ntb/test/ntb_tool.c b/drivers/ntb/test/ntb_tool.c new file mode 100644 index 000000000000..6f5dc6ca673d --- /dev/null +++ b/drivers/ntb/test/ntb_tool.c @@ -0,0 +1,556 @@ +/* + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * GPL LICENSE SUMMARY + * + * Copyright (C) 2015 EMC Corporation. All Rights Reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of version 2 of the GNU General Public License as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * BSD LICENSE + * + * Copyright (C) 2015 EMC Corporation. All Rights Reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copy + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * PCIe NTB Debugging Tool Linux driver + * + * Contact Information: + * Allen Hubbe <Allen.Hubbe@emc.com> + */ + +/* + * How to use this tool, by example. + * + * Assuming $DBG_DIR is something like: + * '/sys/kernel/debug/ntb_tool/0000:00:03.0' + * + * Eg: check if clearing the doorbell mask generates an interrupt. + * + * # Set the doorbell mask + * root@self# echo 's 1' > $DBG_DIR/mask + * + * # Ring the doorbell from the peer + * root@peer# echo 's 1' > $DBG_DIR/peer_db + * + * # Clear the doorbell mask + * root@self# echo 'c 1' > $DBG_DIR/mask + * + * Observe debugging output in dmesg or your console. You should see a + * doorbell event triggered by clearing the mask. If not, this may indicate an + * issue with the hardware that needs to be worked around in the driver. + * + * Eg: read and write scratchpad registers + * + * root@peer# echo '0 0x01010101 1 0x7f7f7f7f' > $DBG_DIR/peer_spad + * + * root@self# cat $DBG_DIR/spad + * + * Observe that spad 0 and 1 have the values set by the peer. + */ + +#include <linux/init.h> +#include <linux/kernel.h> +#include <linux/module.h> + +#include <linux/debugfs.h> +#include <linux/dma-mapping.h> +#include <linux/pci.h> +#include <linux/slab.h> + +#include <linux/ntb.h> + +#define DRIVER_NAME "ntb_tool" +#define DRIVER_DESCRIPTION "PCIe NTB Debugging Tool" + +#define DRIVER_LICENSE "Dual BSD/GPL" +#define DRIVER_VERSION "1.0" +#define DRIVER_RELDATE "22 April 2015" +#define DRIVER_AUTHOR "Allen Hubbe <Allen.Hubbe@emc.com>" + +MODULE_LICENSE(DRIVER_LICENSE); +MODULE_VERSION(DRIVER_VERSION); +MODULE_AUTHOR(DRIVER_AUTHOR); +MODULE_DESCRIPTION(DRIVER_DESCRIPTION); + +static struct dentry *tool_dbgfs; + +struct tool_ctx { + struct ntb_dev *ntb; + struct dentry *dbgfs; +}; + +#define SPAD_FNAME_SIZE 0x10 +#define INT_PTR(x) ((void *)(unsigned long)x) +#define PTR_INT(x) ((int)(unsigned long)x) + +#define TOOL_FOPS_RDWR(__name, __read, __write) \ + const struct file_operations __name = { \ + .owner = THIS_MODULE, \ + .open = simple_open, \ + .read = __read, \ + .write = __write, \ + } + +static void tool_link_event(void *ctx) +{ + struct tool_ctx *tc = ctx; + enum ntb_speed speed; + enum ntb_width width; + int up; + + up = ntb_link_is_up(tc->ntb, &speed, &width); + + dev_dbg(&tc->ntb->dev, "link is %s speed %d width %d\n", + up ? "up" : "down", speed, width); +} + +static void tool_db_event(void *ctx, int vec) +{ + struct tool_ctx *tc = ctx; + u64 db_bits, db_mask; + + db_mask = ntb_db_vector_mask(tc->ntb, vec); + db_bits = ntb_db_read(tc->ntb); + + dev_dbg(&tc->ntb->dev, "doorbell vec %d mask %#llx bits %#llx\n", + vec, db_mask, db_bits); +} + +static const struct ntb_ctx_ops tool_ops = { + .link_event = tool_link_event, + .db_event = tool_db_event, +}; + +static ssize_t tool_dbfn_read(struct tool_ctx *tc, char __user *ubuf, + size_t size, loff_t *offp, + u64 (*db_read_fn)(struct ntb_dev *)) +{ + size_t buf_size; + char *buf; + ssize_t pos, rc; + + if (!db_read_fn) + return -EINVAL; + + buf_size = min_t(size_t, size, 0x20); + + buf = kmalloc(buf_size, GFP_KERNEL); + if (!buf) + return -ENOMEM; + + pos = scnprintf(buf, buf_size, "%#llx\n", + db_read_fn(tc->ntb)); + + rc = simple_read_from_buffer(ubuf, size, offp, buf, pos); + + kfree(buf); + + return rc; +} + +static ssize_t tool_dbfn_write(struct tool_ctx *tc, + const char __user *ubuf, + size_t size, loff_t *offp, + int (*db_set_fn)(struct ntb_dev *, u64), + int (*db_clear_fn)(struct ntb_dev *, u64)) +{ + u64 db_bits; + char *buf, cmd; + ssize_t rc; + int n; + + buf = kmalloc(size + 1, GFP_KERNEL); + if (!buf) + return -ENOMEM; + + rc = simple_write_to_buffer(buf, size, offp, ubuf, size); + if (rc < 0) { + kfree(buf); + return rc; + } + + buf[size] = 0; + + n = sscanf(buf, "%c %lli", &cmd, &db_bits); + + kfree(buf); + + if (n != 2) { + rc = -EINVAL; + } else if (cmd == 's') { + if (!db_set_fn) + rc = -EINVAL; + else + rc = db_set_fn(tc->ntb, db_bits); + } else if (cmd == 'c') { + if (!db_clear_fn) + rc = -EINVAL; + else + rc = db_clear_fn(tc->ntb, db_bits); + } else { + rc = -EINVAL; + } + + return rc ? : size; +} + +static ssize_t tool_spadfn_read(struct tool_ctx *tc, char __user *ubuf, + size_t size, loff_t *offp, + u32 (*spad_read_fn)(struct ntb_dev *, int)) +{ + size_t buf_size; + char *buf; + ssize_t pos, rc; + int i, spad_count; + + if (!spad_read_fn) + return -EINVAL; + + buf_size = min_t(size_t, size, 0x100); + + buf = kmalloc(buf_size, GFP_KERNEL); + if (!buf) + return -ENOMEM; + + pos = 0; + + spad_count = ntb_spad_count(tc->ntb); + for (i = 0; i < spad_count; ++i) { + pos += scnprintf(buf + pos, buf_size - pos, "%d\t%#x\n", + i, spad_read_fn(tc->ntb, i)); + } + + rc = simple_read_from_buffer(ubuf, size, offp, buf, pos); + + kfree(buf); + + return rc; +} + +static ssize_t tool_spadfn_write(struct tool_ctx *tc, + const char __user *ubuf, + size_t size, loff_t *offp, + int (*spad_write_fn)(struct ntb_dev *, + int, u32)) +{ + int spad_idx; + u32 spad_val; + char *buf; + int pos, n; + ssize_t rc; + + if (!spad_write_fn) { + dev_dbg(&tc->ntb->dev, "no spad write fn\n"); + return -EINVAL; + } + + buf = kmalloc(size + 1, GFP_KERNEL); + if (!buf) + return -ENOMEM; + + rc = simple_write_to_buffer(buf, size, offp, ubuf, size); + if (rc < 0) { + kfree(buf); + return rc; + } + + buf[size] = 0; + + n = sscanf(buf, "%d %i%n", &spad_idx, &spad_val, &pos); + while (n == 2) { + rc = spad_write_fn(tc->ntb, spad_idx, spad_val); + if (rc) + break; + + n = sscanf(buf + pos, "%d %i%n", &spad_idx, &spad_val, &pos); + } + + if (n < 0) + rc = n; + + kfree(buf); + + return rc ? : size; +} + +static ssize_t tool_db_read(struct file *filep, char __user *ubuf, + size_t size, loff_t *offp) +{ + struct tool_ctx *tc = filep->private_data; + + return tool_dbfn_read(tc, ubuf, size, offp, + tc->ntb->ops->db_read); +} + +static ssize_t tool_db_write(struct file *filep, const char __user *ubuf, + size_t size, loff_t *offp) +{ + struct tool_ctx *tc = filep->private_data; + + return tool_dbfn_write(tc, ubuf, size, offp, + tc->ntb->ops->db_set, + tc->ntb->ops->db_clear); +} + +static TOOL_FOPS_RDWR(tool_db_fops, + tool_db_read, + tool_db_write); + +static ssize_t tool_mask_read(struct file *filep, char __user *ubuf, + size_t size, loff_t *offp) +{ + struct tool_ctx *tc = filep->private_data; + + return tool_dbfn_read(tc, ubuf, size, offp, + tc->ntb->ops->db_read_mask); +} + +static ssize_t tool_mask_write(struct file *filep, const char __user *ubuf, + size_t size, loff_t *offp) +{ + struct tool_ctx *tc = filep->private_data; + + return tool_dbfn_write(tc, ubuf, size, offp, + tc->ntb->ops->db_set_mask, + tc->ntb->ops->db_clear_mask); +} + +static TOOL_FOPS_RDWR(tool_mask_fops, + tool_mask_read, + tool_mask_write); + +static ssize_t tool_peer_db_read(struct file *filep, char __user *ubuf, + size_t size, loff_t *offp) +{ + struct tool_ctx *tc = filep->private_data; + + return tool_dbfn_read(tc, ubuf, size, offp, + tc->ntb->ops->peer_db_read); +} + +static ssize_t tool_peer_db_write(struct file *filep, const char __user *ubuf, + size_t size, loff_t *offp) +{ + struct tool_ctx *tc = filep->private_data; + + return tool_dbfn_write(tc, ubuf, size, offp, + tc->ntb->ops->peer_db_set, + tc->ntb->ops->peer_db_clear); +} + +static TOOL_FOPS_RDWR(tool_peer_db_fops, + tool_peer_db_read, + tool_peer_db_write); + +static ssize_t tool_peer_mask_read(struct file *filep, char __user *ubuf, + size_t size, loff_t *offp) +{ + struct tool_ctx *tc = filep->private_data; + + return tool_dbfn_read(tc, ubuf, size, offp, + tc->ntb->ops->peer_db_read_mask); +} + +static ssize_t tool_peer_mask_write(struct file *filep, const char __user *ubuf, + size_t size, loff_t *offp) +{ + struct tool_ctx *tc = filep->private_data; + + return tool_dbfn_write(tc, ubuf, size, offp, + tc->ntb->ops->peer_db_set_mask, + tc->ntb->ops->peer_db_clear_mask); +} + +static TOOL_FOPS_RDWR(tool_peer_mask_fops, + tool_peer_mask_read, + tool_peer_mask_write); + +static ssize_t tool_spad_read(struct file *filep, char __user *ubuf, + size_t size, loff_t *offp) +{ + struct tool_ctx *tc = filep->private_data; + + return tool_spadfn_read(tc, ubuf, size, offp, + tc->ntb->ops->spad_read); +} + +static ssize_t tool_spad_write(struct file *filep, const char __user *ubuf, + size_t size, loff_t *offp) +{ + struct tool_ctx *tc = filep->private_data; + + return tool_spadfn_write(tc, ubuf, size, offp, + tc->ntb->ops->spad_write); +} + +static TOOL_FOPS_RDWR(tool_spad_fops, + tool_spad_read, + tool_spad_write); + +static ssize_t tool_peer_spad_read(struct file *filep, char __user *ubuf, + size_t size, loff_t *offp) +{ + struct tool_ctx *tc = filep->private_data; + + return tool_spadfn_read(tc, ubuf, size, offp, + tc->ntb->ops->peer_spad_read); +} + +static ssize_t tool_peer_spad_write(struct file *filep, const char __user *ubuf, + size_t size, loff_t *offp) +{ + struct tool_ctx *tc = filep->private_data; + + return tool_spadfn_write(tc, ubuf, size, offp, + tc->ntb->ops->peer_spad_write); +} + +static TOOL_FOPS_RDWR(tool_peer_spad_fops, + tool_peer_spad_read, + tool_peer_spad_write); + +static void tool_setup_dbgfs(struct tool_ctx *tc) +{ + /* This modules is useless without dbgfs... */ + if (!tool_dbgfs) { + tc->dbgfs = NULL; + return; + } + + tc->dbgfs = debugfs_create_dir(dev_name(&tc->ntb->dev), + tool_dbgfs); + if (!tc->dbgfs) + return; + + debugfs_create_file("db", S_IRUSR | S_IWUSR, tc->dbgfs, + tc, &tool_db_fops); + + debugfs_create_file("mask", S_IRUSR | S_IWUSR, tc->dbgfs, + tc, &tool_mask_fops); + + debugfs_create_file("peer_db", S_IRUSR | S_IWUSR, tc->dbgfs, + tc, &tool_peer_db_fops); + + debugfs_create_file("peer_mask", S_IRUSR | S_IWUSR, tc->dbgfs, + tc, &tool_peer_mask_fops); + + debugfs_create_file("spad", S_IRUSR | S_IWUSR, tc->dbgfs, + tc, &tool_spad_fops); + + debugfs_create_file("peer_spad", S_IRUSR | S_IWUSR, tc->dbgfs, + tc, &tool_peer_spad_fops); +} + +static int tool_probe(struct ntb_client *self, struct ntb_dev *ntb) +{ + struct tool_ctx *tc; + int rc; + + if (ntb_db_is_unsafe(ntb)) + dev_dbg(&ntb->dev, "doorbell is unsafe\n"); + + if (ntb_spad_is_unsafe(ntb)) + dev_dbg(&ntb->dev, "scratchpad is unsafe\n"); + + tc = kmalloc(sizeof(*tc), GFP_KERNEL); + if (!tc) { + rc = -ENOMEM; + goto err_tc; + } + + tc->ntb = ntb; + + tool_setup_dbgfs(tc); + + rc = ntb_set_ctx(ntb, tc, &tool_ops); + if (rc) + goto err_ctx; + + ntb_link_enable(ntb, NTB_SPEED_AUTO, NTB_WIDTH_AUTO); + ntb_link_event(ntb); + + return 0; + +err_ctx: + debugfs_remove_recursive(tc->dbgfs); + kfree(tc); +err_tc: + return rc; +} + +static void tool_remove(struct ntb_client *self, struct ntb_dev *ntb) +{ + struct tool_ctx *tc = ntb->ctx; + + ntb_clear_ctx(ntb); + ntb_link_disable(ntb); + + debugfs_remove_recursive(tc->dbgfs); + kfree(tc); +} + +static struct ntb_client tool_client = { + .ops = { + .probe = tool_probe, + .remove = tool_remove, + }, +}; + +static int __init tool_init(void) +{ + int rc; + + if (debugfs_initialized()) + tool_dbgfs = debugfs_create_dir(KBUILD_MODNAME, NULL); + + rc = ntb_register_client(&tool_client); + if (rc) + goto err_client; + + return 0; + +err_client: + debugfs_remove_recursive(tool_dbgfs); + return rc; +} +module_init(tool_init); + +static void __exit tool_exit(void) +{ + ntb_unregister_client(&tool_client); + debugfs_remove_recursive(tool_dbgfs); +} +module_exit(tool_exit); |