From 4330e4dad780467d930b394b5119c0218a1e2dbe Mon Sep 17 00:00:00 2001 From: Michael Albaugh Date: Wed, 16 Apr 2008 21:09:26 -0700 Subject: IB/ipath: Prevent link-recovery code from negating admin disable The link can be put in LINKDOWN_DISABLE state either locally or via a MAD. However, the link-recovery code will take it out of that state as a side-effect of attempts to clear SerDes/XGXS issues. We add a flag to indicate "link is down on purpose, leave it alone." Signed-off-by: Michael Albaugh Signed-off-by: Roland Dreier --- drivers/infiniband/hw/ipath/ipath_debug.h | 1 + drivers/infiniband/hw/ipath/ipath_driver.c | 70 ++++++++++++++++----------- drivers/infiniband/hw/ipath/ipath_init_chip.c | 5 +- drivers/infiniband/hw/ipath/ipath_intr.c | 2 + drivers/infiniband/hw/ipath/ipath_kernel.h | 2 + drivers/infiniband/hw/ipath/ipath_mad.c | 4 ++ 6 files changed, 55 insertions(+), 29 deletions(-) diff --git a/drivers/infiniband/hw/ipath/ipath_debug.h b/drivers/infiniband/hw/ipath/ipath_debug.h index 7170bd20cfbb..65926cd35759 100644 --- a/drivers/infiniband/hw/ipath/ipath_debug.h +++ b/drivers/infiniband/hw/ipath/ipath_debug.h @@ -90,6 +90,7 @@ #define __IPATH_IPATHERR 0x0 /* Ethernet (IPATH) errors on */ #define __IPATH_IPATHPD 0x0 /* Ethernet (IPATH) packet dump on */ #define __IPATH_IPATHTABLE 0x0 /* Ethernet (IPATH) packet dump on */ +#define __IPATH_LINKVERBDBG 0x0 /* very verbose linkchange debug */ #endif /* _IPATH_DEBUGGING */ diff --git a/drivers/infiniband/hw/ipath/ipath_driver.c b/drivers/infiniband/hw/ipath/ipath_driver.c index 130f4db7b8c1..1299171c1e50 100644 --- a/drivers/infiniband/hw/ipath/ipath_driver.c +++ b/drivers/infiniband/hw/ipath/ipath_driver.c @@ -1664,30 +1664,48 @@ void ipath_cancel_sends(struct ipath_devdata *dd, int restore_sendctrl) ipath_read_kreg64(dd, dd->ipath_kregs->kr_scratch); } - -static void ipath_set_ib_lstate(struct ipath_devdata *dd, int which) +static void ipath_set_ib_lstate(struct ipath_devdata *dd, int linkcmd, + int linitcmd) { + u64 mod_wd; static const char *what[4] = { [0] = "NOP", [INFINIPATH_IBCC_LINKCMD_DOWN] = "DOWN", [INFINIPATH_IBCC_LINKCMD_ARMED] = "ARMED", [INFINIPATH_IBCC_LINKCMD_ACTIVE] = "ACTIVE" }; - int linkcmd = (which >> INFINIPATH_IBCC_LINKCMD_SHIFT) & - INFINIPATH_IBCC_LINKCMD_MASK; - ipath_cdbg(VERBOSE, "Trying to move unit %u to %s, current ltstate " - "is %s\n", dd->ipath_unit, - what[linkcmd], - ipath_ibcstatus_str[ipath_ib_linktrstate(dd, + if (linitcmd == INFINIPATH_IBCC_LINKINITCMD_DISABLE) { + /* + * If we are told to disable, note that so link-recovery + * code does not attempt to bring us back up. + */ + preempt_disable(); + dd->ipath_flags |= IPATH_IB_LINK_DISABLED; + preempt_enable(); + } else if (linitcmd) { + /* + * Any other linkinitcmd will lead to LINKDOWN and then + * to INIT (if all is well), so clear flag to let + * link-recovery code attempt to bring us back up. + */ + preempt_disable(); + dd->ipath_flags &= ~IPATH_IB_LINK_DISABLED; + preempt_enable(); + } + + mod_wd = (linkcmd << dd->ibcc_lc_shift) | + (linitcmd << INFINIPATH_IBCC_LINKINITCMD_SHIFT); + ipath_cdbg(VERBOSE, + "Moving unit %u to %s (initcmd=0x%x), current ltstate is %s\n", + dd->ipath_unit, what[linkcmd], linitcmd, + ipath_ibcstatus_str[ipath_ib_linktrstate(dd, ipath_read_kreg64(dd, dd->ipath_kregs->kr_ibcstatus))]); - /* flush all queued sends when going to DOWN to be sure that - * they don't block MAD packets */ - if (linkcmd == INFINIPATH_IBCC_LINKCMD_DOWN) - ipath_cancel_sends(dd, 1); ipath_write_kreg(dd, dd->ipath_kregs->kr_ibcctrl, - dd->ipath_ibcctrl | which); + dd->ipath_ibcctrl | mod_wd); + /* read from chip so write is flushed */ + (void) ipath_read_kreg64(dd, dd->ipath_kregs->kr_ibcstatus); } int ipath_set_linkstate(struct ipath_devdata *dd, u8 newstate) @@ -1697,30 +1715,28 @@ int ipath_set_linkstate(struct ipath_devdata *dd, u8 newstate) switch (newstate) { case IPATH_IB_LINKDOWN_ONLY: - ipath_set_ib_lstate(dd, INFINIPATH_IBCC_LINKCMD_DOWN << - INFINIPATH_IBCC_LINKCMD_SHIFT); + ipath_set_ib_lstate(dd, INFINIPATH_IBCC_LINKCMD_DOWN, 0); /* don't wait */ ret = 0; goto bail; case IPATH_IB_LINKDOWN: - ipath_set_ib_lstate(dd, INFINIPATH_IBCC_LINKINITCMD_POLL << - INFINIPATH_IBCC_LINKINITCMD_SHIFT); + ipath_set_ib_lstate(dd, INFINIPATH_IBCC_LINKCMD_DOWN, + INFINIPATH_IBCC_LINKINITCMD_POLL); /* don't wait */ ret = 0; goto bail; case IPATH_IB_LINKDOWN_SLEEP: - ipath_set_ib_lstate(dd, INFINIPATH_IBCC_LINKINITCMD_SLEEP << - INFINIPATH_IBCC_LINKINITCMD_SHIFT); + ipath_set_ib_lstate(dd, INFINIPATH_IBCC_LINKCMD_DOWN, + INFINIPATH_IBCC_LINKINITCMD_SLEEP); /* don't wait */ ret = 0; goto bail; case IPATH_IB_LINKDOWN_DISABLE: - ipath_set_ib_lstate(dd, - INFINIPATH_IBCC_LINKINITCMD_DISABLE << - INFINIPATH_IBCC_LINKINITCMD_SHIFT); + ipath_set_ib_lstate(dd, INFINIPATH_IBCC_LINKCMD_DOWN, + INFINIPATH_IBCC_LINKINITCMD_DISABLE); /* don't wait */ ret = 0; goto bail; @@ -1735,8 +1751,8 @@ int ipath_set_linkstate(struct ipath_devdata *dd, u8 newstate) ret = -EINVAL; goto bail; } - ipath_set_ib_lstate(dd, INFINIPATH_IBCC_LINKCMD_ARMED << - INFINIPATH_IBCC_LINKCMD_SHIFT); + ipath_set_ib_lstate(dd, INFINIPATH_IBCC_LINKCMD_ARMED, 0); + /* * Since the port can transition to ACTIVE by receiving * a non VL 15 packet, wait for either state. @@ -1753,8 +1769,7 @@ int ipath_set_linkstate(struct ipath_devdata *dd, u8 newstate) ret = -EINVAL; goto bail; } - ipath_set_ib_lstate(dd, INFINIPATH_IBCC_LINKCMD_ACTIVE << - INFINIPATH_IBCC_LINKCMD_SHIFT); + ipath_set_ib_lstate(dd, INFINIPATH_IBCC_LINKCMD_ACTIVE, 0); lstate = IPATH_LINKACTIVE; break; @@ -2026,8 +2041,7 @@ void ipath_shutdown_device(struct ipath_devdata *dd) */ udelay(5); - ipath_set_ib_lstate(dd, INFINIPATH_IBCC_LINKINITCMD_DISABLE << - INFINIPATH_IBCC_LINKINITCMD_SHIFT); + ipath_set_ib_lstate(dd, 0, INFINIPATH_IBCC_LINKINITCMD_DISABLE); ipath_cancel_sends(dd, 0); signal_ib_event(dd, IB_EVENT_PORT_ERR); diff --git a/drivers/infiniband/hw/ipath/ipath_init_chip.c b/drivers/infiniband/hw/ipath/ipath_init_chip.c index 524fdf7d8c88..46c70656d538 100644 --- a/drivers/infiniband/hw/ipath/ipath_init_chip.c +++ b/drivers/infiniband/hw/ipath/ipath_init_chip.c @@ -180,10 +180,13 @@ static int bringup_link(struct ipath_devdata *dd) /* * Want to start out with both LINKCMD and LINKINITCMD in NOP * (0 and 0). Don't put linkinitcmd in ipath_ibcctrl, want that - * to stay a NOP + * to stay a NOP. Flag that we are disabled, for the (unlikely) + * case that some recovery path is trying to bring the link up + * before we are ready. */ ibc |= INFINIPATH_IBCC_LINKINITCMD_DISABLE << INFINIPATH_IBCC_LINKINITCMD_SHIFT; + dd->ipath_flags |= IPATH_IB_LINK_DISABLED; ipath_cdbg(VERBOSE, "Writing 0x%llx to ibcctrl\n", (unsigned long long) ibc); ipath_write_kreg(dd, dd->ipath_kregs->kr_ibcctrl, ibc); diff --git a/drivers/infiniband/hw/ipath/ipath_intr.c b/drivers/infiniband/hw/ipath/ipath_intr.c index ed82ecbb02da..5608e3268a62 100644 --- a/drivers/infiniband/hw/ipath/ipath_intr.c +++ b/drivers/infiniband/hw/ipath/ipath_intr.c @@ -309,6 +309,8 @@ static void handle_e_ibstatuschanged(struct ipath_devdata *dd, lastlstate == INFINIPATH_IBCS_L_STATE_DOWN) { /* transitioned to UP */ if (dd->ipath_f_ib_updown(dd, 1, ibcs)) { + /* link came up, so we must no longer be disabled */ + dd->ipath_flags &= ~IPATH_IB_LINK_DISABLED; ipath_cdbg(LINKVERB, "LinkUp handled, skipped\n"); goto skip_ibchange; /* chip-code handled */ } diff --git a/drivers/infiniband/hw/ipath/ipath_kernel.h b/drivers/infiniband/hw/ipath/ipath_kernel.h index b8b81cb745b9..2510359b1c83 100644 --- a/drivers/infiniband/hw/ipath/ipath_kernel.h +++ b/drivers/infiniband/hw/ipath/ipath_kernel.h @@ -848,6 +848,8 @@ void ipath_hol_event(unsigned long); /* Suppress heartbeat, even if turning off loopback */ #define IPATH_NO_HRTBT 0x1000000 #define IPATH_HAS_MULT_IB_SPEED 0x8000000 + /* Linkdown-disable intentionally, Do not attempt to bring up */ +#define IPATH_IB_LINK_DISABLED 0x40000000 #define IPATH_IB_FORCE_NOTIFY 0x80000000 /* force notify on next ib change */ /* Bits in GPIO for the added interrupts */ diff --git a/drivers/infiniband/hw/ipath/ipath_mad.c b/drivers/infiniband/hw/ipath/ipath_mad.c index 7516a2608988..30b2f44406b1 100644 --- a/drivers/infiniband/hw/ipath/ipath_mad.c +++ b/drivers/infiniband/hw/ipath/ipath_mad.c @@ -590,6 +590,10 @@ static int recv_subn_set_portinfo(struct ib_smp *smp, else goto err; ipath_set_linkstate(dd, lstate); + if (lstate == IPATH_IB_LINKDOWN_DISABLE) { + ret = IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_CONSUMED; + goto done; + } ipath_wait_linkstate(dd, IPATH_LINKINIT | IPATH_LINKARMED | IPATH_LINKACTIVE, 1000); break; -- cgit v1.2.3