From b25bbc2f24dcab9cd186ef4003c39bf51ad0454c Mon Sep 17 00:00:00 2001 From: Alex Vesker Date: Thu, 28 Jun 2018 15:05:58 +0300 Subject: net/mlx5: Add Vendor Specific Capability access gateway The Vendor Specific Capability (VSC) is used to activate a gateway interfacing with the device. The gateway is used to read or write device configurations, which are organized in different domains (spaces). A configuration access may result in multiple actions, reads, writes. Example usages are accessing the Crspace domain to read the crspace or locking a device semaphore using the Semaphore domain. The configuration access use pci_cfg_access to prevent parallel access to the VSC space by the driver and userspace calls. Signed-off-by: Alex Vesker Signed-off-by: Feras Daoud Signed-off-by: Saeed Mahameed --- include/linux/mlx5/driver.h | 1 + 1 file changed, 1 insertion(+) (limited to 'include/linux') diff --git a/include/linux/mlx5/driver.h b/include/linux/mlx5/driver.h index 3a810bf043fe..f732445bcbdb 100644 --- a/include/linux/mlx5/driver.h +++ b/include/linux/mlx5/driver.h @@ -693,6 +693,7 @@ struct mlx5_core_dev { struct mlx5_clock clock; struct mlx5_ib_clock_info *clock_info; struct mlx5_fw_tracer *tracer; + u32 vsc_addr; }; struct mlx5_db { -- cgit v1.2.3 From 8b9d8baae1de7400f19058020ee8f0f27d436687 Mon Sep 17 00:00:00 2001 From: Alex Vesker Date: Tue, 17 Jul 2018 11:18:26 +0300 Subject: net/mlx5: Add Crdump support Crdump allows the driver to retrieve a dump of the FW PCI crspace. This is useful in case of catastrophic issues which may require FW reset. The crspace dump can be used for later debug. Signed-off-by: Alex Vesker Signed-off-by: Moshe Shemesh Reviewed-by: Feras Daoud Signed-off-by: Saeed Mahameed --- include/linux/mlx5/driver.h | 1 + 1 file changed, 1 insertion(+) (limited to 'include/linux') diff --git a/include/linux/mlx5/driver.h b/include/linux/mlx5/driver.h index f732445bcbdb..4ae533b3da07 100644 --- a/include/linux/mlx5/driver.h +++ b/include/linux/mlx5/driver.h @@ -435,6 +435,7 @@ struct mlx5_core_health { u32 prev; int miss_counter; bool sick; + u32 crdump_size; /* wq spinlock to synchronize draining */ spinlock_t wq_lock; struct workqueue_struct *wq; -- cgit v1.2.3 From 63cbc552eebf08818af2025aef4589a48ef849c0 Mon Sep 17 00:00:00 2001 From: Feras Daoud Date: Mon, 12 Nov 2018 15:23:02 +0200 Subject: net/mlx5: Handle SW reset of FW in error flow New mlx5 adapters allow the driver to reset the FW in the event of an error, this action called "SW Reset". When an SW reset is issued on any PF all PFs enter reset state which is a recoverable condition. The existing recovery flow was designed to allow the recovery of a VF after a PF driver reload. This patch adds the sw reset to the NIC states as a preparation for sw reset handling. When a software reset is issued the following occurs: 1. The NIC interface mode is set to 7 while the reset is in progress. 2. Once the reset completes the NIC interface mode is set to 1. Signed-off-by: Feras Daoud Signed-off-by: Moshe Shemesh Signed-off-by: Daniel Jurgens Reviewed-by: Alex Vesker Signed-off-by: Saeed Mahameed --- include/linux/mlx5/driver.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/mlx5/driver.h b/include/linux/mlx5/driver.h index 4ae533b3da07..cc7fd8e62844 100644 --- a/include/linux/mlx5/driver.h +++ b/include/linux/mlx5/driver.h @@ -435,6 +435,7 @@ struct mlx5_core_health { u32 prev; int miss_counter; bool sick; + u32 fatal_error; u32 crdump_size; /* wq spinlock to synchronize draining */ spinlock_t wq_lock; @@ -906,7 +907,6 @@ void mlx5_start_health_poll(struct mlx5_core_dev *dev); void mlx5_stop_health_poll(struct mlx5_core_dev *dev, bool disable_health); void mlx5_drain_health_wq(struct mlx5_core_dev *dev); void mlx5_trigger_health_work(struct mlx5_core_dev *dev); -void mlx5_drain_health_recovery(struct mlx5_core_dev *dev); int mlx5_buf_alloc_node(struct mlx5_core_dev *dev, int size, struct mlx5_frag_buf *buf, int node); int mlx5_buf_alloc(struct mlx5_core_dev *dev, -- cgit v1.2.3 From 3e5b72ac2f298423902169db7893fef43365e0a6 Mon Sep 17 00:00:00 2001 From: Feras Daoud Date: Mon, 12 Nov 2018 16:40:17 +0200 Subject: net/mlx5: Issue SW reset on FW assert If a FW assert is considered fatal, indicated by a new bit in the health buffer, reset the FW. After the reset go through the normal recovery flow. Only one PF needs to issue the reset, so an attempt is made to prevent the 2nd function from also issuing the reset. It's not an error if that happens, it just slows recovery. Signed-off-by: Feras Daoud Signed-off-by: Alex Vesker Signed-off-by: Moshe Shemesh Signed-off-by: Daniel Jurgens Signed-off-by: Saeed Mahameed --- include/linux/mlx5/device.h | 10 +++++++++- include/linux/mlx5/driver.h | 1 + 2 files changed, 10 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/mlx5/device.h b/include/linux/mlx5/device.h index 5e760067ac41..35ed38c2ae6c 100644 --- a/include/linux/mlx5/device.h +++ b/include/linux/mlx5/device.h @@ -510,6 +510,10 @@ struct mlx5_cmd_layout { u8 status_own; }; +enum mlx5_fatal_assert_bit_offsets { + MLX5_RFR_OFFSET = 31, +}; + struct health_buffer { __be32 assert_var[5]; __be32 rsvd0[3]; @@ -518,12 +522,16 @@ struct health_buffer { __be32 rsvd1[2]; __be32 fw_ver; __be32 hw_id; - __be32 rsvd2; + __be32 rfr; u8 irisc_index; u8 synd; __be16 ext_synd; }; +enum mlx5_initializing_bit_offsets { + MLX5_FW_RESET_SUPPORTED_OFFSET = 30, +}; + enum mlx5_cmd_addr_l_sz_offset { MLX5_NIC_IFC_OFFSET = 8, }; diff --git a/include/linux/mlx5/driver.h b/include/linux/mlx5/driver.h index cc7fd8e62844..89205b6cc7ef 100644 --- a/include/linux/mlx5/driver.h +++ b/include/linux/mlx5/driver.h @@ -583,6 +583,7 @@ struct mlx5_priv { }; enum mlx5_device_state { + MLX5_DEVICE_STATE_UNINITIALIZED, MLX5_DEVICE_STATE_UP, MLX5_DEVICE_STATE_INTERNAL_ERROR, }; -- cgit v1.2.3 From 1e34f3efd413a6318c3edd6e8e7e091f1214b2e6 Mon Sep 17 00:00:00 2001 From: Moshe Shemesh Date: Tue, 11 Dec 2018 16:09:53 +0200 Subject: net/mlx5: Create FW devlink_health_reporter Create mlx5_devlink_health_reporter for FW reporter. The FW reporter implements devlink_health_reporter diagnose callback. The fw reporter diagnose command can be triggered any time by the user to check current fw status. In healthy status, it will return clear syndrome. Otherwise it will return the syndrome and description of the error type. Command example and output on healthy status: $ devlink health diagnose pci/0000:82:00.0 reporter fw Syndrome: 0 Command example and output on non healthy status: $ devlink health diagnose pci/0000:82:00.0 reporter fw Syndrome: 8 Description: unrecoverable hardware error Signed-off-by: Moshe Shemesh Signed-off-by: Eran Ben Elisha Signed-off-by: Saeed Mahameed --- include/linux/mlx5/driver.h | 2 ++ 1 file changed, 2 insertions(+) (limited to 'include/linux') diff --git a/include/linux/mlx5/driver.h b/include/linux/mlx5/driver.h index 89205b6cc7ef..8d5d065d1aa6 100644 --- a/include/linux/mlx5/driver.h +++ b/include/linux/mlx5/driver.h @@ -53,6 +53,7 @@ #include #include #include +#include enum { MLX5_BOARD_ID_LEN = 64, @@ -443,6 +444,7 @@ struct mlx5_core_health { unsigned long flags; struct work_struct work; struct delayed_work recover_work; + struct devlink_health_reporter *fw_reporter; }; struct mlx5_qp_table { -- cgit v1.2.3 From d1bf0e2cc4a6e66c2bff48176b8b2930098468ef Mon Sep 17 00:00:00 2001 From: Moshe Shemesh Date: Tue, 11 Dec 2018 16:09:56 +0200 Subject: net/mlx5: Report devlink health on FW issues Use devlink_health_report() to report any symptom of FW issue as FW counter miss or new health syndrome. The FW issues detected in mlx5 during poll_health which is called in timer atomic context and so health work queue is used to schedule the reports. Signed-off-by: Moshe Shemesh Signed-off-by: Eran Ben Elisha Signed-off-by: Saeed Mahameed --- include/linux/mlx5/driver.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/mlx5/driver.h b/include/linux/mlx5/driver.h index 8d5d065d1aa6..1931a4080d78 100644 --- a/include/linux/mlx5/driver.h +++ b/include/linux/mlx5/driver.h @@ -435,7 +435,7 @@ struct mlx5_core_health { struct timer_list timer; u32 prev; int miss_counter; - bool sick; + u8 synd; u32 fatal_error; u32 crdump_size; /* wq spinlock to synchronize draining */ @@ -443,6 +443,7 @@ struct mlx5_core_health { struct workqueue_struct *wq; unsigned long flags; struct work_struct work; + struct work_struct report_work; struct delayed_work recover_work; struct devlink_health_reporter *fw_reporter; }; -- cgit v1.2.3 From 96c82cdfe77b5e769624af71ec0554434037b82f Mon Sep 17 00:00:00 2001 From: Moshe Shemesh Date: Tue, 11 Dec 2018 16:09:57 +0200 Subject: net/mlx5: Add fw fatal devlink_health_reporter Create mlx5_devlink_health_reporter for fw fatal reporter. The fw fatal reporter is added in addition to the fw reporter and implements the recover callback. The point of having two reporters for FW issues, is that we don't want to run FW recover on any issue, but only fatal ones. Signed-off-by: Moshe Shemesh Signed-off-by: Eran Ben Elisha Signed-off-by: Saeed Mahameed --- include/linux/mlx5/driver.h | 1 + 1 file changed, 1 insertion(+) (limited to 'include/linux') diff --git a/include/linux/mlx5/driver.h b/include/linux/mlx5/driver.h index 1931a4080d78..caac96bf9c0d 100644 --- a/include/linux/mlx5/driver.h +++ b/include/linux/mlx5/driver.h @@ -446,6 +446,7 @@ struct mlx5_core_health { struct work_struct report_work; struct delayed_work recover_work; struct devlink_health_reporter *fw_reporter; + struct devlink_health_reporter *fw_fatal_reporter; }; struct mlx5_qp_table { -- cgit v1.2.3 From b3bd076f7501afea2871bb4738ab53498fd32cd5 Mon Sep 17 00:00:00 2001 From: Moshe Shemesh Date: Sun, 27 Jan 2019 18:38:39 +0200 Subject: net/mlx5: Report devlink health on FW fatal issues Report devlink health on FW fatal issues via fw_fatal_reporter. The driver recover flow for FW fatal error is now being handled by the devlink health. Having the recovery controlled by devlink health, the user has the ability to cancel the auto-recovery for debug session and run it manually. Call mlx5_enter_error_state() before calling devlink_health_report() to ensure entering device error state even if auto-recovery is off. Signed-off-by: Moshe Shemesh Signed-off-by: Saeed Mahameed --- include/linux/mlx5/driver.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/mlx5/driver.h b/include/linux/mlx5/driver.h index caac96bf9c0d..25847beabd3f 100644 --- a/include/linux/mlx5/driver.h +++ b/include/linux/mlx5/driver.h @@ -442,7 +442,7 @@ struct mlx5_core_health { spinlock_t wq_lock; struct workqueue_struct *wq; unsigned long flags; - struct work_struct work; + struct work_struct fatal_report_work; struct work_struct report_work; struct delayed_work recover_work; struct devlink_health_reporter *fw_reporter; -- cgit v1.2.3