diff options
| -rw-r--r-- | drivers/net/ethernet/mellanox/mlx5/core/wc.c | 28 |
1 files changed, 26 insertions, 2 deletions
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/wc.c b/drivers/net/ethernet/mellanox/mlx5/core/wc.c index 999d6216648a..c281153bd411 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/wc.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/wc.c @@ -7,6 +7,10 @@ #include "mlx5_core.h" #include "wq.h" +#if IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && IS_ENABLED(CONFIG_ARM64) +#include <asm/neon.h> +#endif + #define TEST_WC_NUM_WQES 255 #define TEST_WC_LOG_CQ_SZ (order_base_2(TEST_WC_NUM_WQES)) #define TEST_WC_SQ_LOG_WQ_SZ TEST_WC_LOG_CQ_SZ @@ -255,6 +259,27 @@ static void mlx5_wc_destroy_sq(struct mlx5_wc_sq *sq) mlx5_wq_destroy(&sq->wq_ctrl); } +static void mlx5_iowrite64_copy(struct mlx5_wc_sq *sq, __be32 mmio_wqe[16], + size_t mmio_wqe_size, unsigned int offset) +{ +#if IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && IS_ENABLED(CONFIG_ARM64) + if (cpu_has_neon()) { + kernel_neon_begin(); + asm volatile + (".arch_extension simd;\n\t" + "ld1 {v0.16b, v1.16b, v2.16b, v3.16b}, [%0]\n\t" + "st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [%1]" + : + : "r"(mmio_wqe), "r"(sq->bfreg.map + offset) + : "memory", "v0", "v1", "v2", "v3"); + kernel_neon_end(); + return; + } +#endif + __iowrite64_copy(sq->bfreg.map + offset, mmio_wqe, + mmio_wqe_size / 8); +} + static void mlx5_wc_post_nop(struct mlx5_wc_sq *sq, unsigned int *offset, bool signaled) { @@ -289,8 +314,7 @@ static void mlx5_wc_post_nop(struct mlx5_wc_sq *sq, unsigned int *offset, */ wmb(); - __iowrite64_copy(sq->bfreg.map + *offset, mmio_wqe, - sizeof(mmio_wqe) / 8); + mlx5_iowrite64_copy(sq, mmio_wqe, sizeof(mmio_wqe), *offset); *offset ^= buf_size; } |