diff options
Diffstat (limited to 'drivers/net/ethernet/broadcom/bnxt/bnxt.c')
| -rw-r--r-- | drivers/net/ethernet/broadcom/bnxt/bnxt.c | 1329 |
1 files changed, 921 insertions, 408 deletions
diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt.c b/drivers/net/ethernet/broadcom/bnxt/bnxt.c index d17d0ea89c36..d7728d0c5b6e 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt.c @@ -59,10 +59,10 @@ #include <net/netdev_rx_queue.h> #include <linux/pci-tph.h> #include <linux/bnxt/hsi.h> +#include <linux/bnxt/ulp.h> #include "bnxt.h" #include "bnxt_hwrm.h" -#include "bnxt_ulp.h" #include "bnxt_sriov.h" #include "bnxt_ethtool.h" #include "bnxt_dcb.h" @@ -74,6 +74,8 @@ #include "bnxt_debugfs.h" #include "bnxt_coredump.h" #include "bnxt_hwmon.h" +#include "bnxt_gso.h" +#include <net/tso.h> #define BNXT_TX_TIMEOUT (5 * HZ) #define BNXT_DEF_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_HW | \ @@ -447,7 +449,7 @@ const u16 bnxt_lhint_arr[] = { TX_BD_FLAGS_LHINT_2048_AND_LARGER, }; -static u16 bnxt_xmit_get_cfa_action(struct sk_buff *skb) +u16 bnxt_xmit_get_cfa_action(struct sk_buff *skb) { struct metadata_dst *md_dst = skb_metadata_dst(skb); @@ -460,6 +462,16 @@ static u16 bnxt_xmit_get_cfa_action(struct sk_buff *skb) static void bnxt_txr_db_kick(struct bnxt *bp, struct bnxt_tx_ring_info *txr, u16 prod) { + /* If the most recent BD has its completion suppressed, unset the bit + * so that a completion is generated, otherwise nothing is left to + * clean the ring and wake the queue. + */ + if (txr->kick_txbd0) { + txr->kick_txbd0->tx_bd_len_flags_type &= + cpu_to_le32(~TX_BD_FLAGS_NO_CMPL); + txr->kick_txbd0 = NULL; + } + /* Sync BD data before updating doorbell */ wmb(); bnxt_db_write(bp, &txr->tx_db, prod); @@ -499,13 +511,31 @@ static netdev_tx_t bnxt_start_xmit(struct sk_buff *skb, struct net_device *dev) if (skb_shinfo(skb)->nr_frags > TX_MAX_FRAGS) { netdev_warn_once(dev, "SKB has too many (%d) fragments, max supported is %d. SKB will be linearized.\n", skb_shinfo(skb)->nr_frags, TX_MAX_FRAGS); - if (skb_linearize(skb)) { - dev_kfree_skb_any(skb); - dev_core_stats_tx_dropped_inc(dev); - return NETDEV_TX_OK; - } + if (skb_linearize(skb)) + goto tx_free; } #endif + if (skb_is_gso(skb) && + (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) && + !(bp->flags & BNXT_FLAG_UDP_GSO_CAP)) { + int rc = bnxt_sw_udp_gso_xmit(bp, txr, txq, skb); + + /* if SW USO queued a packet, the doorbell will be written + * below and there is no reason to track the last BD with + * suppressed completions + */ + if (rc > 0) + txr->kick_txbd0 = NULL; + + /* if a packet was queued by SW USO or a doorbell was pending + * from a previous xmit that was deferred, write the doorbell. + */ + if (rc > 0 || txr->kick_pending) + bnxt_txr_db_kick(bp, txr, txr->tx_prod); + + return rc < 0 ? NETDEV_TX_BUSY : NETDEV_TX_OK; + } + free_size = bnxt_tx_avail(bp, txr); if (unlikely(free_size < skb_shinfo(skb)->nr_frags + 2)) { /* We must have raced with NAPI cleanup */ @@ -517,9 +547,6 @@ static netdev_tx_t bnxt_start_xmit(struct sk_buff *skb, struct net_device *dev) return NETDEV_TX_BUSY; } - if (unlikely(ipv6_hopopt_jumbo_remove(skb))) - goto tx_free; - length = skb->len; len = skb_headlen(skb); last_frag = skb_shinfo(skb)->nr_frags; @@ -659,6 +686,7 @@ normal_tx: goto tx_free; dma_unmap_addr_set(tx_buf, mapping, mapping); + dma_unmap_len_set(tx_buf, len, len); flags = (len << TX_BD_LEN_SHIFT) | TX_BD_TYPE_LONG_TX_BD | TX_BD_CNT(last_frag + 2); @@ -666,10 +694,9 @@ normal_tx: txbd->tx_bd_opaque = SET_TX_OPAQUE(bp, txr, prod, 2 + last_frag); prod = NEXT_TX(prod); - txbd1 = (struct tx_bd_ext *) - &txr->tx_desc_ring[TX_RING(bp, prod)][TX_IDX(prod)]; + txbd1 = bnxt_init_ext_bd(bp, txr, prod, lflags, vlan_tag_flags, + cfa_action); - txbd1->tx_bd_hsize_lflags = lflags; if (skb_is_gso(skb)) { bool udp_gso = !!(skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4); u32 hdr_len; @@ -696,7 +723,6 @@ normal_tx: } else if (skb->ip_summed == CHECKSUM_PARTIAL) { txbd1->tx_bd_hsize_lflags |= cpu_to_le32(TX_BD_FLAGS_TCP_UDP_CHKSUM); - txbd1->tx_bd_mss = 0; } length >>= 9; @@ -709,9 +735,6 @@ normal_tx: flags |= bnxt_lhint_arr[length]; txbd->tx_bd_len_flags_type = cpu_to_le32(flags); - txbd1->tx_bd_cfa_meta = cpu_to_le32(vlan_tag_flags); - txbd1->tx_bd_cfa_action = - cpu_to_le32(cfa_action << TX_BD_CFA_ACTION_SHIFT); txbd0 = txbd; for (i = 0; i < last_frag; i++) { frag = &skb_shinfo(skb)->frags[i]; @@ -728,6 +751,7 @@ normal_tx: tx_buf = &txr->tx_buf_ring[RING_TX(bp, prod)]; netmem_dma_unmap_addr_set(skb_frag_netmem(frag), tx_buf, mapping, mapping); + dma_unmap_len_set(tx_buf, len, len); txbd->tx_bd_haddr = cpu_to_le64(mapping); @@ -747,23 +771,23 @@ normal_tx: prod = NEXT_TX(prod); WRITE_ONCE(txr->tx_prod, prod); + txr->kick_txbd0 = NULL; if (!netdev_xmit_more() || netif_xmit_stopped(txq)) { bnxt_txr_db_kick(bp, txr, prod); } else { - if (free_size >= bp->tx_wake_thresh) + if (free_size >= bp->tx_wake_thresh) { txbd0->tx_bd_len_flags_type |= cpu_to_le32(TX_BD_FLAGS_NO_CMPL); + txr->kick_txbd0 = txbd0; + } txr->kick_pending = 1; } tx_done: if (unlikely(bnxt_tx_avail(bp, txr) <= MAX_SKB_FRAGS + 1)) { - if (netdev_xmit_more() && !tx_buf->is_push) { - txbd0->tx_bd_len_flags_type &= - cpu_to_le32(~TX_BD_FLAGS_NO_CMPL); + if (txr->kick_pending) bnxt_txr_db_kick(bp, txr, prod); - } netif_txq_try_stop(txq, bnxt_tx_avail(bp, txr), bp->tx_wake_thresh); @@ -817,17 +841,19 @@ static bool __bnxt_tx_int(struct bnxt *bp, struct bnxt_tx_ring_info *txr, u16 hw_cons = txr->tx_hw_cons; unsigned int tx_bytes = 0; u16 cons = txr->tx_cons; - skb_frag_t *frag; + unsigned int dma_len; + dma_addr_t dma_addr; int tx_pkts = 0; bool rc = false; while (RING_TX(bp, cons) != hw_cons) { - struct bnxt_sw_tx_bd *tx_buf; + struct bnxt_sw_tx_bd *tx_buf, *head_buf; struct sk_buff *skb; bool is_ts_pkt; int j, last; tx_buf = &txr->tx_buf_ring[RING_TX(bp, cons)]; + head_buf = tx_buf; skb = tx_buf->skb; if (unlikely(!skb)) { @@ -852,20 +878,44 @@ static bool __bnxt_tx_int(struct bnxt *bp, struct bnxt_tx_ring_info *txr, goto next_tx_int; } - dma_unmap_single(&pdev->dev, dma_unmap_addr(tx_buf, mapping), - skb_headlen(skb), DMA_TO_DEVICE); + if (dma_unmap_len(tx_buf, len)) { + dma_addr = dma_unmap_addr(tx_buf, mapping); + dma_len = dma_unmap_len(tx_buf, len); + + dma_unmap_single(&pdev->dev, dma_addr, dma_len, + DMA_TO_DEVICE); + } + last = tx_buf->nr_frags; for (j = 0; j < last; j++) { - frag = &skb_shinfo(skb)->frags[j]; cons = NEXT_TX(cons); tx_buf = &txr->tx_buf_ring[RING_TX(bp, cons)]; - netmem_dma_unmap_page_attrs(&pdev->dev, - dma_unmap_addr(tx_buf, - mapping), - skb_frag_size(frag), - DMA_TO_DEVICE, 0); + if (dma_unmap_len(tx_buf, len)) { + dma_addr = dma_unmap_addr(tx_buf, mapping); + dma_len = dma_unmap_len(tx_buf, len); + + netmem_dma_unmap_page_attrs(&pdev->dev, + dma_addr, dma_len, + DMA_TO_DEVICE, 0); + } } + + if (unlikely(head_buf->is_sw_gso)) { + u16 inline_cons = txr->tx_inline_cons + 1; + + WRITE_ONCE(txr->tx_inline_cons, inline_cons); + if (head_buf->is_sw_gso == BNXT_SW_GSO_LAST) { + tso_dma_map_complete(&pdev->dev, + &head_buf->sw_gso_cstate); + } else { + tx_pkts--; + tx_bytes -= skb->len; + skb = NULL; + } + head_buf->is_sw_gso = 0; + } + if (unlikely(is_ts_pkt)) { if (BNXT_CHIP_P5(bp)) { /* PTP worker takes ownership of the skb */ @@ -905,7 +955,7 @@ static void bnxt_tx_int(struct bnxt *bp, struct bnxt_napi *bnapi, int budget) static bool bnxt_separate_head_pool(struct bnxt_rx_ring_info *rxr) { - return rxr->need_head_pool || PAGE_SIZE > BNXT_RX_PAGE_SIZE; + return rxr->need_head_pool || rxr->rx_page_size < PAGE_SIZE; } static struct page *__bnxt_alloc_rx_page(struct bnxt *bp, dma_addr_t *mapping, @@ -915,9 +965,9 @@ static struct page *__bnxt_alloc_rx_page(struct bnxt *bp, dma_addr_t *mapping, { struct page *page; - if (PAGE_SIZE > BNXT_RX_PAGE_SIZE) { + if (rxr->rx_page_size < PAGE_SIZE) { page = page_pool_dev_alloc_frag(rxr->page_pool, offset, - BNXT_RX_PAGE_SIZE); + rxr->rx_page_size); } else { page = page_pool_dev_alloc_pages(rxr->page_pool); *offset = 0; @@ -936,8 +986,9 @@ static netmem_ref __bnxt_alloc_rx_netmem(struct bnxt *bp, dma_addr_t *mapping, { netmem_ref netmem; - if (PAGE_SIZE > BNXT_RX_PAGE_SIZE) { - netmem = page_pool_alloc_frag_netmem(rxr->page_pool, offset, BNXT_RX_PAGE_SIZE, gfp); + if (rxr->rx_page_size < PAGE_SIZE) { + netmem = page_pool_alloc_frag_netmem(rxr->page_pool, offset, + rxr->rx_page_size, gfp); } else { netmem = page_pool_alloc_netmems(rxr->page_pool, gfp); *offset = 0; @@ -983,6 +1034,7 @@ int bnxt_alloc_rx_data(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, mapping += bp->rx_dma_offset; rx_buf->data = page; rx_buf->data_ptr = page_address(page) + offset + bp->rx_offset; + rx_buf->offset = offset; } else { u8 *data = __bnxt_alloc_rx_frag(bp, &mapping, rxr, gfp); @@ -991,6 +1043,7 @@ int bnxt_alloc_rx_data(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, rx_buf->data = data; rx_buf->data_ptr = data + bp->rx_offset; + rx_buf->offset = 0; } rx_buf->mapping = mapping; @@ -1012,6 +1065,7 @@ void bnxt_reuse_rx_data(struct bnxt_rx_ring_info *rxr, u16 cons, void *data) prod_rx_buf->data_ptr = cons_rx_buf->data_ptr; prod_rx_buf->mapping = cons_rx_buf->mapping; + prod_rx_buf->offset = cons_rx_buf->offset; prod_bd = &rxr->rx_desc_ring[RX_RING(bp, prod)][RX_IDX(prod)]; cons_bd = &rxr->rx_desc_ring[RX_RING(bp, cons)][RX_IDX(cons)]; @@ -1147,23 +1201,26 @@ static struct sk_buff *bnxt_rx_multi_page_skb(struct bnxt *bp, struct page *page = data; u16 prod = rxr->rx_prod; struct sk_buff *skb; + void *frag_start; int err; + frag_start = page_address(page) + rxr->rx_buf_ring[cons].offset; + err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC); if (unlikely(err)) { bnxt_reuse_rx_data(rxr, cons, data); return NULL; } dma_addr -= bp->rx_dma_offset; - dma_sync_single_for_cpu(&bp->pdev->dev, dma_addr, BNXT_RX_PAGE_SIZE, + dma_sync_single_for_cpu(&bp->pdev->dev, dma_addr, rxr->rx_page_size, bp->rx_dir); - skb = napi_build_skb(data_ptr - bp->rx_offset, BNXT_RX_PAGE_SIZE); + skb = napi_build_skb(frag_start, rxr->rx_page_size); if (!skb) { page_pool_recycle_direct(rxr->page_pool, page); return NULL; } skb_mark_for_recycle(skb); - skb_reserve(skb, bp->rx_offset); + skb_reserve(skb, data_ptr - (u8 *)frag_start); __skb_put(skb, len); return skb; @@ -1189,7 +1246,7 @@ static struct sk_buff *bnxt_rx_page_skb(struct bnxt *bp, return NULL; } dma_addr -= bp->rx_dma_offset; - dma_sync_single_for_cpu(&bp->pdev->dev, dma_addr, BNXT_RX_PAGE_SIZE, + dma_sync_single_for_cpu(&bp->pdev->dev, dma_addr, rxr->rx_page_size, bp->rx_dir); if (unlikely(!payload)) @@ -1203,7 +1260,7 @@ static struct sk_buff *bnxt_rx_page_skb(struct bnxt *bp, skb_mark_for_recycle(skb); off = (void *)data_ptr - page_address(page); - skb_add_rx_frag(skb, 0, page, off, len, BNXT_RX_PAGE_SIZE); + skb_add_rx_frag(skb, 0, page, off, len, rxr->rx_page_size); memcpy(skb->data - NET_IP_ALIGN, data_ptr - NET_IP_ALIGN, payload + NET_IP_ALIGN); @@ -1288,7 +1345,7 @@ static u32 __bnxt_rx_agg_netmems(struct bnxt *bp, if (skb) { skb_add_rx_frag_netmem(skb, i, cons_rx_buf->netmem, cons_rx_buf->offset, - frag_len, BNXT_RX_PAGE_SIZE); + frag_len, rxr->rx_page_size); } else { skb_frag_t *frag = &shinfo->frags[i]; @@ -1313,7 +1370,7 @@ static u32 __bnxt_rx_agg_netmems(struct bnxt *bp, if (skb) { skb->len -= frag_len; skb->data_len -= frag_len; - skb->truesize -= BNXT_RX_PAGE_SIZE; + skb->truesize -= rxr->rx_page_size; } --shinfo->nr_frags; @@ -1328,7 +1385,7 @@ static u32 __bnxt_rx_agg_netmems(struct bnxt *bp, } page_pool_dma_sync_netmem_for_cpu(rxr->page_pool, netmem, 0, - BNXT_RX_PAGE_SIZE); + rxr->rx_page_size); total_frag_len += frag_len; prod = NEXT_RX_AGG(prod); @@ -1477,14 +1534,18 @@ static int bnxt_discard_rx(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, return 0; } -static u16 bnxt_alloc_agg_idx(struct bnxt_rx_ring_info *rxr, u16 agg_id) +static u16 bnxt_alloc_agg_idx(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, + u16 agg_id) { struct bnxt_tpa_idx_map *map = rxr->rx_tpa_idx_map; - u16 idx = agg_id & MAX_TPA_P5_MASK; + u16 idx = agg_id & (bp->max_tpa_roundup_size - 1); - if (test_bit(idx, map->agg_idx_bmap)) + if (test_bit(idx, map->agg_idx_bmap)) { idx = find_first_zero_bit(map->agg_idx_bmap, - BNXT_AGG_IDX_BMAP_SIZE); + bp->max_tpa_roundup_size); + if (idx >= bp->max_tpa_roundup_size) + return INVALID_HW_RING_ID; + } __set_bit(idx, map->agg_idx_bmap); map->agg_id_tbl[agg_id] = idx; return idx; @@ -1547,7 +1608,14 @@ static void bnxt_tpa_start(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) { agg_id = TPA_START_AGG_ID_P5(tpa_start); - agg_id = bnxt_alloc_agg_idx(rxr, agg_id); + agg_id = bnxt_alloc_agg_idx(bp, rxr, agg_id); + if (unlikely(agg_id == INVALID_HW_RING_ID)) { + netdev_warn(bp->dev, "Unable to allocate agg ID for ring %d, agg 0x%x\n", + rxr->bnapi->index, + TPA_START_AGG_ID_P5(tpa_start)); + bnxt_sched_reset_rxr(bp, rxr); + return; + } } else { agg_id = TPA_START_AGG_ID(tpa_start); } @@ -1794,7 +1862,8 @@ static inline struct sk_buff *bnxt_gro_skb(struct bnxt *bp, struct bnxt_tpa_info *tpa_info, struct rx_tpa_end_cmp *tpa_end, struct rx_tpa_end_cmp_ext *tpa_end1, - struct sk_buff *skb) + struct sk_buff *skb, + struct bnxt_rx_sw_stats *rx_stats) { #ifdef CONFIG_INET int payload_off; @@ -1804,6 +1873,9 @@ static inline struct sk_buff *bnxt_gro_skb(struct bnxt *bp, if (segs == 1) return skb; + rx_stats->rx_hw_gro_packets++; + rx_stats->rx_hw_gro_wire_packets += segs; + NAPI_GRO_CB(skb)->count = segs; skb_shinfo(skb)->gso_size = le32_to_cpu(tpa_end1->rx_tpa_end_cmp_seg_len); @@ -1977,7 +2049,8 @@ static inline struct sk_buff *bnxt_tpa_end(struct bnxt *bp, } if (gro) - skb = bnxt_gro_skb(bp, tpa_info, tpa_end, tpa_end1, skb); + skb = bnxt_gro_skb(bp, tpa_info, tpa_end, tpa_end1, skb, + &cpr->sw_stats->rx); return skb; } @@ -2065,23 +2138,6 @@ vlan_err: return NULL; } -static enum pkt_hash_types bnxt_rss_ext_op(struct bnxt *bp, - struct rx_cmp *rxcmp) -{ - u8 ext_op; - - ext_op = RX_CMP_V3_HASH_TYPE(bp, rxcmp); - switch (ext_op) { - case EXT_OP_INNER_4: - case EXT_OP_OUTER_4: - case EXT_OP_INNFL_3: - case EXT_OP_OUTFL_3: - return PKT_HASH_TYPE_L4; - default: - return PKT_HASH_TYPE_L3; - } -} - /* returns the following: * 1 - 1 packet successfully received * 0 - successful TPA_START, packet not completed yet @@ -2100,13 +2156,13 @@ static int bnxt_rx_pkt(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, u32 tmp_raw_cons = *raw_cons; u16 cons, prod, cp_cons = RING_CMP(tmp_raw_cons); struct skb_shared_info *sinfo; + struct bnxt_xdp_buff bnxt_xdp; struct bnxt_sw_rx_bd *rx_buf; unsigned int len; u8 *data_ptr, agg_bufs, cmp_type; bool xdp_active = false; dma_addr_t dma_addr; struct sk_buff *skb; - struct xdp_buff xdp; u32 flags, misc; u32 cmpl_ts; void *data; @@ -2219,9 +2275,14 @@ static int bnxt_rx_pkt(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, dma_addr = rx_buf->mapping; if (bnxt_xdp_attached(bp, rxr)) { - bnxt_xdp_buff_init(bp, rxr, cons, data_ptr, len, &xdp); + bnxt_xdp.rxcmp = rxcmp; + bnxt_xdp.rxcmp1 = rxcmp1; + bnxt_xdp.cmp_type = cmp_type; + + bnxt_xdp_buff_init(bp, rxr, cons, data_ptr, len, &bnxt_xdp.xdp); if (agg_bufs) { - u32 frag_len = bnxt_rx_agg_netmems_xdp(bp, cpr, &xdp, + u32 frag_len = bnxt_rx_agg_netmems_xdp(bp, cpr, + &bnxt_xdp.xdp, cp_cons, agg_bufs, false); @@ -2233,12 +2294,13 @@ static int bnxt_rx_pkt(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, } if (xdp_active) { - if (bnxt_rx_xdp(bp, rxr, cons, &xdp, data, &data_ptr, &len, event)) { + if (bnxt_rx_xdp(bp, rxr, cons, &bnxt_xdp.xdp, data, &data_ptr, + &len, event)) { rc = 1; goto next_rx; } - if (xdp_buff_has_frags(&xdp)) { - sinfo = xdp_get_shared_info_from_buff(&xdp); + if (xdp_buff_has_frags(&bnxt_xdp.xdp)) { + sinfo = xdp_get_shared_info_from_buff(&bnxt_xdp.xdp); agg_bufs = sinfo->nr_frags; } else { agg_bufs = 0; @@ -2249,7 +2311,8 @@ static int bnxt_rx_pkt(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, if (!xdp_active) skb = bnxt_copy_skb(bnapi, data_ptr, len, dma_addr); else - skb = bnxt_copy_xdp(bnapi, &xdp, len, dma_addr); + skb = bnxt_copy_xdp(bnapi, &bnxt_xdp.xdp, len, + dma_addr); bnxt_reuse_rx_data(rxr, cons, data); if (!skb) { if (agg_bufs) { @@ -2257,7 +2320,8 @@ static int bnxt_rx_pkt(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, bnxt_reuse_rx_agg_bufs(cpr, cp_cons, 0, agg_bufs, false); else - bnxt_xdp_buff_frags_free(rxr, &xdp); + bnxt_xdp_buff_frags_free(rxr, + &bnxt_xdp.xdp); } goto oom_next_rx; } @@ -2282,10 +2346,10 @@ static int bnxt_rx_pkt(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, goto oom_next_rx; } else { skb = bnxt_xdp_build_skb(bp, skb, agg_bufs, - rxr->page_pool, &xdp); + rxr, &bnxt_xdp.xdp); if (!skb) { /* we should be able to free the old skb here */ - bnxt_xdp_buff_frags_free(rxr, &xdp); + bnxt_xdp_buff_frags_free(rxr, &bnxt_xdp.xdp); goto oom_next_rx; } } @@ -2918,6 +2982,8 @@ static int bnxt_async_event_process(struct bnxt *bp, u16 type = (u16)BNXT_EVENT_BUF_PRODUCER_TYPE(data1); u32 offset = BNXT_EVENT_BUF_PRODUCER_OFFSET(data2); + if (type >= ARRAY_SIZE(bp->bs_trace)) + goto async_event_process_exit; bnxt_bs_trace_check_wrap(&bp->bs_trace[type], offset); goto async_event_process_exit; } @@ -3394,19 +3460,23 @@ static void bnxt_free_one_tx_ring_skbs(struct bnxt *bp, { int i, max_idx; struct pci_dev *pdev = bp->pdev; + unsigned int dma_len; + dma_addr_t dma_addr; max_idx = bp->tx_nr_pages * TX_DESC_CNT; for (i = 0; i < max_idx;) { struct bnxt_sw_tx_bd *tx_buf = &txr->tx_buf_ring[i]; + struct bnxt_sw_tx_bd *head_buf = tx_buf; struct sk_buff *skb; int j, last; if (idx < bp->tx_nr_rings_xdp && tx_buf->action == XDP_REDIRECT) { - dma_unmap_single(&pdev->dev, - dma_unmap_addr(tx_buf, mapping), - dma_unmap_len(tx_buf, len), + dma_addr = dma_unmap_addr(tx_buf, mapping); + dma_len = dma_unmap_len(tx_buf, len); + + dma_unmap_single(&pdev->dev, dma_addr, dma_len, DMA_TO_DEVICE); xdp_return_frame(tx_buf->xdpf); tx_buf->action = 0; @@ -3429,25 +3499,43 @@ static void bnxt_free_one_tx_ring_skbs(struct bnxt *bp, continue; } - dma_unmap_single(&pdev->dev, - dma_unmap_addr(tx_buf, mapping), - skb_headlen(skb), - DMA_TO_DEVICE); + if (dma_unmap_len(tx_buf, len)) { + dma_addr = dma_unmap_addr(tx_buf, mapping); + dma_len = dma_unmap_len(tx_buf, len); + + dma_unmap_single(&pdev->dev, dma_addr, dma_len, + DMA_TO_DEVICE); + } last = tx_buf->nr_frags; i += 2; for (j = 0; j < last; j++, i++) { int ring_idx = i & bp->tx_ring_mask; - skb_frag_t *frag = &skb_shinfo(skb)->frags[j]; tx_buf = &txr->tx_buf_ring[ring_idx]; - netmem_dma_unmap_page_attrs(&pdev->dev, - dma_unmap_addr(tx_buf, - mapping), - skb_frag_size(frag), - DMA_TO_DEVICE, 0); + if (dma_unmap_len(tx_buf, len)) { + dma_addr = dma_unmap_addr(tx_buf, mapping); + dma_len = dma_unmap_len(tx_buf, len); + + netmem_dma_unmap_page_attrs(&pdev->dev, + dma_addr, dma_len, + DMA_TO_DEVICE, 0); + } } - dev_kfree_skb(skb); + if (head_buf->is_sw_gso) { + u16 inline_cons = txr->tx_inline_cons + 1; + + WRITE_ONCE(txr->tx_inline_cons, inline_cons); + if (head_buf->is_sw_gso == BNXT_SW_GSO_LAST) { + tso_dma_map_complete(&pdev->dev, + &head_buf->sw_gso_cstate); + } else { + skb = NULL; + } + head_buf->is_sw_gso = 0; + } + if (skb) + dev_kfree_skb(skb); } netdev_tx_reset_queue(netdev_get_tx_queue(bp->dev, idx)); } @@ -3518,7 +3606,7 @@ static void bnxt_free_one_tpa_info_data(struct bnxt *bp, { int i; - for (i = 0; i < bp->max_tpa; i++) { + for (i = 0; i < bp->max_tpa_roundup_size; i++) { struct bnxt_tpa_info *tpa_info = &rxr->rx_tpa[i]; u8 *data = tpa_info->data; @@ -3715,7 +3803,7 @@ static void bnxt_free_one_tpa_info(struct bnxt *bp, kfree(rxr->rx_tpa_idx_map); rxr->rx_tpa_idx_map = NULL; if (rxr->rx_tpa) { - for (i = 0; i < bp->max_tpa; i++) { + for (i = 0; i < bp->max_tpa_roundup_size; i++) { kfree(rxr->rx_tpa[i].agg_arr); rxr->rx_tpa[i].agg_arr = NULL; } @@ -3741,21 +3829,20 @@ static int bnxt_alloc_one_tpa_info(struct bnxt *bp, struct rx_agg_cmp *agg; int i; - rxr->rx_tpa = kcalloc(bp->max_tpa, sizeof(struct bnxt_tpa_info), - GFP_KERNEL); + rxr->rx_tpa = kzalloc_objs(struct bnxt_tpa_info, + bp->max_tpa_roundup_size); if (!rxr->rx_tpa) return -ENOMEM; if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS)) return 0; - for (i = 0; i < bp->max_tpa; i++) { - agg = kcalloc(MAX_SKB_FRAGS, sizeof(*agg), GFP_KERNEL); + for (i = 0; i < bp->max_tpa_roundup_size; i++) { + agg = kzalloc_objs(*agg, MAX_SKB_FRAGS); if (!agg) return -ENOMEM; rxr->rx_tpa[i].agg_arr = agg; } - rxr->rx_tpa_idx_map = kzalloc(sizeof(*rxr->rx_tpa_idx_map), - GFP_KERNEL); + rxr->rx_tpa_idx_map = kzalloc_obj(*rxr->rx_tpa_idx_map); if (!rxr->rx_tpa_idx_map) return -ENOMEM; @@ -3768,10 +3855,17 @@ static int bnxt_alloc_tpa_info(struct bnxt *bp) bp->max_tpa = MAX_TPA; if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) { + /* TPA is not supported at all, so there is nothing to + * allocate. + */ if (!bp->max_tpa_v2) return 0; - bp->max_tpa = max_t(u16, bp->max_tpa_v2, MAX_TPA_P5); + bp->max_tpa = min_t(u16, bp->max_tpa_v2, MAX_TPA_P5); + /* Older P5 FW sets max_tpa_v2 low by mistake except NPAR */ + if (bp->max_tpa <= 32 && BNXT_CHIP_P5(bp) && !BNXT_NPAR(bp)) + bp->max_tpa = MAX_TPA_P5; } + bp->max_tpa_roundup_size = roundup_pow_of_two(bp->max_tpa); for (i = 0; i < bp->rx_nr_rings; i++) { struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; @@ -3816,23 +3910,40 @@ static void bnxt_free_rx_rings(struct bnxt *bp) } } +static int bnxt_rx_agg_ring_fill_level(struct bnxt *bp, + struct bnxt_rx_ring_info *rxr) +{ + /* User may have chosen larger than default rx_page_size, + * we keep the ring sizes uniform and also want uniform amount + * of bytes consumed per ring, so cap how much of the rings we fill. + */ + int fill_level = bp->rx_agg_ring_size; + + if (rxr->rx_page_size > BNXT_RX_PAGE_SIZE) + fill_level /= rxr->rx_page_size / BNXT_RX_PAGE_SIZE; + + return fill_level; +} + static int bnxt_alloc_rx_page_pool(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, int numa_node) { - const unsigned int agg_size_fac = PAGE_SIZE / BNXT_RX_PAGE_SIZE; + unsigned int agg_size_fac = rxr->rx_page_size / BNXT_RX_PAGE_SIZE; const unsigned int rx_size_fac = PAGE_SIZE / SZ_4K; struct page_pool_params pp = { 0 }; struct page_pool *pool; - pp.pool_size = bp->rx_agg_ring_size / agg_size_fac; + pp.pool_size = bnxt_rx_agg_ring_fill_level(bp, rxr) / agg_size_fac; if (BNXT_RX_PAGE_MODE(bp)) pp.pool_size += bp->rx_ring_size / rx_size_fac; + + pp.order = get_order(rxr->rx_page_size); pp.nid = numa_node; pp.netdev = bp->dev; pp.dev = &bp->pdev->dev; pp.dma_dir = bp->rx_dir; - pp.max_len = PAGE_SIZE; + pp.max_len = PAGE_SIZE << pp.order; pp.flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV | PP_FLAG_ALLOW_UNREADABLE_NETMEM; pp.queue_idx = rxr->bnapi->index; @@ -3843,7 +3954,10 @@ static int bnxt_alloc_rx_page_pool(struct bnxt *bp, rxr->page_pool = pool; rxr->need_head_pool = page_pool_is_unreadable(pool); + rxr->need_head_pool |= !!pp.order; if (bnxt_separate_head_pool(rxr)) { + pp.order = 0; + pp.max_len = PAGE_SIZE; pp.pool_size = min(bp->rx_ring_size / rx_size_fac, 1024); pp.flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV; pool = page_pool_create(&pp); @@ -3942,6 +4056,39 @@ static int bnxt_alloc_rx_rings(struct bnxt *bp) return rc; } +static void bnxt_free_tx_inline_buf(struct bnxt_tx_ring_info *txr, + struct pci_dev *pdev) +{ + if (!txr->tx_inline_buf) + return; + + dma_unmap_single(&pdev->dev, txr->tx_inline_dma, + txr->tx_inline_size, DMA_TO_DEVICE); + kfree(txr->tx_inline_buf); + txr->tx_inline_buf = NULL; + txr->tx_inline_size = 0; +} + +static int bnxt_alloc_tx_inline_buf(struct bnxt_tx_ring_info *txr, + struct pci_dev *pdev, + unsigned int size) +{ + txr->tx_inline_buf = kmalloc(size, GFP_KERNEL); + if (!txr->tx_inline_buf) + return -ENOMEM; + + txr->tx_inline_dma = dma_map_single(&pdev->dev, txr->tx_inline_buf, + size, DMA_TO_DEVICE); + if (dma_mapping_error(&pdev->dev, txr->tx_inline_dma)) { + kfree(txr->tx_inline_buf); + txr->tx_inline_buf = NULL; + return -ENOMEM; + } + txr->tx_inline_size = size; + + return 0; +} + static void bnxt_free_tx_rings(struct bnxt *bp) { int i; @@ -3960,6 +4107,8 @@ static void bnxt_free_tx_rings(struct bnxt *bp) txr->tx_push = NULL; } + bnxt_free_tx_inline_buf(txr, pdev); + ring = &txr->tx_ring_struct; bnxt_free_ring(bp, &ring->ring_mem); @@ -4025,6 +4174,13 @@ static int bnxt_alloc_tx_rings(struct bnxt *bp) sizeof(struct tx_push_bd); txr->data_mapping = cpu_to_le64(mapping); } + if (!(bp->flags & BNXT_FLAG_UDP_GSO_CAP)) { + rc = bnxt_alloc_tx_inline_buf(txr, pdev, + BNXT_SW_USO_MAX_SEGS * + TSO_HEADER_SIZE); + if (rc) + return rc; + } qidx = bp->tc_to_qidx[j]; ring->queue_id = bp->q_info[qidx].queue_id; spin_lock_init(&txr->xdp_tx_lock); @@ -4050,11 +4206,10 @@ static void bnxt_free_cp_arrays(struct bnxt_cp_ring_info *cpr) static int bnxt_alloc_cp_arrays(struct bnxt_cp_ring_info *cpr, int n) { - cpr->cp_desc_ring = kcalloc(n, sizeof(*cpr->cp_desc_ring), GFP_KERNEL); + cpr->cp_desc_ring = kzalloc_objs(*cpr->cp_desc_ring, n); if (!cpr->cp_desc_ring) return -ENOMEM; - cpr->cp_desc_mapping = kcalloc(n, sizeof(*cpr->cp_desc_mapping), - GFP_KERNEL); + cpr->cp_desc_mapping = kzalloc_objs(*cpr->cp_desc_mapping, n); if (!cpr->cp_desc_mapping) return -ENOMEM; return 0; @@ -4201,8 +4356,7 @@ static int bnxt_alloc_cp_rings(struct bnxt *bp) tx = 1; } - cpr->cp_ring_arr = kcalloc(cp_count, sizeof(*cpr), - GFP_KERNEL); + cpr->cp_ring_arr = kzalloc_objs(*cpr, cp_count); if (!cpr->cp_ring_arr) return -ENOMEM; cpr->cp_ring_count = cp_count; @@ -4297,6 +4451,7 @@ static void bnxt_init_ring_struct(struct bnxt *bp) for (i = 0; i < bp->cp_nr_rings; i++) { struct bnxt_napi *bnapi = bp->bnapi[i]; + struct netdev_queue_config qcfg; struct bnxt_ring_mem_info *rmem; struct bnxt_cp_ring_info *cpr; struct bnxt_rx_ring_info *rxr; @@ -4319,6 +4474,9 @@ static void bnxt_init_ring_struct(struct bnxt *bp) if (!rxr) goto skip_rx; + netdev_queue_config(bp->dev, i, &qcfg); + rxr->rx_page_size = qcfg.rx_page_size; + ring = &rxr->rx_ring_struct; rmem = &ring->ring_mem; rmem->nr_pages = bp->rx_nr_pages; @@ -4396,11 +4554,13 @@ static void bnxt_alloc_one_rx_ring_netmem(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, int ring_nr) { + int fill_level, i; u32 prod; - int i; + + fill_level = bnxt_rx_agg_ring_fill_level(bp, rxr); prod = rxr->rx_agg_prod; - for (i = 0; i < bp->rx_agg_ring_size; i++) { + for (i = 0; i < fill_level; i++) { if (bnxt_alloc_rx_netmem(bp, rxr, prod, GFP_KERNEL)) { netdev_warn(bp->dev, "init'ed rx ring %d with %d/%d pages only\n", ring_nr, i, bp->rx_agg_ring_size); @@ -4418,7 +4578,7 @@ static int bnxt_alloc_one_tpa_info_data(struct bnxt *bp, u8 *data; int i; - for (i = 0; i < bp->max_tpa; i++) { + for (i = 0; i < bp->max_tpa_roundup_size; i++) { data = __bnxt_alloc_rx_frag(bp, &mapping, rxr, GFP_KERNEL); if (!data) @@ -4478,14 +4638,17 @@ static void bnxt_init_one_rx_agg_ring_rxbd(struct bnxt *bp, ring = &rxr->rx_agg_ring_struct; ring->fw_ring_id = INVALID_HW_RING_ID; if ((bp->flags & BNXT_FLAG_AGG_RINGS)) { - type = ((u32)BNXT_RX_PAGE_SIZE << RX_BD_LEN_SHIFT) | + type = ((u32)rxr->rx_page_size << RX_BD_LEN_SHIFT) | RX_BD_TYPE_RX_AGG_BD; - /* On P7, setting EOP will cause the chip to disable - * Relaxed Ordering (RO) for TPA data. Disable EOP for - * potentially higher performance with RO. + /* Disable EOP if TPA is enabled to prevent overlapping zero + * padding with the next segment's data. On P7_PLUS, EOP will + * automatically disable Relaxed Ordering (RO) to prevent + * potential data corruption (and may degrade performance). On + * older chips, RO will not be automatically disabled and may + * cause corruption. */ - if (BNXT_CHIP_P5_AND_MINUS(bp) || !(bp->flags & BNXT_FLAG_TPA)) + if (!(bp->flags & BNXT_FLAG_TPA)) type |= RX_BD_FLAGS_AGG_EOP; bnxt_init_rxbd_pages(ring, type); @@ -4559,10 +4722,13 @@ static int bnxt_init_rx_rings(struct bnxt *bp) static int bnxt_init_tx_rings(struct bnxt *bp) { + netdev_features_t features; u16 i; + features = bp->dev->features; + bp->tx_wake_thresh = max_t(int, bp->tx_ring_size / 2, - BNXT_MIN_TX_DESC_CNT); + bnxt_min_tx_desc_cnt(bp, features)); for (i = 0; i < bp->tx_nr_rings; i++) { struct bnxt_tx_ring_info *txr = &bp->tx_ring[i]; @@ -4590,9 +4756,8 @@ static int bnxt_init_ring_grps(struct bnxt *bp, bool irq_re_init) int i; if (irq_re_init) { - bp->grp_info = kcalloc(bp->cp_nr_rings, - sizeof(struct bnxt_ring_grp_info), - GFP_KERNEL); + bp->grp_info = kzalloc_objs(struct bnxt_ring_grp_info, + bp->cp_nr_rings); if (!bp->grp_info) return -ENOMEM; } @@ -4630,8 +4795,7 @@ static int bnxt_alloc_vnics(struct bnxt *bp) if (BNXT_CHIP_TYPE_NITRO_A0(bp)) num_vnics++; - bp->vnic_info = kcalloc(num_vnics, sizeof(struct bnxt_vnic_info), - GFP_KERNEL); + bp->vnic_info = kzalloc_objs(struct bnxt_vnic_info, num_vnics); if (!bp->vnic_info) return -ENOMEM; @@ -4721,11 +4885,11 @@ static void bnxt_init_ring_params(struct bnxt *bp) { unsigned int rx_size; - bp->rx_copybreak = BNXT_DEFAULT_RX_COPYBREAK; + bp->rx_copybreak = 0; /* rx-copybreak disabled by default */ /* Try to fit 4 chunks into a 4k page */ rx_size = SZ_1K - NET_SKB_PAD - SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); - bp->dev->cfg->hds_thresh = max(BNXT_DEFAULT_RX_COPYBREAK, rx_size); + bp->dev->cfg->hds_thresh = max(BNXT_MIN_RX_HDR_BUF, rx_size); } /* bp->rx_ring_size, bp->tx_ring_size, dev->mtu, BNXT_FLAG_{G|L}RO flags must @@ -4786,7 +4950,7 @@ void bnxt_set_ring_params(struct bnxt *bp) ALIGN(max(NET_SKB_PAD, XDP_PACKET_HEADROOM), 8) - SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); } else { - rx_size = max3(BNXT_DEFAULT_RX_COPYBREAK, + rx_size = max3(BNXT_MIN_RX_HDR_BUF, bp->rx_copybreak, bp->dev->cfg_pending->hds_thresh); rx_size = SKB_DATA_ALIGN(rx_size + NET_IP_ALIGN); @@ -4869,7 +5033,8 @@ void bnxt_set_rx_skb_mode(struct bnxt *bp, bool page_mode) bnxt_get_max_rings(bp, &rx, &tx, true); if (rx > 1) { bp->flags &= ~BNXT_FLAG_NO_AGG_RINGS; - bp->dev->hw_features |= NETIF_F_LRO; + if (BNXT_SUPPORTS_TPA(bp)) + bp->dev->hw_features |= NETIF_F_LRO; } } @@ -5211,7 +5376,7 @@ static int bnxt_alloc_stats(struct bnxt *bp) struct bnxt_napi *bnapi = bp->bnapi[i]; struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; - cpr->sw_stats = kzalloc(sizeof(*cpr->sw_stats), GFP_KERNEL); + cpr->sw_stats = kzalloc_obj(*cpr->sw_stats); if (!cpr->sw_stats) return -ENOMEM; @@ -5290,6 +5455,8 @@ static void bnxt_clear_ring_indices(struct bnxt *bp) txr->tx_prod = 0; txr->tx_cons = 0; txr->tx_hw_cons = 0; + txr->kick_pending = 0; + txr->kick_txbd0 = NULL; } rxr = bnapi->rx_ring; @@ -5479,9 +5646,8 @@ static int bnxt_alloc_mem(struct bnxt *bp, bool irq_re_init) } } - bp->rx_ring = kcalloc(bp->rx_nr_rings, - sizeof(struct bnxt_rx_ring_info), - GFP_KERNEL); + bp->rx_ring = kzalloc_objs(struct bnxt_rx_ring_info, + bp->rx_nr_rings); if (!bp->rx_ring) return -ENOMEM; @@ -5500,9 +5666,8 @@ static int bnxt_alloc_mem(struct bnxt *bp, bool irq_re_init) bp->bnapi[i]->rx_ring = &bp->rx_ring[i]; } - bp->tx_ring = kcalloc(bp->tx_nr_rings, - sizeof(struct bnxt_tx_ring_info), - GFP_KERNEL); + bp->tx_ring = kzalloc_objs(struct bnxt_tx_ring_info, + bp->tx_nr_rings); if (!bp->tx_ring) return -ENOMEM; @@ -5625,7 +5790,7 @@ static void bnxt_disable_int_sync(struct bnxt *bp) { int i; - if (!bp->irq_tbl) + if (!bp->irq_tbl || !bp->bnapi) return; atomic_inc(&bp->intr_sem); @@ -6059,7 +6224,7 @@ static struct bnxt_l2_filter *bnxt_alloc_l2_filter(struct bnxt *bp, if (fltr) return fltr; - fltr = kzalloc(sizeof(*fltr), gfp); + fltr = kzalloc_obj(*fltr, gfp); if (!fltr) return ERR_PTR(-ENOMEM); spin_lock_bh(&bp->ntp_fltr_lock); @@ -6088,7 +6253,7 @@ struct bnxt_l2_filter *bnxt_alloc_new_l2_filter(struct bnxt *bp, fltr = ERR_PTR(-EEXIST); goto l2_filter_exit; } - fltr = kzalloc(sizeof(*fltr), GFP_ATOMIC); + fltr = kzalloc_obj(*fltr, GFP_ATOMIC); if (!fltr) { fltr = ERR_PTR(-ENOMEM); goto l2_filter_exit; @@ -6203,6 +6368,9 @@ int bnxt_hwrm_cfa_ntuple_filter_free(struct bnxt *bp, int rc; set_bit(BNXT_FLTR_FW_DELETED, &fltr->base.state); + if (!test_bit(BNXT_STATE_OPEN, &bp->state)) + return 0; + rc = hwrm_req_init(bp, req, HWRM_CFA_NTUPLE_FILTER_FREE); if (rc) return rc; @@ -6558,6 +6726,9 @@ int bnxt_get_nr_rss_ctxs(struct bnxt *bp, int rx_rings) if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) { if (!rx_rings) return 0; + if (bp->rss_cap & BNXT_RSS_CAP_LARGE_RSS_CTX) + return BNXT_RSS_TABLE_MAX_TBL_P5; + return bnxt_calc_nr_ring_pages(rx_rings - 1, BNXT_RSS_TABLE_ENTRIES_P5); } @@ -6569,22 +6740,36 @@ int bnxt_get_nr_rss_ctxs(struct bnxt *bp, int rx_rings) static void bnxt_fill_hw_rss_tbl(struct bnxt *bp, struct bnxt_vnic_info *vnic) { bool no_rss = !(vnic->flags & BNXT_VNIC_RSS_FLAG); - u16 i, j; + u16 i, j, min_j = bp->rx_nr_rings - 1; + + if (!vnic->rss_table) + goto skip_rss_tbl; /* Fill the RSS indirection table with ring group ids */ for (i = 0, j = 0; i < HW_HASH_INDEX_SIZE; i++) { if (!no_rss) j = bp->rss_indir_tbl[i]; + min_j = min(j, min_j); vnic->rss_table[i] = cpu_to_le16(vnic->fw_grp_ids[j]); } + +skip_rss_tbl: + if (vnic->rss_table && !no_rss) + vnic->default_rx_ring = min_j; + else if (vnic->flags & BNXT_VNIC_RFS_FLAG) + vnic->default_rx_ring = vnic->vnic_id - 1; + else if ((vnic->vnic_id == 1) && BNXT_CHIP_TYPE_NITRO_A0(bp)) + vnic->default_rx_ring = bp->rx_nr_rings - 1; + else + vnic->default_rx_ring = 0; } static void bnxt_fill_hw_rss_tbl_p5(struct bnxt *bp, struct bnxt_vnic_info *vnic) { + u16 tbl_size, i, min_j = bp->rx_nr_rings - 1; __le16 *ring_tbl = vnic->rss_table; struct bnxt_rx_ring_info *rxr; - u16 tbl_size, i; tbl_size = bnxt_get_rxfh_indir_size(bp->dev); @@ -6597,6 +6782,7 @@ static void bnxt_fill_hw_rss_tbl_p5(struct bnxt *bp, j = ethtool_rxfh_context_indir(vnic->rss_ctx)[i]; else j = bp->rss_indir_tbl[i]; + min_j = min(j, min_j); rxr = &bp->rx_ring[j]; ring_id = rxr->rx_ring_struct.fw_ring_id; @@ -6604,19 +6790,15 @@ static void bnxt_fill_hw_rss_tbl_p5(struct bnxt *bp, ring_id = bnxt_cp_ring_for_rx(bp, rxr); *ring_tbl++ = cpu_to_le16(ring_id); } + vnic->default_rx_ring = min_j; } static void __bnxt_hwrm_vnic_set_rss(struct bnxt *bp, struct hwrm_vnic_rss_cfg_input *req, struct bnxt_vnic_info *vnic) { - if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) { - bnxt_fill_hw_rss_tbl_p5(bp, vnic); - if (bp->flags & BNXT_FLAG_CHIP_P7) - req->flags |= VNIC_RSS_CFG_REQ_FLAGS_IPSEC_HASH_TYPE_CFG_SUPPORT; - } else { - bnxt_fill_hw_rss_tbl(bp, vnic); - } + if (bp->flags & BNXT_FLAG_CHIP_P7) + req->flags |= VNIC_RSS_CFG_REQ_FLAGS_IPSEC_HASH_TYPE_CFG_SUPPORT; if (bp->rss_hash_delta) { req->hash_type = cpu_to_le32(bp->rss_hash_delta); @@ -6668,6 +6850,7 @@ static int bnxt_hwrm_vnic_set_rss_p5(struct bnxt *bp, if (!set_rss) return hwrm_req_send(bp, req); + bnxt_fill_hw_rss_tbl_p5(bp, vnic); __bnxt_hwrm_vnic_set_rss(bp, req, vnic); ring_tbl_map = vnic->rss_table_dma_addr; nr_ctxs = bnxt_get_nr_rss_ctxs(bp, bp->rx_nr_rings); @@ -6804,8 +6987,9 @@ int bnxt_hwrm_vnic_cfg(struct bnxt *bp, struct bnxt_vnic_info *vnic) return rc; if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) { - struct bnxt_rx_ring_info *rxr = &bp->rx_ring[0]; + struct bnxt_rx_ring_info *rxr; + rxr = &bp->rx_ring[vnic->default_rx_ring]; req->default_rx_ring_id = cpu_to_le16(rxr->rx_ring_struct.fw_ring_id); req->default_cmpl_ring_id = @@ -6838,13 +7022,7 @@ int bnxt_hwrm_vnic_cfg(struct bnxt *bp, struct bnxt_vnic_info *vnic) req->cos_rule = cpu_to_le16(0xffff); } - if (vnic->flags & BNXT_VNIC_RSS_FLAG) - ring = 0; - else if (vnic->flags & BNXT_VNIC_RFS_FLAG) - ring = vnic->vnic_id - 1; - else if ((vnic->vnic_id == 1) && BNXT_CHIP_TYPE_NITRO_A0(bp)) - ring = bp->rx_nr_rings - 1; - + ring = vnic->default_rx_ring; grp_idx = bp->rx_ring[ring].bnapi->index; req->dflt_ring_grp = cpu_to_le16(bp->grp_info[grp_idx].fw_grp_id); req->lb_rule = cpu_to_le16(0xffff); @@ -6859,7 +7037,8 @@ vnic_mru: #endif if ((bp->flags & BNXT_FLAG_STRIP_VLAN) || def_vlan) req->flags |= cpu_to_le32(VNIC_CFG_REQ_FLAGS_VLAN_STRIP_MODE); - if (vnic->vnic_id == BNXT_VNIC_DEFAULT && bnxt_ulp_registered(bp->edev)) + if (vnic->vnic_id == BNXT_VNIC_DEFAULT && + bnxt_ulp_registered(bp->edev[BNXT_AUXDEV_RDMA])) req->flags |= cpu_to_le32(bnxt_get_roce_vnic_mode(bp)); return hwrm_req_send(bp, req); @@ -7056,6 +7235,7 @@ static void bnxt_hwrm_ring_grp_free(struct bnxt *bp) static void bnxt_set_rx_ring_params_p5(struct bnxt *bp, u32 ring_type, struct hwrm_ring_alloc_input *req, + struct bnxt_rx_ring_info *rxr, struct bnxt_ring_struct *ring) { struct bnxt_ring_grp_info *grp_info = &bp->grp_info[ring->grp_idx]; @@ -7065,7 +7245,7 @@ static void bnxt_set_rx_ring_params_p5(struct bnxt *bp, u32 ring_type, if (ring_type == HWRM_RING_ALLOC_AGG) { req->ring_type = RING_ALLOC_REQ_RING_TYPE_RX_AGG; req->rx_ring_id = cpu_to_le16(grp_info->rx_fw_ring_id); - req->rx_buf_size = cpu_to_le16(BNXT_RX_PAGE_SIZE); + req->rx_buf_size = cpu_to_le16(rxr->rx_page_size); enables |= RING_ALLOC_REQ_ENABLES_RX_RING_ID_VALID; } else { req->rx_buf_size = cpu_to_le16(bp->rx_buf_use_size); @@ -7079,6 +7259,7 @@ static void bnxt_set_rx_ring_params_p5(struct bnxt *bp, u32 ring_type, } static int hwrm_ring_alloc_send_msg(struct bnxt *bp, + struct bnxt_rx_ring_info *rxr, struct bnxt_ring_struct *ring, u32 ring_type, u32 map_index) { @@ -7135,7 +7316,8 @@ static int hwrm_ring_alloc_send_msg(struct bnxt *bp, cpu_to_le32(bp->rx_ring_mask + 1) : cpu_to_le32(bp->rx_agg_ring_mask + 1); if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) - bnxt_set_rx_ring_params_p5(bp, ring_type, req, ring); + bnxt_set_rx_ring_params_p5(bp, ring_type, req, + rxr, ring); break; case HWRM_RING_ALLOC_CMPL: req->ring_type = RING_ALLOC_REQ_RING_TYPE_L2_CMPL; @@ -7283,7 +7465,7 @@ static int bnxt_hwrm_rx_ring_alloc(struct bnxt *bp, u32 map_idx = bnapi->index; int rc; - rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx); + rc = hwrm_ring_alloc_send_msg(bp, rxr, ring, type, map_idx); if (rc) return rc; @@ -7303,7 +7485,7 @@ static int bnxt_hwrm_rx_agg_ring_alloc(struct bnxt *bp, int rc; map_idx = grp_idx + bp->rx_nr_rings; - rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx); + rc = hwrm_ring_alloc_send_msg(bp, rxr, ring, type, map_idx); if (rc) return rc; @@ -7327,7 +7509,7 @@ static int bnxt_hwrm_cp_ring_alloc_p5(struct bnxt *bp, ring = &cpr->cp_ring_struct; ring->handle = BNXT_SET_NQ_HDL(cpr); - rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx); + rc = hwrm_ring_alloc_send_msg(bp, NULL, ring, type, map_idx); if (rc) return rc; bnxt_set_db(bp, &cpr->cp_db, type, map_idx, ring->fw_ring_id); @@ -7342,7 +7524,7 @@ static int bnxt_hwrm_tx_ring_alloc(struct bnxt *bp, const u32 type = HWRM_RING_ALLOC_TX; int rc; - rc = hwrm_ring_alloc_send_msg(bp, ring, type, tx_idx); + rc = hwrm_ring_alloc_send_msg(bp, NULL, ring, type, tx_idx); if (rc) return rc; bnxt_set_db(bp, &txr->tx_db, type, tx_idx, ring->fw_ring_id); @@ -7368,7 +7550,7 @@ static int bnxt_hwrm_ring_alloc(struct bnxt *bp) vector = bp->irq_tbl[map_idx].vector; disable_irq_nosync(vector); - rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx); + rc = hwrm_ring_alloc_send_msg(bp, NULL, ring, type, map_idx); if (rc) { enable_irq(vector); goto err_out; @@ -7908,13 +8090,28 @@ static int bnxt_get_total_vnics(struct bnxt *bp, int rx_rings) return 1; } +static void bnxt_get_total_resources(struct bnxt *bp, struct bnxt_hw_rings *hwr) +{ + hwr->cp = bnxt_nq_rings_in_use(bp); + hwr->cp_p5 = 0; + if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) + hwr->cp_p5 = bnxt_cp_rings_in_use(bp); + hwr->tx = bp->tx_nr_rings; + hwr->rx = bp->rx_nr_rings; + hwr->grp = hwr->rx; + hwr->vnic = bnxt_get_total_vnics(bp, hwr->rx); + hwr->rss_ctx = bnxt_get_total_rss_ctxs(bp, hwr); + if (bp->flags & BNXT_FLAG_AGG_RINGS) + hwr->rx <<= 1; + hwr->stat = bnxt_get_func_stat_ctxs(bp); +} + static bool bnxt_need_reserve_rings(struct bnxt *bp) { struct bnxt_hw_resc *hw_resc = &bp->hw_resc; - int cp = bnxt_cp_rings_in_use(bp); - int nq = bnxt_nq_rings_in_use(bp); - int rx = bp->rx_nr_rings, stat; - int vnic, grp = rx; + struct bnxt_hw_rings hwr; + + bnxt_get_total_resources(bp, &hwr); /* Old firmware does not need RX ring reservations but we still * need to setup a default RSS map when needed. With new firmware @@ -7924,25 +8121,27 @@ static bool bnxt_need_reserve_rings(struct bnxt *bp) if (!BNXT_NEW_RM(bp)) bnxt_check_rss_tbl_no_rmgr(bp); - if (hw_resc->resv_tx_rings != bp->tx_nr_rings && - bp->hwrm_spec_code >= 0x10601) + if (hw_resc->resv_tx_rings != hwr.tx && bp->hwrm_spec_code >= 0x10601) return true; if (!BNXT_NEW_RM(bp)) return false; - vnic = bnxt_get_total_vnics(bp, rx); - - if (bp->flags & BNXT_FLAG_AGG_RINGS) - rx <<= 1; - stat = bnxt_get_func_stat_ctxs(bp); - if (hw_resc->resv_rx_rings != rx || hw_resc->resv_cp_rings != cp || - hw_resc->resv_vnics != vnic || hw_resc->resv_stat_ctxs != stat || - (hw_resc->resv_hw_ring_grps != grp && + if (hw_resc->resv_rx_rings != hwr.rx || + hw_resc->resv_vnics != hwr.vnic || + hw_resc->resv_stat_ctxs != hwr.stat || + hw_resc->resv_rsscos_ctxs != hwr.rss_ctx || + (hw_resc->resv_hw_ring_grps != hwr.grp && !(bp->flags & BNXT_FLAG_CHIP_P5_PLUS))) return true; + if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) { + if (hw_resc->resv_cp_rings != hwr.cp_p5) + return true; + } else if (hw_resc->resv_cp_rings != hwr.cp) { + return true; + } if ((bp->flags & BNXT_FLAG_CHIP_P5_PLUS) && BNXT_PF(bp) && - hw_resc->resv_irqs != nq) + hw_resc->resv_irqs != hwr.cp) return true; return false; } @@ -7974,6 +8173,7 @@ static int bnxt_get_avail_msix(struct bnxt *bp, int num); static int __bnxt_reserve_rings(struct bnxt *bp) { + struct bnxt_en_dev *edev = bp->edev[BNXT_AUXDEV_RDMA]; struct bnxt_hw_rings hwr = {0}; int rx_rings, old_rx_rings, rc; int cp = bp->cp_nr_rings; @@ -7984,10 +8184,12 @@ static int __bnxt_reserve_rings(struct bnxt *bp) if (!bnxt_need_reserve_rings(bp)) return 0; - if (BNXT_NEW_RM(bp) && !bnxt_ulp_registered(bp->edev)) { + if (BNXT_NEW_RM(bp) && !bnxt_ulp_registered(edev)) { ulp_msix = bnxt_get_avail_msix(bp, bp->ulp_num_msix_want); if (!ulp_msix) bnxt_set_ulp_stat_ctxs(bp, 0); + else + bnxt_set_dflt_ulp_stat_ctxs(bp); if (ulp_msix > bp->ulp_num_msix_want) ulp_msix = bp->ulp_num_msix_want; @@ -8035,8 +8237,7 @@ static int __bnxt_reserve_rings(struct bnxt *bp) } rx_rings = min_t(int, rx_rings, hwr.grp); hwr.cp = min_t(int, hwr.cp, bp->cp_nr_rings); - if (bnxt_ulp_registered(bp->edev) && - hwr.stat > bnxt_get_ulp_stat_ctxs(bp)) + if (bnxt_ulp_registered(edev) && hwr.stat > bnxt_get_ulp_stat_ctxs(bp)) hwr.stat -= bnxt_get_ulp_stat_ctxs(bp); hwr.cp = min_t(int, hwr.cp, hwr.stat); rc = bnxt_trim_rings(bp, &rx_rings, &hwr.tx, hwr.cp, sh); @@ -8061,13 +8262,17 @@ static int __bnxt_reserve_rings(struct bnxt *bp) (bnxt_get_nr_rss_ctxs(bp, bp->rx_nr_rings) != bnxt_get_nr_rss_ctxs(bp, rx_rings) || bnxt_get_max_rss_ring(bp) >= rx_rings)) { - netdev_warn(bp->dev, "RSS table entries reverting to default\n"); - bp->dev->priv_flags &= ~IFF_RXFH_CONFIGURED; + ethtool_rxfh_indir_lost(bp->dev); } } bp->rx_nr_rings = rx_rings; bp->cp_nr_rings = hwr.cp; + /* Fall back if we cannot reserve enough HW RSS contexts */ + if ((bp->rss_cap & BNXT_RSS_CAP_LARGE_RSS_CTX) && + hwr.rss_ctx < bnxt_get_total_rss_ctxs(bp, &hwr)) + bp->rss_cap &= ~BNXT_RSS_CAP_LARGE_RSS_CTX; + if (!bnxt_rings_ok(bp, &hwr)) return -ENOMEM; @@ -8075,7 +8280,7 @@ static int __bnxt_reserve_rings(struct bnxt *bp) !netif_is_rxfh_configured(bp->dev)) bnxt_set_dflt_rss_indir_tbl(bp, NULL); - if (!bnxt_ulp_registered(bp->edev) && BNXT_NEW_RM(bp)) { + if (!bnxt_ulp_registered(edev) && BNXT_NEW_RM(bp)) { int resv_msix, resv_ctx, ulp_ctxs; struct bnxt_hw_resc *hw_resc; @@ -8590,7 +8795,7 @@ static int bnxt_alloc_all_ctx_pg_info(struct bnxt *bp, int ctx_max) if (ctxm->instance_bmap) n = hweight32(ctxm->instance_bmap); - ctxm->pg_info = kcalloc(n, sizeof(*ctxm->pg_info), GFP_KERNEL); + ctxm->pg_info = kzalloc_objs(*ctxm->pg_info, n); if (!ctxm->pg_info) return -ENOMEM; } @@ -8609,7 +8814,7 @@ static int bnxt_hwrm_func_backing_store_qcaps_v2(struct bnxt *bp) struct hwrm_func_backing_store_qcaps_v2_output *resp; struct hwrm_func_backing_store_qcaps_v2_input *req; struct bnxt_ctx_mem_info *ctx = bp->ctx; - u16 type; + u16 type, next_type = 0; int rc; rc = hwrm_req_init(bp, req, HWRM_FUNC_BACKING_STORE_QCAPS_V2); @@ -8617,7 +8822,7 @@ static int bnxt_hwrm_func_backing_store_qcaps_v2(struct bnxt *bp) return rc; if (!ctx) { - ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); + ctx = kzalloc_obj(*ctx); if (!ctx) return -ENOMEM; bp->ctx = ctx; @@ -8625,7 +8830,7 @@ static int bnxt_hwrm_func_backing_store_qcaps_v2(struct bnxt *bp) resp = hwrm_req_hold(bp, req); - for (type = 0; type < BNXT_CTX_V2_MAX; ) { + for (type = 0; type < BNXT_CTX_V2_MAX; type = next_type) { struct bnxt_ctx_mem_type *ctxm = &ctx->ctx_arr[type]; u8 init_val, init_off, i; u32 max_entries; @@ -8638,7 +8843,7 @@ static int bnxt_hwrm_func_backing_store_qcaps_v2(struct bnxt *bp) if (rc) goto ctx_done; flags = le32_to_cpu(resp->flags); - type = le16_to_cpu(resp->next_valid_type); + next_type = le16_to_cpu(resp->next_valid_type); if (!(flags & BNXT_CTX_MEM_TYPE_VALID)) { bnxt_free_one_ctx_mem(bp, ctxm, true); continue; @@ -8653,7 +8858,7 @@ static int bnxt_hwrm_func_backing_store_qcaps_v2(struct bnxt *bp) else continue; } - ctxm->type = le16_to_cpu(resp->type); + ctxm->type = type; ctxm->entry_size = entry_size; ctxm->flags = flags; ctxm->instance_bmap = le32_to_cpu(resp->instance_bit_map); @@ -8704,7 +8909,7 @@ static int bnxt_hwrm_func_backing_store_qcaps(struct bnxt *bp) ctx = bp->ctx; if (!ctx) { - ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); + ctx = kzalloc_obj(*ctx); if (!ctx) { rc = -ENOMEM; goto ctx_err; @@ -8983,8 +9188,7 @@ static int bnxt_alloc_ctx_pg_tbls(struct bnxt *bp, int nr_tbls, i; rmem->depth = 2; - ctx_pg->ctx_pg_tbl = kcalloc(MAX_CTX_PAGES, sizeof(ctx_pg), - GFP_KERNEL); + ctx_pg->ctx_pg_tbl = kzalloc_objs(ctx_pg, MAX_CTX_PAGES); if (!ctx_pg->ctx_pg_tbl) return -ENOMEM; nr_tbls = DIV_ROUND_UP(ctx_pg->nr_pages, MAX_CTX_PAGES); @@ -8995,7 +9199,7 @@ static int bnxt_alloc_ctx_pg_tbls(struct bnxt *bp, for (i = 0; i < nr_tbls; i++) { struct bnxt_ctx_pg_info *pg_tbl; - pg_tbl = kzalloc(sizeof(*pg_tbl), GFP_KERNEL); + pg_tbl = kzalloc_obj(*pg_tbl); if (!pg_tbl) return -ENOMEM; ctx_pg->ctx_pg_tbl[i] = pg_tbl; @@ -9504,8 +9708,7 @@ static int bnxt_alloc_crash_dump_mem(struct bnxt *bp) if (bp->fw_crash_mem) bnxt_free_ctx_pg_tbls(bp, bp->fw_crash_mem); else - bp->fw_crash_mem = kzalloc(sizeof(*bp->fw_crash_mem), - GFP_KERNEL); + bp->fw_crash_mem = kzalloc_obj(*bp->fw_crash_mem); if (!bp->fw_crash_mem) return -ENOMEM; @@ -9558,6 +9761,10 @@ int bnxt_hwrm_func_resc_qcaps(struct bnxt *bp, bool all) hw_resc->min_stat_ctxs = le16_to_cpu(resp->min_stat_ctx); hw_resc->max_stat_ctxs = le16_to_cpu(resp->max_stat_ctx); + if (hw_resc->max_rsscos_ctxs >= + hw_resc->max_vnics * BNXT_LARGE_RSS_TO_VNIC_RATIO) + bp->rss_cap |= BNXT_RSS_CAP_LARGE_RSS_CTX; + if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) { u16 max_msix = le16_to_cpu(resp->max_msix); @@ -9608,7 +9815,7 @@ static int __bnxt_hwrm_ptp_qcfg(struct bnxt *bp) goto exit; } if (!ptp) { - ptp = kzalloc(sizeof(*ptp), GFP_KERNEL); + ptp = kzalloc_obj(*ptp); if (!ptp) { rc = -ENOMEM; goto exit; @@ -9691,6 +9898,8 @@ static int __bnxt_hwrm_func_qcaps(struct bnxt *bp) bp->fw_cap |= BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED; if (BNXT_PF(bp) && (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_PTP_PPS_SUPPORTED)) bp->fw_cap |= BNXT_FW_CAP_PTP_PPS; + if (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_PTP_PTM_SUPPORTED) + bp->fw_cap |= BNXT_FW_CAP_PTP_PTM; if (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_PTP_64BIT_RTC_SUPPORTED) bp->fw_cap |= BNXT_FW_CAP_PTP_RTC; if (BNXT_PF(bp) && (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_HOT_RESET_IF_SUPPORT)) @@ -9878,7 +10087,7 @@ static int __bnxt_alloc_fw_health(struct bnxt *bp) if (bp->fw_health) return 0; - bp->fw_health = kzalloc(sizeof(*bp->fw_health), GFP_KERNEL); + bp->fw_health = kzalloc_obj(*bp->fw_health); if (!bp->fw_health) return -ENOMEM; @@ -10364,7 +10573,7 @@ static void bnxt_accumulate_stats(struct bnxt_stats_mem *stats) stats->hw_masks, stats->len / 8, false); } -static void bnxt_accumulate_all_stats(struct bnxt *bp) +static void bnxt_accumulate_ring_stats(struct bnxt *bp) { struct bnxt_stats_mem *ring0_stats; bool ignore_zero = false; @@ -10387,6 +10596,10 @@ static void bnxt_accumulate_all_stats(struct bnxt *bp) ring0_stats->hw_masks, ring0_stats->len / 8, ignore_zero); } +} + +static void bnxt_accumulate_port_stats(struct bnxt *bp) +{ if (bp->flags & BNXT_FLAG_PORT_STATS) { struct bnxt_stats_mem *stats = &bp->port_stats; __le64 *hw_stats = stats->hw_stats; @@ -10409,6 +10622,41 @@ static void bnxt_accumulate_all_stats(struct bnxt *bp) } } +static void bnxt_accumulate_all_stats(struct bnxt *bp) +{ + bnxt_accumulate_ring_stats(bp); + bnxt_accumulate_port_stats(bp); +} + +/* Re-accumulate ring stats from DMA buffers if stale. + * uAPIs for reading sw_stats should call this first. + * + * We promise user space update frequency of bp->stats_coal_ticks but + * the update is a two step process - first device updates the DMA buffer, + * then we have to update from that buffer to driver stats in the service work. + * Worst case we would be 2x off from the desired frequency. + * Sync the stats sooner, if stale. The 20% threshold was chosen arbitrarily. + * + * Ideally we would split the user-configured time into two portions, + * i.e. also lower the DMA period by the 20%. But the DMA timer seems to have + * too coarse granularity to play such tricks. + */ +void bnxt_sync_ring_stats(struct bnxt *bp) +{ + unsigned long stale; + + if (!netif_running(bp->dev) || !bp->stats_coal_ticks) + return; + + spin_lock(&bp->stats_lock); + stale = usecs_to_jiffies(bp->stats_coal_ticks / 5); + if (time_after_eq(jiffies, bp->stats_updated_jiffies + stale)) { + bnxt_accumulate_ring_stats(bp); + bp->stats_updated_jiffies = jiffies; + } + spin_unlock(&bp->stats_lock); +} + static int bnxt_hwrm_port_qstats(struct bnxt *bp, u8 flags) { struct hwrm_port_qstats_input *req; @@ -10661,6 +10909,7 @@ static int __bnxt_setup_vnic(struct bnxt *bp, struct bnxt_vnic_info *vnic) } skip_rss_ctx: + bnxt_fill_hw_rss_tbl(bp, vnic); /* configure default vnic, ring grp */ rc = bnxt_hwrm_vnic_cfg(bp, vnic); if (rc) { @@ -10818,12 +11067,10 @@ void bnxt_del_one_rss_ctx(struct bnxt *bp, struct bnxt_rss_ctx *rss_ctx, struct bnxt_ntuple_filter *ntp_fltr; int i; - if (netif_running(bp->dev)) { - bnxt_hwrm_vnic_free_one(bp, &rss_ctx->vnic); - for (i = 0; i < BNXT_MAX_CTX_PER_VNIC; i++) { - if (vnic->fw_rss_cos_lb_ctx[i] != INVALID_HW_RING_ID) - bnxt_hwrm_vnic_ctx_free_one(bp, vnic, i); - } + bnxt_hwrm_vnic_free_one(bp, &rss_ctx->vnic); + for (i = 0; i < BNXT_MAX_CTX_PER_VNIC; i++) { + if (vnic->fw_rss_cos_lb_ctx[i] != INVALID_HW_RING_ID) + bnxt_hwrm_vnic_ctx_free_one(bp, vnic, i); } if (!all) return; @@ -10887,6 +11134,11 @@ static int bnxt_set_vnic_mru_p5(struct bnxt *bp, struct bnxt_vnic_info *vnic, vnic->vnic_id, rc); return rc; } + if (rxr_id == vnic->default_rx_ring) { + rc = bnxt_hwrm_vnic_cfg(bp, vnic); + if (rc) + return rc; + } } vnic->mru = mru; bnxt_hwrm_vnic_update(bp, vnic, @@ -10968,6 +11220,9 @@ static int bnxt_setup_nitroa0_vnic(struct bnxt *bp) return rc; } + /* Setup the proper default RX ring */ + bnxt_fill_hw_rss_tbl(bp, vnic); + rc = bnxt_hwrm_vnic_cfg(bp, vnic); if (rc) { netdev_err(bp->dev, "Cannot allocate special vnic for NS2 A0: %x\n", @@ -10977,8 +11232,9 @@ static int bnxt_setup_nitroa0_vnic(struct bnxt *bp) return rc; } -static int bnxt_cfg_rx_mode(struct bnxt *); -static bool bnxt_mc_list_updated(struct bnxt *, u32 *); +static int bnxt_cfg_rx_mode(struct bnxt *, struct netdev_hw_addr_list *, bool); +static bool bnxt_mc_list_updated(struct bnxt *, u32 *, + const struct netdev_hw_addr_list *); static int bnxt_init_chip(struct bnxt *bp, bool irq_re_init) { @@ -11068,13 +11324,17 @@ static int bnxt_init_chip(struct bnxt *bp, bool irq_re_init) } else if (bp->dev->flags & IFF_MULTICAST) { u32 mask = 0; - bnxt_mc_list_updated(bp, &mask); + bnxt_mc_list_updated(bp, &mask, &bp->dev->mc); vnic->rx_mask |= mask; } - rc = bnxt_cfg_rx_mode(bp); - if (rc) + rc = bnxt_cfg_rx_mode(bp, &bp->dev->uc, true); + if (rc == -EAGAIN) { + netif_rx_mode_schedule_retry(bp->dev); + rc = 0; + } else if (rc) { goto err_out; + } skip_rx_mask: rc = bnxt_hwrm_set_coal(bp); @@ -11110,8 +11370,13 @@ static int bnxt_shutdown_nic(struct bnxt *bp, bool irq_re_init) static int bnxt_init_nic(struct bnxt *bp, bool irq_re_init) { + int rc; + bnxt_init_cp_rings(bp); - bnxt_init_rx_rings(bp); + rc = bnxt_init_rx_rings(bp); + if (rc) + return rc; + bnxt_init_tx_rings(bp); bnxt_init_ring_grps(bp, irq_re_init); bnxt_init_vnics(bp); @@ -11365,7 +11630,7 @@ static int bnxt_get_num_msix(struct bnxt *bp) static int bnxt_init_int_mode(struct bnxt *bp) { - int i, total_vecs, max, rc = 0, min = 1, ulp_msix, tx_cp, tbl_size; + int i, total_vecs, max, rc, min = 1, ulp_msix, tx_cp, tbl_size; total_vecs = bnxt_get_num_msix(bp); max = bnxt_get_max_func_irqs(bp); @@ -11389,27 +11654,27 @@ static int bnxt_init_int_mode(struct bnxt *bp) tbl_size = total_vecs; if (pci_msix_can_alloc_dyn(bp->pdev)) tbl_size = max; - bp->irq_tbl = kcalloc(tbl_size, sizeof(*bp->irq_tbl), GFP_KERNEL); - if (bp->irq_tbl) { - for (i = 0; i < total_vecs; i++) - bp->irq_tbl[i].vector = pci_irq_vector(bp->pdev, i); - - bp->total_irqs = total_vecs; - /* Trim rings based upon num of vectors allocated */ - rc = bnxt_trim_rings(bp, &bp->rx_nr_rings, &bp->tx_nr_rings, - total_vecs - ulp_msix, min == 1); - if (rc) - goto msix_setup_exit; - - tx_cp = bnxt_num_tx_to_cp(bp, bp->tx_nr_rings); - bp->cp_nr_rings = (min == 1) ? - max_t(int, tx_cp, bp->rx_nr_rings) : - tx_cp + bp->rx_nr_rings; - - } else { + bp->irq_tbl = kzalloc_objs(*bp->irq_tbl, tbl_size); + if (!bp->irq_tbl) { rc = -ENOMEM; goto msix_setup_exit; } + + for (i = 0; i < total_vecs; i++) + bp->irq_tbl[i].vector = pci_irq_vector(bp->pdev, i); + + bp->total_irqs = total_vecs; + /* Trim rings based upon num of vectors allocated */ + rc = bnxt_trim_rings(bp, &bp->rx_nr_rings, &bp->tx_nr_rings, + total_vecs - ulp_msix, min == 1); + if (rc) + goto msix_setup_exit; + + tx_cp = bnxt_num_tx_to_cp(bp, bp->tx_nr_rings); + bp->cp_nr_rings = (min == 1) ? + max_t(int, tx_cp, bp->rx_nr_rings) : + tx_cp + bp->rx_nr_rings; + return 0; msix_setup_exit: @@ -11430,6 +11695,7 @@ static void bnxt_clear_int_mode(struct bnxt *bp) int bnxt_reserve_rings(struct bnxt *bp, bool irq_re_init) { + struct bnxt_en_dev *edev = bp->edev[BNXT_AUXDEV_RDMA]; bool irq_cleared = false; bool irq_change = false; int tcs = bp->num_tc; @@ -11439,7 +11705,7 @@ int bnxt_reserve_rings(struct bnxt *bp, bool irq_re_init) if (!bnxt_need_reserve_rings(bp)) return 0; - if (BNXT_NEW_RM(bp) && !bnxt_ulp_registered(bp->edev)) { + if (BNXT_NEW_RM(bp) && !bnxt_ulp_registered(edev)) { int ulp_msix = bnxt_get_avail_msix(bp, bp->ulp_num_msix_want); if (ulp_msix > bp->ulp_num_msix_want) @@ -11541,6 +11807,8 @@ static int bnxt_tx_queue_start(struct bnxt *bp, int idx) txr->tx_prod = 0; txr->tx_cons = 0; txr->tx_hw_cons = 0; + txr->kick_pending = 0; + txr->kick_txbd0 = NULL; start_tx: WRITE_ONCE(txr->dev_state, 0); synchronize_net(); @@ -11561,33 +11829,36 @@ static void bnxt_irq_affinity_notify(struct irq_affinity_notify *notify, { struct bnxt_irq *irq; u16 tag; - int err; irq = container_of(notify, struct bnxt_irq, affinity_notify); + cpumask_copy(irq->bp->ring_cpu_mask[irq->ring_nr], mask); + set_bit(irq->ring_nr, irq->bp->ring_affinity_set); + +#ifdef CONFIG_RFS_ACCEL + if (irq->bp->dev->rx_cpu_rmap && irq->ring_nr < irq->bp->rx_nr_rings) { + int err; + + err = cpu_rmap_update(irq->bp->dev->rx_cpu_rmap, irq->ring_nr, + mask); + if (err) + netdev_warn(irq->bp->dev, + "aRFS rmap update failed: %d\n", err); + } +#endif + if (!irq->bp->tph_mode) return; - cpumask_copy(irq->cpu_mask, mask); - if (irq->ring_nr >= irq->bp->rx_nr_rings) return; if (pcie_tph_get_cpu_st(irq->bp->pdev, TPH_MEM_TYPE_VM, - cpumask_first(irq->cpu_mask), &tag)) - return; - - if (pcie_tph_set_st_entry(irq->bp->pdev, irq->msix_nr, tag)) + cpumask_first(mask), &tag)) return; - netdev_lock(irq->bp->dev); - if (netif_running(irq->bp->dev)) { - err = netdev_rx_queue_restart(irq->bp->dev, irq->ring_nr); - if (err) - netdev_err(irq->bp->dev, - "RX queue restart failed: err=%d\n", err); - } - netdev_unlock(irq->bp->dev); + WRITE_ONCE(irq->new_tag, tag); + bnxt_queue_sp_work(irq->bp, BNXT_TPH_UPDATE_SP_EVENT); } static void bnxt_irq_affinity_release(struct kref *ref) @@ -11620,10 +11891,6 @@ static void bnxt_register_irq_notifier(struct bnxt *bp, struct bnxt_irq *irq) irq->bp = bp; - /* Nothing to do if TPH is not enabled */ - if (!bp->tph_mode) - return; - /* Register IRQ affinity notifier */ notify = &irq->affinity_notify; notify->irq = irq->vector; @@ -11633,15 +11900,46 @@ static void bnxt_register_irq_notifier(struct bnxt *bp, struct bnxt_irq *irq) irq_set_affinity_notifier(irq->vector, notify); } +static int bnxt_alloc_ring_cpu_masks(struct bnxt *bp) +{ + int i; + + bp->ring_cpu_mask = kzalloc_objs(*bp->ring_cpu_mask, bp->max_irqs); + if (!bp->ring_cpu_mask) + return -ENOMEM; + + bp->ring_affinity_set = bitmap_zalloc(bp->max_irqs, GFP_KERNEL); + if (!bp->ring_affinity_set) + return -ENOMEM; + + for (i = 0; i < bp->max_irqs; i++) + if (!zalloc_cpumask_var(&bp->ring_cpu_mask[i], GFP_KERNEL)) + return -ENOMEM; + + return 0; +} + +static void bnxt_free_ring_cpu_masks(struct bnxt *bp) +{ + int i; + + if (!bp->ring_cpu_mask) + return; + + for (i = 0; i < bp->max_irqs; i++) + free_cpumask_var(bp->ring_cpu_mask[i]); + + bitmap_free(bp->ring_affinity_set); + bp->ring_affinity_set = NULL; + kfree(bp->ring_cpu_mask); + bp->ring_cpu_mask = NULL; +} + static void bnxt_free_irq(struct bnxt *bp) { struct bnxt_irq *irq; int i; -#ifdef CONFIG_RFS_ACCEL - free_irq_cpu_rmap(bp->dev->rx_cpu_rmap); - bp->dev->rx_cpu_rmap = NULL; -#endif if (!bp->irq_tbl || !bp->bnapi) return; @@ -11650,27 +11948,29 @@ static void bnxt_free_irq(struct bnxt *bp) irq = &bp->irq_tbl[map_idx]; if (irq->requested) { - if (irq->have_cpumask) { - irq_update_affinity_hint(irq->vector, NULL); - free_cpumask_var(irq->cpu_mask); - irq->have_cpumask = 0; - } - bnxt_release_irq_notifier(irq); - + irq_update_affinity_hint(irq->vector, NULL); free_irq(irq->vector, bp->bnapi[i]); } irq->requested = 0; + irq->tag = 0; + irq->new_tag = 0; } /* Disable TPH support */ pcie_disable_tph(bp->pdev); bp->tph_mode = 0; + +#ifdef CONFIG_RFS_ACCEL + free_irq_cpu_rmap(bp->dev->rx_cpu_rmap); + bp->dev->rx_cpu_rmap = NULL; +#endif } static int bnxt_request_irq(struct bnxt *bp) { + const int numa_node = dev_to_node(&bp->pdev->dev); struct cpu_rmap *rmap = NULL; int i, j, rc = 0; unsigned long flags = 0; @@ -11686,13 +11986,17 @@ static int bnxt_request_irq(struct bnxt *bp) #endif /* Enable TPH support as part of IRQ request */ - rc = pcie_enable_tph(bp->pdev, PCI_TPH_ST_IV_MODE); - if (!rc) - bp->tph_mode = PCI_TPH_ST_IV_MODE; + if (BNXT_SUPPORTS_QUEUE_API(bp)) { + rc = pcie_enable_tph(bp->pdev, PCI_TPH_ST_IV_MODE); + if (!rc) + bp->tph_mode = PCI_TPH_ST_IV_MODE; + } for (i = 0, j = 0; i < bp->cp_nr_rings; i++) { + struct cpumask *cpu_mask = bp->ring_cpu_mask[i]; int map_idx = bnxt_cp_num_to_irq_num(bp, i); struct bnxt_irq *irq = &bp->irq_tbl[map_idx]; + u16 tag; if (IS_ENABLED(CONFIG_RFS_ACCEL) && rmap && bp->bnapi[i]->rx_ring) { @@ -11710,33 +12014,46 @@ static int bnxt_request_irq(struct bnxt *bp) netif_napi_set_irq_locked(&bp->bnapi[i]->napi, irq->vector); irq->requested = 1; + irq->msix_nr = map_idx; + irq->ring_nr = i; - if (zalloc_cpumask_var(&irq->cpu_mask, GFP_KERNEL)) { - int numa_node = dev_to_node(&bp->pdev->dev); - u16 tag; - - irq->have_cpumask = 1; - irq->msix_nr = map_idx; - irq->ring_nr = i; + /* Reuse the mask recorded before the IRQs were freed. Nothing + * was recorded yet on the very first request, and the mask + * may have gone stale if the CPUs went offline in between. + */ + if (!test_bit(i, bp->ring_affinity_set) || + !cpumask_intersects(cpu_mask, cpu_online_mask)) { + clear_bit(i, bp->ring_affinity_set); + cpumask_clear(cpu_mask); cpumask_set_cpu(cpumask_local_spread(i, numa_node), - irq->cpu_mask); - rc = irq_update_affinity_hint(irq->vector, irq->cpu_mask); - if (rc) { - netdev_warn(bp->dev, - "Update affinity hint failed, IRQ = %d\n", - irq->vector); - break; - } + cpu_mask); + } - bnxt_register_irq_notifier(bp, irq); + /* Init ST table entry if we can get the mapping */ + if (!pcie_tph_get_cpu_st(bp->pdev, TPH_MEM_TYPE_VM, + cpumask_first(cpu_mask), &tag)) { + pcie_tph_set_st_entry(bp->pdev, irq->msix_nr, tag); + irq->tag = tag; + irq->new_tag = tag; + } - /* Init ST table entry */ - if (pcie_tph_get_cpu_st(irq->bp->pdev, TPH_MEM_TYPE_VM, - cpumask_first(irq->cpu_mask), - &tag)) - continue; + bnxt_register_irq_notifier(bp, irq); - pcie_tph_set_st_entry(irq->bp->pdev, irq->msix_nr, tag); + /* Only put the IRQ back where it was configured to be, our own + * placement is just a hint, the core spreads within + * irq_default_affinity which we know nothing about. + * Set after installing the notifier, if we race with the user + * it's better to overwrite than miss the notification. + */ + if (test_bit(i, bp->ring_affinity_set)) + rc = irq_set_affinity_and_hint(irq->vector, cpu_mask); + else + rc = irq_update_affinity_hint(irq->vector, cpu_mask); + if (rc) { + netdev_warn(bp->dev, + "Setting IRQ affinity failed, IRQ = %d\n", + irq->vector); + break; } } return rc; @@ -11892,6 +12209,26 @@ static char *bnxt_report_fec(struct bnxt_link_info *link_info) } } +static char *bnxt_link_down_reason(struct bnxt_link_info *link_info) +{ + u8 reason = link_info->link_down_reason; + + /* Multiple bits can be set, we report 1 bit only in order of + * priority. + */ + if (reason & PORT_PHY_QCFG_RESP_LINK_DOWN_REASON_RF) + return "(Remote fault)"; + if (reason & PORT_PHY_QCFG_RESP_LINK_DOWN_REASON_OTP_SPEED_VIOLATION) + return "(OTP Speed limit violation)"; + if (reason & PORT_PHY_QCFG_RESP_LINK_DOWN_REASON_CABLE_REMOVED) + return "(Cable removed)"; + if (reason & PORT_PHY_QCFG_RESP_LINK_DOWN_REASON_MODULE_FAULT) + return "(Module fault)"; + if (reason & PORT_PHY_QCFG_RESP_LINK_DOWN_REASON_BMC_REQUEST) + return "(BMC request down)"; + return ""; +} + void bnxt_report_link(struct bnxt *bp) { if (BNXT_LINK_IS_UP(bp)) { @@ -11949,8 +12286,10 @@ void bnxt_report_link(struct bnxt *bp) (fec & BNXT_FEC_AUTONEG) ? "on" : "off", bnxt_report_fec(&bp->link_info)); } else { + char *str = bnxt_link_down_reason(&bp->link_info); + netif_carrier_off(bp->dev); - netdev_err(bp->dev, "NIC Link is Down\n"); + netdev_err(bp->dev, "NIC Link is Down %s\n", str); } } @@ -12150,6 +12489,7 @@ int bnxt_update_link(struct bnxt *bp, bool chng_link_state) link_info->phy_addr = resp->eee_config_phy_addr & PORT_PHY_QCFG_RESP_PHY_ADDR_MASK; link_info->module_status = resp->module_status; + link_info->link_down_reason = resp->link_down_reason; if (bp->phy_flags & BNXT_PHY_FL_EEE_CAP) { struct ethtool_keee *eee = &bp->eee; @@ -12863,7 +13203,7 @@ static int bnxt_set_xps_mapping(struct bnxt *bp) cpumask_t *q_map; int rc = 0; - q_map = kcalloc(bp->tx_nr_rings_per_tc, sizeof(*q_map), GFP_KERNEL); + q_map = kzalloc_objs(*q_map, bp->tx_nr_rings_per_tc); if (!q_map) return -ENOMEM; @@ -12905,6 +13245,21 @@ static int bnxt_tx_nr_rings_per_tc(struct bnxt *bp) return bp->num_tc ? bp->tx_nr_rings / bp->num_tc : bp->tx_nr_rings; } +static void bnxt_set_xdp_tx_rings(struct bnxt *bp) +{ + bp->tx_nr_rings_xdp = bp->tx_nr_rings_per_tc; + bp->tx_nr_rings += bp->tx_nr_rings_xdp; +} + +static void bnxt_adj_tx_rings(struct bnxt *bp) +{ + /* Make adjustments if reserved TX rings are less than requested */ + bp->tx_nr_rings -= bp->tx_nr_rings_xdp; + bp->tx_nr_rings_per_tc = bnxt_tx_nr_rings_per_tc(bp); + if (bp->tx_nr_rings_xdp) + bnxt_set_xdp_tx_rings(bp); +} + static int __bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init) { int rc = 0; @@ -12922,13 +13277,7 @@ static int __bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init) if (rc) return rc; - /* Make adjustments if reserved TX rings are less than requested */ - bp->tx_nr_rings -= bp->tx_nr_rings_xdp; - bp->tx_nr_rings_per_tc = bnxt_tx_nr_rings_per_tc(bp); - if (bp->tx_nr_rings_xdp) { - bp->tx_nr_rings_xdp = bp->tx_nr_rings_per_tc; - bp->tx_nr_rings += bp->tx_nr_rings_xdp; - } + bnxt_adj_tx_rings(bp); rc = bnxt_alloc_mem(bp, irq_re_init); if (rc) { netdev_err(bp->dev, "bnxt_alloc_mem err: %x\n", rc); @@ -13158,7 +13507,7 @@ static void __bnxt_close_nic(struct bnxt *bp, bool irq_re_init, /* Save ring stats before shutdown */ if (bp->bnapi && irq_re_init) { bnxt_get_ring_stats(bp, &bp->net_stats_prev); - bnxt_get_ring_err_stats(bp, &bp->ring_err_stats_prev); + bnxt_get_ring_drv_stats(bp, &bp->ring_drv_stats_prev); } if (irq_re_init) { bnxt_free_irq(bp); @@ -13375,6 +13724,7 @@ bnxt_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats) return; } + bnxt_sync_ring_stats(bp); bnxt_get_ring_stats(bp, stats); bnxt_add_prev_stats(bp, stats); @@ -13403,8 +13753,8 @@ bnxt_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats) clear_bit(BNXT_STATE_READ_STATS, &bp->state); } -static void bnxt_get_one_ring_err_stats(struct bnxt *bp, - struct bnxt_total_ring_err_stats *stats, +static void bnxt_get_one_ring_drv_stats(struct bnxt *bp, + struct bnxt_total_ring_drv_stats *stats, struct bnxt_cp_ring_info *cpr) { struct bnxt_sw_stats *sw_stats = cpr->sw_stats; @@ -13417,32 +13767,34 @@ static void bnxt_get_one_ring_err_stats(struct bnxt *bp, stats->rx_total_netpoll_discards += sw_stats->rx.rx_netpoll_discards; stats->rx_total_ring_discards += BNXT_GET_RING_STATS64(hw_stats, rx_discard_pkts); + stats->rx_total_hw_gro_packets += sw_stats->rx.rx_hw_gro_packets; + stats->rx_total_hw_gro_wire_packets += sw_stats->rx.rx_hw_gro_wire_packets; stats->tx_total_resets += sw_stats->tx.tx_resets; stats->tx_total_ring_discards += BNXT_GET_RING_STATS64(hw_stats, tx_discard_pkts); stats->total_missed_irqs += sw_stats->cmn.missed_irqs; } -void bnxt_get_ring_err_stats(struct bnxt *bp, - struct bnxt_total_ring_err_stats *stats) +void bnxt_get_ring_drv_stats(struct bnxt *bp, + struct bnxt_total_ring_drv_stats *stats) { int i; for (i = 0; i < bp->cp_nr_rings; i++) - bnxt_get_one_ring_err_stats(bp, stats, &bp->bnapi[i]->cp_ring); + bnxt_get_one_ring_drv_stats(bp, stats, &bp->bnapi[i]->cp_ring); } -static bool bnxt_mc_list_updated(struct bnxt *bp, u32 *rx_mask) +static bool bnxt_mc_list_updated(struct bnxt *bp, u32 *rx_mask, + const struct netdev_hw_addr_list *mc) { struct bnxt_vnic_info *vnic = &bp->vnic_info[BNXT_VNIC_DEFAULT]; - struct net_device *dev = bp->dev; struct netdev_hw_addr *ha; u8 *haddr; int mc_count = 0; bool update = false; int off = 0; - netdev_for_each_mc_addr(ha, dev) { + netdev_hw_addr_list_for_each(ha, mc) { if (mc_count >= BNXT_MAX_MC_ADDRS) { *rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST; vnic->mc_list_count = 0; @@ -13466,17 +13818,17 @@ static bool bnxt_mc_list_updated(struct bnxt *bp, u32 *rx_mask) return update; } -static bool bnxt_uc_list_updated(struct bnxt *bp) +static bool bnxt_uc_list_updated(struct bnxt *bp, + const struct netdev_hw_addr_list *uc) { - struct net_device *dev = bp->dev; struct bnxt_vnic_info *vnic = &bp->vnic_info[BNXT_VNIC_DEFAULT]; struct netdev_hw_addr *ha; int off = 0; - if (netdev_uc_count(dev) != (vnic->uc_filter_count - 1)) + if (netdev_hw_addr_list_count(uc) != (vnic->uc_filter_count - 1)) return true; - netdev_for_each_uc_addr(ha, dev) { + netdev_hw_addr_list_for_each(ha, uc) { if (!ether_addr_equal(ha->addr, vnic->uc_list + off)) return true; @@ -13485,7 +13837,9 @@ static bool bnxt_uc_list_updated(struct bnxt *bp) return false; } -static void bnxt_set_rx_mode(struct net_device *dev) +static int bnxt_set_rx_mode(struct net_device *dev, + struct netdev_hw_addr_list *uc, + struct netdev_hw_addr_list *mc) { struct bnxt *bp = netdev_priv(dev); struct bnxt_vnic_info *vnic; @@ -13494,7 +13848,7 @@ static void bnxt_set_rx_mode(struct net_device *dev) u32 mask; if (!test_bit(BNXT_STATE_OPEN, &bp->state)) - return; + return 0; vnic = &bp->vnic_info[BNXT_VNIC_DEFAULT]; mask = vnic->rx_mask; @@ -13506,7 +13860,7 @@ static void bnxt_set_rx_mode(struct net_device *dev) if (dev->flags & IFF_PROMISC) mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS; - uc_update = bnxt_uc_list_updated(bp); + uc_update = bnxt_uc_list_updated(bp, uc); if (dev->flags & IFF_BROADCAST) mask |= CFA_L2_SET_RX_MASK_REQ_MASK_BCAST; @@ -13514,27 +13868,25 @@ static void bnxt_set_rx_mode(struct net_device *dev) mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST; vnic->mc_list_count = 0; } else if (dev->flags & IFF_MULTICAST) { - mc_update = bnxt_mc_list_updated(bp, &mask); + mc_update = bnxt_mc_list_updated(bp, &mask, mc); } if (mask != vnic->rx_mask || uc_update || mc_update) { vnic->rx_mask = mask; - bnxt_queue_sp_work(bp, BNXT_RX_MASK_SP_EVENT); + return bnxt_cfg_rx_mode(bp, uc, uc_update); } + + return 0; } -static int bnxt_cfg_rx_mode(struct bnxt *bp) +static int bnxt_cfg_rx_mode(struct bnxt *bp, struct netdev_hw_addr_list *uc, + bool uc_update) { struct net_device *dev = bp->dev; struct bnxt_vnic_info *vnic = &bp->vnic_info[BNXT_VNIC_DEFAULT]; struct netdev_hw_addr *ha; int i, off = 0, rc; - bool uc_update; - - netif_addr_lock_bh(dev); - uc_update = bnxt_uc_list_updated(bp); - netif_addr_unlock_bh(dev); if (!uc_update) goto skip_uc; @@ -13549,10 +13901,10 @@ static int bnxt_cfg_rx_mode(struct bnxt *bp) vnic->uc_filter_count = 1; netif_addr_lock_bh(dev); - if (netdev_uc_count(dev) > (BNXT_MAX_UC_ADDRS - 1)) { + if (netdev_hw_addr_list_count(uc) > (BNXT_MAX_UC_ADDRS - 1)) { vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS; } else { - netdev_for_each_uc_addr(ha, dev) { + netdev_hw_addr_list_for_each(ha, uc) { memcpy(vnic->uc_list + off, ha->addr, ETH_ALEN); off += ETH_ALEN; vnic->uc_filter_count++; @@ -13564,11 +13916,10 @@ static int bnxt_cfg_rx_mode(struct bnxt *bp) rc = bnxt_hwrm_set_vnic_filter(bp, 0, i, vnic->uc_list + off); if (rc) { if (BNXT_VF(bp) && rc == -ENODEV) { - if (!test_and_set_bit(BNXT_STATE_L2_FILTER_RETRY, &bp->state)) - netdev_warn(bp->dev, "Cannot configure L2 filters while PF is unavailable, will retry\n"); - else - netdev_dbg(bp->dev, "PF still unavailable while configuring L2 filters.\n"); - rc = 0; + netdev_warn(bp->dev, "Cannot configure L2 filters while PF is unavailable, will retry\n"); + rc = -EAGAIN; + } else if (rc == -EAGAIN) { + netdev_warn(bp->dev, "FW busy while setting vnic filter, will retry\n"); } else { netdev_err(bp->dev, "HWRM vnic filter failure rc: %x\n", rc); } @@ -13576,8 +13927,6 @@ static int bnxt_cfg_rx_mode(struct bnxt *bp) return rc; } } - if (test_and_clear_bit(BNXT_STATE_L2_FILTER_RETRY, &bp->state)) - netdev_notice(bp->dev, "Retry of L2 filter configuration successful.\n"); skip_uc: if ((vnic->rx_mask & CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS) && @@ -13697,6 +14046,11 @@ static netdev_features_t bnxt_fix_features(struct net_device *dev, if ((features & NETIF_F_NTUPLE) && !bnxt_rfs_capable(bp, false)) features &= ~NETIF_F_NTUPLE; + if ((features & NETIF_F_GSO_UDP_L4) && + !(bp->flags & BNXT_FLAG_UDP_GSO_CAP) && + bp->tx_ring_size < 2 * BNXT_SW_USO_MAX_DESCS) + features &= ~NETIF_F_GSO_UDP_L4; + if ((bp->flags & BNXT_FLAG_NO_AGG_RINGS) || bp->xdp_prog) features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW); @@ -13742,6 +14096,9 @@ static int bnxt_set_features(struct net_device *dev, netdev_features_t features) int rc = 0; bool re_init = false; + bp->tx_wake_thresh = max_t(int, bp->tx_ring_size / 2, + bnxt_min_tx_desc_cnt(bp, features)); + flags &= ~BNXT_FLAG_ALL_CONFIG_FEATS; if (features & NETIF_F_GRO_HW) flags |= BNXT_FLAG_GRO; @@ -13806,7 +14163,6 @@ static bool bnxt_exthdr_check(struct bnxt *bp, struct sk_buff *skb, int nw_off, u8 **nextp) { struct ipv6hdr *ip6h = (struct ipv6hdr *)(skb->data + nw_off); - struct hop_jumbo_hdr *jhdr; int hdr_count = 0; u8 *nexthdr; int start; @@ -13835,24 +14191,7 @@ static bool bnxt_exthdr_check(struct bnxt *bp, struct sk_buff *skb, int nw_off, if (hdrlen > 64) return false; - /* The ext header may be a hop-by-hop header inserted for - * big TCP purposes. This will be removed before sending - * from NIC, so do not count it. - */ - if (*nexthdr == NEXTHDR_HOP) { - if (likely(skb->len <= GRO_LEGACY_MAX_SIZE)) - goto increment_hdr; - - jhdr = (struct hop_jumbo_hdr *)hp; - if (jhdr->tlv_type != IPV6_TLV_JUMBO || jhdr->hdrlen != 0 || - jhdr->nexthdr != IPPROTO_TCP) - goto increment_hdr; - - goto next_hdr; - } -increment_hdr: hdr_count++; -next_hdr: nexthdr = &hp->nexthdr; start += hdrlen; } @@ -14178,9 +14517,6 @@ static void bnxt_timer(struct timer_list *t) } } - if (test_bit(BNXT_STATE_L2_FILTER_RETRY, &bp->state)) - bnxt_queue_sp_work(bp, BNXT_RX_MASK_SP_EVENT); - if ((BNXT_CHIP_P5(bp)) && !bp->chip_rev && netif_carrier_ok(dev)) bnxt_queue_sp_work(bp, BNXT_RING_COAL_NOW_SP_EVENT); @@ -14206,13 +14542,65 @@ static void bnxt_unlock_sp(struct bnxt *bp) netdev_unlock(bp->dev); } +/* Same as bnxt_lock_sp() with additional rtnl_lock */ +static void bnxt_rtnl_lock_sp(struct bnxt *bp) +{ + clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state); + rtnl_lock(); + netdev_lock(bp->dev); +} + +static void bnxt_rtnl_unlock_sp(struct bnxt *bp) +{ + set_bit(BNXT_STATE_IN_SP_TASK, &bp->state); + netdev_unlock(bp->dev); + rtnl_unlock(); +} + +static void bnxt_tph_update(struct bnxt *bp) +{ + struct net_device *dev = bp->dev; + int i; + + bnxt_lock_sp(bp); + if (!test_bit(BNXT_STATE_OPEN, &bp->state)) + goto unlock; + + for (i = 0; i < bp->rx_nr_rings; i++) { + struct bnxt_irq *irq; + int map_idx, err; + u16 tag; + + map_idx = bnxt_cp_num_to_irq_num(bp, i); + irq = &bp->irq_tbl[map_idx]; + tag = READ_ONCE(irq->new_tag); + if (irq->tag == tag) + continue; + + if (pcie_tph_set_st_entry(bp->pdev, irq->msix_nr, tag)) + continue; + + err = netdev_rx_queue_restart(dev, irq->ring_nr); + if (err) { + netdev_err(dev, "RX queue restart failed: err=%d\n", + err); + continue; + } + + irq->tag = tag; + } + +unlock: + bnxt_unlock_sp(bp); +} + /* Only called from bnxt_sp_task() */ static void bnxt_reset(struct bnxt *bp, bool silent) { - bnxt_lock_sp(bp); + bnxt_rtnl_lock_sp(bp); if (test_bit(BNXT_STATE_OPEN, &bp->state)) bnxt_reset_task(bp, silent); - bnxt_unlock_sp(bp); + bnxt_rtnl_unlock_sp(bp); } /* Only called from bnxt_sp_task() */ @@ -14220,9 +14608,9 @@ static void bnxt_rx_ring_reset(struct bnxt *bp) { int i; - bnxt_lock_sp(bp); + bnxt_rtnl_lock_sp(bp); if (!test_bit(BNXT_STATE_OPEN, &bp->state)) { - bnxt_unlock_sp(bp); + bnxt_rtnl_unlock_sp(bp); return; } /* Disable and flush TPA before resetting the RX ring */ @@ -14252,7 +14640,14 @@ static void bnxt_rx_ring_reset(struct bnxt *bp) rxr->rx_sw_agg_prod = 0; rxr->rx_next_cons = 0; rxr->bnapi->in_reset = false; - bnxt_alloc_one_rx_ring(bp, i); + rc = bnxt_alloc_one_rx_ring(bp, i); + if (rc) { + netdev_warn(bp->dev, "RX ring reset failed to allocate buffers, rc = %d, falling back to global reset\n", + rc); + bnxt_reset_task(bp, true); + bnxt_rtnl_unlock_sp(bp); + return; + } cpr = &rxr->bnapi->cp_ring; cpr->sw_stats->rx.rx_resets++; if (bp->flags & BNXT_FLAG_AGG_RINGS) @@ -14261,7 +14656,7 @@ static void bnxt_rx_ring_reset(struct bnxt *bp) } if (bp->flags & BNXT_FLAG_TPA) bnxt_set_tpa(bp, true); - bnxt_unlock_sp(bp); + bnxt_rtnl_unlock_sp(bp); } static void bnxt_fw_fatal_close(struct bnxt *bp) @@ -14522,7 +14917,7 @@ static void bnxt_fw_echo_reply(struct bnxt *bp) static void bnxt_ulp_restart(struct bnxt *bp) { bnxt_ulp_stop(bp); - bnxt_ulp_start(bp, 0); + bnxt_ulp_start(bp); } static void bnxt_sp_task(struct work_struct *work) @@ -14541,9 +14936,6 @@ static void bnxt_sp_task(struct work_struct *work) bnxt_reenable_sriov(bp); } - if (test_and_clear_bit(BNXT_RX_MASK_SP_EVENT, &bp->sp_event)) - bnxt_cfg_rx_mode(bp); - if (test_and_clear_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event)) bnxt_cfg_ntp_filters(bp); if (test_and_clear_bit(BNXT_HWRM_EXEC_FWD_REQ_SP_EVENT, &bp->sp_event)) @@ -14553,7 +14945,10 @@ static void bnxt_sp_task(struct work_struct *work) if (test_and_clear_bit(BNXT_PERIODIC_STATS_SP_EVENT, &bp->sp_event)) { bnxt_hwrm_port_qstats(bp, 0); bnxt_hwrm_port_qstats_ext(bp, 0); + spin_lock(&bp->stats_lock); bnxt_accumulate_all_stats(bp); + bp->stats_updated_jiffies = jiffies; + spin_unlock(&bp->stats_lock); } if (test_and_clear_bit(BNXT_LINK_CHNG_SP_EVENT, &bp->sp_event)) { @@ -14630,6 +15025,9 @@ static void bnxt_sp_task(struct work_struct *work) bnxt_devlink_health_fw_report(bp); } + if (test_and_clear_bit(BNXT_TPH_UPDATE_SP_EVENT, &bp->sp_event)) + bnxt_tph_update(bp); + smp_mb__before_atomic(); clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state); } @@ -14679,7 +15077,7 @@ int bnxt_check_rings(struct bnxt *bp, int tx, int rx, bool sh, int tcs, hwr.cp_p5 = hwr.tx + rx; rc = bnxt_hwrm_check_rings(bp, &hwr); if (!rc && pci_msix_can_alloc_dyn(bp->pdev)) { - if (!bnxt_ulp_registered(bp->edev)) { + if (!bnxt_ulp_registered(bp->edev[BNXT_AUXDEV_RDMA])) { hwr.cp += bnxt_get_ulp_msix_num(bp); hwr.cp = min_t(int, hwr.cp, bnxt_get_max_func_irqs(bp)); } @@ -14716,6 +15114,7 @@ static void bnxt_unmap_bars(struct bnxt *bp, struct pci_dev *pdev) static void bnxt_cleanup_pci(struct bnxt *bp) { + pci_disable_ptm(bp->pdev); bnxt_unmap_bars(bp, bp->pdev); pci_release_regions(bp->pdev); if (pci_is_enabled(bp->pdev)) @@ -15171,15 +15570,17 @@ static void bnxt_fw_reset_task(struct work_struct *work) bp->fw_reset_state = BNXT_FW_RESET_STATE_OPENING; fallthrough; case BNXT_FW_RESET_STATE_OPENING: - while (!netdev_trylock(bp->dev)) { + while (!rtnl_trylock()) { bnxt_queue_fw_reset_work(bp, HZ / 10); return; } + netdev_lock(bp->dev); rc = bnxt_open(bp->dev); if (rc) { netdev_err(bp->dev, "bnxt_open() failed during FW reset\n"); bnxt_fw_reset_abort(bp, rc); netdev_unlock(bp->dev); + rtnl_unlock(); goto ulp_start; } @@ -15199,7 +15600,8 @@ static void bnxt_fw_reset_task(struct work_struct *work) bnxt_dl_health_fw_status_update(bp, true); } netdev_unlock(bp->dev); - bnxt_ulp_start(bp, 0); + rtnl_unlock(); + bnxt_ulp_start(bp); bnxt_reenable_sriov(bp); netdev_lock(bp->dev); bnxt_vf_reps_alloc(bp); @@ -15221,7 +15623,8 @@ fw_reset_abort: bnxt_fw_reset_abort(bp, rc); netdev_unlock(bp->dev); ulp_start: - bnxt_ulp_start(bp, rc); + if (!rc) + bnxt_ulp_start(bp); } static int bnxt_init_board(struct pci_dev *pdev, struct net_device *dev) @@ -15280,10 +15683,13 @@ static int bnxt_init_board(struct pci_dev *pdev, struct net_device *dev) goto init_err_release; } + pci_enable_ptm(pdev); + INIT_WORK(&bp->sp_task, bnxt_sp_task); INIT_DELAYED_WORK(&bp->fw_reset_task, bnxt_fw_reset_task); spin_lock_init(&bp->ntp_fltr_lock); + spin_lock_init(&bp->stats_lock); #if BITS_PER_LONG == 32 spin_lock_init(&bp->db_lock); #endif @@ -15365,11 +15771,19 @@ static int bnxt_change_mtu(struct net_device *dev, int new_mtu) return 0; } +void bnxt_set_cp_rings(struct bnxt *bp, bool sh) +{ + int tx_cp = bnxt_num_tx_to_cp(bp, bp->tx_nr_rings); + + bp->cp_nr_rings = sh ? max_t(int, tx_cp, bp->rx_nr_rings) : + tx_cp + bp->rx_nr_rings; +} + int bnxt_setup_mq_tc(struct net_device *dev, u8 tc) { struct bnxt *bp = netdev_priv(dev); bool sh = false; - int rc, tx_cp; + int rc; if (tc > bp->max_tc) { netdev_err(dev, "Too many traffic classes requested: %d. Max supported is %d.\n", @@ -15402,9 +15816,7 @@ int bnxt_setup_mq_tc(struct net_device *dev, u8 tc) bp->num_tc = 0; } bp->tx_nr_rings += bp->tx_nr_rings_xdp; - tx_cp = bnxt_num_tx_to_cp(bp, bp->tx_nr_rings); - bp->cp_nr_rings = sh ? max_t(int, tx_cp, bp->rx_nr_rings) : - tx_cp + bp->rx_nr_rings; + bnxt_set_cp_rings(bp, sh); if (netif_running(bp->dev)) return bnxt_open_nic(bp, true, false); @@ -15572,7 +15984,7 @@ static int bnxt_rx_flow_steer(struct net_device *dev, const struct sk_buff *skb, return -EINVAL; } } - new_fltr = kzalloc(sizeof(*new_fltr), GFP_ATOMIC); + new_fltr = kzalloc_obj(*new_fltr, GFP_ATOMIC); if (!new_fltr) { bnxt_del_l2_filter(bp, l2_fltr); return -ENOMEM; @@ -15802,7 +16214,7 @@ static const struct net_device_ops bnxt_netdev_ops = { .ndo_start_xmit = bnxt_start_xmit, .ndo_stop = bnxt_close, .ndo_get_stats64 = bnxt_get_stats64, - .ndo_set_rx_mode = bnxt_set_rx_mode, + .ndo_set_rx_mode_async = bnxt_set_rx_mode, .ndo_eth_ioctl = bnxt_ioctl, .ndo_validate_addr = eth_validate_addr, .ndo_set_mac_address = bnxt_change_mac_addr, @@ -15832,6 +16244,10 @@ static const struct net_device_ops bnxt_netdev_ops = { .ndo_hwtstamp_set = bnxt_hwtstamp_set, }; +static const struct xdp_metadata_ops bnxt_xdp_metadata_ops = { + .xmo_rx_hash = bnxt_xdp_rx_hash, +}; + static void bnxt_get_queue_stats_rx(struct net_device *dev, int i, struct netdev_queue_stats_rx *stats) { @@ -15842,6 +16258,7 @@ static void bnxt_get_queue_stats_rx(struct net_device *dev, int i, if (!bp->bnapi) return; + bnxt_sync_ring_stats(bp); cpr = &bp->bnapi[i]->cp_ring; sw = cpr->stats.sw_stats; @@ -15856,6 +16273,8 @@ static void bnxt_get_queue_stats_rx(struct net_device *dev, int i, stats->bytes += BNXT_GET_RING_STATS64(sw, rx_bcast_bytes); stats->alloc_fail = cpr->sw_stats->rx.rx_oom_discards; + stats->hw_gro_packets = cpr->sw_stats->rx.rx_hw_gro_packets; + stats->hw_gro_wire_packets = cpr->sw_stats->rx.rx_hw_gro_wire_packets; } static void bnxt_get_queue_stats_tx(struct net_device *dev, int i, @@ -15868,6 +16287,7 @@ static void bnxt_get_queue_stats_tx(struct net_device *dev, int i, if (!bp->tx_ring) return; + bnxt_sync_ring_stats(bp); bnapi = bp->tx_ring[bp->tx_ring_map[i]].bnapi; sw = bnapi->cp_ring.stats.sw_stats; @@ -15890,7 +16310,9 @@ static void bnxt_get_base_stats(struct net_device *dev, rx->packets = bp->net_stats_prev.rx_packets; rx->bytes = bp->net_stats_prev.rx_bytes; - rx->alloc_fail = bp->ring_err_stats_prev.rx_total_oom_discards; + rx->alloc_fail = bp->ring_drv_stats_prev.rx_total_oom_discards; + rx->hw_gro_packets = bp->ring_drv_stats_prev.rx_total_hw_gro_packets; + rx->hw_gro_wire_packets = bp->ring_drv_stats_prev.rx_total_hw_gro_wire_packets; tx->packets = bp->net_stats_prev.tx_packets; tx->bytes = bp->net_stats_prev.tx_bytes; @@ -15902,7 +16324,36 @@ static const struct netdev_stat_ops bnxt_stat_ops = { .get_base_stats = bnxt_get_base_stats, }; -static int bnxt_queue_mem_alloc(struct net_device *dev, void *qmem, int idx) +static void bnxt_queue_default_qcfg(struct net_device *dev, + struct netdev_queue_config *qcfg) +{ + qcfg->rx_page_size = BNXT_RX_PAGE_SIZE; +} + +static int bnxt_validate_qcfg(struct net_device *dev, + struct netdev_queue_config *qcfg, + struct netlink_ext_ack *extack) +{ + struct bnxt *bp = netdev_priv(dev); + + /* Older chips need MSS calc so rx_page_size is not supported */ + if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS) && + qcfg->rx_page_size != BNXT_RX_PAGE_SIZE) + return -EINVAL; + + if (!is_power_of_2(qcfg->rx_page_size)) + return -ERANGE; + + if (qcfg->rx_page_size < BNXT_RX_PAGE_SIZE || + qcfg->rx_page_size > BNXT_MAX_RX_PAGE_SIZE) + return -ERANGE; + + return 0; +} + +static int bnxt_queue_mem_alloc(struct net_device *dev, + struct netdev_queue_config *qcfg, + void *qmem, int idx) { struct bnxt_rx_ring_info *rxr, *clone; struct bnxt *bp = netdev_priv(dev); @@ -15923,6 +16374,10 @@ static int bnxt_queue_mem_alloc(struct net_device *dev, void *qmem, int idx) clone->rx_sw_agg_prod = 0; clone->rx_next_cons = 0; clone->need_head_pool = false; + clone->rx_page_size = qcfg->rx_page_size; + clone->rx_agg_bmap = NULL; + clone->rx_tpa = NULL; + clone->rx_tpa_idx_map = NULL; rc = bnxt_alloc_rx_page_pool(bp, clone, rxr->page_pool->p.nid); if (rc) @@ -15966,15 +16421,22 @@ static int bnxt_queue_mem_alloc(struct net_device *dev, void *qmem, int idx) bnxt_alloc_one_rx_ring_skb(bp, clone, idx); if (bp->flags & BNXT_FLAG_AGG_RINGS) bnxt_alloc_one_rx_ring_netmem(bp, clone, idx); - if (bp->flags & BNXT_FLAG_TPA) - bnxt_alloc_one_tpa_info_data(bp, clone); + if (bp->flags & BNXT_FLAG_TPA) { + rc = bnxt_alloc_one_tpa_info_data(bp, clone); + if (rc) + goto err_free_rx_ring_skbs; + } return 0; +err_free_rx_ring_skbs: + bnxt_free_one_rx_ring_skbs(bp, clone); err_free_tpa_info: bnxt_free_one_tpa_info(bp, clone); err_free_rx_agg_ring: bnxt_free_ring(bp, &clone->rx_agg_ring_struct.ring_mem); + kfree(clone->rx_agg_bmap); + clone->rx_agg_bmap = NULL; err_free_rx_ring: bnxt_free_ring(bp, &clone->rx_ring_struct.ring_mem); err_rxq_info_unreg: @@ -16049,6 +16511,8 @@ static void bnxt_copy_rx_ring(struct bnxt *bp, src_ring = &src->rx_agg_ring_struct; src_rmem = &src_ring->ring_mem; + dst->rx_page_size = src->rx_page_size; + WARN_ON(dst_rmem->nr_pages != src_rmem->nr_pages); WARN_ON(dst_rmem->page_size != src_rmem->page_size); WARN_ON(dst_rmem->flags != src_rmem->flags); @@ -16068,7 +16532,9 @@ static void bnxt_copy_rx_ring(struct bnxt *bp, dst->rx_agg_bmap = src->rx_agg_bmap; } -static int bnxt_queue_start(struct net_device *dev, void *qmem, int idx) +static int bnxt_queue_start(struct net_device *dev, + struct netdev_queue_config *qcfg, + void *qmem, int idx) { struct bnxt *bp = netdev_priv(dev); struct bnxt_rx_ring_info *rxr, *clone; @@ -16143,7 +16609,7 @@ err_reset: rc); napi_enable_locked(&bnapi->napi); bnxt_db_nq_arm(bp, &cpr->cp_db, cpr->cp_raw_cons); - bnxt_reset_task(bp, true); + netif_close(dev); return rc; } @@ -16201,6 +16667,13 @@ static const struct netdev_queue_mgmt_ops bnxt_queue_mgmt_ops = { .ndo_queue_mem_free = bnxt_queue_mem_free, .ndo_queue_start = bnxt_queue_start, .ndo_queue_stop = bnxt_queue_stop, + .ndo_default_qcfg = bnxt_queue_default_qcfg, + .ndo_validate_qcfg = bnxt_validate_qcfg, + .supported_params = QCFG_RX_PAGE_SIZE, +}; + +static const struct netdev_queue_mgmt_ops bnxt_queue_mgmt_ops_unsupp = { + .ndo_default_qcfg = bnxt_queue_default_qcfg, }; static void bnxt_remove_one(struct pci_dev *pdev) @@ -16211,12 +16684,13 @@ static void bnxt_remove_one(struct pci_dev *pdev) if (BNXT_PF(bp)) __bnxt_sriov_disable(bp); - bnxt_rdma_aux_device_del(bp); + bnxt_aux_devices_del(bp); unregister_netdev(dev); bnxt_ptp_clear(bp); - bnxt_rdma_aux_device_uninit(bp); + bnxt_aux_devices_uninit(bp); + bnxt_auxdev_id_free(bp, bp->auxdev_id); bnxt_free_l2_filters(bp, true); bnxt_free_ntp_fltrs(bp, true); @@ -16232,6 +16706,7 @@ static void bnxt_remove_one(struct pci_dev *pdev) bnxt_shutdown_tc(bp); bnxt_clear_int_mode(bp); + bnxt_free_ring_cpu_masks(bp); bnxt_hwrm_func_drv_unrgtr(bp); bnxt_free_hwrm_resources(bp); bnxt_hwmon_uninit(bp); @@ -16409,6 +16884,19 @@ static void bnxt_trim_dflt_sh_rings(struct bnxt *bp) bp->tx_nr_rings = bnxt_tx_nr_rings(bp); } +static void bnxt_adj_dflt_rings(struct bnxt *bp, bool sh) +{ + if (sh) + bnxt_trim_dflt_sh_rings(bp); + else + bp->cp_nr_rings = bp->tx_nr_rings_per_tc + bp->rx_nr_rings; + bp->tx_nr_rings = bnxt_tx_nr_rings(bp); + if (sh && READ_ONCE(bp->xdp_prog)) { + bnxt_set_xdp_tx_rings(bp); + bnxt_set_cp_rings(bp, true); + } +} + static int bnxt_set_dflt_rings(struct bnxt *bp, bool sh) { int dflt_rings, max_rx_rings, max_tx_rings, rc; @@ -16434,11 +16922,8 @@ static int bnxt_set_dflt_rings(struct bnxt *bp, bool sh) return rc; bp->rx_nr_rings = min_t(int, dflt_rings, max_rx_rings); bp->tx_nr_rings_per_tc = min_t(int, dflt_rings, max_tx_rings); - if (sh) - bnxt_trim_dflt_sh_rings(bp); - else - bp->cp_nr_rings = bp->tx_nr_rings_per_tc + bp->rx_nr_rings; - bp->tx_nr_rings = bnxt_tx_nr_rings(bp); + + bnxt_adj_dflt_rings(bp, sh); avail_msix = bnxt_get_max_func_irqs(bp) - bp->cp_nr_rings; if (avail_msix >= BNXT_MIN_ROCE_CP_RINGS) { @@ -16451,16 +16936,17 @@ static int bnxt_set_dflt_rings(struct bnxt *bp, bool sh) rc = __bnxt_reserve_rings(bp); if (rc && rc != -ENODEV) netdev_warn(bp->dev, "Unable to reserve tx rings\n"); - bp->tx_nr_rings_per_tc = bnxt_tx_nr_rings_per_tc(bp); + + bnxt_adj_tx_rings(bp); if (sh) - bnxt_trim_dflt_sh_rings(bp); + bnxt_adj_dflt_rings(bp, true); /* Rings may have been trimmed, re-reserve the trimmed rings. */ if (bnxt_need_reserve_rings(bp)) { rc = __bnxt_reserve_rings(bp); if (rc && rc != -ENODEV) netdev_warn(bp->dev, "2nd rings reservation failed.\n"); - bp->tx_nr_rings_per_tc = bnxt_tx_nr_rings_per_tc(bp); + bnxt_adj_tx_rings(bp); } if (BNXT_CHIP_TYPE_NITRO_A0(bp)) { bp->rx_nr_rings++; @@ -16494,7 +16980,7 @@ static int bnxt_init_dflt_ring_mode(struct bnxt *bp) if (rc) goto init_dflt_ring_err; - bp->tx_nr_rings_per_tc = bnxt_tx_nr_rings_per_tc(bp); + bnxt_adj_tx_rings(bp); bnxt_set_dflt_rfs(bp); @@ -16507,7 +16993,7 @@ int bnxt_restore_pf_fw_resources(struct bnxt *bp) { int rc; - netdev_ops_assert_locked(bp->dev); + netdev_assert_locked_ops_compat(bp->dev); bnxt_hwrm_func_qcaps(bp); if (netif_running(bp->dev)) @@ -16659,6 +17145,11 @@ static int bnxt_init_one(struct pci_dev *pdev, const struct pci_device_id *ent) bp->msg_enable = BNXT_DEF_MSG_ENABLE; bnxt_set_max_func_irqs(bp, max_irqs); + bp->max_irqs = max_irqs; + rc = bnxt_alloc_ring_cpu_masks(bp); + if (rc) + goto init_err_free; + if (bnxt_vf_pciid(bp->board_idx)) bp->flags |= BNXT_FLAG_VF; @@ -16671,6 +17162,7 @@ static int bnxt_init_one(struct pci_dev *pdev, const struct pci_device_id *ent) goto init_err_free; dev->netdev_ops = &bnxt_netdev_ops; + dev->xdp_metadata_ops = &bnxt_xdp_metadata_ops; dev->stat_ops = &bnxt_stat_ops; dev->watchdog_timeo = BNXT_TX_TIMEOUT; dev->ethtool_ops = &bnxt_ethtool_ops; @@ -16712,21 +17204,20 @@ static int bnxt_init_one(struct pci_dev *pdev, const struct pci_device_id *ent) } dev->hw_features = NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_SG | - NETIF_F_TSO | NETIF_F_TSO6 | + NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_TSO_ECN | NETIF_F_GSO_UDP_TUNNEL | NETIF_F_GSO_GRE | NETIF_F_GSO_IPXIP4 | NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_GSO_GRE_CSUM | NETIF_F_GSO_PARTIAL | NETIF_F_RXHASH | NETIF_F_RXCSUM | NETIF_F_GRO; - if (bp->flags & BNXT_FLAG_UDP_GSO_CAP) - dev->hw_features |= NETIF_F_GSO_UDP_L4; + dev->hw_features |= NETIF_F_GSO_UDP_L4; if (BNXT_SUPPORTS_TPA(bp)) dev->hw_features |= NETIF_F_LRO; dev->hw_enc_features = NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_SG | - NETIF_F_TSO | NETIF_F_TSO6 | + NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_TSO_ECN | NETIF_F_GSO_UDP_TUNNEL | NETIF_F_GSO_GRE | NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_GSO_GRE_CSUM | NETIF_F_GSO_IPXIP4 | NETIF_F_GSO_PARTIAL; @@ -16752,8 +17243,15 @@ static int bnxt_init_one(struct pci_dev *pdev, const struct pci_device_id *ent) dev->priv_flags |= IFF_UNICAST_FLT; netif_set_tso_max_size(dev, GSO_MAX_SIZE); - if (bp->tso_max_segs) + if (!(bp->flags & BNXT_FLAG_UDP_GSO_CAP)) { + u16 max_segs = BNXT_SW_USO_MAX_SEGS; + + if (bp->tso_max_segs) + max_segs = min_t(u16, max_segs, bp->tso_max_segs); + netif_set_tso_max_segs(dev, max_segs); + } else if (bp->tso_max_segs) { netif_set_tso_max_segs(dev, bp->tso_max_segs); + } dev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT | NETDEV_XDP_ACT_RX_SG; @@ -16802,7 +17300,9 @@ static int bnxt_init_one(struct pci_dev *pdev, const struct pci_device_id *ent) bnxt_set_tpa_flags(bp); bnxt_init_ring_params(bp); bnxt_set_ring_params(bp); - bnxt_rdma_aux_device_init(bp); + mutex_init(&bp->auxdev_lock); + if (!bnxt_auxdev_id_alloc(bp)) + bnxt_aux_devices_init(bp); rc = bnxt_set_dflt_rings(bp, true); if (rc) { if (BNXT_VF(bp) && rc == -ENODEV) { @@ -16855,10 +17355,11 @@ static int bnxt_init_one(struct pci_dev *pdev, const struct pci_device_id *ent) if (BNXT_SUPPORTS_NTUPLE_VNIC(bp)) bp->rss_cap |= BNXT_RSS_CAP_MULTI_RSS_CTX; + + dev->queue_mgmt_ops = &bnxt_queue_mgmt_ops_unsupp; if (BNXT_SUPPORTS_QUEUE_API(bp)) dev->queue_mgmt_ops = &bnxt_queue_mgmt_ops; - dev->request_ops_lock = true; - dev->netmem_tx = true; + dev->netmem_tx = NETMEM_TX_DMA; rc = register_netdev(dev); if (rc) @@ -16866,7 +17367,7 @@ static int bnxt_init_one(struct pci_dev *pdev, const struct pci_device_id *ent) bnxt_dl_fw_reporters_create(bp); - bnxt_rdma_aux_device_add(bp); + bnxt_aux_devices_add(bp); bnxt_print_device_info(bp); @@ -16874,7 +17375,8 @@ static int bnxt_init_one(struct pci_dev *pdev, const struct pci_device_id *ent) return 0; init_err_cleanup: - bnxt_rdma_aux_device_uninit(bp); + bnxt_aux_devices_uninit(bp); + bnxt_auxdev_id_free(bp, bp->auxdev_id); bnxt_dl_unregister(bp); init_err_dl: bnxt_shutdown_tc(bp); @@ -16882,12 +17384,12 @@ init_err_dl: init_err_pci_clean: bnxt_hwrm_func_drv_unrgtr(bp); - bnxt_free_hwrm_resources(bp); - bnxt_hwmon_uninit(bp); - bnxt_ethtool_free(bp); bnxt_ptp_clear(bp); kfree(bp->ptp_cfg); bp->ptp_cfg = NULL; + bnxt_free_hwrm_resources(bp); + bnxt_hwmon_uninit(bp); + bnxt_ethtool_free(bp); kfree(bp->fw_health); bp->fw_health = NULL; bnxt_cleanup_pci(bp); @@ -16897,6 +17399,7 @@ init_err_pci_clean: bp->rss_indir_tbl = NULL; init_err_free: + bnxt_free_ring_cpu_masks(bp); free_netdev(dev); return rc; } @@ -16964,6 +17467,7 @@ static int bnxt_resume(struct device *device) struct bnxt *bp = netdev_priv(dev); int rc = 0; + rtnl_lock(); netdev_lock(dev); rc = pci_enable_device(bp->pdev); if (rc) { @@ -17008,9 +17512,11 @@ static int bnxt_resume(struct device *device) resume_exit: netdev_unlock(bp->dev); - bnxt_ulp_start(bp, rc); - if (!rc) + rtnl_unlock(); + if (!rc) { + bnxt_ulp_start(bp); bnxt_reenable_sriov(bp); + } return rc; } @@ -17096,9 +17602,14 @@ static pci_ers_result_t bnxt_io_slot_reset(struct pci_dev *pdev) netdev_info(bp->dev, "PCI Slot Reset\n"); - if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS) && - test_bit(BNXT_STATE_PCI_CHANNEL_IO_FROZEN, &bp->state)) - msleep(900); + if (test_bit(BNXT_STATE_PCI_CHANNEL_IO_FROZEN, &bp->state)) { + /* After DPC, the chip should return CRS when the vendor ID + * config register is read until it is ready. On all chips, + * this is not happening reliably so add a 5-second delay as a + * workaround. + */ + msleep(5000); + } netdev_lock(netdev); @@ -17122,7 +17633,6 @@ static pci_ers_result_t bnxt_io_slot_reset(struct pci_dev *pdev) pci_write_config_dword(bp->pdev, off, 0); } pci_restore_state(pdev); - pci_save_state(pdev); bnxt_inv_fw_health_reg(bp); bnxt_try_map_fw_health_reg(bp); @@ -17173,6 +17683,7 @@ static void bnxt_io_resume(struct pci_dev *pdev) int err; netdev_info(bp->dev, "PCI Slot Resume\n"); + rtnl_lock(); netdev_lock(netdev); err = bnxt_hwrm_func_qcaps(bp); @@ -17190,9 +17701,11 @@ static void bnxt_io_resume(struct pci_dev *pdev) netif_device_attach(netdev); netdev_unlock(netdev); - bnxt_ulp_start(bp, err); - if (!err) + rtnl_unlock(); + if (!err) { + bnxt_ulp_start(bp); bnxt_reenable_sriov(bp); + } } static const struct pci_error_handlers bnxt_err_handler = { |